Ansel 0.0
A darktable fork - bloat + design vision
Loading...
Searching...
No Matches
database.c
Go to the documentation of this file.
1/*
2 This file is part of darktable,
3 Copyright (C) 2011-2012 Edouard Gomez.
4 Copyright (C) 2011 Henrik Andersson.
5 Copyright (C) 2011-2012 johannes hanika.
6 Copyright (C) 2011 Moritz Lipp.
7 Copyright (C) 2012, 2017 Christian Tellefsen.
8 Copyright (C) 2012 José Carlos García Sogo.
9 Copyright (C) 2012 Jérémy Rosen.
10 Copyright (C) 2012 Richard Wonka.
11 Copyright (C) 2012 Simon Spannagel.
12 Copyright (C) 2012-2020 Tobias Ellinghaus.
13 Copyright (C) 2013-2014, 2018-2022 Pascal Obry.
14 Copyright (C) 2014-2016 Roman Lebedev.
15 Copyright (C) 2015 Pedro Côrte-Real.
16 Copyright (C) 2016 piterdias.
17 Copyright (C) 2017, 2019, 2021 luzpaz.
18 Copyright (C) 2017-2018 Rikard Öxler.
19 Copyright (C) 2018-2019 Edgardo Hoszowski.
20 Copyright (C) 2018 Mario Lueder.
21 Copyright (C) 2018 Peter Budai.
22 Copyright (C) 2019 jakubfi.
23 Copyright (C) 2019-2022 Philippe Weyland.
24 Copyright (C) 2020 Heiko Bauke.
25 Copyright (C) 2020-2021 Hubert Kowalski.
26 Copyright (C) 2020 JP Verrue.
27 Copyright (C) 2020 jpverrue.
28 Copyright (C) 2021 Diederik Ter Rahe.
29 Copyright (C) 2021 Hanno Schwalm.
30 Copyright (C) 2021 HansBull.
31 Copyright (C) 2021 Harald.
32 Copyright (C) 2021 Ralf Brown.
33 Copyright (C) 2022 Aldric Renaudin.
34 Copyright (C) 2022-2023, 2025-2026 Aurélien PIERRE.
35 Copyright (C) 2022 Martin Bařinka.
36 Copyright (C) 2022 Miloš Komarčević.
37 Copyright (C) 2023 Maurizio Paglia.
38 Copyright (C) 2025 Alynx Zhou.
39 Copyright (C) 2025-2026 Guillaume Stutin.
40
41 darktable is free software: you can redistribute it and/or modify
42 it under the terms of the GNU General Public License as published by
43 the Free Software Foundation, either version 3 of the License, or
44 (at your option) any later version.
45
46 darktable is distributed in the hope that it will be useful,
47 but WITHOUT ANY WARRANTY; without even the implied warranty of
48 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
49 GNU General Public License for more details.
50
51 You should have received a copy of the GNU General Public License
52 along with darktable. If not, see <http://www.gnu.org/licenses/>.
53*/
54
55#ifdef HAVE_CONFIG_H
56#include "config.h"
57#endif
58
59#include "system/atomic.h"
60#include "common/paths.h" // DT_PATH_MAX
61#include "system/dtpthread.h"
62#include "common/image.h"
63#include "database/database.h"
64#include "database/sql_debug.h"
65#include "common/datetime.h"
66#include "common/file_location.h"
67#include "develop/iop_order.h"
68#ifdef HAVE_ICU
69/* sqlite3IcuInit(), called by _load_icu_collation() below. This block was empty: the
70 * declaration had been lost, and since HAVE_ICU is off in every local and CI build the
71 * only configuration that would have caught it is one nobody runs. */
72#include "database/sqliteicu.h" // conditional-ok: sqlite3IcuInit() is only called inside this same #ifdef, and sqliteicu.c is only compiled with HAVE_ICU
73#endif
74#include "database/legacy_presets.h"
75
76#include <gio/gio.h>
77#include <glib.h>
78#include <glib/gstdio.h>
79
80#include <errno.h>
81#include <fcntl.h>
82#include <signal.h>
83#include <sys/stat.h>
84#include <sys/types.h>
85#include "common/utility.h"
86
87// whenever _create_*_schema() gets changed you HAVE to bump this version and add an update path to
88// _upgrade_*_schema_step()!
89#define CURRENT_DATABASE_VERSION_LIBRARY 36
90#define CURRENT_DATABASE_VERSION_DATA 9
91
92// #define USE_NESTED_TRANSACTIONS
93#define MAX_NESTED_TRANSACTIONS 0
94/* transaction id */
95static dt_atomic_int _trxid;
96static dt_atomic_int _trx_batch_level;
97static gpointer _trx_owner = NULL;
98static gpointer _trx_batch_owner = NULL;
99
100typedef struct dt_database_t
101{
102 gboolean lock_acquired;
103
104 /* data database filename */
105 gchar *dbfilename_data, *lockfile_data;
106
107 /* library database filename */
108 gchar *dbfilename_library, *lockfile_library;
109
110 /* ondisk DB */
111 sqlite3 *handle;
112
113 gchar *error_message, *error_dbfilename;
114 int error_other_pid;
115} dt_database_t;
116
117/* ---------------------------------------------------------------------------------------
118 * Module state. All of it private: the connection, the policy, and the session constants
119 * the orchestrator told us at open time.
120 * ------------------------------------------------------------------------------------- */
121
123static dt_database_t *_db = NULL;
124
141static dt_pthread_rwlock_t _db_lock;
142static gboolean _db_lock_inited = FALSE;
143
147static dt_database_settings_t _settings = { NULL, 0, NULL, 0 };
148static dt_pthread_mutex_t _settings_lock;
149static gboolean _settings_lock_inited = FALSE;
150
152static gboolean _has_gui = FALSE;
153static gboolean _verbose = FALSE;
154
156static dt_database_renamed_handler_t _renamed_handler = NULL;
157
161#define _db_print(...) \
162 do { \
163 if(_verbose) dt_print(DT_DEBUG_SQL, __VA_ARGS__); \
164 } while(0)
165
166
167/* migrates database from old place to new */
168static gchar *_database_migrate_to_xdg_structure(const char *configured_db);
169
170/* delete old mipmaps files */
171static void _database_delete_mipmaps_files();
172
173/* tear down one connection; used both by dt_database_close() and by the failure paths of
174 * _database_init(), which have a database to free but nothing published yet */
175static void _database_free(dt_database_t *db);
176
177#define _SQLITE3_EXEC(a, b, c, d, e) \
178 if(sqlite3_exec(a, b, c, d, e) != SQLITE_OK) \
179 { \
180 all_ok = FALSE; \
181 failing_query = b; \
182 goto end; \
183 }
184
185/* migrate from the legacy db format (with the 'settings' blob) to the first version this system knows */
186static gboolean _migrate_schema(dt_database_t *db, int version)
187{
188 gboolean all_ok = TRUE;
189 const char *failing_query = NULL;
190 sqlite3_stmt *stmt;
191 sqlite3_stmt *innerstmt;
192
193 if(version != 36) // if anyone shows up with an older db we can probably add extra code
194 return FALSE;
195
196 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
197 // clang-format off
198 // remove stuff that is either no longer needed or that got renamed
199 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.lock", NULL, NULL, NULL);
200 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.settings", NULL, NULL, NULL); // yes, we do this in many
201 // places. because it's really
202 // important to not miss it in
203 // any code path.
204 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.group_id_index", NULL, NULL, NULL);
205 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.imgid_index", NULL, NULL, NULL);
206 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.mipmaps", NULL, NULL, NULL);
207 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.mipmap_timestamps", NULL, NULL, NULL);
208 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.dt_migration_table", NULL, NULL, NULL);
209
210 // using _create_library_schema() and filling that with the old data doesn't work since we always want to generate
211 // version 1 tables
213 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)",
214 NULL, NULL, NULL);
215 _SQLITE3_EXEC(db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', 1)",
216 NULL, NULL, NULL);
218 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.film_rolls_folder_index ON film_rolls (folder)",
219 NULL, NULL, NULL);
221 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN orientation INTEGER", NULL, NULL, NULL);
222 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN focus_distance REAL", NULL, NULL, NULL);
223 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN group_id INTEGER", NULL, NULL, NULL);
224 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN histogram BLOB", NULL, NULL, NULL);
225 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN lightmap BLOB", NULL, NULL, NULL);
226 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN longitude REAL", NULL, NULL, NULL);
227 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN latitude REAL", NULL, NULL, NULL);
228 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN color_matrix BLOB", NULL, NULL, NULL);
229 // the colorspace as specified in some image types
230 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN colorspace INTEGER", NULL, NULL, NULL);
231 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN version INTEGER", NULL, NULL, NULL);
232 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN max_version INTEGER", NULL, NULL, NULL);
233 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET orientation = -1 WHERE orientation IS NULL", NULL, NULL, NULL);
234 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET focus_distance = -1 WHERE focus_distance IS NULL",
235 NULL, NULL, NULL);
236 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET group_id = id WHERE group_id IS NULL", NULL, NULL, NULL);
237 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET max_version = (SELECT COUNT(*)-1 FROM main.images i WHERE "
238 "i.filename = main.images.filename AND "
239 "i.film_id = main.images.film_id) WHERE max_version IS NULL",
240 NULL, NULL, NULL);
241 _SQLITE3_EXEC(
242 db->handle,
243 "UPDATE main.images SET version = (SELECT COUNT(*) FROM main.images i "
244 "WHERE i.filename = main.images.filename AND "
245 "i.film_id = main.images.film_id AND i.id < main.images.id) WHERE version IS NULL",
246 NULL, NULL, NULL);
247 // make sure we have AUTOINCREMENT on imgid --> move the whole thing away and recreate the table :(
248 _SQLITE3_EXEC(db->handle, "ALTER TABLE main.images RENAME TO dt_migration_table", NULL, NULL, NULL);
249 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.images_group_id_index", NULL, NULL, NULL);
250 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.images_film_id_index", NULL, NULL, NULL);
251 _SQLITE3_EXEC(
252 db->handle,
253 "CREATE TABLE main.images (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
254 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
255 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
256 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
257 "output_width INTEGER, output_height INTEGER, crop REAL, "
258 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
259 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
260 "caption VARCHAR, description VARCHAR, license VARCHAR, sha1sum CHAR(40), "
261 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
262 "latitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, max_version INTEGER)",
263 NULL, NULL, NULL);
264 _SQLITE3_EXEC(db->handle, "CREATE INDEX main.images_group_id_index ON images (group_id)", NULL, NULL, NULL);
265 _SQLITE3_EXEC(db->handle, "CREATE INDEX main.images_film_id_index ON images (film_id)", NULL, NULL, NULL);
266 _SQLITE3_EXEC(
267 db->handle,
268 "INSERT INTO main.images (id, group_id, film_id, width, height, filename, maker, model, "
269 "lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags, "
270 "output_width, output_height, crop, raw_parameters, raw_denoise_threshold, "
271 "raw_auto_bright_threshold, raw_black, raw_maximum, caption, description, license, sha1sum, "
272 "orientation, histogram, lightmap, longitude, latitude, color_matrix, colorspace, version, "
273 "max_version) "
274 "SELECT id, group_id, film_id, width, height, filename, maker, model, lens, exposure, aperture, iso, "
275 "focal_length, focus_distance, datetime_taken, flags, output_width, output_height, crop, "
276 "raw_parameters, raw_denoise_threshold, raw_auto_bright_threshold, raw_black, raw_maximum, "
277 "caption, description, license, sha1sum, orientation, histogram, lightmap, longitude, "
278 "latitude, color_matrix, colorspace, version, max_version FROM dt_migration_table",
279 NULL, NULL, NULL);
280 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
282 // selected_images should have a primary key. add it if it's missing:
283 _SQLITE3_EXEC(db->handle, "CREATE TEMPORARY TABLE dt_migration_table (imgid INTEGER)", NULL, NULL, NULL);
284 _SQLITE3_EXEC(db->handle, "INSERT INTO dt_migration_table SELECT imgid FROM main.selected_images", NULL, NULL,
285 NULL);
286 _SQLITE3_EXEC(db->handle, "DROP TABLE main.selected_images", NULL, NULL, NULL);
287 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.selected_images (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
288 _SQLITE3_EXEC(db->handle, "INSERT OR IGNORE INTO main.selected_images SELECT imgid FROM dt_migration_table",
289 NULL, NULL, NULL);
290 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
292 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
293 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
294 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
295 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
296 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.history_imgid_index ON history (imgid)", NULL, NULL,
297 NULL);
298 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
299 NULL, NULL);
300 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET multi_priority = 0 WHERE multi_priority IS NULL", NULL, NULL,
301 NULL);
302 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL, NULL);
304 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.mask (imgid INTEGER, formid INTEGER, form INTEGER, "
305 "name VARCHAR(256), version INTEGER, "
306 "points BLOB, points_count INTEGER, source BLOB)",
307 NULL, NULL, NULL);
308 sqlite3_exec(db->handle, "ALTER TABLE main.mask ADD COLUMN source BLOB", NULL, NULL,
309 NULL); // in case the table was there already but missed that column
311 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.tagged_images_tagid_index ON tagged_images (tagid)",
312 NULL, NULL, NULL);
314 _SQLITE3_EXEC(db->handle,
315 "CREATE TABLE IF NOT EXISTS main.styles (id INTEGER, name VARCHAR, description VARCHAR)", NULL,
316 NULL, NULL);
317 sqlite3_exec(db->handle, "ALTER TABLE main.styles ADD COLUMN id INTEGER", NULL, NULL, NULL);
318 _SQLITE3_EXEC(db->handle, "UPDATE main.styles SET id = rowid WHERE id IS NULL", NULL, NULL, NULL);
320 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.style_items (styleid INTEGER, num INTEGER, module "
321 "INTEGER, operation VARCHAR(256), op_params BLOB, "
322 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, multi_priority "
323 "INTEGER, multi_name VARCHAR(256))",
324 NULL, NULL, NULL);
325 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
326 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
327 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
328 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
329 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
330 NULL, NULL);
331 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET multi_priority = 0 WHERE multi_priority IS NULL", NULL,
332 NULL, NULL);
333 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL,
334 NULL);
336 // color_labels could have a PRIMARY KEY that we don't want
337 _SQLITE3_EXEC(db->handle, "CREATE TEMPORARY TABLE dt_migration_table (imgid INTEGER, color INTEGER)", NULL,
338 NULL, NULL);
339 _SQLITE3_EXEC(db->handle, "INSERT INTO dt_migration_table SELECT imgid, color FROM main.color_labels", NULL,
340 NULL, NULL);
341 _SQLITE3_EXEC(db->handle, "DROP TABLE main.color_labels", NULL, NULL, NULL);
342 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.color_labels (imgid INTEGER, color INTEGER)", NULL, NULL, NULL);
343 _SQLITE3_EXEC(db->handle, "CREATE UNIQUE INDEX main.color_labels_idx ON color_labels (imgid, color)", NULL, NULL,
344 NULL);
345 _SQLITE3_EXEC(db->handle, "INSERT OR IGNORE INTO main.color_labels SELECT imgid, color FROM dt_migration_table",
346 NULL, NULL, NULL);
347 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
349 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.meta_data (id INTEGER, key INTEGER, value VARCHAR)",
350 NULL, NULL, NULL);
351 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.metadata_index ON meta_data (id, key)", NULL, NULL,
352 NULL);
354 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.presets (name VARCHAR, description VARCHAR, "
355 "operation VARCHAR, op_version INTEGER, op_params BLOB, "
356 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, multi_priority "
357 "INTEGER, multi_name VARCHAR(256), "
358 "model VARCHAR, maker VARCHAR, lens VARCHAR, iso_min REAL, iso_max REAL, "
359 "exposure_min REAL, exposure_max REAL, "
360 "aperture_min REAL, aperture_max REAL, focal_length_min REAL, focal_length_max "
361 "REAL, writeprotect INTEGER, "
362 "autoapply INTEGER, filter INTEGER, def INTEGER, isldr INTEGER)",
363 NULL, NULL, NULL);
364 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN op_version INTEGER", NULL, NULL, NULL);
365 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
366 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
367 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
368 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
369 // the unique index only works if the db doesn't have any (name, operation, op_version) more than once.
370 // apparently there are dbs out there which do have that. :(
371 sqlite3_prepare_v2(db->handle,
372 "SELECT p.rowid, p.name, p.operation, p.op_version FROM main.presets p INNER JOIN "
373 "(SELECT * FROM (SELECT rowid, name, operation, op_version, COUNT(*) AS count "
374 "FROM main.presets GROUP BY name, operation, op_version) WHERE count > 1) s "
375 "ON p.name = s.name AND p.operation = s.operation AND p.op_version = s.op_version",
376 -1, &stmt, NULL);
377 // clang-format on
378 char *last_name = NULL, *last_operation = NULL;
379 int last_op_version = 0;
380 int i = 0;
381 while(sqlite3_step(stmt) == SQLITE_ROW)
382 {
383 const int rowid = sqlite3_column_int(stmt, 0);
384 const char *name = (const char *)sqlite3_column_text(stmt, 1);
385 const char *operation = (const char *)sqlite3_column_text(stmt, 2);
386 const int op_version = sqlite3_column_int(stmt, 3);
387
388 // is it still the same (name, operation, op_version) triple?
389 if(IS_NULL_PTR(last_name) || strcmp(last_name, name) || IS_NULL_PTR(last_operation) || strcmp(last_operation, operation)
390 || last_op_version != op_version)
391 {
392 dt_free(last_name);
393 dt_free(last_operation);
394 last_name = g_strdup(name);
395 last_operation = g_strdup(operation);
396 last_op_version = op_version;
397 i = 0;
398 }
399
400 // find the next free amended version of name
401 // clang-format off
402 sqlite3_prepare_v2(db->handle, "SELECT name FROM main.presets WHERE name = ?1 || ' (' || ?2 || ')' AND "
403 "operation = ?3 AND op_version = ?4",
404 -1, &innerstmt, NULL);
405 // clang-format on
406 while(1)
407 {
408 sqlite3_bind_text(innerstmt, 1, name, -1, SQLITE_TRANSIENT);
409 sqlite3_bind_int(innerstmt, 2, i);
410 sqlite3_bind_text(innerstmt, 3, operation, -1, SQLITE_TRANSIENT);
411 sqlite3_bind_int(innerstmt, 4, op_version);
412 if(sqlite3_step(innerstmt) != SQLITE_ROW) break;
413 sqlite3_reset(innerstmt);
414 sqlite3_clear_bindings(innerstmt);
415 i++;
416 }
417 sqlite3_finalize(innerstmt);
418
419 // rename preset
420 // clang-format off
421 const char *query = "UPDATE main.presets SET name = name || ' (' || ?1 || ')' WHERE rowid = ?2";
422 // clang-format on
423 sqlite3_prepare_v2(db->handle, query, -1, &innerstmt, NULL);
424 sqlite3_bind_int(innerstmt, 1, i);
425 sqlite3_bind_int(innerstmt, 2, rowid);
426 if(sqlite3_step(innerstmt) != SQLITE_DONE)
427 {
428 all_ok = FALSE;
429 failing_query = query;
430 goto end;
431 }
432 sqlite3_finalize(innerstmt);
433 }
434 sqlite3_finalize(stmt);
435 dt_free(last_name);
436 dt_free(last_operation);
437 // now we should be able to create the index
438 // clang-format off
439 _SQLITE3_EXEC(db->handle,
440 "CREATE UNIQUE INDEX IF NOT EXISTS main.presets_idx ON presets (name, operation, op_version)",
441 NULL, NULL, NULL);
442 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
443 NULL, NULL);
444 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET multi_priority = 0 WHERE multi_priority IS NULL", NULL, NULL,
445 NULL);
446 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL, NULL);
447 // clang-format on
448
449
450 // There are systems where absolute paths don't start with '/' (like Windows).
451 // Since the bug which introduced absolute paths to the db was fixed before a
452 // Windows build was available this shouldn't matter though.
453 // clang-format off
454 sqlite3_prepare_v2(db->handle, "SELECT id, filename FROM main.images WHERE filename LIKE '/%'", -1, &stmt, NULL);
455 sqlite3_prepare_v2(db->handle, "UPDATE main.images SET filename = ?1 WHERE id = ?2", -1, &innerstmt, NULL);
456 // clang-format on
457 while(sqlite3_step(stmt) == SQLITE_ROW)
458 {
459 const int id = sqlite3_column_int(stmt, 0);
460 const char *path = (const char *)sqlite3_column_text(stmt, 1);
461 gchar *filename = g_path_get_basename(path);
462 sqlite3_bind_text(innerstmt, 1, filename, -1, SQLITE_TRANSIENT);
463 sqlite3_bind_int(innerstmt, 2, id);
464 sqlite3_step(innerstmt);
465 sqlite3_reset(innerstmt);
466 sqlite3_clear_bindings(innerstmt);
467 dt_free(filename);
468 }
469 sqlite3_finalize(stmt);
470 sqlite3_finalize(innerstmt);
471
472 // We used to insert datetime_taken entries with '-' as date separators. Since that doesn't work well with
473 // the regular ':' when parsing
474 // or sorting we changed it to ':'. This takes care to change what we have as leftovers
475 // clang-format off
476 _SQLITE3_EXEC(
477 db->handle,
478 "UPDATE main.images SET datetime_taken = REPLACE(datetime_taken, '-', ':') WHERE datetime_taken LIKE '%-%'",
479 NULL, NULL, NULL);
480 // clang-format on
481
482end:
483 if(all_ok)
484 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
485 else
486 {
487 fprintf(stderr, "[init] failing query: `%s'\n", failing_query);
488 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle));
489 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL);
490 }
491
492 return all_ok;
493}
494
495#undef _SQLITE3_EXEC
496
497#define TRY_EXEC(_query, _message) \
498 do \
499 { \
500 if(sqlite3_exec(db->handle, _query, NULL, NULL, NULL) != SQLITE_OK) \
501 { \
502 fprintf(stderr, _message); \
503 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
504 FINALIZE; \
505 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
506 return version; \
507 } \
508 } while(0)
509
510#define TRY_STEP(_stmt, _expected, _message) \
511 do \
512 { \
513 if(sqlite3_step(_stmt) != _expected) \
514 { \
515 fprintf(stderr, _message); \
516 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
517 FINALIZE; \
518 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
519 return version; \
520 } \
521 } while(0)
522
523#define TRY_PREPARE(_stmt, _query, _message) \
524 do \
525 { \
526 if(sqlite3_prepare_v2(db->handle, _query, -1, &_stmt, NULL) != SQLITE_OK) \
527 { \
528 fprintf(stderr, _message); \
529 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
530 FINALIZE; \
531 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
532 return version; \
533 } \
534 } while(0)
535
536// redefine this where needed
537#define FINALIZE
538
539/* do the real migration steps, returns the version the db was converted to */
540static int _upgrade_library_schema_step(dt_database_t *db, int version)
541{
542 sqlite3_stmt *stmt;
543 int new_version = version;
544 if(version == CURRENT_DATABASE_VERSION_LIBRARY)
545 return version;
546 else if(version == 0)
547 {
548 // this can't happen, we started with 1, but it's a good example how this function works
549 // <do some magic to the db>
550 new_version = 1; // the version we transformed the db to. this way it might be possible to roll back or
551 // add fast paths
552 }
553 else if(version == 1)
554 {
555 // 1 -> 2 added write_timestamp
556 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
557 // clang-format off
558 TRY_EXEC("ALTER TABLE main.images ADD COLUMN write_timestamp INTEGER",
559 "[init] can't add `write_timestamp' column to database\n");
560 TRY_EXEC("UPDATE main.images SET write_timestamp = STRFTIME('%s', 'now') WHERE write_timestamp IS NULL",
561 "[init] can't initialize `write_timestamp' with current point in time\n");
562 // clang-format on
563 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
564 new_version = 2;
565 }
566 else if(version == 2)
567 {
568 // 2 -> 3 reset raw_black and raw_maximum. in theory we should change the columns from REAL to INTEGER,
569 // but sqlite doesn't care about types so whatever
570 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
571 // clang-format off
572 TRY_EXEC("UPDATE main.images SET raw_black = 0, raw_maximum = 16384",
573 "[init] can't reset raw_black and raw_maximum\n");
574 // clang-format on
575 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
576 new_version = 3;
577 }
578 else if(version == 3)
579 {
580 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
581 // clang-format off
582 TRY_EXEC("CREATE TRIGGER insert_tag AFTER INSERT ON main.tags"
583 " BEGIN"
584 " INSERT INTO tagxtag SELECT id, new.id, 0 FROM TAGS;"
585 " UPDATE tagxtag SET count = 1000000 WHERE id1=new.id AND id2=new.id;"
586 " END",
587 "[init] can't create insert_tag trigger\n");
588 TRY_EXEC("CREATE TRIGGER delete_tag BEFORE DELETE ON main.tags"
589 " BEGIN"
590 " DELETE FROM tagxtag WHERE id1=old.id OR id2=old.id;"
591 " DELETE FROM tagged_images WHERE tagid=old.id;"
592 " END",
593 "[init] can't create delete_tag trigger\n");
594 TRY_EXEC("CREATE TRIGGER attach_tag AFTER INSERT ON main.tagged_images"
595 " BEGIN"
596 " UPDATE tagxtag"
597 " SET count = count + 1"
598 " WHERE (id1=new.tagid AND id2 IN (SELECT tagid FROM tagged_images WHERE imgid=new.imgid))"
599 " OR (id2=new.tagid AND id1 IN (SELECT tagid FROM tagged_images WHERE imgid=new.imgid));"
600 " END",
601 "[init] can't create attach_tag trigger\n");
602 TRY_EXEC("CREATE TRIGGER detach_tag BEFORE DELETE ON main.tagged_images"
603 " BEGIN"
604 " UPDATE tagxtag"
605 " SET count = count - 1"
606 " WHERE (id1=old.tagid AND id2 IN (SELECT tagid FROM tagged_images WHERE imgid=old.imgid))"
607 " OR (id2=old.tagid AND id1 IN (SELECT tagid FROM tagged_images WHERE imgid=old.imgid));"
608 " END",
609 "[init] can't create detach_tag trigger\n");
610 // clang-format on
611 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
612 new_version = 4;
613 }
614 else if(version == 4)
615 {
616 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
617 // clang-format off
618 TRY_EXEC("ALTER TABLE main.presets RENAME TO tmp_presets", "[init] can't rename table presets\n");
619
620 TRY_EXEC("CREATE TABLE main.presets (name VARCHAR, description VARCHAR, operation VARCHAR, op_params BLOB,"
621 "enabled INTEGER, blendop_params BLOB, model VARCHAR, maker VARCHAR, lens VARCHAR,"
622 "iso_min REAL, iso_max REAL, exposure_min REAL, exposure_max REAL, aperture_min REAL,"
623 "aperture_max REAL, focal_length_min REAL, focal_length_max REAL, writeprotect INTEGER,"
624 "autoapply INTEGER, filter INTEGER, def INTEGER, format INTEGER, op_version INTEGER,"
625 "blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
626 "[init] can't create new presets table\n");
627
628 TRY_EXEC("INSERT INTO main.presets (name, description, operation, op_params, enabled, blendop_params, model, "
629 "maker, lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, aperture_max,"
630 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, def, format, op_version, "
631 "blendop_version, multi_priority, multi_name) SELECT name, description, operation, op_params, "
632 "enabled, blendop_params, model, maker, lens, iso_min, iso_max, exposure_min, exposure_max, "
633 "aperture_min, aperture_max, focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
634 "def, isldr, op_version, blendop_version, multi_priority, multi_name FROM tmp_presets",
635 "[init] can't populate presets table from tmp_presets\n");
636
637 TRY_EXEC("DROP TABLE tmp_presets", "[init] can't delete table tmp_presets\n");
638 // clang-format on
639 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
640 new_version = 5;
641 }
642 else if(version == 5)
643 {
644 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
645 // clang-format off
646 TRY_EXEC("CREATE INDEX main.images_filename_index ON images (filename)",
647 "[init] can't create index on image filename\n");
648 // clang-format on
649 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
650 new_version = 6;
651 }
652 else if(version == 6)
653 {
654 // some ancient tables can have the styleid column of style_items be called style_id. fix that.
655 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
656
657 if(sqlite3_exec(db->handle, "SELECT style_id FROM main.style_items", NULL, NULL, NULL) == SQLITE_OK)
658 {
659 // clang-format off
660 TRY_EXEC("ALTER TABLE main.style_items RENAME TO tmp_style_items",
661 "[init] can't rename table style_items\n");
662
663 TRY_EXEC("CREATE TABLE main.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
664 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
665 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
666 "[init] can't create new style_items table\n");
667
668 TRY_EXEC("INSERT INTO main.style_items (styleid, num, module, operation, op_params, enabled,"
669 " blendop_params, blendop_version, multi_priority, multi_name)"
670 " SELECT style_id, num, module, operation, op_params, enabled,"
671 " blendop_params, blendop_version, multi_priority, multi_name"
672 " FROM tmp_style_items",
673 "[init] can't populate style_items table from tmp_style_items\n");
674
675 TRY_EXEC("DROP TABLE tmp_style_items", "[init] can't delete table tmp_style_items\n");
676 // clang-format on
677 }
678
679 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
680 new_version = 7;
681 }
682 else if(version == 7)
683 {
684 // make sure that we have no film rolls with a NULL folder
685 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
686 // clang-format off
687 TRY_EXEC("ALTER TABLE main.film_rolls RENAME TO tmp_film_rolls", "[init] can't rename table film_rolls\n");
688
689 TRY_EXEC("CREATE TABLE main.film_rolls "
690 "(id INTEGER PRIMARY KEY, datetime_accessed CHAR(20), "
691 "folder VARCHAR(1024) NOT NULL)",
692 "[init] can't create new film_rolls table\n");
693
694 TRY_EXEC("INSERT INTO main.film_rolls (id, datetime_accessed, folder) "
695 "SELECT id, datetime_accessed, folder "
696 "FROM tmp_film_rolls "
697 "WHERE folder IS NOT NULL",
698 "[init] can't populate film_rolls table from tmp_film_rolls\n");
699
700 TRY_EXEC("DROP TABLE tmp_film_rolls", "[init] can't delete table tmp_film_rolls\n");
701 // clang-format on
702 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
703 new_version = 8;
704 }
705 else if(version == 8)
706 {
707 // 8 -> 9 added history_end column to images
708 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
709 // clang-format off
710 TRY_EXEC("ALTER TABLE main.images ADD COLUMN history_end INTEGER",
711 "[init] can't add `history_end' column to database\n");
712
713 TRY_EXEC("UPDATE main.images SET history_end = (SELECT IFNULL(MAX(num) + 1, 0) FROM main.history "
714 "WHERE imgid = id)", "[init] can't initialize `history_end' with last history entry\n");
715 // clang-format on
716 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
717 new_version = 9;
718 }
719 else if(version == 9)
720 {
721 // 9 -> 10 cleanup of last update :(
722 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
723 // clang-format off
724 TRY_EXEC("UPDATE main.images SET history_end = (SELECT IFNULL(MAX(num) + 1, 0) FROM main.history "
725 "WHERE imgid = id)", "[init] can't set `history_end' to 0 where it was NULL\n");
726 // clang-format on
727 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
728 new_version = 10;
729 }
730 else if(version == 10)
731 {
732 // 10 -> 11 added altitude column to images
733 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
734 // clang-format off
735 TRY_EXEC("ALTER TABLE main.images ADD COLUMN altitude REAL",
736 "[init] can't add `altitude' column to database\n");
737
738 TRY_EXEC("UPDATE main.images SET altitude = NULL", "[init] can't initialize `altitude' with NULL\n");
739 // clang-format on
740 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
741 new_version = 11;
742 }
743 else if(version == 11)
744 {
745 // 11 -> 12 tagxtag was removed in order to reduce database size
746 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
747 // clang-format off
748 TRY_EXEC("DROP TRIGGER main.detach_tag", "[init] can't drop trigger `detach_tag' from database\n");
749
750 TRY_EXEC("DROP TRIGGER main.attach_tag", "[init] can't drop trigger `attach_tag' from database\n");
751
752 TRY_EXEC("DROP TRIGGER main.delete_tag", "[init] can't drop trigger `delete_tag' from database\n");
753
754 TRY_EXEC("DROP TRIGGER main.insert_tag", "[init] can't drop trigger `insert_tag' from database\n");
755
756 TRY_EXEC("DROP TABLE main.tagxtag", "[init] can't drop table `tagxtag' from database\n");
757 // clang-format on
758 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
759 new_version = 12;
760 }
761 else if(version == 12)
762 {
763 // 11 -> 12 move presets, styles and tags over to the data database
764 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
765
767#undef FINALIZE
768#define FINALIZE \
769 do \
770 { \
771 sqlite3_finalize(stmt); \
772 sqlite3_finalize(select_stmt); \
773 sqlite3_finalize(count_clashes_stmt); \
774 sqlite3_finalize(update_name_stmt); \
775 sqlite3_finalize(insert_stmt); \
776 sqlite3_finalize(delete_stmt); \
777 } while(0)
778
779 stmt = NULL;
780 sqlite3_stmt *insert_stmt = NULL, *delete_stmt = NULL, *select_stmt = NULL, *count_clashes_stmt = NULL,
781 *update_name_stmt = NULL;
782 // clang-format off
783 // remove presets that are already in data.
784 // we can't use a NATURAL JOIN here as that fails when columns have NULL values. :-(
785 TRY_EXEC("DELETE FROM main.presets WHERE rowid IN (SELECT p1.rowid FROM main.presets p1 "
786 "JOIN data.presets p2 ON "
787 "p1.name IS p2.name AND "
788 "p1.description IS p2.description AND "
789 "p1.operation IS p2.operation AND "
790 "p1.op_version IS p2.op_version AND "
791 "p1.op_params IS p2.op_params AND "
792 "p1.enabled IS p2.enabled AND "
793 "p1.blendop_params IS p2.blendop_params AND "
794 "p1.blendop_version IS p2.blendop_version AND "
795 "p1.multi_priority IS p2.multi_priority AND "
796 "p1.multi_name IS p2.multi_name AND "
797 "p1.model IS p2.model AND "
798 "p1.maker IS p2.maker AND "
799 "p1.lens IS p2.lens AND "
800 "p1.iso_min IS p2.iso_min AND "
801 "p1.iso_max IS p2.iso_max AND "
802 "p1.exposure_min IS p2.exposure_min AND "
803 "p1.exposure_max IS p2.exposure_max AND "
804 "p1.aperture_min IS p2.aperture_min AND "
805 "p1.aperture_max IS p2.aperture_max AND "
806 "p1.focal_length_min IS p2.focal_length_min AND "
807 "p1.focal_length_max IS p2.focal_length_max AND "
808 "p1.writeprotect IS p2.writeprotect AND "
809 "p1.autoapply IS p2.autoapply AND "
810 "p1.filter IS p2.filter AND "
811 "p1.def IS p2.def AND "
812 "p1.format IS p2.format "
813 "WHERE p1.writeprotect = 0)",
814 "[init] can't delete already migrated presets from database\n");
815
816 // find all presets that are clashing with something else in presets. that can happen as we introduced an
817 // index on presets in data which wasn't in place in library.
818 TRY_PREPARE(select_stmt, "SELECT p.rowid, r FROM main.presets AS p, (SELECT rowid AS r, name, operation, "
819 "op_version FROM main.presets GROUP BY name, operation, op_version HAVING "
820 "COUNT(*) > 1) USING (name, operation, op_version) WHERE p.rowid != r",
821 "[init] can't prepare selecting presets with same name, operation, op_version from database\n");
822
823 // see if an updated preset name still causes problems
824 TRY_PREPARE(count_clashes_stmt, "SELECT COUNT(*) FROM main.presets AS p, (SELECT name, operation, op_version "
825 "FROM main.presets WHERE rowid = ?1) AS i ON p.name = i.name || \" #\" || ?2 "
826 "AND p.operation = i.operation AND p.op_version = i.op_version",
827 "[init] can't prepare selection of preset count by name from database\n");
828
829 // update the preset name for good
830 TRY_PREPARE(update_name_stmt, "UPDATE main.presets SET name = name || \" #\" || ?1 WHERE rowid = ?2",
831 "[init] can't prepare updating of preset name in database\n");
832
833 // find all presets that would be clashing with something in data
834 TRY_PREPARE(stmt, "SELECT p1.rowid FROM main.presets p1 INNER JOIN data.presets p2 "
835 "USING (name, operation, op_version) WHERE p1.writeprotect = 0",
836 "[init] can't access table `presets' in database\n");
837
838 // ... and move them over with a new name
839 TRY_PREPARE(insert_stmt, "INSERT OR FAIL INTO data.presets (name, description, operation, op_version, "
840 "op_params, enabled, blendop_params, blendop_version, multi_priority, multi_name, "
841 "model, maker, lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, "
842 "aperture_max, focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
843 "def, format) "
844 "SELECT name || \" #\" || ?1, description, operation, op_version, op_params, "
845 "enabled, blendop_params, blendop_version, multi_priority, multi_name, model, maker, "
846 "lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, aperture_max, "
847 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, def, format "
848 "FROM main.presets p1 WHERE p1.rowid = ?2",
849 "[init] can't prepare insertion statement\n");
850
851 TRY_PREPARE(delete_stmt, "DELETE FROM main.presets WHERE rowid = ?1", "[init] can't prepare deletion statement\n");
852 // clang-format on
853 // first rename presets with (name, operation, op_version) not being unique
854 while(sqlite3_step(select_stmt) == SQLITE_ROW)
855 {
856 const int own_rowid = sqlite3_column_int(select_stmt, 0);
857 const int other_rowid = sqlite3_column_int(select_stmt, 1);
858 int preset_version = 0;
859
860 do
861 {
862 preset_version++;
863 sqlite3_reset(count_clashes_stmt);
864 sqlite3_clear_bindings(count_clashes_stmt);
865 sqlite3_bind_int(count_clashes_stmt, 1, other_rowid);
866 sqlite3_bind_int(count_clashes_stmt, 2, preset_version);
867 }
868 while(sqlite3_step(count_clashes_stmt) == SQLITE_ROW && sqlite3_column_int(count_clashes_stmt, 0) > 0);
869
870 sqlite3_bind_int(update_name_stmt, 1, preset_version);
871 sqlite3_bind_int(update_name_stmt, 2, own_rowid);
872 TRY_STEP(update_name_stmt, SQLITE_DONE, "[init] can't rename preset in database\n");
873 sqlite3_reset(update_name_stmt);
874 sqlite3_reset(update_name_stmt);
875 }
876
877 // now rename to avoid clashes with data.presets
878 while(sqlite3_step(stmt) == SQLITE_ROW)
879 {
880 int preset_version = 0;
881 const int rowid = sqlite3_column_int(stmt, 0);
882
883 do
884 {
885 preset_version++;
886 sqlite3_reset(insert_stmt);
887 sqlite3_clear_bindings(insert_stmt);
888 sqlite3_bind_int(insert_stmt, 1, preset_version);
889 sqlite3_bind_int(insert_stmt, 2, rowid);
890 } while(sqlite3_step(insert_stmt) != SQLITE_DONE);
891
892 sqlite3_reset(delete_stmt);
893 sqlite3_clear_bindings(delete_stmt);
894 sqlite3_bind_int(delete_stmt, 1, rowid);
895 TRY_STEP(delete_stmt, SQLITE_DONE, "[init] can't delete preset from database\n");
896 }
897 // clang-format off
898 // all that is left in presets should be those that can be moved over without any further concerns
899 TRY_EXEC("INSERT OR FAIL INTO data.presets SELECT name, description, operation, "
900 "op_version, op_params, enabled, blendop_params, blendop_version, "
901 "multi_priority, multi_name, model, maker, lens, iso_min, iso_max, "
902 "exposure_min, exposure_max, aperture_min, aperture_max, "
903 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
904 "def, format FROM main.presets WHERE writeprotect = 0",
905 "[init] can't copy presets to the data database\n");
906 // ... delete them on the old side
907 TRY_EXEC("DELETE FROM main.presets WHERE writeprotect = 0",
908 "[init] can't copy presets to the data database\n");
909 // clang-format on
910
911 FINALIZE;
912#undef FINALIZE
913
915#define FINALIZE \
916 do \
917 { \
918 sqlite3_finalize(stmt); \
919 sqlite3_finalize(insert_stmt); \
920 sqlite3_finalize(select_stmt); \
921 sqlite3_finalize(delete_stmt); \
922 sqlite3_finalize(update_name_stmt); \
923 sqlite3_finalize(select_new_stmt); \
924 sqlite3_finalize(copy_style_items_stmt); \
925 sqlite3_finalize(delete_style_items_stmt); \
926 } while(0)
927
928 stmt = NULL;
929 select_stmt = NULL;
930 update_name_stmt = NULL;
931 insert_stmt = NULL;
932 delete_stmt = NULL;
933 sqlite3_stmt *select_new_stmt = NULL, *copy_style_items_stmt = NULL, *delete_style_items_stmt = NULL;
934 // clang-format off
935 TRY_PREPARE(stmt, "SELECT id, name FROM main.styles", "[init] can't prepare style selection from database\n");
936 TRY_PREPARE(select_stmt, "SELECT rowid FROM data.styles WHERE name = ?1 LIMIT 1",
937 "[init] can't prepare style item selection from database\n");
938 TRY_PREPARE(update_name_stmt, "UPDATE main.styles SET name = ?1 WHERE id = ?2",
939 "[init] can't prepare style name update\n");
940 TRY_PREPARE(insert_stmt, "INSERT INTO data.styles (id, name, description) "
941 "SELECT (SELECT COALESCE(MAX(id),0)+1 FROM data.styles), name, description "
942 "FROM main.styles where id = ?1",
943 "[init] can't prepare style insertion for database\n");
944 TRY_PREPARE(delete_stmt, "DELETE FROM main.styles WHERE id = ?1",
945 "[init] can't prepare style deletion for database\n");
946 TRY_PREPARE(select_new_stmt, "SELECT id FROM data.styles WHERE rowid = ?1",
947 "[init] can't prepare style selection from data database\n");
948 TRY_PREPARE(copy_style_items_stmt, "INSERT INTO data.style_items "
949 "(styleid, num, module, operation, op_params, enabled, blendop_params, "
950 "blendop_version, multi_priority, multi_name) "
951 "SELECT ?1, num, module, operation, op_params, enabled, blendop_params, "
952 "blendop_version, multi_priority, multi_name FROM main.style_items "
953 "WHERE styleid = ?2",
954 "[init] can't prepare style item copy into data database\n");
955 TRY_PREPARE(delete_style_items_stmt, "DELETE FROM main.style_items WHERE styleid = ?1",
956 "[init] can't prepare style item deletion for database\n");
957 // clang-format on
958 while(sqlite3_step(stmt) == SQLITE_ROW)
959 {
960 const int id = sqlite3_column_int(stmt, 0);
961 const char *name = (const char *)sqlite3_column_text(stmt, 1);
962
963 // find a unique name of the style for data.styles
964 sqlite3_bind_text(select_stmt, 1, name, -1, SQLITE_TRANSIENT);
965 if(sqlite3_step(select_stmt) == SQLITE_ROW)
966 {
967 // we need to append a version
968 int style_version = 0;
969 char *new_name = NULL;
970 do
971 {
972 style_version++;
973 dt_free(new_name);
974 new_name = g_strdup_printf("%s #%d", name, style_version);
975 sqlite3_reset(select_stmt);
976 sqlite3_clear_bindings(select_stmt);
977 sqlite3_bind_text(select_stmt, 1, new_name, -1, SQLITE_TRANSIENT);
978 } while(sqlite3_step(select_stmt) == SQLITE_ROW);
979
980 // update the name in the old place
981 sqlite3_bind_text(update_name_stmt, 1, new_name, -1, SQLITE_TRANSIENT);
982 sqlite3_bind_int(update_name_stmt, 2, id);
983 TRY_STEP(update_name_stmt, SQLITE_DONE, "[init] can't update name of style in database\n");
984 sqlite3_reset(update_name_stmt);
985 sqlite3_clear_bindings(update_name_stmt);
986 dt_free(new_name);
987 }
988
989 // move the style to data.styles and get the rowid
990 sqlite3_bind_int(insert_stmt, 1, id);
991 TRY_STEP(insert_stmt, SQLITE_DONE, "[init] can't insert style into data database\n");
992 sqlite3_int64 last_rowid = sqlite3_last_insert_rowid(db->handle);
993
994 // delete style from styles
995 sqlite3_bind_int(delete_stmt, 1, id);
996 TRY_STEP(delete_stmt, SQLITE_DONE, "[init] can't delete style from database\n");
997
998 sqlite3_bind_int(select_new_stmt, 1, last_rowid);
999 TRY_STEP(select_new_stmt, SQLITE_ROW, "[init] can't select new style from data database\n");
1000 const int new_id = sqlite3_column_int(select_new_stmt, 0);
1001
1002 // now that we have the style over in data.styles and the new id we can just copy over all style items
1003 sqlite3_bind_int(copy_style_items_stmt, 1, new_id);
1004 sqlite3_bind_int(copy_style_items_stmt, 2, id);
1005 TRY_STEP(copy_style_items_stmt, SQLITE_DONE, "[init] can't copy style items into data database\n");
1006
1007 // delete the style items from the old table
1008 sqlite3_bind_int(delete_style_items_stmt, 1, id);
1009 TRY_STEP(delete_style_items_stmt, SQLITE_DONE, "[init] can't delete style items from database\n");
1010
1011 // cleanup for the next round
1012 sqlite3_reset(insert_stmt);
1013 sqlite3_clear_bindings(insert_stmt);
1014 sqlite3_reset(select_stmt);
1015 sqlite3_clear_bindings(select_stmt);
1016 sqlite3_reset(delete_stmt);
1017 sqlite3_clear_bindings(delete_stmt);
1018 sqlite3_reset(select_new_stmt);
1019 sqlite3_clear_bindings(select_new_stmt);
1020 sqlite3_reset(copy_style_items_stmt);
1021 sqlite3_clear_bindings(copy_style_items_stmt);
1022 sqlite3_reset(delete_style_items_stmt);
1023 sqlite3_clear_bindings(delete_style_items_stmt);
1024 }
1025 FINALIZE;
1026#undef FINALIZE
1027
1029#define FINALIZE
1030 // clang-format off
1031 // tags
1032 TRY_EXEC("INSERT OR IGNORE INTO data.tags (name, icon, description, flags) "
1033 "SELECT name, icon, description, flags FROM main.tags",
1034 "[init] can't prepare insertion of used tags into data database\n");
1035
1036 // tagged images
1037 // we need a temp table to update tagged_images due to its primary key
1038 TRY_EXEC("CREATE TEMPORARY TABLE tagged_images_tmp (imgid INTEGER, tagid INTEGER)",
1039 "[init] can't create temporary table for updating `tagged_images'\n");
1040
1041 TRY_EXEC("INSERT INTO tagged_images_tmp (imgid, tagid) "
1042 "SELECT imgid, (SELECT t2.id FROM main.tags t1, data.tags t2 USING (name) WHERE t1.id = tagid) "
1043 "FROM main.tagged_images", "[init] can't insert into `tagged_images_tmp'\n");
1044
1045 TRY_EXEC("DELETE FROM main.tagged_images", "[init] can't delete tagged images in database\n");
1046
1047 TRY_EXEC("INSERT OR IGNORE INTO main.tagged_images (imgid, tagid) SELECT imgid, tagid FROM tagged_images_tmp",
1048 "[init] can't copy updated values back to `tagged_images'\n");
1049
1050 TRY_EXEC("DROP TABLE tagged_images_tmp", "[init] can't drop table `tagged_images_tmp' from database\n");
1051
1053 TRY_EXEC("DROP INDEX IF EXISTS main.presets_idx", "[init] can't drop index `presets_idx' from database\n");
1054 TRY_EXEC("DROP TABLE main.presets", "[init] can't drop table `presets' from database\n");
1055 TRY_EXEC("DROP TABLE main.style_items", "[init] can't drop table `style_items' from database\n");
1056 TRY_EXEC("DROP TABLE main.styles", "[init] can't drop table `styles' from database\n");
1057 TRY_EXEC("DROP TABLE main.tags", "[init] can't drop table `tags' from database\n");
1058 // clang-format on
1059 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1060 new_version = 13;
1061 } else if(version == 13)
1062 {
1063 // 12 -> 13 bring back the used tag names to library.db so people can use it independently of data.db
1064 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1065 // clang-format off
1066 TRY_EXEC("CREATE TABLE main.used_tags (id INTEGER, name VARCHAR NOT NULL)",
1067 "[init] can't create `used_tags` table\n");
1068
1069 TRY_EXEC("CREATE INDEX main.used_tags_idx ON used_tags (id, name)",
1070 "[init] can't create index on table `used_tags' in database\n");
1071
1072 TRY_EXEC("INSERT INTO main.used_tags (id, name) SELECT t.id, t.name FROM data.tags AS t, main.tagged_images "
1073 "AS i ON t.id = i.tagid GROUP BY t.id",
1074 "[init] can't insert used tags into `used_tags` table in database\n");
1075 // clang-format on
1076 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1077 new_version = 14;
1078 }
1079 else if(version == 14)
1080 {
1081 // 13 -> fix the index on used_tags to be a UNIQUE index :-/
1082 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1083 // clang-format off
1084 TRY_EXEC("DELETE FROM main.used_tags WHERE rowid NOT IN (SELECT rowid FROM used_tags GROUP BY id)",
1085 "[init] can't delete duplicated entries from `used_tags' in database\n");
1086
1087 TRY_EXEC("DROP INDEX main.used_tags_idx", "[init] can't drop index `used_tags_idx' from database\n");
1088
1089 TRY_EXEC("CREATE UNIQUE INDEX main.used_tags_idx ON used_tags (id, name)",
1090 "[init] can't create index `used_tags_idx' in database\n");
1091
1092 TRY_EXEC("DELETE FROM main.tagged_images WHERE tagid IS NULL",
1093 "[init] can't delete NULL entries from `tagged_images' in database");
1094
1095 TRY_EXEC("DELETE FROM main.used_tags WHERE id NOT IN (SELECT DISTINCT tagid FROM main.tagged_images)",
1096 "[init] can't delete unused tags from `used_tags' in database\n");
1097 // clang-format on
1098 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1099 new_version = 15;
1100 }
1101 else if(version == 15)
1102 {
1103 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1105 // clang-format off
1106 TRY_EXEC("ALTER TABLE main.images ADD COLUMN position INTEGER",
1107 "[init] can't add `position' column to images table in database\n");
1108 TRY_EXEC("CREATE INDEX main.image_position_index ON images (position)",
1109 "[init] can't create index for custom image order table\n");
1110
1111 // Set the initial image sequence. The image id - the sequence images were imported -
1112 // defines the initial order of images.
1113 //
1114 // An int64 is used for the position index. The upper 31 bits define the initial order.
1115 // The lower 32bit provide space to reorder images.
1116 //
1117 // see: dt_collection_move_before()
1118 //
1119 TRY_EXEC("UPDATE main.images SET position = id << 32",
1120 "[init] can't update positions custom image order table\n");
1121 // clang-format on
1122 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1123 new_version = 16;
1124 }
1125 else if(version == 16)
1126 {
1127 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1129 // clang-format off
1130 TRY_EXEC("ALTER TABLE main.images ADD COLUMN aspect_ratio REAL",
1131 "[init] can't add `aspect_ratio' column to images table in database\n");
1132 TRY_EXEC("UPDATE main.images SET aspect_ratio = 0.0",
1133 "[init] can't update aspect_ratio in database\n");
1134 // clang-format on
1135 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1136 new_version = 17;
1137 }
1138 else if(version == 17)
1139 {
1140 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1141
1143 // clang-format off
1144 TRY_EXEC("CREATE TABLE main.masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, name VARCHAR(256), "
1145 "version INTEGER, points BLOB, points_count INTEGER, source BLOB)",
1146 "[init] can't create `masks_history` table\n");
1147
1148 TRY_EXEC("CREATE INDEX main.masks_history_imgid_index ON masks_history (imgid)",
1149 "[init] can't create index `masks_history_imgid_index' in database\n");
1150
1151 // to speed up the mask look-up, and makes the following UPDATE instantaneous whereas it could takes hours
1152 TRY_EXEC("CREATE INDEX main.mask_imgid_index ON mask (imgid);",
1153 "[init] can't create index `mask_imgid_index' in database\n");
1154
1155 // create a mask manager entry on history for all images containing all forms
1156 // make room for mask manager history entry
1157 TRY_EXEC("UPDATE main.history SET num=num+1 WHERE imgid IN (SELECT imgid FROM main.mask WHERE main.mask.imgid=main.history.imgid)",
1158 "[init] can't update `num' with num+1\n");
1159
1160 // update history end
1161 TRY_EXEC("UPDATE main.images SET history_end = history_end+1 WHERE id IN (SELECT imgid FROM main.mask WHERE main.mask.imgid=main.images.id)",
1162 "[init] can't update `history_end' with history_end+1\n");
1163
1164 // copy all masks into history
1165 TRY_EXEC("INSERT INTO main.masks_history (imgid, num, formid, form, name, version, points, points_count, source) SELECT "
1166 "imgid, 0, formid, form, name, version, points, points_count, source FROM main.mask",
1167 "[init] can't insert into masks_history\n");
1168
1169 // create a mask manager entry for each image that has masks
1170 TRY_EXEC("INSERT INTO main.history (imgid, num, operation, op_params, module, enabled, "
1171 "blendop_params, blendop_version, multi_priority, multi_name) "
1172 "SELECT DISTINCT imgid, 0, 'mask_manager', NULL, 1, 0, NULL, 0, 0, '' FROM main.mask "
1173 "GROUP BY imgid",
1174 "[init] can't insert mask manager into history\n");
1175
1176 TRY_EXEC("DROP TABLE main.mask", "[init] can't drop table `mask' from database\n");
1177
1179 GList *prior_v1 = dt_ioppr_get_iop_order_list_version(DT_IOP_ORDER_LEGACY);
1180
1181 TRY_EXEC("ALTER TABLE main.images ADD COLUMN iop_order_version INTEGER",
1182 "[init] can't add `iop_order_version' column to images table in database\n");
1183
1184 TRY_EXEC("UPDATE main.images SET iop_order_version = 0",
1185 "[init] can't update iop_order_version in database\n");
1186
1187 TRY_EXEC("UPDATE main.images SET iop_order_version = 1 WHERE "
1188 "EXISTS(SELECT * FROM main.history WHERE main.history.imgid = main.images.id)",
1189 "[init] can't update iop_order_version in database\n");
1190
1191 TRY_EXEC("ALTER TABLE main.history ADD COLUMN iop_order REAL",
1192 "[init] can't add `iop_order' column to history table in database\n");
1193
1194 // create a temp table with the previous priorities
1195 TRY_EXEC("CREATE TEMPORARY TABLE iop_order_tmp (iop_order REAL, operation VARCHAR(256))",
1196 "[init] can't create temporary table for updating `main.history'\n");
1197 // clang-format on
1198 // fill temp table with all operations up to this release
1199 // it will be used to create the pipe and update the iop_order on history
1200 for(GList *priorities = prior_v1; priorities; priorities = g_list_next(priorities))
1201 {
1202 dt_iop_order_entry_t *prior = (dt_iop_order_entry_t *)priorities->data;
1203
1204 sqlite3_prepare_v2(
1205 db->handle,
1206 "INSERT INTO iop_order_tmp (iop_order, operation) VALUES (?1, ?2)",
1207 -1, &stmt, NULL);
1208 sqlite3_bind_double(stmt, 1, prior->o.iop_order_f);
1209 sqlite3_bind_text(stmt, 2, prior->operation, -1, SQLITE_TRANSIENT);
1210 TRY_STEP(stmt, SQLITE_DONE, "[init] can't insert default value in iop_order_tmp\n");
1211 sqlite3_finalize(stmt);
1212 }
1213 g_list_free_full(prior_v1, dt_free_gpointer);
1214 prior_v1 = NULL;
1215
1216 // create the order of the pipe
1217 // iop_order is by default the module priority
1218 // if there's multi-instances we add the multi_priority
1219 // multi_priority is in reverse order in this version,
1220 // so we assume that is always less than 1000 and reverse it
1221 // it is possible that multi_priority = 0 don't appear in history
1222 // so just in case 1 / 1000 to every instance
1223 // clang-format off
1224 TRY_EXEC("UPDATE main.history SET iop_order = ((("
1225 "SELECT MAX(multi_priority) FROM main.history hist1 WHERE hist1.imgid = main.history.imgid AND hist1.operation = main.history.operation "
1226 ") + 1. - multi_priority) / 1000.) + "
1227 "IFNULL((SELECT iop_order FROM iop_order_tmp WHERE iop_order_tmp.operation = "
1228 "main.history.operation), -999999.) ",
1229 "[init] can't update iop_order in history table\n");
1230
1231 // check if there's any entry in history that was not updated
1232 sqlite3_stmt *sel_stmt;
1233 TRY_PREPARE(sel_stmt, "SELECT DISTINCT operation FROM main.history WHERE iop_order <= 0 OR iop_order IS NULL",
1234 "[init] can't prepare selecting history iop_order\n");
1235 // clang-format on
1236 while(sqlite3_step(sel_stmt) == SQLITE_ROW)
1237 {
1238 const char *op_name = (const char *)sqlite3_column_text(sel_stmt, 0);
1239 printf("operation %s with no iop_order while upgrading database\n", op_name);
1240 }
1241 sqlite3_finalize(sel_stmt);
1242 // clang-format off
1243 TRY_EXEC("DROP TABLE iop_order_tmp", "[init] can't drop table `iop_order_tmp' from database\n");
1244 // clang-format on
1245 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1246 new_version = 18;
1247 }
1248 // maybe in the future, see commented out code elsewhere
1249 // else if(version == XXX)
1250 // {
1251 // sqlite3_exec(db->handle, "ALTER TABLE film_rolls ADD COLUMN external_drive VARCHAR(1024)", NULL,
1252 // NULL, NULL);
1253 // }
1254 else if(version == 18)
1255 {
1256 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1257 // clang-format off
1258 TRY_EXEC("UPDATE images SET orientation=-2 WHERE orientation=1;",
1259 "[init] can't update images orientation 1 from database\n");
1260
1261 TRY_EXEC("UPDATE images SET orientation=1 WHERE orientation=2;",
1262 "[init] can't update images orientation 2 from database\n");
1263
1264 TRY_EXEC("UPDATE images SET orientation=-6 WHERE orientation=5;",
1265 "[init] can't update images orientation 5 from database\n");
1266
1267 TRY_EXEC("UPDATE images SET orientation=5 WHERE orientation=6;",
1268 "[init] can't update images orientation 6 from database\n");
1269
1270 TRY_EXEC("UPDATE images SET orientation=2 WHERE orientation=-2;",
1271 "[init] can't update images orientation -1 from database\n");
1272
1273 TRY_EXEC("UPDATE images SET orientation=6 WHERE orientation=-6;",
1274 "[init] can't update images orientation -6 from database\n");
1275 // clang-format on
1276 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1277 new_version = 19;
1278 }
1279 else if(version == 19)
1280 {
1281 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1282 // clang-format off
1283 // create a temp table to invert all multi_priority
1284 TRY_EXEC("CREATE TEMPORARY TABLE m_prio (id INTEGER, operation VARCHAR(256), prio INTEGER)",
1285 "[init] can't create temporary table for updating `history and style_items'\n");
1286
1287 TRY_EXEC("CREATE INDEX m_prio_id_index ON m_prio (id)",
1288 "[init] can't create temporary index for updating `history and style_items'\n");
1289 TRY_EXEC("CREATE INDEX m_prio_op_index ON m_prio (operation)",
1290 "[init] can't create temporary index for updating `history and style_items'\n");
1291
1292 TRY_EXEC("INSERT INTO m_prio SELECT imgid, operation, MAX(multi_priority)"
1293 " FROM main.history GROUP BY imgid, operation",
1294 "[init] can't populate m_prio\n");
1295
1296 TRY_EXEC("UPDATE main.history SET multi_priority = "
1297 "(SELECT prio FROM m_prio "
1298 " WHERE main.history.operation = operation AND main.history.imgid = id) - main.history.multi_priority",
1299 "[init] can't update multi_priority for history\n");
1300
1301 TRY_EXEC("DROP TABLE m_prio", "[init] can't drop table `m_prio' from database\n");
1302 // clang-format on
1303 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1304 new_version = 20;
1305 }
1306 else if(version == 20)
1307 {
1308 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1309 // clang-format off
1310 TRY_EXEC("DROP INDEX IF EXISTS main.used_tags_idx", "[init] can't drop index `used_tags_idx' from database\n");
1311 TRY_EXEC("DROP TABLE used_tags", "[init] can't delete table used_tags\n");
1312 // clang-format on
1313 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1314
1315 new_version = 21;
1316 }
1317 else if(version == 21)
1318 {
1319 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1320 // create a temp table to invert all multi_priority
1321 // clang-format off
1322 TRY_EXEC("CREATE TABLE module_order (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR)",
1323 "[init] can't create module_order table'\n");
1324
1325 // for all images:
1326 sqlite3_stmt *mig_stmt;
1327 TRY_PREPARE(mig_stmt, "SELECT imgid, operation, multi_priority, iop_order, mi.iop_order_version"
1328 " FROM main.history AS hi, main.images AS mi"
1329 " WHERE hi.imgid = mi.id"
1330 " GROUP BY imgid, operation, multi_priority"
1331 " ORDER BY imgid, iop_order",
1332 "[init] can't prepare selecting history for iop_order migration (v21)\n");
1333 // clang-format on
1334 GList *item_list = NULL;
1335 int current_imgid = UNKNOWN_IMAGE;
1336 int current_order_version = -1;
1337
1338 gboolean has_row = (sqlite3_step(mig_stmt) == SQLITE_ROW);
1339
1340 while(has_row)
1341 {
1342 const int32_t imgid = sqlite3_column_int(mig_stmt, 0);
1343 char operation[20] = { 0 };
1344 g_strlcpy(operation, (const char *)sqlite3_column_text(mig_stmt, 1), sizeof(operation));
1345 const int multi_priority = sqlite3_column_int(mig_stmt, 2);
1346 const double iop_order = sqlite3_column_double(mig_stmt, 3);
1347 const int iop_order_version = sqlite3_column_int(mig_stmt, 4);
1348
1349 has_row = (sqlite3_step(mig_stmt) == SQLITE_ROW);
1350
1351 // a new image, let's initialize the iop_order_version
1352 if(imgid != current_imgid || !has_row)
1353 {
1354 // new image, let's handle it
1355 if(!IS_NULL_PTR(item_list))
1356 {
1357 // we keep legacy, everything else is migrated to v3.0
1358 const dt_iop_order_t new_order_version = current_order_version == 2 ? DT_IOP_ORDER_LEGACY : DT_IOP_ORDER_ANSEL_RAW;
1359
1360 GList *iop_order_list = dt_ioppr_get_iop_order_list_version(new_order_version);
1361
1362 // merge entries into iop_order_list
1363
1364 // first remove all item_list iop from the iop_order_list
1365
1366 GList *e = item_list;
1367 GList *n = NULL;
1368 dt_iop_order_entry_t *n_entry = NULL;
1369
1370 while(e)
1371 {
1372 dt_iop_order_entry_t *e_entry = (dt_iop_order_entry_t *)e->data;
1373
1374 GList *s = iop_order_list;
1375 while(s && strcmp(((dt_iop_order_entry_t *)s->data)->operation, e_entry->operation))
1376 {
1377 s = g_list_next(s);
1378 }
1379 if(s)
1380 {
1381 iop_order_list = g_list_delete_link(iop_order_list, s);
1382 }
1383
1384 // skip all multipe instances
1385 n = e;
1386 do
1387 {
1388 n = g_list_next(n);
1389 if(IS_NULL_PTR(n)) break;
1390 n_entry = (dt_iop_order_entry_t *)n->data;
1391 } while(!strcmp(n_entry->operation, e_entry->operation));
1392 e = n;
1393 }
1394
1395 // then add all item_list into iop_order_list
1396
1397 for(e = item_list; e; e = g_list_next(e))
1398 {
1399 dt_iop_order_entry_t *e_entry = (dt_iop_order_entry_t *)e->data;
1400 iop_order_list = g_list_prepend(iop_order_list, e_entry);
1401 }
1402
1403 // and finally reorder the full list based on the iop-order
1404
1405 iop_order_list = g_list_sort(iop_order_list, dt_sort_iop_list_by_order_f);
1406
1407 const dt_iop_order_t kind = dt_ioppr_get_iop_order_list_kind(iop_order_list);
1408
1409 // check if we have some multi-instances
1410
1411 gboolean has_multiple_instances = FALSE;
1412 GList *l = iop_order_list;
1413
1414 while(l)
1415 {
1416 GList *next = g_list_next(l);
1417 if(next
1418 && (strcmp(((dt_iop_order_entry_t *)(l->data))->operation,
1419 ((dt_iop_order_entry_t *)(next->data))->operation) == 0))
1420 {
1421 has_multiple_instances = TRUE;
1422 break;
1423 }
1424 l = next;
1425 }
1426
1427 // write iop_order_list and/or version into module_order
1428
1429 sqlite3_stmt *ins_stmt = NULL;
1430 if(kind == DT_IOP_ORDER_CUSTOM || has_multiple_instances)
1431 {
1432 char *iop_list_txt = dt_ioppr_serialize_text_iop_order_list(iop_order_list);
1433
1434 sqlite3_prepare_v2(db->handle,
1435 "INSERT INTO module_order VALUES (?1, ?2, ?3)", -1,
1436 &ins_stmt, NULL);
1437 sqlite3_bind_int(ins_stmt, 1, current_imgid);
1438 sqlite3_bind_int(ins_stmt, 2, kind);
1439 sqlite3_bind_text(ins_stmt, 3, iop_list_txt, -1, SQLITE_TRANSIENT);
1440 TRY_STEP(ins_stmt, SQLITE_DONE, "[init] can't insert into module_order (custom order)\n");
1441 sqlite3_finalize(ins_stmt);
1442
1443 dt_free(iop_list_txt);
1444 }
1445 else
1446 {
1447 sqlite3_prepare_v2(db->handle,
1448 "INSERT INTO module_order VALUES (?1, ?2, NULL)", -1,
1449 &ins_stmt, NULL);
1450 sqlite3_bind_int(ins_stmt, 1, current_imgid);
1451 sqlite3_bind_int(ins_stmt, 2, kind);
1452 TRY_STEP(ins_stmt, SQLITE_DONE, "[init] can't insert into module_order (standard order)\n");
1453 sqlite3_finalize(ins_stmt);
1454 }
1455
1456 g_list_free(item_list);
1457 item_list = NULL;
1458 g_list_free_full(iop_order_list, dt_free_gpointer);
1459 iop_order_list = NULL;
1460
1461 item_list = NULL;
1462 }
1463
1464 current_imgid = imgid;
1465 current_order_version = iop_order_version;
1466 }
1467
1468 dt_iop_order_entry_t *item = (dt_iop_order_entry_t *)malloc(sizeof(dt_iop_order_entry_t));
1469 memcpy(item->operation, operation, sizeof(item->operation));
1470 item->instance = multi_priority;
1471 item->o.iop_order_f = iop_order; // used to order the enties only
1472 item_list = g_list_append(item_list, item);
1473 }
1474 sqlite3_finalize(mig_stmt);
1475
1476 // remove iop_order from history table
1477 // clang-format off
1478 TRY_EXEC("CREATE TABLE h (imgid INTEGER, num INTEGER, module INTEGER, "
1479 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
1480 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
1481 "[init] can't create module_order table\n");
1482 TRY_EXEC("CREATE INDEX h_imgid_index ON h (imgid)",
1483 "[init] can't create index h_imgid_index\n");
1484 TRY_EXEC("INSERT INTO h SELECT imgid, num, module, operation, op_params, enabled, "
1485 "blendop_params, blendop_version, multi_priority, multi_name FROM main.history",
1486 "[init] can't create module_order table\n");
1487 TRY_EXEC("DROP TABLE history",
1488 "[init] can't drop table history\n");
1489 TRY_EXEC("ALTER TABLE h RENAME TO history",
1490 "[init] can't rename h to history\n");
1491 TRY_EXEC("DROP INDEX h_imgid_index",
1492 "[init] can't drop index h_imgid_index\n");
1493 TRY_EXEC("CREATE INDEX main.history_imgid_index ON history (imgid)",
1494 "[init] can't create index images_imgid_index\n");
1495
1496 // remove iop_order_version from images
1497
1498 TRY_EXEC("CREATE TABLE i (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1499 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
1500 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
1501 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
1502 "output_width INTEGER, output_height INTEGER, crop REAL, "
1503 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
1504 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
1505 "caption VARCHAR, description VARCHAR, license VARCHAR, sha1sum CHAR(40), "
1506 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
1507 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
1508 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, aspect_ratio REAL)",
1509 "[init] can't create table i\n");
1510 TRY_EXEC("INSERT INTO i SELECT id, group_id, film_id, width, height, filename, maker, model,"
1511 " lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags,"
1512 " output_width, output_height, crop, raw_parameters, raw_denoise_threshold,"
1513 " raw_auto_bright_threshold, raw_black, raw_maximum, caption, description, license, sha1sum,"
1514 " orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version,"
1515 " max_version, write_timestamp, history_end, position, aspect_ratio "
1516 "FROM images",
1517 "[init] can't populate table h\n");
1518 TRY_EXEC("DROP TABLE images",
1519 "[init] can't drop table images\n");
1520 TRY_EXEC("ALTER TABLE i RENAME TO images",
1521 "[init] can't rename i to images\n");
1522 // clang-format on
1523 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1524
1525 new_version = 22;
1526 }
1527 else if(version == 22)
1528 {
1529 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1530 // clang-format off
1531 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_group_id_index ON images (group_id)",
1532 "[init] can't create group_id index on image\n");
1533 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_film_id_index ON images (film_id)",
1534 "[init] can't create film_id index on image\n");
1535 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_filename_index ON images (filename)",
1536 "[init] can't create filename index on image\n");
1537 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.image_position_index ON images (position)",
1538 "[init] can't create position index on image\n");
1539
1540 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.film_rolls_folder_index ON film_rolls (folder)",
1541 "[init] can't create folder index on film_rolls\n");
1542 // clang-format on
1543 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1544
1545 new_version = 23;
1546 }
1547 else if(version == 23)
1548 {
1549 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1550 // clang-format off
1551 TRY_EXEC("CREATE TABLE main.history_hash (imgid INTEGER PRIMARY KEY, "
1552 "basic_hash BLOB, auto_hash BLOB, current_hash BLOB)",
1553 "[init] can't create table history_hash\n");
1554 // clang-format on
1555
1556 // history_hash backfill is deferred: it will be filled through the image cache.
1557
1558 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1559
1560 new_version = 24;
1561 }
1562 else if(version == 24)
1563 {
1564 // clang-format off
1565 TRY_EXEC("ALTER TABLE main.history_hash ADD COLUMN mipmap_hash BLOB",
1566 "[init] can't add `mipmap_hash' column to history_hash table in database\n");
1567 // clang-format on
1568
1569 new_version = 25;
1570 }
1571 else if(version == 25)
1572 {
1573 // clang-format of
1574 TRY_EXEC("ALTER TABLE main.images ADD COLUMN exposure_bias REAL",
1575 "[init] can't add `exposure_bias' column to images table in database\n");
1576 // clang-format on
1577
1578 new_version = 26;
1579 }
1580 else if(version == 26)
1581 {
1582 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1583 // clang-format off
1584 TRY_EXEC("CREATE TABLE main.new_film_rolls "
1585 "(id INTEGER PRIMARY KEY, "
1586 "access_timestamp INTEGER, "
1587 "folder VARCHAR(1024) NOT NULL)",
1588 "[init] can't create new_film_rolls table\n");
1589
1590 TRY_EXEC("INSERT INTO main.new_film_rolls"
1591 "(id, access_timestamp, folder) "
1592 "SELECT id, "
1593 "strftime('%s', replace(substr(datetime_accessed, 1, 10), ':', '-') || substr(datetime_accessed, 11), 'utc'), "
1594 "folder "
1595 "FROM film_rolls "
1596 "WHERE folder IS NOT NULL",
1597 "[init] can't populate new_film_rolls table from film_rolls\n");
1598
1599 TRY_EXEC("DROP TABLE film_rolls",
1600 "[init] can't delete table film_rolls\n");
1601
1602 TRY_EXEC("ALTER TABLE main.new_film_rolls RENAME TO film_rolls",
1603 "[init] can't rename table new_film_rolls to film_rolls\n");
1604
1605 TRY_EXEC("CREATE INDEX main.film_rolls_folder_index ON film_rolls (folder)",
1606 "[init] can't create index `film_rolls_folder_index' on table `film_rolls'\n");
1607 // clang-format on
1608 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1609 new_version = 27;
1610 }
1611 else if(version == 27)
1612 {
1613 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1614 // clang-format off
1615 TRY_EXEC("ALTER TABLE main.images ADD COLUMN import_timestamp INTEGER DEFAULT -1",
1616 "[init] can't add `import_timestamp' column to images table in database\n");
1617 TRY_EXEC("ALTER TABLE main.images ADD COLUMN change_timestamp INTEGER DEFAULT -1",
1618 "[init] can't add `change_timestamp' column to images table in database\n");
1619 TRY_EXEC("ALTER TABLE main.images ADD COLUMN export_timestamp INTEGER DEFAULT -1",
1620 "[init] can't add `export_timestamp' column to images table in database\n");
1621 TRY_EXEC("ALTER TABLE main.images ADD COLUMN print_timestamp INTEGER DEFAULT -1",
1622 "[init] can't add `print_timestamp' column to images table in database\n");
1623
1624 TRY_EXEC("UPDATE main.images SET import_timestamp = (SELECT access_timestamp "
1625 "FROM main.film_rolls WHERE film_rolls.id = images.film_id)",
1626 "[init] can't populate import_timestamp column from film_rolls.access_timestamp.\n");
1627
1628 TRY_EXEC("UPDATE main.images SET change_timestamp = images.write_timestamp "
1629 "WHERE images.write_timestamp IS NOT NULL "
1630 "AND images.id = (SELECT imgid FROM tagged_images "
1631 "JOIN data.tags ON tags.id = tagged_images.tagid "
1632 "WHERE data.tags.name = 'darktable|changed')",
1633 "[init] can't populate change_timestamp column from images.write_timestamp.\n");
1634 // clang-format on
1635 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1636 new_version = 28;
1637 }
1638 else if(version == 28)
1639 {
1640 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1641 // clang-format off
1642 // clear flag DT_IMAGE_REJECTED (was not used)
1643 TRY_EXEC("UPDATE main.images SET flags = (flags & ~8)",
1644 "[init] can't clear rejected flags");
1645
1646 // add DT_IMAGE_REJECTED and clear rating for all images being rejected
1647 TRY_EXEC("UPDATE main.images SET flags = (flags | 8) & ~7 WHERE (flags & 7) = 6",
1648 "[init] can't set rejected flags");
1649 // clang-format on
1650 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1651 new_version = 29;
1652 }
1653 else if(version == 29)
1654 {
1655 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1656 // clang-format off
1657 // add position in tagged_images table
1658 TRY_EXEC("ALTER TABLE main.tagged_images ADD COLUMN position INTEGER",
1659 "[init] can't add `position' column to tagged_images table in database\n");
1660
1661 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.tagged_images_imgid_index ON tagged_images (imgid)",
1662 "[init] can't create image index on tagged_images\n");
1663 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.tagged_images_position_index ON tagged_images (position)",
1664 "[init] can't create position index on tagged_images\n");
1665 TRY_EXEC("UPDATE main.tagged_images SET position = (tagid + imgid) << 32",
1666 "[init] can't populate position on tagged_images\n");
1667
1668 // remove caption and description fields from images table
1669
1670 TRY_EXEC("CREATE TABLE main.i (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1671 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
1672 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
1673 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
1674 "output_width INTEGER, output_height INTEGER, crop REAL, "
1675 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
1676 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
1677 "license VARCHAR, sha1sum CHAR(40), "
1678 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
1679 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
1680 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
1681 "aspect_ratio REAL, exposure_bias REAL, "
1682 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, "
1683 "export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1)",
1684 "[init] can't create table i\n");
1685
1686 TRY_EXEC("INSERT INTO main.i SELECT id, group_id, film_id, width, height, filename, maker, model,"
1687 " lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags,"
1688 " output_width, output_height, crop, raw_parameters, raw_denoise_threshold,"
1689 " raw_auto_bright_threshold, raw_black, raw_maximum, license, sha1sum,"
1690 " orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version,"
1691 " max_version, write_timestamp, history_end, position, aspect_ratio, exposure_bias,"
1692 " import_timestamp, change_timestamp, export_timestamp, print_timestamp "
1693 "FROM main.images",
1694 "[init] can't populate table i\n");
1695 TRY_EXEC("DROP TABLE main.images",
1696 "[init] can't drop table images\n");
1697 TRY_EXEC("ALTER TABLE main.i RENAME TO images",
1698 "[init] can't rename i to images\n");
1699
1700 TRY_EXEC("CREATE INDEX main.images_group_id_index ON images (group_id)",
1701 "[init] can't create group_id index on images table\n");
1702 TRY_EXEC("CREATE INDEX main.images_film_id_index ON images (film_id)",
1703 "[init] can't create film_id index on images table\n");
1704 TRY_EXEC("CREATE INDEX main.images_filename_index ON images (filename)",
1705 "[init] can't create filename index on images table\n");
1706 TRY_EXEC("CREATE INDEX main.image_position_index ON images (position)",
1707 "[init] can't create position index on images table\n");
1708 // clang-format on
1709 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1710 new_version = 30;
1711 }
1712 else if(version == 30)
1713 {
1714 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1715 // clang-format off
1716 // add second columns to speed up sorting
1717 TRY_EXEC("DROP INDEX IF EXISTS `history_imgid_index`",
1718 "[init] can't drop history_imgid_index\n");
1719 TRY_EXEC("CREATE INDEX `history_imgid_index` ON `history` ( `imgid`, `operation` )",
1720 "[init] can't recreate history_imgid_index\n");
1721
1722 TRY_EXEC("DROP INDEX IF EXISTS `images_filename_index`",
1723 "[init] can't drop images_filename_index\n");
1724 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
1725 "[init] can't recreate images_filename_index\n");
1726
1727 TRY_EXEC("DROP INDEX IF EXISTS `images_film_id_index`",
1728 "[init] can't drop images_film_id_index\n");
1729 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
1730 "[init] can't recreate images_film_id_index\n");
1731
1732 TRY_EXEC("DROP INDEX IF EXISTS `images_group_id_index`",
1733 "[init] can't drop images_group_id_index\n");
1734 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
1735 "[init] can't recreate images_group_id_index\n");
1736
1737 TRY_EXEC("DROP INDEX IF EXISTS `masks_history_imgid_index`",
1738 "[init] can't drop masks_history_imgid_index\n");
1739 TRY_EXEC("CREATE INDEX `masks_history_imgid_index` ON `masks_history` ( `imgid`, `num` )",
1740 "[init] can't recreate masks_history_imgid_index\n");
1741
1742 // map refinement: avoid full table scan
1743 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( `latitude` DESC, `longitude` DESC )",
1744 "[init] can't create images_latlong_index\n");
1745 // clang-format on
1746 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1747 new_version = 31;
1748 }
1749 else if(version == 31)
1750 {
1751 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1752 // clang-format off
1753 // remove duplicates
1754 TRY_EXEC("DELETE FROM main.meta_data WHERE rowid NOT IN (SELECT MIN(rowid) "
1755 "FROM main.meta_data GROUP BY id, key)",
1756 "[init] can't remove duplicates from meta_data\n");
1757
1758 // recreate the index with UNIQUE option
1759 TRY_EXEC("DROP INDEX IF EXISTS metadata_index",
1760 "[init] can't drop metadata_index\n");
1761 TRY_EXEC("CREATE UNIQUE INDEX main.metadata_index ON meta_data (id, key)",
1762 "[init] can't create metadata_index\n");
1763 // clang-format on
1764 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1765 new_version = 32;
1766 }
1767 else if(version == 32)
1768 {
1769 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1770 // clang-format off
1771 // add foreign keys for database consistency. ON UPDATE CASCADE since you never know
1772 // if a future version will change image_id
1773 // Unfortunately sqlite does not support adding foreign keys to existing tables
1774 // so we have to rename the existing tables, recreate them and copy back the old values
1775 // images first
1776 // needs to delete orphaned entries
1777 TRY_EXEC("ALTER TABLE `images` RENAME TO `images_old`",
1778 "[init] can't rename images\n");
1779
1780 TRY_EXEC("CREATE TABLE `images` (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1781 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, lens VARCHAR, "
1782 "exposure REAL, aperture REAL, iso REAL, focal_length REAL, focus_distance REAL, datetime_taken CHAR(20), "
1783 "flags INTEGER, output_width INTEGER, output_height INTEGER, crop REAL, "
1784 "raw_parameters INTEGER, raw_denoise_threshold REAL, raw_auto_bright_threshold REAL, "
1785 "raw_black INTEGER, raw_maximum INTEGER, license VARCHAR, sha1sum CHAR(40), "
1786 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, latitude REAL, altitude REAL, "
1787 "color_matrix BLOB, colorspace INTEGER, version INTEGER, max_version INTEGER, write_timestamp INTEGER, "
1788 "history_end INTEGER, position INTEGER, aspect_ratio REAL, exposure_bias REAL, "
1789 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1, "
1790 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
1791 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
1792 "[init] can't create new images table\n");
1793
1794 // corner case: database inconsistency with images having invalid film id
1795 TRY_EXEC("DELETE FROM `images_old` WHERE film_id NOT IN (SELECT id FROM `film_rolls`)",
1796 "[init] can't delete images with invalid film id\n");
1797
1798 TRY_EXEC("UPDATE `images_old` SET group_id=id WHERE group_id NOT IN (SELECT id from `images_old`)",
1799 "[init] can't fix invalid group ids\n");
1800
1801 TRY_EXEC("INSERT INTO `images` SELECT * FROM `images_old`",
1802 "[init] can't copy back from images_old\n");
1803
1804 // pita: need to recreate index
1805 TRY_EXEC("DROP INDEX IF EXISTS `image_position_index`",
1806 "[init] can't drop image_position_index\n");
1807 TRY_EXEC("CREATE INDEX `image_position_index` ON `images` (position)",
1808 "[init] can't add image_position_index\n");
1809
1810 // second columns
1811 TRY_EXEC("DROP INDEX IF EXISTS `images_filename_index`",
1812 "[init] can't drop images_filename_index\n");
1813 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
1814 "[init] can't recreate images_filename_index\n");
1815
1816 TRY_EXEC("DROP INDEX IF EXISTS `images_film_id_index`",
1817 "[init] can't drop images_film_id_index\n");
1818 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
1819 "[init] can't recreate images_film_id_index\n");
1820
1821 TRY_EXEC("DROP INDEX IF EXISTS `images_group_id_index`",
1822 "[init] can't drop images_group_id_index\n");
1823 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
1824 "[init] can't recreate images_group_id_index\n");
1825
1826 TRY_EXEC("DROP INDEX IF EXISTS `images_latlong_index`",
1827 "[init] can't drop images_latlong_index\n");
1828 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( latitude DESC, longitude DESC )",
1829 "[init] can't add images_latlong_index\n");
1830
1831 TRY_EXEC("DROP TABLE `images_old`",
1832 "[init] can't drop table images_old\n");
1833
1834 // history
1835 TRY_EXEC("ALTER TABLE `history` RENAME TO `history_old`",
1836 "[init] can't rename history\n");
1837
1838 TRY_EXEC("CREATE TABLE `history` (imgid INTEGER, num INTEGER, module INTEGER, "
1839 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, "
1840 "multi_priority INTEGER, multi_name VARCHAR(256), "
1841 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1842 "[init] can't create new history table\n");
1843
1844 TRY_EXEC("DELETE FROM `history_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1845 "[init] can't delete orphaned history elements\n");
1846
1847 TRY_EXEC("INSERT INTO history SELECT * FROM history_old",
1848 "[init] can't copy back from history_old\n");
1849
1850 TRY_EXEC("DROP INDEX IF EXISTS `history_imgid_index`",
1851 "[init] can't drop history_imgid_index\n");
1852 TRY_EXEC("CREATE INDEX `history_imgid_op_index` ON `history` ( `imgid`, `operation` )",
1853 "[init] can't recreate history_imgid_index\n");
1854 TRY_EXEC("CREATE INDEX `history_imgid_num_index` ON `history` ( `imgid`, `num` DESC )",
1855 "[init] can't recreate history_imgid_index\n");
1856
1857 TRY_EXEC("DROP TABLE `history_old`",
1858 "[init] can't drop table history_old\n");
1859
1860 // history hash
1861 TRY_EXEC("ALTER TABLE `history_hash` RENAME TO `history_hash_old`",
1862 "[init] can't rename history_hash\n");
1863
1864 TRY_EXEC("CREATE TABLE `history_hash` (imgid INTEGER PRIMARY KEY, basic_hash BLOB, auto_hash BLOB, current_hash BLOB, "
1865 "mipmap_hash BLOB, FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1866 "[init] can't create new history_hash table\n");
1867
1868 TRY_EXEC("DELETE FROM `history_hash_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1869 "[init] can't delete orphaned history_hash elements\n");
1870
1871 TRY_EXEC("INSERT INTO `history_hash` SELECT * FROM `history_hash_old`",
1872 "[init] can't copy back from history_hash_old\n");
1873
1874 TRY_EXEC("DROP TABLE `history_hash_old`",
1875 "[init] can't drop table history_hash_old\n");
1876
1877 // tagged images
1878 TRY_EXEC("ALTER TABLE `tagged_images` RENAME TO `tagged_images_old`",
1879 "[init] can't rename tagged_images\n");
1880
1881 TRY_EXEC("CREATE TABLE `tagged_images` (imgid integer, tagid integer, position INTEGER, "
1882 "primary key(imgid, tagid), FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1883 "[init] can't create new tagged_images table\n");
1884
1885 TRY_EXEC("DELETE FROM `tagged_images_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1886 "[init] can't delete orphaned tagged_images elements\n");
1887
1888 TRY_EXEC("INSERT INTO `tagged_images` SELECT * FROM `tagged_images_old`",
1889 "[init] can't copy back from tagged_images_old\n");
1890
1891 // old indices
1892 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_imgid_index",
1893 "[init] can't drop tagged_images_imgid_index\n");
1894 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_position_index",
1895 "[init] can't drop tagged_images_position_index\n");
1896 TRY_EXEC("CREATE INDEX tagged_images_position_index ON tagged_images (position)",
1897 "[init] can't add index tagged_images_position_index\n");
1898 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_tagid_index",
1899 "[init] can't drop tagged_images_tagid_index\n");
1900 TRY_EXEC("CREATE INDEX tagged_images_tagid_index ON tagged_images (tagid)",
1901 "[init] can't add index tagged_images_tagid_index\n");
1902
1903 TRY_EXEC("DROP TABLE `tagged_images_old`",
1904 "[init] can't drop table tagged_images_old\n");
1905
1906 // masks history
1907 TRY_EXEC("ALTER TABLE `masks_history` RENAME TO `masks_history_old`",
1908 "[init] can't rename masks_history\n");
1909
1910 TRY_EXEC("CREATE TABLE masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, "
1911 "name VARCHAR(256), version INTEGER, points BLOB, points_count INTEGER, source BLOB, "
1912 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1913 "[init] can't create new masks_history table\n");
1914
1915 TRY_EXEC("DELETE FROM `masks_history_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1916 "[init] can't delete orphaned masks_history elements\n");
1917
1918 TRY_EXEC("INSERT INTO `masks_history` SELECT * FROM `masks_history_old`",
1919 "[init] can't copy back from masks_history\n");
1920
1921 TRY_EXEC("DROP INDEX IF EXISTS `masks_history_imgid_index`",
1922 "[init] can't drop masks_history_imgid_index\n");
1923 TRY_EXEC("CREATE INDEX `masks_history_imgid_index` ON `masks_history` ( imgid, num )",
1924 "[init] can't recreate masks_history_imgid_index\n");
1925
1926 TRY_EXEC("DROP TABLE masks_history_old",
1927 "[init] can't drop table masks_history_old\n");
1928
1929 // color labels
1930 TRY_EXEC("ALTER TABLE `color_labels` RENAME TO `color_labels_old`",
1931 "[init] can't rename color_labels\n");
1932 TRY_EXEC("CREATE TABLE `color_labels` (imgid INTEGER, color INTEGER, "
1933 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1934 "[init] can't create new color_labels table\n");
1935
1936 TRY_EXEC("DELETE FROM `color_labels_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1937 "[init] can't delete orphaned color_labels elements\n");
1938
1939 TRY_EXEC("INSERT INTO `color_labels` SELECT * FROM `color_labels_old`",
1940 "[init] can't copy back from color_labels\n");
1941
1942 TRY_EXEC("DROP TABLE color_labels_old",
1943 "[init] can't drop table color_labels_old\n");
1944
1945 TRY_EXEC("CREATE UNIQUE INDEX `color_labels_idx` ON `color_labels` (imgid, color)",
1946 "[init] can't recreate color_labels_idx\n");
1947
1948 // meta data
1949 TRY_EXEC("ALTER TABLE `meta_data` RENAME TO `meta_data_old`",
1950 "[init] can't rename meta_data\n");
1951 TRY_EXEC("CREATE TABLE `meta_data` (id integer, key integer, value varchar, "
1952 "FOREIGN KEY(id) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1953 "[init] can't create new meta_data table\n");
1954
1955 TRY_EXEC("DELETE FROM `meta_data_old` WHERE id NOT IN (SELECT id FROM `images`)",
1956 "[init] can't delete orphaned meta_data elements\n");
1957
1958 TRY_EXEC("INSERT INTO `meta_data` SELECT * FROM `meta_data_old`",
1959 "[init] can't copy back from meta_data\n");
1960
1961 TRY_EXEC("DROP TABLE meta_data_old",
1962 "[init] can't drop table meta_data_old\n");
1963
1964 TRY_EXEC("CREATE UNIQUE INDEX `metadata_index` ON `meta_data` (id, key, value)",
1965 "[init] can't recreate metadata_index\n");
1966
1967 // selected images
1968 TRY_EXEC("ALTER TABLE `selected_images` RENAME TO `selected_images_old`",
1969 "[init] can't rename selected_images\n");
1970 TRY_EXEC("CREATE TABLE `selected_images` (imgid INTEGER PRIMARY KEY, "
1971 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1972 "[init] can't create new selected_images table\n");
1973
1974 TRY_EXEC("DELETE FROM `selected_images_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1975 "[init] can't delete orphaned selected_images elements\n");
1976
1977 TRY_EXEC("INSERT INTO `selected_images` SELECT * FROM `selected_images_old`",
1978 "[init] can't copy back selected_images meta_data\n");
1979
1980 TRY_EXEC("DROP TABLE selected_images_old",
1981 "[init] can't drop table selected_images_old\n");
1982
1983 // module order
1984 TRY_EXEC("ALTER TABLE `module_order` RENAME TO `module_order_old`",
1985 "[init] can't rename module_order\n");
1986 TRY_EXEC("CREATE TABLE `module_order` (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR, "
1987 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1988 "[init] can't create new module_order table\n");
1989
1990 TRY_EXEC("DELETE FROM `module_order_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1991 "[init] can't delete orphaned module_order elements\n");
1992
1993 TRY_EXEC("INSERT INTO `module_order` SELECT * FROM `module_order_old`",
1994 "[init] can't copy back module_order meta_data\n");
1995
1996 TRY_EXEC("DROP TABLE module_order_old",
1997 "[init] can't drop table module_order_old\n");
1998 // clang-format on
1999 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2000 new_version = 33;
2001 }
2002 else if(version == 33)
2003 {
2004 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2005
2006 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_datetime_taken_nc ON images (datetime_taken COLLATE NOCASE)",
2007 "[init] can't create images_datetime_taken\n");
2008 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.metadata_index_key ON meta_data (key)",
2009 "[init] can't create metadata_index_key\n");
2010
2011 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2012 new_version = 34;
2013 }
2014 else if(version == 34)
2015 {
2016 sqlite3_exec(db->handle, "PRAGMA foreign_keys = OFF", NULL, NULL, NULL);
2017 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2018
2019 TRY_EXEC("CREATE TABLE main.images_new (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
2020 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
2021 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
2022 "focus_distance REAL, datetime_taken INTEGER, flags INTEGER, "
2023 "output_width INTEGER, output_height INTEGER, crop REAL, "
2024 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
2025 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
2026 "license VARCHAR, sha1sum CHAR(40), "
2027 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
2028 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
2029 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
2030 "aspect_ratio REAL, exposure_bias REAL, "
2031 "import_timestamp INTEGER, change_timestamp INTEGER, "
2032 "export_timestamp INTEGER, print_timestamp INTEGER, "
2033 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
2034 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
2035 "[init] can't create new images table\n");
2036
2037 TRY_EXEC("INSERT INTO `images_new` SELECT "
2038 "id, group_id, film_id, width, height, filename, maker, model, "
2039 "lens, exposure, aperture, iso, focal_length, focus_distance, NULL AS datetime_taken, flags, "
2040 "output_width, output_height, crop, raw_parameters, raw_denoise_threshold, raw_auto_bright_threshold, raw_black, raw_maximum, "
2041 "license, sha1sum, orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version, "
2042 "max_version, write_timestamp, history_end, position, aspect_ratio, exposure_bias, "
2043 "NULL AS import_timestamp, NULL AS change_timestamp, NULL AS export_timestamp, NULL AS print_timestamp "
2044 "FROM `images`",
2045 "[init] can't copy back from images\n");
2046
2047 TRY_PREPARE(stmt, "SELECT id,"
2048 " CASE WHEN datetime_taken = '' THEN NULL ELSE datetime_taken END,"
2049 " CASE WHEN import_timestamp = -1 THEN NULL ELSE import_timestamp END,"
2050 " CASE WHEN change_timestamp = -1 THEN NULL ELSE change_timestamp END,"
2051 " CASE WHEN export_timestamp = -1 THEN NULL ELSE export_timestamp END,"
2052 " CASE WHEN print_timestamp = -1 THEN NULL ELSE print_timestamp END "
2053 "FROM `images`",
2054 "[init] can't get datetime from images\n");
2055 while(sqlite3_step(stmt) == SQLITE_ROW)
2056 {
2057 sqlite3_stmt *stmt2;
2058 sqlite3_prepare_v2(db->handle,
2059 "UPDATE `images_new` SET"
2060 " (datetime_taken, import_timestamp,"
2061 " change_timestamp, export_timestamp, print_timestamp) = "
2062 " (?2, ?3, ?4, ?5, ?6) WHERE id = ?1",
2063 -1, &stmt2, NULL);
2064 sqlite3_bind_int(stmt2, 1, sqlite3_column_int(stmt, 0));
2065 if(sqlite3_column_type(stmt, 1) != SQLITE_NULL)
2066 {
2067 GDateTime *gdt = dt_datetime_exif_to_gdatetime((const char *)sqlite3_column_text(stmt, 1), dt_datetime_utc_tz());
2068 if(gdt)
2069 {
2070 sqlite3_bind_int64(stmt2, 2, dt_datetime_gdatetime_to_gtimespan(gdt));
2071 g_date_time_unref(gdt);
2072 }
2073 }
2074 for(int i = 0; i < 4; i++)
2075 {
2076 if(sqlite3_column_type(stmt, i + 2) != SQLITE_NULL)
2077 {
2078 GDateTime *gdt = g_date_time_new_from_unix_utc(sqlite3_column_int(stmt, i + 2));
2079 if(gdt)
2080 {
2081 sqlite3_bind_int64(stmt2, i + 3, dt_datetime_gdatetime_to_gtimespan(gdt));
2082 g_date_time_unref(gdt);
2083 }
2084 }
2085 }
2086 TRY_STEP(stmt2, SQLITE_DONE, "[init] can't update datetimes into images_new table\n");
2087 sqlite3_finalize(stmt2);
2088 }
2089 sqlite3_finalize(stmt);
2090
2091 TRY_EXEC("DROP TABLE `images`", "[init] can't drop images table\n");
2092 // that's the way to keep the other tables foreign keys references valid
2093 TRY_EXEC("ALTER TABLE `images_new` RENAME TO `images`", "[init] can't rename images_new table to images");
2094
2095 // pita: need to recreate indexes
2096 TRY_EXEC("CREATE INDEX `image_position_index` ON `images` (position)",
2097 "[init] can't add image_position_index\n");
2098 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
2099 "[init] can't recreate images_filename_index\n");
2100 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
2101 "[init] can't recreate images_film_id_index\n");
2102 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
2103 "[init] can't recreate images_group_id_index\n");
2104 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( latitude DESC, longitude DESC )",
2105 "[init] can't add images_latlong_index\n");
2106 TRY_EXEC("CREATE INDEX `images_datetime_taken` ON images (datetime_taken)",
2107 "[init] can't create images_datetime_taken\n");
2108
2109 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2110 sqlite3_exec(db->handle, "PRAGMA foreign_keys = ON", NULL, NULL, NULL);
2111 new_version = 35;
2112 }
2113 else if(version == 35)
2114 {
2115 TRY_EXEC("CREATE TABLE main.images_new (id INTEGER, filename VARCHAR, flags INTEGER)",
2116 "[init] can't create new images table\n");
2117
2118 gchar *query = g_strdup_printf("INSERT INTO `images_new` "
2119 "SELECT id, filename, flags"
2120 " FROM images"
2121 " WHERE (flags & %d == 0)",
2122 DT_IMAGE_RAW | DT_IMAGE_LDR | DT_IMAGE_HDR);
2123 TRY_EXEC(query, "[init] can't copy back from images\n");
2124
2125 TRY_PREPARE(stmt, "SELECT id, filename, flags FROM `images_new`",
2126 "[init] can't prepare selecting images flags\n");
2127
2128 while(sqlite3_step(stmt) == SQLITE_ROW)
2129 {
2130 sqlite3_stmt *stmt2;
2131 sqlite3_prepare_v2(db->handle,
2132 "UPDATE `images` SET"
2133 " (flags) = "
2134 " (?2) WHERE id = ?1",
2135 -1, &stmt2, NULL);
2136 sqlite3_bind_int(stmt2, 1, sqlite3_column_int(stmt, 0));
2137
2138 dt_image_flags_t flags = sqlite3_column_int(stmt, 2);
2139 gchar *ext = g_strrstr((const char *)sqlite3_column_text(stmt, 1), ".");
2140 flags |= dt_image_flags_from_extension(ext);
2141 sqlite3_bind_int(stmt2, 2, flags);
2142
2143 TRY_STEP(stmt2, SQLITE_DONE, "[init] can't update flags\n");
2144 sqlite3_finalize(stmt2);
2145 }
2146 sqlite3_finalize(stmt);
2147
2148 TRY_EXEC("DROP TABLE `images_new`", "[init] can't drop temp images table\n");
2149 new_version = 36;
2150 }
2151 else
2152 new_version = version; // should be the fallback so that calling code sees that we are in an infinite loop
2153
2154 // write the new version to db
2155 sqlite3_prepare_v2(db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', ?1)", -1, &stmt,
2156 NULL);
2157 sqlite3_bind_int(stmt, 1, new_version);
2158 sqlite3_step(stmt);
2159 sqlite3_finalize(stmt);
2160
2161 return new_version;
2162}
2163
2164/* do the real migration steps, returns the version the db was converted to */
2165static int _upgrade_data_schema_step(dt_database_t *db, int version)
2166{
2167 sqlite3_stmt *stmt;
2168 int new_version = version;
2169 if(version == CURRENT_DATABASE_VERSION_DATA)
2170 return version;
2171 else if(version == 0)
2172 {
2173 // this can't happen, we started with 1, but it's a good example how this function works
2174 // <do some magic to the db>
2175 new_version = 1; // the version we transformed the db to. this way it might be possible to roll back or
2176 // add fast paths
2177 }
2178 else if(version == 1)
2179 {
2180 // clang-format off
2181 // style_items:
2182 // NO TRY_EXEC has the column could be there before version 1 (master build)
2183 // TRY_EXEC("ALTER TABLE data.style_items ADD COLUMN iop_order REAL",
2184 // "[init] can't add `iop_order' column to style_items table in database\n");
2185 sqlite3_exec(db->handle, "ALTER TABLE data.style_items ADD COLUMN iop_order REAL", NULL, NULL, NULL);
2186 // clang-format on
2187 sqlite3_stmt *sel_stmt = NULL;
2188 GList *prior_v1 = dt_ioppr_get_iop_order_list_version(DT_IOP_ORDER_LEGACY);
2189 // clang-format off
2190 // create a temp table with the previous priorities
2191 TRY_EXEC("CREATE TEMPORARY TABLE iop_order_tmp (iop_order REAL, operation VARCHAR(256))",
2192 "[init] can't create temporary table for updating `data.style_items'\n");
2193 // clang-format on
2194 // fill temp table with all operations up to this release
2195 // it will be used to create the pipe and update the iop_order on history
2196 for(GList *priorities = prior_v1; priorities; priorities = g_list_next(priorities))
2197 {
2198 dt_iop_order_entry_t *prior = (dt_iop_order_entry_t *)priorities->data;
2199
2200 sqlite3_prepare_v2(
2201 db->handle,
2202 "INSERT INTO iop_order_tmp (iop_order, operation) VALUES (?1, ?2)",
2203 -1, &stmt, NULL);
2204 sqlite3_bind_double(stmt, 1, prior->o.iop_order_f);
2205 sqlite3_bind_text(stmt, 2, prior->operation, -1, SQLITE_TRANSIENT);
2206 TRY_STEP(stmt, SQLITE_DONE, "[init] can't insert default value in iop_order_tmp\n");
2207 sqlite3_finalize(stmt);
2208 }
2209 g_list_free_full(prior_v1, dt_free_gpointer);
2210 prior_v1 = NULL;
2211
2212 // do the same as for history
2213 // clang-format off
2214 TRY_EXEC("UPDATE data.style_items SET iop_order = ((("
2215 "SELECT MAX(multi_priority) FROM data.style_items style1 WHERE style1.styleid = data.style_items.styleid AND style1.operation = data.style_items.operation "
2216 ") + 1. - multi_priority) / 1000.) + "
2217 "IFNULL((SELECT iop_order FROM iop_order_tmp WHERE iop_order_tmp.operation = "
2218 "data.style_items.operation), -999999.) ",
2219 "[init] can't update iop_order in style_items table\n");
2220
2221 TRY_PREPARE(sel_stmt, "SELECT DISTINCT operation FROM data.style_items WHERE iop_order <= 0 OR iop_order IS NULL",
2222 "[init] can't prepare selecting style_items iop_order\n");
2223 // clang-format on
2224 while(sqlite3_step(sel_stmt) == SQLITE_ROW)
2225 {
2226 const char *op_name = (const char *)sqlite3_column_text(sel_stmt, 0);
2227 printf("operation %s with no iop_order while upgrading style_items in database\n", op_name);
2228 }
2229 sqlite3_finalize(sel_stmt);
2230 // clang-format off
2231 TRY_EXEC("DROP TABLE iop_order_tmp", "[init] can't drop table `iop_order_tmp' from database\n");
2232 // clang-format on
2233 new_version = 2;
2234 }
2235 else if(version == 2)
2236 {
2237 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2238
2239 // With sqlite above or equal to 3.25.0 RENAME COLUMN can be used instead of the following code
2240 // TRY_EXEC("ALTER TABLE data.tags RENAME COLUMN description TO synonyms;",
2241 // "[init] can't change tags column name from description to synonyms\n");
2242 // clang-format off
2243 TRY_EXEC("ALTER TABLE data.tags RENAME TO tmp_tags", "[init] can't rename table tags\n");
2244
2245 TRY_EXEC("CREATE TABLE data.tags (id INTEGER PRIMARY KEY, name VARCHAR, "
2246 "synonyms VARCHAR, flags INTEGER)",
2247 "[init] can't create new tags table\n");
2248
2249 TRY_EXEC("INSERT INTO data.tags (id, name, synonyms, flags) SELECT id, name, description, flags "
2250 "FROM tmp_tags",
2251 "[init] can't populate tags table from tmp_tags\n");
2252
2253 TRY_EXEC("DROP TABLE tmp_tags", "[init] can't delete table tmp_tags\n");
2254
2255 TRY_EXEC("CREATE UNIQUE INDEX data.tags_name_idx ON tags (name)",
2256 "[init] can't create tags_name_idx on tags table\n");
2257 // clang-format on
2258 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2259
2260 new_version = 3;
2261 }
2262 else if(version == 3)
2263 {
2264 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2265 // clang-format off
2266 // create a temp table to invert all multi_priority
2267 TRY_EXEC("CREATE TEMPORARY TABLE m_prio (id INTEGER, operation VARCHAR(256), prio INTEGER)",
2268 "[init] can't create temporary table for updating `history and style_items'\n");
2269
2270 TRY_EXEC("INSERT INTO m_prio SELECT styleid, operation, MAX(multi_priority)"
2271 " FROM data.style_items GROUP BY styleid, operation",
2272 "[init] can't populate m_prio\n");
2273
2274 // update multi_priority for style items and history
2275 TRY_EXEC("UPDATE data.style_items SET multi_priority = "
2276 "(SELECT prio FROM m_prio "
2277 " WHERE data.style_items.operation = operation AND data.style_items.styleid = id)"
2278 " - data.style_items.multi_priority",
2279 "[init] can't update multi_priority for style_items\n");
2280
2281 TRY_EXEC("DROP TABLE m_prio", "[init] can't drop table `m_prio' from database\n");
2282 // clang-format on
2283 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2284
2285 new_version = 4;
2286 }
2287 else if(version == 4)
2288 {
2289 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2290 // clang-format off
2291 // remove iop_order from style_item table
2292 TRY_EXEC("ALTER TABLE data.style_items RENAME TO s",
2293 "[init] can't rename style_items to s\n");
2294 TRY_EXEC("CREATE TABLE data.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
2295 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2296 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2297 "[init] can't create style_items table'\n");
2298 TRY_EXEC("INSERT INTO data.style_items SELECT styleid, num, module, operation, op_params, enabled, "
2299 " blendop_params, blendop_version, multi_priority, multi_name "
2300 "FROM s",
2301 "[init] can't populate style_items table'\n");
2302 TRY_EXEC("DROP TABLE s",
2303 "[init] can't drop table s'\n");
2304 // clang-format on
2305 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2306
2307 new_version = 5;
2308 }
2309 else if(version == 5)
2310 {
2311 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2312 // clang-format of
2313 // make style.id a PRIMARY KEY and add iop_list
2314 TRY_EXEC("ALTER TABLE data.styles RENAME TO s",
2315 "[init] can't rename styles to s\n");
2316 TRY_EXEC("CREATE TABLE data.styles (id INTEGER PRIMARY KEY, name VARCHAR, description VARCHAR, iop_list VARCHAR)",
2317 "[init] can't create styles table\n");
2318 TRY_EXEC("INSERT INTO data.styles SELECT id, name, description, NULL FROM s",
2319 "[init] can't populate styles table\n");
2320 TRY_EXEC("DROP TABLE s",
2321 "[init] can't drop table s\n");
2322
2323 TRY_EXEC("CREATE INDEX IF NOT EXISTS data.styles_name_index ON styles (name)",
2324 "[init] can't create styles_nmae_index\n");
2325
2326 // make style_items.styleid index
2327
2328 TRY_EXEC("CREATE INDEX IF NOT EXISTS data.style_items_styleid_index ON style_items (styleid)",
2329 "[init] can't create style_items_styleid_index\n");
2330 // clang-format on
2331 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2332
2333 new_version = 6;
2334 }
2335 else if(version == 6)
2336 {
2337 // clang-format off
2338 TRY_EXEC("CREATE TABLE data.locations "
2339 "(tagid INTEGER PRIMARY KEY, type INTEGER, longitude REAL, latitude REAL, "
2340 "delta1 REAL, delta2 REAL, FOREIGN KEY(tagid) REFERENCES tags(id))",
2341 "[init] can't create new locations table\n");
2342 // clang-format on
2343 new_version = 7;
2344 }
2345 else if(version == 7)
2346 {
2347 // clang-format off
2348 TRY_EXEC("ALTER TABLE data.locations ADD COLUMN ratio FLOAT DEFAULT 1",
2349 "[init] can't add column `ratio' column to locations table\n");
2350 // clang-format on
2351 new_version = 8;
2352 }
2353 else if(version == 8)
2354 {
2355 // clang-format off
2356 TRY_EXEC("ALTER TABLE data.locations ADD COLUMN polygons BLOB",
2357 "[init] can't add column `polygons' column to locations table\n");
2358 // clang-format on
2359 new_version = 9;
2360 }
2361 else
2362 new_version = version; // should be the fallback so that calling code sees that we are in an infinite loop
2363
2364 // write the new version to db
2365 // clang-format off¨
2366 sqlite3_prepare_v2(db->handle, "INSERT OR REPLACE INTO data.db_info (key, value) VALUES ('version', ?1)", -1, &stmt,
2367 NULL);
2368 // clang-format on
2369 sqlite3_bind_int(stmt, 1, new_version);
2370 sqlite3_step(stmt);
2371 sqlite3_finalize(stmt);
2372
2373 return new_version;
2374}
2375
2376#undef FINALIZE
2377
2378#undef TRY_EXEC
2379#undef TRY_STEP
2380#undef TRY_PREPARE
2381
2382/* upgrade library db from 'version' to CURRENT_DATABASE_VERSION_LIBRARY. don't touch this function but
2383 * _upgrade_library_schema_step() instead. */
2384static gboolean _upgrade_library_schema(dt_database_t *db, int version)
2385{
2386 while(version < CURRENT_DATABASE_VERSION_LIBRARY)
2387 {
2388 const int new_version = _upgrade_library_schema_step(db, version);
2389 if(new_version == version)
2390 return FALSE; // we don't know how to upgrade this db. probably a bug in _upgrade_library_schema_step
2391 else
2392 version = new_version;
2393 }
2394 return TRUE;
2395}
2396
2397/* upgrade data db from 'version' to CURRENT_DATABASE_VERSION_DATA. don't touch this function but
2398 * _upgrade_data_schema_step() instead. */
2399static gboolean _upgrade_data_schema(dt_database_t *db, int version)
2400{
2401 while(version < CURRENT_DATABASE_VERSION_DATA)
2402 {
2403 const int new_version = _upgrade_data_schema_step(db, version);
2404 if(new_version == version)
2405 return FALSE; // we don't know how to upgrade this db. probably a bug in _upgrade_data_schema_step
2406 else
2407 version = new_version;
2408 }
2409 return TRUE;
2410}
2411
2412/* create the current database schema and set the version in db_info accordingly */
2413static void _create_library_schema(dt_database_t *db)
2414{
2415 sqlite3_stmt *stmt;
2417 // clang-format off
2418 sqlite3_exec(db->handle, "CREATE TABLE main.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)", NULL,
2419 NULL, NULL);
2420 sqlite3_prepare_v2(
2421 db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', ?1)", -1, &stmt, NULL);
2422 // clang-format on
2423 sqlite3_bind_int(stmt, 1, CURRENT_DATABASE_VERSION_LIBRARY);
2424 sqlite3_step(stmt);
2425 sqlite3_finalize(stmt);
2427 // clang-format off
2428 sqlite3_exec(db->handle,
2429 "CREATE TABLE main.film_rolls "
2430 "(id INTEGER PRIMARY KEY, access_timestamp INTEGER, "
2431 // "folder VARCHAR(1024), external_drive VARCHAR(1024))", //
2432 // FIXME: make sure to bump CURRENT_DATABASE_VERSION_LIBRARY and add a
2433 // case to _upgrade_library_schema_step when adding this!
2434 "folder VARCHAR(1024) NOT NULL)",
2435 NULL, NULL, NULL);
2436 sqlite3_exec(db->handle, "CREATE INDEX main.film_rolls_folder_index ON film_rolls (folder)", NULL, NULL, NULL);
2438 sqlite3_exec(
2439 db->handle,
2440 "CREATE TABLE main.images (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
2441 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
2442 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
2443 "focus_distance REAL, datetime_taken INTEGER, flags INTEGER, "
2444 "output_width INTEGER, output_height INTEGER, crop REAL, "
2445 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
2446 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
2447 "license VARCHAR, sha1sum CHAR(40), "
2448 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
2449 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
2450 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
2451 "aspect_ratio REAL, exposure_bias REAL, "
2452 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, "
2453 "export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1, "
2454 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
2455 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
2456 NULL, NULL, NULL);
2457 sqlite3_exec(db->handle, "CREATE INDEX main.images_group_id_index ON images (group_id, id)", NULL, NULL, NULL);
2458 sqlite3_exec(db->handle, "CREATE INDEX main.images_film_id_index ON images (film_id, filename)", NULL, NULL, NULL);
2459 sqlite3_exec(db->handle, "CREATE INDEX main.images_filename_index ON images (filename, version)", NULL, NULL, NULL);
2460 sqlite3_exec(db->handle, "CREATE INDEX main.image_position_index ON images (position)", NULL, NULL, NULL);
2461 sqlite3_exec(db->handle, "CREATE INDEX main.images_datetime_taken_nc ON images (datetime_taken)", NULL, NULL, NULL);
2462
2464 sqlite3_exec(db->handle, "CREATE TABLE main.selected_images (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2466 sqlite3_exec(
2467 db->handle,
2468 "CREATE TABLE main.history (imgid INTEGER, num INTEGER, module INTEGER, "
2469 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2470 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256), "
2471 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2472 NULL, NULL, NULL);
2473 sqlite3_exec(db->handle, "CREATE INDEX main.history_imgid_op_index ON history (imgid, operation)", NULL, NULL, NULL);
2474 sqlite3_exec(db->handle, "CREATE INDEX main.history_imgid_num_index ON history (imgid, num DESC)", NULL, NULL, NULL);
2476 sqlite3_exec(db->handle,
2477 "CREATE TABLE main.masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, name VARCHAR(256), "
2478 "version INTEGER, points BLOB, points_count INTEGER, source BLOB, "
2479 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2480 NULL, NULL, NULL);
2481
2482 sqlite3_exec(db->handle,
2483 "CREATE INDEX main.masks_history_imgid_index ON masks_history (imgid, num)",
2484 NULL, NULL, NULL);
2485
2486 sqlite3_exec(db->handle, "CREATE INDEX main.images_latlong_index ON images (latitude DESC, longitude DESC)",
2487 NULL, NULL, NULL);
2488
2490 sqlite3_exec(db->handle, "CREATE TABLE main.tagged_images (imgid INTEGER, tagid INTEGER, position INTEGER, "
2491 "PRIMARY KEY (imgid, tagid),"
2492 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)", NULL, NULL, NULL);
2493 sqlite3_exec(db->handle, "CREATE INDEX main.tagged_images_tagid_index ON tagged_images (tagid)", NULL, NULL, NULL);
2494 sqlite3_exec(db->handle, "CREATE INDEX main.tagged_images_position_index ON tagged_images (position)", NULL, NULL, NULL);
2496 sqlite3_exec(db->handle, "CREATE TABLE main.color_labels (imgid INTEGER, color INTEGER)", NULL, NULL, NULL);
2497 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX main.color_labels_idx ON color_labels (imgid, color)", NULL, NULL,
2498 NULL);
2500 sqlite3_exec(db->handle, "CREATE TABLE main.meta_data (id INTEGER, key INTEGER, value VARCHAR)", NULL, NULL, NULL);
2501 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX main.metadata_index ON meta_data (id, key, value)", NULL, NULL, NULL);
2502
2503 sqlite3_exec(db->handle, "CREATE INDEX main.metadata_index_key ON meta_data (key)", NULL, NULL, NULL);
2504 sqlite3_exec(db->handle, "CREATE TABLE main.module_order (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR)",
2505 NULL, NULL, NULL);
2506 sqlite3_exec(db->handle, "CREATE TABLE main.history_hash (imgid INTEGER PRIMARY KEY, "
2507 "basic_hash BLOB, auto_hash BLOB, current_hash BLOB, mipmap_hash BLOB, "
2508 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2509 NULL, NULL, NULL);
2510
2511 // v34
2512 sqlite3_exec(db->handle, "CREATE INDEX main.images_datetime_taken_nc ON images (datetime_taken COLLATE NOCASE)",
2513 NULL, NULL, NULL);
2514 sqlite3_exec(db->handle, "CREATE INDEX main.metadata_index_key ON meta_data (key)", NULL, NULL, NULL);
2515 // clang-format on
2516}
2517
2518/* create the current database schema and set the version in db_info accordingly */
2519static void _create_data_schema(dt_database_t *db)
2520{
2521 sqlite3_stmt *stmt;
2522 // clang-format off
2524 sqlite3_exec(db->handle, "CREATE TABLE data.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)", NULL,
2525 NULL, NULL);
2526 sqlite3_prepare_v2(
2527 db->handle, "INSERT OR REPLACE INTO data.db_info (key, value) VALUES ('version', ?1)", -1, &stmt, NULL);
2528 sqlite3_bind_int(stmt, 1, CURRENT_DATABASE_VERSION_DATA);
2529 sqlite3_step(stmt);
2530 sqlite3_finalize(stmt);
2532 sqlite3_exec(db->handle, "CREATE TABLE data.tags (id INTEGER PRIMARY KEY, name VARCHAR, "
2533 "synonyms VARCHAR, flags INTEGER)", NULL, NULL, NULL);
2534 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX data.tags_name_idx ON tags (name)", NULL, NULL, NULL);
2536 sqlite3_exec(db->handle, "CREATE TABLE data.styles (id INTEGER PRIMARY KEY, name VARCHAR, description VARCHAR, iop_list VARCHAR)",
2537 NULL, NULL, NULL);
2538 sqlite3_exec(db->handle, "CREATE INDEX data.styles_name_index ON styles (name)", NULL, NULL, NULL);
2540 sqlite3_exec(
2541 db->handle,
2542 "CREATE TABLE data.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
2543 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2544 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2545 NULL, NULL, NULL);
2546 sqlite3_exec(
2547 db->handle,
2548 "CREATE INDEX IF NOT EXISTS data.style_items_styleid_index ON style_items (styleid)",
2549 NULL, NULL, NULL);
2551 sqlite3_exec(db->handle, "CREATE TABLE data.presets (name VARCHAR, description VARCHAR, operation "
2552 "VARCHAR, op_version INTEGER, op_params BLOB, "
2553 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, "
2554 "multi_priority INTEGER, multi_name VARCHAR(256), "
2555 "model VARCHAR, maker VARCHAR, lens VARCHAR, iso_min REAL, iso_max REAL, "
2556 "exposure_min REAL, exposure_max REAL, "
2557 "aperture_min REAL, aperture_max REAL, focal_length_min REAL, "
2558 "focal_length_max REAL, writeprotect INTEGER, "
2559 "autoapply INTEGER, filter INTEGER, def INTEGER, format INTEGER)",
2560 NULL, NULL, NULL);
2561 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX data.presets_idx ON presets (name, operation, op_version)",
2562 NULL, NULL, NULL);
2564 sqlite3_exec(db->handle, "CREATE TABLE data.locations (tagid INTEGER PRIMARY KEY, "
2565 "type INTEGER, longitude REAL, latitude REAL, delta1 REAL, delta2 REAL, ratio FLOAT, polygons BLOB, "
2566 "FOREIGN KEY(tagid) REFERENCES tags(id))", NULL, NULL, NULL);
2567 // clang-format on
2568}
2569
2570// create the in-memory tables
2571// temporary stuff for some ops, need this for some reason with newer sqlite3:
2572static void _create_memory_schema(dt_database_t *db)
2573{
2574 // clang-format off
2575 sqlite3_exec(db->handle, "CREATE TABLE memory.color_labels_temp (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2576 sqlite3_exec(
2577 db->handle,
2578 "CREATE TABLE memory.collected_images (rowid INTEGER PRIMARY KEY AUTOINCREMENT, imgid INTEGER)", NULL,
2579 NULL, NULL);
2580 sqlite3_exec(
2581 db->handle,
2582 "CREATE TABLE memory.collected_backup (rowid INTEGER PRIMARY KEY AUTOINCREMENT, imgid INTEGER)", NULL,
2583 NULL, NULL); sqlite3_exec(db->handle, "CREATE TABLE memory.selected_backup (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2584 sqlite3_exec(db->handle, "CREATE TABLE memory.selected_backup (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2585 sqlite3_exec(db->handle, "CREATE TABLE memory.tmp_selection (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2586 sqlite3_exec(db->handle, "CREATE TABLE memory.taglist "
2587 "(tmpid INTEGER PRIMARY KEY, id INTEGER UNIQUE ON CONFLICT IGNORE, "
2588 "count INTEGER DEFAULT 0, count2 INTEGER DEFAULT 0)",
2589 NULL, NULL, NULL);
2590 sqlite3_exec(db->handle, "CREATE TABLE memory.similar_tags (tagid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2591 sqlite3_exec(db->handle, "CREATE TABLE memory.darktable_tags (tagid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2592 sqlite3_exec(
2593 db->handle,
2594 "CREATE TABLE memory.undo_history (id INTEGER, imgid INTEGER, num INTEGER, module INTEGER, "
2595 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2596 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2597 NULL, NULL, NULL);
2598 sqlite3_exec(
2599 db->handle,
2600 "CREATE TABLE memory.undo_masks_history (id INTEGER, imgid INTEGER, num INTEGER, formid INTEGER,"
2601 " form INTEGER, name VARCHAR(256), version INTEGER, points BLOB, points_count INTEGER, source BLOB)",
2602 NULL, NULL, NULL);
2603 sqlite3_exec(
2604 db->handle,
2605 "CREATE TABLE memory.undo_module_order (id INTEGER, imgid INTEGER, version INTEGER, iop_list VARCHAR)",
2606 NULL, NULL, NULL);
2607 sqlite3_exec(db->handle,
2608 "CREATE TABLE memory.darktable_iop_names (operation VARCHAR(256) PRIMARY KEY, name VARCHAR(256))",
2609 NULL, NULL, NULL);
2610 sqlite3_exec(db->handle,
2611 "CREATE TABLE memory.film_folder (id INTEGER PRIMARY KEY, status INTEGER)",
2612 NULL, NULL, NULL);
2613 // clang-format on
2614
2615 /* Staging area for "undo remove from library": one twin per table a removed image owns
2616 * rows in. They are created as `CREATE TABLE ... AS SELECT` over the live tables rather
2617 * than declared column by column, so their shape follows whatever main holds and a schema
2618 * migration reaches database/removed_image_repository.c without an edit there -- that file
2619 * names its columns from PRAGMA table_info() on these very tables. `WHERE 0` copies the
2620 * shape and no row; the two leading columns it prepends are the key the repository works
2621 * on. This runs after the library schema is created or upgraded, which is what makes the
2622 * SELECT resolvable. */
2623 static const char *const removed_twins[]
2624 = { "film_rolls", "images", "history", "masks_history", "module_order",
2625 "tagged_images", "color_labels", "meta_data", "history_hash" };
2626
2627 for(gsize i = 0; i < G_N_ELEMENTS(removed_twins); i++)
2628 {
2629 gchar *query = g_strdup_printf("CREATE TABLE memory.removed_%s AS"
2630 " SELECT 0 AS snap_id, 0 AS undo_imgid, * FROM main.%s WHERE 0",
2631 removed_twins[i], removed_twins[i]);
2632 sqlite3_exec(db->handle, query, NULL, NULL, NULL);
2633 g_free(query);
2634 }
2635
2636 /* Group membership is the one thing a removal changes outside the removed image's own
2637 * rows: dropping a group's leader hands the group to a survivor, rewriting a group_id that
2638 * belongs to an image nobody asked to remove. */
2639 sqlite3_exec(db->handle,
2640 "CREATE TABLE memory.removed_groups (snap_id INTEGER, undo_imgid INTEGER, id INTEGER, group_id INTEGER)",
2641 NULL, NULL, NULL);
2642}
2643
2644static void _sanitize_db(dt_database_t *db)
2645{
2646 sqlite3_stmt *stmt, *innerstmt;
2647 // clang-format off
2648 /* first let's get rid of non-utf8 tags. */
2649 sqlite3_prepare_v2(db->handle, "SELECT id, name FROM data.tags", -1, &stmt, NULL);
2650 sqlite3_prepare_v2(db->handle, "UPDATE data.tags SET name = ?1 WHERE id = ?2", -1, &innerstmt, NULL);
2651 // clang-format on
2652 while(sqlite3_step(stmt) == SQLITE_ROW)
2653 {
2654 const int id = sqlite3_column_int(stmt, 0);
2655 const char *tag = (const char *)sqlite3_column_text(stmt, 1);
2656
2657 if(!g_utf8_validate(tag, -1, NULL))
2658 {
2659 gchar *new_tag = dt_util_foo_to_utf8(tag);
2660 fprintf(stderr, "[init]: tag `%s' is not valid utf8, replacing it with `%s'\n", tag, new_tag);
2661 if(tag)
2662 {
2663 sqlite3_bind_text(innerstmt, 1, new_tag, -1, SQLITE_TRANSIENT);
2664 sqlite3_bind_int(innerstmt, 2, id);
2665 sqlite3_step(innerstmt);
2666 sqlite3_reset(innerstmt);
2667 sqlite3_clear_bindings(innerstmt);
2668 dt_free(new_tag);
2669 }
2670 }
2671 }
2672 sqlite3_finalize(stmt);
2673 sqlite3_finalize(innerstmt);
2674 // clang-format off
2675 // make sure film_roll folders don't end in "/", that will result in empty entries in the collect module
2676 sqlite3_exec(db->handle,
2677 "UPDATE main.film_rolls SET folder = substr(folder, 1, length(folder) - 1) WHERE folder LIKE '%/'",
2678 NULL, NULL, NULL);
2679 // clang-format on
2680}
2681
2682// in library we keep the names of the tags used in tagged_images. however, using that table at runtime results
2683// in some overhead not necessary so instead we just use the used_tags table to update tagged_images on startup
2684#define TRY_EXEC(_query, _message) \
2685 do \
2686 { \
2687 if(sqlite3_exec(db->handle, _query, NULL, NULL, NULL) != SQLITE_OK) \
2688 { \
2689 fprintf(stderr, _message); \
2690 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2691 FINALIZE; \
2692 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2693 return FALSE; \
2694 } \
2695 } while(0)
2696
2697#define TRY_STEP(_stmt, _expected, _message) \
2698 do \
2699 { \
2700 if(sqlite3_step(_stmt) != _expected) \
2701 { \
2702 fprintf(stderr, _message); \
2703 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2704 FINALIZE; \
2705 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2706 return FALSE; \
2707 } \
2708 } while(0)
2709
2710#define TRY_PREPARE(_stmt, _query, _message) \
2711 do \
2712 { \
2713 if(sqlite3_prepare_v2(db->handle, _query, -1, &_stmt, NULL) != SQLITE_OK) \
2714 { \
2715 fprintf(stderr, _message); \
2716 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2717 FINALIZE; \
2718 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2719 return FALSE; \
2720 } \
2721 } while(0)
2722
2723#define FINALIZE \
2724 do \
2725 { \
2726 sqlite3_finalize(stmt); stmt = NULL; /* NULL so that finalize becomes a NOP */ \
2727 } while(0)
2728
2729#undef TRY_EXEC
2730#undef TRY_STEP
2731#undef TRY_PREPARE
2732#undef FINALIZE
2733
2734void dt_database_take_error(dt_database_error_t *error)
2735{
2736 dt_database_t *const db = _db;
2737 if(IS_NULL_PTR(db)) return;
2738 if(IS_NULL_PTR(db) || IS_NULL_PTR(error)) return;
2739
2740 // Ownership of the strings moves to the caller; the database keeps no pending error.
2741 error->lock_acquired = db->lock_acquired;
2742 error->other_pid = db->error_other_pid;
2743 error->message = db->error_message;
2744 error->dbfilename = db->error_dbfilename;
2745
2746 db->error_other_pid = 0;
2747 db->error_message = NULL;
2748 db->error_dbfilename = NULL;
2749}
2750
2751void dt_database_error_free(dt_database_error_t *error)
2752{
2753 if(IS_NULL_PTR(error)) return;
2754 dt_free(error->message);
2755 dt_free(error->dbfilename);
2756 error->message = NULL;
2757 error->dbfilename = NULL;
2758}
2759
2760int dt_database_delete_lock_files(const char *dbfilename)
2761{
2762 if(IS_NULL_PTR(dbfilename)) return -1;
2763
2764 char lck_pathname[1024];
2765 snprintf(lck_pathname, sizeof(lck_pathname), "%s.lock", dbfilename);
2766 char *lck_dirname = g_strdup(lck_pathname);
2767 char *slash_pos = g_strrstr(lck_dirname, "/");
2768 if(!IS_NULL_PTR(slash_pos)) *slash_pos = '\0';
2769
2770 int status = 0;
2771 const char *names[] = { "/data.db.lock", "/library.db.lock" };
2772 for(size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++)
2773 {
2774 char *lck_filename = g_strconcat(lck_dirname, names[i], NULL);
2775 if(g_access(lck_filename, F_OK) != -1) status += remove(lck_filename);
2776 dt_free(lck_filename);
2777 }
2778
2779 dt_free(lck_dirname);
2780 return status;
2781}
2782
2783
2784static gboolean pid_is_alive(int pid)
2785{
2786 gboolean pid_is_alive;
2787
2788#ifdef _WIN32
2789 pid_is_alive = FALSE;
2790 HANDLE h = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid);
2791 if(h)
2792 {
2793 wchar_t wfilename[MAX_PATH];
2794 long unsigned int n_filename = sizeof(wfilename);
2795 int ret = QueryFullProcessImageNameW(h, 0, wfilename, &n_filename);
2796 char *filename = g_utf16_to_utf8(wfilename, -1, NULL, NULL, NULL);
2797 if(ret && n_filename > 0 && filename && g_str_has_suffix(filename, "ansel.exe"))
2798 pid_is_alive = TRUE;
2799 dt_free(filename);
2800 CloseHandle(h);
2801 }
2802#else
2803 pid_is_alive = !((kill(pid, 0) == -1) && errno == ESRCH);
2804
2805#ifdef __linux__
2806 // If this is Linux, we can query /proc to see if the pid is
2807 // actually a darktable instance.
2808 if(pid_is_alive)
2809 {
2810 gchar *contents;
2811 gsize length;
2812 gchar filename[64];
2813 snprintf(filename, sizeof(filename), "/proc/%d/cmdline", pid);
2814
2815 if(g_file_get_contents("", &contents, &length, NULL))
2816 {
2817 if(strstr(contents, "ansel") == NULL)
2818 {
2819 pid_is_alive = FALSE;
2820 }
2821 dt_free(contents);
2822 }
2823 }
2824#endif
2825
2826#endif
2827
2828 return pid_is_alive;
2829}
2830
2831static gboolean _lock_single_database(dt_database_t *db, const char *dbfilename, char **lockfile)
2832{
2833 gboolean lock_acquired = FALSE;
2834 mode_t old_mode;
2835 int lock_tries = 0;
2836 gchar *pid = g_strdup_printf("%d", getpid());
2837
2838 if(!g_strcmp0(dbfilename, ":memory:"))
2839 {
2840 lock_acquired = TRUE;
2841 }
2842 else
2843 {
2844 *lockfile = g_strconcat(dbfilename, ".lock", NULL);
2845lock_again:
2846 lock_tries++;
2847 old_mode = umask(0);
2848 int fd = g_open(*lockfile, O_RDWR | O_CREAT | O_EXCL, 0666);
2849 umask(old_mode);
2850
2851 if(fd != -1) // the lockfile was successfully created - write our PID into it
2852 {
2853 if(write(fd, pid, strlen(pid) + 1) > -1) lock_acquired = TRUE;
2854 close(fd);
2855 // we created the file; if the PID could not be written it is ours to remove
2856 if(!lock_acquired) g_unlink(*lockfile);
2857 }
2858 else // the lockfile already exists - see if it's a stale one left over from a crashed instance
2859 {
2860 char buf[64];
2861 memset(buf, 0, sizeof(buf));
2862 fd = g_open(*lockfile, O_RDWR | O_CREAT, 0666);
2863
2864 if(fd == -1)
2865 {
2866 // failed to open the lock file
2867 const int err = errno;
2868 fprintf(stderr, "[init] error opening the database lock file for reading: %s\n", strerror(err));
2869 db->error_message = g_strdup_printf(_("error opening the database lock file for reading: %s"), strerror(err));
2870 }
2871 else
2872 {
2873 // successfully opened the lock file, read the PID
2874 const int bytes_read = read(fd, buf, sizeof(buf) - 1);
2875
2876 if(bytes_read > 0)
2877 {
2878 // lock file contains a PID
2879 db->error_other_pid = atoi(buf);
2880
2881 if(!pid_is_alive(db->error_other_pid))
2882 {
2883 // stale lock file - close fd before unlinking (required on Windows)
2884 close(fd);
2885 g_unlink(*lockfile);
2886
2887 if(lock_tries < 5)
2888 {
2889 goto lock_again;
2890 }
2891 else
2892 {
2893 fprintf(stderr, "[init] tried several times to acquire the database lock, giving up\n");
2894 db->error_message = g_strdup_printf(_("tried several times to acquire the database lock, giving up"));
2895 }
2896 }
2897 else
2898 {
2899 // lock file contains a live PID
2900 close(fd);
2901 fprintf(stderr, "[init] the database lock file contains a pid that seems to be alive in your system: %d\n",
2902 db->error_other_pid);
2903 db->error_message = g_strdup_printf(_("the database lock file contains a pid that seems to be alive in your system: %d"),
2904 db->error_other_pid);
2905 }
2906 }
2907 else
2908 {
2909 // lock file is empty or unreadable
2910 close(fd);
2911 fprintf(stderr, "[init] the database lock file seems to be empty\n");
2912 db->error_message = g_strdup_printf(_("the database lock file seems to be empty"));
2913 }
2914 }
2915 }
2916 }
2917
2918 dt_free(pid);
2919
2920 if(db->error_message)
2921 db->error_dbfilename = g_strdup(dbfilename);
2922
2923 /* The stored path must outlive this call ONLY when this process owns the file:
2924 * _database_free() unlinks whatever these fields point at, unconditionally, and a
2925 * lock-failed database still travels -- dt_database_open() publishes it so the caller
2926 * can read the error, and the retry loop closes it before trying again. Left set on
2927 * failure, the path points at the OTHER instance's live lock file, and that close
2928 * deletes it -- after which the retry "succeeds" and two instances write the same
2929 * database. (dt_free NULLs the field.) */
2930 if(!lock_acquired) dt_free(*lockfile);
2931
2932 return lock_acquired;
2933}
2934
2935static gboolean _lock_databases(dt_database_t *db)
2936{
2937 if(!_lock_single_database(db, db->dbfilename_data, &db->lockfile_data))
2938 return FALSE;
2939 if(!_lock_single_database(db, db->dbfilename_library, &db->lockfile_library))
2940 {
2941 // unlock data.db to not leave a stale lock file around -- and drop the path, so
2942 // _database_free() does not unlink whatever lives at that name by the time it runs
2943 if(db->lockfile_data)
2944 {
2945 g_unlink(db->lockfile_data);
2946 dt_free(db->lockfile_data);
2947 }
2948 return FALSE;
2949 }
2950 return TRUE;
2951}
2952
2953/* Forward declaration: this asks a question, and the asking machinery is defined further
2954 * down next to the handler it goes through. */
2955static dt_database_response_t _ask_upgrade(const char *dbfilename);
2956
2957void ask_for_upgrade(const gchar *dbname, const gboolean has_gui)
2958{
2959 // if there's no gui just leave
2960 if(!has_gui)
2961 {
2962 fprintf(stderr, "[init] database `%s' is out-of-date. aborting.\n", dbname);
2963 exit(1);
2964 }
2965
2966 // the database has to be upgraded, let's ask user. What that looks like -- the wording,
2967 // the warning that it may take a while on a large library, the button labels -- belongs
2968 // to whoever renders it; this file only states that the schema is out of date.
2969 if(_ask_upgrade(dbname) != DT_DATABASE_RESPONSE_PROCEED)
2970 {
2971 fprintf(stderr, "[init] we shall not update the database, aborting.\n");
2972 exit(1);
2973 }
2974}
2975
2976void dt_database_backup(const char *filename)
2977{
2978 char *version = g_strdup(darktable_package_version);
2979 int k = 0;
2980 // get plain version (no commit id)
2981 while(version[k])
2982 {
2983 if((version[k] < '0' || version[k] > '9') && (version[k] != '.'))
2984 {
2985 version[k] = '\0';
2986 break;
2987 }
2988 k++;
2989 }
2990
2991 gchar *backup = g_strdup_printf("%s-pre-%s", filename, version);
2992
2993 GError *gerror = NULL;
2994 if(!g_file_test(backup, G_FILE_TEST_EXISTS))
2995 {
2996 GFile *src = g_file_new_for_path(filename);
2997 GFile *dest = g_file_new_for_path(backup);
2998 gboolean copy_status = TRUE;
2999 if(g_file_test(filename, G_FILE_TEST_EXISTS))
3000 {
3001 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
3002 if(copy_status) copy_status = g_chmod(backup, S_IRUSR) == 0;
3003 }
3004 else
3005 {
3006 // there is nothing to backup, create an empty file to prevent further backup attempts
3007 int fd = g_open(backup, O_CREAT, S_IRUSR);
3008 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
3009 }
3010 if(!copy_status) fprintf(stderr, "[backup failed] %s -> %s\n", filename, backup);
3011
3012 g_object_unref(src);
3013 g_object_unref(dest);
3014 }
3015
3016 dt_free(version);
3017 dt_free(backup);
3018}
3019
3020int _get_pragma_int_val(sqlite3 *db, const char* pragma)
3021{
3022 gchar* query= g_strdup_printf("PRAGMA %s", pragma);
3023 int val = -1;
3024 sqlite3_stmt *stmt;
3025 const int rc = sqlite3_prepare_v2(db, query,-1, &stmt, NULL);
3026 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3027 {
3028 val = sqlite3_column_int(stmt, 0);
3029 }
3030 sqlite3_finalize(stmt);
3031 dt_free(query);
3032
3033 return val;
3034}
3035
3036gchar* _get_pragma_string_val(sqlite3 *db, const char* pragma)
3037{
3038 gchar* query= g_strdup_printf("PRAGMA %s", pragma);
3039 sqlite3_stmt *stmt;
3040 gchar* val = NULL;
3041 const int rc = sqlite3_prepare_v2(db, query,-1, &stmt, NULL);
3042 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3043 {
3044 val = g_strdup((const char *)sqlite3_column_text(stmt, 0));
3045 while(sqlite3_step(stmt) == SQLITE_ROW)
3046 {
3047 gchar* cur_val = g_strdup((const char *)sqlite3_column_text(stmt, 0));
3048 gchar* tmp_val = g_strdup(val);
3049 dt_free(val);
3050 val = g_strconcat(tmp_val, "\n", cur_val, NULL);
3051 dt_free(cur_val);
3052 dt_free(tmp_val);
3053 }
3054 }
3055 sqlite3_finalize(stmt);
3056 dt_free(query);
3057 return val;
3058}
3059
3060static dt_database_prompt_handler_t _prompt_handler = NULL;
3061
3066
3067/* Ask, or answer CLOSE if there is nobody to ask. Deliberately not a fallback that guesses:
3068 * deleting or restoring a database is not something to do on nobody's authority, and
3069 * neither is spending a minute vacuuming one. */
3070static dt_database_response_t _ask_user(const dt_database_prompt_context_t *context)
3071{
3072 if(IS_NULL_PTR(_prompt_handler))
3073 {
3074 fprintf(stderr, "[init] cannot ask the user about `%s': no prompt handler registered\n",
3075 context->dbfilename);
3076 return DT_DATABASE_RESPONSE_CLOSE;
3077 }
3078
3079 return _prompt_handler(context);
3080}
3081
3082/* The questions that used to be asked with gtk_dialog calls written inline in this file --
3083 * the reason `widgets/dialog.h` was included by the SQL layer at all. */
3084
3085static dt_database_response_t _ask_corrupted(const char *dbfilename, const char *quick_check,
3086 const gboolean snapshot_available)
3087{
3088 const dt_database_prompt_context_t context = { .prompt = DT_DATABASE_PROMPT_CORRUPTED,
3089 .dbfilename = dbfilename,
3090 .quick_check = quick_check,
3091 .snapshot_available = snapshot_available };
3092 return _ask_user(&context);
3093}
3094
3095static dt_database_response_t _ask_readonly(const char *dbfilename)
3096{
3097 const dt_database_prompt_context_t context = { .prompt = DT_DATABASE_PROMPT_READONLY,
3098 .dbfilename = dbfilename };
3099 return _ask_user(&context);
3100}
3101
3102static dt_database_response_t _ask_upgrade(const char *dbfilename)
3103{
3104 const dt_database_prompt_context_t context = { .prompt = DT_DATABASE_PROMPT_UPGRADE,
3105 .dbfilename = dbfilename };
3106 return _ask_user(&context);
3107}
3108
3109static dt_database_t *_database_init(const dt_database_params_t *params)
3110{
3111 const char *const alternative = params->alternative;
3112 const gboolean load_data = params->load_data;
3113 const gboolean has_gui = params->has_gui;
3114
3115 /* set the threading mode to Serialized */
3116 sqlite3_config(SQLITE_CONFIG_SERIALIZED);
3117
3118 sqlite3_initialize();
3119
3120 /* Declared before `start:` because the two corrupt-database paths jump back to it, and
3121 * the effective name must survive the retry.
3122 *
3123 * `library` is what the orchestrator configured, EXCEPT when the legacy pre-XDG file
3124 * under $HOME was just migrated -- then it is the name it was migrated to. Reading the
3125 * conf key after the migration used to do this implicitly, and getting it wrong here
3126 * means opening a path that does not exist and silently creating an empty library
3127 * beside the user's real one. */
3128 gchar *migrated_name = NULL;
3129 const char *library = params->library;
3130
3131start:
3132 if(IS_NULL_PTR(alternative))
3133 {
3134 /* migrate default database location to new default. On a corrupt-database retry,
3135 * `library` may still alias the PREVIOUS pass's migrated name -- so that allocation
3136 * must survive until a new name actually replaces it. Freeing it first and then
3137 * passing `library` into the migration reads freed memory. */
3138 gchar *previous_name = migrated_name;
3139 migrated_name = _database_migrate_to_xdg_structure(library);
3140 if(!IS_NULL_PTR(migrated_name))
3141 {
3142 library = migrated_name;
3143 dt_free(previous_name);
3144 }
3145 else
3146 {
3147 migrated_name = previous_name; // keep ownership: `library` may point into it
3148 }
3149 }
3150
3151 /* delete old mipmaps files */
3152 _database_delete_mipmaps_files();
3153
3154 /* lets construct the db filename */
3155 gchar *dbname = NULL;
3156 gchar dbfilename_library[DT_PATH_MAX] = { 0 };
3157 gchar datadir[DT_PATH_MAX] = { 0 };
3158
3159 dt_loc_get_user_config_dir(datadir, sizeof(datadir));
3160
3161 if(IS_NULL_PTR(alternative))
3162 {
3163 /* The configured library name, told to us rather than read from conf here. */
3164 dbname = g_strdup(library);
3165 if(IS_NULL_PTR(dbname))
3166 dt_concat_path_file(dbfilename_library, datadir, "library.db");
3167 else if(!strcmp(dbname, ":memory:"))
3168 g_strlcpy(dbfilename_library, dbname, sizeof(dbfilename_library));
3169 else if(dbname[0] != '/')
3170 dt_concat_path_file(dbfilename_library, datadir, dbname);
3171 else
3172 g_strlcpy(dbfilename_library, dbname, sizeof(dbfilename_library));
3173 }
3174 else
3175 {
3176 g_strlcpy(dbfilename_library, alternative, sizeof(dbfilename_library));
3177
3178 GFile *galternative = g_file_new_for_path(alternative);
3179 dbname = g_file_get_basename(galternative);
3180 g_object_unref(galternative);
3181 }
3182
3183 /* we also need a 2nd db with permanent data like presets, styles and tags */
3184 char dbfilename_data[DT_PATH_MAX] = { 0 };
3185 if(load_data)
3186 dt_concat_path_file(dbfilename_data, datadir, "data.db");
3187 else
3188 snprintf(dbfilename_data, sizeof(dbfilename_data), ":memory:");
3189
3190 /* create database */
3191 dt_database_t *db = (dt_database_t *)g_malloc0(sizeof(dt_database_t));
3192 db->dbfilename_data = g_strdup(dbfilename_data);
3193 db->dbfilename_library = g_strdup(dbfilename_library);
3194
3195 dt_atomic_set_int(&_trxid, 0);
3196 dt_atomic_set_int(&_trx_batch_level, 0);
3197 g_atomic_pointer_set(&_trx_batch_owner, NULL);
3198
3199 /* make sure the folder exists. this might not be the case for new databases */
3200 /* also check if a database backup is needed */
3201 if(g_strcmp0(dbfilename_data, ":memory:"))
3202 {
3203 char *data_path = g_path_get_dirname(dbfilename_data);
3204 g_mkdir_with_parents(data_path, 0750);
3205 dt_free(data_path);
3206 dt_database_backup(dbfilename_data);
3207 }
3208 if(g_strcmp0(dbfilename_library, ":memory:"))
3209 {
3210 char *library_path = g_path_get_dirname(dbfilename_library);
3211 g_mkdir_with_parents(library_path, 0750);
3212 dt_free(library_path);
3213 dt_database_backup(dbfilename_library);
3214 }
3215
3216 _db_print("[init sql] library: %s, data: %s\n", dbfilename_library, dbfilename_data);
3217
3218 /* having more than one instance of darktable using the same database is a bad idea */
3219 /* try to get locks for the databases */
3220 db->lock_acquired = _lock_databases(db);
3221
3222 if(!db->lock_acquired)
3223 {
3224 fprintf(stderr, "[init] database is locked, probably another process is already using it\n");
3225 dt_free(dbname);
3226 dt_free(migrated_name);
3227 return db;
3228 }
3229
3230 /* opening / creating database */
3231 if(sqlite3_open(db->dbfilename_library, &db->handle))
3232 {
3233 fprintf(stderr, "[init] could not find database ");
3234 if(dbname)
3235 fprintf(stderr, "`%s'!\n", dbname);
3236 else
3237 fprintf(stderr, "\n");
3238 fprintf(stderr, "[init] maybe your %s/anselrc is corrupt?\n", datadir);
3239 dt_loc_get_datadir(dbfilename_library, sizeof(dbfilename_library));
3240 fprintf(stderr, "[init] try `cp %s/anselrc %s/anselrc'\n", dbfilename_library, datadir);
3241 sqlite3_close(db->handle);
3242 dt_free(dbname);
3243 dt_free(migrated_name);
3244 dt_free(db->lockfile_data);
3245 dt_free(db->dbfilename_data);
3246 dt_free(db->lockfile_library);
3247 dt_free(db->dbfilename_library);
3248 dt_free(db);
3249 return NULL;
3250 }
3251
3252 _db_print("[sql] Opened database: '%s'\n", db->dbfilename_library);
3253
3254 sqlite3_stmt *stmt;
3255 int rc;
3256
3257 if(sqlite3_db_readonly(db->handle, "main") != 0)
3258 {
3259 fprintf(stderr, "%s database is read only. Abort...\n", db->dbfilename_library);
3260
3261 // Informational: there is nothing the user can decide here, only be told.
3262 _ask_readonly(db->dbfilename_library);
3263 _database_free(db);
3264 dt_free(dbname);
3265 dt_free(migrated_name);
3266 db = NULL;
3267 return NULL;
3268 }
3269
3270 _db_print("[sql] Checked that we can write in database: '%s'\n", db->dbfilename_library);
3271
3272 /* attach a memory database to db connection for use with temporary tables
3273 used during instance life time, which is discarded on exit.
3274 */
3275 sqlite3_exec(db->handle, "attach database ':memory:' as memory", NULL, NULL, NULL);
3276
3277 // attach the data database which contains presets, styles, tags and similar things not tied to single images
3278 gboolean have_data_db = load_data && g_file_test(dbfilename_data, G_FILE_TEST_EXISTS);
3279 rc = sqlite3_prepare_v2(db->handle, "ATTACH DATABASE ?1 AS data", -1, &stmt, NULL);
3280 sqlite3_bind_text(stmt, 1, dbfilename_data, -1, SQLITE_TRANSIENT);
3281 if(rc != SQLITE_OK || sqlite3_step(stmt) != SQLITE_DONE)
3282 {
3283 sqlite3_finalize(stmt);
3284 fprintf(stderr, "[init] database `%s' couldn't be opened. aborting\n", dbfilename_data);
3285 _database_free(db);
3286 db = NULL;
3287 goto error;
3288 }
3289 sqlite3_finalize(stmt);
3290
3291 _db_print("[sql] Opened database: '%s'\n", dbfilename_data);
3292
3293 // some sqlite3 config
3294 sqlite3_exec(db->handle, "PRAGMA synchronous = OFF", NULL, NULL, NULL);
3295 sqlite3_exec(db->handle, "PRAGMA journal_mode = MEMORY", NULL, NULL, NULL);
3296 sqlite3_exec(db->handle, "PRAGMA page_size = 32768", NULL, NULL, NULL);
3297
3298 // WARNING: the foreign_keys pragma must not be used, the integrity of the
3299 // database rely on it.
3300 sqlite3_exec(db->handle, "PRAGMA foreign_keys = ON", NULL, NULL, NULL);
3301
3302 _db_print("[sql] Configured database\n");
3303
3304 /* now that we got functional databases that are locked for us we can make sure that the schema is set up */
3305
3306 // first we update the data database to the latest version so that we can potentially move data from the library
3307 // over when updating that one
3308 if(!have_data_db)
3309 {
3310 _create_data_schema(db); // a brand new db it seems
3311 }
3312 else
3313 {
3314 gchar* data_status = _get_pragma_string_val(db->handle, "data.quick_check");
3315 rc = sqlite3_prepare_v2(db->handle, "select value from data.db_info where key = 'version'", -1, &stmt, NULL);
3316 if(!g_strcmp0(data_status, "ok") && rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3317 {
3318 dt_free(data_status); // status is OK and we don't need to care :)
3319
3320 // compare the version of the db with what is current for this executable
3321 const int db_version = sqlite3_column_int(stmt, 0);
3322 sqlite3_finalize(stmt);
3323 if(db_version < CURRENT_DATABASE_VERSION_DATA)
3324 {
3325 ask_for_upgrade(dbfilename_data, has_gui);
3326
3327 // older: upgrade
3328 if(!_upgrade_data_schema(db, db_version))
3329 {
3330 // we couldn't upgrade the db for some reason. bail out.
3331 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version %d to %d. aborting\n",
3332 dbfilename_data, db_version, CURRENT_DATABASE_VERSION_DATA);
3333 _database_free(db);
3334 db = NULL;
3335 goto error;
3336 }
3337 }
3338 else if(db_version > CURRENT_DATABASE_VERSION_DATA)
3339 {
3340 // newer: bail out
3341 fprintf(stderr, "[init] database version of `%s' is too new for this build of darktable. aborting\n",
3342 dbfilename_data);
3343 _database_free(db);
3344 db = NULL;
3345 goto error;
3346 }
3347 // else: the current version, do nothing
3348 }
3349 else
3350 {
3351 // oh, bad situation. the database is corrupt and can't be read!
3352 // we inform the user here and let him decide what to do: exit or delete and try again.
3353
3354 gchar* quick_check_text = NULL;
3355 if(g_strcmp0(data_status, "ok")) // data_status is not ok
3356 {
3357 quick_check_text = g_strdup_printf(_("quick_check said:\n"
3358 "%s \n"), data_status);
3359 }
3360 else
3361 {
3362 quick_check_text = g_strdup(""); // a trick;
3363 }
3364
3365 gchar *data_snap = dt_database_get_most_recent_snap(dbfilename_data);
3366
3367 // The user's answer decides whether init aborts, restores or starts fresh, so it has to
3368 // come back as a value. Everything about how it is obtained -- and whether anybody can be
3369 // asked at all -- belongs to whoever registered the handler.
3370 const dt_database_response_t resp
3371 = _ask_corrupted(dbfilename_data, quick_check_text, data_snap != NULL);
3372
3373 dt_free(quick_check_text);
3374
3375 _database_free(db);
3376 db = NULL;
3377
3378 if(resp != DT_DATABASE_RESPONSE_RESTORE && resp != DT_DATABASE_RESPONSE_DELETE)
3379 {
3380 fprintf(stderr, "[init] database `%s' is corrupt and can't be opened! either replace it from a backup or "
3381 "delete the file so that darktable can create a new one the next time. aborting\n", dbfilename_data);
3382 dt_free(data_snap);
3383 goto error;
3384 }
3385
3386 //here were sure that response is either accept (restore from snap) or reject (just delete the damaged db)
3387
3388 fprintf(stderr, "[init] deleting `%s' on user request", dbfilename_data);
3389
3390 if(g_unlink(dbfilename_data) == 0)
3391 fprintf(stderr, " ... ok\n");
3392 else
3393 fprintf(stderr, " ... failed\n");
3394
3395 if(resp == DT_DATABASE_RESPONSE_RESTORE && data_snap)
3396 {
3397 fprintf(stderr, "[init] restoring `%s' from `%s'...", dbfilename_data, data_snap);
3398 GError *gerror = NULL;
3399 if(!g_file_test(dbfilename_data, G_FILE_TEST_EXISTS))
3400 {
3401 GFile *src = g_file_new_for_path(data_snap);
3402 GFile *dest = g_file_new_for_path(dbfilename_data);
3403 gboolean copy_status = TRUE;
3404 if(g_file_test(data_snap, G_FILE_TEST_EXISTS))
3405 {
3406 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
3407 if(copy_status) copy_status = g_chmod(dbfilename_data, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == 0;
3408 }
3409 else
3410 {
3411 // there is nothing to restore, create an empty file
3412 const int fd = g_open(dbfilename_data, O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
3413 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
3414 }
3415 if(copy_status)
3416 fprintf(stderr, " success!\n");
3417 else
3418 fprintf(stderr, " failed!\n");
3419
3420 g_object_unref(src);
3421 g_object_unref(dest);
3422 }
3423 }
3424 dt_free(data_snap);
3425 dt_free(dbname);
3426 goto start;
3427 }
3428 }
3429
3430 _db_print("[sql] Checked database schema and migrated if needed\n");
3431
3432 gchar* libdb_status = _get_pragma_string_val(db->handle, "main.quick_check");
3433 // next we are looking at the library database
3434 // does the db contain the new 'db_info' table?
3435 rc = sqlite3_prepare_v2(db->handle, "select value from main.db_info where key = 'version'", -1, &stmt, NULL);
3436 if(!g_strcmp0(libdb_status, "ok") && rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3437 {
3438 dt_free(libdb_status);//it's ok :)
3439
3440 // compare the version of the db with what is current for this executable
3441 const int db_version = sqlite3_column_int(stmt, 0);
3442 sqlite3_finalize(stmt);
3443
3444 if(db_version < CURRENT_DATABASE_VERSION_LIBRARY)
3445 {
3446 ask_for_upgrade(dbfilename_library, has_gui);
3447
3448 // older: upgrade
3449 if(!_upgrade_library_schema(db, db_version))
3450 {
3451 // we couldn't upgrade the db for some reason. bail out.
3452 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version %d to %d. aborting\n", dbname,
3453 db_version, CURRENT_DATABASE_VERSION_LIBRARY);
3454 _database_free(db);
3455 db = NULL;
3456 goto error;
3457 }
3458 }
3459 else if(db_version > CURRENT_DATABASE_VERSION_LIBRARY)
3460 {
3461 // newer: bail out. it's better than what we did before: delete everything
3462 fprintf(stderr, "[init] database version of `%s' is too new for this build of darktable. aborting\n",
3463 dbname);
3464 _database_free(db);
3465 db = NULL;
3466 goto error;
3467 }
3468 // else: the current version, do nothing
3469 }
3470 else if(g_strcmp0(libdb_status, "ok") || rc == SQLITE_CORRUPT || rc == SQLITE_NOTADB)
3471 {
3472 // oh, bad situation. the database is corrupt and can't be read!
3473 // we inform the user here and let him decide what to do: exit or delete and try again.
3474
3475 gchar* quick_check_text = NULL;
3476 if(g_strcmp0(libdb_status, "ok")) // data_status is not ok
3477 {
3478 quick_check_text = g_strdup_printf(_("quick_check said:\n"
3479 "%s \n"), libdb_status);
3480 }
3481 else
3482 {
3483 quick_check_text = g_strdup(""); // a trick;
3484 }
3485
3486 gchar *data_snap = dt_database_get_most_recent_snap(dbfilename_library);
3487
3488 // The user's answer decides whether init aborts, restores or starts fresh, so it has to
3489 // come back as a value. Everything about how it is obtained -- and whether anybody can be
3490 // asked at all -- belongs to whoever registered the handler.
3491 const dt_database_response_t resp
3492 = _ask_corrupted(dbfilename_library, quick_check_text, data_snap != NULL);
3493
3494 dt_free(quick_check_text);
3495
3496 _database_free(db);
3497 db = NULL;
3498
3499 if(resp != DT_DATABASE_RESPONSE_RESTORE && resp != DT_DATABASE_RESPONSE_DELETE)
3500 {
3501 fprintf(stderr, "[init] database `%s' is corrupt and can't be opened! either replace it from a backup or "
3502 "delete the file so that darktable can create a new one the next time. aborting\n", dbfilename_library);
3503 dt_free(data_snap);
3504 goto error;
3505 }
3506
3507 //here were sure that response is either accept (restore from snap) or reject (just delete the damaged db)
3508
3509 fprintf(stderr, "[init] deleting `%s' on user request", dbfilename_library);
3510
3511 if(g_unlink(dbfilename_library) == 0)
3512 fprintf(stderr, " ... ok\n");
3513 else
3514 fprintf(stderr, " ... failed\n");
3515
3516 if(resp == DT_DATABASE_RESPONSE_RESTORE && data_snap)
3517 {
3518 fprintf(stderr, "[init] restoring `%s' from `%s'...", dbfilename_library, data_snap);
3519 GError *gerror = NULL;
3520 if(!g_file_test(dbfilename_library, G_FILE_TEST_EXISTS))
3521 {
3522 GFile *src = g_file_new_for_path(data_snap);
3523 GFile *dest = g_file_new_for_path(dbfilename_library);
3524 gboolean copy_status = TRUE;
3525 if(g_file_test(data_snap, G_FILE_TEST_EXISTS))
3526 {
3527 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
3528 if(copy_status) copy_status = g_chmod(dbfilename_library, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == 0;
3529 }
3530 else
3531 {
3532 // there is nothing to restore, create an empty file to prevent further backup attempts
3533 const int fd = g_open(dbfilename_library, O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
3534 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
3535 }
3536 if(copy_status)
3537 fprintf(stderr, " success!\n");
3538 else
3539 fprintf(stderr, " failed!\n");
3540 g_object_unref(src);
3541 g_object_unref(dest);
3542 }
3543 }
3544 dt_free(data_snap);
3545 dt_free(dbname);
3546 goto start;
3547 }
3548 else
3549 {
3550 // does it contain the legacy 'settings' table?
3551 sqlite3_finalize(stmt);
3552 rc = sqlite3_prepare_v2(db->handle, "select settings from main.settings", -1, &stmt, NULL);
3553 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3554 {
3555 // the old blob had the version as an int in the first place
3556 const void *set = sqlite3_column_blob(stmt, 0);
3557 const int db_version = *(int *)set;
3558 sqlite3_finalize(stmt);
3559 if(!_migrate_schema(db, db_version)) // bring the legacy layout to the first one known to our upgrade
3560 // path ...
3561 {
3562 // we couldn't migrate the db for some reason. bail out.
3563 fprintf(stderr, "[init] database `%s' couldn't be migrated from the legacy version %d. aborting\n",
3564 dbname, db_version);
3565 _database_free(db);
3566 db = NULL;
3567 goto error;
3568 }
3569 if(!_upgrade_library_schema(db, 1)) // ... and upgrade it
3570 {
3571 // we couldn't upgrade the db for some reason. bail out.
3572 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version 1 to %d. aborting\n", dbname,
3573 CURRENT_DATABASE_VERSION_LIBRARY);
3574 _database_free(db);
3575 db = NULL;
3576 goto error;
3577 }
3578 }
3579 else
3580 {
3581 sqlite3_finalize(stmt);
3582 _create_library_schema(db); // a brand new db it seems
3583 }
3584 }
3585
3586 _db_print("[sql] Checked db_info\n");
3587
3588 // create the in-memory tables
3589 _create_memory_schema(db);
3590
3591 _db_print("[sql] Created in-memory DB\n");
3592
3593 // create a table legacy_presets with all the presets from pre-auto-apply-cleanup darktable.
3594 dt_legacy_presets_create(db->handle);
3595
3596 _db_print("[sql] Loaded legacy presets\n");
3597
3598 // drop table settings -- we don't want old versions of dt to drop our tables
3599 sqlite3_exec(db->handle, "drop table main.settings", NULL, NULL, NULL);
3600
3601 // take care of potential bad data in the db.
3602 _sanitize_db(db);
3603
3604#ifdef HAVE_ICU
3605 // check if sqlite is already icu enabled
3606 // if not enabled expected error: no such function:icu_load_collation
3607 rc = sqlite3_prepare_v2(db->handle,
3608 "SELECT icu_load_collation('en_US', 'english')",
3609 -1, &stmt, NULL);
3610 sqlite3_finalize(stmt);
3611
3612 if(rc != SQLITE_OK)
3613 {
3614 rc = sqlite3IcuInit(db->handle);
3615 if(rc != SQLITE_OK)
3616 fprintf(stderr, "[sqlite] init icu extension error %d\n", rc);
3617 }
3618#endif
3619
3620error:
3621 dt_free(dbname);
3622 dt_free(migrated_name);
3623
3624 return db;
3625}
3626
3631
3632dt_database_open_result_t dt_database_open(const dt_database_params_t *params)
3633{
3634 if(IS_NULL_PTR(params)) return DT_DATABASE_OPEN_FAILED;
3635
3636 if(!IS_NULL_PTR(_db))
3637 {
3638 fprintf(stderr, "[database] a database is already open; close it first\n");
3639 return DT_DATABASE_OPEN_FAILED;
3640 }
3641
3642 if(!_db_lock_inited)
3643 {
3644 dt_pthread_rwlock_init(&_db_lock, NULL);
3645 _db_lock_inited = TRUE;
3646 }
3647 if(!_settings_lock_inited)
3648 {
3649 dt_pthread_mutex_init(&_settings_lock, NULL);
3650 _settings_lock_inited = TRUE;
3651 }
3652
3653 /* Session constants. Read once, here -- nothing below consults the debug flags or the
3654 * configuration again. */
3655 _has_gui = params->has_gui;
3656 _verbose = params->verbose;
3657
3658 _db = _database_init(params);
3659
3660 if(IS_NULL_PTR(_db)) return DT_DATABASE_OPEN_FAILED;
3661 if(!_db->lock_acquired) return DT_DATABASE_OPEN_LOCKED;
3662
3663 return DT_DATABASE_OPEN_OK;
3664}
3665
3684
3685void dt_database_get_settings(dt_database_settings_t *settings)
3686{
3687 if(IS_NULL_PTR(settings)) return;
3688
3689 if(!_settings_lock_inited)
3690 {
3691 /* Nothing has been told to us yet: report the neutral policy rather than read a
3692 * half-initialised struct behind an uninitialised mutex. */
3693 settings->maintenance_check = NULL;
3694 settings->create_snapshot = NULL;
3695 settings->maintenance_freepage_ratio = 0;
3696 settings->keep_snapshots = 0;
3697 return;
3698 }
3699
3700 /* One snapshot under the lock. Four fields read one at a time can be a mix of old and
3701 * new -- the GUI thread replaces them from the preferences dialog while a maintenance
3702 * decision may be part-way through reading them. */
3703 dt_pthread_mutex_lock(&_settings_lock);
3704 settings->maintenance_check = g_strdup(_settings.maintenance_check);
3705 settings->create_snapshot = g_strdup(_settings.create_snapshot);
3706 settings->maintenance_freepage_ratio = _settings.maintenance_freepage_ratio;
3707 settings->keep_snapshots = _settings.keep_snapshots;
3708 dt_pthread_mutex_unlock(&_settings_lock);
3709}
3710
3711void dt_database_settings_free(dt_database_settings_t *settings)
3712{
3713 if(IS_NULL_PTR(settings)) return;
3714 dt_free(settings->maintenance_check);
3715 dt_free(settings->create_snapshot);
3716 settings->maintenance_check = NULL;
3717 settings->create_snapshot = NULL;
3718}
3719
3720/* Close a connection and release its lock files.
3721 *
3722 * Separate from dt_database_close() because _database_init() calls it on a database that
3723 * was never published -- it has no lock to take and nothing to unpublish. The error
3724 * strings are freed here, which the old dt_database_destroy() did not do; on the ordinary
3725 * shutdown path they are always NULL, but every failure path through _database_init()
3726 * sets one and then tore the database down. */
3727static void _database_free(dt_database_t *db)
3728{
3729 if(IS_NULL_PTR(db)) return;
3730
3731 sqlite3_close(db->handle);
3732 if (db->lockfile_data)
3733 {
3734 g_unlink(db->lockfile_data);
3735 dt_free(db->lockfile_data);
3736 }
3737 if (db->lockfile_library)
3738 {
3739 g_unlink(db->lockfile_library);
3740 dt_free(db->lockfile_library);
3741 }
3742 dt_free(db->dbfilename_data);
3743 dt_free(db->dbfilename_library);
3744 dt_free(db->error_message);
3745 dt_free(db->error_dbfilename);
3746 dt_free(db);
3747}
3748
3749void dt_database_close(void)
3750{
3751 /* Exclusive against the TRANSACTION writers -- begin/end and their batch variants are
3752 * the only readers-side acquirers of _db_lock today, so this waits for open transactions
3753 * and for nothing else. It does NOT wait for a repository mid-sqlite3_step on another
3754 * thread (no per-query read lock exists yet), and it cannot wait for the call sites
3755 * holding a raw connection -- nothing can, which is the whole argument for getting rid
3756 * of them. Callers must stop the workers before closing. */
3757 if(_db_lock_inited) dt_pthread_rwlock_wrlock(&_db_lock);
3758
3759 dt_database_t *const db = _db;
3760 _db = NULL;
3761
3762 if(!IS_NULL_PTR(db))
3763 {
3764 _database_free(db);
3765 sqlite3_shutdown();
3766 }
3767
3768 if(_db_lock_inited) dt_pthread_rwlock_unlock(&_db_lock);
3769}
3770
3771gboolean dt_database_is_open(void)
3772{
3773 return !IS_NULL_PTR(_db);
3774}
3775
3776const char *dt_database_get_last_error(void)
3777{
3778 if(IS_NULL_PTR(_db) || IS_NULL_PTR(_db->handle)) return "no database connection";
3779 return sqlite3_errmsg(_db->handle);
3780}
3781
3782sqlite3 *dt_database_get_sqlite3_global(void)
3783{
3784 return _db ? _db->handle : NULL;
3785}
3786
3787const gchar *dt_database_get_path(void)
3788{
3789 return _db ? _db->dbfilename_library : NULL;
3790}
3791
3792/* @p configured_db is the library name as the orchestrator has it (the "database" conf
3793 * key). When the legacy file under $HOME is moved into the XDG data directory the module
3794 * does not write that key back itself -- it reports the new name through
3795 * ::_renamed_handler and lets whoever owns the configuration persist it, and returns it
3796 * so this run opens the file that now exists. */
3797static gchar *_database_migrate_to_xdg_structure(const char *configured_db)
3798{
3799 gchar dbfilename[DT_PATH_MAX] = { 0 };
3800
3801 /* The destination must be the directory _database_init() resolves a relative library
3802 * name against -- the user config dir -- because the name this returns is exactly such a
3803 * relative name. It used to be dt_loc_get_datadir(), a fossil from when "datadir" WAS the
3804 * user's directory; today that is the read-only <prefix>/share/ansel, where the rename
3805 * fails on a system install (EACCES, or EXDEV across mounts) and would land the file
3806 * where the open path never looks on any install where it could succeed. */
3807 gchar configdir[DT_PATH_MAX] = { 0 };
3808 dt_loc_get_user_config_dir(configdir, sizeof(configdir));
3809
3810 if(configured_db && configured_db[0] != '/')
3811 {
3812 const char *homedir = getenv("HOME");
3813 snprintf(dbfilename, sizeof(dbfilename), "%s/%s", homedir, configured_db);
3814 if(g_file_test(dbfilename, G_FILE_TEST_EXISTS))
3815 {
3816 char destdbname[DT_PATH_MAX] = { 0 };
3817 dt_concat_path_file(destdbname, configdir, "library.db");
3818 if(!g_file_test(destdbname, G_FILE_TEST_EXISTS))
3819 {
3820 fprintf(stderr, "[init] moving database into new XDG directory structure\n");
3821 if(rename(dbfilename, destdbname) != 0)
3822 {
3823 /* Report the migration ONLY when it happened: the old code returned success on a
3824 * failed rename, so conf was rewritten to name a file that was never created and
3825 * the next open built an empty library while the real one sat orphaned in $HOME. */
3826 fprintf(stderr, "[init] could not move `%s' to `%s' (%s), keeping the old location\n",
3827 dbfilename, destdbname, strerror(errno));
3828 return NULL;
3829 }
3830 if(!IS_NULL_PTR(_renamed_handler)) _renamed_handler("library.db");
3831 return g_strdup("library.db");
3832 }
3833 }
3834 }
3835
3836 return NULL;
3837}
3838
3839/* delete old mipmaps files */
3840static void _database_delete_mipmaps_files()
3841{
3842 /* This migration is intended to be run only from 0.9.x to new cache in 1.0 */
3843
3844 // Directory
3845 char cachedir[DT_PATH_MAX] = { 0 }, mipmapfilename[DT_PATH_MAX] = { 0 };
3846 dt_loc_get_user_cache_dir(cachedir, sizeof(cachedir));
3847 dt_concat_path_file(mipmapfilename, cachedir, "mipmaps");
3848
3849 if(g_access(mipmapfilename, F_OK) != -1)
3850 {
3851 fprintf(stderr, "[mipmap_cache] dropping old version file: %s\n", mipmapfilename);
3852 g_unlink(mipmapfilename);
3853 dt_concat_path_file(mipmapfilename, cachedir, "mipmaps.fallback");
3854
3855 if(g_access(mipmapfilename, F_OK) != -1) g_unlink(mipmapfilename);
3856 }
3857}
3858
3859
3860
3861void dt_database_cleanup_busy_statements(void)
3862{
3863 /* was a parameter; the module owns the one connection now */
3864 const dt_database_t *const db = _db;
3865 if(IS_NULL_PTR(db)) return;
3866 sqlite3_stmt *stmt = NULL;
3867 while( (stmt = sqlite3_next_stmt(db->handle, NULL)) != NULL)
3868 {
3869 const char* sql = sqlite3_sql(stmt);
3870 if(sqlite3_stmt_busy(stmt))
3871 {
3872 _db_print("[db busy stmt] non-finalized nor stepped through statement: '%s'\n",sql);
3873 sqlite3_reset(stmt);
3874 }
3875 else {
3876 _db_print("[db busy stmt] non-finalized statement: '%s'\n",sql);
3877 }
3878 sqlite3_finalize(stmt);
3879 }
3880}
3881
3882#define ERRCHECK {if (err!=NULL) {_db_print("[db maintenance] maintenance error: '%s'\n",err); sqlite3_free(err); err=NULL;}}
3883void dt_database_perform_maintenance(void)
3884{
3885 /* was a parameter; the module owns the one connection now */
3886 const dt_database_t *const db = _db;
3887 if(IS_NULL_PTR(db)) return;
3888 char* err = NULL;
3889
3890 const int main_pre_free_count = _get_pragma_int_val(db->handle, "main.freelist_count");
3891 const int main_page_size = _get_pragma_int_val(db->handle, "main.page_size");
3892 const int data_pre_free_count = _get_pragma_int_val(db->handle, "data.freelist_count");
3893 const int data_page_size = _get_pragma_int_val(db->handle, "data.page_size");
3894
3895 const guint64 calc_pre_size = (main_pre_free_count*main_page_size) + (data_pre_free_count*data_page_size);
3896
3897 if(calc_pre_size == 0)
3898 {
3899 _db_print("[db maintenance] maintenance deemed unnecesary, performing only analyze.\n");
3900 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE data", NULL, NULL, &err);
3901 ERRCHECK
3902 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE main", NULL, NULL, &err);
3903 ERRCHECK
3904 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE", NULL, NULL, &err);
3905 ERRCHECK
3906 return;
3907 }
3908
3909 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM data", NULL, NULL, &err);
3910 ERRCHECK
3911 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM main", NULL, NULL, &err);
3912 ERRCHECK
3913 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE data", NULL, NULL, &err);
3914 ERRCHECK
3915 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE main", NULL, NULL, &err);
3916 ERRCHECK
3917
3918 // for some reason this is needed in some cases
3919 // in case above performed vacuum+analyze properly, this is noop.
3920 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM", NULL, NULL, &err);
3921 ERRCHECK
3922 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE", NULL, NULL, &err);
3923 ERRCHECK
3924
3925 const int main_post_free_count = _get_pragma_int_val(db->handle, "main.freelist_count");
3926 const int data_post_free_count = _get_pragma_int_val(db->handle, "data.freelist_count");
3927
3928 const guint64 calc_post_size = (main_post_free_count*main_page_size) + (data_post_free_count*data_page_size);
3929 const gint64 bytes_freed = calc_pre_size - calc_post_size;
3930
3931 _db_print("[db maintenance] maintenance done, %" G_GINT64_FORMAT " bytes freed.\n", bytes_freed);
3932
3933 if(calc_post_size >= calc_pre_size)
3934 {
3935 _db_print("[db maintenance] maintenance problem. if no errors logged, it should work fine next time.\n");
3936 }
3937}
3938#undef ERRCHECK
3939
3940/* State the facts and take back an answer.
3941 *
3942 * The prose this used to build here -- how much would be freed, and which of three
3943 * "you'll be asked again..." sentences applies -- is the handler's to write. What the
3944 * module knows, and all it passes, is the byte count and the booleans that decide which
3945 * sentence is true. */
3946static gboolean _ask_for_maintenance(const char *maintenance_check, const gboolean closing_time,
3947 const guint64 size)
3948{
3949 if(!_has_gui)
3950 return FALSE;
3951
3952 const gboolean on_both = !g_strcmp0(maintenance_check, "on both");
3953
3954 const dt_database_prompt_context_t context
3955 = { .prompt = DT_DATABASE_PROMPT_MAINTENANCE,
3956 .dbfilename = _db->dbfilename_library,
3957 .reclaimable_bytes = size,
3958 .at_close = closing_time,
3959 .ask_on_startup = (closing_time && on_both) || !g_strcmp0(maintenance_check, "on startup"),
3960 .ask_on_close = (!closing_time && on_both) || !g_strcmp0(maintenance_check, "on close") };
3961
3962 return _ask_user(&context) == DT_DATABASE_RESPONSE_PROCEED;
3963}
3964
3965static inline gboolean _is_mem_db(void)
3966{
3967 return !g_strcmp0(_db->dbfilename_data, ":memory:") || !g_strcmp0(_db->dbfilename_library, ":memory:");
3968}
3969
3970gboolean dt_database_maybe_maintenance(const gboolean closing_time)
3971{
3972 if(IS_NULL_PTR(_db) || _is_mem_db())
3973 return FALSE;
3974
3975 dt_database_settings_t settings = { 0 };
3976 dt_database_get_settings(&settings);
3977 const char *config = settings.maintenance_check;
3978
3979 if(!g_strcmp0(config, "never"))
3980 {
3981 // early bail out on "never"
3982 _db_print("[db maintenance] please consider enabling database maintenance.\n");
3983 dt_database_settings_free(&settings);
3984 return FALSE;
3985 }
3986
3987 gboolean check_for_maintenance = FALSE;
3988 const gboolean force_maintenance = g_str_has_suffix (config, "(don't ask)");
3989
3990 if(config)
3991 {
3992 if((strstr(config, "on both")) // should cover "(don't ask) suffix
3993 || (closing_time && (strstr(config, "on close")))
3994 || (!closing_time && (strstr(config, "on startup"))))
3995 {
3996 // we have "on both/on close/on startup" setting, so - checking!
3997 _db_print("[db maintenance] checking for maintenance, due to rule: '%s'.\n", config);
3998 check_for_maintenance = TRUE;
3999 }
4000 // if the config was "never", check_for_vacuum is false.
4001 }
4002
4003 if(!check_for_maintenance)
4004 {
4005 dt_database_settings_free(&settings);
4006 return FALSE;
4007 }
4008
4009 // checking free pages
4010 const int main_free_count = _get_pragma_int_val(_db->handle, "main.freelist_count");
4011 const int main_page_count = _get_pragma_int_val(_db->handle, "main.page_count");
4012 const int main_page_size = _get_pragma_int_val(_db->handle, "main.page_size");
4013
4014 const int data_free_count = _get_pragma_int_val(_db->handle, "data.freelist_count");
4015 const int data_page_count = _get_pragma_int_val(_db->handle, "data.page_count");
4016 const int data_page_size = _get_pragma_int_val(_db->handle, "data.page_size");
4017
4018 _db_print("[db maintenance] main: [%d/%d pages], data: [%d/%d pages].\n",
4019 main_free_count, main_page_count, data_free_count, data_page_count);
4020
4021 if(main_page_count <= 0 || data_page_count <= 0)
4022 {
4023 //something's wrong with PRAGMA page_size returns. early bail.
4024 _db_print("[db maintenance] page_count <= 0 : main.page_count: %d, data.page_count: %d \n",
4025 main_page_count, data_page_count);
4026 dt_database_settings_free(&settings);
4027 return FALSE;
4028 }
4029
4030 // we don't need fine-grained percentages, so let's do ints
4031 const int main_free_percentage = (main_free_count * 100 ) / main_page_count;
4032 const int data_free_percentage = (data_free_count * 100 ) / data_page_count;
4033
4034 const int freepage_ratio = settings.maintenance_freepage_ratio;
4035 gboolean maintenance = FALSE;
4036
4037 if((main_free_percentage >= freepage_ratio)
4038 || (data_free_percentage >= freepage_ratio))
4039 {
4040 const guint64 calc_size = (main_free_count*main_page_size) + (data_free_count*data_page_size);
4041 _db_print("[db maintenance] maintenance suggested, %" G_GUINT64_FORMAT " bytes to free.\n", calc_size);
4042
4043 maintenance = force_maintenance || _ask_for_maintenance(config, closing_time, calc_size);
4044 }
4045
4046 dt_database_settings_free(&settings);
4047 return maintenance;
4048}
4049
4050void dt_database_optimize(void)
4051{
4052 /* was a parameter; the module owns the one connection now */
4053 const dt_database_t *const db = _db;
4054 if(IS_NULL_PTR(db) || _is_mem_db())
4055 return;
4056 // optimize should in most cases be no-op and have no noticeable downsides
4057 // this should be ran on every exit
4058 // see: https://www.sqlite.org/pragma.html#pragma_optimize
4059 DT_DEBUG_SQLITE3_EXEC(db->handle, "PRAGMA optimize", NULL, NULL, NULL);
4060}
4061
4062static void _print_backup_progress(int remaining, int total)
4063{
4064 // TODO if we have closing splashpage - this can be used to advance progressbar :)
4065 _db_print("[db backup] %d out of %d done\n", total - remaining, total);
4066}
4067
4068static int _backup_db(
4069 sqlite3 *src_db, /* Database handle to back up */
4070 const char *src_db_name, /* Database name to back up */
4071 const char *dest_filename, /* Name of file to back up to */
4072 void(*xProgress)(int, int) /* Progress function to invoke */
4073)
4074{
4075 sqlite3 *dest_db; /* Database connection opened on zFilename */
4076 sqlite3_backup *sb_dest; /* Backup handle used to copy data */
4077
4078 /* Open the database file identified by zFilename. */
4079 int rc = sqlite3_open(dest_filename, &dest_db);
4080
4081 if(rc == SQLITE_OK)
4082 {
4083 /* Open the sqlite3_backup object used to accomplish the transfer */
4084 sb_dest = sqlite3_backup_init(dest_db, "main", src_db, src_db_name);
4085 if(sb_dest)
4086 {
4087 _db_print("[db backup] %s to %s\n", src_db_name, dest_filename);
4088 gchar *pragma = g_strdup_printf("%s.page_count", src_db_name);
4089 const int spc = _get_pragma_int_val(src_db, pragma);
4090 dt_free(pragma);
4091 const int pc = MIN(spc, MAX(5,spc/100));
4092 do
4093 {
4094 rc = sqlite3_backup_step(sb_dest, pc);
4095 if(xProgress)
4096 xProgress(
4097 sqlite3_backup_remaining(sb_dest),
4098 sqlite3_backup_pagecount(sb_dest)
4099 );
4100 if( rc==SQLITE_OK || rc==SQLITE_BUSY || rc==SQLITE_LOCKED )
4101 {
4102 sqlite3_sleep(25);
4103 }
4104 }
4105 while( rc==SQLITE_OK || rc==SQLITE_BUSY || rc==SQLITE_LOCKED );
4106
4107 /* Release resources allocated by backup_init(). */
4108 (void)sqlite3_backup_finish(sb_dest);
4109 }
4110 rc = sqlite3_errcode(dest_db);
4111 }
4112 /* Close the database connection opened on database file zFilename
4113 ** and return the result of this function. */
4114 (void)sqlite3_close(dest_db);
4115 return rc;
4116}
4117
4118gboolean dt_database_snapshot(void)
4119{
4120 /* was a parameter; the module owns the one connection now */
4121 const dt_database_t *const db = _db;
4122 if(IS_NULL_PTR(db)) return FALSE;
4123 // backing up memory db is pointelss
4124 if(_is_mem_db())
4125 return FALSE;
4126 GDateTime *date_now = g_date_time_new_now_local();
4127 gchar *date_suffix = g_date_time_format(date_now, "%Y%m%d%H%M%S");
4128 g_date_time_unref(date_now);
4129
4130 const char *file_pattern = "%s-snp-%s";
4131 const char *temp_pattern = "%s-tmp-%s";
4132
4133 gchar *lib_backup_file = g_strdup_printf(file_pattern, db->dbfilename_library, date_suffix);
4134 gchar *lib_tmpbackup_file = g_strdup_printf(temp_pattern, db->dbfilename_library, date_suffix);
4135
4136 int rc = _backup_db(db->handle, "main", lib_tmpbackup_file, _print_backup_progress);
4137 if(!(rc==SQLITE_OK))
4138 {
4139 g_unlink(lib_tmpbackup_file);
4140 dt_free(lib_tmpbackup_file);
4141 dt_free(lib_backup_file);
4142 dt_free(date_suffix);
4143 return FALSE;
4144 }
4145 g_rename(lib_tmpbackup_file, lib_backup_file);
4146 g_chmod(lib_backup_file, S_IRUSR);
4147 dt_free(lib_tmpbackup_file);
4148 dt_free(lib_backup_file);
4149
4150 gchar *dat_backup_file = g_strdup_printf(file_pattern, db->dbfilename_data, date_suffix);
4151 gchar *dat_tmpbackup_file = g_strdup_printf(temp_pattern, db->dbfilename_data, date_suffix);
4152
4153 dt_free(date_suffix);
4154
4155 rc = _backup_db(db->handle, "data", dat_tmpbackup_file, _print_backup_progress);
4156 if(!(rc==SQLITE_OK))
4157 {
4158 g_unlink(dat_tmpbackup_file);
4159 dt_free(dat_tmpbackup_file);
4160 dt_free(dat_backup_file);
4161 return FALSE;
4162 }
4163 g_rename(dat_tmpbackup_file, dat_backup_file);
4164 g_chmod(dat_backup_file, S_IRUSR);
4165 dt_free(dat_tmpbackup_file);
4166 dt_free(dat_backup_file);
4167
4168 return TRUE;
4169}
4170
4171/* Turn the "create_snapshot" policy into a snapshot interval.
4172 *
4173 * Returns TRUE when the policy answers the question outright -- "never", "on close", or
4174 * an unrecognised value -- with the answer in @p answer. Returns FALSE when the caller
4175 * has to go and look at what is on disk, with the required interval in @p interval.
4176 *
4177 * It exists so that the settings copy is taken and released in one small scope: the
4178 * function below has a dozen early returns and no business holding a couple of strings it
4179 * would have to free on each of them. */
4180static gboolean _snapshot_policy_decides(GTimeSpan *interval, gboolean *answer)
4181{
4182 dt_database_settings_t settings = { 0 };
4183 dt_database_get_settings(&settings);
4184 const char *const config = settings.create_snapshot;
4185
4186 gboolean decides = TRUE;
4187
4188 if(!g_strcmp0(config, "never"))
4189 {
4190 // early bail out on "never"
4191 _db_print("[db backup] please consider enabling database snapshots.\n");
4192 *answer = FALSE;
4193 }
4194 else if(!g_strcmp0(config, "on close"))
4195 {
4196 // early bail out on "on close"
4197 _db_print("[db backup] performing unconditional snapshot.\n");
4198 *answer = TRUE;
4199 }
4200 else if(!g_strcmp0(config, "once a day"))
4201 {
4202 *interval = G_TIME_SPAN_DAY;
4203 decides = FALSE;
4204 }
4205 else if(!g_strcmp0(config, "once a week"))
4206 {
4207 *interval = G_TIME_SPAN_DAY * 7;
4208 decides = FALSE;
4209 }
4210 else if(!g_strcmp0(config, "once a month"))
4211 {
4212 //average month ;)
4213 *interval = G_TIME_SPAN_DAY * 30;
4214 decides = FALSE;
4215 }
4216 else
4217 {
4218 // early bail out on "invalid value"
4219 _db_print("[db backup] invalid timespan requirement expecting never/on close/once a [day/week/month], got %s.\n", config);
4220 *answer = TRUE;
4221 }
4222
4223 dt_database_settings_free(&settings);
4224 return decides;
4225}
4226
4227gboolean dt_database_maybe_snapshot(void)
4228{
4229 /* was a parameter; the module owns the one connection now */
4230 const dt_database_t *const db = _db;
4231 if(IS_NULL_PTR(db)) return FALSE;
4232 if(_is_mem_db())
4233 return FALSE;
4234
4235 /* Resolve the policy into an interval and be done with the settings copy right here. */
4236 GTimeSpan span_from_last_snap_required = 0;
4237 gboolean decided_answer = FALSE;
4238 if(_snapshot_policy_decides(&span_from_last_snap_required, &decided_answer))
4239 return decided_answer;
4240
4241 //we're in trouble zone - we have to determine when was the last snapshot done (including version upgrade snapshot) :/
4242 //this could be easy if we wrote date of last successful backup to config, but that's not really an option since
4243 //backup may done as last db operation, way after config file is closed. Plus we might be mixing dates of backups for
4244 //various library.db
4245
4246 _db_print("[db backup] checking snapshots existence.\n");
4247 GFile *library = g_file_parse_name(db->dbfilename_library);
4248 GFile *parent = g_file_get_parent(library);
4249
4250 if(IS_NULL_PTR(parent))
4251 {
4252 _db_print("[db backup] couldn't get library parent!.\n");
4253 g_object_unref(library);
4254 return FALSE;
4255 }
4256
4257 GError *error = NULL;
4258 GFileEnumerator *library_dir_files = g_file_enumerate_children(parent, G_FILE_ATTRIBUTE_STANDARD_NAME "," G_FILE_ATTRIBUTE_TIME_MODIFIED, G_FILE_QUERY_INFO_NONE, NULL, &error);
4259
4260 if(IS_NULL_PTR(library_dir_files))
4261 {
4262 _db_print("[db backup] couldn't enumerate library parent: %s.\n", error->message);
4263 g_object_unref(parent);
4264 g_object_unref(library);
4265 g_error_free(error);
4266 return FALSE;
4267 }
4268
4269 gchar *lib_basename = g_file_get_basename(library);
4270 g_object_unref(library);
4271
4272 gchar *lib_snap_format = g_strdup_printf("%s-snp-", lib_basename);
4273 gchar *lib_backup_format = g_strdup_printf("%s-pre-", lib_basename);
4274 dt_free(lib_basename);
4275
4276 GFileInfo *info = NULL;
4277 guint64 last_snap = 0;
4278
4279 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4280 {
4281 const char* fname = g_file_info_get_name(info);
4282 if(g_str_has_prefix(fname, lib_snap_format) || g_str_has_prefix(fname, lib_backup_format))
4283 {
4284 _db_print("[db backup] found file: %s.\n", fname);
4285 if(last_snap == 0)
4286 {
4287 last_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4288 g_object_unref(info);
4289 continue;
4290 }
4291 const guint64 try_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4292 if(try_snap > last_snap)
4293 {
4294 last_snap = try_snap;
4295 }
4296 }
4297 g_object_unref(info);
4298 }
4299 g_object_unref(parent);
4300 dt_free(lib_snap_format);
4301 dt_free(lib_backup_format);
4302
4303 if(error)
4304 {
4305 _db_print("[db backup] problem enumerating library parent: %s.\n", error->message);
4306 g_file_enumerator_close(library_dir_files, NULL, NULL);
4307 g_object_unref(library_dir_files);
4308 g_error_free(error);
4309 return FALSE;
4310 }
4311
4312 g_file_enumerator_close(library_dir_files, NULL, NULL);
4313 g_object_unref(library_dir_files);
4314
4315 GDateTime *date_now = g_date_time_new_now_local();
4316
4317 // Even if last_snap is 0 (didn't found any snaps) it produces proper date - unix epoch
4318 GDateTime *date_last_snap = g_date_time_new_from_unix_local(last_snap);
4319
4320 gchar *now_txt = g_date_time_format(date_now, "%Y%m%d%H%M%S");
4321 gchar *ls_txt = g_date_time_format(date_last_snap, "%Y%m%d%H%M%S");
4322 _db_print("[db backup] last snap: %s; curr date: %s.\n", ls_txt, now_txt);
4323 dt_free(now_txt);
4324 dt_free(ls_txt);
4325
4326 GTimeSpan span_from_last_snap = g_date_time_difference(date_now, date_last_snap);
4327
4328 g_date_time_unref(date_now);
4329 g_date_time_unref(date_last_snap);
4330
4331 return span_from_last_snap > span_from_last_snap_required;
4332}
4333
4334/* Parse integers in the form d (week days), dd (hours etc), ddd (ordinal days) or dddd (years) */
4335static gboolean _get_iso8601_int (const gchar *text, gsize length, gint *value)
4336{
4337 gsize i;
4338 guint v = 0;
4339
4340 if (length < 1 || length > 4)
4341 return FALSE;
4342
4343 for (i = 0; i < length; i++)
4344 {
4345 const gchar c = text[i];
4346 if (c < '0' || c > '9')
4347 return FALSE;
4348 v = v * 10 + (c - '0');
4349 }
4350
4351 *value = v;
4352 return TRUE;
4353}
4354
4355static gint _db_snap_sort(gconstpointer a, gconstpointer b, gpointer user_data)
4356{
4357 const gchar* e1 = (gchar *) a;
4358 const gchar* e2 = (gchar *) b;
4359
4360 //we assume that both end with date in
4361 //"%Y%m%d%H%M%S" format
4362
4363 gchar *datepos1 = g_strrstr(e1, "-snp-");
4364 gchar *datepos2 = g_strrstr(e2, "-snp-");
4365 if(IS_NULL_PTR(datepos1) || IS_NULL_PTR(datepos2))
4366 return 0;
4367
4368 datepos1 +=5; //skip "-snp-"
4369 datepos2 +=5; //skip "-snp-"
4370
4371 int year,month,day,hour,minute,second;
4372
4373 if(!_get_iso8601_int(datepos1, 4, &year) ||
4374 !_get_iso8601_int(datepos1+4, 2, &month) ||
4375 !_get_iso8601_int(datepos1+6, 2, &day) ||
4376 !_get_iso8601_int(datepos1+8, 2, &hour) ||
4377 !_get_iso8601_int(datepos1+10, 2, &minute) ||
4378 !_get_iso8601_int(datepos1+12, 2, &second)
4379 )
4380 {
4381 return 0;
4382 }
4383
4384 GDateTime *d1=g_date_time_new_local(year, month, day, hour, minute, second);
4385
4386 if(!_get_iso8601_int(datepos2, 4, &year) ||
4387 !_get_iso8601_int(datepos2+4, 2, &month) ||
4388 !_get_iso8601_int(datepos2+6, 2, &day) ||
4389 !_get_iso8601_int(datepos2+8, 2, &hour) ||
4390 !_get_iso8601_int(datepos2+10, 2, &minute) ||
4391 !_get_iso8601_int(datepos2+12, 2, &second)
4392 )
4393 {
4394 g_date_time_unref(d1);
4395 return 0;
4396 }
4397
4398 GDateTime *d2=g_date_time_new_local(year, month, day, hour, minute, second);
4399
4400 const gint ret = g_date_time_compare(d1, d2);
4401
4402 g_date_time_unref(d1);
4403 g_date_time_unref(d2);
4404
4405 return ret;
4406}
4407
4408char **dt_database_snaps_to_remove(void)
4409{
4410 /* was a parameter; the module owns the one connection now */
4411 const dt_database_t *const db = _db;
4412 if(IS_NULL_PTR(db)) return NULL;
4413 if(_is_mem_db())
4414 return NULL;
4415
4416 dt_database_settings_t settings = { 0 };
4417 dt_database_get_settings(&settings);
4418 const int keep_snaps = settings.keep_snapshots;
4419 dt_database_settings_free(&settings);
4420
4421 if(keep_snaps < 0)
4422 return NULL;
4423
4424 _db_print("[db backup] checking snapshots existence.\n");
4425 GFile *lib_file = g_file_parse_name(db->dbfilename_library);
4426 GFile *lib_parent = g_file_get_parent(lib_file);
4427
4428 if(IS_NULL_PTR(lib_parent))
4429 {
4430 _db_print("[db backup] couldn't get library parent!.\n");
4431 g_object_unref(lib_file);
4432 return NULL;
4433 }
4434
4435 GFile *dat_file = g_file_parse_name(db->dbfilename_data);
4436 GFile *dat_parent = g_file_get_parent(dat_file);
4437
4438 if(IS_NULL_PTR(dat_parent))
4439 {
4440 _db_print("[db backup] couldn't get data parent!.\n");
4441 g_object_unref(dat_file);
4442 g_object_unref(lib_file);
4443 g_object_unref(lib_parent);
4444 }
4445
4446 gchar *lib_basename = g_file_get_basename(lib_file);
4447 g_object_unref(lib_file);
4448 gchar *lib_snap_format = g_strdup_printf("%s-snp-", lib_basename);
4449 gchar *lib_tmp_format = g_strdup_printf("%s-tmp-", lib_basename);
4450 dt_free(lib_basename);
4451
4452 gchar *dat_basename = g_file_get_basename(dat_file);
4453 g_object_unref(dat_file);
4454 gchar *dat_snap_format = g_strdup_printf("%s-snp-", dat_basename);
4455 gchar *dat_tmp_format = g_strdup_printf("%s-tmp-", dat_basename);
4456 dt_free(dat_basename);
4457
4458 GQueue *lib_snaps = g_queue_new();
4459 GQueue *dat_snaps = g_queue_new();
4460 GQueue *tmplib_snaps = g_queue_new();
4461 GQueue *tmpdat_snaps = g_queue_new();
4462
4463 if(g_file_equal(lib_parent, dat_parent))
4464 {
4465 //slight optimization if library and data are in same dir, we only have to scan one
4466 GError *error = NULL;
4467 GFileEnumerator *library_dir_files = g_file_enumerate_children(lib_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4468
4469 if(IS_NULL_PTR(library_dir_files))
4470 {
4471 _db_print("[db backup] couldn't enumerate library parent: %s.\n", error->message);
4472 g_object_unref(lib_parent);
4473 g_object_unref(dat_parent);
4474 dt_free(lib_snap_format);
4475 dt_free(dat_snap_format);
4476 dt_free(lib_tmp_format);
4477 dt_free(dat_tmp_format);
4478 g_queue_free(lib_snaps);
4479 g_queue_free(dat_snaps);
4480 g_queue_free(tmplib_snaps);
4481 g_queue_free(tmpdat_snaps);
4482 g_error_free(error);
4483 return NULL;
4484 }
4485
4486 GFileInfo *info = NULL;
4487
4488 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4489 {
4490 const char* fname = g_file_info_get_name(info);
4491 if(g_str_has_prefix(fname, lib_snap_format))
4492 {
4493 _db_print("[db backup] found file: %s.\n", fname);
4494 g_queue_insert_sorted(lib_snaps, g_strdup(fname), _db_snap_sort, NULL);
4495 }
4496 else if(g_str_has_prefix(fname, dat_snap_format))
4497 {
4498 _db_print("[db backup] found file: %s.\n", fname);
4499 g_queue_insert_sorted(dat_snaps, g_strdup(fname), _db_snap_sort, NULL);
4500 }
4501 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4502 {
4503 //we insert into single queue, since it's just dependent on parent
4504 g_queue_push_head(tmplib_snaps, g_strdup(fname));
4505 }
4506 g_object_unref(info);
4507 }
4508 dt_free(lib_snap_format);
4509 dt_free(dat_snap_format);
4510
4511 if(error)
4512 {
4513 _db_print("[db backup] problem enumerating library parent: %s.\n", error->message);
4514 g_object_unref(lib_parent);
4515 g_object_unref(dat_parent);
4516 dt_free(lib_tmp_format);
4517 dt_free(dat_tmp_format);
4518 g_queue_free_full(lib_snaps, g_free);
4519 g_queue_free_full(dat_snaps, g_free);
4520 g_queue_free_full(tmplib_snaps, g_free);
4521 g_queue_free_full(tmpdat_snaps, g_free);
4522 g_file_enumerator_close(library_dir_files, NULL, NULL);
4523 g_object_unref(library_dir_files);
4524 g_error_free(error);
4525 return NULL;
4526 }
4527 g_file_enumerator_close(library_dir_files, NULL, NULL);
4528 g_object_unref(library_dir_files);
4529 }
4530 else
4531 {
4532 //well... fun.
4533
4534 GError *error = NULL;
4535 GFileEnumerator *library_dir_files = g_file_enumerate_children(lib_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4536 if(IS_NULL_PTR(library_dir_files))
4537 {
4538 _db_print("[db backup] couldn't enumerate library parent: %s.\n", error->message);
4539 g_object_unref(lib_parent);
4540 g_object_unref(dat_parent);
4541 dt_free(lib_snap_format);
4542 dt_free(dat_snap_format);
4543 dt_free(lib_tmp_format);
4544 dt_free(dat_tmp_format);
4545 g_error_free(error);
4546 g_queue_free(lib_snaps);
4547 g_queue_free(dat_snaps);
4548 g_queue_free(tmplib_snaps);
4549 g_queue_free(tmpdat_snaps);
4550 return NULL;
4551 }
4552
4553 GFileEnumerator *data_dir_files = g_file_enumerate_children(dat_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4554 if(IS_NULL_PTR(data_dir_files))
4555 {
4556 _db_print("[db backup] couldn't enumerate data parent: %s.\n", error->message);
4557 g_object_unref(lib_parent);
4558 g_object_unref(dat_parent);
4559 dt_free(lib_snap_format);
4560 dt_free(dat_snap_format);
4561 dt_free(lib_tmp_format);
4562 dt_free(dat_tmp_format);
4563 g_file_enumerator_close(library_dir_files, NULL, NULL);
4564 g_object_unref(library_dir_files);
4565 g_error_free(error);
4566 g_queue_free(lib_snaps);
4567 g_queue_free(dat_snaps);
4568 g_queue_free(tmplib_snaps);
4569 g_queue_free(tmpdat_snaps);
4570 return NULL;
4571 }
4572
4573 GFileInfo *info = NULL;
4574
4575 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4576 {
4577 const char* fname = g_file_info_get_name(info);
4578 if(g_str_has_prefix(fname, lib_snap_format))
4579 {
4580 _db_print("[db backup] found file: %s.\n", fname);
4581 g_queue_insert_sorted(lib_snaps, g_strdup(fname), _db_snap_sort, NULL);
4582 }
4583 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4584 {
4585 // we remove all incomplete snaps matching pattern in BOTH dirs
4586 g_queue_push_head(tmplib_snaps, g_strdup(fname));
4587 }
4588 g_object_unref(info);
4589 }
4590 dt_free(lib_snap_format);
4591
4592 if(error)
4593 {
4594 _db_print("[db backup] problem enumerating library parent: %s.\n", error->message);
4595 g_object_unref(lib_parent);
4596 g_object_unref(dat_parent);
4597 dt_free(lib_tmp_format);
4598 dt_free(dat_tmp_format);
4599 g_queue_free_full(lib_snaps, g_free);
4600 g_queue_free(dat_snaps);
4601 g_queue_free_full(tmplib_snaps, g_free);
4602 g_queue_free(tmpdat_snaps);
4603 g_file_enumerator_close(library_dir_files, NULL, NULL);
4604 g_object_unref(library_dir_files);
4605 g_file_enumerator_close(data_dir_files, NULL, NULL);
4606 g_object_unref(data_dir_files);
4607 g_error_free(error);
4608 return NULL;
4609 }
4610 g_file_enumerator_close(library_dir_files, NULL, NULL);
4611 g_object_unref(library_dir_files);
4612
4613 while ((info = g_file_enumerator_next_file(data_dir_files, NULL, &error)))
4614 {
4615 const char* fname = g_file_info_get_name(info);
4616 if(g_str_has_prefix(fname, dat_snap_format))
4617 {
4618 _db_print("[db backup] found file: %s.\n", fname);
4619 g_queue_insert_sorted(dat_snaps, g_strdup(fname), _db_snap_sort, NULL);
4620 }
4621 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4622 {
4623 //we add to queue both matches - it just depends on parent
4624 g_queue_push_head(tmpdat_snaps, g_strdup(fname));
4625 }
4626 g_object_unref(info);
4627 }
4628 dt_free(dat_snap_format);
4629 dt_free(lib_tmp_format);
4630 dt_free(dat_tmp_format);
4631
4632 if(error)
4633 {
4634 _db_print("[db backup] problem enumerating data parent: %s.\n", error->message);
4635 g_object_unref(lib_parent);
4636 g_object_unref(dat_parent);
4637 g_queue_free_full(lib_snaps, g_free);
4638 g_queue_free_full(dat_snaps, g_free);
4639 g_queue_free_full(tmplib_snaps, g_free);
4640 g_queue_free_full(tmpdat_snaps, g_free);
4641 g_file_enumerator_close(data_dir_files, NULL, NULL);
4642 g_object_unref(data_dir_files);
4643 g_error_free(error);
4644 return NULL;
4645 }
4646
4647 g_file_enumerator_close(data_dir_files, NULL, NULL);
4648 g_object_unref(data_dir_files);
4649 }
4650
4651 // here we have list of snaps sorted in date order, now we have to
4652 // create from that list of snaps to be deleted and return that :D
4653
4654 GPtrArray *ret = g_ptr_array_new();
4655
4656 gchar *lib_parent_path = g_file_get_path(lib_parent);
4657 g_object_unref(lib_parent);
4658
4659 while(g_queue_get_length(lib_snaps) > keep_snaps)
4660 {
4661 gchar *head = g_queue_pop_head(lib_snaps);
4662 g_ptr_array_add(ret, g_strconcat(lib_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4663 dt_free(head);
4664 }
4665 while(!g_queue_is_empty(tmplib_snaps))
4666 {
4667 gchar *head = g_queue_pop_head(tmplib_snaps);
4668 g_ptr_array_add(ret, g_strconcat(lib_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4669 dt_free(head);
4670 }
4671 dt_free(lib_parent_path);
4672 g_queue_free_full(lib_snaps, g_free);
4673 g_queue_free_full(tmplib_snaps, g_free); // should be totally freed, but eh - this won't make doublefree
4674
4675 gchar *dat_parent_path = g_file_get_path(dat_parent);
4676 g_object_unref(dat_parent);
4677
4678 while(g_queue_get_length(dat_snaps) > keep_snaps)
4679 {
4680 gchar *head = g_queue_pop_head(dat_snaps);
4681 g_ptr_array_add(ret, g_strconcat(dat_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4682 dt_free(head);
4683 }
4684 while(!g_queue_is_empty(tmpdat_snaps))
4685 {
4686 gchar *head = g_queue_pop_head(tmpdat_snaps);
4687 g_ptr_array_add(ret, g_strconcat(dat_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4688 dt_free(head);
4689 }
4690 dt_free(dat_parent_path);
4691 g_queue_free_full(dat_snaps, g_free);
4692 g_queue_free_full(tmpdat_snaps, g_free);
4693
4694 g_ptr_array_add (ret, NULL);
4695
4696 return (char**)g_ptr_array_free(ret, FALSE);
4697}
4698
4699gchar *dt_database_get_most_recent_snap(const char* db_filename)
4700{
4701 if(!g_strcmp0(db_filename, ":memory:"))
4702 return NULL;
4703
4704 _db_print("[db backup] checking snapshots existence.\n");
4705 GFile *db_file = g_file_parse_name(db_filename);
4706 GFile *parent = g_file_get_parent(db_file);
4707
4708 if(IS_NULL_PTR(parent))
4709 {
4710 _db_print("[db backup] couldn't get database parent!.\n");
4711 g_object_unref(db_file);
4712 return NULL;
4713 }
4714
4715 GError *error = NULL;
4716 GFileEnumerator *db_dir_files = g_file_enumerate_children(parent, G_FILE_ATTRIBUTE_STANDARD_NAME "," G_FILE_ATTRIBUTE_TIME_MODIFIED, G_FILE_QUERY_INFO_NONE, NULL, &error);
4717
4718 if(IS_NULL_PTR(db_dir_files))
4719 {
4720 _db_print("[db backup] couldn't enumerate database parent: %s.\n", error->message);
4721 g_object_unref(parent);
4722 g_object_unref(db_file);
4723 g_error_free(error);
4724 return NULL;
4725 }
4726
4727 gchar *db_basename = g_file_get_basename(db_file);
4728 g_object_unref(db_file);
4729
4730 gchar *db_snap_format = g_strdup_printf("%s-snp-", db_basename);
4731 gchar *db_backup_format = g_strdup_printf("%s-pre-", db_basename);
4732 dt_free(db_basename);
4733
4734 GFileInfo *info = NULL;
4735 guint64 last_snap = 0;
4736 gchar *last_snap_name = NULL;
4737
4738 while ((info = g_file_enumerator_next_file(db_dir_files, NULL, &error)))
4739 {
4740 const char* fname = g_file_info_get_name(info);
4741 if(g_str_has_prefix(fname, db_snap_format) || g_str_has_prefix(fname, db_backup_format))
4742 {
4743 _db_print("[db backup] found file: %s.\n", fname);
4744 if(last_snap == 0)
4745 {
4746 last_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4747 last_snap_name = g_strdup(fname);
4748 g_object_unref(info);
4749 continue;
4750 }
4751 guint64 try_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4752 if(try_snap > last_snap)
4753 {
4754 last_snap = try_snap;
4755 dt_free(last_snap_name);
4756 last_snap_name = g_strdup(fname);
4757 }
4758 }
4759 g_object_unref(info);
4760 }
4761 dt_free(db_snap_format);
4762 dt_free(db_backup_format);
4763
4764 if(error)
4765 {
4766 _db_print("[db backup] problem enumerating database parent: %s.\n", error->message);
4767 g_file_enumerator_close(db_dir_files, NULL, NULL);
4768 g_object_unref(db_dir_files);
4769 g_error_free(error);
4770 dt_free(last_snap_name);
4771 return NULL;
4772 }
4773
4774 g_file_enumerator_close(db_dir_files, NULL, NULL);
4775 g_object_unref(db_dir_files);
4776
4777 if(IS_NULL_PTR(last_snap_name))
4778 {
4779 g_object_unref(parent);
4780 return NULL;
4781 }
4782
4783 gchar *parent_path = g_file_get_path(parent);
4784 g_object_unref(parent);
4785
4786 gchar *ret = g_strconcat(parent_path, G_DIR_SEPARATOR_S, last_snap_name, NULL);
4787 dt_free(parent_path);
4788 dt_free(last_snap_name);
4789
4790 return ret;
4791}
4792
4793// Nested transactions support
4794//
4795// NOTE: the nested support is actually not activated (see || TRUE below). This current
4796// implementation is just a refactoring of the previous code using:
4797// - dt_database_start_transaction()
4798// - dt_database_release_transaction()
4799// - dt_database_rollback_transaction()
4800//
4801// With this refactoring we can count and check for nested transaction and unmatched
4802// transaction routines. And it has been done to help further implementation for
4803// proper threading and nested transaction support.
4804//
4805void dt_database_start_transaction_debug(void)
4806{
4807 /* was a parameter; the module owns the one connection now */
4808 const dt_database_t *const db = _db;
4809 gpointer const owner = g_thread_self();
4810 const int batch_level = dt_atomic_get_int(&_trx_batch_level);
4811 if(batch_level > 0)
4812 {
4813 // If a batch transaction is active on this thread, suppress nested BEGIN/COMMIT
4814 // to avoid per-module transaction overhead during presets initialization.
4815 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4816 {
4817 dt_atomic_add_int(&_trxid, 1);
4818 return;
4819 }
4820 }
4821
4822 if(g_atomic_pointer_get(&_trx_owner) == owner)
4823 {
4824 dt_atomic_add_int(&_trxid, 1);
4825 return;
4826 }
4827
4828 dt_pthread_rwlock_wrlock(&_db_lock);
4829 g_atomic_pointer_set(&_trx_owner, owner);
4830
4831 const int trxid = dt_atomic_add_int(&_trxid, 1);
4832
4833 // if top level a simple unamed transaction is used BEGIN / COMMIT / ROLLBACK
4834 // otherwise we use a savepoint (named transaction).
4835
4836 if(trxid == 0)
4837 {
4838 // In theads application it may be safer to use an IMMEDIATE transaction:
4839 // "BEGIN IMMEDIATE TRANSACTION"
4840 // This implies "BEGIN DEFERRED TRANSACTION", which means
4841 // no write event is dispatched to DB until the first "COMMIT"
4842 DT_DEBUG_SQLITE3_EXEC(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
4843 }
4844#ifdef USE_NESTED_TRANSACTIONS
4845 else
4846 {
4847 char SQLTRX[32] = { 0 };
4848 g_snprintf(SQLTRX, sizeof(SQLTRX), "SAVEPOINT trx%d", trxid);
4849 DT_DEBUG_SQLITE3_EXEC(db->handle, SQLTRX, NULL, NULL, NULL);
4850 }
4851#endif
4852
4853 if(trxid > MAX_NESTED_TRANSACTIONS)
4854 fprintf(stderr, "[dt_database_start_transaction] more than %d nested transaction\n", MAX_NESTED_TRANSACTIONS);
4855}
4856
4857void dt_database_release_transaction_debug(void)
4858{
4859 /* was a parameter; the module owns the one connection now */
4860 const dt_database_t *const db = _db;
4861 gpointer const owner = g_thread_self();
4862 const int batch_level = dt_atomic_get_int(&_trx_batch_level);
4863 if(batch_level > 0)
4864 {
4865 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4866 {
4867 dt_atomic_sub_int(&_trxid, 1);
4868 return;
4869 }
4870 }
4871
4872 if(g_atomic_pointer_get(&_trx_owner) != owner)
4873 {
4874 fprintf(stderr, "[dt_database_release_transaction] COMMIT from non-owner thread\n");
4875 return;
4876 }
4877
4878 const int trxid = dt_atomic_sub_int(&_trxid, 1);
4879
4880 if(trxid <= 0)
4881 fprintf(stderr, "[dt_database_release_transaction] COMMIT outside a transaction\n");
4882
4883 if(trxid == 1)
4884 {
4885 DT_DEBUG_SQLITE3_EXEC(db->handle, "COMMIT TRANSACTION", NULL, NULL, NULL);
4886 g_atomic_pointer_set(&_trx_owner, NULL);
4887 dt_pthread_rwlock_unlock(&_db_lock);
4888 }
4889#ifdef USE_NESTED_TRANSACTIONS
4890 else
4891 {
4892 char SQLTRX[64] = { 0 };
4893 g_snprintf(SQLTRX, sizeof(SQLTRX), "RELEASE SAVEPOINT trx%d", trxid - 1);
4894 DT_DEBUG_SQLITE3_EXEC(db->handle, SQLTRX, NULL, NULL, NULL);
4895 }
4896#endif
4897}
4898
4899void dt_database_rollback_transaction(void)
4900{
4901 /* was a parameter; the module owns the one connection now */
4902 const dt_database_t *const db = _db;
4903 gpointer const owner = g_thread_self();
4904 if(g_atomic_pointer_get(&_trx_owner) != owner && g_atomic_pointer_get(&_trx_batch_owner) != owner)
4905 {
4906 fprintf(stderr, "[dt_database_rollback_transaction] ROLLBACK from non-owner thread\n");
4907 return;
4908 }
4909
4910 const int trxid = dt_atomic_sub_int(&_trxid, 1);
4911
4912 if(trxid <= 0)
4913 fprintf(stderr, "[dt_database_rollback_transaction] ROLLBACK outside a transaction\n");
4914
4915 if(trxid >= 1)
4916 {
4917 DT_DEBUG_SQLITE3_EXEC(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL);
4918 dt_atomic_set_int(&_trxid, 0);
4919 dt_atomic_set_int(&_trx_batch_level, 0);
4920 g_atomic_pointer_set(&_trx_owner, NULL);
4921 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4922 dt_pthread_rwlock_unlock(&_db_lock);
4923 }
4924#ifdef USE_NESTED_TRANSACTIONS
4925 else
4926 {
4927 char SQLTRX[64] = { 0 };
4928 g_snprintf(SQLTRX, sizeof(SQLTRX), "ROLLBACK TRANSACTION TO SAVEPOINT trx%d", trxid - 1);
4929 DT_DEBUG_SQLITE3_EXEC(db->handle, SQLTRX, NULL, NULL, NULL);
4930 }
4931#endif
4932}
4933
4934void dt_database_begin_transaction_batch(void)
4935{
4936 /* was a parameter; the module owns the one connection now */
4937 const dt_database_t *const db = _db;
4938 gpointer const owner = g_thread_self();
4939 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4940 {
4941 dt_atomic_add_int(&_trx_batch_level, 1);
4942 return;
4943 }
4944
4945 dt_pthread_rwlock_wrlock(&_db_lock);
4946 g_atomic_pointer_set(&_trx_owner, owner);
4947 g_atomic_pointer_set(&_trx_batch_owner, owner);
4948
4949 const int level = dt_atomic_add_int(&_trx_batch_level, 1);
4950 if(level == 0)
4951 {
4952 DT_DEBUG_SQLITE3_EXEC(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
4953 }
4954}
4955
4956void dt_database_end_transaction_batch(void)
4957{
4958 /* was a parameter; the module owns the one connection now */
4959 const dt_database_t *const db = _db;
4960 gpointer const owner = g_thread_self();
4961 if(g_atomic_pointer_get(&_trx_batch_owner) != owner)
4962 {
4963 _db_print("[dt_database_end_transaction_batch] COMMIT from non-owner thread\n");
4964 return;
4965 }
4966
4967 const int level = dt_atomic_sub_int(&_trx_batch_level, 1);
4968 if(level <= 0)
4969 {
4970 _db_print("[dt_database_end_transaction_batch] COMMIT outside a batch transaction\n");
4971 dt_atomic_set_int(&_trx_batch_level, 0);
4972 g_atomic_pointer_set(&_trx_owner, NULL);
4973 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4974 dt_pthread_rwlock_unlock(&_db_lock);
4975 return;
4976 }
4977
4978 if(level == 1)
4979 {
4980 DT_DEBUG_SQLITE3_EXEC(db->handle, "COMMIT TRANSACTION", NULL, NULL, NULL);
4981 g_atomic_pointer_set(&_trx_owner, NULL);
4982 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4983 dt_pthread_rwlock_unlock(&_db_lock);
4984 }
4985}
4986
4987// clang-format off
4988// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
4989// vim: shiftwidth=2 expandtab tabstop=2 cindent
4990// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
4991// clang-format on
error
static void error(char *msg)
Definition ashift_lsd.c:202
TRUE
#define TRUE
Definition ashift_lsd.c:162
FALSE
#define FALSE
Definition ashift_lsd.c:158
dt_atomic_set_int
void dt_atomic_set_int(dt_atomic_int *var, int value)
dt_atomic_get_int
int dt_atomic_get_int(dt_atomic_int *var)
dt_atomic_sub_int
int dt_atomic_sub_int(dt_atomic_int *var, int decr)
dt_atomic_add_int
int dt_atomic_add_int(dt_atomic_int *var, int incr)
dt_atomic_int
atomic_int dt_atomic_int
Definition atomic.h:68
void
typedef void((*dt_cache_allocate_t)(void *userdata, dt_cache_entry_t *entry))
kind
static const dt_adaptation_t kind
Definition chromatic_adaptation.h:125
v
const float v
Definition colorprofiles/iop_profile.h:536
i
const float i
Definition colorspaces_inline_conversions.h:441
n
const float n
Definition colorspaces_inline_conversions.h:679
dt_image_flags_from_extension
dt_image_flags_t dt_image_flags_from_extension(const char *extension)
Definition common/image.c:1895
darktable_package_version
const char darktable_package_version[]
_db_lock
static dt_pthread_rwlock_t _db_lock
Definition database.c:141
_lock_single_database
static gboolean _lock_single_database(dt_database_t *db, const char *dbfilename, char **lockfile)
Definition database.c:2831
_print_backup_progress
static void _print_backup_progress(int remaining, int total)
Definition database.c:4062
dt_database_set_renamed_handler
void dt_database_set_renamed_handler(dt_database_renamed_handler_t handler)
Definition database.c:3627
dt_database_settings_free
void dt_database_settings_free(dt_database_settings_t *settings)
Definition database.c:3711
dt_database_backup
void dt_database_backup(const char *filename)
Definition database.c:2976
dt_database_optimize
void dt_database_optimize(void)
Definition database.c:4050
dt_database_start_transaction_debug
void dt_database_start_transaction_debug(void)
Definition database.c:4805
TRY_PREPARE
#define TRY_PREPARE(_stmt, _query, _message)
Definition database.c:523
_settings
static dt_database_settings_t _settings
Definition database.c:147
dt_database_get_most_recent_snap
gchar * dt_database_get_most_recent_snap(const char *db_filename)
Definition database.c:4699
_trx_batch_owner
static gpointer _trx_batch_owner
Definition database.c:98
pid_is_alive
static gboolean pid_is_alive(int pid)
Definition database.c:2784
dt_database_set_settings
void dt_database_set_settings(const dt_database_settings_t *settings)
Definition database.c:3666
MAX_NESTED_TRANSACTIONS
#define MAX_NESTED_TRANSACTIONS
Definition database.c:93
dt_database_begin_transaction_batch
void dt_database_begin_transaction_batch(void)
Definition database.c:4934
_is_mem_db
static gboolean _is_mem_db(void)
Definition database.c:3965
dt_database_release_transaction_debug
void dt_database_release_transaction_debug(void)
Definition database.c:4857
ask_for_upgrade
void ask_for_upgrade(const gchar *dbname, const gboolean has_gui)
Definition database.c:2957
_get_pragma_string_val
gchar * _get_pragma_string_val(sqlite3 *db, const char *pragma)
Definition database.c:3036
_ask_readonly
static dt_database_response_t _ask_readonly(const char *dbfilename)
Definition database.c:3095
_trxid
static dt_atomic_int _trxid
Definition database.c:95
_ask_user
static dt_database_response_t _ask_user(const dt_database_prompt_context_t *context)
Definition database.c:3070
dt_database_snaps_to_remove
char ** dt_database_snaps_to_remove(void)
Definition database.c:4408
_settings_lock
static dt_pthread_mutex_t _settings_lock
Definition database.c:148
dt_database_take_error
void dt_database_take_error(dt_database_error_t *error)
Definition database.c:2734
_verbose
static gboolean _verbose
Definition database.c:153
dt_database_error_free
void dt_database_error_free(dt_database_error_t *error)
Definition database.c:2751
dt_database_get_settings
void dt_database_get_settings(dt_database_settings_t *settings)
Definition database.c:3685
_trx_owner
static gpointer _trx_owner
Definition database.c:97
_sanitize_db
static void _sanitize_db(dt_database_t *db)
Definition database.c:2644
dt_database_get_sqlite3_global
sqlite3 * dt_database_get_sqlite3_global(void)
Definition database.c:3782
dt_database_get_last_error
const char * dt_database_get_last_error(void)
Definition database.c:3776
dt_database_set_prompt_handler
void dt_database_set_prompt_handler(dt_database_prompt_handler_t handler)
Definition database.c:3062
FINALIZE
#define FINALIZE
Definition database.c:537
dt_database_close
void dt_database_close(void)
Definition database.c:3749
ERRCHECK
#define ERRCHECK
Definition database.c:3882
_migrate_schema
static gboolean _migrate_schema(dt_database_t *db, int version)
Definition database.c:186
_ask_corrupted
static dt_database_response_t _ask_corrupted(const char *dbfilename, const char *quick_check, const gboolean snapshot_available)
Definition database.c:3085
_database_delete_mipmaps_files
static void _database_delete_mipmaps_files()
Definition database.c:3840
_upgrade_library_schema_step
static int _upgrade_library_schema_step(dt_database_t *db, int version)
Definition database.c:540
TRY_EXEC
#define TRY_EXEC(_query, _message)
Definition database.c:497
dt_database_snapshot
gboolean dt_database_snapshot(void)
Definition database.c:4118
dt_database_get_path
const gchar * dt_database_get_path(void)
Definition database.c:3787
_has_gui
static gboolean _has_gui
Definition database.c:152
_ask_for_maintenance
static gboolean _ask_for_maintenance(const char *maintenance_check, const gboolean closing_time, const guint64 size)
Definition database.c:3946
_database_init
static dt_database_t * _database_init(const dt_database_params_t *params)
Definition database.c:3109
_upgrade_data_schema_step
static int _upgrade_data_schema_step(dt_database_t *db, int version)
Definition database.c:2165
dt_database_perform_maintenance
void dt_database_perform_maintenance(void)
Definition database.c:3883
dt_database_cleanup_busy_statements
void dt_database_cleanup_busy_statements(void)
Definition database.c:3861
_SQLITE3_EXEC
#define _SQLITE3_EXEC(a, b, c, d, e)
Definition database.c:177
dt_database_rollback_transaction
void dt_database_rollback_transaction(void)
Definition database.c:4899
_db_lock_inited
static gboolean _db_lock_inited
Definition database.c:142
_get_pragma_int_val
int _get_pragma_int_val(sqlite3 *db, const char *pragma)
Definition database.c:3020
_create_library_schema
static void _create_library_schema(dt_database_t *db)
Definition database.c:2413
_snapshot_policy_decides
static gboolean _snapshot_policy_decides(GTimeSpan *interval, gboolean *answer)
Definition database.c:4180
_upgrade_library_schema
static gboolean _upgrade_library_schema(dt_database_t *db, int version)
Definition database.c:2384
_backup_db
static int _backup_db(sqlite3 *src_db, const char *src_db_name, const char *dest_filename, void(*xProgress)(int, int))
Definition database.c:4068
_trx_batch_level
static dt_atomic_int _trx_batch_level
Definition database.c:96
_database_free
static void _database_free(dt_database_t *db)
Definition database.c:3727
_db_print
#define _db_print(...)
Definition database.c:161
_db
static dt_database_t * _db
Definition database.c:123
_settings_lock_inited
static gboolean _settings_lock_inited
Definition database.c:149
CURRENT_DATABASE_VERSION_LIBRARY
#define CURRENT_DATABASE_VERSION_LIBRARY
Definition database.c:89
_renamed_handler
static dt_database_renamed_handler_t _renamed_handler
Definition database.c:156
_upgrade_data_schema
static gboolean _upgrade_data_schema(dt_database_t *db, int version)
Definition database.c:2399
_create_memory_schema
static void _create_memory_schema(dt_database_t *db)
Definition database.c:2572
TRY_STEP
#define TRY_STEP(_stmt, _expected, _message)
Definition database.c:510
dt_database_maybe_maintenance
gboolean dt_database_maybe_maintenance(const gboolean closing_time)
Definition database.c:3970
dt_database_is_open
gboolean dt_database_is_open(void)
Definition database.c:3771
dt_database_open
dt_database_open_result_t dt_database_open(const dt_database_params_t *params)
Definition database.c:3632
_database_migrate_to_xdg_structure
static gchar * _database_migrate_to_xdg_structure(const char *configured_db)
Definition database.c:3797
dt_database_maybe_snapshot
gboolean dt_database_maybe_snapshot(void)
Definition database.c:4227
_ask_upgrade
static dt_database_response_t _ask_upgrade(const char *dbfilename)
Definition database.c:3102
dt_database_delete_lock_files
int dt_database_delete_lock_files(const char *dbfilename)
Definition database.c:2760
_lock_databases
static gboolean _lock_databases(dt_database_t *db)
Definition database.c:2935
dt_database_end_transaction_batch
void dt_database_end_transaction_batch(void)
Definition database.c:4956
_get_iso8601_int
static gboolean _get_iso8601_int(const gchar *text, gsize length, gint *value)
Definition database.c:4335
_create_data_schema
static void _create_data_schema(dt_database_t *db)
Definition database.c:2519
_prompt_handler
static dt_database_prompt_handler_t _prompt_handler
Definition database.c:3060
_db_snap_sort
static gint _db_snap_sort(gconstpointer a, gconstpointer b, gpointer user_data)
Definition database.c:4355
CURRENT_DATABASE_VERSION_DATA
#define CURRENT_DATABASE_VERSION_DATA
Definition database.c:90
dt_database_prompt_handler_t
dt_database_response_t(* dt_database_prompt_handler_t)(const dt_database_prompt_context_t *context)
Definition database.h:245
dt_database_renamed_handler_t
void(* dt_database_renamed_handler_t)(const char *new_library_name)
Definition database.h:158
dt_database_response_t
dt_database_response_t
Definition database.h:209
DT_DATABASE_RESPONSE_CLOSE
@ DT_DATABASE_RESPONSE_CLOSE
Definition database.h:212
DT_DATABASE_RESPONSE_PROCEED
@ DT_DATABASE_RESPONSE_PROCEED
Definition database.h:218
DT_DATABASE_RESPONSE_RESTORE
@ DT_DATABASE_RESPONSE_RESTORE
Definition database.h:214
DT_DATABASE_RESPONSE_DELETE
@ DT_DATABASE_RESPONSE_DELETE
Definition database.h:216
dt_database_open_result_t
dt_database_open_result_t
Definition database.h:74
DT_DATABASE_OPEN_FAILED
@ DT_DATABASE_OPEN_FAILED
Definition database.h:76
DT_DATABASE_OPEN_LOCKED
@ DT_DATABASE_OPEN_LOCKED
Definition database.h:82
DT_DATABASE_OPEN_OK
@ DT_DATABASE_OPEN_OK
Definition database.h:78
DT_DATABASE_PROMPT_UPGRADE
@ DT_DATABASE_PROMPT_UPGRADE
Definition database.h:202
DT_DATABASE_PROMPT_MAINTENANCE
@ DT_DATABASE_PROMPT_MAINTENANCE
Definition database.h:205
DT_DATABASE_PROMPT_CORRUPTED
@ DT_DATABASE_PROMPT_CORRUPTED
Definition database.h:199
DT_DATABASE_PROMPT_READONLY
@ DT_DATABASE_PROMPT_READONLY
Definition database.h:197
dt_datetime_exif_to_gdatetime
GDateTime * dt_datetime_exif_to_gdatetime(const char *exif, const GTimeZone *tz)
Definition datetime.c:260
dt_datetime_utc_tz
GTimeZone * dt_datetime_utc_tz(void)
Definition datetime.c:40
dt_datetime_gdatetime_to_gtimespan
GTimeSpan dt_datetime_gdatetime_to_gtimespan(GDateTime *gdt)
Definition datetime.c:459
names
GHashTable * names
GtkWidget* -> the name the application knows it by.
Definition doc_screenshot.c:112
status
GtkWidget * status
result of the last capture
Definition doc_screenshot.c:109
dt_pthread_rwlock_wrlock
static int dt_pthread_rwlock_wrlock(dt_pthread_rwlock_t *rwlock) ACQUIRE(rwlock) NO_THREAD_SAFETY_ANALYSIS
Definition dtpthread.h:299
dt_pthread_rwlock_unlock
static int dt_pthread_rwlock_unlock(dt_pthread_rwlock_t *rwlock) RELEASE_GENERIC(rwlock) NO_THREAD_SAFETY_ANALYSIS
Definition dtpthread.h:217
dt_pthread_mutex_unlock
static int dt_pthread_mutex_unlock(dt_pthread_mutex_t *mutex) RELEASE(mutex) NO_THREAD_SAFETY_ANALYSIS
Definition dtpthread.h:127
dt_pthread_mutex_init
static int dt_pthread_mutex_init(dt_pthread_mutex_t *mutex, const pthread_mutexattr_t *mutexattr)
Initialise a mutex. With mutexattr NULL – which is how 54 of the 56 call sites in this tree spell it ...
Definition dtpthread.h:104
dt_pthread_rwlock_init
static int dt_pthread_rwlock_init(dt_pthread_rwlock_t *lock, const pthread_rwlockattr_t *attr)
Definition dtpthread.h:192
dt_pthread_mutex_lock
static int dt_pthread_mutex_lock(dt_pthread_mutex_t *mutex) ACQUIRE(mutex) NO_THREAD_SAFETY_ANALYSIS
Definition dtpthread.h:117
dt_loc_get_user_cache_dir
void dt_loc_get_user_cache_dir(char *cachedir, size_t bufsize)
Definition file_location.c:358
dt_loc_get_datadir
void dt_loc_get_datadir(char *datadir, size_t bufsize)
Definition file_location.c:366
dt_loc_get_user_config_dir
void dt_loc_get_user_config_dir(char *configdir, size_t bufsize)
Definition file_location.c:354
file_location.h
UNKNOWN_IMAGE
#define UNKNOWN_IMAGE
Definition image.h:78
dt_image_flags_t
dt_image_flags_t
Definition image.h:104
DT_IMAGE_RAW
@ DT_IMAGE_RAW
Definition image.h:124
DT_IMAGE_HDR
@ DT_IMAGE_HDR
Definition image.h:126
DT_IMAGE_LDR
@ DT_IMAGE_LDR
Definition image.h:122
dt_ioppr_get_iop_order_list_version
GList * dt_ioppr_get_iop_order_list_version(dt_iop_order_t version)
Return the built-in order list for a given version.
Definition iop_order.c:1066
dt_sort_iop_list_by_order_f
gint dt_sort_iop_list_by_order_f(gconstpointer a, gconstpointer b)
Compare two list nodes holding modules by iop_order.
Definition iop_order.c:939
dt_ioppr_get_iop_order_list_kind
dt_iop_order_t dt_ioppr_get_iop_order_list_kind(GList *iop_order_list)
Determine the kind of an order list by inspecting its content.
Definition iop_order.c:951
dt_ioppr_serialize_text_iop_order_list
char * dt_ioppr_serialize_text_iop_order_list(GList *iop_order_list)
Serialize an order list to a text representation.
Definition iop_order.c:2507
iop_order.h
dt_iop_order_t
dt_iop_order_t
Definition iop_order.h:149
DT_IOP_ORDER_ANSEL_RAW
@ DT_IOP_ORDER_ANSEL_RAW
Definition iop_order.h:154
DT_IOP_ORDER_LEGACY
@ DT_IOP_ORDER_LEGACY
Definition iop_order.h:151
DT_IOP_ORDER_CUSTOM
@ DT_IOP_ORDER_CUSTOM
Definition iop_order.h:150
dt_legacy_presets_create
void dt_legacy_presets_create(sqlite3 *handle)
Definition legacy_presets.c:1134
legacy_presets.h
k
float *const restrict const size_t k
Definition luminance_mask.h:81
IS_NULL_PTR
#define IS_NULL_PTR(p)
C is way too permissive with !=, == and if(var) checks, which can mean too many things depending on w...
Definition macros.h:96
dt_free_gpointer
static void dt_free_gpointer(gpointer ptr)
g_free() one pointer, with the signature GDestroyNotify wants.
Definition mem_alloc.h:184
dt_free
#define dt_free(ptr)
g_free() ptr and set it to NULL, skipping both if it is already NULL.
Definition mem_alloc.h:171
size
size_t size
Definition mipmap_cache.c:3
flags
dt_mipmap_buffer_dsc_flags flags
Definition mipmap_cache.c:4
DT_PATH_MAX
#define DT_PATH_MAX
Buffer size for a filesystem path anywhere in Ansel.
Definition paths.h:57
dt_concat_path_file
void dt_concat_path_file(char destination[4096], const char path[4096], const char *const file)
Append a constant filename to a variable, stack-based, fixed-sized, directory, and add a / in-between...
Definition darktable.c:2674
name
const char * name
Definition pdf.h:90
value
static const dt_aligned_pixel_simd_t value
Definition simd.h:144
sql_debug.h
Checked wrappers around the sqlite3_* calls: trace the statement, assert the return code,...
DT_DEBUG_SQLITE3_EXEC
#define DT_DEBUG_SQLITE3_EXEC(a, b, c, d, e)
Definition sql_debug.h:129
sqlite3IcuInit
int sqlite3IcuInit(sqlite3 *db)
Definition sqliteicu.c:523
day
static const char *const day[7]
Definition strptime.c:97
dt_database_error_t
Definition database.h:167
dt_database_params_t
Definition database.h:58
dt_database_prompt_context_t
Definition database.h:224
dt_database_prompt_context_t::prompt
dt_database_prompt_t prompt
Definition database.h:225
dt_database_prompt_context_t::dbfilename
const char * dbfilename
Definition database.h:227
dt_database_settings_t
Definition database.h:132
dt_database_settings_t::create_snapshot
char * create_snapshot
Definition database.h:138
dt_database_settings_t::maintenance_check
char * maintenance_check
Definition database.h:134
dt_database_settings_t::keep_snapshots
int keep_snapshots
Definition database.h:140
dt_database_settings_t::maintenance_freepage_ratio
int maintenance_freepage_ratio
Definition database.h:136
dt_database_t
Definition database.c:101
dt_database_t::error_other_pid
int error_other_pid
Definition database.c:114
dt_database_t::lockfile_data
gchar * lockfile_data
Definition database.c:105
dt_database_t::dbfilename_library
gchar * dbfilename_library
Definition database.c:108
dt_database_t::handle
sqlite3 * handle
Definition database.c:111
dt_database_t::lockfile_library
gchar * lockfile_library
Definition database.c:108
dt_database_t::error_message
gchar * error_message
Definition database.c:113
dt_database_t::lock_acquired
gboolean lock_acquired
Definition database.c:102
dt_database_t::dbfilename_data
gchar * dbfilename_data
Definition database.c:105
dt_database_t::error_dbfilename
gchar * error_dbfilename
Definition database.c:113
dt_iop_order_entry_t
Definition iop_order.h:160
dt_iop_order_entry_t::operation
char operation[20]
Definition iop_order.h:166
dt_iop_order_entry_t::iop_order_f
double iop_order_f
Definition iop_order.h:162
dt_iop_order_entry_t::o
union dt_iop_order_entry_t::@26 o
dt_iop_order_entry_t::instance
int32_t instance
Definition iop_order.h:167
MIN
#define MIN(a, b)
Definition thinplate.c:32
MAX
#define MAX(a, b)
Definition thinplate.c:29
dt_util_foo_to_utf8
gchar * dt_util_foo_to_utf8(const char *string)
Definition utility.c:396