Ansel 0.0
A darktable fork - bloat + design vision
Loading...
Searching...
No Matches
database.c
Go to the documentation of this file.
1/*
2 This file is part of darktable,
3 Copyright (C) 2011-2012 Edouard Gomez.
4 Copyright (C) 2011 Henrik Andersson.
5 Copyright (C) 2011-2012 johannes hanika.
6 Copyright (C) 2011 Moritz Lipp.
7 Copyright (C) 2012, 2017 Christian Tellefsen.
8 Copyright (C) 2012 José Carlos García Sogo.
9 Copyright (C) 2012 Jérémy Rosen.
10 Copyright (C) 2012 Richard Wonka.
11 Copyright (C) 2012 Simon Spannagel.
12 Copyright (C) 2012-2020 Tobias Ellinghaus.
13 Copyright (C) 2013-2014, 2018-2022 Pascal Obry.
14 Copyright (C) 2014-2016 Roman Lebedev.
15 Copyright (C) 2015 Pedro Côrte-Real.
16 Copyright (C) 2016 piterdias.
17 Copyright (C) 2017, 2019, 2021 luzpaz.
18 Copyright (C) 2017-2018 Rikard Öxler.
19 Copyright (C) 2018-2019 Edgardo Hoszowski.
20 Copyright (C) 2018 Mario Lueder.
21 Copyright (C) 2018 Peter Budai.
22 Copyright (C) 2019 jakubfi.
23 Copyright (C) 2019-2022 Philippe Weyland.
24 Copyright (C) 2020 Heiko Bauke.
25 Copyright (C) 2020-2021 Hubert Kowalski.
26 Copyright (C) 2020 JP Verrue.
27 Copyright (C) 2020 jpverrue.
28 Copyright (C) 2021 Diederik Ter Rahe.
29 Copyright (C) 2021 Hanno Schwalm.
30 Copyright (C) 2021 HansBull.
31 Copyright (C) 2021 Harald.
32 Copyright (C) 2021 Ralf Brown.
33 Copyright (C) 2022 Aldric Renaudin.
34 Copyright (C) 2022-2023, 2025-2026 Aurélien PIERRE.
35 Copyright (C) 2022 Martin Bařinka.
36 Copyright (C) 2022 Miloš Komarčević.
37 Copyright (C) 2023 Maurizio Paglia.
38 Copyright (C) 2025 Alynx Zhou.
39 Copyright (C) 2025-2026 Guillaume Stutin.
40
41 darktable is free software: you can redistribute it and/or modify
42 it under the terms of the GNU General Public License as published by
43 the Free Software Foundation, either version 3 of the License, or
44 (at your option) any later version.
45
46 darktable is distributed in the hope that it will be useful,
47 but WITHOUT ANY WARRANTY; without even the implied warranty of
48 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
49 GNU General Public License for more details.
50
51 You should have received a copy of the GNU General Public License
52 along with darktable. If not, see <http://www.gnu.org/licenses/>.
53*/
54
55#ifdef HAVE_CONFIG_H
56#include "config.h"
57#endif
58
59#include "common/atomic.h"
60#include "common/database.h"
61#include "common/darktable.h"
62#include "common/datetime.h"
63#include "common/debug.h"
64#include "common/file_location.h"
65#include "common/iop_order.h"
66#include "common/styles.h"
67#include "common/history.h"
68#ifdef HAVE_ICU
69#include "common/sqliteicu.h"
70#endif
71#include "control/conf.h"
72#include "control/control.h"
73#include "gui/legacy_presets.h"
74
75#include <gio/gio.h>
76#include <glib.h>
77#include <glib/gstdio.h>
78
79#include <errno.h>
80#include <fcntl.h>
81#include <signal.h>
82#include <sys/stat.h>
83#include <sys/types.h>
84
85// whenever _create_*_schema() gets changed you HAVE to bump this version and add an update path to
86// _upgrade_*_schema_step()!
87#define CURRENT_DATABASE_VERSION_LIBRARY 36
88#define CURRENT_DATABASE_VERSION_DATA 9
89
90// #define USE_NESTED_TRANSACTIONS
91#define MAX_NESTED_TRANSACTIONS 0
92/* transaction id */
93static dt_atomic_int _trxid;
94static dt_atomic_int _trx_batch_level;
95static gpointer _trx_owner = NULL;
96static gpointer _trx_batch_owner = NULL;
97
98typedef struct dt_database_t
99{
100 gboolean lock_acquired;
101
102 /* data database filename */
103 gchar *dbfilename_data, *lockfile_data;
104
105 /* library database filename */
106 gchar *dbfilename_library, *lockfile_library;
107
108 /* ondisk DB */
109 sqlite3 *handle;
110
111 gchar *error_message, *error_dbfilename;
112 int error_other_pid;
113} dt_database_t;
114
115
116/* migrates database from old place to new */
117static void _database_migrate_to_xdg_structure();
118
119/* delete old mipmaps files */
120static void _database_delete_mipmaps_files();
121
122#define _SQLITE3_EXEC(a, b, c, d, e) \
123 if(sqlite3_exec(a, b, c, d, e) != SQLITE_OK) \
124 { \
125 all_ok = FALSE; \
126 failing_query = b; \
127 goto end; \
128 }
129
130/* migrate from the legacy db format (with the 'settings' blob) to the first version this system knows */
131static gboolean _migrate_schema(dt_database_t *db, int version)
132{
133 gboolean all_ok = TRUE;
134 const char *failing_query = NULL;
135 sqlite3_stmt *stmt;
136 sqlite3_stmt *innerstmt;
137
138 if(version != 36) // if anyone shows up with an older db we can probably add extra code
139 return FALSE;
140
141 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
142 // clang-format off
143 // remove stuff that is either no longer needed or that got renamed
144 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.lock", NULL, NULL, NULL);
145 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.settings", NULL, NULL, NULL); // yes, we do this in many
146 // places. because it's really
147 // important to not miss it in
148 // any code path.
149 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.group_id_index", NULL, NULL, NULL);
150 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.imgid_index", NULL, NULL, NULL);
151 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.mipmaps", NULL, NULL, NULL);
152 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.mipmap_timestamps", NULL, NULL, NULL);
153 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.dt_migration_table", NULL, NULL, NULL);
154
155 // using _create_library_schema() and filling that with the old data doesn't work since we always want to generate
156 // version 1 tables
158 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)",
159 NULL, NULL, NULL);
160 _SQLITE3_EXEC(db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', 1)",
161 NULL, NULL, NULL);
163 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.film_rolls_folder_index ON film_rolls (folder)",
164 NULL, NULL, NULL);
166 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN orientation INTEGER", NULL, NULL, NULL);
167 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN focus_distance REAL", NULL, NULL, NULL);
168 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN group_id INTEGER", NULL, NULL, NULL);
169 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN histogram BLOB", NULL, NULL, NULL);
170 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN lightmap BLOB", NULL, NULL, NULL);
171 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN longitude REAL", NULL, NULL, NULL);
172 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN latitude REAL", NULL, NULL, NULL);
173 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN color_matrix BLOB", NULL, NULL, NULL);
174 // the colorspace as specified in some image types
175 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN colorspace INTEGER", NULL, NULL, NULL);
176 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN version INTEGER", NULL, NULL, NULL);
177 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN max_version INTEGER", NULL, NULL, NULL);
178 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET orientation = -1 WHERE orientation IS NULL", NULL, NULL, NULL);
179 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET focus_distance = -1 WHERE focus_distance IS NULL",
180 NULL, NULL, NULL);
181 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET group_id = id WHERE group_id IS NULL", NULL, NULL, NULL);
182 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET max_version = (SELECT COUNT(*)-1 FROM main.images i WHERE "
183 "i.filename = main.images.filename AND "
184 "i.film_id = main.images.film_id) WHERE max_version IS NULL",
185 NULL, NULL, NULL);
186 _SQLITE3_EXEC(
187 db->handle,
188 "UPDATE main.images SET version = (SELECT COUNT(*) FROM main.images i "
189 "WHERE i.filename = main.images.filename AND "
190 "i.film_id = main.images.film_id AND i.id < main.images.id) WHERE version IS NULL",
191 NULL, NULL, NULL);
192 // make sure we have AUTOINCREMENT on imgid --> move the whole thing away and recreate the table :(
193 _SQLITE3_EXEC(db->handle, "ALTER TABLE main.images RENAME TO dt_migration_table", NULL, NULL, NULL);
194 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.images_group_id_index", NULL, NULL, NULL);
195 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.images_film_id_index", NULL, NULL, NULL);
196 _SQLITE3_EXEC(
197 db->handle,
198 "CREATE TABLE main.images (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
199 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
200 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
201 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
202 "output_width INTEGER, output_height INTEGER, crop REAL, "
203 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
204 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
205 "caption VARCHAR, description VARCHAR, license VARCHAR, sha1sum CHAR(40), "
206 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
207 "latitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, max_version INTEGER)",
208 NULL, NULL, NULL);
209 _SQLITE3_EXEC(db->handle, "CREATE INDEX main.images_group_id_index ON images (group_id)", NULL, NULL, NULL);
210 _SQLITE3_EXEC(db->handle, "CREATE INDEX main.images_film_id_index ON images (film_id)", NULL, NULL, NULL);
211 _SQLITE3_EXEC(
212 db->handle,
213 "INSERT INTO main.images (id, group_id, film_id, width, height, filename, maker, model, "
214 "lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags, "
215 "output_width, output_height, crop, raw_parameters, raw_denoise_threshold, "
216 "raw_auto_bright_threshold, raw_black, raw_maximum, caption, description, license, sha1sum, "
217 "orientation, histogram, lightmap, longitude, latitude, color_matrix, colorspace, version, "
218 "max_version) "
219 "SELECT id, group_id, film_id, width, height, filename, maker, model, lens, exposure, aperture, iso, "
220 "focal_length, focus_distance, datetime_taken, flags, output_width, output_height, crop, "
221 "raw_parameters, raw_denoise_threshold, raw_auto_bright_threshold, raw_black, raw_maximum, "
222 "caption, description, license, sha1sum, orientation, histogram, lightmap, longitude, "
223 "latitude, color_matrix, colorspace, version, max_version FROM dt_migration_table",
224 NULL, NULL, NULL);
225 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
227 // selected_images should have a primary key. add it if it's missing:
228 _SQLITE3_EXEC(db->handle, "CREATE TEMPORARY TABLE dt_migration_table (imgid INTEGER)", NULL, NULL, NULL);
229 _SQLITE3_EXEC(db->handle, "INSERT INTO dt_migration_table SELECT imgid FROM main.selected_images", NULL, NULL,
230 NULL);
231 _SQLITE3_EXEC(db->handle, "DROP TABLE main.selected_images", NULL, NULL, NULL);
232 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.selected_images (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
233 _SQLITE3_EXEC(db->handle, "INSERT OR IGNORE INTO main.selected_images SELECT imgid FROM dt_migration_table",
234 NULL, NULL, NULL);
235 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
237 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
238 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
239 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
240 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
241 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.history_imgid_index ON history (imgid)", NULL, NULL,
242 NULL);
243 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
244 NULL, NULL);
245 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET multi_priority = 0 WHERE multi_priority IS NULL", NULL, NULL,
246 NULL);
247 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL, NULL);
249 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.mask (imgid INTEGER, formid INTEGER, form INTEGER, "
250 "name VARCHAR(256), version INTEGER, "
251 "points BLOB, points_count INTEGER, source BLOB)",
252 NULL, NULL, NULL);
253 sqlite3_exec(db->handle, "ALTER TABLE main.mask ADD COLUMN source BLOB", NULL, NULL,
254 NULL); // in case the table was there already but missed that column
256 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.tagged_images_tagid_index ON tagged_images (tagid)",
257 NULL, NULL, NULL);
259 _SQLITE3_EXEC(db->handle,
260 "CREATE TABLE IF NOT EXISTS main.styles (id INTEGER, name VARCHAR, description VARCHAR)", NULL,
261 NULL, NULL);
262 sqlite3_exec(db->handle, "ALTER TABLE main.styles ADD COLUMN id INTEGER", NULL, NULL, NULL);
263 _SQLITE3_EXEC(db->handle, "UPDATE main.styles SET id = rowid WHERE id IS NULL", NULL, NULL, NULL);
265 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.style_items (styleid INTEGER, num INTEGER, module "
266 "INTEGER, operation VARCHAR(256), op_params BLOB, "
267 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, multi_priority "
268 "INTEGER, multi_name VARCHAR(256))",
269 NULL, NULL, NULL);
270 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
271 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
272 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
273 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
274 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
275 NULL, NULL);
276 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET multi_priority = 0 WHERE multi_priority IS NULL", NULL,
277 NULL, NULL);
278 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL,
279 NULL);
281 // color_labels could have a PRIMARY KEY that we don't want
282 _SQLITE3_EXEC(db->handle, "CREATE TEMPORARY TABLE dt_migration_table (imgid INTEGER, color INTEGER)", NULL,
283 NULL, NULL);
284 _SQLITE3_EXEC(db->handle, "INSERT INTO dt_migration_table SELECT imgid, color FROM main.color_labels", NULL,
285 NULL, NULL);
286 _SQLITE3_EXEC(db->handle, "DROP TABLE main.color_labels", NULL, NULL, NULL);
287 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.color_labels (imgid INTEGER, color INTEGER)", NULL, NULL, NULL);
288 _SQLITE3_EXEC(db->handle, "CREATE UNIQUE INDEX main.color_labels_idx ON color_labels (imgid, color)", NULL, NULL,
289 NULL);
290 _SQLITE3_EXEC(db->handle, "INSERT OR IGNORE INTO main.color_labels SELECT imgid, color FROM dt_migration_table",
291 NULL, NULL, NULL);
292 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
294 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.meta_data (id INTEGER, key INTEGER, value VARCHAR)",
295 NULL, NULL, NULL);
296 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.metadata_index ON meta_data (id, key)", NULL, NULL,
297 NULL);
299 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.presets (name VARCHAR, description VARCHAR, "
300 "operation VARCHAR, op_version INTEGER, op_params BLOB, "
301 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, multi_priority "
302 "INTEGER, multi_name VARCHAR(256), "
303 "model VARCHAR, maker VARCHAR, lens VARCHAR, iso_min REAL, iso_max REAL, "
304 "exposure_min REAL, exposure_max REAL, "
305 "aperture_min REAL, aperture_max REAL, focal_length_min REAL, focal_length_max "
306 "REAL, writeprotect INTEGER, "
307 "autoapply INTEGER, filter INTEGER, def INTEGER, isldr INTEGER)",
308 NULL, NULL, NULL);
309 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN op_version INTEGER", NULL, NULL, NULL);
310 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
311 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
312 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
313 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
314 // the unique index only works if the db doesn't have any (name, operation, op_version) more than once.
315 // apparently there are dbs out there which do have that. :(
316 sqlite3_prepare_v2(db->handle,
317 "SELECT p.rowid, p.name, p.operation, p.op_version FROM main.presets p INNER JOIN "
318 "(SELECT * FROM (SELECT rowid, name, operation, op_version, COUNT(*) AS count "
319 "FROM main.presets GROUP BY name, operation, op_version) WHERE count > 1) s "
320 "ON p.name = s.name AND p.operation = s.operation AND p.op_version = s.op_version",
321 -1, &stmt, NULL);
322 // clang-format on
323 char *last_name = NULL, *last_operation = NULL;
324 int last_op_version = 0;
325 int i = 0;
326 while(sqlite3_step(stmt) == SQLITE_ROW)
327 {
328 const int rowid = sqlite3_column_int(stmt, 0);
329 const char *name = (const char *)sqlite3_column_text(stmt, 1);
330 const char *operation = (const char *)sqlite3_column_text(stmt, 2);
331 const int op_version = sqlite3_column_int(stmt, 3);
332
333 // is it still the same (name, operation, op_version) triple?
334 if(IS_NULL_PTR(last_name) || strcmp(last_name, name) || IS_NULL_PTR(last_operation) || strcmp(last_operation, operation)
335 || last_op_version != op_version)
336 {
337 dt_free(last_name);
338 dt_free(last_operation);
339 last_name = g_strdup(name);
340 last_operation = g_strdup(operation);
341 last_op_version = op_version;
342 i = 0;
343 }
344
345 // find the next free amended version of name
346 // clang-format off
347 sqlite3_prepare_v2(db->handle, "SELECT name FROM main.presets WHERE name = ?1 || ' (' || ?2 || ')' AND "
348 "operation = ?3 AND op_version = ?4",
349 -1, &innerstmt, NULL);
350 // clang-format on
351 while(1)
352 {
353 sqlite3_bind_text(innerstmt, 1, name, -1, SQLITE_TRANSIENT);
354 sqlite3_bind_int(innerstmt, 2, i);
355 sqlite3_bind_text(innerstmt, 3, operation, -1, SQLITE_TRANSIENT);
356 sqlite3_bind_int(innerstmt, 4, op_version);
357 if(sqlite3_step(innerstmt) != SQLITE_ROW) break;
358 sqlite3_reset(innerstmt);
359 sqlite3_clear_bindings(innerstmt);
360 i++;
361 }
362 sqlite3_finalize(innerstmt);
363
364 // rename preset
365 // clang-format off
366 const char *query = "UPDATE main.presets SET name = name || ' (' || ?1 || ')' WHERE rowid = ?2";
367 // clang-format on
368 sqlite3_prepare_v2(db->handle, query, -1, &innerstmt, NULL);
369 sqlite3_bind_int(innerstmt, 1, i);
370 sqlite3_bind_int(innerstmt, 2, rowid);
371 if(sqlite3_step(innerstmt) != SQLITE_DONE)
372 {
373 all_ok = FALSE;
374 failing_query = query;
375 goto end;
376 }
377 sqlite3_finalize(innerstmt);
378 }
379 sqlite3_finalize(stmt);
380 dt_free(last_name);
381 dt_free(last_operation);
382 // now we should be able to create the index
383 // clang-format off
384 _SQLITE3_EXEC(db->handle,
385 "CREATE UNIQUE INDEX IF NOT EXISTS main.presets_idx ON presets (name, operation, op_version)",
386 NULL, NULL, NULL);
387 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
388 NULL, NULL);
389 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET multi_priority = 0 WHERE multi_priority IS NULL", NULL, NULL,
390 NULL);
391 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL, NULL);
392 // clang-format on
393
394
395 // There are systems where absolute paths don't start with '/' (like Windows).
396 // Since the bug which introduced absolute paths to the db was fixed before a
397 // Windows build was available this shouldn't matter though.
398 // clang-format off
399 sqlite3_prepare_v2(db->handle, "SELECT id, filename FROM main.images WHERE filename LIKE '/%'", -1, &stmt, NULL);
400 sqlite3_prepare_v2(db->handle, "UPDATE main.images SET filename = ?1 WHERE id = ?2", -1, &innerstmt, NULL);
401 // clang-format on
402 while(sqlite3_step(stmt) == SQLITE_ROW)
403 {
404 const int id = sqlite3_column_int(stmt, 0);
405 const char *path = (const char *)sqlite3_column_text(stmt, 1);
406 gchar *filename = g_path_get_basename(path);
407 sqlite3_bind_text(innerstmt, 1, filename, -1, SQLITE_TRANSIENT);
408 sqlite3_bind_int(innerstmt, 2, id);
409 sqlite3_step(innerstmt);
410 sqlite3_reset(innerstmt);
411 sqlite3_clear_bindings(innerstmt);
412 dt_free(filename);
413 }
414 sqlite3_finalize(stmt);
415 sqlite3_finalize(innerstmt);
416
417 // We used to insert datetime_taken entries with '-' as date separators. Since that doesn't work well with
418 // the regular ':' when parsing
419 // or sorting we changed it to ':'. This takes care to change what we have as leftovers
420 // clang-format off
421 _SQLITE3_EXEC(
422 db->handle,
423 "UPDATE main.images SET datetime_taken = REPLACE(datetime_taken, '-', ':') WHERE datetime_taken LIKE '%-%'",
424 NULL, NULL, NULL);
425 // clang-format on
426
427end:
428 if(all_ok)
429 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
430 else
431 {
432 fprintf(stderr, "[init] failing query: `%s'\n", failing_query);
433 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle));
434 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL);
435 }
436
437 return all_ok;
438}
439
440#undef _SQLITE3_EXEC
441
442#define TRY_EXEC(_query, _message) \
443 do \
444 { \
445 if(sqlite3_exec(db->handle, _query, NULL, NULL, NULL) != SQLITE_OK) \
446 { \
447 fprintf(stderr, _message); \
448 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
449 FINALIZE; \
450 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
451 return version; \
452 } \
453 } while(0)
454
455#define TRY_STEP(_stmt, _expected, _message) \
456 do \
457 { \
458 if(sqlite3_step(_stmt) != _expected) \
459 { \
460 fprintf(stderr, _message); \
461 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
462 FINALIZE; \
463 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
464 return version; \
465 } \
466 } while(0)
467
468#define TRY_PREPARE(_stmt, _query, _message) \
469 do \
470 { \
471 if(sqlite3_prepare_v2(db->handle, _query, -1, &_stmt, NULL) != SQLITE_OK) \
472 { \
473 fprintf(stderr, _message); \
474 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
475 FINALIZE; \
476 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
477 return version; \
478 } \
479 } while(0)
480
481// redefine this where needed
482#define FINALIZE
483
484/* do the real migration steps, returns the version the db was converted to */
485static int _upgrade_library_schema_step(dt_database_t *db, int version)
486{
487 sqlite3_stmt *stmt;
488 int new_version = version;
489 if(version == CURRENT_DATABASE_VERSION_LIBRARY)
490 return version;
491 else if(version == 0)
492 {
493 // this can't happen, we started with 1, but it's a good example how this function works
494 // <do some magic to the db>
495 new_version = 1; // the version we transformed the db to. this way it might be possible to roll back or
496 // add fast paths
497 }
498 else if(version == 1)
499 {
500 // 1 -> 2 added write_timestamp
501 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
502 // clang-format off
503 TRY_EXEC("ALTER TABLE main.images ADD COLUMN write_timestamp INTEGER",
504 "[init] can't add `write_timestamp' column to database\n");
505 TRY_EXEC("UPDATE main.images SET write_timestamp = STRFTIME('%s', 'now') WHERE write_timestamp IS NULL",
506 "[init] can't initialize `write_timestamp' with current point in time\n");
507 // clang-format on
508 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
509 new_version = 2;
510 }
511 else if(version == 2)
512 {
513 // 2 -> 3 reset raw_black and raw_maximum. in theory we should change the columns from REAL to INTEGER,
514 // but sqlite doesn't care about types so whatever
515 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
516 // clang-format off
517 TRY_EXEC("UPDATE main.images SET raw_black = 0, raw_maximum = 16384",
518 "[init] can't reset raw_black and raw_maximum\n");
519 // clang-format on
520 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
521 new_version = 3;
522 }
523 else if(version == 3)
524 {
525 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
526 // clang-format off
527 TRY_EXEC("CREATE TRIGGER insert_tag AFTER INSERT ON main.tags"
528 " BEGIN"
529 " INSERT INTO tagxtag SELECT id, new.id, 0 FROM TAGS;"
530 " UPDATE tagxtag SET count = 1000000 WHERE id1=new.id AND id2=new.id;"
531 " END",
532 "[init] can't create insert_tag trigger\n");
533 TRY_EXEC("CREATE TRIGGER delete_tag BEFORE DELETE ON main.tags"
534 " BEGIN"
535 " DELETE FROM tagxtag WHERE id1=old.id OR id2=old.id;"
536 " DELETE FROM tagged_images WHERE tagid=old.id;"
537 " END",
538 "[init] can't create delete_tag trigger\n");
539 TRY_EXEC("CREATE TRIGGER attach_tag AFTER INSERT ON main.tagged_images"
540 " BEGIN"
541 " UPDATE tagxtag"
542 " SET count = count + 1"
543 " WHERE (id1=new.tagid AND id2 IN (SELECT tagid FROM tagged_images WHERE imgid=new.imgid))"
544 " OR (id2=new.tagid AND id1 IN (SELECT tagid FROM tagged_images WHERE imgid=new.imgid));"
545 " END",
546 "[init] can't create attach_tag trigger\n");
547 TRY_EXEC("CREATE TRIGGER detach_tag BEFORE DELETE ON main.tagged_images"
548 " BEGIN"
549 " UPDATE tagxtag"
550 " SET count = count - 1"
551 " WHERE (id1=old.tagid AND id2 IN (SELECT tagid FROM tagged_images WHERE imgid=old.imgid))"
552 " OR (id2=old.tagid AND id1 IN (SELECT tagid FROM tagged_images WHERE imgid=old.imgid));"
553 " END",
554 "[init] can't create detach_tag trigger\n");
555 // clang-format on
556 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
557 new_version = 4;
558 }
559 else if(version == 4)
560 {
561 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
562 // clang-format off
563 TRY_EXEC("ALTER TABLE main.presets RENAME TO tmp_presets", "[init] can't rename table presets\n");
564
565 TRY_EXEC("CREATE TABLE main.presets (name VARCHAR, description VARCHAR, operation VARCHAR, op_params BLOB,"
566 "enabled INTEGER, blendop_params BLOB, model VARCHAR, maker VARCHAR, lens VARCHAR,"
567 "iso_min REAL, iso_max REAL, exposure_min REAL, exposure_max REAL, aperture_min REAL,"
568 "aperture_max REAL, focal_length_min REAL, focal_length_max REAL, writeprotect INTEGER,"
569 "autoapply INTEGER, filter INTEGER, def INTEGER, format INTEGER, op_version INTEGER,"
570 "blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
571 "[init] can't create new presets table\n");
572
573 TRY_EXEC("INSERT INTO main.presets (name, description, operation, op_params, enabled, blendop_params, model, "
574 "maker, lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, aperture_max,"
575 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, def, format, op_version, "
576 "blendop_version, multi_priority, multi_name) SELECT name, description, operation, op_params, "
577 "enabled, blendop_params, model, maker, lens, iso_min, iso_max, exposure_min, exposure_max, "
578 "aperture_min, aperture_max, focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
579 "def, isldr, op_version, blendop_version, multi_priority, multi_name FROM tmp_presets",
580 "[init] can't populate presets table from tmp_presets\n");
581
582 TRY_EXEC("DROP TABLE tmp_presets", "[init] can't delete table tmp_presets\n");
583 // clang-format on
584 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
585 new_version = 5;
586 }
587 else if(version == 5)
588 {
589 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
590 // clang-format off
591 TRY_EXEC("CREATE INDEX main.images_filename_index ON images (filename)",
592 "[init] can't create index on image filename\n");
593 // clang-format on
594 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
595 new_version = 6;
596 }
597 else if(version == 6)
598 {
599 // some ancient tables can have the styleid column of style_items be called style_id. fix that.
600 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
601
602 if(sqlite3_exec(db->handle, "SELECT style_id FROM main.style_items", NULL, NULL, NULL) == SQLITE_OK)
603 {
604 // clang-format off
605 TRY_EXEC("ALTER TABLE main.style_items RENAME TO tmp_style_items",
606 "[init] can't rename table style_items\n");
607
608 TRY_EXEC("CREATE TABLE main.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
609 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
610 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
611 "[init] can't create new style_items table\n");
612
613 TRY_EXEC("INSERT INTO main.style_items (styleid, num, module, operation, op_params, enabled,"
614 " blendop_params, blendop_version, multi_priority, multi_name)"
615 " SELECT style_id, num, module, operation, op_params, enabled,"
616 " blendop_params, blendop_version, multi_priority, multi_name"
617 " FROM tmp_style_items",
618 "[init] can't populate style_items table from tmp_style_items\n");
619
620 TRY_EXEC("DROP TABLE tmp_style_items", "[init] can't delete table tmp_style_items\n");
621 // clang-format on
622 }
623
624 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
625 new_version = 7;
626 }
627 else if(version == 7)
628 {
629 // make sure that we have no film rolls with a NULL folder
630 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
631 // clang-format off
632 TRY_EXEC("ALTER TABLE main.film_rolls RENAME TO tmp_film_rolls", "[init] can't rename table film_rolls\n");
633
634 TRY_EXEC("CREATE TABLE main.film_rolls "
635 "(id INTEGER PRIMARY KEY, datetime_accessed CHAR(20), "
636 "folder VARCHAR(1024) NOT NULL)",
637 "[init] can't create new film_rolls table\n");
638
639 TRY_EXEC("INSERT INTO main.film_rolls (id, datetime_accessed, folder) "
640 "SELECT id, datetime_accessed, folder "
641 "FROM tmp_film_rolls "
642 "WHERE folder IS NOT NULL",
643 "[init] can't populate film_rolls table from tmp_film_rolls\n");
644
645 TRY_EXEC("DROP TABLE tmp_film_rolls", "[init] can't delete table tmp_film_rolls\n");
646 // clang-format on
647 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
648 new_version = 8;
649 }
650 else if(version == 8)
651 {
652 // 8 -> 9 added history_end column to images
653 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
654 // clang-format off
655 TRY_EXEC("ALTER TABLE main.images ADD COLUMN history_end INTEGER",
656 "[init] can't add `history_end' column to database\n");
657
658 TRY_EXEC("UPDATE main.images SET history_end = (SELECT IFNULL(MAX(num) + 1, 0) FROM main.history "
659 "WHERE imgid = id)", "[init] can't initialize `history_end' with last history entry\n");
660 // clang-format on
661 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
662 new_version = 9;
663 }
664 else if(version == 9)
665 {
666 // 9 -> 10 cleanup of last update :(
667 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
668 // clang-format off
669 TRY_EXEC("UPDATE main.images SET history_end = (SELECT IFNULL(MAX(num) + 1, 0) FROM main.history "
670 "WHERE imgid = id)", "[init] can't set `history_end' to 0 where it was NULL\n");
671 // clang-format on
672 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
673 new_version = 10;
674 }
675 else if(version == 10)
676 {
677 // 10 -> 11 added altitude column to images
678 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
679 // clang-format off
680 TRY_EXEC("ALTER TABLE main.images ADD COLUMN altitude REAL",
681 "[init] can't add `altitude' column to database\n");
682
683 TRY_EXEC("UPDATE main.images SET altitude = NULL", "[init] can't initialize `altitude' with NULL\n");
684 // clang-format on
685 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
686 new_version = 11;
687 }
688 else if(version == 11)
689 {
690 // 11 -> 12 tagxtag was removed in order to reduce database size
691 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
692 // clang-format off
693 TRY_EXEC("DROP TRIGGER main.detach_tag", "[init] can't drop trigger `detach_tag' from database\n");
694
695 TRY_EXEC("DROP TRIGGER main.attach_tag", "[init] can't drop trigger `attach_tag' from database\n");
696
697 TRY_EXEC("DROP TRIGGER main.delete_tag", "[init] can't drop trigger `delete_tag' from database\n");
698
699 TRY_EXEC("DROP TRIGGER main.insert_tag", "[init] can't drop trigger `insert_tag' from database\n");
700
701 TRY_EXEC("DROP TABLE main.tagxtag", "[init] can't drop table `tagxtag' from database\n");
702 // clang-format on
703 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
704 new_version = 12;
705 }
706 else if(version == 12)
707 {
708 // 11 -> 12 move presets, styles and tags over to the data database
709 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
710
712#undef FINALIZE
713#define FINALIZE \
714 do \
715 { \
716 sqlite3_finalize(stmt); \
717 sqlite3_finalize(select_stmt); \
718 sqlite3_finalize(count_clashes_stmt); \
719 sqlite3_finalize(update_name_stmt); \
720 sqlite3_finalize(insert_stmt); \
721 sqlite3_finalize(delete_stmt); \
722 } while(0)
723
724 stmt = NULL;
725 sqlite3_stmt *insert_stmt = NULL, *delete_stmt = NULL, *select_stmt = NULL, *count_clashes_stmt = NULL,
726 *update_name_stmt = NULL;
727 // clang-format off
728 // remove presets that are already in data.
729 // we can't use a NATURAL JOIN here as that fails when columns have NULL values. :-(
730 TRY_EXEC("DELETE FROM main.presets WHERE rowid IN (SELECT p1.rowid FROM main.presets p1 "
731 "JOIN data.presets p2 ON "
732 "p1.name IS p2.name AND "
733 "p1.description IS p2.description AND "
734 "p1.operation IS p2.operation AND "
735 "p1.op_version IS p2.op_version AND "
736 "p1.op_params IS p2.op_params AND "
737 "p1.enabled IS p2.enabled AND "
738 "p1.blendop_params IS p2.blendop_params AND "
739 "p1.blendop_version IS p2.blendop_version AND "
740 "p1.multi_priority IS p2.multi_priority AND "
741 "p1.multi_name IS p2.multi_name AND "
742 "p1.model IS p2.model AND "
743 "p1.maker IS p2.maker AND "
744 "p1.lens IS p2.lens AND "
745 "p1.iso_min IS p2.iso_min AND "
746 "p1.iso_max IS p2.iso_max AND "
747 "p1.exposure_min IS p2.exposure_min AND "
748 "p1.exposure_max IS p2.exposure_max AND "
749 "p1.aperture_min IS p2.aperture_min AND "
750 "p1.aperture_max IS p2.aperture_max AND "
751 "p1.focal_length_min IS p2.focal_length_min AND "
752 "p1.focal_length_max IS p2.focal_length_max AND "
753 "p1.writeprotect IS p2.writeprotect AND "
754 "p1.autoapply IS p2.autoapply AND "
755 "p1.filter IS p2.filter AND "
756 "p1.def IS p2.def AND "
757 "p1.format IS p2.format "
758 "WHERE p1.writeprotect = 0)",
759 "[init] can't delete already migrated presets from database\n");
760
761 // find all presets that are clashing with something else in presets. that can happen as we introduced an
762 // index on presets in data which wasn't in place in library.
763 TRY_PREPARE(select_stmt, "SELECT p.rowid, r FROM main.presets AS p, (SELECT rowid AS r, name, operation, "
764 "op_version FROM main.presets GROUP BY name, operation, op_version HAVING "
765 "COUNT(*) > 1) USING (name, operation, op_version) WHERE p.rowid != r",
766 "[init] can't prepare selecting presets with same name, operation, op_version from database\n");
767
768 // see if an updated preset name still causes problems
769 TRY_PREPARE(count_clashes_stmt, "SELECT COUNT(*) FROM main.presets AS p, (SELECT name, operation, op_version "
770 "FROM main.presets WHERE rowid = ?1) AS i ON p.name = i.name || \" #\" || ?2 "
771 "AND p.operation = i.operation AND p.op_version = i.op_version",
772 "[init] can't prepare selection of preset count by name from database\n");
773
774 // update the preset name for good
775 TRY_PREPARE(update_name_stmt, "UPDATE main.presets SET name = name || \" #\" || ?1 WHERE rowid = ?2",
776 "[init] can't prepare updating of preset name in database\n");
777
778 // find all presets that would be clashing with something in data
779 TRY_PREPARE(stmt, "SELECT p1.rowid FROM main.presets p1 INNER JOIN data.presets p2 "
780 "USING (name, operation, op_version) WHERE p1.writeprotect = 0",
781 "[init] can't access table `presets' in database\n");
782
783 // ... and move them over with a new name
784 TRY_PREPARE(insert_stmt, "INSERT OR FAIL INTO data.presets (name, description, operation, op_version, "
785 "op_params, enabled, blendop_params, blendop_version, multi_priority, multi_name, "
786 "model, maker, lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, "
787 "aperture_max, focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
788 "def, format) "
789 "SELECT name || \" #\" || ?1, description, operation, op_version, op_params, "
790 "enabled, blendop_params, blendop_version, multi_priority, multi_name, model, maker, "
791 "lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, aperture_max, "
792 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, def, format "
793 "FROM main.presets p1 WHERE p1.rowid = ?2",
794 "[init] can't prepare insertion statement\n");
795
796 TRY_PREPARE(delete_stmt, "DELETE FROM main.presets WHERE rowid = ?1", "[init] can't prepare deletion statement\n");
797 // clang-format on
798 // first rename presets with (name, operation, op_version) not being unique
799 while(sqlite3_step(select_stmt) == SQLITE_ROW)
800 {
801 const int own_rowid = sqlite3_column_int(select_stmt, 0);
802 const int other_rowid = sqlite3_column_int(select_stmt, 1);
803 int preset_version = 0;
804
805 do
806 {
807 preset_version++;
808 sqlite3_reset(count_clashes_stmt);
809 sqlite3_clear_bindings(count_clashes_stmt);
810 sqlite3_bind_int(count_clashes_stmt, 1, other_rowid);
811 sqlite3_bind_int(count_clashes_stmt, 2, preset_version);
812 }
813 while(sqlite3_step(count_clashes_stmt) == SQLITE_ROW && sqlite3_column_int(count_clashes_stmt, 0) > 0);
814
815 sqlite3_bind_int(update_name_stmt, 1, preset_version);
816 sqlite3_bind_int(update_name_stmt, 2, own_rowid);
817 TRY_STEP(update_name_stmt, SQLITE_DONE, "[init] can't rename preset in database\n");
818 sqlite3_reset(update_name_stmt);
819 sqlite3_reset(update_name_stmt);
820 }
821
822 // now rename to avoid clashes with data.presets
823 while(sqlite3_step(stmt) == SQLITE_ROW)
824 {
825 int preset_version = 0;
826 const int rowid = sqlite3_column_int(stmt, 0);
827
828 do
829 {
830 preset_version++;
831 sqlite3_reset(insert_stmt);
832 sqlite3_clear_bindings(insert_stmt);
833 sqlite3_bind_int(insert_stmt, 1, preset_version);
834 sqlite3_bind_int(insert_stmt, 2, rowid);
835 } while(sqlite3_step(insert_stmt) != SQLITE_DONE);
836
837 sqlite3_reset(delete_stmt);
838 sqlite3_clear_bindings(delete_stmt);
839 sqlite3_bind_int(delete_stmt, 1, rowid);
840 TRY_STEP(delete_stmt, SQLITE_DONE, "[init] can't delete preset from database\n");
841 }
842 // clang-format off
843 // all that is left in presets should be those that can be moved over without any further concerns
844 TRY_EXEC("INSERT OR FAIL INTO data.presets SELECT name, description, operation, "
845 "op_version, op_params, enabled, blendop_params, blendop_version, "
846 "multi_priority, multi_name, model, maker, lens, iso_min, iso_max, "
847 "exposure_min, exposure_max, aperture_min, aperture_max, "
848 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
849 "def, format FROM main.presets WHERE writeprotect = 0",
850 "[init] can't copy presets to the data database\n");
851 // ... delete them on the old side
852 TRY_EXEC("DELETE FROM main.presets WHERE writeprotect = 0",
853 "[init] can't copy presets to the data database\n");
854 // clang-format on
855
856 FINALIZE;
857#undef FINALIZE
858
860#define FINALIZE \
861 do \
862 { \
863 sqlite3_finalize(stmt); \
864 sqlite3_finalize(insert_stmt); \
865 sqlite3_finalize(select_stmt); \
866 sqlite3_finalize(delete_stmt); \
867 sqlite3_finalize(update_name_stmt); \
868 sqlite3_finalize(select_new_stmt); \
869 sqlite3_finalize(copy_style_items_stmt); \
870 sqlite3_finalize(delete_style_items_stmt); \
871 } while(0)
872
873 stmt = NULL;
874 select_stmt = NULL;
875 update_name_stmt = NULL;
876 insert_stmt = NULL;
877 delete_stmt = NULL;
878 sqlite3_stmt *select_new_stmt = NULL, *copy_style_items_stmt = NULL, *delete_style_items_stmt = NULL;
879 // clang-format off
880 TRY_PREPARE(stmt, "SELECT id, name FROM main.styles", "[init] can't prepare style selection from database\n");
881 TRY_PREPARE(select_stmt, "SELECT rowid FROM data.styles WHERE name = ?1 LIMIT 1",
882 "[init] can't prepare style item selection from database\n");
883 TRY_PREPARE(update_name_stmt, "UPDATE main.styles SET name = ?1 WHERE id = ?2",
884 "[init] can't prepare style name update\n");
885 TRY_PREPARE(insert_stmt, "INSERT INTO data.styles (id, name, description) "
886 "SELECT (SELECT COALESCE(MAX(id),0)+1 FROM data.styles), name, description "
887 "FROM main.styles where id = ?1",
888 "[init] can't prepare style insertion for database\n");
889 TRY_PREPARE(delete_stmt, "DELETE FROM main.styles WHERE id = ?1",
890 "[init] can't prepare style deletion for database\n");
891 TRY_PREPARE(select_new_stmt, "SELECT id FROM data.styles WHERE rowid = ?1",
892 "[init] can't prepare style selection from data database\n");
893 TRY_PREPARE(copy_style_items_stmt, "INSERT INTO data.style_items "
894 "(styleid, num, module, operation, op_params, enabled, blendop_params, "
895 "blendop_version, multi_priority, multi_name) "
896 "SELECT ?1, num, module, operation, op_params, enabled, blendop_params, "
897 "blendop_version, multi_priority, multi_name FROM main.style_items "
898 "WHERE styleid = ?2",
899 "[init] can't prepare style item copy into data database\n");
900 TRY_PREPARE(delete_style_items_stmt, "DELETE FROM main.style_items WHERE styleid = ?1",
901 "[init] can't prepare style item deletion for database\n");
902 // clang-format on
903 while(sqlite3_step(stmt) == SQLITE_ROW)
904 {
905 const int id = sqlite3_column_int(stmt, 0);
906 const char *name = (const char *)sqlite3_column_text(stmt, 1);
907
908 // find a unique name of the style for data.styles
909 sqlite3_bind_text(select_stmt, 1, name, -1, SQLITE_TRANSIENT);
910 if(sqlite3_step(select_stmt) == SQLITE_ROW)
911 {
912 // we need to append a version
913 int style_version = 0;
914 char *new_name = NULL;
915 do
916 {
917 style_version++;
918 dt_free(new_name);
919 new_name = g_strdup_printf("%s #%d", name, style_version);
920 sqlite3_reset(select_stmt);
921 sqlite3_clear_bindings(select_stmt);
922 sqlite3_bind_text(select_stmt, 1, new_name, -1, SQLITE_TRANSIENT);
923 } while(sqlite3_step(select_stmt) == SQLITE_ROW);
924
925 // update the name in the old place
926 sqlite3_bind_text(update_name_stmt, 1, new_name, -1, SQLITE_TRANSIENT);
927 sqlite3_bind_int(update_name_stmt, 2, id);
928 TRY_STEP(update_name_stmt, SQLITE_DONE, "[init] can't update name of style in database\n");
929 sqlite3_reset(update_name_stmt);
930 sqlite3_clear_bindings(update_name_stmt);
931 dt_free(new_name);
932 }
933
934 // move the style to data.styles and get the rowid
935 sqlite3_bind_int(insert_stmt, 1, id);
936 TRY_STEP(insert_stmt, SQLITE_DONE, "[init] can't insert style into data database\n");
937 sqlite3_int64 last_rowid = sqlite3_last_insert_rowid(db->handle);
938
939 // delete style from styles
940 sqlite3_bind_int(delete_stmt, 1, id);
941 TRY_STEP(delete_stmt, SQLITE_DONE, "[init] can't delete style from database\n");
942
943 sqlite3_bind_int(select_new_stmt, 1, last_rowid);
944 TRY_STEP(select_new_stmt, SQLITE_ROW, "[init] can't select new style from data database\n");
945 const int new_id = sqlite3_column_int(select_new_stmt, 0);
946
947 // now that we have the style over in data.styles and the new id we can just copy over all style items
948 sqlite3_bind_int(copy_style_items_stmt, 1, new_id);
949 sqlite3_bind_int(copy_style_items_stmt, 2, id);
950 TRY_STEP(copy_style_items_stmt, SQLITE_DONE, "[init] can't copy style items into data database\n");
951
952 // delete the style items from the old table
953 sqlite3_bind_int(delete_style_items_stmt, 1, id);
954 TRY_STEP(delete_style_items_stmt, SQLITE_DONE, "[init] can't delete style items from database\n");
955
956 // cleanup for the next round
957 sqlite3_reset(insert_stmt);
958 sqlite3_clear_bindings(insert_stmt);
959 sqlite3_reset(select_stmt);
960 sqlite3_clear_bindings(select_stmt);
961 sqlite3_reset(delete_stmt);
962 sqlite3_clear_bindings(delete_stmt);
963 sqlite3_reset(select_new_stmt);
964 sqlite3_clear_bindings(select_new_stmt);
965 sqlite3_reset(copy_style_items_stmt);
966 sqlite3_clear_bindings(copy_style_items_stmt);
967 sqlite3_reset(delete_style_items_stmt);
968 sqlite3_clear_bindings(delete_style_items_stmt);
969 }
970 FINALIZE;
971#undef FINALIZE
972
974#define FINALIZE
975 // clang-format off
976 // tags
977 TRY_EXEC("INSERT OR IGNORE INTO data.tags (name, icon, description, flags) "
978 "SELECT name, icon, description, flags FROM main.tags",
979 "[init] can't prepare insertion of used tags into data database\n");
980
981 // tagged images
982 // we need a temp table to update tagged_images due to its primary key
983 TRY_EXEC("CREATE TEMPORARY TABLE tagged_images_tmp (imgid INTEGER, tagid INTEGER)",
984 "[init] can't create temporary table for updating `tagged_images'\n");
985
986 TRY_EXEC("INSERT INTO tagged_images_tmp (imgid, tagid) "
987 "SELECT imgid, (SELECT t2.id FROM main.tags t1, data.tags t2 USING (name) WHERE t1.id = tagid) "
988 "FROM main.tagged_images", "[init] can't insert into `tagged_images_tmp'\n");
989
990 TRY_EXEC("DELETE FROM main.tagged_images", "[init] can't delete tagged images in database\n");
991
992 TRY_EXEC("INSERT OR IGNORE INTO main.tagged_images (imgid, tagid) SELECT imgid, tagid FROM tagged_images_tmp",
993 "[init] can't copy updated values back to `tagged_images'\n");
994
995 TRY_EXEC("DROP TABLE tagged_images_tmp", "[init] can't drop table `tagged_images_tmp' from database\n");
996
998 TRY_EXEC("DROP INDEX IF EXISTS main.presets_idx", "[init] can't drop index `presets_idx' from database\n");
999 TRY_EXEC("DROP TABLE main.presets", "[init] can't drop table `presets' from database\n");
1000 TRY_EXEC("DROP TABLE main.style_items", "[init] can't drop table `style_items' from database\n");
1001 TRY_EXEC("DROP TABLE main.styles", "[init] can't drop table `styles' from database\n");
1002 TRY_EXEC("DROP TABLE main.tags", "[init] can't drop table `tags' from database\n");
1003 // clang-format on
1004 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1005 new_version = 13;
1006 } else if(version == 13)
1007 {
1008 // 12 -> 13 bring back the used tag names to library.db so people can use it independently of data.db
1009 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1010 // clang-format off
1011 TRY_EXEC("CREATE TABLE main.used_tags (id INTEGER, name VARCHAR NOT NULL)",
1012 "[init] can't create `used_tags` table\n");
1013
1014 TRY_EXEC("CREATE INDEX main.used_tags_idx ON used_tags (id, name)",
1015 "[init] can't create index on table `used_tags' in database\n");
1016
1017 TRY_EXEC("INSERT INTO main.used_tags (id, name) SELECT t.id, t.name FROM data.tags AS t, main.tagged_images "
1018 "AS i ON t.id = i.tagid GROUP BY t.id",
1019 "[init] can't insert used tags into `used_tags` table in database\n");
1020 // clang-format on
1021 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1022 new_version = 14;
1023 }
1024 else if(version == 14)
1025 {
1026 // 13 -> fix the index on used_tags to be a UNIQUE index :-/
1027 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1028 // clang-format off
1029 TRY_EXEC("DELETE FROM main.used_tags WHERE rowid NOT IN (SELECT rowid FROM used_tags GROUP BY id)",
1030 "[init] can't delete duplicated entries from `used_tags' in database\n");
1031
1032 TRY_EXEC("DROP INDEX main.used_tags_idx", "[init] can't drop index `used_tags_idx' from database\n");
1033
1034 TRY_EXEC("CREATE UNIQUE INDEX main.used_tags_idx ON used_tags (id, name)",
1035 "[init] can't create index `used_tags_idx' in database\n");
1036
1037 TRY_EXEC("DELETE FROM main.tagged_images WHERE tagid IS NULL",
1038 "[init] can't delete NULL entries from `tagged_images' in database");
1039
1040 TRY_EXEC("DELETE FROM main.used_tags WHERE id NOT IN (SELECT DISTINCT tagid FROM main.tagged_images)",
1041 "[init] can't delete unused tags from `used_tags' in database\n");
1042 // clang-format on
1043 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1044 new_version = 15;
1045 }
1046 else if(version == 15)
1047 {
1048 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1050 // clang-format off
1051 TRY_EXEC("ALTER TABLE main.images ADD COLUMN position INTEGER",
1052 "[init] can't add `position' column to images table in database\n");
1053 TRY_EXEC("CREATE INDEX main.image_position_index ON images (position)",
1054 "[init] can't create index for custom image order table\n");
1055
1056 // Set the initial image sequence. The image id - the sequence images were imported -
1057 // defines the initial order of images.
1058 //
1059 // An int64 is used for the position index. The upper 31 bits define the initial order.
1060 // The lower 32bit provide space to reorder images.
1061 //
1062 // see: dt_collection_move_before()
1063 //
1064 TRY_EXEC("UPDATE main.images SET position = id << 32",
1065 "[init] can't update positions custom image order table\n");
1066 // clang-format on
1067 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1068 new_version = 16;
1069 }
1070 else if(version == 16)
1071 {
1072 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1074 // clang-format off
1075 TRY_EXEC("ALTER TABLE main.images ADD COLUMN aspect_ratio REAL",
1076 "[init] can't add `aspect_ratio' column to images table in database\n");
1077 TRY_EXEC("UPDATE main.images SET aspect_ratio = 0.0",
1078 "[init] can't update aspect_ratio in database\n");
1079 // clang-format on
1080 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1081 new_version = 17;
1082 }
1083 else if(version == 17)
1084 {
1085 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1086
1088 // clang-format off
1089 TRY_EXEC("CREATE TABLE main.masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, name VARCHAR(256), "
1090 "version INTEGER, points BLOB, points_count INTEGER, source BLOB)",
1091 "[init] can't create `masks_history` table\n");
1092
1093 TRY_EXEC("CREATE INDEX main.masks_history_imgid_index ON masks_history (imgid)",
1094 "[init] can't create index `masks_history_imgid_index' in database\n");
1095
1096 // to speed up the mask look-up, and makes the following UPDATE instantaneous whereas it could takes hours
1097 TRY_EXEC("CREATE INDEX main.mask_imgid_index ON mask (imgid);",
1098 "[init] can't create index `mask_imgid_index' in database\n");
1099
1100 // create a mask manager entry on history for all images containing all forms
1101 // make room for mask manager history entry
1102 TRY_EXEC("UPDATE main.history SET num=num+1 WHERE imgid IN (SELECT imgid FROM main.mask WHERE main.mask.imgid=main.history.imgid)",
1103 "[init] can't update `num' with num+1\n");
1104
1105 // update history end
1106 TRY_EXEC("UPDATE main.images SET history_end = history_end+1 WHERE id IN (SELECT imgid FROM main.mask WHERE main.mask.imgid=main.images.id)",
1107 "[init] can't update `history_end' with history_end+1\n");
1108
1109 // copy all masks into history
1110 TRY_EXEC("INSERT INTO main.masks_history (imgid, num, formid, form, name, version, points, points_count, source) SELECT "
1111 "imgid, 0, formid, form, name, version, points, points_count, source FROM main.mask",
1112 "[init] can't insert into masks_history\n");
1113
1114 // create a mask manager entry for each image that has masks
1115 TRY_EXEC("INSERT INTO main.history (imgid, num, operation, op_params, module, enabled, "
1116 "blendop_params, blendop_version, multi_priority, multi_name) "
1117 "SELECT DISTINCT imgid, 0, 'mask_manager', NULL, 1, 0, NULL, 0, 0, '' FROM main.mask "
1118 "GROUP BY imgid",
1119 "[init] can't insert mask manager into history\n");
1120
1121 TRY_EXEC("DROP TABLE main.mask", "[init] can't drop table `mask' from database\n");
1122
1124 GList *prior_v1 = dt_ioppr_get_iop_order_list_version(DT_IOP_ORDER_LEGACY);
1125
1126 TRY_EXEC("ALTER TABLE main.images ADD COLUMN iop_order_version INTEGER",
1127 "[init] can't add `iop_order_version' column to images table in database\n");
1128
1129 TRY_EXEC("UPDATE main.images SET iop_order_version = 0",
1130 "[init] can't update iop_order_version in database\n");
1131
1132 TRY_EXEC("UPDATE main.images SET iop_order_version = 1 WHERE "
1133 "EXISTS(SELECT * FROM main.history WHERE main.history.imgid = main.images.id)",
1134 "[init] can't update iop_order_version in database\n");
1135
1136 TRY_EXEC("ALTER TABLE main.history ADD COLUMN iop_order REAL",
1137 "[init] can't add `iop_order' column to history table in database\n");
1138
1139 // create a temp table with the previous priorities
1140 TRY_EXEC("CREATE TEMPORARY TABLE iop_order_tmp (iop_order REAL, operation VARCHAR(256))",
1141 "[init] can't create temporary table for updating `main.history'\n");
1142 // clang-format on
1143 // fill temp table with all operations up to this release
1144 // it will be used to create the pipe and update the iop_order on history
1145 for(GList *priorities = prior_v1; priorities; priorities = g_list_next(priorities))
1146 {
1147 dt_iop_order_entry_t *prior = (dt_iop_order_entry_t *)priorities->data;
1148
1149 sqlite3_prepare_v2(
1150 db->handle,
1151 "INSERT INTO iop_order_tmp (iop_order, operation) VALUES (?1, ?2)",
1152 -1, &stmt, NULL);
1153 sqlite3_bind_double(stmt, 1, prior->o.iop_order_f);
1154 sqlite3_bind_text(stmt, 2, prior->operation, -1, SQLITE_TRANSIENT);
1155 TRY_STEP(stmt, SQLITE_DONE, "[init] can't insert default value in iop_order_tmp\n");
1156 sqlite3_finalize(stmt);
1157 }
1158 g_list_free_full(prior_v1, dt_free_gpointer);
1159 prior_v1 = NULL;
1160
1161 // create the order of the pipe
1162 // iop_order is by default the module priority
1163 // if there's multi-instances we add the multi_priority
1164 // multi_priority is in reverse order in this version,
1165 // so we assume that is always less than 1000 and reverse it
1166 // it is possible that multi_priority = 0 don't appear in history
1167 // so just in case 1 / 1000 to every instance
1168 // clang-format off
1169 TRY_EXEC("UPDATE main.history SET iop_order = ((("
1170 "SELECT MAX(multi_priority) FROM main.history hist1 WHERE hist1.imgid = main.history.imgid AND hist1.operation = main.history.operation "
1171 ") + 1. - multi_priority) / 1000.) + "
1172 "IFNULL((SELECT iop_order FROM iop_order_tmp WHERE iop_order_tmp.operation = "
1173 "main.history.operation), -999999.) ",
1174 "[init] can't update iop_order in history table\n");
1175
1176 // check if there's any entry in history that was not updated
1177 sqlite3_stmt *sel_stmt;
1178 TRY_PREPARE(sel_stmt, "SELECT DISTINCT operation FROM main.history WHERE iop_order <= 0 OR iop_order IS NULL",
1179 "[init] can't prepare selecting history iop_order\n");
1180 // clang-format on
1181 while(sqlite3_step(sel_stmt) == SQLITE_ROW)
1182 {
1183 const char *op_name = (const char *)sqlite3_column_text(sel_stmt, 0);
1184 printf("operation %s with no iop_order while upgrading database\n", op_name);
1185 }
1186 sqlite3_finalize(sel_stmt);
1187 // clang-format off
1188 TRY_EXEC("DROP TABLE iop_order_tmp", "[init] can't drop table `iop_order_tmp' from database\n");
1189 // clang-format on
1190 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1191 new_version = 18;
1192 }
1193 // maybe in the future, see commented out code elsewhere
1194 // else if(version == XXX)
1195 // {
1196 // sqlite3_exec(db->handle, "ALTER TABLE film_rolls ADD COLUMN external_drive VARCHAR(1024)", NULL,
1197 // NULL, NULL);
1198 // }
1199 else if(version == 18)
1200 {
1201 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1202 // clang-format off
1203 TRY_EXEC("UPDATE images SET orientation=-2 WHERE orientation=1;",
1204 "[init] can't update images orientation 1 from database\n");
1205
1206 TRY_EXEC("UPDATE images SET orientation=1 WHERE orientation=2;",
1207 "[init] can't update images orientation 2 from database\n");
1208
1209 TRY_EXEC("UPDATE images SET orientation=-6 WHERE orientation=5;",
1210 "[init] can't update images orientation 5 from database\n");
1211
1212 TRY_EXEC("UPDATE images SET orientation=5 WHERE orientation=6;",
1213 "[init] can't update images orientation 6 from database\n");
1214
1215 TRY_EXEC("UPDATE images SET orientation=2 WHERE orientation=-2;",
1216 "[init] can't update images orientation -1 from database\n");
1217
1218 TRY_EXEC("UPDATE images SET orientation=6 WHERE orientation=-6;",
1219 "[init] can't update images orientation -6 from database\n");
1220 // clang-format on
1221 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1222 new_version = 19;
1223 }
1224 else if(version == 19)
1225 {
1226 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1227 // clang-format off
1228 // create a temp table to invert all multi_priority
1229 TRY_EXEC("CREATE TEMPORARY TABLE m_prio (id INTEGER, operation VARCHAR(256), prio INTEGER)",
1230 "[init] can't create temporary table for updating `history and style_items'\n");
1231
1232 TRY_EXEC("CREATE INDEX m_prio_id_index ON m_prio (id)",
1233 "[init] can't create temporary index for updating `history and style_items'\n");
1234 TRY_EXEC("CREATE INDEX m_prio_op_index ON m_prio (operation)",
1235 "[init] can't create temporary index for updating `history and style_items'\n");
1236
1237 TRY_EXEC("INSERT INTO m_prio SELECT imgid, operation, MAX(multi_priority)"
1238 " FROM main.history GROUP BY imgid, operation",
1239 "[init] can't populate m_prio\n");
1240
1241 TRY_EXEC("UPDATE main.history SET multi_priority = "
1242 "(SELECT prio FROM m_prio "
1243 " WHERE main.history.operation = operation AND main.history.imgid = id) - main.history.multi_priority",
1244 "[init] can't update multi_priority for history\n");
1245
1246 TRY_EXEC("DROP TABLE m_prio", "[init] can't drop table `m_prio' from database\n");
1247 // clang-format on
1248 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1249 new_version = 20;
1250 }
1251 else if(version == 20)
1252 {
1253 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1254 // clang-format off
1255 TRY_EXEC("DROP INDEX IF EXISTS main.used_tags_idx", "[init] can't drop index `used_tags_idx' from database\n");
1256 TRY_EXEC("DROP TABLE used_tags", "[init] can't delete table used_tags\n");
1257 // clang-format on
1258 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1259
1260 new_version = 21;
1261 }
1262 else if(version == 21)
1263 {
1264 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1265 // create a temp table to invert all multi_priority
1266 // clang-format off
1267 TRY_EXEC("CREATE TABLE module_order (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR)",
1268 "[init] can't create module_order table'\n");
1269
1270 // for all images:
1271 sqlite3_stmt *mig_stmt;
1272 TRY_PREPARE(mig_stmt, "SELECT imgid, operation, multi_priority, iop_order, mi.iop_order_version"
1273 " FROM main.history AS hi, main.images AS mi"
1274 " WHERE hi.imgid = mi.id"
1275 " GROUP BY imgid, operation, multi_priority"
1276 " ORDER BY imgid, iop_order",
1277 "[init] can't prepare selecting history for iop_order migration (v21)\n");
1278 // clang-format on
1279 GList *item_list = NULL;
1280 int current_imgid = UNKNOWN_IMAGE;
1281 int current_order_version = -1;
1282
1283 gboolean has_row = (sqlite3_step(mig_stmt) == SQLITE_ROW);
1284
1285 while(has_row)
1286 {
1287 const int32_t imgid = sqlite3_column_int(mig_stmt, 0);
1288 char operation[20] = { 0 };
1289 g_strlcpy(operation, (const char *)sqlite3_column_text(mig_stmt, 1), sizeof(operation));
1290 const int multi_priority = sqlite3_column_int(mig_stmt, 2);
1291 const double iop_order = sqlite3_column_double(mig_stmt, 3);
1292 const int iop_order_version = sqlite3_column_int(mig_stmt, 4);
1293
1294 has_row = (sqlite3_step(mig_stmt) == SQLITE_ROW);
1295
1296 // a new image, let's initialize the iop_order_version
1297 if(imgid != current_imgid || !has_row)
1298 {
1299 // new image, let's handle it
1300 if(!IS_NULL_PTR(item_list))
1301 {
1302 // we keep legacy, everything else is migrated to v3.0
1303 const dt_iop_order_t new_order_version = current_order_version == 2 ? DT_IOP_ORDER_LEGACY : DT_IOP_ORDER_ANSEL_RAW;
1304
1305 GList *iop_order_list = dt_ioppr_get_iop_order_list_version(new_order_version);
1306
1307 // merge entries into iop_order_list
1308
1309 // first remove all item_list iop from the iop_order_list
1310
1311 GList *e = item_list;
1312 GList *n = NULL;
1313 dt_iop_order_entry_t *n_entry = NULL;
1314
1315 while(e)
1316 {
1317 dt_iop_order_entry_t *e_entry = (dt_iop_order_entry_t *)e->data;
1318
1319 GList *s = iop_order_list;
1320 while(s && strcmp(((dt_iop_order_entry_t *)s->data)->operation, e_entry->operation))
1321 {
1322 s = g_list_next(s);
1323 }
1324 if(s)
1325 {
1326 iop_order_list = g_list_delete_link(iop_order_list, s);
1327 }
1328
1329 // skip all multipe instances
1330 n = e;
1331 do
1332 {
1333 n = g_list_next(n);
1334 if(IS_NULL_PTR(n)) break;
1335 n_entry = (dt_iop_order_entry_t *)n->data;
1336 } while(!strcmp(n_entry->operation, e_entry->operation));
1337 e = n;
1338 }
1339
1340 // then add all item_list into iop_order_list
1341
1342 for(e = item_list; e; e = g_list_next(e))
1343 {
1344 dt_iop_order_entry_t *e_entry = (dt_iop_order_entry_t *)e->data;
1345 iop_order_list = g_list_prepend(iop_order_list, e_entry);
1346 }
1347
1348 // and finally reorder the full list based on the iop-order
1349
1350 iop_order_list = g_list_sort(iop_order_list, dt_sort_iop_list_by_order_f);
1351
1352 const dt_iop_order_t kind = dt_ioppr_get_iop_order_list_kind(iop_order_list);
1353
1354 // check if we have some multi-instances
1355
1356 gboolean has_multiple_instances = FALSE;
1357 GList *l = iop_order_list;
1358
1359 while(l)
1360 {
1361 GList *next = g_list_next(l);
1362 if(next
1363 && (strcmp(((dt_iop_order_entry_t *)(l->data))->operation,
1364 ((dt_iop_order_entry_t *)(next->data))->operation) == 0))
1365 {
1366 has_multiple_instances = TRUE;
1367 break;
1368 }
1369 l = next;
1370 }
1371
1372 // write iop_order_list and/or version into module_order
1373
1374 sqlite3_stmt *ins_stmt = NULL;
1375 if(kind == DT_IOP_ORDER_CUSTOM || has_multiple_instances)
1376 {
1377 char *iop_list_txt = dt_ioppr_serialize_text_iop_order_list(iop_order_list);
1378
1379 sqlite3_prepare_v2(db->handle,
1380 "INSERT INTO module_order VALUES (?1, ?2, ?3)", -1,
1381 &ins_stmt, NULL);
1382 sqlite3_bind_int(ins_stmt, 1, current_imgid);
1383 sqlite3_bind_int(ins_stmt, 2, kind);
1384 sqlite3_bind_text(ins_stmt, 3, iop_list_txt, -1, SQLITE_TRANSIENT);
1385 TRY_STEP(ins_stmt, SQLITE_DONE, "[init] can't insert into module_order (custom order)\n");
1386 sqlite3_finalize(ins_stmt);
1387
1388 dt_free(iop_list_txt);
1389 }
1390 else
1391 {
1392 sqlite3_prepare_v2(db->handle,
1393 "INSERT INTO module_order VALUES (?1, ?2, NULL)", -1,
1394 &ins_stmt, NULL);
1395 sqlite3_bind_int(ins_stmt, 1, current_imgid);
1396 sqlite3_bind_int(ins_stmt, 2, kind);
1397 TRY_STEP(ins_stmt, SQLITE_DONE, "[init] can't insert into module_order (standard order)\n");
1398 sqlite3_finalize(ins_stmt);
1399 }
1400
1401 g_list_free(item_list);
1402 item_list = NULL;
1403 g_list_free_full(iop_order_list, dt_free_gpointer);
1404 iop_order_list = NULL;
1405
1406 item_list = NULL;
1407 }
1408
1409 current_imgid = imgid;
1410 current_order_version = iop_order_version;
1411 }
1412
1413 dt_iop_order_entry_t *item = (dt_iop_order_entry_t *)malloc(sizeof(dt_iop_order_entry_t));
1414 memcpy(item->operation, operation, sizeof(item->operation));
1415 item->instance = multi_priority;
1416 item->o.iop_order_f = iop_order; // used to order the enties only
1417 item_list = g_list_append(item_list, item);
1418 }
1419 sqlite3_finalize(mig_stmt);
1420
1421 // remove iop_order from history table
1422 // clang-format off
1423 TRY_EXEC("CREATE TABLE h (imgid INTEGER, num INTEGER, module INTEGER, "
1424 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
1425 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
1426 "[init] can't create module_order table\n");
1427 TRY_EXEC("CREATE INDEX h_imgid_index ON h (imgid)",
1428 "[init] can't create index h_imgid_index\n");
1429 TRY_EXEC("INSERT INTO h SELECT imgid, num, module, operation, op_params, enabled, "
1430 "blendop_params, blendop_version, multi_priority, multi_name FROM main.history",
1431 "[init] can't create module_order table\n");
1432 TRY_EXEC("DROP TABLE history",
1433 "[init] can't drop table history\n");
1434 TRY_EXEC("ALTER TABLE h RENAME TO history",
1435 "[init] can't rename h to history\n");
1436 TRY_EXEC("DROP INDEX h_imgid_index",
1437 "[init] can't drop index h_imgid_index\n");
1438 TRY_EXEC("CREATE INDEX main.history_imgid_index ON history (imgid)",
1439 "[init] can't create index images_imgid_index\n");
1440
1441 // remove iop_order_version from images
1442
1443 TRY_EXEC("CREATE TABLE i (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1444 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
1445 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
1446 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
1447 "output_width INTEGER, output_height INTEGER, crop REAL, "
1448 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
1449 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
1450 "caption VARCHAR, description VARCHAR, license VARCHAR, sha1sum CHAR(40), "
1451 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
1452 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
1453 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, aspect_ratio REAL)",
1454 "[init] can't create table i\n");
1455 TRY_EXEC("INSERT INTO i SELECT id, group_id, film_id, width, height, filename, maker, model,"
1456 " lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags,"
1457 " output_width, output_height, crop, raw_parameters, raw_denoise_threshold,"
1458 " raw_auto_bright_threshold, raw_black, raw_maximum, caption, description, license, sha1sum,"
1459 " orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version,"
1460 " max_version, write_timestamp, history_end, position, aspect_ratio "
1461 "FROM images",
1462 "[init] can't populate table h\n");
1463 TRY_EXEC("DROP TABLE images",
1464 "[init] can't drop table images\n");
1465 TRY_EXEC("ALTER TABLE i RENAME TO images",
1466 "[init] can't rename i to images\n");
1467 // clang-format on
1468 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1469
1470 new_version = 22;
1471 }
1472 else if(version == 22)
1473 {
1474 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1475 // clang-format off
1476 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_group_id_index ON images (group_id)",
1477 "[init] can't create group_id index on image\n");
1478 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_film_id_index ON images (film_id)",
1479 "[init] can't create film_id index on image\n");
1480 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_filename_index ON images (filename)",
1481 "[init] can't create filename index on image\n");
1482 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.image_position_index ON images (position)",
1483 "[init] can't create position index on image\n");
1484
1485 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.film_rolls_folder_index ON film_rolls (folder)",
1486 "[init] can't create folder index on film_rolls\n");
1487 // clang-format on
1488 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1489
1490 new_version = 23;
1491 }
1492 else if(version == 23)
1493 {
1494 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1495 // clang-format off
1496 TRY_EXEC("CREATE TABLE main.history_hash (imgid INTEGER PRIMARY KEY, "
1497 "basic_hash BLOB, auto_hash BLOB, current_hash BLOB)",
1498 "[init] can't create table history_hash\n");
1499 // clang-format on
1500
1501 // history_hash backfill is deferred: it will be filled through the image cache.
1502
1503 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1504
1505 new_version = 24;
1506 }
1507 else if(version == 24)
1508 {
1509 // clang-format off
1510 TRY_EXEC("ALTER TABLE main.history_hash ADD COLUMN mipmap_hash BLOB",
1511 "[init] can't add `mipmap_hash' column to history_hash table in database\n");
1512 // clang-format on
1513
1514 new_version = 25;
1515 }
1516 else if(version == 25)
1517 {
1518 // clang-format of
1519 TRY_EXEC("ALTER TABLE main.images ADD COLUMN exposure_bias REAL",
1520 "[init] can't add `exposure_bias' column to images table in database\n");
1521 // clang-format on
1522
1523 new_version = 26;
1524 }
1525 else if(version == 26)
1526 {
1527 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1528 // clang-format off
1529 TRY_EXEC("CREATE TABLE main.new_film_rolls "
1530 "(id INTEGER PRIMARY KEY, "
1531 "access_timestamp INTEGER, "
1532 "folder VARCHAR(1024) NOT NULL)",
1533 "[init] can't create new_film_rolls table\n");
1534
1535 TRY_EXEC("INSERT INTO main.new_film_rolls"
1536 "(id, access_timestamp, folder) "
1537 "SELECT id, "
1538 "strftime('%s', replace(substr(datetime_accessed, 1, 10), ':', '-') || substr(datetime_accessed, 11), 'utc'), "
1539 "folder "
1540 "FROM film_rolls "
1541 "WHERE folder IS NOT NULL",
1542 "[init] can't populate new_film_rolls table from film_rolls\n");
1543
1544 TRY_EXEC("DROP TABLE film_rolls",
1545 "[init] can't delete table film_rolls\n");
1546
1547 TRY_EXEC("ALTER TABLE main.new_film_rolls RENAME TO film_rolls",
1548 "[init] can't rename table new_film_rolls to film_rolls\n");
1549
1550 TRY_EXEC("CREATE INDEX main.film_rolls_folder_index ON film_rolls (folder)",
1551 "[init] can't create index `film_rolls_folder_index' on table `film_rolls'\n");
1552 // clang-format on
1553 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1554 new_version = 27;
1555 }
1556 else if(version == 27)
1557 {
1558 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1559 // clang-format off
1560 TRY_EXEC("ALTER TABLE main.images ADD COLUMN import_timestamp INTEGER DEFAULT -1",
1561 "[init] can't add `import_timestamp' column to images table in database\n");
1562 TRY_EXEC("ALTER TABLE main.images ADD COLUMN change_timestamp INTEGER DEFAULT -1",
1563 "[init] can't add `change_timestamp' column to images table in database\n");
1564 TRY_EXEC("ALTER TABLE main.images ADD COLUMN export_timestamp INTEGER DEFAULT -1",
1565 "[init] can't add `export_timestamp' column to images table in database\n");
1566 TRY_EXEC("ALTER TABLE main.images ADD COLUMN print_timestamp INTEGER DEFAULT -1",
1567 "[init] can't add `print_timestamp' column to images table in database\n");
1568
1569 TRY_EXEC("UPDATE main.images SET import_timestamp = (SELECT access_timestamp "
1570 "FROM main.film_rolls WHERE film_rolls.id = images.film_id)",
1571 "[init] can't populate import_timestamp column from film_rolls.access_timestamp.\n");
1572
1573 TRY_EXEC("UPDATE main.images SET change_timestamp = images.write_timestamp "
1574 "WHERE images.write_timestamp IS NOT NULL "
1575 "AND images.id = (SELECT imgid FROM tagged_images "
1576 "JOIN data.tags ON tags.id = tagged_images.tagid "
1577 "WHERE data.tags.name = 'darktable|changed')",
1578 "[init] can't populate change_timestamp column from images.write_timestamp.\n");
1579 // clang-format on
1580 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1581 new_version = 28;
1582 }
1583 else if(version == 28)
1584 {
1585 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1586 // clang-format off
1587 // clear flag DT_IMAGE_REJECTED (was not used)
1588 TRY_EXEC("UPDATE main.images SET flags = (flags & ~8)",
1589 "[init] can't clear rejected flags");
1590
1591 // add DT_IMAGE_REJECTED and clear rating for all images being rejected
1592 TRY_EXEC("UPDATE main.images SET flags = (flags | 8) & ~7 WHERE (flags & 7) = 6",
1593 "[init] can't set rejected flags");
1594 // clang-format on
1595 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1596 new_version = 29;
1597 }
1598 else if(version == 29)
1599 {
1600 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1601 // clang-format off
1602 // add position in tagged_images table
1603 TRY_EXEC("ALTER TABLE main.tagged_images ADD COLUMN position INTEGER",
1604 "[init] can't add `position' column to tagged_images table in database\n");
1605
1606 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.tagged_images_imgid_index ON tagged_images (imgid)",
1607 "[init] can't create image index on tagged_images\n");
1608 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.tagged_images_position_index ON tagged_images (position)",
1609 "[init] can't create position index on tagged_images\n");
1610 TRY_EXEC("UPDATE main.tagged_images SET position = (tagid + imgid) << 32",
1611 "[init] can't populate position on tagged_images\n");
1612
1613 // remove caption and description fields from images table
1614
1615 TRY_EXEC("CREATE TABLE main.i (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1616 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
1617 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
1618 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
1619 "output_width INTEGER, output_height INTEGER, crop REAL, "
1620 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
1621 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
1622 "license VARCHAR, sha1sum CHAR(40), "
1623 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
1624 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
1625 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
1626 "aspect_ratio REAL, exposure_bias REAL, "
1627 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, "
1628 "export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1)",
1629 "[init] can't create table i\n");
1630
1631 TRY_EXEC("INSERT INTO main.i SELECT id, group_id, film_id, width, height, filename, maker, model,"
1632 " lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags,"
1633 " output_width, output_height, crop, raw_parameters, raw_denoise_threshold,"
1634 " raw_auto_bright_threshold, raw_black, raw_maximum, license, sha1sum,"
1635 " orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version,"
1636 " max_version, write_timestamp, history_end, position, aspect_ratio, exposure_bias,"
1637 " import_timestamp, change_timestamp, export_timestamp, print_timestamp "
1638 "FROM main.images",
1639 "[init] can't populate table i\n");
1640 TRY_EXEC("DROP TABLE main.images",
1641 "[init] can't drop table images\n");
1642 TRY_EXEC("ALTER TABLE main.i RENAME TO images",
1643 "[init] can't rename i to images\n");
1644
1645 TRY_EXEC("CREATE INDEX main.images_group_id_index ON images (group_id)",
1646 "[init] can't create group_id index on images table\n");
1647 TRY_EXEC("CREATE INDEX main.images_film_id_index ON images (film_id)",
1648 "[init] can't create film_id index on images table\n");
1649 TRY_EXEC("CREATE INDEX main.images_filename_index ON images (filename)",
1650 "[init] can't create filename index on images table\n");
1651 TRY_EXEC("CREATE INDEX main.image_position_index ON images (position)",
1652 "[init] can't create position index on images table\n");
1653 // clang-format on
1654 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1655 new_version = 30;
1656 }
1657 else if(version == 30)
1658 {
1659 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1660 // clang-format off
1661 // add second columns to speed up sorting
1662 TRY_EXEC("DROP INDEX IF EXISTS `history_imgid_index`",
1663 "[init] can't drop history_imgid_index\n");
1664 TRY_EXEC("CREATE INDEX `history_imgid_index` ON `history` ( `imgid`, `operation` )",
1665 "[init] can't recreate history_imgid_index\n");
1666
1667 TRY_EXEC("DROP INDEX IF EXISTS `images_filename_index`",
1668 "[init] can't drop images_filename_index\n");
1669 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
1670 "[init] can't recreate images_filename_index\n");
1671
1672 TRY_EXEC("DROP INDEX IF EXISTS `images_film_id_index`",
1673 "[init] can't drop images_film_id_index\n");
1674 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
1675 "[init] can't recreate images_film_id_index\n");
1676
1677 TRY_EXEC("DROP INDEX IF EXISTS `images_group_id_index`",
1678 "[init] can't drop images_group_id_index\n");
1679 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
1680 "[init] can't recreate images_group_id_index\n");
1681
1682 TRY_EXEC("DROP INDEX IF EXISTS `masks_history_imgid_index`",
1683 "[init] can't drop masks_history_imgid_index\n");
1684 TRY_EXEC("CREATE INDEX `masks_history_imgid_index` ON `masks_history` ( `imgid`, `num` )",
1685 "[init] can't recreate masks_history_imgid_index\n");
1686
1687 // map refinement: avoid full table scan
1688 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( `latitude` DESC, `longitude` DESC )",
1689 "[init] can't create images_latlong_index\n");
1690 // clang-format on
1691 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1692 new_version = 31;
1693 }
1694 else if(version == 31)
1695 {
1696 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1697 // clang-format off
1698 // remove duplicates
1699 TRY_EXEC("DELETE FROM main.meta_data WHERE rowid NOT IN (SELECT MIN(rowid) "
1700 "FROM main.meta_data GROUP BY id, key)",
1701 "[init] can't remove duplicates from meta_data\n");
1702
1703 // recreate the index with UNIQUE option
1704 TRY_EXEC("DROP INDEX IF EXISTS metadata_index",
1705 "[init] can't drop metadata_index\n");
1706 TRY_EXEC("CREATE UNIQUE INDEX main.metadata_index ON meta_data (id, key)",
1707 "[init] can't create metadata_index\n");
1708 // clang-format on
1709 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1710 new_version = 32;
1711 }
1712 else if(version == 32)
1713 {
1714 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1715 // clang-format off
1716 // add foreign keys for database consistency. ON UPDATE CASCADE since you never know
1717 // if a future version will change image_id
1718 // Unfortunately sqlite does not support adding foreign keys to existing tables
1719 // so we have to rename the existing tables, recreate them and copy back the old values
1720 // images first
1721 // needs to delete orphaned entries
1722 TRY_EXEC("ALTER TABLE `images` RENAME TO `images_old`",
1723 "[init] can't rename images\n");
1724
1725 TRY_EXEC("CREATE TABLE `images` (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1726 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, lens VARCHAR, "
1727 "exposure REAL, aperture REAL, iso REAL, focal_length REAL, focus_distance REAL, datetime_taken CHAR(20), "
1728 "flags INTEGER, output_width INTEGER, output_height INTEGER, crop REAL, "
1729 "raw_parameters INTEGER, raw_denoise_threshold REAL, raw_auto_bright_threshold REAL, "
1730 "raw_black INTEGER, raw_maximum INTEGER, license VARCHAR, sha1sum CHAR(40), "
1731 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, latitude REAL, altitude REAL, "
1732 "color_matrix BLOB, colorspace INTEGER, version INTEGER, max_version INTEGER, write_timestamp INTEGER, "
1733 "history_end INTEGER, position INTEGER, aspect_ratio REAL, exposure_bias REAL, "
1734 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1, "
1735 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
1736 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
1737 "[init] can't create new images table\n");
1738
1739 // corner case: database inconsistency with images having invalid film id
1740 TRY_EXEC("DELETE FROM `images_old` WHERE film_id NOT IN (SELECT id FROM `film_rolls`)",
1741 "[init] can't delete images with invalid film id\n");
1742
1743 TRY_EXEC("UPDATE `images_old` SET group_id=id WHERE group_id NOT IN (SELECT id from `images_old`)",
1744 "[init] can't fix invalid group ids\n");
1745
1746 TRY_EXEC("INSERT INTO `images` SELECT * FROM `images_old`",
1747 "[init] can't copy back from images_old\n");
1748
1749 // pita: need to recreate index
1750 TRY_EXEC("DROP INDEX IF EXISTS `image_position_index`",
1751 "[init] can't drop image_position_index\n");
1752 TRY_EXEC("CREATE INDEX `image_position_index` ON `images` (position)",
1753 "[init] can't add image_position_index\n");
1754
1755 // second columns
1756 TRY_EXEC("DROP INDEX IF EXISTS `images_filename_index`",
1757 "[init] can't drop images_filename_index\n");
1758 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
1759 "[init] can't recreate images_filename_index\n");
1760
1761 TRY_EXEC("DROP INDEX IF EXISTS `images_film_id_index`",
1762 "[init] can't drop images_film_id_index\n");
1763 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
1764 "[init] can't recreate images_film_id_index\n");
1765
1766 TRY_EXEC("DROP INDEX IF EXISTS `images_group_id_index`",
1767 "[init] can't drop images_group_id_index\n");
1768 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
1769 "[init] can't recreate images_group_id_index\n");
1770
1771 TRY_EXEC("DROP INDEX IF EXISTS `images_latlong_index`",
1772 "[init] can't drop images_latlong_index\n");
1773 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( latitude DESC, longitude DESC )",
1774 "[init] can't add images_latlong_index\n");
1775
1776 TRY_EXEC("DROP TABLE `images_old`",
1777 "[init] can't drop table images_old\n");
1778
1779 // history
1780 TRY_EXEC("ALTER TABLE `history` RENAME TO `history_old`",
1781 "[init] can't rename history\n");
1782
1783 TRY_EXEC("CREATE TABLE `history` (imgid INTEGER, num INTEGER, module INTEGER, "
1784 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, "
1785 "multi_priority INTEGER, multi_name VARCHAR(256), "
1786 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1787 "[init] can't create new history table\n");
1788
1789 TRY_EXEC("DELETE FROM `history_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1790 "[init] can't delete orphaned history elements\n");
1791
1792 TRY_EXEC("INSERT INTO history SELECT * FROM history_old",
1793 "[init] can't copy back from history_old\n");
1794
1795 TRY_EXEC("DROP INDEX IF EXISTS `history_imgid_index`",
1796 "[init] can't drop history_imgid_index\n");
1797 TRY_EXEC("CREATE INDEX `history_imgid_op_index` ON `history` ( `imgid`, `operation` )",
1798 "[init] can't recreate history_imgid_index\n");
1799 TRY_EXEC("CREATE INDEX `history_imgid_num_index` ON `history` ( `imgid`, `num` DESC )",
1800 "[init] can't recreate history_imgid_index\n");
1801
1802 TRY_EXEC("DROP TABLE `history_old`",
1803 "[init] can't drop table history_old\n");
1804
1805 // history hash
1806 TRY_EXEC("ALTER TABLE `history_hash` RENAME TO `history_hash_old`",
1807 "[init] can't rename history_hash\n");
1808
1809 TRY_EXEC("CREATE TABLE `history_hash` (imgid INTEGER PRIMARY KEY, basic_hash BLOB, auto_hash BLOB, current_hash BLOB, "
1810 "mipmap_hash BLOB, FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1811 "[init] can't create new history_hash table\n");
1812
1813 TRY_EXEC("DELETE FROM `history_hash_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1814 "[init] can't delete orphaned history_hash elements\n");
1815
1816 TRY_EXEC("INSERT INTO `history_hash` SELECT * FROM `history_hash_old`",
1817 "[init] can't copy back from history_hash_old\n");
1818
1819 TRY_EXEC("DROP TABLE `history_hash_old`",
1820 "[init] can't drop table history_hash_old\n");
1821
1822 // tagged images
1823 TRY_EXEC("ALTER TABLE `tagged_images` RENAME TO `tagged_images_old`",
1824 "[init] can't rename tagged_images\n");
1825
1826 TRY_EXEC("CREATE TABLE `tagged_images` (imgid integer, tagid integer, position INTEGER, "
1827 "primary key(imgid, tagid), FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1828 "[init] can't create new tagged_images table\n");
1829
1830 TRY_EXEC("DELETE FROM `tagged_images_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1831 "[init] can't delete orphaned tagged_images elements\n");
1832
1833 TRY_EXEC("INSERT INTO `tagged_images` SELECT * FROM `tagged_images_old`",
1834 "[init] can't copy back from tagged_images_old\n");
1835
1836 // old indices
1837 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_imgid_index",
1838 "[init] can't drop tagged_images_imgid_index\n");
1839 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_position_index",
1840 "[init] can't drop tagged_images_position_index\n");
1841 TRY_EXEC("CREATE INDEX tagged_images_position_index ON tagged_images (position)",
1842 "[init] can't add index tagged_images_position_index\n");
1843 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_tagid_index",
1844 "[init] can't drop tagged_images_tagid_index\n");
1845 TRY_EXEC("CREATE INDEX tagged_images_tagid_index ON tagged_images (tagid)",
1846 "[init] can't add index tagged_images_tagid_index\n");
1847
1848 TRY_EXEC("DROP TABLE `tagged_images_old`",
1849 "[init] can't drop table tagged_images_old\n");
1850
1851 // masks history
1852 TRY_EXEC("ALTER TABLE `masks_history` RENAME TO `masks_history_old`",
1853 "[init] can't rename masks_history\n");
1854
1855 TRY_EXEC("CREATE TABLE masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, "
1856 "name VARCHAR(256), version INTEGER, points BLOB, points_count INTEGER, source BLOB, "
1857 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1858 "[init] can't create new masks_history table\n");
1859
1860 TRY_EXEC("DELETE FROM `masks_history_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1861 "[init] can't delete orphaned masks_history elements\n");
1862
1863 TRY_EXEC("INSERT INTO `masks_history` SELECT * FROM `masks_history_old`",
1864 "[init] can't copy back from masks_history\n");
1865
1866 TRY_EXEC("DROP INDEX IF EXISTS `masks_history_imgid_index`",
1867 "[init] can't drop masks_history_imgid_index\n");
1868 TRY_EXEC("CREATE INDEX `masks_history_imgid_index` ON `masks_history` ( imgid, num )",
1869 "[init] can't recreate masks_history_imgid_index\n");
1870
1871 TRY_EXEC("DROP TABLE masks_history_old",
1872 "[init] can't drop table masks_history_old\n");
1873
1874 // color labels
1875 TRY_EXEC("ALTER TABLE `color_labels` RENAME TO `color_labels_old`",
1876 "[init] can't rename color_labels\n");
1877 TRY_EXEC("CREATE TABLE `color_labels` (imgid INTEGER, color INTEGER, "
1878 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1879 "[init] can't create new color_labels table\n");
1880
1881 TRY_EXEC("DELETE FROM `color_labels_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1882 "[init] can't delete orphaned color_labels elements\n");
1883
1884 TRY_EXEC("INSERT INTO `color_labels` SELECT * FROM `color_labels_old`",
1885 "[init] can't copy back from color_labels\n");
1886
1887 TRY_EXEC("DROP TABLE color_labels_old",
1888 "[init] can't drop table color_labels_old\n");
1889
1890 TRY_EXEC("CREATE UNIQUE INDEX `color_labels_idx` ON `color_labels` (imgid, color)",
1891 "[init] can't recreate color_labels_idx\n");
1892
1893 // meta data
1894 TRY_EXEC("ALTER TABLE `meta_data` RENAME TO `meta_data_old`",
1895 "[init] can't rename meta_data\n");
1896 TRY_EXEC("CREATE TABLE `meta_data` (id integer, key integer, value varchar, "
1897 "FOREIGN KEY(id) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1898 "[init] can't create new meta_data table\n");
1899
1900 TRY_EXEC("DELETE FROM `meta_data_old` WHERE id NOT IN (SELECT id FROM `images`)",
1901 "[init] can't delete orphaned meta_data elements\n");
1902
1903 TRY_EXEC("INSERT INTO `meta_data` SELECT * FROM `meta_data_old`",
1904 "[init] can't copy back from meta_data\n");
1905
1906 TRY_EXEC("DROP TABLE meta_data_old",
1907 "[init] can't drop table meta_data_old\n");
1908
1909 TRY_EXEC("CREATE UNIQUE INDEX `metadata_index` ON `meta_data` (id, key, value)",
1910 "[init] can't recreate metadata_index\n");
1911
1912 // selected images
1913 TRY_EXEC("ALTER TABLE `selected_images` RENAME TO `selected_images_old`",
1914 "[init] can't rename selected_images\n");
1915 TRY_EXEC("CREATE TABLE `selected_images` (imgid INTEGER PRIMARY KEY, "
1916 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1917 "[init] can't create new selected_images table\n");
1918
1919 TRY_EXEC("DELETE FROM `selected_images_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1920 "[init] can't delete orphaned selected_images elements\n");
1921
1922 TRY_EXEC("INSERT INTO `selected_images` SELECT * FROM `selected_images_old`",
1923 "[init] can't copy back selected_images meta_data\n");
1924
1925 TRY_EXEC("DROP TABLE selected_images_old",
1926 "[init] can't drop table selected_images_old\n");
1927
1928 // module order
1929 TRY_EXEC("ALTER TABLE `module_order` RENAME TO `module_order_old`",
1930 "[init] can't rename module_order\n");
1931 TRY_EXEC("CREATE TABLE `module_order` (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR, "
1932 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1933 "[init] can't create new module_order table\n");
1934
1935 TRY_EXEC("DELETE FROM `module_order_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1936 "[init] can't delete orphaned module_order elements\n");
1937
1938 TRY_EXEC("INSERT INTO `module_order` SELECT * FROM `module_order_old`",
1939 "[init] can't copy back module_order meta_data\n");
1940
1941 TRY_EXEC("DROP TABLE module_order_old",
1942 "[init] can't drop table module_order_old\n");
1943 // clang-format on
1944 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1945 new_version = 33;
1946 }
1947 else if(version == 33)
1948 {
1949 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1950
1951 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_datetime_taken_nc ON images (datetime_taken COLLATE NOCASE)",
1952 "[init] can't create images_datetime_taken\n");
1953 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.metadata_index_key ON meta_data (key)",
1954 "[init] can't create metadata_index_key\n");
1955
1956 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1957 new_version = 34;
1958 }
1959 else if(version == 34)
1960 {
1961 sqlite3_exec(db->handle, "PRAGMA foreign_keys = OFF", NULL, NULL, NULL);
1962 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1963
1964 TRY_EXEC("CREATE TABLE main.images_new (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1965 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
1966 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
1967 "focus_distance REAL, datetime_taken INTEGER, flags INTEGER, "
1968 "output_width INTEGER, output_height INTEGER, crop REAL, "
1969 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
1970 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
1971 "license VARCHAR, sha1sum CHAR(40), "
1972 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
1973 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
1974 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
1975 "aspect_ratio REAL, exposure_bias REAL, "
1976 "import_timestamp INTEGER, change_timestamp INTEGER, "
1977 "export_timestamp INTEGER, print_timestamp INTEGER, "
1978 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
1979 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
1980 "[init] can't create new images table\n");
1981
1982 TRY_EXEC("INSERT INTO `images_new` SELECT "
1983 "id, group_id, film_id, width, height, filename, maker, model, "
1984 "lens, exposure, aperture, iso, focal_length, focus_distance, NULL AS datetime_taken, flags, "
1985 "output_width, output_height, crop, raw_parameters, raw_denoise_threshold, raw_auto_bright_threshold, raw_black, raw_maximum, "
1986 "license, sha1sum, orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version, "
1987 "max_version, write_timestamp, history_end, position, aspect_ratio, exposure_bias, "
1988 "NULL AS import_timestamp, NULL AS change_timestamp, NULL AS export_timestamp, NULL AS print_timestamp "
1989 "FROM `images`",
1990 "[init] can't copy back from images\n");
1991
1992 TRY_PREPARE(stmt, "SELECT id,"
1993 " CASE WHEN datetime_taken = '' THEN NULL ELSE datetime_taken END,"
1994 " CASE WHEN import_timestamp = -1 THEN NULL ELSE import_timestamp END,"
1995 " CASE WHEN change_timestamp = -1 THEN NULL ELSE change_timestamp END,"
1996 " CASE WHEN export_timestamp = -1 THEN NULL ELSE export_timestamp END,"
1997 " CASE WHEN print_timestamp = -1 THEN NULL ELSE print_timestamp END "
1998 "FROM `images`",
1999 "[init] can't get datetime from images\n");
2000 while(sqlite3_step(stmt) == SQLITE_ROW)
2001 {
2002 sqlite3_stmt *stmt2;
2003 sqlite3_prepare_v2(db->handle,
2004 "UPDATE `images_new` SET"
2005 " (datetime_taken, import_timestamp,"
2006 " change_timestamp, export_timestamp, print_timestamp) = "
2007 " (?2, ?3, ?4, ?5, ?6) WHERE id = ?1",
2008 -1, &stmt2, NULL);
2009 sqlite3_bind_int(stmt2, 1, sqlite3_column_int(stmt, 0));
2010 if(sqlite3_column_type(stmt, 1) != SQLITE_NULL)
2011 {
2012 GDateTime *gdt = dt_datetime_exif_to_gdatetime((const char *)sqlite3_column_text(stmt, 1), darktable.utc_tz);
2013 if(gdt)
2014 {
2015 sqlite3_bind_int64(stmt2, 2, dt_datetime_gdatetime_to_gtimespan(gdt));
2016 g_date_time_unref(gdt);
2017 }
2018 }
2019 for(int i = 0; i < 4; i++)
2020 {
2021 if(sqlite3_column_type(stmt, i + 2) != SQLITE_NULL)
2022 {
2023 GDateTime *gdt = g_date_time_new_from_unix_utc(sqlite3_column_int(stmt, i + 2));
2024 if(gdt)
2025 {
2026 sqlite3_bind_int64(stmt2, i + 3, dt_datetime_gdatetime_to_gtimespan(gdt));
2027 g_date_time_unref(gdt);
2028 }
2029 }
2030 }
2031 TRY_STEP(stmt2, SQLITE_DONE, "[init] can't update datetimes into images_new table\n");
2032 sqlite3_finalize(stmt2);
2033 }
2034 sqlite3_finalize(stmt);
2035
2036 TRY_EXEC("DROP TABLE `images`", "[init] can't drop images table\n");
2037 // that's the way to keep the other tables foreign keys references valid
2038 TRY_EXEC("ALTER TABLE `images_new` RENAME TO `images`", "[init] can't rename images_new table to images");
2039
2040 // pita: need to recreate indexes
2041 TRY_EXEC("CREATE INDEX `image_position_index` ON `images` (position)",
2042 "[init] can't add image_position_index\n");
2043 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
2044 "[init] can't recreate images_filename_index\n");
2045 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
2046 "[init] can't recreate images_film_id_index\n");
2047 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
2048 "[init] can't recreate images_group_id_index\n");
2049 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( latitude DESC, longitude DESC )",
2050 "[init] can't add images_latlong_index\n");
2051 TRY_EXEC("CREATE INDEX `images_datetime_taken` ON images (datetime_taken)",
2052 "[init] can't create images_datetime_taken\n");
2053
2054 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2055 sqlite3_exec(db->handle, "PRAGMA foreign_keys = ON", NULL, NULL, NULL);
2056 new_version = 35;
2057 }
2058 else if(version == 35)
2059 {
2060 TRY_EXEC("CREATE TABLE main.images_new (id INTEGER, filename VARCHAR, flags INTEGER)",
2061 "[init] can't create new images table\n");
2062
2063 gchar *query = g_strdup_printf("INSERT INTO `images_new` "
2064 "SELECT id, filename, flags"
2065 " FROM images"
2066 " WHERE (flags & %d == 0)",
2067 DT_IMAGE_RAW | DT_IMAGE_LDR | DT_IMAGE_HDR);
2068 TRY_EXEC(query, "[init] can't copy back from images\n");
2069
2070 TRY_PREPARE(stmt, "SELECT id, filename, flags FROM `images_new`",
2071 "[init] can't prepare selecting images flags\n");
2072
2073 while(sqlite3_step(stmt) == SQLITE_ROW)
2074 {
2075 sqlite3_stmt *stmt2;
2076 sqlite3_prepare_v2(db->handle,
2077 "UPDATE `images` SET"
2078 " (flags) = "
2079 " (?2) WHERE id = ?1",
2080 -1, &stmt2, NULL);
2081 sqlite3_bind_int(stmt2, 1, sqlite3_column_int(stmt, 0));
2082
2083 dt_image_flags_t flags = sqlite3_column_int(stmt, 2);
2084 gchar *ext = g_strrstr((const char *)sqlite3_column_text(stmt, 1), ".");
2085 flags |= dt_imageio_get_type_from_extension(ext);
2086 sqlite3_bind_int(stmt2, 2, flags);
2087
2088 TRY_STEP(stmt2, SQLITE_DONE, "[init] can't update flags\n");
2089 sqlite3_finalize(stmt2);
2090 }
2091 sqlite3_finalize(stmt);
2092
2093 TRY_EXEC("DROP TABLE `images_new`", "[init] can't drop temp images table\n");
2094 new_version = 36;
2095 }
2096 else
2097 new_version = version; // should be the fallback so that calling code sees that we are in an infinite loop
2098
2099 // write the new version to db
2100 sqlite3_prepare_v2(db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', ?1)", -1, &stmt,
2101 NULL);
2102 sqlite3_bind_int(stmt, 1, new_version);
2103 sqlite3_step(stmt);
2104 sqlite3_finalize(stmt);
2105
2106 return new_version;
2107}
2108
2109/* do the real migration steps, returns the version the db was converted to */
2110static int _upgrade_data_schema_step(dt_database_t *db, int version)
2111{
2112 sqlite3_stmt *stmt;
2113 int new_version = version;
2114 if(version == CURRENT_DATABASE_VERSION_DATA)
2115 return version;
2116 else if(version == 0)
2117 {
2118 // this can't happen, we started with 1, but it's a good example how this function works
2119 // <do some magic to the db>
2120 new_version = 1; // the version we transformed the db to. this way it might be possible to roll back or
2121 // add fast paths
2122 }
2123 else if(version == 1)
2124 {
2125 // clang-format off
2126 // style_items:
2127 // NO TRY_EXEC has the column could be there before version 1 (master build)
2128 // TRY_EXEC("ALTER TABLE data.style_items ADD COLUMN iop_order REAL",
2129 // "[init] can't add `iop_order' column to style_items table in database\n");
2130 sqlite3_exec(db->handle, "ALTER TABLE data.style_items ADD COLUMN iop_order REAL", NULL, NULL, NULL);
2131 // clang-format on
2132 sqlite3_stmt *sel_stmt = NULL;
2133 GList *prior_v1 = dt_ioppr_get_iop_order_list_version(DT_IOP_ORDER_LEGACY);
2134 // clang-format off
2135 // create a temp table with the previous priorities
2136 TRY_EXEC("CREATE TEMPORARY TABLE iop_order_tmp (iop_order REAL, operation VARCHAR(256))",
2137 "[init] can't create temporary table for updating `data.style_items'\n");
2138 // clang-format on
2139 // fill temp table with all operations up to this release
2140 // it will be used to create the pipe and update the iop_order on history
2141 for(GList *priorities = prior_v1; priorities; priorities = g_list_next(priorities))
2142 {
2143 dt_iop_order_entry_t *prior = (dt_iop_order_entry_t *)priorities->data;
2144
2145 sqlite3_prepare_v2(
2146 db->handle,
2147 "INSERT INTO iop_order_tmp (iop_order, operation) VALUES (?1, ?2)",
2148 -1, &stmt, NULL);
2149 sqlite3_bind_double(stmt, 1, prior->o.iop_order_f);
2150 sqlite3_bind_text(stmt, 2, prior->operation, -1, SQLITE_TRANSIENT);
2151 TRY_STEP(stmt, SQLITE_DONE, "[init] can't insert default value in iop_order_tmp\n");
2152 sqlite3_finalize(stmt);
2153 }
2154 g_list_free_full(prior_v1, dt_free_gpointer);
2155 prior_v1 = NULL;
2156
2157 // do the same as for history
2158 // clang-format off
2159 TRY_EXEC("UPDATE data.style_items SET iop_order = ((("
2160 "SELECT MAX(multi_priority) FROM data.style_items style1 WHERE style1.styleid = data.style_items.styleid AND style1.operation = data.style_items.operation "
2161 ") + 1. - multi_priority) / 1000.) + "
2162 "IFNULL((SELECT iop_order FROM iop_order_tmp WHERE iop_order_tmp.operation = "
2163 "data.style_items.operation), -999999.) ",
2164 "[init] can't update iop_order in style_items table\n");
2165
2166 TRY_PREPARE(sel_stmt, "SELECT DISTINCT operation FROM data.style_items WHERE iop_order <= 0 OR iop_order IS NULL",
2167 "[init] can't prepare selecting style_items iop_order\n");
2168 // clang-format on
2169 while(sqlite3_step(sel_stmt) == SQLITE_ROW)
2170 {
2171 const char *op_name = (const char *)sqlite3_column_text(sel_stmt, 0);
2172 printf("operation %s with no iop_order while upgrading style_items in database\n", op_name);
2173 }
2174 sqlite3_finalize(sel_stmt);
2175 // clang-format off
2176 TRY_EXEC("DROP TABLE iop_order_tmp", "[init] can't drop table `iop_order_tmp' from database\n");
2177 // clang-format on
2178 new_version = 2;
2179 }
2180 else if(version == 2)
2181 {
2182 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2183
2184 // With sqlite above or equal to 3.25.0 RENAME COLUMN can be used instead of the following code
2185 // TRY_EXEC("ALTER TABLE data.tags RENAME COLUMN description TO synonyms;",
2186 // "[init] can't change tags column name from description to synonyms\n");
2187 // clang-format off
2188 TRY_EXEC("ALTER TABLE data.tags RENAME TO tmp_tags", "[init] can't rename table tags\n");
2189
2190 TRY_EXEC("CREATE TABLE data.tags (id INTEGER PRIMARY KEY, name VARCHAR, "
2191 "synonyms VARCHAR, flags INTEGER)",
2192 "[init] can't create new tags table\n");
2193
2194 TRY_EXEC("INSERT INTO data.tags (id, name, synonyms, flags) SELECT id, name, description, flags "
2195 "FROM tmp_tags",
2196 "[init] can't populate tags table from tmp_tags\n");
2197
2198 TRY_EXEC("DROP TABLE tmp_tags", "[init] can't delete table tmp_tags\n");
2199
2200 TRY_EXEC("CREATE UNIQUE INDEX data.tags_name_idx ON tags (name)",
2201 "[init] can't create tags_name_idx on tags table\n");
2202 // clang-format on
2203 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2204
2205 new_version = 3;
2206 }
2207 else if(version == 3)
2208 {
2209 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2210 // clang-format off
2211 // create a temp table to invert all multi_priority
2212 TRY_EXEC("CREATE TEMPORARY TABLE m_prio (id INTEGER, operation VARCHAR(256), prio INTEGER)",
2213 "[init] can't create temporary table for updating `history and style_items'\n");
2214
2215 TRY_EXEC("INSERT INTO m_prio SELECT styleid, operation, MAX(multi_priority)"
2216 " FROM data.style_items GROUP BY styleid, operation",
2217 "[init] can't populate m_prio\n");
2218
2219 // update multi_priority for style items and history
2220 TRY_EXEC("UPDATE data.style_items SET multi_priority = "
2221 "(SELECT prio FROM m_prio "
2222 " WHERE data.style_items.operation = operation AND data.style_items.styleid = id)"
2223 " - data.style_items.multi_priority",
2224 "[init] can't update multi_priority for style_items\n");
2225
2226 TRY_EXEC("DROP TABLE m_prio", "[init] can't drop table `m_prio' from database\n");
2227 // clang-format on
2228 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2229
2230 new_version = 4;
2231 }
2232 else if(version == 4)
2233 {
2234 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2235 // clang-format off
2236 // remove iop_order from style_item table
2237 TRY_EXEC("ALTER TABLE data.style_items RENAME TO s",
2238 "[init] can't rename style_items to s\n");
2239 TRY_EXEC("CREATE TABLE data.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
2240 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2241 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2242 "[init] can't create style_items table'\n");
2243 TRY_EXEC("INSERT INTO data.style_items SELECT styleid, num, module, operation, op_params, enabled, "
2244 " blendop_params, blendop_version, multi_priority, multi_name "
2245 "FROM s",
2246 "[init] can't populate style_items table'\n");
2247 TRY_EXEC("DROP TABLE s",
2248 "[init] can't drop table s'\n");
2249 // clang-format on
2250 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2251
2252 new_version = 5;
2253 }
2254 else if(version == 5)
2255 {
2256 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2257 // clang-format of
2258 // make style.id a PRIMARY KEY and add iop_list
2259 TRY_EXEC("ALTER TABLE data.styles RENAME TO s",
2260 "[init] can't rename styles to s\n");
2261 TRY_EXEC("CREATE TABLE data.styles (id INTEGER PRIMARY KEY, name VARCHAR, description VARCHAR, iop_list VARCHAR)",
2262 "[init] can't create styles table\n");
2263 TRY_EXEC("INSERT INTO data.styles SELECT id, name, description, NULL FROM s",
2264 "[init] can't populate styles table\n");
2265 TRY_EXEC("DROP TABLE s",
2266 "[init] can't drop table s\n");
2267
2268 TRY_EXEC("CREATE INDEX IF NOT EXISTS data.styles_name_index ON styles (name)",
2269 "[init] can't create styles_nmae_index\n");
2270
2271 // make style_items.styleid index
2272
2273 TRY_EXEC("CREATE INDEX IF NOT EXISTS data.style_items_styleid_index ON style_items (styleid)",
2274 "[init] can't create style_items_styleid_index\n");
2275 // clang-format on
2276 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2277
2278 new_version = 6;
2279 }
2280 else if(version == 6)
2281 {
2282 // clang-format off
2283 TRY_EXEC("CREATE TABLE data.locations "
2284 "(tagid INTEGER PRIMARY KEY, type INTEGER, longitude REAL, latitude REAL, "
2285 "delta1 REAL, delta2 REAL, FOREIGN KEY(tagid) REFERENCES tags(id))",
2286 "[init] can't create new locations table\n");
2287 // clang-format on
2288 new_version = 7;
2289 }
2290 else if(version == 7)
2291 {
2292 // clang-format off
2293 TRY_EXEC("ALTER TABLE data.locations ADD COLUMN ratio FLOAT DEFAULT 1",
2294 "[init] can't add column `ratio' column to locations table\n");
2295 // clang-format on
2296 new_version = 8;
2297 }
2298 else if(version == 8)
2299 {
2300 // clang-format off
2301 TRY_EXEC("ALTER TABLE data.locations ADD COLUMN polygons BLOB",
2302 "[init] can't add column `polygons' column to locations table\n");
2303 // clang-format on
2304 new_version = 9;
2305 }
2306 else
2307 new_version = version; // should be the fallback so that calling code sees that we are in an infinite loop
2308
2309 // write the new version to db
2310 // clang-format off¨
2311 sqlite3_prepare_v2(db->handle, "INSERT OR REPLACE INTO data.db_info (key, value) VALUES ('version', ?1)", -1, &stmt,
2312 NULL);
2313 // clang-format on
2314 sqlite3_bind_int(stmt, 1, new_version);
2315 sqlite3_step(stmt);
2316 sqlite3_finalize(stmt);
2317
2318 return new_version;
2319}
2320
2321#undef FINALIZE
2322
2323#undef TRY_EXEC
2324#undef TRY_STEP
2325#undef TRY_PREPARE
2326
2327/* upgrade library db from 'version' to CURRENT_DATABASE_VERSION_LIBRARY. don't touch this function but
2328 * _upgrade_library_schema_step() instead. */
2329static gboolean _upgrade_library_schema(dt_database_t *db, int version)
2330{
2331 while(version < CURRENT_DATABASE_VERSION_LIBRARY)
2332 {
2333 const int new_version = _upgrade_library_schema_step(db, version);
2334 if(new_version == version)
2335 return FALSE; // we don't know how to upgrade this db. probably a bug in _upgrade_library_schema_step
2336 else
2337 version = new_version;
2338 }
2339 return TRUE;
2340}
2341
2342/* upgrade data db from 'version' to CURRENT_DATABASE_VERSION_DATA. don't touch this function but
2343 * _upgrade_data_schema_step() instead. */
2344static gboolean _upgrade_data_schema(dt_database_t *db, int version)
2345{
2346 while(version < CURRENT_DATABASE_VERSION_DATA)
2347 {
2348 const int new_version = _upgrade_data_schema_step(db, version);
2349 if(new_version == version)
2350 return FALSE; // we don't know how to upgrade this db. probably a bug in _upgrade_data_schema_step
2351 else
2352 version = new_version;
2353 }
2354 return TRUE;
2355}
2356
2357/* create the current database schema and set the version in db_info accordingly */
2358static void _create_library_schema(dt_database_t *db)
2359{
2360 sqlite3_stmt *stmt;
2362 // clang-format off
2363 sqlite3_exec(db->handle, "CREATE TABLE main.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)", NULL,
2364 NULL, NULL);
2365 sqlite3_prepare_v2(
2366 db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', ?1)", -1, &stmt, NULL);
2367 // clang-format on
2368 sqlite3_bind_int(stmt, 1, CURRENT_DATABASE_VERSION_LIBRARY);
2369 sqlite3_step(stmt);
2370 sqlite3_finalize(stmt);
2372 // clang-format off
2373 sqlite3_exec(db->handle,
2374 "CREATE TABLE main.film_rolls "
2375 "(id INTEGER PRIMARY KEY, access_timestamp INTEGER, "
2376 // "folder VARCHAR(1024), external_drive VARCHAR(1024))", //
2377 // FIXME: make sure to bump CURRENT_DATABASE_VERSION_LIBRARY and add a
2378 // case to _upgrade_library_schema_step when adding this!
2379 "folder VARCHAR(1024) NOT NULL)",
2380 NULL, NULL, NULL);
2381 sqlite3_exec(db->handle, "CREATE INDEX main.film_rolls_folder_index ON film_rolls (folder)", NULL, NULL, NULL);
2383 sqlite3_exec(
2384 db->handle,
2385 "CREATE TABLE main.images (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
2386 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
2387 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
2388 "focus_distance REAL, datetime_taken INTEGER, flags INTEGER, "
2389 "output_width INTEGER, output_height INTEGER, crop REAL, "
2390 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
2391 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
2392 "license VARCHAR, sha1sum CHAR(40), "
2393 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
2394 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
2395 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
2396 "aspect_ratio REAL, exposure_bias REAL, "
2397 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, "
2398 "export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1, "
2399 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
2400 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
2401 NULL, NULL, NULL);
2402 sqlite3_exec(db->handle, "CREATE INDEX main.images_group_id_index ON images (group_id, id)", NULL, NULL, NULL);
2403 sqlite3_exec(db->handle, "CREATE INDEX main.images_film_id_index ON images (film_id, filename)", NULL, NULL, NULL);
2404 sqlite3_exec(db->handle, "CREATE INDEX main.images_filename_index ON images (filename, version)", NULL, NULL, NULL);
2405 sqlite3_exec(db->handle, "CREATE INDEX main.image_position_index ON images (position)", NULL, NULL, NULL);
2406 sqlite3_exec(db->handle, "CREATE INDEX main.images_datetime_taken_nc ON images (datetime_taken)", NULL, NULL, NULL);
2407
2409 sqlite3_exec(db->handle, "CREATE TABLE main.selected_images (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2411 sqlite3_exec(
2412 db->handle,
2413 "CREATE TABLE main.history (imgid INTEGER, num INTEGER, module INTEGER, "
2414 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2415 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256), "
2416 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2417 NULL, NULL, NULL);
2418 sqlite3_exec(db->handle, "CREATE INDEX main.history_imgid_op_index ON history (imgid, operation)", NULL, NULL, NULL);
2419 sqlite3_exec(db->handle, "CREATE INDEX main.history_imgid_num_index ON history (imgid, num DESC)", NULL, NULL, NULL);
2421 sqlite3_exec(db->handle,
2422 "CREATE TABLE main.masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, name VARCHAR(256), "
2423 "version INTEGER, points BLOB, points_count INTEGER, source BLOB, "
2424 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2425 NULL, NULL, NULL);
2426
2427 sqlite3_exec(db->handle,
2428 "CREATE INDEX main.masks_history_imgid_index ON masks_history (imgid, num)",
2429 NULL, NULL, NULL);
2430
2431 sqlite3_exec(db->handle, "CREATE INDEX main.images_latlong_index ON images (latitude DESC, longitude DESC)",
2432 NULL, NULL, NULL);
2433
2435 sqlite3_exec(db->handle, "CREATE TABLE main.tagged_images (imgid INTEGER, tagid INTEGER, position INTEGER, "
2436 "PRIMARY KEY (imgid, tagid),"
2437 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)", NULL, NULL, NULL);
2438 sqlite3_exec(db->handle, "CREATE INDEX main.tagged_images_tagid_index ON tagged_images (tagid)", NULL, NULL, NULL);
2439 sqlite3_exec(db->handle, "CREATE INDEX main.tagged_images_position_index ON tagged_images (position)", NULL, NULL, NULL);
2441 sqlite3_exec(db->handle, "CREATE TABLE main.color_labels (imgid INTEGER, color INTEGER)", NULL, NULL, NULL);
2442 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX main.color_labels_idx ON color_labels (imgid, color)", NULL, NULL,
2443 NULL);
2445 sqlite3_exec(db->handle, "CREATE TABLE main.meta_data (id INTEGER, key INTEGER, value VARCHAR)", NULL, NULL, NULL);
2446 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX main.metadata_index ON meta_data (id, key, value)", NULL, NULL, NULL);
2447
2448 sqlite3_exec(db->handle, "CREATE INDEX main.metadata_index_key ON meta_data (key)", NULL, NULL, NULL);
2449 sqlite3_exec(db->handle, "CREATE TABLE main.module_order (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR)",
2450 NULL, NULL, NULL);
2451 sqlite3_exec(db->handle, "CREATE TABLE main.history_hash (imgid INTEGER PRIMARY KEY, "
2452 "basic_hash BLOB, auto_hash BLOB, current_hash BLOB, mipmap_hash BLOB, "
2453 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2454 NULL, NULL, NULL);
2455
2456 // v34
2457 sqlite3_exec(db->handle, "CREATE INDEX main.images_datetime_taken_nc ON images (datetime_taken COLLATE NOCASE)",
2458 NULL, NULL, NULL);
2459 sqlite3_exec(db->handle, "CREATE INDEX main.metadata_index_key ON meta_data (key)", NULL, NULL, NULL);
2460 // clang-format on
2461}
2462
2463/* create the current database schema and set the version in db_info accordingly */
2464static void _create_data_schema(dt_database_t *db)
2465{
2466 sqlite3_stmt *stmt;
2467 // clang-format off
2469 sqlite3_exec(db->handle, "CREATE TABLE data.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)", NULL,
2470 NULL, NULL);
2471 sqlite3_prepare_v2(
2472 db->handle, "INSERT OR REPLACE INTO data.db_info (key, value) VALUES ('version', ?1)", -1, &stmt, NULL);
2473 sqlite3_bind_int(stmt, 1, CURRENT_DATABASE_VERSION_DATA);
2474 sqlite3_step(stmt);
2475 sqlite3_finalize(stmt);
2477 sqlite3_exec(db->handle, "CREATE TABLE data.tags (id INTEGER PRIMARY KEY, name VARCHAR, "
2478 "synonyms VARCHAR, flags INTEGER)", NULL, NULL, NULL);
2479 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX data.tags_name_idx ON tags (name)", NULL, NULL, NULL);
2481 sqlite3_exec(db->handle, "CREATE TABLE data.styles (id INTEGER PRIMARY KEY, name VARCHAR, description VARCHAR, iop_list VARCHAR)",
2482 NULL, NULL, NULL);
2483 sqlite3_exec(db->handle, "CREATE INDEX data.styles_name_index ON styles (name)", NULL, NULL, NULL);
2485 sqlite3_exec(
2486 db->handle,
2487 "CREATE TABLE data.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
2488 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2489 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2490 NULL, NULL, NULL);
2491 sqlite3_exec(
2492 db->handle,
2493 "CREATE INDEX IF NOT EXISTS data.style_items_styleid_index ON style_items (styleid)",
2494 NULL, NULL, NULL);
2496 sqlite3_exec(db->handle, "CREATE TABLE data.presets (name VARCHAR, description VARCHAR, operation "
2497 "VARCHAR, op_version INTEGER, op_params BLOB, "
2498 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, "
2499 "multi_priority INTEGER, multi_name VARCHAR(256), "
2500 "model VARCHAR, maker VARCHAR, lens VARCHAR, iso_min REAL, iso_max REAL, "
2501 "exposure_min REAL, exposure_max REAL, "
2502 "aperture_min REAL, aperture_max REAL, focal_length_min REAL, "
2503 "focal_length_max REAL, writeprotect INTEGER, "
2504 "autoapply INTEGER, filter INTEGER, def INTEGER, format INTEGER)",
2505 NULL, NULL, NULL);
2506 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX data.presets_idx ON presets (name, operation, op_version)",
2507 NULL, NULL, NULL);
2509 sqlite3_exec(db->handle, "CREATE TABLE data.locations (tagid INTEGER PRIMARY KEY, "
2510 "type INTEGER, longitude REAL, latitude REAL, delta1 REAL, delta2 REAL, ratio FLOAT, polygons BLOB, "
2511 "FOREIGN KEY(tagid) REFERENCES tags(id))", NULL, NULL, NULL);
2512 // clang-format on
2513}
2514
2515// create the in-memory tables
2516// temporary stuff for some ops, need this for some reason with newer sqlite3:
2517static void _create_memory_schema(dt_database_t *db)
2518{
2519 // clang-format off
2520 sqlite3_exec(db->handle, "CREATE TABLE memory.color_labels_temp (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2521 sqlite3_exec(
2522 db->handle,
2523 "CREATE TABLE memory.collected_images (rowid INTEGER PRIMARY KEY AUTOINCREMENT, imgid INTEGER)", NULL,
2524 NULL, NULL);
2525 sqlite3_exec(
2526 db->handle,
2527 "CREATE TABLE memory.collected_backup (rowid INTEGER PRIMARY KEY AUTOINCREMENT, imgid INTEGER)", NULL,
2528 NULL, NULL); sqlite3_exec(db->handle, "CREATE TABLE memory.selected_backup (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2529 sqlite3_exec(db->handle, "CREATE TABLE memory.selected_backup (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2530 sqlite3_exec(db->handle, "CREATE TABLE memory.tmp_selection (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2531 sqlite3_exec(db->handle, "CREATE TABLE memory.taglist "
2532 "(tmpid INTEGER PRIMARY KEY, id INTEGER UNIQUE ON CONFLICT IGNORE, "
2533 "count INTEGER DEFAULT 0, count2 INTEGER DEFAULT 0)",
2534 NULL, NULL, NULL);
2535 sqlite3_exec(db->handle, "CREATE TABLE memory.similar_tags (tagid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2536 sqlite3_exec(db->handle, "CREATE TABLE memory.darktable_tags (tagid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2537 sqlite3_exec(
2538 db->handle,
2539 "CREATE TABLE memory.undo_history (id INTEGER, imgid INTEGER, num INTEGER, module INTEGER, "
2540 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2541 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2542 NULL, NULL, NULL);
2543 sqlite3_exec(
2544 db->handle,
2545 "CREATE TABLE memory.undo_masks_history (id INTEGER, imgid INTEGER, num INTEGER, formid INTEGER,"
2546 " form INTEGER, name VARCHAR(256), version INTEGER, points BLOB, points_count INTEGER, source BLOB)",
2547 NULL, NULL, NULL);
2548 sqlite3_exec(
2549 db->handle,
2550 "CREATE TABLE memory.undo_module_order (id INTEGER, imgid INTEGER, version INTEGER, iop_list VARCHAR)",
2551 NULL, NULL, NULL);
2552 sqlite3_exec(db->handle,
2553 "CREATE TABLE memory.darktable_iop_names (operation VARCHAR(256) PRIMARY KEY, name VARCHAR(256))",
2554 NULL, NULL, NULL);
2555 sqlite3_exec(db->handle,
2556 "CREATE TABLE memory.film_folder (id INTEGER PRIMARY KEY, status INTEGER)",
2557 NULL, NULL, NULL);
2558 // clang-format on
2559}
2560
2561static void _sanitize_db(dt_database_t *db)
2562{
2563 sqlite3_stmt *stmt, *innerstmt;
2564 // clang-format off
2565 /* first let's get rid of non-utf8 tags. */
2566 sqlite3_prepare_v2(db->handle, "SELECT id, name FROM data.tags", -1, &stmt, NULL);
2567 sqlite3_prepare_v2(db->handle, "UPDATE data.tags SET name = ?1 WHERE id = ?2", -1, &innerstmt, NULL);
2568 // clang-format on
2569 while(sqlite3_step(stmt) == SQLITE_ROW)
2570 {
2571 const int id = sqlite3_column_int(stmt, 0);
2572 const char *tag = (const char *)sqlite3_column_text(stmt, 1);
2573
2574 if(!g_utf8_validate(tag, -1, NULL))
2575 {
2576 gchar *new_tag = dt_util_foo_to_utf8(tag);
2577 fprintf(stderr, "[init]: tag `%s' is not valid utf8, replacing it with `%s'\n", tag, new_tag);
2578 if(tag)
2579 {
2580 sqlite3_bind_text(innerstmt, 1, new_tag, -1, SQLITE_TRANSIENT);
2581 sqlite3_bind_int(innerstmt, 2, id);
2582 sqlite3_step(innerstmt);
2583 sqlite3_reset(innerstmt);
2584 sqlite3_clear_bindings(innerstmt);
2585 dt_free(new_tag);
2586 }
2587 }
2588 }
2589 sqlite3_finalize(stmt);
2590 sqlite3_finalize(innerstmt);
2591 // clang-format off
2592 // make sure film_roll folders don't end in "/", that will result in empty entries in the collect module
2593 sqlite3_exec(db->handle,
2594 "UPDATE main.film_rolls SET folder = substr(folder, 1, length(folder) - 1) WHERE folder LIKE '%/'",
2595 NULL, NULL, NULL);
2596 // clang-format on
2597}
2598
2599// in library we keep the names of the tags used in tagged_images. however, using that table at runtime results
2600// in some overhead not necessary so instead we just use the used_tags table to update tagged_images on startup
2601#define TRY_EXEC(_query, _message) \
2602 do \
2603 { \
2604 if(sqlite3_exec(db->handle, _query, NULL, NULL, NULL) != SQLITE_OK) \
2605 { \
2606 fprintf(stderr, _message); \
2607 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2608 FINALIZE; \
2609 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2610 return FALSE; \
2611 } \
2612 } while(0)
2613
2614#define TRY_STEP(_stmt, _expected, _message) \
2615 do \
2616 { \
2617 if(sqlite3_step(_stmt) != _expected) \
2618 { \
2619 fprintf(stderr, _message); \
2620 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2621 FINALIZE; \
2622 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2623 return FALSE; \
2624 } \
2625 } while(0)
2626
2627#define TRY_PREPARE(_stmt, _query, _message) \
2628 do \
2629 { \
2630 if(sqlite3_prepare_v2(db->handle, _query, -1, &_stmt, NULL) != SQLITE_OK) \
2631 { \
2632 fprintf(stderr, _message); \
2633 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2634 FINALIZE; \
2635 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2636 return FALSE; \
2637 } \
2638 } while(0)
2639
2640#define FINALIZE \
2641 do \
2642 { \
2643 sqlite3_finalize(stmt); stmt = NULL; /* NULL so that finalize becomes a NOP */ \
2644 } while(0)
2645
2646#undef TRY_EXEC
2647#undef TRY_STEP
2648#undef TRY_PREPARE
2649#undef FINALIZE
2650
2651gboolean dt_database_show_error(const dt_database_t *db)
2652{
2653 gboolean error = TRUE;
2654
2655 if(!db->lock_acquired)
2656 {
2657 char lck_pathname[1024];
2658 snprintf(lck_pathname, sizeof(lck_pathname), "%s.lock", db->error_dbfilename);
2659 char *lck_dirname = g_strdup(lck_pathname);
2660 char *slash_pos = g_strrstr(lck_dirname, "/");
2661 if(!IS_NULL_PTR(slash_pos)) *slash_pos = '\0';
2662 // clang-format off
2663 char *label_text = g_markup_printf_escaped(
2664 _("\n"
2665 " Sorry, Ansel could not be started because the database is locked.\n"
2666 "\n"
2667 " How to solve this problem?\n"
2668 "\n"
2669 " 1 - If another Ansel instance is already running, \n"
2670 " click \"Quit\" and either use that instance or close it before trying to start Ansel again. \n"
2671 " (process ID <i><b>%d</b></i> created the database lock files)\n"
2672 "\n"
2673 " 2 - If you cannot find any running instance of Ansel, try restarting your session or your computer. \n"
2674 " This will close all running programs and should release any database locks. \n"
2675 "\n"
2676 " 3 - If you have already tried the above steps, or you are certain that no other instances of Ansel are running, \n"
2677 " this likely means the previous instance ended unexpectedly. \n"
2678 " Click the \"Delete database lock files\" button to remove <i>data.db.lock</i> and <i>library.db.lock</i>. \n"
2679 " Ansel will then attempt to load the database again. \n"
2680 "\n\n"
2681 " <i><u>Caution!</u> Do not delete these files without first undertaking the above checks, \n"
2682 " otherwise you risk generating serious inconsistencies in your database.</i>\n"),
2683 db->error_other_pid);
2684 // clang-format on
2685
2686 const int choice = dt_gui_show_standalone_three_choice_dialog(_("Error starting Ansel"), label_text,
2687 _("Quit"), _("Retry"), _("Delete database lock files and try again"));
2688
2689 if(choice == 1)
2690 {
2691 // Just try to acquire the lock again: useful once the other instance that held it has
2692 // since closed. dt_database_show_error() returning FALSE makes the caller's init loop
2693 // (common/darktable.c) re-run dt_database_init() without touching any lock file.
2694 error = FALSE;
2695 }
2696 else if(choice == 2)
2697 {
2698 gboolean really_delete_lockfiles =
2699 dt_gui_show_standalone_yes_no_dialog
2700 (_("Confirmation"),
2701 _("\n<u>Caution!</u> Are you sure you want to delete the database lock files?\n"
2702 "This action should only be performed if you are certain no other Ansel instances are running.\n"), _("Quit"), _("Yes"));
2703 if(really_delete_lockfiles)
2704 {
2705 int status = 0;
2706
2707 char *lck_filename = g_strconcat(lck_dirname, "/data.db.lock", NULL);
2708 if(g_access(lck_filename, F_OK) != -1)
2709 status += remove(lck_filename);
2710
2711 lck_filename = g_strconcat(lck_dirname, "/library.db.lock", NULL);
2712 if(g_access(lck_filename, F_OK) != -1)
2713 status += remove(lck_filename);
2714 dt_free(lck_filename);
2715
2716 if(status==0)
2717 {
2718 dt_gui_show_standalone_yes_no_dialog(_("Done"),
2719 _("\nThe database lock files have been deleted successfully.\n"),
2720 _("Continue"), NULL);
2721 error = FALSE;
2722 }
2723
2724 else
2725 dt_gui_show_standalone_yes_no_dialog
2726 (_("Error"), g_markup_printf_escaped(
2727 _("\nAt least one lock file could not be removed.\n"
2728 "You may try to manually delete the files <i>data.db.lock</i> and <i>library.db.lock</i>\n"
2729 "in folder <a href=\"file:///%s\">%s</a>.\n"), lck_dirname, lck_dirname),
2730 _("Quit"), NULL);
2731 }
2732 }
2733
2734 dt_free(lck_dirname);
2735 dt_free(label_text);
2736 }
2737
2738 dt_free(db->error_message);
2739 dt_free(db->error_dbfilename);
2740 ((dt_database_t *)db)->error_other_pid = 0;
2741 ((dt_database_t *)db)->error_message = NULL;
2742 ((dt_database_t *)db)->error_dbfilename = NULL;
2743 return error;
2744}
2745
2746static gboolean pid_is_alive(int pid)
2747{
2748 gboolean pid_is_alive;
2749
2750#ifdef _WIN32
2751 pid_is_alive = FALSE;
2752 HANDLE h = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid);
2753 if(h)
2754 {
2755 wchar_t wfilename[MAX_PATH];
2756 long unsigned int n_filename = sizeof(wfilename);
2757 int ret = QueryFullProcessImageNameW(h, 0, wfilename, &n_filename);
2758 char *filename = g_utf16_to_utf8(wfilename, -1, NULL, NULL, NULL);
2759 if(ret && n_filename > 0 && filename && g_str_has_suffix(filename, "ansel.exe"))
2760 pid_is_alive = TRUE;
2761 dt_free(filename);
2762 CloseHandle(h);
2763 }
2764#else
2765 pid_is_alive = !((kill(pid, 0) == -1) && errno == ESRCH);
2766
2767#ifdef __linux__
2768 // If this is Linux, we can query /proc to see if the pid is
2769 // actually a darktable instance.
2770 if(pid_is_alive)
2771 {
2772 gchar *contents;
2773 gsize length;
2774 gchar filename[64];
2775 snprintf(filename, sizeof(filename), "/proc/%d/cmdline", pid);
2776
2777 if(g_file_get_contents("", &contents, &length, NULL))
2778 {
2779 if(strstr(contents, "ansel") == NULL)
2780 {
2781 pid_is_alive = FALSE;
2782 }
2783 dt_free(contents);
2784 }
2785 }
2786#endif
2787
2788#endif
2789
2790 return pid_is_alive;
2791}
2792
2793static gboolean _lock_single_database(dt_database_t *db, const char *dbfilename, char **lockfile)
2794{
2795 gboolean lock_acquired = FALSE;
2796 mode_t old_mode;
2797 int lock_tries = 0;
2798 gchar *pid = g_strdup_printf("%d", getpid());
2799
2800 if(!g_strcmp0(dbfilename, ":memory:"))
2801 {
2802 lock_acquired = TRUE;
2803 }
2804 else
2805 {
2806 *lockfile = g_strconcat(dbfilename, ".lock", NULL);
2807lock_again:
2808 lock_tries++;
2809 old_mode = umask(0);
2810 int fd = g_open(*lockfile, O_RDWR | O_CREAT | O_EXCL, 0666);
2811 umask(old_mode);
2812
2813 if(fd != -1) // the lockfile was successfully created - write our PID into it
2814 {
2815 if(write(fd, pid, strlen(pid) + 1) > -1) lock_acquired = TRUE;
2816 close(fd);
2817 }
2818 else // the lockfile already exists - see if it's a stale one left over from a crashed instance
2819 {
2820 char buf[64];
2821 memset(buf, 0, sizeof(buf));
2822 fd = g_open(*lockfile, O_RDWR | O_CREAT, 0666);
2823
2824 if(fd == -1)
2825 {
2826 // failed to open the lock file
2827 const int err = errno;
2828 fprintf(stderr, "[init] error opening the database lock file for reading: %s\n", strerror(err));
2829 db->error_message = g_strdup_printf(_("error opening the database lock file for reading: %s"), strerror(err));
2830 }
2831 else
2832 {
2833 // successfully opened the lock file, read the PID
2834 const int bytes_read = read(fd, buf, sizeof(buf) - 1);
2835
2836 if(bytes_read > 0)
2837 {
2838 // lock file contains a PID
2839 db->error_other_pid = atoi(buf);
2840
2841 if(!pid_is_alive(db->error_other_pid))
2842 {
2843 // stale lock file - close fd before unlinking (required on Windows)
2844 close(fd);
2845 g_unlink(*lockfile);
2846
2847 if(lock_tries < 5)
2848 {
2849 goto lock_again;
2850 }
2851 else
2852 {
2853 fprintf(stderr, "[init] tried several times to acquire the database lock, giving up\n");
2854 db->error_message = g_strdup_printf(_("tried several times to acquire the database lock, giving up"));
2855 }
2856 }
2857 else
2858 {
2859 // lock file contains a live PID
2860 close(fd);
2861 fprintf(stderr, "[init] the database lock file contains a pid that seems to be alive in your system: %d\n",
2862 db->error_other_pid);
2863 db->error_message = g_strdup_printf(_("the database lock file contains a pid that seems to be alive in your system: %d"),
2864 db->error_other_pid);
2865 }
2866 }
2867 else
2868 {
2869 // lock file is empty or unreadable
2870 close(fd);
2871 fprintf(stderr, "[init] the database lock file seems to be empty\n");
2872 db->error_message = g_strdup_printf(_("the database lock file seems to be empty"));
2873 }
2874 }
2875 }
2876 }
2877
2878 dt_free(pid);
2879
2880 if(db->error_message)
2881 db->error_dbfilename = g_strdup(dbfilename);
2882
2883 return lock_acquired;
2884}
2885
2886static gboolean _lock_databases(dt_database_t *db)
2887{
2888 if(!_lock_single_database(db, db->dbfilename_data, &db->lockfile_data))
2889 return FALSE;
2890 if(!_lock_single_database(db, db->dbfilename_library, &db->lockfile_library))
2891 {
2892 // unlock data.db to not leave a stale lock file around
2893 g_unlink(db->lockfile_data);
2894 return FALSE;
2895 }
2896 return TRUE;
2897}
2898
2899void ask_for_upgrade(const gchar *dbname, const gboolean has_gui)
2900{
2901 // if there's no gui just leave
2902 if(!has_gui)
2903 {
2904 fprintf(stderr, "[init] database `%s' is out-of-date. aborting.\n", dbname);
2905 exit(1);
2906 }
2907
2908 // the database has to be upgraded, let's ask user
2909
2910 char *label_text = g_markup_printf_escaped(_("the database schema has to be upgraded for\n"
2911 "\n"
2912 "<span style='italic'>%s</span>\n"
2913 "\nthis might take a long time in case of a large database\n\n"
2914 "do you want to proceed or quit now to do a backup\n"),
2915 dbname);
2916
2917 gboolean shall_we_update_the_db =
2918 dt_gui_show_standalone_yes_no_dialog(_("ansel - schema migration"), label_text,
2919 _("close Ansel"), _("upgrade database"));
2920
2921 dt_free(label_text);
2922
2923 // if no upgrade, we exit now, nothing we can do more
2924 if(!shall_we_update_the_db)
2925 {
2926 fprintf(stderr, "[init] we shall not update the database, aborting.\n");
2927 exit(1);
2928 }
2929}
2930
2931void dt_database_backup(const char *filename)
2932{
2933 char *version = g_strdup(darktable_package_version);
2934 int k = 0;
2935 // get plain version (no commit id)
2936 while(version[k])
2937 {
2938 if((version[k] < '0' || version[k] > '9') && (version[k] != '.'))
2939 {
2940 version[k] = '\0';
2941 break;
2942 }
2943 k++;
2944 }
2945
2946 gchar *backup = g_strdup_printf("%s-pre-%s", filename, version);
2947
2948 GError *gerror = NULL;
2949 if(!g_file_test(backup, G_FILE_TEST_EXISTS))
2950 {
2951 GFile *src = g_file_new_for_path(filename);
2952 GFile *dest = g_file_new_for_path(backup);
2953 gboolean copy_status = TRUE;
2954 if(g_file_test(filename, G_FILE_TEST_EXISTS))
2955 {
2956 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
2957 if(copy_status) copy_status = g_chmod(backup, S_IRUSR) == 0;
2958 }
2959 else
2960 {
2961 // there is nothing to backup, create an empty file to prevent further backup attempts
2962 int fd = g_open(backup, O_CREAT, S_IRUSR);
2963 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
2964 }
2965 if(!copy_status) fprintf(stderr, "[backup failed] %s -> %s\n", filename, backup);
2966
2967 g_object_unref(src);
2968 g_object_unref(dest);
2969 }
2970
2971 dt_free(version);
2972 dt_free(backup);
2973}
2974
2975int _get_pragma_int_val(sqlite3 *db, const char* pragma)
2976{
2977 gchar* query= g_strdup_printf("PRAGMA %s", pragma);
2978 int val = -1;
2979 sqlite3_stmt *stmt;
2980 const int rc = sqlite3_prepare_v2(db, query,-1, &stmt, NULL);
2981 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
2982 {
2983 val = sqlite3_column_int(stmt, 0);
2984 }
2985 sqlite3_finalize(stmt);
2986 dt_free(query);
2987
2988 return val;
2989}
2990
2991gchar* _get_pragma_string_val(sqlite3 *db, const char* pragma)
2992{
2993 gchar* query= g_strdup_printf("PRAGMA %s", pragma);
2994 sqlite3_stmt *stmt;
2995 gchar* val = NULL;
2996 const int rc = sqlite3_prepare_v2(db, query,-1, &stmt, NULL);
2997 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
2998 {
2999 val = g_strdup((const char *)sqlite3_column_text(stmt, 0));
3000 while(sqlite3_step(stmt) == SQLITE_ROW)
3001 {
3002 gchar* cur_val = g_strdup((const char *)sqlite3_column_text(stmt, 0));
3003 gchar* tmp_val = g_strdup(val);
3004 dt_free(val);
3005 val = g_strconcat(tmp_val, "\n", cur_val, NULL);
3006 dt_free(cur_val);
3007 dt_free(tmp_val);
3008 }
3009 }
3010 sqlite3_finalize(stmt);
3011 dt_free(query);
3012 return val;
3013}
3014
3015dt_database_t *dt_database_init(const char *alternative, const gboolean load_data, const gboolean has_gui)
3016{
3017 /* set the threading mode to Serialized */
3018 sqlite3_config(SQLITE_CONFIG_SERIALIZED);
3019
3020 sqlite3_initialize();
3021
3022start:
3023 if(IS_NULL_PTR(alternative))
3024 {
3025 /* migrate default database location to new default */
3026 _database_migrate_to_xdg_structure();
3027 }
3028
3029 /* delete old mipmaps files */
3030 _database_delete_mipmaps_files();
3031
3032 /* lets construct the db filename */
3033 gchar *dbname = NULL;
3034 gchar dbfilename_library[PATH_MAX] = { 0 };
3035 gchar datadir[PATH_MAX] = { 0 };
3036
3037 dt_loc_get_user_config_dir(datadir, sizeof(datadir));
3038
3039 if(IS_NULL_PTR(alternative))
3040 {
3041 dbname = dt_conf_get_string("database");
3042 if(IS_NULL_PTR(dbname))
3043 dt_concat_path_file(dbfilename_library, datadir, "library.db");
3044 else if(!strcmp(dbname, ":memory:"))
3045 g_strlcpy(dbfilename_library, dbname, sizeof(dbfilename_library));
3046 else if(dbname[0] != '/')
3047 dt_concat_path_file(dbfilename_library, datadir, dbname);
3048 else
3049 g_strlcpy(dbfilename_library, dbname, sizeof(dbfilename_library));
3050 }
3051 else
3052 {
3053 g_strlcpy(dbfilename_library, alternative, sizeof(dbfilename_library));
3054
3055 GFile *galternative = g_file_new_for_path(alternative);
3056 dbname = g_file_get_basename(galternative);
3057 g_object_unref(galternative);
3058 }
3059
3060 /* we also need a 2nd db with permanent data like presets, styles and tags */
3061 char dbfilename_data[PATH_MAX] = { 0 };
3062 if(load_data)
3063 dt_concat_path_file(dbfilename_data, datadir, "data.db");
3064 else
3065 snprintf(dbfilename_data, sizeof(dbfilename_data), ":memory:");
3066
3067 /* create database */
3068 dt_database_t *db = (dt_database_t *)g_malloc0(sizeof(dt_database_t));
3069 db->dbfilename_data = g_strdup(dbfilename_data);
3070 db->dbfilename_library = g_strdup(dbfilename_library);
3071
3072 dt_atomic_set_int(&_trxid, 0);
3073 dt_atomic_set_int(&_trx_batch_level, 0);
3074 g_atomic_pointer_set(&_trx_batch_owner, NULL);
3075
3076 /* make sure the folder exists. this might not be the case for new databases */
3077 /* also check if a database backup is needed */
3078 if(g_strcmp0(dbfilename_data, ":memory:"))
3079 {
3080 char *data_path = g_path_get_dirname(dbfilename_data);
3081 g_mkdir_with_parents(data_path, 0750);
3082 dt_free(data_path);
3083 dt_database_backup(dbfilename_data);
3084 }
3085 if(g_strcmp0(dbfilename_library, ":memory:"))
3086 {
3087 char *library_path = g_path_get_dirname(dbfilename_library);
3088 g_mkdir_with_parents(library_path, 0750);
3089 dt_free(library_path);
3090 dt_database_backup(dbfilename_library);
3091 }
3092
3093 dt_print(DT_DEBUG_SQL, "[init sql] library: %s, data: %s\n", dbfilename_library, dbfilename_data);
3094
3095 /* having more than one instance of darktable using the same database is a bad idea */
3096 /* try to get locks for the databases */
3097 db->lock_acquired = _lock_databases(db);
3098
3099 if(!db->lock_acquired)
3100 {
3101 fprintf(stderr, "[init] database is locked, probably another process is already using it\n");
3102 dt_free(dbname);
3103 return db;
3104 }
3105
3106 /* opening / creating database */
3107 if(sqlite3_open(db->dbfilename_library, &db->handle))
3108 {
3109 fprintf(stderr, "[init] could not find database ");
3110 if(dbname)
3111 fprintf(stderr, "`%s'!\n", dbname);
3112 else
3113 fprintf(stderr, "\n");
3114 fprintf(stderr, "[init] maybe your %s/anselrc is corrupt?\n", datadir);
3115 dt_loc_get_datadir(dbfilename_library, sizeof(dbfilename_library));
3116 fprintf(stderr, "[init] try `cp %s/anselrc %s/anselrc'\n", dbfilename_library, datadir);
3117 sqlite3_close(db->handle);
3118 dt_free(dbname);
3119 dt_free(db->lockfile_data);
3120 dt_free(db->dbfilename_data);
3121 dt_free(db->lockfile_library);
3122 dt_free(db->dbfilename_library);
3123 dt_free(db);
3124 return NULL;
3125 }
3126
3127 dt_print(DT_DEBUG_SQL, "[sql] Opened database: '%s'\n", db->dbfilename_library);
3128
3129 sqlite3_stmt *stmt;
3130 int rc;
3131
3132 if(sqlite3_db_readonly(db->handle, "main") != 0)
3133 {
3134 fprintf(stderr, "%s database is read only. Abort...\n", db->dbfilename_library);
3135
3136 GtkWidget *dialog;
3137 GtkDialogFlags dflags = GTK_DIALOG_MODAL | GTK_DIALOG_DESTROY_WITH_PARENT;
3138
3139 dialog = gtk_dialog_new_with_buttons(_("Ansel - Database is read-only"),
3140 NULL, dflags,
3141 _("Close Ansel"), GTK_RESPONSE_CLOSE, NULL);
3142 gtk_dialog_set_default_response(GTK_DIALOG(dialog), GTK_RESPONSE_CLOSE);
3143 char *label_options
3144 = g_strdup_printf(_("<span weight='bold'>Ansel library database is read-only</span>\n\n"
3145 "This happens if you don't have permissions to write on the filesystem\n"
3146 "or if you have restored a write-protected backup snapshot.\n\n"
3147 "Please change the filesystem access permissions for:\n\n"
3148 "\t<span style='italic'>%s</span>"),
3149 db->dbfilename_library);
3150
3151 GtkWidget *content_area = gtk_dialog_get_content_area(GTK_DIALOG (dialog));
3152 GtkWidget *label = gtk_label_new(NULL);
3153 gtk_label_set_markup(GTK_LABEL(label), label_options);
3154 gtk_container_add(GTK_CONTAINER (content_area), label);
3155 gtk_widget_show_all(content_area);
3156 gtk_dialog_run(GTK_DIALOG(dialog));
3157 gtk_widget_destroy(dialog);
3158 dt_database_destroy(db);
3159 dt_free(dbname);
3160 dt_free(label_options);
3161 db = NULL;
3162 return NULL;
3163 }
3164
3165 dt_print(DT_DEBUG_SQL, "[sql] Checked that we can write in database: '%s'\n", db->dbfilename_library);
3166
3167 /* attach a memory database to db connection for use with temporary tables
3168 used during instance life time, which is discarded on exit.
3169 */
3170 sqlite3_exec(db->handle, "attach database ':memory:' as memory", NULL, NULL, NULL);
3171
3172 // attach the data database which contains presets, styles, tags and similar things not tied to single images
3173 gboolean have_data_db = load_data && g_file_test(dbfilename_data, G_FILE_TEST_EXISTS);
3174 rc = sqlite3_prepare_v2(db->handle, "ATTACH DATABASE ?1 AS data", -1, &stmt, NULL);
3175 sqlite3_bind_text(stmt, 1, dbfilename_data, -1, SQLITE_TRANSIENT);
3176 if(rc != SQLITE_OK || sqlite3_step(stmt) != SQLITE_DONE)
3177 {
3178 sqlite3_finalize(stmt);
3179 fprintf(stderr, "[init] database `%s' couldn't be opened. aborting\n", dbfilename_data);
3180 dt_database_destroy(db);
3181 db = NULL;
3182 goto error;
3183 }
3184 sqlite3_finalize(stmt);
3185
3186 dt_print(DT_DEBUG_SQL, "[sql] Opened database: '%s'\n", dbfilename_data);
3187
3188 // some sqlite3 config
3189 sqlite3_exec(db->handle, "PRAGMA synchronous = OFF", NULL, NULL, NULL);
3190 sqlite3_exec(db->handle, "PRAGMA journal_mode = MEMORY", NULL, NULL, NULL);
3191 sqlite3_exec(db->handle, "PRAGMA page_size = 32768", NULL, NULL, NULL);
3192
3193 // WARNING: the foreign_keys pragma must not be used, the integrity of the
3194 // database rely on it.
3195 sqlite3_exec(db->handle, "PRAGMA foreign_keys = ON", NULL, NULL, NULL);
3196
3197 dt_print(DT_DEBUG_SQL, "[sql] Configured database\n");
3198
3199 /* now that we got functional databases that are locked for us we can make sure that the schema is set up */
3200
3201 // first we update the data database to the latest version so that we can potentially move data from the library
3202 // over when updating that one
3203 if(!have_data_db)
3204 {
3205 _create_data_schema(db); // a brand new db it seems
3206 }
3207 else
3208 {
3209 gchar* data_status = _get_pragma_string_val(db->handle, "data.quick_check");
3210 rc = sqlite3_prepare_v2(db->handle, "select value from data.db_info where key = 'version'", -1, &stmt, NULL);
3211 if(!g_strcmp0(data_status, "ok") && rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3212 {
3213 dt_free(data_status); // status is OK and we don't need to care :)
3214
3215 // compare the version of the db with what is current for this executable
3216 const int db_version = sqlite3_column_int(stmt, 0);
3217 sqlite3_finalize(stmt);
3218 if(db_version < CURRENT_DATABASE_VERSION_DATA)
3219 {
3220 ask_for_upgrade(dbfilename_data, has_gui);
3221
3222 // older: upgrade
3223 if(!_upgrade_data_schema(db, db_version))
3224 {
3225 // we couldn't upgrade the db for some reason. bail out.
3226 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version %d to %d. aborting\n",
3227 dbfilename_data, db_version, CURRENT_DATABASE_VERSION_DATA);
3228 dt_database_destroy(db);
3229 db = NULL;
3230 goto error;
3231 }
3232 }
3233 else if(db_version > CURRENT_DATABASE_VERSION_DATA)
3234 {
3235 // newer: bail out
3236 fprintf(stderr, "[init] database version of `%s' is too new for this build of darktable. aborting\n",
3237 dbfilename_data);
3238 dt_database_destroy(db);
3239 db = NULL;
3240 goto error;
3241 }
3242 // else: the current version, do nothing
3243 }
3244 else
3245 {
3246 // oh, bad situation. the database is corrupt and can't be read!
3247 // we inform the user here and let him decide what to do: exit or delete and try again.
3248
3249 gchar* quick_check_text = NULL;
3250 if(g_strcmp0(data_status, "ok")) // data_status is not ok
3251 {
3252 quick_check_text = g_strdup_printf(_("quick_check said:\n"
3253 "%s \n"), data_status);
3254 }
3255 else
3256 {
3257 quick_check_text = g_strdup(""); // a trick;
3258 }
3259
3260 gchar *data_snap = dt_database_get_most_recent_snap(dbfilename_data);
3261
3262 GtkWidget *dialog;
3263 GtkDialogFlags dflags = GTK_DIALOG_MODAL | GTK_DIALOG_DESTROY_WITH_PARENT;
3264
3265 const char* label_options = NULL;
3266
3267 if(data_snap)
3268 {
3269 dialog = gtk_dialog_new_with_buttons(_("ansel - error opening database"),
3270 NULL,
3271 dflags,
3272 _("close Ansel"),
3273 GTK_RESPONSE_CLOSE,
3274 _("attempt restore"),
3275 GTK_RESPONSE_ACCEPT,
3276 _("delete database"),
3277 GTK_RESPONSE_REJECT,
3278 NULL);
3279 gtk_dialog_set_default_response(GTK_DIALOG(dialog), GTK_RESPONSE_ACCEPT);
3280 label_options = _("do you want to close Ansel now to manually restore\n"
3281 "the database from a backup, attempt an automatic restore\n"
3282 "from the most recent snapshot or delete the corrupted database\n"
3283 "and start with a new one?");
3284 }
3285 else
3286 {
3287 dialog = gtk_dialog_new_with_buttons(_("ansel - error opening database"),
3288 NULL,
3289 dflags,
3290 _("close Ansel"),
3291 GTK_RESPONSE_CLOSE,
3292 _("delete database"),
3293 GTK_RESPONSE_REJECT,
3294 NULL);
3295 gtk_dialog_set_default_response(GTK_DIALOG(dialog), GTK_RESPONSE_CLOSE);
3296 label_options = _("do you want to close Ansel now to manually restore\n"
3297 "the database from a backup or delete the corrupted database\n"
3298 "and start with a new one?");
3299 }
3300
3301
3302
3303 char *label_text = g_markup_printf_escaped(_("an error has occurred while trying to open the database from\n"
3304 "\n"
3305 "<span style='italic'>%s</span>\n"
3306 "\n"
3307 "it seems that the database is corrupted.\n"
3308 "%s"
3309 "%s"),
3310 dbfilename_data, quick_check_text, label_options);
3311
3312 dt_free(quick_check_text);
3313 dt_free(data_status);
3314
3315 GtkWidget *content_area = gtk_dialog_get_content_area(GTK_DIALOG (dialog));
3316 GtkWidget *label = gtk_label_new(NULL);
3317 gtk_label_set_markup(GTK_LABEL(label), label_text);
3318 dt_free(label_text);
3319 gtk_container_add(GTK_CONTAINER (content_area), label);
3320
3321 gtk_widget_show_all(content_area);
3322
3323 const int resp = gtk_dialog_run(GTK_DIALOG(dialog));
3324
3325 gtk_widget_destroy(dialog);
3326
3327 dt_database_destroy(db);
3328 db = NULL;
3329
3330 if(resp != GTK_RESPONSE_ACCEPT && resp != GTK_RESPONSE_REJECT)
3331 {
3332 fprintf(stderr, "[init] database `%s' is corrupt and can't be opened! either replace it from a backup or "
3333 "delete the file so that darktable can create a new one the next time. aborting\n", dbfilename_data);
3334 dt_free(data_snap);
3335 goto error;
3336 }
3337
3338 //here were sure that response is either accept (restore from snap) or reject (just delete the damaged db)
3339
3340 fprintf(stderr, "[init] deleting `%s' on user request", dbfilename_data);
3341
3342 if(g_unlink(dbfilename_data) == 0)
3343 fprintf(stderr, " ... ok\n");
3344 else
3345 fprintf(stderr, " ... failed\n");
3346
3347 if(resp == GTK_RESPONSE_ACCEPT && data_snap)
3348 {
3349 fprintf(stderr, "[init] restoring `%s' from `%s'...", dbfilename_data, data_snap);
3350 GError *gerror = NULL;
3351 if(!g_file_test(dbfilename_data, G_FILE_TEST_EXISTS))
3352 {
3353 GFile *src = g_file_new_for_path(data_snap);
3354 GFile *dest = g_file_new_for_path(dbfilename_data);
3355 gboolean copy_status = TRUE;
3356 if(g_file_test(data_snap, G_FILE_TEST_EXISTS))
3357 {
3358 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
3359 if(copy_status) copy_status = g_chmod(dbfilename_data, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == 0;
3360 }
3361 else
3362 {
3363 // there is nothing to restore, create an empty file
3364 const int fd = g_open(dbfilename_data, O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
3365 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
3366 }
3367 if(copy_status)
3368 fprintf(stderr, " success!\n");
3369 else
3370 fprintf(stderr, " failed!\n");
3371
3372 g_object_unref(src);
3373 g_object_unref(dest);
3374 }
3375 }
3376 dt_free(data_snap);
3377 dt_free(dbname);
3378 goto start;
3379 }
3380 }
3381
3382 dt_print(DT_DEBUG_SQL, "[sql] Checked database schema and migrated if needed\n");
3383
3384 gchar* libdb_status = _get_pragma_string_val(db->handle, "main.quick_check");
3385 // next we are looking at the library database
3386 // does the db contain the new 'db_info' table?
3387 rc = sqlite3_prepare_v2(db->handle, "select value from main.db_info where key = 'version'", -1, &stmt, NULL);
3388 if(!g_strcmp0(libdb_status, "ok") && rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3389 {
3390 dt_free(libdb_status);//it's ok :)
3391
3392 // compare the version of the db with what is current for this executable
3393 const int db_version = sqlite3_column_int(stmt, 0);
3394 sqlite3_finalize(stmt);
3395
3396 if(db_version < CURRENT_DATABASE_VERSION_LIBRARY)
3397 {
3398 ask_for_upgrade(dbfilename_library, has_gui);
3399
3400 // older: upgrade
3401 if(!_upgrade_library_schema(db, db_version))
3402 {
3403 // we couldn't upgrade the db for some reason. bail out.
3404 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version %d to %d. aborting\n", dbname,
3405 db_version, CURRENT_DATABASE_VERSION_LIBRARY);
3406 dt_database_destroy(db);
3407 db = NULL;
3408 goto error;
3409 }
3410 }
3411 else if(db_version > CURRENT_DATABASE_VERSION_LIBRARY)
3412 {
3413 // newer: bail out. it's better than what we did before: delete everything
3414 fprintf(stderr, "[init] database version of `%s' is too new for this build of darktable. aborting\n",
3415 dbname);
3416 dt_database_destroy(db);
3417 db = NULL;
3418 goto error;
3419 }
3420 // else: the current version, do nothing
3421 }
3422 else if(g_strcmp0(libdb_status, "ok") || rc == SQLITE_CORRUPT || rc == SQLITE_NOTADB)
3423 {
3424 // oh, bad situation. the database is corrupt and can't be read!
3425 // we inform the user here and let him decide what to do: exit or delete and try again.
3426
3427 gchar* quick_check_text = NULL;
3428 if(g_strcmp0(libdb_status, "ok")) // data_status is not ok
3429 {
3430 quick_check_text = g_strdup_printf(_("quick_check said:\n"
3431 "%s \n"), libdb_status);
3432 }
3433 else
3434 {
3435 quick_check_text = g_strdup(""); // a trick;
3436 }
3437
3438 gchar *data_snap = dt_database_get_most_recent_snap(dbfilename_library);
3439
3440 GtkWidget *dialog;
3441 GtkDialogFlags dflags = GTK_DIALOG_MODAL | GTK_DIALOG_DESTROY_WITH_PARENT;
3442
3443 const char* label_options = NULL;
3444
3445 if(data_snap)
3446 {
3447 dialog = gtk_dialog_new_with_buttons(_("ansel - error opening database"),
3448 NULL,
3449 dflags,
3450 _("close Ansel"),
3451 GTK_RESPONSE_CLOSE,
3452 _("attempt restore"),
3453 GTK_RESPONSE_ACCEPT,
3454 _("delete database"),
3455 GTK_RESPONSE_REJECT,
3456 NULL);
3457 gtk_dialog_set_default_response(GTK_DIALOG(dialog), GTK_RESPONSE_ACCEPT);
3458 label_options = _("do you want to close Ansel now to manually restore\n"
3459 "the database from a backup, attempt an automatic restore\n"
3460 "from the most recent snapshot or delete the corrupted database\n"
3461 "and start with a new one?");
3462 }
3463 else
3464 {
3465 dialog = gtk_dialog_new_with_buttons(_("ansel - error opening database"),
3466 NULL,
3467 dflags,
3468 _("close Ansel"),
3469 GTK_RESPONSE_CLOSE,
3470 _("delete database"),
3471 GTK_RESPONSE_REJECT,
3472 NULL);
3473 gtk_dialog_set_default_response(GTK_DIALOG(dialog), GTK_RESPONSE_CLOSE);
3474 label_options = _("do you want to close Ansel now to manually restore\n"
3475 "the database from a backup or delete the corrupted database\n"
3476 "and start with a new one?");
3477 }
3478
3479 char *label_text = g_markup_printf_escaped(_("an error has occurred while trying to open the database from\n"
3480 "\n"
3481 "<span style='italic'>%s</span>\n"
3482 "\n"
3483 "it seems that the database is corrupted.\n"
3484 "%s"
3485 "%s"),
3486 dbfilename_data, quick_check_text, label_options);
3487
3488 dt_free(quick_check_text);
3489 dt_free(libdb_status);
3490
3491 GtkWidget *content_area = gtk_dialog_get_content_area(GTK_DIALOG (dialog));
3492 GtkWidget *label = gtk_label_new(NULL);
3493 gtk_label_set_markup(GTK_LABEL(label), label_text);
3494 dt_free(label_text);
3495 gtk_container_add(GTK_CONTAINER (content_area), label);
3496
3497 gtk_widget_show_all(content_area);
3498
3499 const int resp = gtk_dialog_run(GTK_DIALOG(dialog));
3500
3501 gtk_widget_destroy(dialog);
3502
3503 dt_database_destroy(db);
3504 db = NULL;
3505
3506 if(resp != GTK_RESPONSE_ACCEPT && resp != GTK_RESPONSE_REJECT)
3507 {
3508 fprintf(stderr, "[init] database `%s' is corrupt and can't be opened! either replace it from a backup or "
3509 "delete the file so that darktable can create a new one the next time. aborting\n", dbfilename_library);
3510 dt_free(data_snap);
3511 goto error;
3512 }
3513
3514 //here were sure that response is either accept (restore from snap) or reject (just delete the damaged db)
3515
3516 fprintf(stderr, "[init] deleting `%s' on user request", dbfilename_library);
3517
3518 if(g_unlink(dbfilename_library) == 0)
3519 fprintf(stderr, " ... ok\n");
3520 else
3521 fprintf(stderr, " ... failed\n");
3522
3523 if(resp == GTK_RESPONSE_ACCEPT && data_snap)
3524 {
3525 fprintf(stderr, "[init] restoring `%s' from `%s'...", dbfilename_library, data_snap);
3526 GError *gerror = NULL;
3527 if(!g_file_test(dbfilename_library, G_FILE_TEST_EXISTS))
3528 {
3529 GFile *src = g_file_new_for_path(data_snap);
3530 GFile *dest = g_file_new_for_path(dbfilename_library);
3531 gboolean copy_status = TRUE;
3532 if(g_file_test(data_snap, G_FILE_TEST_EXISTS))
3533 {
3534 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
3535 if(copy_status) copy_status = g_chmod(dbfilename_library, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == 0;
3536 }
3537 else
3538 {
3539 // there is nothing to restore, create an empty file to prevent further backup attempts
3540 const int fd = g_open(dbfilename_library, O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
3541 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
3542 }
3543 if(copy_status)
3544 fprintf(stderr, " success!\n");
3545 else
3546 fprintf(stderr, " failed!\n");
3547 g_object_unref(src);
3548 g_object_unref(dest);
3549 }
3550 }
3551 dt_free(data_snap);
3552 dt_free(dbname);
3553 goto start;
3554 }
3555 else
3556 {
3557 // does it contain the legacy 'settings' table?
3558 sqlite3_finalize(stmt);
3559 rc = sqlite3_prepare_v2(db->handle, "select settings from main.settings", -1, &stmt, NULL);
3560 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3561 {
3562 // the old blob had the version as an int in the first place
3563 const void *set = sqlite3_column_blob(stmt, 0);
3564 const int db_version = *(int *)set;
3565 sqlite3_finalize(stmt);
3566 if(!_migrate_schema(db, db_version)) // bring the legacy layout to the first one known to our upgrade
3567 // path ...
3568 {
3569 // we couldn't migrate the db for some reason. bail out.
3570 fprintf(stderr, "[init] database `%s' couldn't be migrated from the legacy version %d. aborting\n",
3571 dbname, db_version);
3572 dt_database_destroy(db);
3573 db = NULL;
3574 goto error;
3575 }
3576 if(!_upgrade_library_schema(db, 1)) // ... and upgrade it
3577 {
3578 // we couldn't upgrade the db for some reason. bail out.
3579 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version 1 to %d. aborting\n", dbname,
3580 CURRENT_DATABASE_VERSION_LIBRARY);
3581 dt_database_destroy(db);
3582 db = NULL;
3583 goto error;
3584 }
3585 }
3586 else
3587 {
3588 sqlite3_finalize(stmt);
3589 _create_library_schema(db); // a brand new db it seems
3590 }
3591 }
3592
3593 dt_print(DT_DEBUG_SQL, "[sql] Checked db_info\n");
3594
3595 // create the in-memory tables
3596 _create_memory_schema(db);
3597
3598 dt_print(DT_DEBUG_SQL, "[sql] Created in-memory DB\n");
3599
3600 // create a table legacy_presets with all the presets from pre-auto-apply-cleanup darktable.
3601 dt_legacy_presets_create(db);
3602
3603 dt_print(DT_DEBUG_SQL, "[sql] Loaded legacy presets\n");
3604
3605 // drop table settings -- we don't want old versions of dt to drop our tables
3606 sqlite3_exec(db->handle, "drop table main.settings", NULL, NULL, NULL);
3607
3608 // take care of potential bad data in the db.
3609 _sanitize_db(db);
3610
3611#ifdef HAVE_ICU
3612 // check if sqlite is already icu enabled
3613 // if not enabled expected error: no such function:icu_load_collation
3614 rc = sqlite3_prepare_v2(db->handle,
3615 "SELECT icu_load_collation('en_US', 'english')",
3616 -1, &stmt, NULL);
3617 sqlite3_finalize(stmt);
3618
3619 if(rc != SQLITE_OK)
3620 {
3621 rc = sqlite3IcuInit(db->handle);
3622 if(rc != SQLITE_OK)
3623 fprintf(stderr, "[sqlite] init icu extension error %d\n", rc);
3624 }
3625#endif
3626
3627error:
3628 dt_free(dbname);
3629
3630 return db;
3631}
3632
3633void dt_database_destroy(const dt_database_t *db)
3634{
3635 sqlite3_close(db->handle);
3636 if (db->lockfile_data)
3637 {
3638 g_unlink(db->lockfile_data);
3639 dt_free(db->lockfile_data);
3640 }
3641 if (db->lockfile_library)
3642 {
3643 g_unlink(db->lockfile_library);
3644 dt_free(db->lockfile_library);
3645 }
3646 dt_free(db->dbfilename_data);
3647 dt_free(db->dbfilename_library);
3648 dt_free(db);
3649
3650 sqlite3_shutdown();
3651}
3652
3653sqlite3 *dt_database_get(const dt_database_t *db)
3654{
3655 return db ? db->handle : NULL;
3656}
3657
3658const gchar *dt_database_get_path(const struct dt_database_t *db)
3659{
3660 return db->dbfilename_library;
3661}
3662
3663static void _database_migrate_to_xdg_structure()
3664{
3665 gchar dbfilename[PATH_MAX] = { 0 };
3666 gchar *conf_db = dt_conf_get_string("database");
3667
3668 gchar datadir[PATH_MAX] = { 0 };
3669 dt_loc_get_datadir(datadir, sizeof(datadir));
3670
3671 if(conf_db && conf_db[0] != '/')
3672 {
3673 const char *homedir = getenv("HOME");
3674 snprintf(dbfilename, sizeof(dbfilename), "%s/%s", homedir, conf_db);
3675 if(g_file_test(dbfilename, G_FILE_TEST_EXISTS))
3676 {
3677 char destdbname[PATH_MAX] = { 0 };
3678 dt_concat_path_file(destdbname, datadir, "library.db");
3679 if(!g_file_test(destdbname, G_FILE_TEST_EXISTS))
3680 {
3681 fprintf(stderr, "[init] moving database into new XDG directory structure\n");
3682 rename(dbfilename, destdbname);
3683 dt_conf_set_string("database", "library.db");
3684 }
3685 }
3686 }
3687
3688 dt_free(conf_db);
3689}
3690
3691/* delete old mipmaps files */
3692static void _database_delete_mipmaps_files()
3693{
3694 /* This migration is intended to be run only from 0.9.x to new cache in 1.0 */
3695
3696 // Directory
3697 char cachedir[PATH_MAX] = { 0 }, mipmapfilename[PATH_MAX] = { 0 };
3698 dt_loc_get_user_cache_dir(cachedir, sizeof(cachedir));
3699 dt_concat_path_file(mipmapfilename, cachedir, "mipmaps");
3700
3701 if(g_access(mipmapfilename, F_OK) != -1)
3702 {
3703 fprintf(stderr, "[mipmap_cache] dropping old version file: %s\n", mipmapfilename);
3704 g_unlink(mipmapfilename);
3705 dt_concat_path_file(mipmapfilename, cachedir, "mipmaps.fallback");
3706
3707 if(g_access(mipmapfilename, F_OK) != -1) g_unlink(mipmapfilename);
3708 }
3709}
3710
3711gboolean dt_database_get_lock_acquired(const dt_database_t *db)
3712{
3713 return db->lock_acquired;
3714}
3715
3716void dt_database_cleanup_busy_statements(const struct dt_database_t *db)
3717{
3718 sqlite3_stmt *stmt = NULL;
3719 while( (stmt = sqlite3_next_stmt(db->handle, NULL)) != NULL)
3720 {
3721 const char* sql = sqlite3_sql(stmt);
3722 if(sqlite3_stmt_busy(stmt))
3723 {
3724 dt_print(DT_DEBUG_SQL, "[db busy stmt] non-finalized nor stepped through statement: '%s'\n",sql);
3725 sqlite3_reset(stmt);
3726 }
3727 else {
3728 dt_print(DT_DEBUG_SQL, "[db busy stmt] non-finalized statement: '%s'\n",sql);
3729 }
3730 sqlite3_finalize(stmt);
3731 }
3732}
3733
3734#define ERRCHECK {if (err!=NULL) {dt_print(DT_DEBUG_SQL, "[db maintenance] maintenance error: '%s'\n",err); sqlite3_free(err); err=NULL;}}
3735void dt_database_perform_maintenance(const struct dt_database_t *db)
3736{
3737 char* err = NULL;
3738
3739 const int main_pre_free_count = _get_pragma_int_val(db->handle, "main.freelist_count");
3740 const int main_page_size = _get_pragma_int_val(db->handle, "main.page_size");
3741 const int data_pre_free_count = _get_pragma_int_val(db->handle, "data.freelist_count");
3742 const int data_page_size = _get_pragma_int_val(db->handle, "data.page_size");
3743
3744 const guint64 calc_pre_size = (main_pre_free_count*main_page_size) + (data_pre_free_count*data_page_size);
3745
3746 if(calc_pre_size == 0)
3747 {
3748 dt_print(DT_DEBUG_SQL, "[db maintenance] maintenance deemed unnecesary, performing only analyze.\n");
3749 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE data", NULL, NULL, &err);
3750 ERRCHECK
3751 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE main", NULL, NULL, &err);
3752 ERRCHECK
3753 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE", NULL, NULL, &err);
3754 ERRCHECK
3755 return;
3756 }
3757
3758 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM data", NULL, NULL, &err);
3759 ERRCHECK
3760 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM main", NULL, NULL, &err);
3761 ERRCHECK
3762 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE data", NULL, NULL, &err);
3763 ERRCHECK
3764 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE main", NULL, NULL, &err);
3765 ERRCHECK
3766
3767 // for some reason this is needed in some cases
3768 // in case above performed vacuum+analyze properly, this is noop.
3769 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM", NULL, NULL, &err);
3770 ERRCHECK
3771 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE", NULL, NULL, &err);
3772 ERRCHECK
3773
3774 const int main_post_free_count = _get_pragma_int_val(db->handle, "main.freelist_count");
3775 const int data_post_free_count = _get_pragma_int_val(db->handle, "data.freelist_count");
3776
3777 const guint64 calc_post_size = (main_post_free_count*main_page_size) + (data_post_free_count*data_page_size);
3778 const gint64 bytes_freed = calc_pre_size - calc_post_size;
3779
3780 dt_print(DT_DEBUG_SQL, "[db maintenance] maintenance done, %" G_GINT64_FORMAT " bytes freed.\n", bytes_freed);
3781
3782 if(calc_post_size >= calc_pre_size)
3783 {
3784 dt_print(DT_DEBUG_SQL, "[db maintenance] maintenance problem. if no errors logged, it should work fine next time.\n");
3785 }
3786}
3787#undef ERRCHECK
3788
3789gboolean _ask_for_maintenance(const gboolean has_gui, const gboolean closing_time, const guint64 size)
3790{
3791 if(!has_gui)
3792 {
3793 return FALSE;
3794 }
3795
3796 char *later_info = NULL;
3797 char *size_info = g_format_size(size);
3798 const char *config = dt_conf_get_string_const("database/maintenance_check");
3799 if((closing_time && (!g_strcmp0(config, "on both"))) || !g_strcmp0(config, "on startup"))
3800 {
3801 later_info = _("click later to be asked on next startup");
3802 }
3803 else if (!closing_time && (!g_strcmp0(config, "on both")))
3804 {
3805 later_info = _("click later to be asked when closing Ansel");
3806 }
3807 else if (!g_strcmp0(config, "on close"))
3808 {
3809 later_info = _("click later to be asked next time when closing Ansel");
3810 }
3811
3812 char *label_text = g_markup_printf_escaped(_("the database could use some maintenance\n"
3813 "\n"
3814 "there's <span style='italic'>%s</span> to be freed"
3815 "\n\n"
3816 "do you want to proceed now?\n\n"
3817 "%s\n"
3818 "you can always change maintenance preferences in core options"),
3819 size_info, later_info);
3820
3821 const gboolean shall_perform_maintenance =
3822 dt_gui_show_standalone_yes_no_dialog(_("ansel - schema maintenance"), label_text,
3823 _("later"), _("yes"));
3824
3825 dt_free(label_text);
3826 dt_free(size_info);
3827
3828 return shall_perform_maintenance;
3829}
3830
3831static inline gboolean _is_mem_db(const struct dt_database_t *db)
3832{
3833 return !g_strcmp0(db->dbfilename_data, ":memory:") || !g_strcmp0(db->dbfilename_library, ":memory:");
3834}
3835
3836gboolean dt_database_maybe_maintenance(const struct dt_database_t *db, const gboolean has_gui, const gboolean closing_time)
3837{
3838 if(_is_mem_db(db))
3839 return FALSE;
3840
3841 const char *config = dt_conf_get_string_const("database/maintenance_check");
3842
3843 if(!g_strcmp0(config, "never"))
3844 {
3845 // early bail out on "never"
3846 dt_print(DT_DEBUG_SQL, "[db maintenance] please consider enabling database maintenance.\n");
3847 return FALSE;
3848 }
3849
3850 gboolean check_for_maintenance = FALSE;
3851 const gboolean force_maintenance = g_str_has_suffix (config, "(don't ask)");
3852
3853 if(config)
3854 {
3855 if((strstr(config, "on both")) // should cover "(don't ask) suffix
3856 || (closing_time && (strstr(config, "on close")))
3857 || (!closing_time && (strstr(config, "on startup"))))
3858 {
3859 // we have "on both/on close/on startup" setting, so - checking!
3860 dt_print(DT_DEBUG_SQL, "[db maintenance] checking for maintenance, due to rule: '%s'.\n", config);
3861 check_for_maintenance = TRUE;
3862 }
3863 // if the config was "never", check_for_vacuum is false.
3864 }
3865
3866 if(!check_for_maintenance)
3867 {
3868 return FALSE;
3869 }
3870
3871 // checking free pages
3872 const int main_free_count = _get_pragma_int_val(db->handle, "main.freelist_count");
3873 const int main_page_count = _get_pragma_int_val(db->handle, "main.page_count");
3874 const int main_page_size = _get_pragma_int_val(db->handle, "main.page_size");
3875
3876 const int data_free_count = _get_pragma_int_val(db->handle, "data.freelist_count");
3877 const int data_page_count = _get_pragma_int_val(db->handle, "data.page_count");
3878 const int data_page_size = _get_pragma_int_val(db->handle, "data.page_size");
3879
3880 dt_print(DT_DEBUG_SQL,
3881 "[db maintenance] main: [%d/%d pages], data: [%d/%d pages].\n",
3882 main_free_count, main_page_count, data_free_count, data_page_count);
3883
3884 if(main_page_count <= 0 || data_page_count <= 0)
3885 {
3886 //something's wrong with PRAGMA page_size returns. early bail.
3887 dt_print(DT_DEBUG_SQL,
3888 "[db maintenance] page_count <= 0 : main.page_count: %d, data.page_count: %d \n",
3889 main_page_count, data_page_count);
3890 return FALSE;
3891 }
3892
3893 // we don't need fine-grained percentages, so let's do ints
3894 const int main_free_percentage = (main_free_count * 100 ) / main_page_count;
3895 const int data_free_percentage = (data_free_count * 100 ) / data_page_count;
3896
3897 const int freepage_ratio = dt_conf_get_int("database/maintenance_freepage_ratio");
3898
3899 if((main_free_percentage >= freepage_ratio)
3900 || (data_free_percentage >= freepage_ratio))
3901 {
3902 const guint64 calc_size = (main_free_count*main_page_size) + (data_free_count*data_page_size);
3903 dt_print(DT_DEBUG_SQL, "[db maintenance] maintenance suggested, %" G_GUINT64_FORMAT " bytes to free.\n", calc_size);
3904
3905 if(force_maintenance || _ask_for_maintenance(has_gui, closing_time, calc_size))
3906 {
3907 return TRUE;
3908 }
3909 }
3910 return FALSE;
3911}
3912
3913void dt_database_optimize(const struct dt_database_t *db)
3914{
3915 if(_is_mem_db(db))
3916 return;
3917 // optimize should in most cases be no-op and have no noticeable downsides
3918 // this should be ran on every exit
3919 // see: https://www.sqlite.org/pragma.html#pragma_optimize
3920 DT_DEBUG_SQLITE3_EXEC(db->handle, "PRAGMA optimize", NULL, NULL, NULL);
3921}
3922
3923static void _print_backup_progress(int remaining, int total)
3924{
3925 // TODO if we have closing splashpage - this can be used to advance progressbar :)
3926 dt_print(DT_DEBUG_SQL, "[db backup] %d out of %d done\n", total - remaining, total);
3927}
3928
3929static int _backup_db(
3930 sqlite3 *src_db, /* Database handle to back up */
3931 const char *src_db_name, /* Database name to back up */
3932 const char *dest_filename, /* Name of file to back up to */
3933 void(*xProgress)(int, int) /* Progress function to invoke */
3934)
3935{
3936 sqlite3 *dest_db; /* Database connection opened on zFilename */
3937 sqlite3_backup *sb_dest; /* Backup handle used to copy data */
3938
3939 /* Open the database file identified by zFilename. */
3940 int rc = sqlite3_open(dest_filename, &dest_db);
3941
3942 if(rc == SQLITE_OK)
3943 {
3944 /* Open the sqlite3_backup object used to accomplish the transfer */
3945 sb_dest = sqlite3_backup_init(dest_db, "main", src_db, src_db_name);
3946 if(sb_dest)
3947 {
3948 dt_print(DT_DEBUG_SQL, "[db backup] %s to %s\n", src_db_name, dest_filename);
3949 gchar *pragma = g_strdup_printf("%s.page_count", src_db_name);
3950 const int spc = _get_pragma_int_val(src_db, pragma);
3951 dt_free(pragma);
3952 const int pc = MIN(spc, MAX(5,spc/100));
3953 do
3954 {
3955 rc = sqlite3_backup_step(sb_dest, pc);
3956 if(xProgress)
3957 xProgress(
3958 sqlite3_backup_remaining(sb_dest),
3959 sqlite3_backup_pagecount(sb_dest)
3960 );
3961 if( rc==SQLITE_OK || rc==SQLITE_BUSY || rc==SQLITE_LOCKED )
3962 {
3963 sqlite3_sleep(25);
3964 }
3965 }
3966 while( rc==SQLITE_OK || rc==SQLITE_BUSY || rc==SQLITE_LOCKED );
3967
3968 /* Release resources allocated by backup_init(). */
3969 (void)sqlite3_backup_finish(sb_dest);
3970 }
3971 rc = sqlite3_errcode(dest_db);
3972 }
3973 /* Close the database connection opened on database file zFilename
3974 ** and return the result of this function. */
3975 (void)sqlite3_close(dest_db);
3976 return rc;
3977}
3978
3979gboolean dt_database_snapshot(const struct dt_database_t *db)
3980{
3981 // backing up memory db is pointelss
3982 if(_is_mem_db(db))
3983 return FALSE;
3984 GDateTime *date_now = g_date_time_new_now_local();
3985 gchar *date_suffix = g_date_time_format(date_now, "%Y%m%d%H%M%S");
3986 g_date_time_unref(date_now);
3987
3988 const char *file_pattern = "%s-snp-%s";
3989 const char *temp_pattern = "%s-tmp-%s";
3990
3991 gchar *lib_backup_file = g_strdup_printf(file_pattern, db->dbfilename_library, date_suffix);
3992 gchar *lib_tmpbackup_file = g_strdup_printf(temp_pattern, db->dbfilename_library, date_suffix);
3993
3994 int rc = _backup_db(db->handle, "main", lib_tmpbackup_file, _print_backup_progress);
3995 if(!(rc==SQLITE_OK))
3996 {
3997 g_unlink(lib_tmpbackup_file);
3998 dt_free(lib_tmpbackup_file);
3999 dt_free(lib_backup_file);
4000 dt_free(date_suffix);
4001 return FALSE;
4002 }
4003 g_rename(lib_tmpbackup_file, lib_backup_file);
4004 g_chmod(lib_backup_file, S_IRUSR);
4005 dt_free(lib_tmpbackup_file);
4006 dt_free(lib_backup_file);
4007
4008 gchar *dat_backup_file = g_strdup_printf(file_pattern, db->dbfilename_data, date_suffix);
4009 gchar *dat_tmpbackup_file = g_strdup_printf(temp_pattern, db->dbfilename_data, date_suffix);
4010
4011 dt_free(date_suffix);
4012
4013 rc = _backup_db(db->handle, "data", dat_tmpbackup_file, _print_backup_progress);
4014 if(!(rc==SQLITE_OK))
4015 {
4016 g_unlink(dat_tmpbackup_file);
4017 dt_free(dat_tmpbackup_file);
4018 dt_free(dat_backup_file);
4019 return FALSE;
4020 }
4021 g_rename(dat_tmpbackup_file, dat_backup_file);
4022 g_chmod(dat_backup_file, S_IRUSR);
4023 dt_free(dat_tmpbackup_file);
4024 dt_free(dat_backup_file);
4025
4026 return TRUE;
4027}
4028
4029gboolean dt_database_maybe_snapshot(const struct dt_database_t *db)
4030{
4031 if(_is_mem_db(db))
4032 return FALSE;
4033
4034 const char *config = dt_conf_get_string_const("database/create_snapshot");
4035 if(!g_strcmp0(config, "never"))
4036 {
4037 // early bail out on "never"
4038 dt_print(DT_DEBUG_SQL, "[db backup] please consider enabling database snapshots.\n");
4039 return FALSE;
4040 }
4041 if(!g_strcmp0(config, "on close"))
4042 {
4043 // early bail out on "on close"
4044 dt_print(DT_DEBUG_SQL, "[db backup] performing unconditional snapshot.\n");
4045 return TRUE;
4046 }
4047
4048 GTimeSpan span_from_last_snap_required;
4049
4050 if(!g_strcmp0(config, "once a day"))
4051 {
4052 span_from_last_snap_required = G_TIME_SPAN_DAY;
4053 }
4054 else if(!g_strcmp0(config, "once a week"))
4055 {
4056 span_from_last_snap_required = G_TIME_SPAN_DAY * 7;
4057 }
4058 else if(!g_strcmp0(config, "once a month"))
4059 {
4060 //average month ;)
4061 span_from_last_snap_required = G_TIME_SPAN_DAY * 30;
4062 }
4063 else
4064 {
4065 // early bail out on "invalid value"
4066 dt_print(DT_DEBUG_SQL, "[db backup] invalid timespan requirement expecting never/on close/once a [day/week/month], got %s.\n", config);
4067 return TRUE;
4068 }
4069
4070 //we're in trouble zone - we have to determine when was the last snapshot done (including version upgrade snapshot) :/
4071 //this could be easy if we wrote date of last successful backup to config, but that's not really an option since
4072 //backup may done as last db operation, way after config file is closed. Plus we might be mixing dates of backups for
4073 //various library.db
4074
4075 dt_print(DT_DEBUG_SQL, "[db backup] checking snapshots existence.\n");
4076 GFile *library = g_file_parse_name(db->dbfilename_library);
4077 GFile *parent = g_file_get_parent(library);
4078
4079 if(IS_NULL_PTR(parent))
4080 {
4081 dt_print(DT_DEBUG_SQL, "[db backup] couldn't get library parent!.\n");
4082 g_object_unref(library);
4083 return FALSE;
4084 }
4085
4086 GError *error = NULL;
4087 GFileEnumerator *library_dir_files = g_file_enumerate_children(parent, G_FILE_ATTRIBUTE_STANDARD_NAME "," G_FILE_ATTRIBUTE_TIME_MODIFIED, G_FILE_QUERY_INFO_NONE, NULL, &error);
4088
4089 if(IS_NULL_PTR(library_dir_files))
4090 {
4091 dt_print(DT_DEBUG_SQL, "[db backup] couldn't enumerate library parent: %s.\n", error->message);
4092 g_object_unref(parent);
4093 g_object_unref(library);
4094 g_error_free(error);
4095 return FALSE;
4096 }
4097
4098 gchar *lib_basename = g_file_get_basename(library);
4099 g_object_unref(library);
4100
4101 gchar *lib_snap_format = g_strdup_printf("%s-snp-", lib_basename);
4102 gchar *lib_backup_format = g_strdup_printf("%s-pre-", lib_basename);
4103 dt_free(lib_basename);
4104
4105 GFileInfo *info = NULL;
4106 guint64 last_snap = 0;
4107
4108 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4109 {
4110 const char* fname = g_file_info_get_name(info);
4111 if(g_str_has_prefix(fname, lib_snap_format) || g_str_has_prefix(fname, lib_backup_format))
4112 {
4113 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4114 if(last_snap == 0)
4115 {
4116 last_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4117 g_object_unref(info);
4118 continue;
4119 }
4120 const guint64 try_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4121 if(try_snap > last_snap)
4122 {
4123 last_snap = try_snap;
4124 }
4125 }
4126 g_object_unref(info);
4127 }
4128 g_object_unref(parent);
4129 dt_free(lib_snap_format);
4130 dt_free(lib_backup_format);
4131
4132 if(error)
4133 {
4134 dt_print(DT_DEBUG_SQL, "[db backup] problem enumerating library parent: %s.\n", error->message);
4135 g_file_enumerator_close(library_dir_files, NULL, NULL);
4136 g_object_unref(library_dir_files);
4137 g_error_free(error);
4138 return FALSE;
4139 }
4140
4141 g_file_enumerator_close(library_dir_files, NULL, NULL);
4142 g_object_unref(library_dir_files);
4143
4144 GDateTime *date_now = g_date_time_new_now_local();
4145
4146 // Even if last_snap is 0 (didn't found any snaps) it produces proper date - unix epoch
4147 GDateTime *date_last_snap = g_date_time_new_from_unix_local(last_snap);
4148
4149 gchar *now_txt = g_date_time_format(date_now, "%Y%m%d%H%M%S");
4150 gchar *ls_txt = g_date_time_format(date_last_snap, "%Y%m%d%H%M%S");
4151 dt_print(DT_DEBUG_SQL, "[db backup] last snap: %s; curr date: %s.\n", ls_txt, now_txt);
4152 dt_free(now_txt);
4153 dt_free(ls_txt);
4154
4155 GTimeSpan span_from_last_snap = g_date_time_difference(date_now, date_last_snap);
4156
4157 g_date_time_unref(date_now);
4158 g_date_time_unref(date_last_snap);
4159
4160 return span_from_last_snap > span_from_last_snap_required;
4161}
4162
4163/* Parse integers in the form d (week days), dd (hours etc), ddd (ordinal days) or dddd (years) */
4164static gboolean _get_iso8601_int (const gchar *text, gsize length, gint *value)
4165{
4166 gsize i;
4167 guint v = 0;
4168
4169 if (length < 1 || length > 4)
4170 return FALSE;
4171
4172 for (i = 0; i < length; i++)
4173 {
4174 const gchar c = text[i];
4175 if (c < '0' || c > '9')
4176 return FALSE;
4177 v = v * 10 + (c - '0');
4178 }
4179
4180 *value = v;
4181 return TRUE;
4182}
4183
4184static gint _db_snap_sort(gconstpointer a, gconstpointer b, gpointer user_data)
4185{
4186 const gchar* e1 = (gchar *) a;
4187 const gchar* e2 = (gchar *) b;
4188
4189 //we assume that both end with date in
4190 //"%Y%m%d%H%M%S" format
4191
4192 gchar *datepos1 = g_strrstr(e1, "-snp-");
4193 gchar *datepos2 = g_strrstr(e2, "-snp-");
4194 if(IS_NULL_PTR(datepos1) || IS_NULL_PTR(datepos2))
4195 return 0;
4196
4197 datepos1 +=5; //skip "-snp-"
4198 datepos2 +=5; //skip "-snp-"
4199
4200 int year,month,day,hour,minute,second;
4201
4202 if(!_get_iso8601_int(datepos1, 4, &year) ||
4203 !_get_iso8601_int(datepos1+4, 2, &month) ||
4204 !_get_iso8601_int(datepos1+6, 2, &day) ||
4205 !_get_iso8601_int(datepos1+8, 2, &hour) ||
4206 !_get_iso8601_int(datepos1+10, 2, &minute) ||
4207 !_get_iso8601_int(datepos1+12, 2, &second)
4208 )
4209 {
4210 return 0;
4211 }
4212
4213 GDateTime *d1=g_date_time_new_local(year, month, day, hour, minute, second);
4214
4215 if(!_get_iso8601_int(datepos2, 4, &year) ||
4216 !_get_iso8601_int(datepos2+4, 2, &month) ||
4217 !_get_iso8601_int(datepos2+6, 2, &day) ||
4218 !_get_iso8601_int(datepos2+8, 2, &hour) ||
4219 !_get_iso8601_int(datepos2+10, 2, &minute) ||
4220 !_get_iso8601_int(datepos2+12, 2, &second)
4221 )
4222 {
4223 g_date_time_unref(d1);
4224 return 0;
4225 }
4226
4227 GDateTime *d2=g_date_time_new_local(year, month, day, hour, minute, second);
4228
4229 const gint ret = g_date_time_compare(d1, d2);
4230
4231 g_date_time_unref(d1);
4232 g_date_time_unref(d2);
4233
4234 return ret;
4235}
4236
4237char **dt_database_snaps_to_remove(const struct dt_database_t *db)
4238{
4239 if(_is_mem_db(db))
4240 return NULL;
4241
4242 const int keep_snaps = dt_conf_get_int("database/keep_snapshots");
4243
4244 if(keep_snaps < 0)
4245 return NULL;
4246
4247 dt_print(DT_DEBUG_SQL, "[db backup] checking snapshots existence.\n");
4248 GFile *lib_file = g_file_parse_name(db->dbfilename_library);
4249 GFile *lib_parent = g_file_get_parent(lib_file);
4250
4251 if(IS_NULL_PTR(lib_parent))
4252 {
4253 dt_print(DT_DEBUG_SQL, "[db backup] couldn't get library parent!.\n");
4254 g_object_unref(lib_file);
4255 return NULL;
4256 }
4257
4258 GFile *dat_file = g_file_parse_name(db->dbfilename_data);
4259 GFile *dat_parent = g_file_get_parent(dat_file);
4260
4261 if(IS_NULL_PTR(dat_parent))
4262 {
4263 dt_print(DT_DEBUG_SQL, "[db backup] couldn't get data parent!.\n");
4264 g_object_unref(dat_file);
4265 g_object_unref(lib_file);
4266 g_object_unref(lib_parent);
4267 }
4268
4269 gchar *lib_basename = g_file_get_basename(lib_file);
4270 g_object_unref(lib_file);
4271 gchar *lib_snap_format = g_strdup_printf("%s-snp-", lib_basename);
4272 gchar *lib_tmp_format = g_strdup_printf("%s-tmp-", lib_basename);
4273 dt_free(lib_basename);
4274
4275 gchar *dat_basename = g_file_get_basename(dat_file);
4276 g_object_unref(dat_file);
4277 gchar *dat_snap_format = g_strdup_printf("%s-snp-", dat_basename);
4278 gchar *dat_tmp_format = g_strdup_printf("%s-tmp-", dat_basename);
4279 dt_free(dat_basename);
4280
4281 GQueue *lib_snaps = g_queue_new();
4282 GQueue *dat_snaps = g_queue_new();
4283 GQueue *tmplib_snaps = g_queue_new();
4284 GQueue *tmpdat_snaps = g_queue_new();
4285
4286 if(g_file_equal(lib_parent, dat_parent))
4287 {
4288 //slight optimization if library and data are in same dir, we only have to scan one
4289 GError *error = NULL;
4290 GFileEnumerator *library_dir_files = g_file_enumerate_children(lib_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4291
4292 if(IS_NULL_PTR(library_dir_files))
4293 {
4294 dt_print(DT_DEBUG_SQL, "[db backup] couldn't enumerate library parent: %s.\n", error->message);
4295 g_object_unref(lib_parent);
4296 g_object_unref(dat_parent);
4297 dt_free(lib_snap_format);
4298 dt_free(dat_snap_format);
4299 dt_free(lib_tmp_format);
4300 dt_free(dat_tmp_format);
4301 g_queue_free(lib_snaps);
4302 g_queue_free(dat_snaps);
4303 g_queue_free(tmplib_snaps);
4304 g_queue_free(tmpdat_snaps);
4305 g_error_free(error);
4306 return NULL;
4307 }
4308
4309 GFileInfo *info = NULL;
4310
4311 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4312 {
4313 const char* fname = g_file_info_get_name(info);
4314 if(g_str_has_prefix(fname, lib_snap_format))
4315 {
4316 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4317 g_queue_insert_sorted(lib_snaps, g_strdup(fname), _db_snap_sort, NULL);
4318 }
4319 else if(g_str_has_prefix(fname, dat_snap_format))
4320 {
4321 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4322 g_queue_insert_sorted(dat_snaps, g_strdup(fname), _db_snap_sort, NULL);
4323 }
4324 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4325 {
4326 //we insert into single queue, since it's just dependent on parent
4327 g_queue_push_head(tmplib_snaps, g_strdup(fname));
4328 }
4329 g_object_unref(info);
4330 }
4331 dt_free(lib_snap_format);
4332 dt_free(dat_snap_format);
4333
4334 if(error)
4335 {
4336 dt_print(DT_DEBUG_SQL, "[db backup] problem enumerating library parent: %s.\n", error->message);
4337 g_object_unref(lib_parent);
4338 g_object_unref(dat_parent);
4339 dt_free(lib_tmp_format);
4340 dt_free(dat_tmp_format);
4341 g_queue_free_full(lib_snaps, g_free);
4342 g_queue_free_full(dat_snaps, g_free);
4343 g_queue_free_full(tmplib_snaps, g_free);
4344 g_queue_free_full(tmpdat_snaps, g_free);
4345 g_file_enumerator_close(library_dir_files, NULL, NULL);
4346 g_object_unref(library_dir_files);
4347 g_error_free(error);
4348 return NULL;
4349 }
4350 g_file_enumerator_close(library_dir_files, NULL, NULL);
4351 g_object_unref(library_dir_files);
4352 }
4353 else
4354 {
4355 //well... fun.
4356
4357 GError *error = NULL;
4358 GFileEnumerator *library_dir_files = g_file_enumerate_children(lib_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4359 if(IS_NULL_PTR(library_dir_files))
4360 {
4361 dt_print(DT_DEBUG_SQL, "[db backup] couldn't enumerate library parent: %s.\n", error->message);
4362 g_object_unref(lib_parent);
4363 g_object_unref(dat_parent);
4364 dt_free(lib_snap_format);
4365 dt_free(dat_snap_format);
4366 dt_free(lib_tmp_format);
4367 dt_free(dat_tmp_format);
4368 g_error_free(error);
4369 g_queue_free(lib_snaps);
4370 g_queue_free(dat_snaps);
4371 g_queue_free(tmplib_snaps);
4372 g_queue_free(tmpdat_snaps);
4373 return NULL;
4374 }
4375
4376 GFileEnumerator *data_dir_files = g_file_enumerate_children(dat_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4377 if(IS_NULL_PTR(data_dir_files))
4378 {
4379 dt_print(DT_DEBUG_SQL, "[db backup] couldn't enumerate data parent: %s.\n", error->message);
4380 g_object_unref(lib_parent);
4381 g_object_unref(dat_parent);
4382 dt_free(lib_snap_format);
4383 dt_free(dat_snap_format);
4384 dt_free(lib_tmp_format);
4385 dt_free(dat_tmp_format);
4386 g_file_enumerator_close(library_dir_files, NULL, NULL);
4387 g_object_unref(library_dir_files);
4388 g_error_free(error);
4389 g_queue_free(lib_snaps);
4390 g_queue_free(dat_snaps);
4391 g_queue_free(tmplib_snaps);
4392 g_queue_free(tmpdat_snaps);
4393 return NULL;
4394 }
4395
4396 GFileInfo *info = NULL;
4397
4398 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4399 {
4400 const char* fname = g_file_info_get_name(info);
4401 if(g_str_has_prefix(fname, lib_snap_format))
4402 {
4403 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4404 g_queue_insert_sorted(lib_snaps, g_strdup(fname), _db_snap_sort, NULL);
4405 }
4406 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4407 {
4408 // we remove all incomplete snaps matching pattern in BOTH dirs
4409 g_queue_push_head(tmplib_snaps, g_strdup(fname));
4410 }
4411 g_object_unref(info);
4412 }
4413 dt_free(lib_snap_format);
4414
4415 if(error)
4416 {
4417 dt_print(DT_DEBUG_SQL, "[db backup] problem enumerating library parent: %s.\n", error->message);
4418 g_object_unref(lib_parent);
4419 g_object_unref(dat_parent);
4420 dt_free(lib_tmp_format);
4421 dt_free(dat_tmp_format);
4422 g_queue_free_full(lib_snaps, g_free);
4423 g_queue_free(dat_snaps);
4424 g_queue_free_full(tmplib_snaps, g_free);
4425 g_queue_free(tmpdat_snaps);
4426 g_file_enumerator_close(library_dir_files, NULL, NULL);
4427 g_object_unref(library_dir_files);
4428 g_file_enumerator_close(data_dir_files, NULL, NULL);
4429 g_object_unref(data_dir_files);
4430 g_error_free(error);
4431 return NULL;
4432 }
4433 g_file_enumerator_close(library_dir_files, NULL, NULL);
4434 g_object_unref(library_dir_files);
4435
4436 while ((info = g_file_enumerator_next_file(data_dir_files, NULL, &error)))
4437 {
4438 const char* fname = g_file_info_get_name(info);
4439 if(g_str_has_prefix(fname, dat_snap_format))
4440 {
4441 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4442 g_queue_insert_sorted(dat_snaps, g_strdup(fname), _db_snap_sort, NULL);
4443 }
4444 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4445 {
4446 //we add to queue both matches - it just depends on parent
4447 g_queue_push_head(tmpdat_snaps, g_strdup(fname));
4448 }
4449 g_object_unref(info);
4450 }
4451 dt_free(dat_snap_format);
4452 dt_free(lib_tmp_format);
4453 dt_free(dat_tmp_format);
4454
4455 if(error)
4456 {
4457 dt_print(DT_DEBUG_SQL, "[db backup] problem enumerating data parent: %s.\n", error->message);
4458 g_object_unref(lib_parent);
4459 g_object_unref(dat_parent);
4460 g_queue_free_full(lib_snaps, g_free);
4461 g_queue_free_full(dat_snaps, g_free);
4462 g_queue_free_full(tmplib_snaps, g_free);
4463 g_queue_free_full(tmpdat_snaps, g_free);
4464 g_file_enumerator_close(data_dir_files, NULL, NULL);
4465 g_object_unref(data_dir_files);
4466 g_error_free(error);
4467 return NULL;
4468 }
4469
4470 g_file_enumerator_close(data_dir_files, NULL, NULL);
4471 g_object_unref(data_dir_files);
4472 }
4473
4474 // here we have list of snaps sorted in date order, now we have to
4475 // create from that list of snaps to be deleted and return that :D
4476
4477 GPtrArray *ret = g_ptr_array_new();
4478
4479 gchar *lib_parent_path = g_file_get_path(lib_parent);
4480 g_object_unref(lib_parent);
4481
4482 while(g_queue_get_length(lib_snaps) > keep_snaps)
4483 {
4484 gchar *head = g_queue_pop_head(lib_snaps);
4485 g_ptr_array_add(ret, g_strconcat(lib_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4486 dt_free(head);
4487 }
4488 while(!g_queue_is_empty(tmplib_snaps))
4489 {
4490 gchar *head = g_queue_pop_head(tmplib_snaps);
4491 g_ptr_array_add(ret, g_strconcat(lib_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4492 dt_free(head);
4493 }
4494 dt_free(lib_parent_path);
4495 g_queue_free_full(lib_snaps, g_free);
4496 g_queue_free_full(tmplib_snaps, g_free); // should be totally freed, but eh - this won't make doublefree
4497
4498 gchar *dat_parent_path = g_file_get_path(dat_parent);
4499 g_object_unref(dat_parent);
4500
4501 while(g_queue_get_length(dat_snaps) > keep_snaps)
4502 {
4503 gchar *head = g_queue_pop_head(dat_snaps);
4504 g_ptr_array_add(ret, g_strconcat(dat_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4505 dt_free(head);
4506 }
4507 while(!g_queue_is_empty(tmpdat_snaps))
4508 {
4509 gchar *head = g_queue_pop_head(tmpdat_snaps);
4510 g_ptr_array_add(ret, g_strconcat(dat_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4511 dt_free(head);
4512 }
4513 dt_free(dat_parent_path);
4514 g_queue_free_full(dat_snaps, g_free);
4515 g_queue_free_full(tmpdat_snaps, g_free);
4516
4517 g_ptr_array_add (ret, NULL);
4518
4519 return (char**)g_ptr_array_free(ret, FALSE);
4520}
4521
4522gchar *dt_database_get_most_recent_snap(const char* db_filename)
4523{
4524 if(!g_strcmp0(db_filename, ":memory:"))
4525 return NULL;
4526
4527 dt_print(DT_DEBUG_SQL, "[db backup] checking snapshots existence.\n");
4528 GFile *db_file = g_file_parse_name(db_filename);
4529 GFile *parent = g_file_get_parent(db_file);
4530
4531 if(IS_NULL_PTR(parent))
4532 {
4533 dt_print(DT_DEBUG_SQL, "[db backup] couldn't get database parent!.\n");
4534 g_object_unref(db_file);
4535 return NULL;
4536 }
4537
4538 GError *error = NULL;
4539 GFileEnumerator *db_dir_files = g_file_enumerate_children(parent, G_FILE_ATTRIBUTE_STANDARD_NAME "," G_FILE_ATTRIBUTE_TIME_MODIFIED, G_FILE_QUERY_INFO_NONE, NULL, &error);
4540
4541 if(IS_NULL_PTR(db_dir_files))
4542 {
4543 dt_print(DT_DEBUG_SQL, "[db backup] couldn't enumerate database parent: %s.\n", error->message);
4544 g_object_unref(parent);
4545 g_object_unref(db_file);
4546 g_error_free(error);
4547 return NULL;
4548 }
4549
4550 gchar *db_basename = g_file_get_basename(db_file);
4551 g_object_unref(db_file);
4552
4553 gchar *db_snap_format = g_strdup_printf("%s-snp-", db_basename);
4554 gchar *db_backup_format = g_strdup_printf("%s-pre-", db_basename);
4555 dt_free(db_basename);
4556
4557 GFileInfo *info = NULL;
4558 guint64 last_snap = 0;
4559 gchar *last_snap_name = NULL;
4560
4561 while ((info = g_file_enumerator_next_file(db_dir_files, NULL, &error)))
4562 {
4563 const char* fname = g_file_info_get_name(info);
4564 if(g_str_has_prefix(fname, db_snap_format) || g_str_has_prefix(fname, db_backup_format))
4565 {
4566 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4567 if(last_snap == 0)
4568 {
4569 last_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4570 last_snap_name = g_strdup(fname);
4571 g_object_unref(info);
4572 continue;
4573 }
4574 guint64 try_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4575 if(try_snap > last_snap)
4576 {
4577 last_snap = try_snap;
4578 dt_free(last_snap_name);
4579 last_snap_name = g_strdup(fname);
4580 }
4581 }
4582 g_object_unref(info);
4583 }
4584 dt_free(db_snap_format);
4585 dt_free(db_backup_format);
4586
4587 if(error)
4588 {
4589 dt_print(DT_DEBUG_SQL, "[db backup] problem enumerating database parent: %s.\n", error->message);
4590 g_file_enumerator_close(db_dir_files, NULL, NULL);
4591 g_object_unref(db_dir_files);
4592 g_error_free(error);
4593 dt_free(last_snap_name);
4594 return NULL;
4595 }
4596
4597 g_file_enumerator_close(db_dir_files, NULL, NULL);
4598 g_object_unref(db_dir_files);
4599
4600 if(IS_NULL_PTR(last_snap_name))
4601 {
4602 g_object_unref(parent);
4603 return NULL;
4604 }
4605
4606 gchar *parent_path = g_file_get_path(parent);
4607 g_object_unref(parent);
4608
4609 gchar *ret = g_strconcat(parent_path, G_DIR_SEPARATOR_S, last_snap_name, NULL);
4610 dt_free(parent_path);
4611 dt_free(last_snap_name);
4612
4613 return ret;
4614}
4615
4616// Nested transactions support
4617//
4618// NOTE: the nested support is actually not activated (see || TRUE below). This current
4619// implementation is just a refactoring of the previous code using:
4620// - dt_database_start_transaction()
4621// - dt_database_release_transaction()
4622// - dt_database_rollback_transaction()
4623//
4624// With this refactoring we can count and check for nested transaction and unmatched
4625// transaction routines. And it has been done to help further implementation for
4626// proper threading and nested transaction support.
4627//
4628void dt_database_start_transaction_debug(const struct dt_database_t *db)
4629{
4630 gpointer const owner = g_thread_self();
4631 const int batch_level = dt_atomic_get_int(&_trx_batch_level);
4632 if(batch_level > 0)
4633 {
4634 // If a batch transaction is active on this thread, suppress nested BEGIN/COMMIT
4635 // to avoid per-module transaction overhead during presets initialization.
4636 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4637 {
4638 dt_atomic_add_int(&_trxid, 1);
4639 return;
4640 }
4641 }
4642
4643 if(g_atomic_pointer_get(&_trx_owner) == owner)
4644 {
4645 dt_atomic_add_int(&_trxid, 1);
4646 return;
4647 }
4648
4649 dt_pthread_rwlock_wrlock(&darktable.database_threadsafe);
4650 g_atomic_pointer_set(&_trx_owner, owner);
4651
4652 const int trxid = dt_atomic_add_int(&_trxid, 1);
4653
4654 // if top level a simple unamed transaction is used BEGIN / COMMIT / ROLLBACK
4655 // otherwise we use a savepoint (named transaction).
4656
4657 if(trxid == 0)
4658 {
4659 // In theads application it may be safer to use an IMMEDIATE transaction:
4660 // "BEGIN IMMEDIATE TRANSACTION"
4661 // This implies "BEGIN DEFERRED TRANSACTION", which means
4662 // no write event is dispatched to DB until the first "COMMIT"
4663 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), "BEGIN TRANSACTION", NULL, NULL, NULL);
4664 }
4665#ifdef USE_NESTED_TRANSACTIONS
4666 else
4667 {
4668 char SQLTRX[32] = { 0 };
4669 g_snprintf(SQLTRX, sizeof(SQLTRX), "SAVEPOINT trx%d", trxid);
4670 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), SQLTRX, NULL, NULL, NULL);
4671 }
4672#endif
4673
4674 if(trxid > MAX_NESTED_TRANSACTIONS)
4675 fprintf(stderr, "[dt_database_start_transaction] more than %d nested transaction\n", MAX_NESTED_TRANSACTIONS);
4676}
4677
4678void dt_database_release_transaction_debug(const struct dt_database_t *db)
4679{
4680 gpointer const owner = g_thread_self();
4681 const int batch_level = dt_atomic_get_int(&_trx_batch_level);
4682 if(batch_level > 0)
4683 {
4684 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4685 {
4686 dt_atomic_sub_int(&_trxid, 1);
4687 return;
4688 }
4689 }
4690
4691 if(g_atomic_pointer_get(&_trx_owner) != owner)
4692 {
4693 fprintf(stderr, "[dt_database_release_transaction] COMMIT from non-owner thread\n");
4694 return;
4695 }
4696
4697 const int trxid = dt_atomic_sub_int(&_trxid, 1);
4698
4699 if(trxid <= 0)
4700 fprintf(stderr, "[dt_database_release_transaction] COMMIT outside a transaction\n");
4701
4702 if(trxid == 1)
4703 {
4704 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), "COMMIT TRANSACTION", NULL, NULL, NULL);
4705 g_atomic_pointer_set(&_trx_owner, NULL);
4706 dt_pthread_rwlock_unlock(&darktable.database_threadsafe);
4707 }
4708#ifdef USE_NESTED_TRANSACTIONS
4709 else
4710 {
4711 char SQLTRX[64] = { 0 };
4712 g_snprintf(SQLTRX, sizeof(SQLTRX), "RELEASE SAVEPOINT trx%d", trxid - 1);
4713 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), SQLTRX, NULL, NULL, NULL);
4714 }
4715#endif
4716}
4717
4718void dt_database_rollback_transaction(const struct dt_database_t *db)
4719{
4720 gpointer const owner = g_thread_self();
4721 if(g_atomic_pointer_get(&_trx_owner) != owner && g_atomic_pointer_get(&_trx_batch_owner) != owner)
4722 {
4723 fprintf(stderr, "[dt_database_rollback_transaction] ROLLBACK from non-owner thread\n");
4724 return;
4725 }
4726
4727 const int trxid = dt_atomic_sub_int(&_trxid, 1);
4728
4729 if(trxid <= 0)
4730 fprintf(stderr, "[dt_database_rollback_transaction] ROLLBACK outside a transaction\n");
4731
4732 if(trxid >= 1)
4733 {
4734 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), "ROLLBACK TRANSACTION", NULL, NULL, NULL);
4735 dt_atomic_set_int(&_trxid, 0);
4736 dt_atomic_set_int(&_trx_batch_level, 0);
4737 g_atomic_pointer_set(&_trx_owner, NULL);
4738 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4739 dt_pthread_rwlock_unlock(&darktable.database_threadsafe);
4740 }
4741#ifdef USE_NESTED_TRANSACTIONS
4742 else
4743 {
4744 char SQLTRX[64] = { 0 };
4745 g_snprintf(SQLTRX, sizeof(SQLTRX), "ROLLBACK TRANSACTION TO SAVEPOINT trx%d", trxid - 1);
4746 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), SQLTRX, NULL, NULL, NULL);
4747 }
4748#endif
4749}
4750
4751void dt_database_begin_transaction_batch(const struct dt_database_t *db)
4752{
4753 gpointer const owner = g_thread_self();
4754 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4755 {
4756 dt_atomic_add_int(&_trx_batch_level, 1);
4757 return;
4758 }
4759
4760 dt_pthread_rwlock_wrlock(&darktable.database_threadsafe);
4761 g_atomic_pointer_set(&_trx_owner, owner);
4762 g_atomic_pointer_set(&_trx_batch_owner, owner);
4763
4764 const int level = dt_atomic_add_int(&_trx_batch_level, 1);
4765 if(level == 0)
4766 {
4767 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), "BEGIN TRANSACTION", NULL, NULL, NULL);
4768 }
4769}
4770
4771void dt_database_end_transaction_batch(const struct dt_database_t *db)
4772{
4773 gpointer const owner = g_thread_self();
4774 if(g_atomic_pointer_get(&_trx_batch_owner) != owner)
4775 {
4776 dt_print(DT_DEBUG_SQL, "[dt_database_end_transaction_batch] COMMIT from non-owner thread\n");
4777 return;
4778 }
4779
4780 const int level = dt_atomic_sub_int(&_trx_batch_level, 1);
4781 if(level <= 0)
4782 {
4783 dt_print(DT_DEBUG_SQL, "[dt_database_end_transaction_batch] COMMIT outside a batch transaction\n");
4784 dt_atomic_set_int(&_trx_batch_level, 0);
4785 g_atomic_pointer_set(&_trx_owner, NULL);
4786 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4787 dt_pthread_rwlock_unlock(&darktable.database_threadsafe);
4788 return;
4789 }
4790
4791 if(level == 1)
4792 {
4793 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), "COMMIT TRANSACTION", NULL, NULL, NULL);
4794 g_atomic_pointer_set(&_trx_owner, NULL);
4795 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4796 dt_pthread_rwlock_unlock(&darktable.database_threadsafe);
4797 }
4798}
4799
4800// clang-format off
4801// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
4802// vim: shiftwidth=2 expandtab tabstop=2 cindent
4803// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
4804// clang-format on
error
static void error(char *msg)
Definition ashift_lsd.c:202
TRUE
#define TRUE
Definition ashift_lsd.c:162
FALSE
#define FALSE
Definition ashift_lsd.c:158
dt_atomic_set_int
void dt_atomic_set_int(dt_atomic_int *var, int value)
dt_atomic_get_int
int dt_atomic_get_int(dt_atomic_int *var)
dt_atomic_sub_int
int dt_atomic_sub_int(dt_atomic_int *var, int decr)
dt_atomic_add_int
int dt_atomic_add_int(dt_atomic_int *var, int incr)
dt_atomic_int
atomic_int dt_atomic_int
Definition atomic.h:66
kind
static const dt_adaptation_t kind
Definition chromatic_adaptation.h:124
i
const float i
Definition colorspaces_inline_conversions.h:440
n
const float n
Definition colorspaces_inline_conversions.h:678
void
typedef void((*dt_cache_allocate_t)(void *userdata, dt_cache_entry_t *entry))
name
char * name
Definition common/metadata.c:61
darktable_package_version
const char darktable_package_version[]
dt_conf_get_string
gchar * dt_conf_get_string(const char *name)
Definition control/conf.c:371
dt_conf_get_int
int dt_conf_get_int(const char *name)
Definition control/conf.c:241
dt_conf_set_string
void dt_conf_set_string(const char *name, const char *val)
Definition control/conf.c:167
dt_conf_get_string_const
const char * dt_conf_get_string_const(const char *name)
Definition control/conf.c:377
dt_concat_path_file
void dt_concat_path_file(char destination[PATH_MAX], const char path[PATH_MAX], const char *const file)
Definition darktable.c:1947
darktable
darktable_t darktable
Definition darktable.c:183
dt_print
void dt_print(dt_debug_thread_t thread, const char *msg,...)
Definition darktable.c:1600
UNKNOWN_IMAGE
#define UNKNOWN_IMAGE
Definition darktable.h:194
DT_DEBUG_SQL
@ DT_DEBUG_SQL
Definition darktable.h:745
dt_free_gpointer
static void dt_free_gpointer(gpointer ptr)
Definition darktable.h:485
dt_free
#define dt_free(ptr)
Definition darktable.h:478
value
static const dt_aligned_pixel_simd_t value
Definition darktable.h:599
PATH_MAX
#define PATH_MAX
Definition darktable.h:1189
IS_NULL_PTR
#define IS_NULL_PTR(p)
C is way too permissive with !=, == and if(var) checks, which can mean too many things depending on w...
Definition darktable.h:293
_lock_single_database
static gboolean _lock_single_database(dt_database_t *db, const char *dbfilename, char **lockfile)
Definition database.c:2793
_print_backup_progress
static void _print_backup_progress(int remaining, int total)
Definition database.c:3923
dt_database_end_transaction_batch
void dt_database_end_transaction_batch(const struct dt_database_t *db)
Definition database.c:4771
dt_database_backup
void dt_database_backup(const char *filename)
Definition database.c:2931
TRY_PREPARE
#define TRY_PREPARE(_stmt, _query, _message)
Definition database.c:468
dt_database_get_most_recent_snap
gchar * dt_database_get_most_recent_snap(const char *db_filename)
Definition database.c:4522
dt_database_cleanup_busy_statements
void dt_database_cleanup_busy_statements(const struct dt_database_t *db)
Definition database.c:3716
_trx_batch_owner
static gpointer _trx_batch_owner
Definition database.c:96
pid_is_alive
static gboolean pid_is_alive(int pid)
Definition database.c:2746
MAX_NESTED_TRANSACTIONS
#define MAX_NESTED_TRANSACTIONS
Definition database.c:91
ask_for_upgrade
void ask_for_upgrade(const gchar *dbname, const gboolean has_gui)
Definition database.c:2899
dt_database_optimize
void dt_database_optimize(const struct dt_database_t *db)
Definition database.c:3913
_get_pragma_string_val
gchar * _get_pragma_string_val(sqlite3 *db, const char *pragma)
Definition database.c:2991
_trxid
static dt_atomic_int _trxid
Definition database.c:93
_is_mem_db
static gboolean _is_mem_db(const struct dt_database_t *db)
Definition database.c:3831
dt_database_snapshot
gboolean dt_database_snapshot(const struct dt_database_t *db)
Definition database.c:3979
_trx_owner
static gpointer _trx_owner
Definition database.c:95
_sanitize_db
static void _sanitize_db(dt_database_t *db)
Definition database.c:2561
dt_database_perform_maintenance
void dt_database_perform_maintenance(const struct dt_database_t *db)
Definition database.c:3735
FINALIZE
#define FINALIZE
Definition database.c:482
ERRCHECK
#define ERRCHECK
Definition database.c:3734
dt_database_start_transaction_debug
void dt_database_start_transaction_debug(const struct dt_database_t *db)
Definition database.c:4628
_migrate_schema
static gboolean _migrate_schema(dt_database_t *db, int version)
Definition database.c:131
dt_database_destroy
void dt_database_destroy(const dt_database_t *db)
Definition database.c:3633
dt_database_show_error
gboolean dt_database_show_error(const dt_database_t *db)
Definition database.c:2651
dt_database_begin_transaction_batch
void dt_database_begin_transaction_batch(const struct dt_database_t *db)
Definition database.c:4751
dt_database_get
sqlite3 * dt_database_get(const dt_database_t *db)
Definition database.c:3653
_database_delete_mipmaps_files
static void _database_delete_mipmaps_files()
Definition database.c:3692
_upgrade_library_schema_step
static int _upgrade_library_schema_step(dt_database_t *db, int version)
Definition database.c:485
TRY_EXEC
#define TRY_EXEC(_query, _message)
Definition database.c:442
dt_database_snaps_to_remove
char ** dt_database_snaps_to_remove(const struct dt_database_t *db)
Definition database.c:4237
dt_database_maybe_maintenance
gboolean dt_database_maybe_maintenance(const struct dt_database_t *db, const gboolean has_gui, const gboolean closing_time)
Definition database.c:3836
dt_database_get_lock_acquired
gboolean dt_database_get_lock_acquired(const dt_database_t *db)
Definition database.c:3711
_upgrade_data_schema_step
static int _upgrade_data_schema_step(dt_database_t *db, int version)
Definition database.c:2110
_SQLITE3_EXEC
#define _SQLITE3_EXEC(a, b, c, d, e)
Definition database.c:122
dt_database_init
dt_database_t * dt_database_init(const char *alternative, const gboolean load_data, const gboolean has_gui)
Definition database.c:3015
dt_database_release_transaction_debug
void dt_database_release_transaction_debug(const struct dt_database_t *db)
Definition database.c:4678
_get_pragma_int_val
int _get_pragma_int_val(sqlite3 *db, const char *pragma)
Definition database.c:2975
_create_library_schema
static void _create_library_schema(dt_database_t *db)
Definition database.c:2358
_upgrade_library_schema
static gboolean _upgrade_library_schema(dt_database_t *db, int version)
Definition database.c:2329
_backup_db
static int _backup_db(sqlite3 *src_db, const char *src_db_name, const char *dest_filename, void(*xProgress)(int, int))
Definition database.c:3929
_trx_batch_level
static dt_atomic_int _trx_batch_level
Definition database.c:94
_database_migrate_to_xdg_structure
static void _database_migrate_to_xdg_structure()
Definition database.c:3663
dt_database_get_path
const gchar * dt_database_get_path(const struct dt_database_t *db)
Definition database.c:3658
CURRENT_DATABASE_VERSION_LIBRARY
#define CURRENT_DATABASE_VERSION_LIBRARY
Definition database.c:87
dt_database_rollback_transaction
void dt_database_rollback_transaction(const struct dt_database_t *db)
Definition database.c:4718
_upgrade_data_schema
static gboolean _upgrade_data_schema(dt_database_t *db, int version)
Definition database.c:2344
_create_memory_schema
static void _create_memory_schema(dt_database_t *db)
Definition database.c:2517
TRY_STEP
#define TRY_STEP(_stmt, _expected, _message)
Definition database.c:455
_ask_for_maintenance
gboolean _ask_for_maintenance(const gboolean has_gui, const gboolean closing_time, const guint64 size)
Definition database.c:3789
_lock_databases
static gboolean _lock_databases(dt_database_t *db)
Definition database.c:2886
_get_iso8601_int
static gboolean _get_iso8601_int(const gchar *text, gsize length, gint *value)
Definition database.c:4164
dt_database_maybe_snapshot
gboolean dt_database_maybe_snapshot(const struct dt_database_t *db)
Definition database.c:4029
_create_data_schema
static void _create_data_schema(dt_database_t *db)
Definition database.c:2464
_db_snap_sort
static gint _db_snap_sort(gconstpointer a, gconstpointer b, gpointer user_data)
Definition database.c:4184
CURRENT_DATABASE_VERSION_DATA
#define CURRENT_DATABASE_VERSION_DATA
Definition database.c:88
dt_datetime_exif_to_gdatetime
GDateTime * dt_datetime_exif_to_gdatetime(const char *exif, const GTimeZone *tz)
Definition datetime.c:246
dt_datetime_gdatetime_to_gtimespan
GTimeSpan dt_datetime_gdatetime_to_gtimespan(GDateTime *gdt)
Definition datetime.c:445
DT_DEBUG_SQLITE3_EXEC
#define DT_DEBUG_SQLITE3_EXEC(a, b, c, d, e)
Definition debug.h:99
dt_pthread_rwlock_unlock
static int dt_pthread_rwlock_unlock(dt_pthread_rwlock_t *rwlock)
Definition dtpthread.h:452
dt_pthread_rwlock_wrlock
static int dt_pthread_rwlock_wrlock(dt_pthread_rwlock_t *rwlock)
Definition dtpthread.h:534
dt_loc_get_user_cache_dir
void dt_loc_get_user_cache_dir(char *cachedir, size_t bufsize)
Definition file_location.c:314
dt_loc_get_datadir
void dt_loc_get_datadir(char *datadir, size_t bufsize)
Definition file_location.c:322
dt_loc_get_user_config_dir
void dt_loc_get_user_config_dir(char *configdir, size_t bufsize)
Definition file_location.c:310
file_location.h
dt_gui_show_standalone_three_choice_dialog
int dt_gui_show_standalone_three_choice_dialog(const char *title, const char *markup, const char *first_text, const char *second_text, const char *third_text)
Definition gtk.c:2057
dt_gui_show_standalone_yes_no_dialog
gboolean dt_gui_show_standalone_yes_no_dialog(const char *title, const char *markup, const char *no_text, const char *yes_text)
Definition gtk.c:1952
dt_image_flags_t
dt_image_flags_t
Definition image.h:91
DT_IMAGE_RAW
@ DT_IMAGE_RAW
Definition image.h:111
DT_IMAGE_HDR
@ DT_IMAGE_HDR
Definition image.h:113
DT_IMAGE_LDR
@ DT_IMAGE_LDR
Definition image.h:109
dt_imageio_get_type_from_extension
dt_image_flags_t dt_imageio_get_type_from_extension(const char *extension)
Map Exiv2 preview MIME types to decoder format identifiers.
Definition imageio.c:181
dt_ioppr_get_iop_order_list_version
GList * dt_ioppr_get_iop_order_list_version(dt_iop_order_t version)
Return the built-in order list for a given version.
Definition iop_order.c:1044
dt_sort_iop_list_by_order_f
gint dt_sort_iop_list_by_order_f(gconstpointer a, gconstpointer b)
Compare two list nodes holding modules by iop_order.
Definition iop_order.c:893
dt_ioppr_get_iop_order_list_kind
dt_iop_order_t dt_ioppr_get_iop_order_list_kind(GList *iop_order_list)
Determine the kind of an order list by inspecting its content.
Definition iop_order.c:905
dt_ioppr_serialize_text_iop_order_list
char * dt_ioppr_serialize_text_iop_order_list(GList *iop_order_list)
Serialize an order list to a text representation.
Definition iop_order.c:2539
iop_order.h
dt_iop_order_t
dt_iop_order_t
Definition iop_order.h:143
DT_IOP_ORDER_ANSEL_RAW
@ DT_IOP_ORDER_ANSEL_RAW
Definition iop_order.h:148
DT_IOP_ORDER_LEGACY
@ DT_IOP_ORDER_LEGACY
Definition iop_order.h:145
DT_IOP_ORDER_CUSTOM
@ DT_IOP_ORDER_CUSTOM
Definition iop_order.h:144
v
const float v
Definition iop_profile.h:221
legacy_presets.h
dt_legacy_presets_create
static void dt_legacy_presets_create(const struct dt_database_t *db)
Definition legacy_presets.h:1130
k
float *const restrict const size_t k
Definition luminance_mask.h:78
size
size_t size
Definition mipmap_cache.c:3
flags
dt_mipmap_buffer_dsc_flags flags
Definition mipmap_cache.c:4
GtkWidget
struct _GtkWidget GtkWidget
Definition splash.h:29
sqlite3IcuInit
int sqlite3IcuInit(sqlite3 *db)
Definition sqliteicu.c:525
day
static const char *const day[7]
Definition strptime.c:97
darktable_t::database_threadsafe
dt_pthread_rwlock_t database_threadsafe
Definition darktable.h:844
darktable_t::utc_tz
GTimeZone * utc_tz
Definition darktable.h:860
dt_database_t
Definition database.c:99
dt_database_t::error_other_pid
int error_other_pid
Definition database.c:112
dt_database_t::lockfile_data
gchar * lockfile_data
Definition database.c:103
dt_database_t::dbfilename_library
gchar * dbfilename_library
Definition database.c:106
dt_database_t::handle
sqlite3 * handle
Definition database.c:109
dt_database_t::lockfile_library
gchar * lockfile_library
Definition database.c:106
dt_database_t::error_message
gchar * error_message
Definition database.c:111
dt_database_t::lock_acquired
gboolean lock_acquired
Definition database.c:100
dt_database_t::dbfilename_data
gchar * dbfilename_data
Definition database.c:103
dt_database_t::error_dbfilename
gchar * error_dbfilename
Definition database.c:111
dt_iop_order_entry_t
Definition iop_order.h:154
dt_iop_order_entry_t::operation
char operation[20]
Definition iop_order.h:160
dt_iop_order_entry_t::o
union dt_iop_order_entry_t::@8 o
dt_iop_order_entry_t::iop_order_f
double iop_order_f
Definition iop_order.h:156
dt_iop_order_entry_t::instance
int32_t instance
Definition iop_order.h:161
MIN
#define MIN(a, b)
Definition thinplate.c:32
MAX
#define MAX(a, b)
Definition thinplate.c:29
dt_util_foo_to_utf8
gchar * dt_util_foo_to_utf8(const char *string)
Definition utility.c:392