Ansel 0.0
A darktable fork - bloat + design vision
Loading...
Searching...
No Matches
database.c
Go to the documentation of this file.
1/*
2 This file is part of darktable,
3 Copyright (C) 2011-2012 Edouard Gomez.
4 Copyright (C) 2011 Henrik Andersson.
5 Copyright (C) 2011-2012 johannes hanika.
6 Copyright (C) 2011 Moritz Lipp.
7 Copyright (C) 2012, 2017 Christian Tellefsen.
8 Copyright (C) 2012 José Carlos García Sogo.
9 Copyright (C) 2012 Jérémy Rosen.
10 Copyright (C) 2012 Richard Wonka.
11 Copyright (C) 2012 Simon Spannagel.
12 Copyright (C) 2012-2020 Tobias Ellinghaus.
13 Copyright (C) 2013-2014, 2018-2022 Pascal Obry.
14 Copyright (C) 2014-2016 Roman Lebedev.
15 Copyright (C) 2015 Pedro Côrte-Real.
16 Copyright (C) 2016 piterdias.
17 Copyright (C) 2017, 2019, 2021 luzpaz.
18 Copyright (C) 2017-2018 Rikard Öxler.
19 Copyright (C) 2018-2019 Edgardo Hoszowski.
20 Copyright (C) 2018 Mario Lueder.
21 Copyright (C) 2018 Peter Budai.
22 Copyright (C) 2019 jakubfi.
23 Copyright (C) 2019-2022 Philippe Weyland.
24 Copyright (C) 2020 Heiko Bauke.
25 Copyright (C) 2020-2021 Hubert Kowalski.
26 Copyright (C) 2020 JP Verrue.
27 Copyright (C) 2020 jpverrue.
28 Copyright (C) 2021 Diederik Ter Rahe.
29 Copyright (C) 2021 Hanno Schwalm.
30 Copyright (C) 2021 HansBull.
31 Copyright (C) 2021 Harald.
32 Copyright (C) 2021 Ralf Brown.
33 Copyright (C) 2022 Aldric Renaudin.
34 Copyright (C) 2022-2023, 2025-2026 Aurélien PIERRE.
35 Copyright (C) 2022 Martin Bařinka.
36 Copyright (C) 2022 Miloš Komarčević.
37 Copyright (C) 2023 Maurizio Paglia.
38 Copyright (C) 2025 Alynx Zhou.
39 Copyright (C) 2025-2026 Guillaume Stutin.
40
41 darktable is free software: you can redistribute it and/or modify
42 it under the terms of the GNU General Public License as published by
43 the Free Software Foundation, either version 3 of the License, or
44 (at your option) any later version.
45
46 darktable is distributed in the hope that it will be useful,
47 but WITHOUT ANY WARRANTY; without even the implied warranty of
48 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
49 GNU General Public License for more details.
50
51 You should have received a copy of the GNU General Public License
52 along with darktable. If not, see <http://www.gnu.org/licenses/>.
53*/
54
55#ifdef HAVE_CONFIG_H
56#include "config.h"
57#endif
58
59#include "common/atomic.h"
60#include "common/database.h"
61#include "common/darktable.h"
62#include "common/datetime.h"
63#include "common/debug.h"
64#include "common/file_location.h"
65#include "common/iop_order.h"
66#include "common/styles.h"
67#include "common/history.h"
68#ifdef HAVE_ICU
69#include "common/sqliteicu.h"
70#endif
71#include "control/conf.h"
72#include "control/control.h"
73#include "gui/legacy_presets.h"
74
75#include <gio/gio.h>
76#include <glib.h>
77#include <glib/gstdio.h>
78
79#include <errno.h>
80#include <fcntl.h>
81#include <signal.h>
82#include <sys/stat.h>
83#include <sys/types.h>
84
85// whenever _create_*_schema() gets changed you HAVE to bump this version and add an update path to
86// _upgrade_*_schema_step()!
87#define CURRENT_DATABASE_VERSION_LIBRARY 36
88#define CURRENT_DATABASE_VERSION_DATA 9
89
90// #define USE_NESTED_TRANSACTIONS
91#define MAX_NESTED_TRANSACTIONS 0
92/* transaction id */
93static dt_atomic_int _trxid;
94static dt_atomic_int _trx_batch_level;
95static gpointer _trx_owner = NULL;
96static gpointer _trx_batch_owner = NULL;
97
98typedef struct dt_database_t
99{
100 gboolean lock_acquired;
101
102 /* data database filename */
103 gchar *dbfilename_data, *lockfile_data;
104
105 /* library database filename */
106 gchar *dbfilename_library, *lockfile_library;
107
108 /* ondisk DB */
109 sqlite3 *handle;
110
111 gchar *error_message, *error_dbfilename;
112 int error_other_pid;
113} dt_database_t;
114
115
116/* migrates database from old place to new */
117static void _database_migrate_to_xdg_structure();
118
119/* delete old mipmaps files */
120static void _database_delete_mipmaps_files();
121
122#define _SQLITE3_EXEC(a, b, c, d, e) \
123 if(sqlite3_exec(a, b, c, d, e) != SQLITE_OK) \
124 { \
125 all_ok = FALSE; \
126 failing_query = b; \
127 goto end; \
128 }
129
130/* migrate from the legacy db format (with the 'settings' blob) to the first version this system knows */
131static gboolean _migrate_schema(dt_database_t *db, int version)
132{
133 gboolean all_ok = TRUE;
134 const char *failing_query = NULL;
135 sqlite3_stmt *stmt;
136 sqlite3_stmt *innerstmt;
137
138 if(version != 36) // if anyone shows up with an older db we can probably add extra code
139 return FALSE;
140
141 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
142 // clang-format off
143 // remove stuff that is either no longer needed or that got renamed
144 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.lock", NULL, NULL, NULL);
145 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.settings", NULL, NULL, NULL); // yes, we do this in many
146 // places. because it's really
147 // important to not miss it in
148 // any code path.
149 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.group_id_index", NULL, NULL, NULL);
150 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.imgid_index", NULL, NULL, NULL);
151 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.mipmaps", NULL, NULL, NULL);
152 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.mipmap_timestamps", NULL, NULL, NULL);
153 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.dt_migration_table", NULL, NULL, NULL);
154
155 // using _create_library_schema() and filling that with the old data doesn't work since we always want to generate
156 // version 1 tables
158 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)",
159 NULL, NULL, NULL);
160 _SQLITE3_EXEC(db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', 1)",
161 NULL, NULL, NULL);
163 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.film_rolls_folder_index ON film_rolls (folder)",
164 NULL, NULL, NULL);
166 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN orientation INTEGER", NULL, NULL, NULL);
167 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN focus_distance REAL", NULL, NULL, NULL);
168 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN group_id INTEGER", NULL, NULL, NULL);
169 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN histogram BLOB", NULL, NULL, NULL);
170 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN lightmap BLOB", NULL, NULL, NULL);
171 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN longitude REAL", NULL, NULL, NULL);
172 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN latitude REAL", NULL, NULL, NULL);
173 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN color_matrix BLOB", NULL, NULL, NULL);
174 // the colorspace as specified in some image types
175 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN colorspace INTEGER", NULL, NULL, NULL);
176 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN version INTEGER", NULL, NULL, NULL);
177 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN max_version INTEGER", NULL, NULL, NULL);
178 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET orientation = -1 WHERE orientation IS NULL", NULL, NULL, NULL);
179 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET focus_distance = -1 WHERE focus_distance IS NULL",
180 NULL, NULL, NULL);
181 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET group_id = id WHERE group_id IS NULL", NULL, NULL, NULL);
182 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET max_version = (SELECT COUNT(*)-1 FROM main.images i WHERE "
183 "i.filename = main.images.filename AND "
184 "i.film_id = main.images.film_id) WHERE max_version IS NULL",
185 NULL, NULL, NULL);
186 _SQLITE3_EXEC(
187 db->handle,
188 "UPDATE main.images SET version = (SELECT COUNT(*) FROM main.images i "
189 "WHERE i.filename = main.images.filename AND "
190 "i.film_id = main.images.film_id AND i.id < main.images.id) WHERE version IS NULL",
191 NULL, NULL, NULL);
192 // make sure we have AUTOINCREMENT on imgid --> move the whole thing away and recreate the table :(
193 _SQLITE3_EXEC(db->handle, "ALTER TABLE main.images RENAME TO dt_migration_table", NULL, NULL, NULL);
194 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.images_group_id_index", NULL, NULL, NULL);
195 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.images_film_id_index", NULL, NULL, NULL);
196 _SQLITE3_EXEC(
197 db->handle,
198 "CREATE TABLE main.images (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
199 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
200 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
201 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
202 "output_width INTEGER, output_height INTEGER, crop REAL, "
203 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
204 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
205 "caption VARCHAR, description VARCHAR, license VARCHAR, sha1sum CHAR(40), "
206 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
207 "latitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, max_version INTEGER)",
208 NULL, NULL, NULL);
209 _SQLITE3_EXEC(db->handle, "CREATE INDEX main.images_group_id_index ON images (group_id)", NULL, NULL, NULL);
210 _SQLITE3_EXEC(db->handle, "CREATE INDEX main.images_film_id_index ON images (film_id)", NULL, NULL, NULL);
211 _SQLITE3_EXEC(
212 db->handle,
213 "INSERT INTO main.images (id, group_id, film_id, width, height, filename, maker, model, "
214 "lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags, "
215 "output_width, output_height, crop, raw_parameters, raw_denoise_threshold, "
216 "raw_auto_bright_threshold, raw_black, raw_maximum, caption, description, license, sha1sum, "
217 "orientation, histogram, lightmap, longitude, latitude, color_matrix, colorspace, version, "
218 "max_version) "
219 "SELECT id, group_id, film_id, width, height, filename, maker, model, lens, exposure, aperture, iso, "
220 "focal_length, focus_distance, datetime_taken, flags, output_width, output_height, crop, "
221 "raw_parameters, raw_denoise_threshold, raw_auto_bright_threshold, raw_black, raw_maximum, "
222 "caption, description, license, sha1sum, orientation, histogram, lightmap, longitude, "
223 "latitude, color_matrix, colorspace, version, max_version FROM dt_migration_table",
224 NULL, NULL, NULL);
225 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
227 // selected_images should have a primary key. add it if it's missing:
228 _SQLITE3_EXEC(db->handle, "CREATE TEMPORARY TABLE dt_migration_table (imgid INTEGER)", NULL, NULL, NULL);
229 _SQLITE3_EXEC(db->handle, "INSERT INTO dt_migration_table SELECT imgid FROM main.selected_images", NULL, NULL,
230 NULL);
231 _SQLITE3_EXEC(db->handle, "DROP TABLE main.selected_images", NULL, NULL, NULL);
232 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.selected_images (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
233 _SQLITE3_EXEC(db->handle, "INSERT OR IGNORE INTO main.selected_images SELECT imgid FROM dt_migration_table",
234 NULL, NULL, NULL);
235 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
237 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
238 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
239 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
240 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
241 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.history_imgid_index ON history (imgid)", NULL, NULL,
242 NULL);
243 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
244 NULL, NULL);
245 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET multi_priority = 0 WHERE multi_priority IS NULL", NULL, NULL,
246 NULL);
247 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL, NULL);
249 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.mask (imgid INTEGER, formid INTEGER, form INTEGER, "
250 "name VARCHAR(256), version INTEGER, "
251 "points BLOB, points_count INTEGER, source BLOB)",
252 NULL, NULL, NULL);
253 sqlite3_exec(db->handle, "ALTER TABLE main.mask ADD COLUMN source BLOB", NULL, NULL,
254 NULL); // in case the table was there already but missed that column
256 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.tagged_images_tagid_index ON tagged_images (tagid)",
257 NULL, NULL, NULL);
259 _SQLITE3_EXEC(db->handle,
260 "CREATE TABLE IF NOT EXISTS main.styles (id INTEGER, name VARCHAR, description VARCHAR)", NULL,
261 NULL, NULL);
262 sqlite3_exec(db->handle, "ALTER TABLE main.styles ADD COLUMN id INTEGER", NULL, NULL, NULL);
263 _SQLITE3_EXEC(db->handle, "UPDATE main.styles SET id = rowid WHERE id IS NULL", NULL, NULL, NULL);
265 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.style_items (styleid INTEGER, num INTEGER, module "
266 "INTEGER, operation VARCHAR(256), op_params BLOB, "
267 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, multi_priority "
268 "INTEGER, multi_name VARCHAR(256))",
269 NULL, NULL, NULL);
270 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
271 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
272 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
273 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
274 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
275 NULL, NULL);
276 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET multi_priority = 0 WHERE multi_priority IS NULL", NULL,
277 NULL, NULL);
278 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL,
279 NULL);
281 // color_labels could have a PRIMARY KEY that we don't want
282 _SQLITE3_EXEC(db->handle, "CREATE TEMPORARY TABLE dt_migration_table (imgid INTEGER, color INTEGER)", NULL,
283 NULL, NULL);
284 _SQLITE3_EXEC(db->handle, "INSERT INTO dt_migration_table SELECT imgid, color FROM main.color_labels", NULL,
285 NULL, NULL);
286 _SQLITE3_EXEC(db->handle, "DROP TABLE main.color_labels", NULL, NULL, NULL);
287 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.color_labels (imgid INTEGER, color INTEGER)", NULL, NULL, NULL);
288 _SQLITE3_EXEC(db->handle, "CREATE UNIQUE INDEX main.color_labels_idx ON color_labels (imgid, color)", NULL, NULL,
289 NULL);
290 _SQLITE3_EXEC(db->handle, "INSERT OR IGNORE INTO main.color_labels SELECT imgid, color FROM dt_migration_table",
291 NULL, NULL, NULL);
292 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
294 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.meta_data (id INTEGER, key INTEGER, value VARCHAR)",
295 NULL, NULL, NULL);
296 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.metadata_index ON meta_data (id, key)", NULL, NULL,
297 NULL);
299 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.presets (name VARCHAR, description VARCHAR, "
300 "operation VARCHAR, op_version INTEGER, op_params BLOB, "
301 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, multi_priority "
302 "INTEGER, multi_name VARCHAR(256), "
303 "model VARCHAR, maker VARCHAR, lens VARCHAR, iso_min REAL, iso_max REAL, "
304 "exposure_min REAL, exposure_max REAL, "
305 "aperture_min REAL, aperture_max REAL, focal_length_min REAL, focal_length_max "
306 "REAL, writeprotect INTEGER, "
307 "autoapply INTEGER, filter INTEGER, def INTEGER, isldr INTEGER)",
308 NULL, NULL, NULL);
309 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN op_version INTEGER", NULL, NULL, NULL);
310 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
311 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
312 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
313 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
314 // the unique index only works if the db doesn't have any (name, operation, op_version) more than once.
315 // apparently there are dbs out there which do have that. :(
316 sqlite3_prepare_v2(db->handle,
317 "SELECT p.rowid, p.name, p.operation, p.op_version FROM main.presets p INNER JOIN "
318 "(SELECT * FROM (SELECT rowid, name, operation, op_version, COUNT(*) AS count "
319 "FROM main.presets GROUP BY name, operation, op_version) WHERE count > 1) s "
320 "ON p.name = s.name AND p.operation = s.operation AND p.op_version = s.op_version",
321 -1, &stmt, NULL);
322 // clang-format on
323 char *last_name = NULL, *last_operation = NULL;
324 int last_op_version = 0;
325 int i = 0;
326 while(sqlite3_step(stmt) == SQLITE_ROW)
327 {
328 const int rowid = sqlite3_column_int(stmt, 0);
329 const char *name = (const char *)sqlite3_column_text(stmt, 1);
330 const char *operation = (const char *)sqlite3_column_text(stmt, 2);
331 const int op_version = sqlite3_column_int(stmt, 3);
332
333 // is it still the same (name, operation, op_version) triple?
334 if(IS_NULL_PTR(last_name) || strcmp(last_name, name) || IS_NULL_PTR(last_operation) || strcmp(last_operation, operation)
335 || last_op_version != op_version)
336 {
337 dt_free(last_name);
338 dt_free(last_operation);
339 last_name = g_strdup(name);
340 last_operation = g_strdup(operation);
341 last_op_version = op_version;
342 i = 0;
343 }
344
345 // find the next free amended version of name
346 // clang-format off
347 sqlite3_prepare_v2(db->handle, "SELECT name FROM main.presets WHERE name = ?1 || ' (' || ?2 || ')' AND "
348 "operation = ?3 AND op_version = ?4",
349 -1, &innerstmt, NULL);
350 // clang-format on
351 while(1)
352 {
353 sqlite3_bind_text(innerstmt, 1, name, -1, SQLITE_TRANSIENT);
354 sqlite3_bind_int(innerstmt, 2, i);
355 sqlite3_bind_text(innerstmt, 3, operation, -1, SQLITE_TRANSIENT);
356 sqlite3_bind_int(innerstmt, 4, op_version);
357 if(sqlite3_step(innerstmt) != SQLITE_ROW) break;
358 sqlite3_reset(innerstmt);
359 sqlite3_clear_bindings(innerstmt);
360 i++;
361 }
362 sqlite3_finalize(innerstmt);
363
364 // rename preset
365 // clang-format off
366 const char *query = "UPDATE main.presets SET name = name || ' (' || ?1 || ')' WHERE rowid = ?2";
367 // clang-format on
368 sqlite3_prepare_v2(db->handle, query, -1, &innerstmt, NULL);
369 sqlite3_bind_int(innerstmt, 1, i);
370 sqlite3_bind_int(innerstmt, 2, rowid);
371 if(sqlite3_step(innerstmt) != SQLITE_DONE)
372 {
373 all_ok = FALSE;
374 failing_query = query;
375 goto end;
376 }
377 sqlite3_finalize(innerstmt);
378 }
379 sqlite3_finalize(stmt);
380 dt_free(last_name);
381 dt_free(last_operation);
382 // now we should be able to create the index
383 // clang-format off
384 _SQLITE3_EXEC(db->handle,
385 "CREATE UNIQUE INDEX IF NOT EXISTS main.presets_idx ON presets (name, operation, op_version)",
386 NULL, NULL, NULL);
387 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
388 NULL, NULL);
389 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET multi_priority = 0 WHERE multi_priority IS NULL", NULL, NULL,
390 NULL);
391 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL, NULL);
392 // clang-format on
393
394
395 // There are systems where absolute paths don't start with '/' (like Windows).
396 // Since the bug which introduced absolute paths to the db was fixed before a
397 // Windows build was available this shouldn't matter though.
398 // clang-format off
399 sqlite3_prepare_v2(db->handle, "SELECT id, filename FROM main.images WHERE filename LIKE '/%'", -1, &stmt, NULL);
400 sqlite3_prepare_v2(db->handle, "UPDATE main.images SET filename = ?1 WHERE id = ?2", -1, &innerstmt, NULL);
401 // clang-format on
402 while(sqlite3_step(stmt) == SQLITE_ROW)
403 {
404 const int id = sqlite3_column_int(stmt, 0);
405 const char *path = (const char *)sqlite3_column_text(stmt, 1);
406 gchar *filename = g_path_get_basename(path);
407 sqlite3_bind_text(innerstmt, 1, filename, -1, SQLITE_TRANSIENT);
408 sqlite3_bind_int(innerstmt, 2, id);
409 sqlite3_step(innerstmt);
410 sqlite3_reset(innerstmt);
411 sqlite3_clear_bindings(innerstmt);
412 dt_free(filename);
413 }
414 sqlite3_finalize(stmt);
415 sqlite3_finalize(innerstmt);
416
417 // We used to insert datetime_taken entries with '-' as date separators. Since that doesn't work well with
418 // the regular ':' when parsing
419 // or sorting we changed it to ':'. This takes care to change what we have as leftovers
420 // clang-format off
421 _SQLITE3_EXEC(
422 db->handle,
423 "UPDATE main.images SET datetime_taken = REPLACE(datetime_taken, '-', ':') WHERE datetime_taken LIKE '%-%'",
424 NULL, NULL, NULL);
425 // clang-format on
426
427end:
428 if(all_ok)
429 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
430 else
431 {
432 fprintf(stderr, "[init] failing query: `%s'\n", failing_query);
433 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle));
434 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL);
435 }
436
437 return all_ok;
438}
439
440#undef _SQLITE3_EXEC
441
442#define TRY_EXEC(_query, _message) \
443 do \
444 { \
445 if(sqlite3_exec(db->handle, _query, NULL, NULL, NULL) != SQLITE_OK) \
446 { \
447 fprintf(stderr, _message); \
448 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
449 FINALIZE; \
450 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
451 return version; \
452 } \
453 } while(0)
454
455#define TRY_STEP(_stmt, _expected, _message) \
456 do \
457 { \
458 if(sqlite3_step(_stmt) != _expected) \
459 { \
460 fprintf(stderr, _message); \
461 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
462 FINALIZE; \
463 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
464 return version; \
465 } \
466 } while(0)
467
468#define TRY_PREPARE(_stmt, _query, _message) \
469 do \
470 { \
471 if(sqlite3_prepare_v2(db->handle, _query, -1, &_stmt, NULL) != SQLITE_OK) \
472 { \
473 fprintf(stderr, _message); \
474 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
475 FINALIZE; \
476 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
477 return version; \
478 } \
479 } while(0)
480
481// redefine this where needed
482#define FINALIZE
483
484/* do the real migration steps, returns the version the db was converted to */
485static int _upgrade_library_schema_step(dt_database_t *db, int version)
486{
487 sqlite3_stmt *stmt;
488 int new_version = version;
489 if(version == CURRENT_DATABASE_VERSION_LIBRARY)
490 return version;
491 else if(version == 0)
492 {
493 // this can't happen, we started with 1, but it's a good example how this function works
494 // <do some magic to the db>
495 new_version = 1; // the version we transformed the db to. this way it might be possible to roll back or
496 // add fast paths
497 }
498 else if(version == 1)
499 {
500 // 1 -> 2 added write_timestamp
501 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
502 // clang-format off
503 TRY_EXEC("ALTER TABLE main.images ADD COLUMN write_timestamp INTEGER",
504 "[init] can't add `write_timestamp' column to database\n");
505 TRY_EXEC("UPDATE main.images SET write_timestamp = STRFTIME('%s', 'now') WHERE write_timestamp IS NULL",
506 "[init] can't initialize `write_timestamp' with current point in time\n");
507 // clang-format on
508 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
509 new_version = 2;
510 }
511 else if(version == 2)
512 {
513 // 2 -> 3 reset raw_black and raw_maximum. in theory we should change the columns from REAL to INTEGER,
514 // but sqlite doesn't care about types so whatever
515 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
516 // clang-format off
517 TRY_EXEC("UPDATE main.images SET raw_black = 0, raw_maximum = 16384",
518 "[init] can't reset raw_black and raw_maximum\n");
519 // clang-format on
520 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
521 new_version = 3;
522 }
523 else if(version == 3)
524 {
525 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
526 // clang-format off
527 TRY_EXEC("CREATE TRIGGER insert_tag AFTER INSERT ON main.tags"
528 " BEGIN"
529 " INSERT INTO tagxtag SELECT id, new.id, 0 FROM TAGS;"
530 " UPDATE tagxtag SET count = 1000000 WHERE id1=new.id AND id2=new.id;"
531 " END",
532 "[init] can't create insert_tag trigger\n");
533 TRY_EXEC("CREATE TRIGGER delete_tag BEFORE DELETE ON main.tags"
534 " BEGIN"
535 " DELETE FROM tagxtag WHERE id1=old.id OR id2=old.id;"
536 " DELETE FROM tagged_images WHERE tagid=old.id;"
537 " END",
538 "[init] can't create delete_tag trigger\n");
539 TRY_EXEC("CREATE TRIGGER attach_tag AFTER INSERT ON main.tagged_images"
540 " BEGIN"
541 " UPDATE tagxtag"
542 " SET count = count + 1"
543 " WHERE (id1=new.tagid AND id2 IN (SELECT tagid FROM tagged_images WHERE imgid=new.imgid))"
544 " OR (id2=new.tagid AND id1 IN (SELECT tagid FROM tagged_images WHERE imgid=new.imgid));"
545 " END",
546 "[init] can't create attach_tag trigger\n");
547 TRY_EXEC("CREATE TRIGGER detach_tag BEFORE DELETE ON main.tagged_images"
548 " BEGIN"
549 " UPDATE tagxtag"
550 " SET count = count - 1"
551 " WHERE (id1=old.tagid AND id2 IN (SELECT tagid FROM tagged_images WHERE imgid=old.imgid))"
552 " OR (id2=old.tagid AND id1 IN (SELECT tagid FROM tagged_images WHERE imgid=old.imgid));"
553 " END",
554 "[init] can't create detach_tag trigger\n");
555 // clang-format on
556 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
557 new_version = 4;
558 }
559 else if(version == 4)
560 {
561 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
562 // clang-format off
563 TRY_EXEC("ALTER TABLE main.presets RENAME TO tmp_presets", "[init] can't rename table presets\n");
564
565 TRY_EXEC("CREATE TABLE main.presets (name VARCHAR, description VARCHAR, operation VARCHAR, op_params BLOB,"
566 "enabled INTEGER, blendop_params BLOB, model VARCHAR, maker VARCHAR, lens VARCHAR,"
567 "iso_min REAL, iso_max REAL, exposure_min REAL, exposure_max REAL, aperture_min REAL,"
568 "aperture_max REAL, focal_length_min REAL, focal_length_max REAL, writeprotect INTEGER,"
569 "autoapply INTEGER, filter INTEGER, def INTEGER, format INTEGER, op_version INTEGER,"
570 "blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
571 "[init] can't create new presets table\n");
572
573 TRY_EXEC("INSERT INTO main.presets (name, description, operation, op_params, enabled, blendop_params, model, "
574 "maker, lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, aperture_max,"
575 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, def, format, op_version, "
576 "blendop_version, multi_priority, multi_name) SELECT name, description, operation, op_params, "
577 "enabled, blendop_params, model, maker, lens, iso_min, iso_max, exposure_min, exposure_max, "
578 "aperture_min, aperture_max, focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
579 "def, isldr, op_version, blendop_version, multi_priority, multi_name FROM tmp_presets",
580 "[init] can't populate presets table from tmp_presets\n");
581
582 TRY_EXEC("DROP TABLE tmp_presets", "[init] can't delete table tmp_presets\n");
583 // clang-format on
584 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
585 new_version = 5;
586 }
587 else if(version == 5)
588 {
589 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
590 // clang-format off
591 TRY_EXEC("CREATE INDEX main.images_filename_index ON images (filename)",
592 "[init] can't create index on image filename\n");
593 // clang-format on
594 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
595 new_version = 6;
596 }
597 else if(version == 6)
598 {
599 // some ancient tables can have the styleid column of style_items be called style_id. fix that.
600 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
601
602 if(sqlite3_exec(db->handle, "SELECT style_id FROM main.style_items", NULL, NULL, NULL) == SQLITE_OK)
603 {
604 // clang-format off
605 TRY_EXEC("ALTER TABLE main.style_items RENAME TO tmp_style_items",
606 "[init] can't rename table style_items\n");
607
608 TRY_EXEC("CREATE TABLE main.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
609 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
610 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
611 "[init] can't create new style_items table\n");
612
613 TRY_EXEC("INSERT INTO main.style_items (styleid, num, module, operation, op_params, enabled,"
614 " blendop_params, blendop_version, multi_priority, multi_name)"
615 " SELECT style_id, num, module, operation, op_params, enabled,"
616 " blendop_params, blendop_version, multi_priority, multi_name"
617 " FROM tmp_style_items",
618 "[init] can't populate style_items table from tmp_style_items\n");
619
620 TRY_EXEC("DROP TABLE tmp_style_items", "[init] can't delete table tmp_style_items\n");
621 // clang-format on
622 }
623
624 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
625 new_version = 7;
626 }
627 else if(version == 7)
628 {
629 // make sure that we have no film rolls with a NULL folder
630 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
631 // clang-format off
632 TRY_EXEC("ALTER TABLE main.film_rolls RENAME TO tmp_film_rolls", "[init] can't rename table film_rolls\n");
633
634 TRY_EXEC("CREATE TABLE main.film_rolls "
635 "(id INTEGER PRIMARY KEY, datetime_accessed CHAR(20), "
636 "folder VARCHAR(1024) NOT NULL)",
637 "[init] can't create new film_rolls table\n");
638
639 TRY_EXEC("INSERT INTO main.film_rolls (id, datetime_accessed, folder) "
640 "SELECT id, datetime_accessed, folder "
641 "FROM tmp_film_rolls "
642 "WHERE folder IS NOT NULL",
643 "[init] can't populate film_rolls table from tmp_film_rolls\n");
644
645 TRY_EXEC("DROP TABLE tmp_film_rolls", "[init] can't delete table tmp_film_rolls\n");
646 // clang-format on
647 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
648 new_version = 8;
649 }
650 else if(version == 8)
651 {
652 // 8 -> 9 added history_end column to images
653 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
654 // clang-format off
655 TRY_EXEC("ALTER TABLE main.images ADD COLUMN history_end INTEGER",
656 "[init] can't add `history_end' column to database\n");
657
658 TRY_EXEC("UPDATE main.images SET history_end = (SELECT IFNULL(MAX(num) + 1, 0) FROM main.history "
659 "WHERE imgid = id)", "[init] can't initialize `history_end' with last history entry\n");
660 // clang-format on
661 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
662 new_version = 9;
663 }
664 else if(version == 9)
665 {
666 // 9 -> 10 cleanup of last update :(
667 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
668 // clang-format off
669 TRY_EXEC("UPDATE main.images SET history_end = (SELECT IFNULL(MAX(num) + 1, 0) FROM main.history "
670 "WHERE imgid = id)", "[init] can't set `history_end' to 0 where it was NULL\n");
671 // clang-format on
672 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
673 new_version = 10;
674 }
675 else if(version == 10)
676 {
677 // 10 -> 11 added altitude column to images
678 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
679 // clang-format off
680 TRY_EXEC("ALTER TABLE main.images ADD COLUMN altitude REAL",
681 "[init] can't add `altitude' column to database\n");
682
683 TRY_EXEC("UPDATE main.images SET altitude = NULL", "[init] can't initialize `altitude' with NULL\n");
684 // clang-format on
685 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
686 new_version = 11;
687 }
688 else if(version == 11)
689 {
690 // 11 -> 12 tagxtag was removed in order to reduce database size
691 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
692 // clang-format off
693 TRY_EXEC("DROP TRIGGER main.detach_tag", "[init] can't drop trigger `detach_tag' from database\n");
694
695 TRY_EXEC("DROP TRIGGER main.attach_tag", "[init] can't drop trigger `attach_tag' from database\n");
696
697 TRY_EXEC("DROP TRIGGER main.delete_tag", "[init] can't drop trigger `delete_tag' from database\n");
698
699 TRY_EXEC("DROP TRIGGER main.insert_tag", "[init] can't drop trigger `insert_tag' from database\n");
700
701 TRY_EXEC("DROP TABLE main.tagxtag", "[init] can't drop table `tagxtag' from database\n");
702 // clang-format on
703 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
704 new_version = 12;
705 }
706 else if(version == 12)
707 {
708 // 11 -> 12 move presets, styles and tags over to the data database
709 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
710
712#undef FINALIZE
713#define FINALIZE \
714 do \
715 { \
716 sqlite3_finalize(stmt); \
717 sqlite3_finalize(select_stmt); \
718 sqlite3_finalize(count_clashes_stmt); \
719 sqlite3_finalize(update_name_stmt); \
720 sqlite3_finalize(insert_stmt); \
721 sqlite3_finalize(delete_stmt); \
722 } while(0)
723
724 stmt = NULL;
725 sqlite3_stmt *insert_stmt = NULL, *delete_stmt = NULL, *select_stmt = NULL, *count_clashes_stmt = NULL,
726 *update_name_stmt = NULL;
727 // clang-format off
728 // remove presets that are already in data.
729 // we can't use a NATURAL JOIN here as that fails when columns have NULL values. :-(
730 TRY_EXEC("DELETE FROM main.presets WHERE rowid IN (SELECT p1.rowid FROM main.presets p1 "
731 "JOIN data.presets p2 ON "
732 "p1.name IS p2.name AND "
733 "p1.description IS p2.description AND "
734 "p1.operation IS p2.operation AND "
735 "p1.op_version IS p2.op_version AND "
736 "p1.op_params IS p2.op_params AND "
737 "p1.enabled IS p2.enabled AND "
738 "p1.blendop_params IS p2.blendop_params AND "
739 "p1.blendop_version IS p2.blendop_version AND "
740 "p1.multi_priority IS p2.multi_priority AND "
741 "p1.multi_name IS p2.multi_name AND "
742 "p1.model IS p2.model AND "
743 "p1.maker IS p2.maker AND "
744 "p1.lens IS p2.lens AND "
745 "p1.iso_min IS p2.iso_min AND "
746 "p1.iso_max IS p2.iso_max AND "
747 "p1.exposure_min IS p2.exposure_min AND "
748 "p1.exposure_max IS p2.exposure_max AND "
749 "p1.aperture_min IS p2.aperture_min AND "
750 "p1.aperture_max IS p2.aperture_max AND "
751 "p1.focal_length_min IS p2.focal_length_min AND "
752 "p1.focal_length_max IS p2.focal_length_max AND "
753 "p1.writeprotect IS p2.writeprotect AND "
754 "p1.autoapply IS p2.autoapply AND "
755 "p1.filter IS p2.filter AND "
756 "p1.def IS p2.def AND "
757 "p1.format IS p2.format "
758 "WHERE p1.writeprotect = 0)",
759 "[init] can't delete already migrated presets from database\n");
760
761 // find all presets that are clashing with something else in presets. that can happen as we introduced an
762 // index on presets in data which wasn't in place in library.
763 TRY_PREPARE(select_stmt, "SELECT p.rowid, r FROM main.presets AS p, (SELECT rowid AS r, name, operation, "
764 "op_version FROM main.presets GROUP BY name, operation, op_version HAVING "
765 "COUNT(*) > 1) USING (name, operation, op_version) WHERE p.rowid != r",
766 "[init] can't prepare selecting presets with same name, operation, op_version from database\n");
767
768 // see if an updated preset name still causes problems
769 TRY_PREPARE(count_clashes_stmt, "SELECT COUNT(*) FROM main.presets AS p, (SELECT name, operation, op_version "
770 "FROM main.presets WHERE rowid = ?1) AS i ON p.name = i.name || \" #\" || ?2 "
771 "AND p.operation = i.operation AND p.op_version = i.op_version",
772 "[init] can't prepare selection of preset count by name from database\n");
773
774 // update the preset name for good
775 TRY_PREPARE(update_name_stmt, "UPDATE main.presets SET name = name || \" #\" || ?1 WHERE rowid = ?2",
776 "[init] can't prepare updating of preset name in database\n");
777
778 // find all presets that would be clashing with something in data
779 TRY_PREPARE(stmt, "SELECT p1.rowid FROM main.presets p1 INNER JOIN data.presets p2 "
780 "USING (name, operation, op_version) WHERE p1.writeprotect = 0",
781 "[init] can't access table `presets' in database\n");
782
783 // ... and move them over with a new name
784 TRY_PREPARE(insert_stmt, "INSERT OR FAIL INTO data.presets (name, description, operation, op_version, "
785 "op_params, enabled, blendop_params, blendop_version, multi_priority, multi_name, "
786 "model, maker, lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, "
787 "aperture_max, focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
788 "def, format) "
789 "SELECT name || \" #\" || ?1, description, operation, op_version, op_params, "
790 "enabled, blendop_params, blendop_version, multi_priority, multi_name, model, maker, "
791 "lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, aperture_max, "
792 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, def, format "
793 "FROM main.presets p1 WHERE p1.rowid = ?2",
794 "[init] can't prepare insertion statement\n");
795
796 TRY_PREPARE(delete_stmt, "DELETE FROM main.presets WHERE rowid = ?1", "[init] can't prepare deletion statement\n");
797 // clang-format on
798 // first rename presets with (name, operation, op_version) not being unique
799 while(sqlite3_step(select_stmt) == SQLITE_ROW)
800 {
801 const int own_rowid = sqlite3_column_int(select_stmt, 0);
802 const int other_rowid = sqlite3_column_int(select_stmt, 1);
803 int preset_version = 0;
804
805 do
806 {
807 preset_version++;
808 sqlite3_reset(count_clashes_stmt);
809 sqlite3_clear_bindings(count_clashes_stmt);
810 sqlite3_bind_int(count_clashes_stmt, 1, other_rowid);
811 sqlite3_bind_int(count_clashes_stmt, 2, preset_version);
812 }
813 while(sqlite3_step(count_clashes_stmt) == SQLITE_ROW && sqlite3_column_int(count_clashes_stmt, 0) > 0);
814
815 sqlite3_bind_int(update_name_stmt, 1, preset_version);
816 sqlite3_bind_int(update_name_stmt, 2, own_rowid);
817 TRY_STEP(update_name_stmt, SQLITE_DONE, "[init] can't rename preset in database\n");
818 sqlite3_reset(update_name_stmt);
819 sqlite3_reset(update_name_stmt);
820 }
821
822 // now rename to avoid clashes with data.presets
823 while(sqlite3_step(stmt) == SQLITE_ROW)
824 {
825 int preset_version = 0;
826 const int rowid = sqlite3_column_int(stmt, 0);
827
828 do
829 {
830 preset_version++;
831 sqlite3_reset(insert_stmt);
832 sqlite3_clear_bindings(insert_stmt);
833 sqlite3_bind_int(insert_stmt, 1, preset_version);
834 sqlite3_bind_int(insert_stmt, 2, rowid);
835 } while(sqlite3_step(insert_stmt) != SQLITE_DONE);
836
837 sqlite3_reset(delete_stmt);
838 sqlite3_clear_bindings(delete_stmt);
839 sqlite3_bind_int(delete_stmt, 1, rowid);
840 TRY_STEP(delete_stmt, SQLITE_DONE, "[init] can't delete preset from database\n");
841 }
842 // clang-format off
843 // all that is left in presets should be those that can be moved over without any further concerns
844 TRY_EXEC("INSERT OR FAIL INTO data.presets SELECT name, description, operation, "
845 "op_version, op_params, enabled, blendop_params, blendop_version, "
846 "multi_priority, multi_name, model, maker, lens, iso_min, iso_max, "
847 "exposure_min, exposure_max, aperture_min, aperture_max, "
848 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
849 "def, format FROM main.presets WHERE writeprotect = 0",
850 "[init] can't copy presets to the data database\n");
851 // ... delete them on the old side
852 TRY_EXEC("DELETE FROM main.presets WHERE writeprotect = 0",
853 "[init] can't copy presets to the data database\n");
854 // clang-format on
855
856 FINALIZE;
857#undef FINALIZE
858
860#define FINALIZE \
861 do \
862 { \
863 sqlite3_finalize(stmt); \
864 sqlite3_finalize(insert_stmt); \
865 sqlite3_finalize(select_stmt); \
866 sqlite3_finalize(delete_stmt); \
867 sqlite3_finalize(update_name_stmt); \
868 sqlite3_finalize(select_new_stmt); \
869 sqlite3_finalize(copy_style_items_stmt); \
870 sqlite3_finalize(delete_style_items_stmt); \
871 } while(0)
872
873 stmt = NULL;
874 select_stmt = NULL;
875 update_name_stmt = NULL;
876 insert_stmt = NULL;
877 delete_stmt = NULL;
878 sqlite3_stmt *select_new_stmt = NULL, *copy_style_items_stmt = NULL, *delete_style_items_stmt = NULL;
879 // clang-format off
880 TRY_PREPARE(stmt, "SELECT id, name FROM main.styles", "[init] can't prepare style selection from database\n");
881 TRY_PREPARE(select_stmt, "SELECT rowid FROM data.styles WHERE name = ?1 LIMIT 1",
882 "[init] can't prepare style item selection from database\n");
883 TRY_PREPARE(update_name_stmt, "UPDATE main.styles SET name = ?1 WHERE id = ?2",
884 "[init] can't prepare style name update\n");
885 TRY_PREPARE(insert_stmt, "INSERT INTO data.styles (id, name, description) "
886 "SELECT (SELECT COALESCE(MAX(id),0)+1 FROM data.styles), name, description "
887 "FROM main.styles where id = ?1",
888 "[init] can't prepare style insertion for database\n");
889 TRY_PREPARE(delete_stmt, "DELETE FROM main.styles WHERE id = ?1",
890 "[init] can't prepare style deletion for database\n");
891 TRY_PREPARE(select_new_stmt, "SELECT id FROM data.styles WHERE rowid = ?1",
892 "[init] can't prepare style selection from data database\n");
893 TRY_PREPARE(copy_style_items_stmt, "INSERT INTO data.style_items "
894 "(styleid, num, module, operation, op_params, enabled, blendop_params, "
895 "blendop_version, multi_priority, multi_name) "
896 "SELECT ?1, num, module, operation, op_params, enabled, blendop_params, "
897 "blendop_version, multi_priority, multi_name FROM main.style_items "
898 "WHERE styleid = ?2",
899 "[init] can't prepare style item copy into data database\n");
900 TRY_PREPARE(delete_style_items_stmt, "DELETE FROM main.style_items WHERE styleid = ?1",
901 "[init] can't prepare style item deletion for database\n");
902 // clang-format on
903 while(sqlite3_step(stmt) == SQLITE_ROW)
904 {
905 const int id = sqlite3_column_int(stmt, 0);
906 const char *name = (const char *)sqlite3_column_text(stmt, 1);
907
908 // find a unique name of the style for data.styles
909 sqlite3_bind_text(select_stmt, 1, name, -1, SQLITE_TRANSIENT);
910 if(sqlite3_step(select_stmt) == SQLITE_ROW)
911 {
912 // we need to append a version
913 int style_version = 0;
914 char *new_name = NULL;
915 do
916 {
917 style_version++;
918 dt_free(new_name);
919 new_name = g_strdup_printf("%s #%d", name, style_version);
920 sqlite3_reset(select_stmt);
921 sqlite3_clear_bindings(select_stmt);
922 sqlite3_bind_text(select_stmt, 1, new_name, -1, SQLITE_TRANSIENT);
923 } while(sqlite3_step(select_stmt) == SQLITE_ROW);
924
925 // update the name in the old place
926 sqlite3_bind_text(update_name_stmt, 1, new_name, -1, SQLITE_TRANSIENT);
927 sqlite3_bind_int(update_name_stmt, 2, id);
928 TRY_STEP(update_name_stmt, SQLITE_DONE, "[init] can't update name of style in database\n");
929 sqlite3_reset(update_name_stmt);
930 sqlite3_clear_bindings(update_name_stmt);
931 dt_free(new_name);
932 }
933
934 // move the style to data.styles and get the rowid
935 sqlite3_bind_int(insert_stmt, 1, id);
936 TRY_STEP(insert_stmt, SQLITE_DONE, "[init] can't insert style into data database\n");
937 sqlite3_int64 last_rowid = sqlite3_last_insert_rowid(db->handle);
938
939 // delete style from styles
940 sqlite3_bind_int(delete_stmt, 1, id);
941 TRY_STEP(delete_stmt, SQLITE_DONE, "[init] can't delete style from database\n");
942
943 sqlite3_bind_int(select_new_stmt, 1, last_rowid);
944 TRY_STEP(select_new_stmt, SQLITE_ROW, "[init] can't select new style from data database\n");
945 const int new_id = sqlite3_column_int(select_new_stmt, 0);
946
947 // now that we have the style over in data.styles and the new id we can just copy over all style items
948 sqlite3_bind_int(copy_style_items_stmt, 1, new_id);
949 sqlite3_bind_int(copy_style_items_stmt, 2, id);
950 TRY_STEP(copy_style_items_stmt, SQLITE_DONE, "[init] can't copy style items into data database\n");
951
952 // delete the style items from the old table
953 sqlite3_bind_int(delete_style_items_stmt, 1, id);
954 TRY_STEP(delete_style_items_stmt, SQLITE_DONE, "[init] can't delete style items from database\n");
955
956 // cleanup for the next round
957 sqlite3_reset(insert_stmt);
958 sqlite3_clear_bindings(insert_stmt);
959 sqlite3_reset(select_stmt);
960 sqlite3_clear_bindings(select_stmt);
961 sqlite3_reset(delete_stmt);
962 sqlite3_clear_bindings(delete_stmt);
963 sqlite3_reset(select_new_stmt);
964 sqlite3_clear_bindings(select_new_stmt);
965 sqlite3_reset(copy_style_items_stmt);
966 sqlite3_clear_bindings(copy_style_items_stmt);
967 sqlite3_reset(delete_style_items_stmt);
968 sqlite3_clear_bindings(delete_style_items_stmt);
969 }
970 FINALIZE;
971#undef FINALIZE
972
974#define FINALIZE
975 // clang-format off
976 // tags
977 TRY_EXEC("INSERT OR IGNORE INTO data.tags (name, icon, description, flags) "
978 "SELECT name, icon, description, flags FROM main.tags",
979 "[init] can't prepare insertion of used tags into data database\n");
980
981 // tagged images
982 // we need a temp table to update tagged_images due to its primary key
983 TRY_EXEC("CREATE TEMPORARY TABLE tagged_images_tmp (imgid INTEGER, tagid INTEGER)",
984 "[init] can't create temporary table for updating `tagged_images'\n");
985
986 TRY_EXEC("INSERT INTO tagged_images_tmp (imgid, tagid) "
987 "SELECT imgid, (SELECT t2.id FROM main.tags t1, data.tags t2 USING (name) WHERE t1.id = tagid) "
988 "FROM main.tagged_images", "[init] can't insert into `tagged_images_tmp'\n");
989
990 TRY_EXEC("DELETE FROM main.tagged_images", "[init] can't delete tagged images in database\n");
991
992 TRY_EXEC("INSERT OR IGNORE INTO main.tagged_images (imgid, tagid) SELECT imgid, tagid FROM tagged_images_tmp",
993 "[init] can't copy updated values back to `tagged_images'\n");
994
995 TRY_EXEC("DROP TABLE tagged_images_tmp", "[init] can't drop table `tagged_images_tmp' from database\n");
996
998 TRY_EXEC("DROP INDEX IF EXISTS main.presets_idx", "[init] can't drop index `presets_idx' from database\n");
999 TRY_EXEC("DROP TABLE main.presets", "[init] can't drop table `presets' from database\n");
1000 TRY_EXEC("DROP TABLE main.style_items", "[init] can't drop table `style_items' from database\n");
1001 TRY_EXEC("DROP TABLE main.styles", "[init] can't drop table `styles' from database\n");
1002 TRY_EXEC("DROP TABLE main.tags", "[init] can't drop table `tags' from database\n");
1003 // clang-format on
1004 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1005 new_version = 13;
1006 } else if(version == 13)
1007 {
1008 // 12 -> 13 bring back the used tag names to library.db so people can use it independently of data.db
1009 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1010 // clang-format off
1011 TRY_EXEC("CREATE TABLE main.used_tags (id INTEGER, name VARCHAR NOT NULL)",
1012 "[init] can't create `used_tags` table\n");
1013
1014 TRY_EXEC("CREATE INDEX main.used_tags_idx ON used_tags (id, name)",
1015 "[init] can't create index on table `used_tags' in database\n");
1016
1017 TRY_EXEC("INSERT INTO main.used_tags (id, name) SELECT t.id, t.name FROM data.tags AS t, main.tagged_images "
1018 "AS i ON t.id = i.tagid GROUP BY t.id",
1019 "[init] can't insert used tags into `used_tags` table in database\n");
1020 // clang-format on
1021 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1022 new_version = 14;
1023 }
1024 else if(version == 14)
1025 {
1026 // 13 -> fix the index on used_tags to be a UNIQUE index :-/
1027 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1028 // clang-format off
1029 TRY_EXEC("DELETE FROM main.used_tags WHERE rowid NOT IN (SELECT rowid FROM used_tags GROUP BY id)",
1030 "[init] can't delete duplicated entries from `used_tags' in database\n");
1031
1032 TRY_EXEC("DROP INDEX main.used_tags_idx", "[init] can't drop index `used_tags_idx' from database\n");
1033
1034 TRY_EXEC("CREATE UNIQUE INDEX main.used_tags_idx ON used_tags (id, name)",
1035 "[init] can't create index `used_tags_idx' in database\n");
1036
1037 TRY_EXEC("DELETE FROM main.tagged_images WHERE tagid IS NULL",
1038 "[init] can't delete NULL entries from `tagged_images' in database");
1039
1040 TRY_EXEC("DELETE FROM main.used_tags WHERE id NOT IN (SELECT DISTINCT tagid FROM main.tagged_images)",
1041 "[init] can't delete unused tags from `used_tags' in database\n");
1042 // clang-format on
1043 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1044 new_version = 15;
1045 }
1046 else if(version == 15)
1047 {
1048 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1050 // clang-format off
1051 TRY_EXEC("ALTER TABLE main.images ADD COLUMN position INTEGER",
1052 "[init] can't add `position' column to images table in database\n");
1053 TRY_EXEC("CREATE INDEX main.image_position_index ON images (position)",
1054 "[init] can't create index for custom image order table\n");
1055
1056 // Set the initial image sequence. The image id - the sequence images were imported -
1057 // defines the initial order of images.
1058 //
1059 // An int64 is used for the position index. The upper 31 bits define the initial order.
1060 // The lower 32bit provide space to reorder images.
1061 //
1062 // see: dt_collection_move_before()
1063 //
1064 TRY_EXEC("UPDATE main.images SET position = id << 32",
1065 "[init] can't update positions custom image order table\n");
1066 // clang-format on
1067 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1068 new_version = 16;
1069 }
1070 else if(version == 16)
1071 {
1072 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1074 // clang-format off
1075 TRY_EXEC("ALTER TABLE main.images ADD COLUMN aspect_ratio REAL",
1076 "[init] can't add `aspect_ratio' column to images table in database\n");
1077 TRY_EXEC("UPDATE main.images SET aspect_ratio = 0.0",
1078 "[init] can't update aspect_ratio in database\n");
1079 // clang-format on
1080 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1081 new_version = 17;
1082 }
1083 else if(version == 17)
1084 {
1085 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1086
1088 // clang-format off
1089 TRY_EXEC("CREATE TABLE main.masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, name VARCHAR(256), "
1090 "version INTEGER, points BLOB, points_count INTEGER, source BLOB)",
1091 "[init] can't create `masks_history` table\n");
1092
1093 TRY_EXEC("CREATE INDEX main.masks_history_imgid_index ON masks_history (imgid)",
1094 "[init] can't create index `masks_history_imgid_index' in database\n");
1095
1096 // to speed up the mask look-up, and makes the following UPDATE instantaneous whereas it could takes hours
1097 TRY_EXEC("CREATE INDEX main.mask_imgid_index ON mask (imgid);",
1098 "[init] can't create index `mask_imgid_index' in database\n");
1099
1100 // create a mask manager entry on history for all images containing all forms
1101 // make room for mask manager history entry
1102 TRY_EXEC("UPDATE main.history SET num=num+1 WHERE imgid IN (SELECT imgid FROM main.mask WHERE main.mask.imgid=main.history.imgid)",
1103 "[init] can't update `num' with num+1\n");
1104
1105 // update history end
1106 TRY_EXEC("UPDATE main.images SET history_end = history_end+1 WHERE id IN (SELECT imgid FROM main.mask WHERE main.mask.imgid=main.images.id)",
1107 "[init] can't update `history_end' with history_end+1\n");
1108
1109 // copy all masks into history
1110 TRY_EXEC("INSERT INTO main.masks_history (imgid, num, formid, form, name, version, points, points_count, source) SELECT "
1111 "imgid, 0, formid, form, name, version, points, points_count, source FROM main.mask",
1112 "[init] can't insert into masks_history\n");
1113
1114 // create a mask manager entry for each image that has masks
1115 TRY_EXEC("INSERT INTO main.history (imgid, num, operation, op_params, module, enabled, "
1116 "blendop_params, blendop_version, multi_priority, multi_name) "
1117 "SELECT DISTINCT imgid, 0, 'mask_manager', NULL, 1, 0, NULL, 0, 0, '' FROM main.mask "
1118 "GROUP BY imgid",
1119 "[init] can't insert mask manager into history\n");
1120
1121 TRY_EXEC("DROP TABLE main.mask", "[init] can't drop table `mask' from database\n");
1122
1124 GList *prior_v1 = dt_ioppr_get_iop_order_list_version(DT_IOP_ORDER_LEGACY);
1125
1126 TRY_EXEC("ALTER TABLE main.images ADD COLUMN iop_order_version INTEGER",
1127 "[init] can't add `iop_order_version' column to images table in database\n");
1128
1129 TRY_EXEC("UPDATE main.images SET iop_order_version = 0",
1130 "[init] can't update iop_order_version in database\n");
1131
1132 TRY_EXEC("UPDATE main.images SET iop_order_version = 1 WHERE "
1133 "EXISTS(SELECT * FROM main.history WHERE main.history.imgid = main.images.id)",
1134 "[init] can't update iop_order_version in database\n");
1135
1136 TRY_EXEC("ALTER TABLE main.history ADD COLUMN iop_order REAL",
1137 "[init] can't add `iop_order' column to history table in database\n");
1138
1139 // create a temp table with the previous priorities
1140 TRY_EXEC("CREATE TEMPORARY TABLE iop_order_tmp (iop_order REAL, operation VARCHAR(256))",
1141 "[init] can't create temporary table for updating `main.history'\n");
1142 // clang-format on
1143 // fill temp table with all operations up to this release
1144 // it will be used to create the pipe and update the iop_order on history
1145 for(GList *priorities = prior_v1; priorities; priorities = g_list_next(priorities))
1146 {
1147 dt_iop_order_entry_t *prior = (dt_iop_order_entry_t *)priorities->data;
1148
1149 sqlite3_prepare_v2(
1150 db->handle,
1151 "INSERT INTO iop_order_tmp (iop_order, operation) VALUES (?1, ?2)",
1152 -1, &stmt, NULL);
1153 sqlite3_bind_double(stmt, 1, prior->o.iop_order_f);
1154 sqlite3_bind_text(stmt, 2, prior->operation, -1, SQLITE_TRANSIENT);
1155 TRY_STEP(stmt, SQLITE_DONE, "[init] can't insert default value in iop_order_tmp\n");
1156 sqlite3_finalize(stmt);
1157 }
1158 g_list_free_full(prior_v1, dt_free_gpointer);
1159 prior_v1 = NULL;
1160
1161 // create the order of the pipe
1162 // iop_order is by default the module priority
1163 // if there's multi-instances we add the multi_priority
1164 // multi_priority is in reverse order in this version,
1165 // so we assume that is always less than 1000 and reverse it
1166 // it is possible that multi_priority = 0 don't appear in history
1167 // so just in case 1 / 1000 to every instance
1168 // clang-format off
1169 TRY_EXEC("UPDATE main.history SET iop_order = ((("
1170 "SELECT MAX(multi_priority) FROM main.history hist1 WHERE hist1.imgid = main.history.imgid AND hist1.operation = main.history.operation "
1171 ") + 1. - multi_priority) / 1000.) + "
1172 "IFNULL((SELECT iop_order FROM iop_order_tmp WHERE iop_order_tmp.operation = "
1173 "main.history.operation), -999999.) ",
1174 "[init] can't update iop_order in history table\n");
1175
1176 // check if there's any entry in history that was not updated
1177 sqlite3_stmt *sel_stmt;
1178 TRY_PREPARE(sel_stmt, "SELECT DISTINCT operation FROM main.history WHERE iop_order <= 0 OR iop_order IS NULL",
1179 "[init] can't prepare selecting history iop_order\n");
1180 // clang-format on
1181 while(sqlite3_step(sel_stmt) == SQLITE_ROW)
1182 {
1183 const char *op_name = (const char *)sqlite3_column_text(sel_stmt, 0);
1184 printf("operation %s with no iop_order while upgrading database\n", op_name);
1185 }
1186 sqlite3_finalize(sel_stmt);
1187 // clang-format off
1188 TRY_EXEC("DROP TABLE iop_order_tmp", "[init] can't drop table `iop_order_tmp' from database\n");
1189 // clang-format on
1190 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1191 new_version = 18;
1192 }
1193 // maybe in the future, see commented out code elsewhere
1194 // else if(version == XXX)
1195 // {
1196 // sqlite3_exec(db->handle, "ALTER TABLE film_rolls ADD COLUMN external_drive VARCHAR(1024)", NULL,
1197 // NULL, NULL);
1198 // }
1199 else if(version == 18)
1200 {
1201 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1202 // clang-format off
1203 TRY_EXEC("UPDATE images SET orientation=-2 WHERE orientation=1;",
1204 "[init] can't update images orientation 1 from database\n");
1205
1206 TRY_EXEC("UPDATE images SET orientation=1 WHERE orientation=2;",
1207 "[init] can't update images orientation 2 from database\n");
1208
1209 TRY_EXEC("UPDATE images SET orientation=-6 WHERE orientation=5;",
1210 "[init] can't update images orientation 5 from database\n");
1211
1212 TRY_EXEC("UPDATE images SET orientation=5 WHERE orientation=6;",
1213 "[init] can't update images orientation 6 from database\n");
1214
1215 TRY_EXEC("UPDATE images SET orientation=2 WHERE orientation=-2;",
1216 "[init] can't update images orientation -1 from database\n");
1217
1218 TRY_EXEC("UPDATE images SET orientation=6 WHERE orientation=-6;",
1219 "[init] can't update images orientation -6 from database\n");
1220 // clang-format on
1221 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1222 new_version = 19;
1223 }
1224 else if(version == 19)
1225 {
1226 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1227 // clang-format off
1228 // create a temp table to invert all multi_priority
1229 TRY_EXEC("CREATE TEMPORARY TABLE m_prio (id INTEGER, operation VARCHAR(256), prio INTEGER)",
1230 "[init] can't create temporary table for updating `history and style_items'\n");
1231
1232 TRY_EXEC("CREATE INDEX m_prio_id_index ON m_prio (id)",
1233 "[init] can't create temporary index for updating `history and style_items'\n");
1234 TRY_EXEC("CREATE INDEX m_prio_op_index ON m_prio (operation)",
1235 "[init] can't create temporary index for updating `history and style_items'\n");
1236
1237 TRY_EXEC("INSERT INTO m_prio SELECT imgid, operation, MAX(multi_priority)"
1238 " FROM main.history GROUP BY imgid, operation",
1239 "[init] can't populate m_prio\n");
1240
1241 TRY_EXEC("UPDATE main.history SET multi_priority = "
1242 "(SELECT prio FROM m_prio "
1243 " WHERE main.history.operation = operation AND main.history.imgid = id) - main.history.multi_priority",
1244 "[init] can't update multi_priority for history\n");
1245
1246 TRY_EXEC("DROP TABLE m_prio", "[init] can't drop table `m_prio' from database\n");
1247 // clang-format on
1248 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1249 new_version = 20;
1250 }
1251 else if(version == 20)
1252 {
1253 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1254 // clang-format off
1255 TRY_EXEC("DROP INDEX IF EXISTS main.used_tags_idx", "[init] can't drop index `used_tags_idx' from database\n");
1256 TRY_EXEC("DROP TABLE used_tags", "[init] can't delete table used_tags\n");
1257 // clang-format on
1258 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1259
1260 new_version = 21;
1261 }
1262 else if(version == 21)
1263 {
1264 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1265 // create a temp table to invert all multi_priority
1266 // clang-format off
1267 TRY_EXEC("CREATE TABLE module_order (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR)",
1268 "[init] can't create module_order table'\n");
1269
1270 // for all images:
1271 sqlite3_stmt *mig_stmt;
1272 TRY_PREPARE(mig_stmt, "SELECT imgid, operation, multi_priority, iop_order, mi.iop_order_version"
1273 " FROM main.history AS hi, main.images AS mi"
1274 " WHERE hi.imgid = mi.id"
1275 " GROUP BY imgid, operation, multi_priority"
1276 " ORDER BY imgid, iop_order",
1277 "[init] can't prepare selecting history for iop_order migration (v21)\n");
1278 // clang-format on
1279 GList *item_list = NULL;
1280 int current_imgid = UNKNOWN_IMAGE;
1281 int current_order_version = -1;
1282
1283 gboolean has_row = (sqlite3_step(mig_stmt) == SQLITE_ROW);
1284
1285 while(has_row)
1286 {
1287 const int32_t imgid = sqlite3_column_int(mig_stmt, 0);
1288 char operation[20] = { 0 };
1289 g_strlcpy(operation, (const char *)sqlite3_column_text(mig_stmt, 1), sizeof(operation));
1290 const int multi_priority = sqlite3_column_int(mig_stmt, 2);
1291 const double iop_order = sqlite3_column_double(mig_stmt, 3);
1292 const int iop_order_version = sqlite3_column_int(mig_stmt, 4);
1293
1294 has_row = (sqlite3_step(mig_stmt) == SQLITE_ROW);
1295
1296 // a new image, let's initialize the iop_order_version
1297 if(imgid != current_imgid || !has_row)
1298 {
1299 // new image, let's handle it
1300 if(!IS_NULL_PTR(item_list))
1301 {
1302 // we keep legacy, everything else is migrated to v3.0
1303 const dt_iop_order_t new_order_version = current_order_version == 2 ? DT_IOP_ORDER_LEGACY : DT_IOP_ORDER_ANSEL_RAW;
1304
1305 GList *iop_order_list = dt_ioppr_get_iop_order_list_version(new_order_version);
1306
1307 // merge entries into iop_order_list
1308
1309 // first remove all item_list iop from the iop_order_list
1310
1311 GList *e = item_list;
1312 GList *n = NULL;
1313 dt_iop_order_entry_t *n_entry = NULL;
1314
1315 while(e)
1316 {
1317 dt_iop_order_entry_t *e_entry = (dt_iop_order_entry_t *)e->data;
1318
1319 GList *s = iop_order_list;
1320 while(s && strcmp(((dt_iop_order_entry_t *)s->data)->operation, e_entry->operation))
1321 {
1322 s = g_list_next(s);
1323 }
1324 if(s)
1325 {
1326 iop_order_list = g_list_delete_link(iop_order_list, s);
1327 }
1328
1329 // skip all multipe instances
1330 n = e;
1331 do
1332 {
1333 n = g_list_next(n);
1334 if(IS_NULL_PTR(n)) break;
1335 n_entry = (dt_iop_order_entry_t *)n->data;
1336 } while(!strcmp(n_entry->operation, e_entry->operation));
1337 e = n;
1338 }
1339
1340 // then add all item_list into iop_order_list
1341
1342 for(e = item_list; e; e = g_list_next(e))
1343 {
1344 dt_iop_order_entry_t *e_entry = (dt_iop_order_entry_t *)e->data;
1345 iop_order_list = g_list_prepend(iop_order_list, e_entry);
1346 }
1347
1348 // and finally reorder the full list based on the iop-order
1349
1350 iop_order_list = g_list_sort(iop_order_list, dt_sort_iop_list_by_order_f);
1351
1352 const dt_iop_order_t kind = dt_ioppr_get_iop_order_list_kind(iop_order_list);
1353
1354 // check if we have some multi-instances
1355
1356 gboolean has_multiple_instances = FALSE;
1357 GList *l = iop_order_list;
1358
1359 while(l)
1360 {
1361 GList *next = g_list_next(l);
1362 if(next
1363 && (strcmp(((dt_iop_order_entry_t *)(l->data))->operation,
1364 ((dt_iop_order_entry_t *)(next->data))->operation) == 0))
1365 {
1366 has_multiple_instances = TRUE;
1367 break;
1368 }
1369 l = next;
1370 }
1371
1372 // write iop_order_list and/or version into module_order
1373
1374 sqlite3_stmt *ins_stmt = NULL;
1375 if(kind == DT_IOP_ORDER_CUSTOM || has_multiple_instances)
1376 {
1377 char *iop_list_txt = dt_ioppr_serialize_text_iop_order_list(iop_order_list);
1378
1379 sqlite3_prepare_v2(db->handle,
1380 "INSERT INTO module_order VALUES (?1, ?2, ?3)", -1,
1381 &ins_stmt, NULL);
1382 sqlite3_bind_int(ins_stmt, 1, current_imgid);
1383 sqlite3_bind_int(ins_stmt, 2, kind);
1384 sqlite3_bind_text(ins_stmt, 3, iop_list_txt, -1, SQLITE_TRANSIENT);
1385 TRY_STEP(ins_stmt, SQLITE_DONE, "[init] can't insert into module_order (custom order)\n");
1386 sqlite3_finalize(ins_stmt);
1387
1388 dt_free(iop_list_txt);
1389 }
1390 else
1391 {
1392 sqlite3_prepare_v2(db->handle,
1393 "INSERT INTO module_order VALUES (?1, ?2, NULL)", -1,
1394 &ins_stmt, NULL);
1395 sqlite3_bind_int(ins_stmt, 1, current_imgid);
1396 sqlite3_bind_int(ins_stmt, 2, kind);
1397 TRY_STEP(ins_stmt, SQLITE_DONE, "[init] can't insert into module_order (standard order)\n");
1398 sqlite3_finalize(ins_stmt);
1399 }
1400
1401 g_list_free(item_list);
1402 item_list = NULL;
1403 g_list_free_full(iop_order_list, dt_free_gpointer);
1404 iop_order_list = NULL;
1405
1406 item_list = NULL;
1407 }
1408
1409 current_imgid = imgid;
1410 current_order_version = iop_order_version;
1411 }
1412
1413 dt_iop_order_entry_t *item = (dt_iop_order_entry_t *)malloc(sizeof(dt_iop_order_entry_t));
1414 memcpy(item->operation, operation, sizeof(item->operation));
1415 item->instance = multi_priority;
1416 item->o.iop_order_f = iop_order; // used to order the enties only
1417 item_list = g_list_append(item_list, item);
1418 }
1419 sqlite3_finalize(mig_stmt);
1420
1421 // remove iop_order from history table
1422 // clang-format off
1423 TRY_EXEC("CREATE TABLE h (imgid INTEGER, num INTEGER, module INTEGER, "
1424 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
1425 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
1426 "[init] can't create module_order table\n");
1427 TRY_EXEC("CREATE INDEX h_imgid_index ON h (imgid)",
1428 "[init] can't create index h_imgid_index\n");
1429 TRY_EXEC("INSERT INTO h SELECT imgid, num, module, operation, op_params, enabled, "
1430 "blendop_params, blendop_version, multi_priority, multi_name FROM main.history",
1431 "[init] can't create module_order table\n");
1432 TRY_EXEC("DROP TABLE history",
1433 "[init] can't drop table history\n");
1434 TRY_EXEC("ALTER TABLE h RENAME TO history",
1435 "[init] can't rename h to history\n");
1436 TRY_EXEC("DROP INDEX h_imgid_index",
1437 "[init] can't drop index h_imgid_index\n");
1438 TRY_EXEC("CREATE INDEX main.history_imgid_index ON history (imgid)",
1439 "[init] can't create index images_imgid_index\n");
1440
1441 // remove iop_order_version from images
1442
1443 TRY_EXEC("CREATE TABLE i (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1444 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
1445 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
1446 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
1447 "output_width INTEGER, output_height INTEGER, crop REAL, "
1448 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
1449 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
1450 "caption VARCHAR, description VARCHAR, license VARCHAR, sha1sum CHAR(40), "
1451 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
1452 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
1453 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, aspect_ratio REAL)",
1454 "[init] can't create table i\n");
1455 TRY_EXEC("INSERT INTO i SELECT id, group_id, film_id, width, height, filename, maker, model,"
1456 " lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags,"
1457 " output_width, output_height, crop, raw_parameters, raw_denoise_threshold,"
1458 " raw_auto_bright_threshold, raw_black, raw_maximum, caption, description, license, sha1sum,"
1459 " orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version,"
1460 " max_version, write_timestamp, history_end, position, aspect_ratio "
1461 "FROM images",
1462 "[init] can't populate table h\n");
1463 TRY_EXEC("DROP TABLE images",
1464 "[init] can't drop table images\n");
1465 TRY_EXEC("ALTER TABLE i RENAME TO images",
1466 "[init] can't rename i to images\n");
1467 // clang-format on
1468 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1469
1470 new_version = 22;
1471 }
1472 else if(version == 22)
1473 {
1474 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1475 // clang-format off
1476 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_group_id_index ON images (group_id)",
1477 "[init] can't create group_id index on image\n");
1478 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_film_id_index ON images (film_id)",
1479 "[init] can't create film_id index on image\n");
1480 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_filename_index ON images (filename)",
1481 "[init] can't create filename index on image\n");
1482 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.image_position_index ON images (position)",
1483 "[init] can't create position index on image\n");
1484
1485 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.film_rolls_folder_index ON film_rolls (folder)",
1486 "[init] can't create folder index on film_rolls\n");
1487 // clang-format on
1488 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1489
1490 new_version = 23;
1491 }
1492 else if(version == 23)
1493 {
1494 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1495 // clang-format off
1496 TRY_EXEC("CREATE TABLE main.history_hash (imgid INTEGER PRIMARY KEY, "
1497 "basic_hash BLOB, auto_hash BLOB, current_hash BLOB)",
1498 "[init] can't create table history_hash\n");
1499 // clang-format on
1500
1501 // history_hash backfill is deferred: it will be filled through the image cache.
1502
1503 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1504
1505 new_version = 24;
1506 }
1507 else if(version == 24)
1508 {
1509 // clang-format off
1510 TRY_EXEC("ALTER TABLE main.history_hash ADD COLUMN mipmap_hash BLOB",
1511 "[init] can't add `mipmap_hash' column to history_hash table in database\n");
1512 // clang-format on
1513
1514 new_version = 25;
1515 }
1516 else if(version == 25)
1517 {
1518 // clang-format of
1519 TRY_EXEC("ALTER TABLE main.images ADD COLUMN exposure_bias REAL",
1520 "[init] can't add `exposure_bias' column to images table in database\n");
1521 // clang-format on
1522
1523 new_version = 26;
1524 }
1525 else if(version == 26)
1526 {
1527 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1528 // clang-format off
1529 TRY_EXEC("CREATE TABLE main.new_film_rolls "
1530 "(id INTEGER PRIMARY KEY, "
1531 "access_timestamp INTEGER, "
1532 "folder VARCHAR(1024) NOT NULL)",
1533 "[init] can't create new_film_rolls table\n");
1534
1535 TRY_EXEC("INSERT INTO main.new_film_rolls"
1536 "(id, access_timestamp, folder) "
1537 "SELECT id, "
1538 "strftime('%s', replace(substr(datetime_accessed, 1, 10), ':', '-') || substr(datetime_accessed, 11), 'utc'), "
1539 "folder "
1540 "FROM film_rolls "
1541 "WHERE folder IS NOT NULL",
1542 "[init] can't populate new_film_rolls table from film_rolls\n");
1543
1544 TRY_EXEC("DROP TABLE film_rolls",
1545 "[init] can't delete table film_rolls\n");
1546
1547 TRY_EXEC("ALTER TABLE main.new_film_rolls RENAME TO film_rolls",
1548 "[init] can't rename table new_film_rolls to film_rolls\n");
1549
1550 TRY_EXEC("CREATE INDEX main.film_rolls_folder_index ON film_rolls (folder)",
1551 "[init] can't create index `film_rolls_folder_index' on table `film_rolls'\n");
1552 // clang-format on
1553 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1554 new_version = 27;
1555 }
1556 else if(version == 27)
1557 {
1558 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1559 // clang-format off
1560 TRY_EXEC("ALTER TABLE main.images ADD COLUMN import_timestamp INTEGER DEFAULT -1",
1561 "[init] can't add `import_timestamp' column to images table in database\n");
1562 TRY_EXEC("ALTER TABLE main.images ADD COLUMN change_timestamp INTEGER DEFAULT -1",
1563 "[init] can't add `change_timestamp' column to images table in database\n");
1564 TRY_EXEC("ALTER TABLE main.images ADD COLUMN export_timestamp INTEGER DEFAULT -1",
1565 "[init] can't add `export_timestamp' column to images table in database\n");
1566 TRY_EXEC("ALTER TABLE main.images ADD COLUMN print_timestamp INTEGER DEFAULT -1",
1567 "[init] can't add `print_timestamp' column to images table in database\n");
1568
1569 TRY_EXEC("UPDATE main.images SET import_timestamp = (SELECT access_timestamp "
1570 "FROM main.film_rolls WHERE film_rolls.id = images.film_id)",
1571 "[init] can't populate import_timestamp column from film_rolls.access_timestamp.\n");
1572
1573 TRY_EXEC("UPDATE main.images SET change_timestamp = images.write_timestamp "
1574 "WHERE images.write_timestamp IS NOT NULL "
1575 "AND images.id = (SELECT imgid FROM tagged_images "
1576 "JOIN data.tags ON tags.id = tagged_images.tagid "
1577 "WHERE data.tags.name = 'darktable|changed')",
1578 "[init] can't populate change_timestamp column from images.write_timestamp.\n");
1579 // clang-format on
1580 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1581 new_version = 28;
1582 }
1583 else if(version == 28)
1584 {
1585 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1586 // clang-format off
1587 // clear flag DT_IMAGE_REJECTED (was not used)
1588 TRY_EXEC("UPDATE main.images SET flags = (flags & ~8)",
1589 "[init] can't clear rejected flags");
1590
1591 // add DT_IMAGE_REJECTED and clear rating for all images being rejected
1592 TRY_EXEC("UPDATE main.images SET flags = (flags | 8) & ~7 WHERE (flags & 7) = 6",
1593 "[init] can't set rejected flags");
1594 // clang-format on
1595 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1596 new_version = 29;
1597 }
1598 else if(version == 29)
1599 {
1600 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1601 // clang-format off
1602 // add position in tagged_images table
1603 TRY_EXEC("ALTER TABLE main.tagged_images ADD COLUMN position INTEGER",
1604 "[init] can't add `position' column to tagged_images table in database\n");
1605
1606 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.tagged_images_imgid_index ON tagged_images (imgid)",
1607 "[init] can't create image index on tagged_images\n");
1608 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.tagged_images_position_index ON tagged_images (position)",
1609 "[init] can't create position index on tagged_images\n");
1610 TRY_EXEC("UPDATE main.tagged_images SET position = (tagid + imgid) << 32",
1611 "[init] can't populate position on tagged_images\n");
1612
1613 // remove caption and description fields from images table
1614
1615 TRY_EXEC("CREATE TABLE main.i (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1616 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
1617 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
1618 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
1619 "output_width INTEGER, output_height INTEGER, crop REAL, "
1620 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
1621 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
1622 "license VARCHAR, sha1sum CHAR(40), "
1623 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
1624 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
1625 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
1626 "aspect_ratio REAL, exposure_bias REAL, "
1627 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, "
1628 "export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1)",
1629 "[init] can't create table i\n");
1630
1631 TRY_EXEC("INSERT INTO main.i SELECT id, group_id, film_id, width, height, filename, maker, model,"
1632 " lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags,"
1633 " output_width, output_height, crop, raw_parameters, raw_denoise_threshold,"
1634 " raw_auto_bright_threshold, raw_black, raw_maximum, license, sha1sum,"
1635 " orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version,"
1636 " max_version, write_timestamp, history_end, position, aspect_ratio, exposure_bias,"
1637 " import_timestamp, change_timestamp, export_timestamp, print_timestamp "
1638 "FROM main.images",
1639 "[init] can't populate table i\n");
1640 TRY_EXEC("DROP TABLE main.images",
1641 "[init] can't drop table images\n");
1642 TRY_EXEC("ALTER TABLE main.i RENAME TO images",
1643 "[init] can't rename i to images\n");
1644
1645 TRY_EXEC("CREATE INDEX main.images_group_id_index ON images (group_id)",
1646 "[init] can't create group_id index on images table\n");
1647 TRY_EXEC("CREATE INDEX main.images_film_id_index ON images (film_id)",
1648 "[init] can't create film_id index on images table\n");
1649 TRY_EXEC("CREATE INDEX main.images_filename_index ON images (filename)",
1650 "[init] can't create filename index on images table\n");
1651 TRY_EXEC("CREATE INDEX main.image_position_index ON images (position)",
1652 "[init] can't create position index on images table\n");
1653 // clang-format on
1654 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1655 new_version = 30;
1656 }
1657 else if(version == 30)
1658 {
1659 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1660 // clang-format off
1661 // add second columns to speed up sorting
1662 TRY_EXEC("DROP INDEX IF EXISTS `history_imgid_index`",
1663 "[init] can't drop history_imgid_index\n");
1664 TRY_EXEC("CREATE INDEX `history_imgid_index` ON `history` ( `imgid`, `operation` )",
1665 "[init] can't recreate history_imgid_index\n");
1666
1667 TRY_EXEC("DROP INDEX IF EXISTS `images_filename_index`",
1668 "[init] can't drop images_filename_index\n");
1669 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
1670 "[init] can't recreate images_filename_index\n");
1671
1672 TRY_EXEC("DROP INDEX IF EXISTS `images_film_id_index`",
1673 "[init] can't drop images_film_id_index\n");
1674 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
1675 "[init] can't recreate images_film_id_index\n");
1676
1677 TRY_EXEC("DROP INDEX IF EXISTS `images_group_id_index`",
1678 "[init] can't drop images_group_id_index\n");
1679 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
1680 "[init] can't recreate images_group_id_index\n");
1681
1682 TRY_EXEC("DROP INDEX IF EXISTS `masks_history_imgid_index`",
1683 "[init] can't drop masks_history_imgid_index\n");
1684 TRY_EXEC("CREATE INDEX `masks_history_imgid_index` ON `masks_history` ( `imgid`, `num` )",
1685 "[init] can't recreate masks_history_imgid_index\n");
1686
1687 // map refinement: avoid full table scan
1688 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( `latitude` DESC, `longitude` DESC )",
1689 "[init] can't create images_latlong_index\n");
1690 // clang-format on
1691 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1692 new_version = 31;
1693 }
1694 else if(version == 31)
1695 {
1696 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1697 // clang-format off
1698 // remove duplicates
1699 TRY_EXEC("DELETE FROM main.meta_data WHERE rowid NOT IN (SELECT MIN(rowid) "
1700 "FROM main.meta_data GROUP BY id, key)",
1701 "[init] can't remove duplicates from meta_data\n");
1702
1703 // recreate the index with UNIQUE option
1704 TRY_EXEC("DROP INDEX IF EXISTS metadata_index",
1705 "[init] can't drop metadata_index\n");
1706 TRY_EXEC("CREATE UNIQUE INDEX main.metadata_index ON meta_data (id, key)",
1707 "[init] can't create metadata_index\n");
1708 // clang-format on
1709 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1710 new_version = 32;
1711 }
1712 else if(version == 32)
1713 {
1714 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1715 // clang-format off
1716 // add foreign keys for database consistency. ON UPDATE CASCADE since you never know
1717 // if a future version will change image_id
1718 // Unfortunately sqlite does not support adding foreign keys to existing tables
1719 // so we have to rename the existing tables, recreate them and copy back the old values
1720 // images first
1721 // needs to delete orphaned entries
1722 TRY_EXEC("ALTER TABLE `images` RENAME TO `images_old`",
1723 "[init] can't rename images\n");
1724
1725 TRY_EXEC("CREATE TABLE `images` (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1726 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, lens VARCHAR, "
1727 "exposure REAL, aperture REAL, iso REAL, focal_length REAL, focus_distance REAL, datetime_taken CHAR(20), "
1728 "flags INTEGER, output_width INTEGER, output_height INTEGER, crop REAL, "
1729 "raw_parameters INTEGER, raw_denoise_threshold REAL, raw_auto_bright_threshold REAL, "
1730 "raw_black INTEGER, raw_maximum INTEGER, license VARCHAR, sha1sum CHAR(40), "
1731 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, latitude REAL, altitude REAL, "
1732 "color_matrix BLOB, colorspace INTEGER, version INTEGER, max_version INTEGER, write_timestamp INTEGER, "
1733 "history_end INTEGER, position INTEGER, aspect_ratio REAL, exposure_bias REAL, "
1734 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1, "
1735 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
1736 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
1737 "[init] can't create new images table\n");
1738
1739 // corner case: database inconsistency with images having invalid film id
1740 TRY_EXEC("DELETE FROM `images_old` WHERE film_id NOT IN (SELECT id FROM `film_rolls`)",
1741 "[init] can't delete images with invalid film id\n");
1742
1743 TRY_EXEC("UPDATE `images_old` SET group_id=id WHERE group_id NOT IN (SELECT id from `images_old`)",
1744 "[init] can't fix invalid group ids\n");
1745
1746 TRY_EXEC("INSERT INTO `images` SELECT * FROM `images_old`",
1747 "[init] can't copy back from images_old\n");
1748
1749 // pita: need to recreate index
1750 TRY_EXEC("DROP INDEX IF EXISTS `image_position_index`",
1751 "[init] can't drop image_position_index\n");
1752 TRY_EXEC("CREATE INDEX `image_position_index` ON `images` (position)",
1753 "[init] can't add image_position_index\n");
1754
1755 // second columns
1756 TRY_EXEC("DROP INDEX IF EXISTS `images_filename_index`",
1757 "[init] can't drop images_filename_index\n");
1758 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
1759 "[init] can't recreate images_filename_index\n");
1760
1761 TRY_EXEC("DROP INDEX IF EXISTS `images_film_id_index`",
1762 "[init] can't drop images_film_id_index\n");
1763 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
1764 "[init] can't recreate images_film_id_index\n");
1765
1766 TRY_EXEC("DROP INDEX IF EXISTS `images_group_id_index`",
1767 "[init] can't drop images_group_id_index\n");
1768 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
1769 "[init] can't recreate images_group_id_index\n");
1770
1771 TRY_EXEC("DROP INDEX IF EXISTS `images_latlong_index`",
1772 "[init] can't drop images_latlong_index\n");
1773 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( latitude DESC, longitude DESC )",
1774 "[init] can't add images_latlong_index\n");
1775
1776 TRY_EXEC("DROP TABLE `images_old`",
1777 "[init] can't drop table images_old\n");
1778
1779 // history
1780 TRY_EXEC("ALTER TABLE `history` RENAME TO `history_old`",
1781 "[init] can't rename history\n");
1782
1783 TRY_EXEC("CREATE TABLE `history` (imgid INTEGER, num INTEGER, module INTEGER, "
1784 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, "
1785 "multi_priority INTEGER, multi_name VARCHAR(256), "
1786 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1787 "[init] can't create new history table\n");
1788
1789 TRY_EXEC("DELETE FROM `history_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1790 "[init] can't delete orphaned history elements\n");
1791
1792 TRY_EXEC("INSERT INTO history SELECT * FROM history_old",
1793 "[init] can't copy back from history_old\n");
1794
1795 TRY_EXEC("DROP INDEX IF EXISTS `history_imgid_index`",
1796 "[init] can't drop history_imgid_index\n");
1797 TRY_EXEC("CREATE INDEX `history_imgid_op_index` ON `history` ( `imgid`, `operation` )",
1798 "[init] can't recreate history_imgid_index\n");
1799 TRY_EXEC("CREATE INDEX `history_imgid_num_index` ON `history` ( `imgid`, `num` DESC )",
1800 "[init] can't recreate history_imgid_index\n");
1801
1802 TRY_EXEC("DROP TABLE `history_old`",
1803 "[init] can't drop table history_old\n");
1804
1805 // history hash
1806 TRY_EXEC("ALTER TABLE `history_hash` RENAME TO `history_hash_old`",
1807 "[init] can't rename history_hash\n");
1808
1809 TRY_EXEC("CREATE TABLE `history_hash` (imgid INTEGER PRIMARY KEY, basic_hash BLOB, auto_hash BLOB, current_hash BLOB, "
1810 "mipmap_hash BLOB, FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1811 "[init] can't create new history_hash table\n");
1812
1813 TRY_EXEC("DELETE FROM `history_hash_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1814 "[init] can't delete orphaned history_hash elements\n");
1815
1816 TRY_EXEC("INSERT INTO `history_hash` SELECT * FROM `history_hash_old`",
1817 "[init] can't copy back from history_hash_old\n");
1818
1819 TRY_EXEC("DROP TABLE `history_hash_old`",
1820 "[init] can't drop table history_hash_old\n");
1821
1822 // tagged images
1823 TRY_EXEC("ALTER TABLE `tagged_images` RENAME TO `tagged_images_old`",
1824 "[init] can't rename tagged_images\n");
1825
1826 TRY_EXEC("CREATE TABLE `tagged_images` (imgid integer, tagid integer, position INTEGER, "
1827 "primary key(imgid, tagid), FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1828 "[init] can't create new tagged_images table\n");
1829
1830 TRY_EXEC("DELETE FROM `tagged_images_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1831 "[init] can't delete orphaned tagged_images elements\n");
1832
1833 TRY_EXEC("INSERT INTO `tagged_images` SELECT * FROM `tagged_images_old`",
1834 "[init] can't copy back from tagged_images_old\n");
1835
1836 // old indices
1837 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_imgid_index",
1838 "[init] can't drop tagged_images_imgid_index\n");
1839 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_position_index",
1840 "[init] can't drop tagged_images_position_index\n");
1841 TRY_EXEC("CREATE INDEX tagged_images_position_index ON tagged_images (position)",
1842 "[init] can't add index tagged_images_position_index\n");
1843 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_tagid_index",
1844 "[init] can't drop tagged_images_tagid_index\n");
1845 TRY_EXEC("CREATE INDEX tagged_images_tagid_index ON tagged_images (tagid)",
1846 "[init] can't add index tagged_images_tagid_index\n");
1847
1848 TRY_EXEC("DROP TABLE `tagged_images_old`",
1849 "[init] can't drop table tagged_images_old\n");
1850
1851 // masks history
1852 TRY_EXEC("ALTER TABLE `masks_history` RENAME TO `masks_history_old`",
1853 "[init] can't rename masks_history\n");
1854
1855 TRY_EXEC("CREATE TABLE masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, "
1856 "name VARCHAR(256), version INTEGER, points BLOB, points_count INTEGER, source BLOB, "
1857 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1858 "[init] can't create new masks_history table\n");
1859
1860 TRY_EXEC("DELETE FROM `masks_history_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1861 "[init] can't delete orphaned masks_history elements\n");
1862
1863 TRY_EXEC("INSERT INTO `masks_history` SELECT * FROM `masks_history_old`",
1864 "[init] can't copy back from masks_history\n");
1865
1866 TRY_EXEC("DROP INDEX IF EXISTS `masks_history_imgid_index`",
1867 "[init] can't drop masks_history_imgid_index\n");
1868 TRY_EXEC("CREATE INDEX `masks_history_imgid_index` ON `masks_history` ( imgid, num )",
1869 "[init] can't recreate masks_history_imgid_index\n");
1870
1871 TRY_EXEC("DROP TABLE masks_history_old",
1872 "[init] can't drop table masks_history_old\n");
1873
1874 // color labels
1875 TRY_EXEC("ALTER TABLE `color_labels` RENAME TO `color_labels_old`",
1876 "[init] can't rename color_labels\n");
1877 TRY_EXEC("CREATE TABLE `color_labels` (imgid INTEGER, color INTEGER, "
1878 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1879 "[init] can't create new color_labels table\n");
1880
1881 TRY_EXEC("DELETE FROM `color_labels_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1882 "[init] can't delete orphaned color_labels elements\n");
1883
1884 TRY_EXEC("INSERT INTO `color_labels` SELECT * FROM `color_labels_old`",
1885 "[init] can't copy back from color_labels\n");
1886
1887 TRY_EXEC("DROP TABLE color_labels_old",
1888 "[init] can't drop table color_labels_old\n");
1889
1890 TRY_EXEC("CREATE UNIQUE INDEX `color_labels_idx` ON `color_labels` (imgid, color)",
1891 "[init] can't recreate color_labels_idx\n");
1892
1893 // meta data
1894 TRY_EXEC("ALTER TABLE `meta_data` RENAME TO `meta_data_old`",
1895 "[init] can't rename meta_data\n");
1896 TRY_EXEC("CREATE TABLE `meta_data` (id integer, key integer, value varchar, "
1897 "FOREIGN KEY(id) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1898 "[init] can't create new meta_data table\n");
1899
1900 TRY_EXEC("DELETE FROM `meta_data_old` WHERE id NOT IN (SELECT id FROM `images`)",
1901 "[init] can't delete orphaned meta_data elements\n");
1902
1903 TRY_EXEC("INSERT INTO `meta_data` SELECT * FROM `meta_data_old`",
1904 "[init] can't copy back from meta_data\n");
1905
1906 TRY_EXEC("DROP TABLE meta_data_old",
1907 "[init] can't drop table meta_data_old\n");
1908
1909 TRY_EXEC("CREATE UNIQUE INDEX `metadata_index` ON `meta_data` (id, key, value)",
1910 "[init] can't recreate metadata_index\n");
1911
1912 // selected images
1913 TRY_EXEC("ALTER TABLE `selected_images` RENAME TO `selected_images_old`",
1914 "[init] can't rename selected_images\n");
1915 TRY_EXEC("CREATE TABLE `selected_images` (imgid INTEGER PRIMARY KEY, "
1916 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1917 "[init] can't create new selected_images table\n");
1918
1919 TRY_EXEC("DELETE FROM `selected_images_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1920 "[init] can't delete orphaned selected_images elements\n");
1921
1922 TRY_EXEC("INSERT INTO `selected_images` SELECT * FROM `selected_images_old`",
1923 "[init] can't copy back selected_images meta_data\n");
1924
1925 TRY_EXEC("DROP TABLE selected_images_old",
1926 "[init] can't drop table selected_images_old\n");
1927
1928 // module order
1929 TRY_EXEC("ALTER TABLE `module_order` RENAME TO `module_order_old`",
1930 "[init] can't rename module_order\n");
1931 TRY_EXEC("CREATE TABLE `module_order` (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR, "
1932 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1933 "[init] can't create new module_order table\n");
1934
1935 TRY_EXEC("DELETE FROM `module_order_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1936 "[init] can't delete orphaned module_order elements\n");
1937
1938 TRY_EXEC("INSERT INTO `module_order` SELECT * FROM `module_order_old`",
1939 "[init] can't copy back module_order meta_data\n");
1940
1941 TRY_EXEC("DROP TABLE module_order_old",
1942 "[init] can't drop table module_order_old\n");
1943 // clang-format on
1944 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1945 new_version = 33;
1946 }
1947 else if(version == 33)
1948 {
1949 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1950
1951 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_datetime_taken_nc ON images (datetime_taken COLLATE NOCASE)",
1952 "[init] can't create images_datetime_taken\n");
1953 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.metadata_index_key ON meta_data (key)",
1954 "[init] can't create metadata_index_key\n");
1955
1956 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1957 new_version = 34;
1958 }
1959 else if(version == 34)
1960 {
1961 sqlite3_exec(db->handle, "PRAGMA foreign_keys = OFF", NULL, NULL, NULL);
1962 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1963
1964 TRY_EXEC("CREATE TABLE main.images_new (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1965 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
1966 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
1967 "focus_distance REAL, datetime_taken INTEGER, flags INTEGER, "
1968 "output_width INTEGER, output_height INTEGER, crop REAL, "
1969 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
1970 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
1971 "license VARCHAR, sha1sum CHAR(40), "
1972 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
1973 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
1974 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
1975 "aspect_ratio REAL, exposure_bias REAL, "
1976 "import_timestamp INTEGER, change_timestamp INTEGER, "
1977 "export_timestamp INTEGER, print_timestamp INTEGER, "
1978 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
1979 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
1980 "[init] can't create new images table\n");
1981
1982 TRY_EXEC("INSERT INTO `images_new` SELECT "
1983 "id, group_id, film_id, width, height, filename, maker, model, "
1984 "lens, exposure, aperture, iso, focal_length, focus_distance, NULL AS datetime_taken, flags, "
1985 "output_width, output_height, crop, raw_parameters, raw_denoise_threshold, raw_auto_bright_threshold, raw_black, raw_maximum, "
1986 "license, sha1sum, orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version, "
1987 "max_version, write_timestamp, history_end, position, aspect_ratio, exposure_bias, "
1988 "NULL AS import_timestamp, NULL AS change_timestamp, NULL AS export_timestamp, NULL AS print_timestamp "
1989 "FROM `images`",
1990 "[init] can't copy back from images\n");
1991
1992 TRY_PREPARE(stmt, "SELECT id,"
1993 " CASE WHEN datetime_taken = '' THEN NULL ELSE datetime_taken END,"
1994 " CASE WHEN import_timestamp = -1 THEN NULL ELSE import_timestamp END,"
1995 " CASE WHEN change_timestamp = -1 THEN NULL ELSE change_timestamp END,"
1996 " CASE WHEN export_timestamp = -1 THEN NULL ELSE export_timestamp END,"
1997 " CASE WHEN print_timestamp = -1 THEN NULL ELSE print_timestamp END "
1998 "FROM `images`",
1999 "[init] can't get datetime from images\n");
2000 while(sqlite3_step(stmt) == SQLITE_ROW)
2001 {
2002 sqlite3_stmt *stmt2;
2003 sqlite3_prepare_v2(db->handle,
2004 "UPDATE `images_new` SET"
2005 " (datetime_taken, import_timestamp,"
2006 " change_timestamp, export_timestamp, print_timestamp) = "
2007 " (?2, ?3, ?4, ?5, ?6) WHERE id = ?1",
2008 -1, &stmt2, NULL);
2009 sqlite3_bind_int(stmt2, 1, sqlite3_column_int(stmt, 0));
2010 if(sqlite3_column_type(stmt, 1) != SQLITE_NULL)
2011 {
2012 GDateTime *gdt = dt_datetime_exif_to_gdatetime((const char *)sqlite3_column_text(stmt, 1), darktable.utc_tz);
2013 if(gdt)
2014 {
2015 sqlite3_bind_int64(stmt2, 2, dt_datetime_gdatetime_to_gtimespan(gdt));
2016 g_date_time_unref(gdt);
2017 }
2018 }
2019 for(int i = 0; i < 4; i++)
2020 {
2021 if(sqlite3_column_type(stmt, i + 2) != SQLITE_NULL)
2022 {
2023 GDateTime *gdt = g_date_time_new_from_unix_utc(sqlite3_column_int(stmt, i + 2));
2024 if(gdt)
2025 {
2026 sqlite3_bind_int64(stmt2, i + 3, dt_datetime_gdatetime_to_gtimespan(gdt));
2027 g_date_time_unref(gdt);
2028 }
2029 }
2030 }
2031 TRY_STEP(stmt2, SQLITE_DONE, "[init] can't update datetimes into images_new table\n");
2032 sqlite3_finalize(stmt2);
2033 }
2034 sqlite3_finalize(stmt);
2035
2036 TRY_EXEC("DROP TABLE `images`", "[init] can't drop images table\n");
2037 // that's the way to keep the other tables foreign keys references valid
2038 TRY_EXEC("ALTER TABLE `images_new` RENAME TO `images`", "[init] can't rename images_new table to images");
2039
2040 // pita: need to recreate indexes
2041 TRY_EXEC("CREATE INDEX `image_position_index` ON `images` (position)",
2042 "[init] can't add image_position_index\n");
2043 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
2044 "[init] can't recreate images_filename_index\n");
2045 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
2046 "[init] can't recreate images_film_id_index\n");
2047 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
2048 "[init] can't recreate images_group_id_index\n");
2049 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( latitude DESC, longitude DESC )",
2050 "[init] can't add images_latlong_index\n");
2051 TRY_EXEC("CREATE INDEX `images_datetime_taken` ON images (datetime_taken)",
2052 "[init] can't create images_datetime_taken\n");
2053
2054 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2055 sqlite3_exec(db->handle, "PRAGMA foreign_keys = ON", NULL, NULL, NULL);
2056 new_version = 35;
2057 }
2058 else if(version == 35)
2059 {
2060 TRY_EXEC("CREATE TABLE main.images_new (id INTEGER, filename VARCHAR, flags INTEGER)",
2061 "[init] can't create new images table\n");
2062
2063 gchar *query = g_strdup_printf("INSERT INTO `images_new` "
2064 "SELECT id, filename, flags"
2065 " FROM images"
2066 " WHERE (flags & %d == 0)",
2067 DT_IMAGE_RAW | DT_IMAGE_LDR | DT_IMAGE_HDR);
2068 TRY_EXEC(query, "[init] can't copy back from images\n");
2069
2070 TRY_PREPARE(stmt, "SELECT id, filename, flags FROM `images_new`",
2071 "[init] can't prepare selecting images flags\n");
2072
2073 while(sqlite3_step(stmt) == SQLITE_ROW)
2074 {
2075 sqlite3_stmt *stmt2;
2076 sqlite3_prepare_v2(db->handle,
2077 "UPDATE `images` SET"
2078 " (flags) = "
2079 " (?2) WHERE id = ?1",
2080 -1, &stmt2, NULL);
2081 sqlite3_bind_int(stmt2, 1, sqlite3_column_int(stmt, 0));
2082
2083 dt_image_flags_t flags = sqlite3_column_int(stmt, 2);
2084 gchar *ext = g_strrstr((const char *)sqlite3_column_text(stmt, 1), ".");
2085 flags |= dt_imageio_get_type_from_extension(ext);
2086 sqlite3_bind_int(stmt2, 2, flags);
2087
2088 TRY_STEP(stmt2, SQLITE_DONE, "[init] can't update flags\n");
2089 sqlite3_finalize(stmt2);
2090 }
2091 sqlite3_finalize(stmt);
2092
2093 TRY_EXEC("DROP TABLE `images_new`", "[init] can't drop temp images table\n");
2094 new_version = 36;
2095 }
2096 else
2097 new_version = version; // should be the fallback so that calling code sees that we are in an infinite loop
2098
2099 // write the new version to db
2100 sqlite3_prepare_v2(db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', ?1)", -1, &stmt,
2101 NULL);
2102 sqlite3_bind_int(stmt, 1, new_version);
2103 sqlite3_step(stmt);
2104 sqlite3_finalize(stmt);
2105
2106 return new_version;
2107}
2108
2109/* do the real migration steps, returns the version the db was converted to */
2110static int _upgrade_data_schema_step(dt_database_t *db, int version)
2111{
2112 sqlite3_stmt *stmt;
2113 int new_version = version;
2114 if(version == CURRENT_DATABASE_VERSION_DATA)
2115 return version;
2116 else if(version == 0)
2117 {
2118 // this can't happen, we started with 1, but it's a good example how this function works
2119 // <do some magic to the db>
2120 new_version = 1; // the version we transformed the db to. this way it might be possible to roll back or
2121 // add fast paths
2122 }
2123 else if(version == 1)
2124 {
2125 // clang-format off
2126 // style_items:
2127 // NO TRY_EXEC has the column could be there before version 1 (master build)
2128 // TRY_EXEC("ALTER TABLE data.style_items ADD COLUMN iop_order REAL",
2129 // "[init] can't add `iop_order' column to style_items table in database\n");
2130 sqlite3_exec(db->handle, "ALTER TABLE data.style_items ADD COLUMN iop_order REAL", NULL, NULL, NULL);
2131 // clang-format on
2132 sqlite3_stmt *sel_stmt = NULL;
2133 GList *prior_v1 = dt_ioppr_get_iop_order_list_version(DT_IOP_ORDER_LEGACY);
2134 // clang-format off
2135 // create a temp table with the previous priorities
2136 TRY_EXEC("CREATE TEMPORARY TABLE iop_order_tmp (iop_order REAL, operation VARCHAR(256))",
2137 "[init] can't create temporary table for updating `data.style_items'\n");
2138 // clang-format on
2139 // fill temp table with all operations up to this release
2140 // it will be used to create the pipe and update the iop_order on history
2141 for(GList *priorities = prior_v1; priorities; priorities = g_list_next(priorities))
2142 {
2143 dt_iop_order_entry_t *prior = (dt_iop_order_entry_t *)priorities->data;
2144
2145 sqlite3_prepare_v2(
2146 db->handle,
2147 "INSERT INTO iop_order_tmp (iop_order, operation) VALUES (?1, ?2)",
2148 -1, &stmt, NULL);
2149 sqlite3_bind_double(stmt, 1, prior->o.iop_order_f);
2150 sqlite3_bind_text(stmt, 2, prior->operation, -1, SQLITE_TRANSIENT);
2151 TRY_STEP(stmt, SQLITE_DONE, "[init] can't insert default value in iop_order_tmp\n");
2152 sqlite3_finalize(stmt);
2153 }
2154 g_list_free_full(prior_v1, dt_free_gpointer);
2155 prior_v1 = NULL;
2156
2157 // do the same as for history
2158 // clang-format off
2159 TRY_EXEC("UPDATE data.style_items SET iop_order = ((("
2160 "SELECT MAX(multi_priority) FROM data.style_items style1 WHERE style1.styleid = data.style_items.styleid AND style1.operation = data.style_items.operation "
2161 ") + 1. - multi_priority) / 1000.) + "
2162 "IFNULL((SELECT iop_order FROM iop_order_tmp WHERE iop_order_tmp.operation = "
2163 "data.style_items.operation), -999999.) ",
2164 "[init] can't update iop_order in style_items table\n");
2165
2166 TRY_PREPARE(sel_stmt, "SELECT DISTINCT operation FROM data.style_items WHERE iop_order <= 0 OR iop_order IS NULL",
2167 "[init] can't prepare selecting style_items iop_order\n");
2168 // clang-format on
2169 while(sqlite3_step(sel_stmt) == SQLITE_ROW)
2170 {
2171 const char *op_name = (const char *)sqlite3_column_text(sel_stmt, 0);
2172 printf("operation %s with no iop_order while upgrading style_items in database\n", op_name);
2173 }
2174 sqlite3_finalize(sel_stmt);
2175 // clang-format off
2176 TRY_EXEC("DROP TABLE iop_order_tmp", "[init] can't drop table `iop_order_tmp' from database\n");
2177 // clang-format on
2178 new_version = 2;
2179 }
2180 else if(version == 2)
2181 {
2182 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2183
2184 // With sqlite above or equal to 3.25.0 RENAME COLUMN can be used instead of the following code
2185 // TRY_EXEC("ALTER TABLE data.tags RENAME COLUMN description TO synonyms;",
2186 // "[init] can't change tags column name from description to synonyms\n");
2187 // clang-format off
2188 TRY_EXEC("ALTER TABLE data.tags RENAME TO tmp_tags", "[init] can't rename table tags\n");
2189
2190 TRY_EXEC("CREATE TABLE data.tags (id INTEGER PRIMARY KEY, name VARCHAR, "
2191 "synonyms VARCHAR, flags INTEGER)",
2192 "[init] can't create new tags table\n");
2193
2194 TRY_EXEC("INSERT INTO data.tags (id, name, synonyms, flags) SELECT id, name, description, flags "
2195 "FROM tmp_tags",
2196 "[init] can't populate tags table from tmp_tags\n");
2197
2198 TRY_EXEC("DROP TABLE tmp_tags", "[init] can't delete table tmp_tags\n");
2199
2200 TRY_EXEC("CREATE UNIQUE INDEX data.tags_name_idx ON tags (name)",
2201 "[init] can't create tags_name_idx on tags table\n");
2202 // clang-format on
2203 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2204
2205 new_version = 3;
2206 }
2207 else if(version == 3)
2208 {
2209 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2210 // clang-format off
2211 // create a temp table to invert all multi_priority
2212 TRY_EXEC("CREATE TEMPORARY TABLE m_prio (id INTEGER, operation VARCHAR(256), prio INTEGER)",
2213 "[init] can't create temporary table for updating `history and style_items'\n");
2214
2215 TRY_EXEC("INSERT INTO m_prio SELECT styleid, operation, MAX(multi_priority)"
2216 " FROM data.style_items GROUP BY styleid, operation",
2217 "[init] can't populate m_prio\n");
2218
2219 // update multi_priority for style items and history
2220 TRY_EXEC("UPDATE data.style_items SET multi_priority = "
2221 "(SELECT prio FROM m_prio "
2222 " WHERE data.style_items.operation = operation AND data.style_items.styleid = id)"
2223 " - data.style_items.multi_priority",
2224 "[init] can't update multi_priority for style_items\n");
2225
2226 TRY_EXEC("DROP TABLE m_prio", "[init] can't drop table `m_prio' from database\n");
2227 // clang-format on
2228 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2229
2230 new_version = 4;
2231 }
2232 else if(version == 4)
2233 {
2234 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2235 // clang-format off
2236 // remove iop_order from style_item table
2237 TRY_EXEC("ALTER TABLE data.style_items RENAME TO s",
2238 "[init] can't rename style_items to s\n");
2239 TRY_EXEC("CREATE TABLE data.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
2240 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2241 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2242 "[init] can't create style_items table'\n");
2243 TRY_EXEC("INSERT INTO data.style_items SELECT styleid, num, module, operation, op_params, enabled, "
2244 " blendop_params, blendop_version, multi_priority, multi_name "
2245 "FROM s",
2246 "[init] can't populate style_items table'\n");
2247 TRY_EXEC("DROP TABLE s",
2248 "[init] can't drop table s'\n");
2249 // clang-format on
2250 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2251
2252 new_version = 5;
2253 }
2254 else if(version == 5)
2255 {
2256 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2257 // clang-format of
2258 // make style.id a PRIMARY KEY and add iop_list
2259 TRY_EXEC("ALTER TABLE data.styles RENAME TO s",
2260 "[init] can't rename styles to s\n");
2261 TRY_EXEC("CREATE TABLE data.styles (id INTEGER PRIMARY KEY, name VARCHAR, description VARCHAR, iop_list VARCHAR)",
2262 "[init] can't create styles table\n");
2263 TRY_EXEC("INSERT INTO data.styles SELECT id, name, description, NULL FROM s",
2264 "[init] can't populate styles table\n");
2265 TRY_EXEC("DROP TABLE s",
2266 "[init] can't drop table s\n");
2267
2268 TRY_EXEC("CREATE INDEX IF NOT EXISTS data.styles_name_index ON styles (name)",
2269 "[init] can't create styles_nmae_index\n");
2270
2271 // make style_items.styleid index
2272
2273 TRY_EXEC("CREATE INDEX IF NOT EXISTS data.style_items_styleid_index ON style_items (styleid)",
2274 "[init] can't create style_items_styleid_index\n");
2275 // clang-format on
2276 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2277
2278 new_version = 6;
2279 }
2280 else if(version == 6)
2281 {
2282 // clang-format off
2283 TRY_EXEC("CREATE TABLE data.locations "
2284 "(tagid INTEGER PRIMARY KEY, type INTEGER, longitude REAL, latitude REAL, "
2285 "delta1 REAL, delta2 REAL, FOREIGN KEY(tagid) REFERENCES tags(id))",
2286 "[init] can't create new locations table\n");
2287 // clang-format on
2288 new_version = 7;
2289 }
2290 else if(version == 7)
2291 {
2292 // clang-format off
2293 TRY_EXEC("ALTER TABLE data.locations ADD COLUMN ratio FLOAT DEFAULT 1",
2294 "[init] can't add column `ratio' column to locations table\n");
2295 // clang-format on
2296 new_version = 8;
2297 }
2298 else if(version == 8)
2299 {
2300 // clang-format off
2301 TRY_EXEC("ALTER TABLE data.locations ADD COLUMN polygons BLOB",
2302 "[init] can't add column `polygons' column to locations table\n");
2303 // clang-format on
2304 new_version = 9;
2305 }
2306 else
2307 new_version = version; // should be the fallback so that calling code sees that we are in an infinite loop
2308
2309 // write the new version to db
2310 // clang-format off¨
2311 sqlite3_prepare_v2(db->handle, "INSERT OR REPLACE INTO data.db_info (key, value) VALUES ('version', ?1)", -1, &stmt,
2312 NULL);
2313 // clang-format on
2314 sqlite3_bind_int(stmt, 1, new_version);
2315 sqlite3_step(stmt);
2316 sqlite3_finalize(stmt);
2317
2318 return new_version;
2319}
2320
2321#undef FINALIZE
2322
2323#undef TRY_EXEC
2324#undef TRY_STEP
2325#undef TRY_PREPARE
2326
2327/* upgrade library db from 'version' to CURRENT_DATABASE_VERSION_LIBRARY. don't touch this function but
2328 * _upgrade_library_schema_step() instead. */
2329static gboolean _upgrade_library_schema(dt_database_t *db, int version)
2330{
2331 while(version < CURRENT_DATABASE_VERSION_LIBRARY)
2332 {
2333 const int new_version = _upgrade_library_schema_step(db, version);
2334 if(new_version == version)
2335 return FALSE; // we don't know how to upgrade this db. probably a bug in _upgrade_library_schema_step
2336 else
2337 version = new_version;
2338 }
2339 return TRUE;
2340}
2341
2342/* upgrade data db from 'version' to CURRENT_DATABASE_VERSION_DATA. don't touch this function but
2343 * _upgrade_data_schema_step() instead. */
2344static gboolean _upgrade_data_schema(dt_database_t *db, int version)
2345{
2346 while(version < CURRENT_DATABASE_VERSION_DATA)
2347 {
2348 const int new_version = _upgrade_data_schema_step(db, version);
2349 if(new_version == version)
2350 return FALSE; // we don't know how to upgrade this db. probably a bug in _upgrade_data_schema_step
2351 else
2352 version = new_version;
2353 }
2354 return TRUE;
2355}
2356
2357/* create the current database schema and set the version in db_info accordingly */
2358static void _create_library_schema(dt_database_t *db)
2359{
2360 sqlite3_stmt *stmt;
2362 // clang-format off
2363 sqlite3_exec(db->handle, "CREATE TABLE main.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)", NULL,
2364 NULL, NULL);
2365 sqlite3_prepare_v2(
2366 db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', ?1)", -1, &stmt, NULL);
2367 // clang-format on
2368 sqlite3_bind_int(stmt, 1, CURRENT_DATABASE_VERSION_LIBRARY);
2369 sqlite3_step(stmt);
2370 sqlite3_finalize(stmt);
2372 // clang-format off
2373 sqlite3_exec(db->handle,
2374 "CREATE TABLE main.film_rolls "
2375 "(id INTEGER PRIMARY KEY, access_timestamp INTEGER, "
2376 // "folder VARCHAR(1024), external_drive VARCHAR(1024))", //
2377 // FIXME: make sure to bump CURRENT_DATABASE_VERSION_LIBRARY and add a
2378 // case to _upgrade_library_schema_step when adding this!
2379 "folder VARCHAR(1024) NOT NULL)",
2380 NULL, NULL, NULL);
2381 sqlite3_exec(db->handle, "CREATE INDEX main.film_rolls_folder_index ON film_rolls (folder)", NULL, NULL, NULL);
2383 sqlite3_exec(
2384 db->handle,
2385 "CREATE TABLE main.images (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
2386 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
2387 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
2388 "focus_distance REAL, datetime_taken INTEGER, flags INTEGER, "
2389 "output_width INTEGER, output_height INTEGER, crop REAL, "
2390 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
2391 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
2392 "license VARCHAR, sha1sum CHAR(40), "
2393 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
2394 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
2395 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
2396 "aspect_ratio REAL, exposure_bias REAL, "
2397 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, "
2398 "export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1, "
2399 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
2400 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
2401 NULL, NULL, NULL);
2402 sqlite3_exec(db->handle, "CREATE INDEX main.images_group_id_index ON images (group_id, id)", NULL, NULL, NULL);
2403 sqlite3_exec(db->handle, "CREATE INDEX main.images_film_id_index ON images (film_id, filename)", NULL, NULL, NULL);
2404 sqlite3_exec(db->handle, "CREATE INDEX main.images_filename_index ON images (filename, version)", NULL, NULL, NULL);
2405 sqlite3_exec(db->handle, "CREATE INDEX main.image_position_index ON images (position)", NULL, NULL, NULL);
2406 sqlite3_exec(db->handle, "CREATE INDEX main.images_datetime_taken_nc ON images (datetime_taken)", NULL, NULL, NULL);
2407
2409 sqlite3_exec(db->handle, "CREATE TABLE main.selected_images (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2411 sqlite3_exec(
2412 db->handle,
2413 "CREATE TABLE main.history (imgid INTEGER, num INTEGER, module INTEGER, "
2414 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2415 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256), "
2416 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2417 NULL, NULL, NULL);
2418 sqlite3_exec(db->handle, "CREATE INDEX main.history_imgid_op_index ON history (imgid, operation)", NULL, NULL, NULL);
2419 sqlite3_exec(db->handle, "CREATE INDEX main.history_imgid_num_index ON history (imgid, num DESC)", NULL, NULL, NULL);
2421 sqlite3_exec(db->handle,
2422 "CREATE TABLE main.masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, name VARCHAR(256), "
2423 "version INTEGER, points BLOB, points_count INTEGER, source BLOB, "
2424 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2425 NULL, NULL, NULL);
2426
2427 sqlite3_exec(db->handle,
2428 "CREATE INDEX main.masks_history_imgid_index ON masks_history (imgid, num)",
2429 NULL, NULL, NULL);
2430
2431 sqlite3_exec(db->handle, "CREATE INDEX main.images_latlong_index ON images (latitude DESC, longitude DESC)",
2432 NULL, NULL, NULL);
2433
2435 sqlite3_exec(db->handle, "CREATE TABLE main.tagged_images (imgid INTEGER, tagid INTEGER, position INTEGER, "
2436 "PRIMARY KEY (imgid, tagid),"
2437 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)", NULL, NULL, NULL);
2438 sqlite3_exec(db->handle, "CREATE INDEX main.tagged_images_tagid_index ON tagged_images (tagid)", NULL, NULL, NULL);
2439 sqlite3_exec(db->handle, "CREATE INDEX main.tagged_images_position_index ON tagged_images (position)", NULL, NULL, NULL);
2441 sqlite3_exec(db->handle, "CREATE TABLE main.color_labels (imgid INTEGER, color INTEGER)", NULL, NULL, NULL);
2442 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX main.color_labels_idx ON color_labels (imgid, color)", NULL, NULL,
2443 NULL);
2445 sqlite3_exec(db->handle, "CREATE TABLE main.meta_data (id INTEGER, key INTEGER, value VARCHAR)", NULL, NULL, NULL);
2446 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX main.metadata_index ON meta_data (id, key, value)", NULL, NULL, NULL);
2447
2448 sqlite3_exec(db->handle, "CREATE INDEX main.metadata_index_key ON meta_data (key)", NULL, NULL, NULL);
2449 sqlite3_exec(db->handle, "CREATE TABLE main.module_order (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR)",
2450 NULL, NULL, NULL);
2451 sqlite3_exec(db->handle, "CREATE TABLE main.history_hash (imgid INTEGER PRIMARY KEY, "
2452 "basic_hash BLOB, auto_hash BLOB, current_hash BLOB, mipmap_hash BLOB, "
2453 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2454 NULL, NULL, NULL);
2455
2456 // v34
2457 sqlite3_exec(db->handle, "CREATE INDEX main.images_datetime_taken_nc ON images (datetime_taken COLLATE NOCASE)",
2458 NULL, NULL, NULL);
2459 sqlite3_exec(db->handle, "CREATE INDEX main.metadata_index_key ON meta_data (key)", NULL, NULL, NULL);
2460 // clang-format on
2461}
2462
2463/* create the current database schema and set the version in db_info accordingly */
2464static void _create_data_schema(dt_database_t *db)
2465{
2466 sqlite3_stmt *stmt;
2467 // clang-format off
2469 sqlite3_exec(db->handle, "CREATE TABLE data.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)", NULL,
2470 NULL, NULL);
2471 sqlite3_prepare_v2(
2472 db->handle, "INSERT OR REPLACE INTO data.db_info (key, value) VALUES ('version', ?1)", -1, &stmt, NULL);
2473 sqlite3_bind_int(stmt, 1, CURRENT_DATABASE_VERSION_DATA);
2474 sqlite3_step(stmt);
2475 sqlite3_finalize(stmt);
2477 sqlite3_exec(db->handle, "CREATE TABLE data.tags (id INTEGER PRIMARY KEY, name VARCHAR, "
2478 "synonyms VARCHAR, flags INTEGER)", NULL, NULL, NULL);
2479 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX data.tags_name_idx ON tags (name)", NULL, NULL, NULL);
2481 sqlite3_exec(db->handle, "CREATE TABLE data.styles (id INTEGER PRIMARY KEY, name VARCHAR, description VARCHAR, iop_list VARCHAR)",
2482 NULL, NULL, NULL);
2483 sqlite3_exec(db->handle, "CREATE INDEX data.styles_name_index ON styles (name)", NULL, NULL, NULL);
2485 sqlite3_exec(
2486 db->handle,
2487 "CREATE TABLE data.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
2488 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2489 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2490 NULL, NULL, NULL);
2491 sqlite3_exec(
2492 db->handle,
2493 "CREATE INDEX IF NOT EXISTS data.style_items_styleid_index ON style_items (styleid)",
2494 NULL, NULL, NULL);
2496 sqlite3_exec(db->handle, "CREATE TABLE data.presets (name VARCHAR, description VARCHAR, operation "
2497 "VARCHAR, op_version INTEGER, op_params BLOB, "
2498 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, "
2499 "multi_priority INTEGER, multi_name VARCHAR(256), "
2500 "model VARCHAR, maker VARCHAR, lens VARCHAR, iso_min REAL, iso_max REAL, "
2501 "exposure_min REAL, exposure_max REAL, "
2502 "aperture_min REAL, aperture_max REAL, focal_length_min REAL, "
2503 "focal_length_max REAL, writeprotect INTEGER, "
2504 "autoapply INTEGER, filter INTEGER, def INTEGER, format INTEGER)",
2505 NULL, NULL, NULL);
2506 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX data.presets_idx ON presets (name, operation, op_version)",
2507 NULL, NULL, NULL);
2509 sqlite3_exec(db->handle, "CREATE TABLE data.locations (tagid INTEGER PRIMARY KEY, "
2510 "type INTEGER, longitude REAL, latitude REAL, delta1 REAL, delta2 REAL, ratio FLOAT, polygons BLOB, "
2511 "FOREIGN KEY(tagid) REFERENCES tags(id))", NULL, NULL, NULL);
2512 // clang-format on
2513}
2514
2515// create the in-memory tables
2516// temporary stuff for some ops, need this for some reason with newer sqlite3:
2517static void _create_memory_schema(dt_database_t *db)
2518{
2519 // clang-format off
2520 sqlite3_exec(db->handle, "CREATE TABLE memory.color_labels_temp (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2521 sqlite3_exec(
2522 db->handle,
2523 "CREATE TABLE memory.collected_images (rowid INTEGER PRIMARY KEY AUTOINCREMENT, imgid INTEGER)", NULL,
2524 NULL, NULL);
2525 sqlite3_exec(
2526 db->handle,
2527 "CREATE TABLE memory.collected_backup (rowid INTEGER PRIMARY KEY AUTOINCREMENT, imgid INTEGER)", NULL,
2528 NULL, NULL); sqlite3_exec(db->handle, "CREATE TABLE memory.selected_backup (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2529 sqlite3_exec(db->handle, "CREATE TABLE memory.selected_backup (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2530 sqlite3_exec(db->handle, "CREATE TABLE memory.tmp_selection (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2531 sqlite3_exec(db->handle, "CREATE TABLE memory.taglist "
2532 "(tmpid INTEGER PRIMARY KEY, id INTEGER UNIQUE ON CONFLICT IGNORE, "
2533 "count INTEGER DEFAULT 0, count2 INTEGER DEFAULT 0)",
2534 NULL, NULL, NULL);
2535 sqlite3_exec(db->handle, "CREATE TABLE memory.similar_tags (tagid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2536 sqlite3_exec(db->handle, "CREATE TABLE memory.darktable_tags (tagid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2537 sqlite3_exec(
2538 db->handle,
2539 "CREATE TABLE memory.undo_history (id INTEGER, imgid INTEGER, num INTEGER, module INTEGER, "
2540 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2541 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2542 NULL, NULL, NULL);
2543 sqlite3_exec(
2544 db->handle,
2545 "CREATE TABLE memory.undo_masks_history (id INTEGER, imgid INTEGER, num INTEGER, formid INTEGER,"
2546 " form INTEGER, name VARCHAR(256), version INTEGER, points BLOB, points_count INTEGER, source BLOB)",
2547 NULL, NULL, NULL);
2548 sqlite3_exec(
2549 db->handle,
2550 "CREATE TABLE memory.undo_module_order (id INTEGER, imgid INTEGER, version INTEGER, iop_list VARCHAR)",
2551 NULL, NULL, NULL);
2552 sqlite3_exec(db->handle,
2553 "CREATE TABLE memory.darktable_iop_names (operation VARCHAR(256) PRIMARY KEY, name VARCHAR(256))",
2554 NULL, NULL, NULL);
2555 sqlite3_exec(db->handle,
2556 "CREATE TABLE memory.film_folder (id INTEGER PRIMARY KEY, status INTEGER)",
2557 NULL, NULL, NULL);
2558 // clang-format on
2559}
2560
2561static void _sanitize_db(dt_database_t *db)
2562{
2563 sqlite3_stmt *stmt, *innerstmt;
2564 // clang-format off
2565 /* first let's get rid of non-utf8 tags. */
2566 sqlite3_prepare_v2(db->handle, "SELECT id, name FROM data.tags", -1, &stmt, NULL);
2567 sqlite3_prepare_v2(db->handle, "UPDATE data.tags SET name = ?1 WHERE id = ?2", -1, &innerstmt, NULL);
2568 // clang-format on
2569 while(sqlite3_step(stmt) == SQLITE_ROW)
2570 {
2571 const int id = sqlite3_column_int(stmt, 0);
2572 const char *tag = (const char *)sqlite3_column_text(stmt, 1);
2573
2574 if(!g_utf8_validate(tag, -1, NULL))
2575 {
2576 gchar *new_tag = dt_util_foo_to_utf8(tag);
2577 fprintf(stderr, "[init]: tag `%s' is not valid utf8, replacing it with `%s'\n", tag, new_tag);
2578 if(tag)
2579 {
2580 sqlite3_bind_text(innerstmt, 1, new_tag, -1, SQLITE_TRANSIENT);
2581 sqlite3_bind_int(innerstmt, 2, id);
2582 sqlite3_step(innerstmt);
2583 sqlite3_reset(innerstmt);
2584 sqlite3_clear_bindings(innerstmt);
2585 dt_free(new_tag);
2586 }
2587 }
2588 }
2589 sqlite3_finalize(stmt);
2590 sqlite3_finalize(innerstmt);
2591 // clang-format off
2592 // make sure film_roll folders don't end in "/", that will result in empty entries in the collect module
2593 sqlite3_exec(db->handle,
2594 "UPDATE main.film_rolls SET folder = substr(folder, 1, length(folder) - 1) WHERE folder LIKE '%/'",
2595 NULL, NULL, NULL);
2596 // clang-format on
2597}
2598
2599// in library we keep the names of the tags used in tagged_images. however, using that table at runtime results
2600// in some overhead not necessary so instead we just use the used_tags table to update tagged_images on startup
2601#define TRY_EXEC(_query, _message) \
2602 do \
2603 { \
2604 if(sqlite3_exec(db->handle, _query, NULL, NULL, NULL) != SQLITE_OK) \
2605 { \
2606 fprintf(stderr, _message); \
2607 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2608 FINALIZE; \
2609 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2610 return FALSE; \
2611 } \
2612 } while(0)
2613
2614#define TRY_STEP(_stmt, _expected, _message) \
2615 do \
2616 { \
2617 if(sqlite3_step(_stmt) != _expected) \
2618 { \
2619 fprintf(stderr, _message); \
2620 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2621 FINALIZE; \
2622 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2623 return FALSE; \
2624 } \
2625 } while(0)
2626
2627#define TRY_PREPARE(_stmt, _query, _message) \
2628 do \
2629 { \
2630 if(sqlite3_prepare_v2(db->handle, _query, -1, &_stmt, NULL) != SQLITE_OK) \
2631 { \
2632 fprintf(stderr, _message); \
2633 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2634 FINALIZE; \
2635 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2636 return FALSE; \
2637 } \
2638 } while(0)
2639
2640#define FINALIZE \
2641 do \
2642 { \
2643 sqlite3_finalize(stmt); stmt = NULL; /* NULL so that finalize becomes a NOP */ \
2644 } while(0)
2645
2646#undef TRY_EXEC
2647#undef TRY_STEP
2648#undef TRY_PREPARE
2649#undef FINALIZE
2650
2651gboolean dt_database_show_error(const dt_database_t *db)
2652{
2653 gboolean error = TRUE;
2654
2655 if(!db->lock_acquired)
2656 {
2657 char lck_pathname[1024];
2658 snprintf(lck_pathname, sizeof(lck_pathname), "%s.lock", db->error_dbfilename);
2659 char *lck_dirname = g_strdup(lck_pathname);
2660 char *slash_pos = g_strrstr(lck_dirname, "/");
2661 if(!IS_NULL_PTR(slash_pos)) *slash_pos = '\0';
2662 // clang-format off
2663 char *label_text = g_markup_printf_escaped(
2664 _("\n"
2665 " Sorry, Ansel could not be started because the database is locked.\n"
2666 "\n"
2667 " How to solve this problem?\n"
2668 "\n"
2669 " 1 - If another Ansel instance is already running, \n"
2670 " click \"Quit\" and either use that instance or close it before trying to start Ansel again. \n"
2671 " (process ID <i><b>%d</b></i> created the database lock files)\n"
2672 "\n"
2673 " 2 - If you cannot find any running instance of Ansel, try restarting your session or your computer. \n"
2674 " This will close all running programs and should release any database locks. \n"
2675 "\n"
2676 " 3 - If you have already tried the above steps, or you are certain that no other instances of Ansel are running, \n"
2677 " this likely means the previous instance ended unexpectedly. \n"
2678 " Click the \"Delete database lock files\" button to remove <i>data.db.lock</i> and <i>library.db.lock</i>. \n"
2679 " Ansel will then attempt to load the database again. \n"
2680 "\n\n"
2681 " <i><u>Caution!</u> Do not delete these files without first undertaking the above checks, \n"
2682 " otherwise you risk generating serious inconsistencies in your database.</i>\n"),
2683 db->error_other_pid);
2684 // clang-format on
2685
2686 gboolean delete_lockfiles = dt_gui_show_standalone_yes_no_dialog(_("Error starting Ansel"),
2687 label_text, _("Quit"), _("Delete database lock files and try again"));
2688
2689 if(delete_lockfiles)
2690 {
2691 gboolean really_delete_lockfiles =
2692 dt_gui_show_standalone_yes_no_dialog
2693 (_("Confirmation"),
2694 _("\n<u>Caution!</u> Are you sure you want to delete the database lock files?\n"
2695 "This action should only be performed if you are certain no other Ansel instances are running.\n"), _("Quit"), _("Yes"));
2696 if(really_delete_lockfiles)
2697 {
2698 int status = 0;
2699
2700 char *lck_filename = g_strconcat(lck_dirname, "/data.db.lock", NULL);
2701 if(g_access(lck_filename, F_OK) != -1)
2702 status += remove(lck_filename);
2703
2704 lck_filename = g_strconcat(lck_dirname, "/library.db.lock", NULL);
2705 if(g_access(lck_filename, F_OK) != -1)
2706 status += remove(lck_filename);
2707 dt_free(lck_filename);
2708
2709 if(status==0)
2710 {
2711 dt_gui_show_standalone_yes_no_dialog(_("Done"),
2712 _("\nThe database lock files have been deleted successfully.\n"),
2713 _("Continue"), NULL);
2714 error = FALSE;
2715 }
2716
2717 else
2718 dt_gui_show_standalone_yes_no_dialog
2719 (_("Error"), g_markup_printf_escaped(
2720 _("\nAt least one lock file could not be removed.\n"
2721 "You may try to manually delete the files <i>data.db.lock</i> and <i>library.db.lock</i>\n"
2722 "in folder <a href=\"file:///%s\">%s</a>.\n"), lck_dirname, lck_dirname),
2723 _("Quit"), NULL);
2724 }
2725 }
2726
2727 dt_free(lck_dirname);
2728 dt_free(label_text);
2729 }
2730
2731 dt_free(db->error_message);
2732 dt_free(db->error_dbfilename);
2733 ((dt_database_t *)db)->error_other_pid = 0;
2734 ((dt_database_t *)db)->error_message = NULL;
2735 ((dt_database_t *)db)->error_dbfilename = NULL;
2736 return error;
2737}
2738
2739static gboolean pid_is_alive(int pid)
2740{
2741 gboolean pid_is_alive;
2742
2743#ifdef _WIN32
2744 pid_is_alive = FALSE;
2745 HANDLE h = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid);
2746 if(h)
2747 {
2748 wchar_t wfilename[MAX_PATH];
2749 long unsigned int n_filename = sizeof(wfilename);
2750 int ret = QueryFullProcessImageNameW(h, 0, wfilename, &n_filename);
2751 char *filename = g_utf16_to_utf8(wfilename, -1, NULL, NULL, NULL);
2752 if(ret && n_filename > 0 && filename && g_str_has_suffix(filename, "ansel.exe"))
2753 pid_is_alive = TRUE;
2754 dt_free(filename);
2755 CloseHandle(h);
2756 }
2757#else
2758 pid_is_alive = !((kill(pid, 0) == -1) && errno == ESRCH);
2759
2760#ifdef __linux__
2761 // If this is Linux, we can query /proc to see if the pid is
2762 // actually a darktable instance.
2763 if(pid_is_alive)
2764 {
2765 gchar *contents;
2766 gsize length;
2767 gchar filename[64];
2768 snprintf(filename, sizeof(filename), "/proc/%d/cmdline", pid);
2769
2770 if(g_file_get_contents("", &contents, &length, NULL))
2771 {
2772 if(strstr(contents, "ansel") == NULL)
2773 {
2774 pid_is_alive = FALSE;
2775 }
2776 dt_free(contents);
2777 }
2778 }
2779#endif
2780
2781#endif
2782
2783 return pid_is_alive;
2784}
2785
2786static gboolean _lock_single_database(dt_database_t *db, const char *dbfilename, char **lockfile)
2787{
2788 gboolean lock_acquired = FALSE;
2789 mode_t old_mode;
2790 int lock_tries = 0;
2791 gchar *pid = g_strdup_printf("%d", getpid());
2792
2793 if(!g_strcmp0(dbfilename, ":memory:"))
2794 {
2795 lock_acquired = TRUE;
2796 }
2797 else
2798 {
2799 *lockfile = g_strconcat(dbfilename, ".lock", NULL);
2800lock_again:
2801 lock_tries++;
2802 old_mode = umask(0);
2803 int fd = g_open(*lockfile, O_RDWR | O_CREAT | O_EXCL, 0666);
2804 umask(old_mode);
2805
2806 if(fd != -1) // the lockfile was successfully created - write our PID into it
2807 {
2808 if(write(fd, pid, strlen(pid) + 1) > -1) lock_acquired = TRUE;
2809 close(fd);
2810 }
2811 else // the lockfile already exists - see if it's a stale one left over from a crashed instance
2812 {
2813 char buf[64];
2814 memset(buf, 0, sizeof(buf));
2815 fd = g_open(*lockfile, O_RDWR | O_CREAT, 0666);
2816
2817 if(fd == -1)
2818 {
2819 // failed to open the lock file
2820 const int err = errno;
2821 fprintf(stderr, "[init] error opening the database lock file for reading: %s\n", strerror(err));
2822 db->error_message = g_strdup_printf(_("error opening the database lock file for reading: %s"), strerror(err));
2823 }
2824 else
2825 {
2826 // successfully opened the lock file, read the PID
2827 const int bytes_read = read(fd, buf, sizeof(buf) - 1);
2828
2829 if(bytes_read > 0)
2830 {
2831 // lock file contains a PID
2832 db->error_other_pid = atoi(buf);
2833
2834 if(!pid_is_alive(db->error_other_pid))
2835 {
2836 // stale lock file - close fd before unlinking (required on Windows)
2837 close(fd);
2838 g_unlink(*lockfile);
2839
2840 if(lock_tries < 5)
2841 {
2842 goto lock_again;
2843 }
2844 else
2845 {
2846 fprintf(stderr, "[init] tried several times to acquire the database lock, giving up\n");
2847 db->error_message = g_strdup_printf(_("tried several times to acquire the database lock, giving up"));
2848 }
2849 }
2850 else
2851 {
2852 // lock file contains a live PID
2853 close(fd);
2854 fprintf(stderr, "[init] the database lock file contains a pid that seems to be alive in your system: %d\n",
2855 db->error_other_pid);
2856 db->error_message = g_strdup_printf(_("the database lock file contains a pid that seems to be alive in your system: %d"),
2857 db->error_other_pid);
2858 }
2859 }
2860 else
2861 {
2862 // lock file is empty or unreadable
2863 close(fd);
2864 fprintf(stderr, "[init] the database lock file seems to be empty\n");
2865 db->error_message = g_strdup_printf(_("the database lock file seems to be empty"));
2866 }
2867 }
2868 }
2869 }
2870
2871 dt_free(pid);
2872
2873 if(db->error_message)
2874 db->error_dbfilename = g_strdup(dbfilename);
2875
2876 return lock_acquired;
2877}
2878
2879static gboolean _lock_databases(dt_database_t *db)
2880{
2881 if(!_lock_single_database(db, db->dbfilename_data, &db->lockfile_data))
2882 return FALSE;
2883 if(!_lock_single_database(db, db->dbfilename_library, &db->lockfile_library))
2884 {
2885 // unlock data.db to not leave a stale lock file around
2886 g_unlink(db->lockfile_data);
2887 return FALSE;
2888 }
2889 return TRUE;
2890}
2891
2892void ask_for_upgrade(const gchar *dbname, const gboolean has_gui)
2893{
2894 // if there's no gui just leave
2895 if(!has_gui)
2896 {
2897 fprintf(stderr, "[init] database `%s' is out-of-date. aborting.\n", dbname);
2898 exit(1);
2899 }
2900
2901 // the database has to be upgraded, let's ask user
2902
2903 char *label_text = g_markup_printf_escaped(_("the database schema has to be upgraded for\n"
2904 "\n"
2905 "<span style='italic'>%s</span>\n"
2906 "\nthis might take a long time in case of a large database\n\n"
2907 "do you want to proceed or quit now to do a backup\n"),
2908 dbname);
2909
2910 gboolean shall_we_update_the_db =
2911 dt_gui_show_standalone_yes_no_dialog(_("ansel - schema migration"), label_text,
2912 _("close Ansel"), _("upgrade database"));
2913
2914 dt_free(label_text);
2915
2916 // if no upgrade, we exit now, nothing we can do more
2917 if(!shall_we_update_the_db)
2918 {
2919 fprintf(stderr, "[init] we shall not update the database, aborting.\n");
2920 exit(1);
2921 }
2922}
2923
2924void dt_database_backup(const char *filename)
2925{
2926 char *version = g_strdup(darktable_package_version);
2927 int k = 0;
2928 // get plain version (no commit id)
2929 while(version[k])
2930 {
2931 if((version[k] < '0' || version[k] > '9') && (version[k] != '.'))
2932 {
2933 version[k] = '\0';
2934 break;
2935 }
2936 k++;
2937 }
2938
2939 gchar *backup = g_strdup_printf("%s-pre-%s", filename, version);
2940
2941 GError *gerror = NULL;
2942 if(!g_file_test(backup, G_FILE_TEST_EXISTS))
2943 {
2944 GFile *src = g_file_new_for_path(filename);
2945 GFile *dest = g_file_new_for_path(backup);
2946 gboolean copy_status = TRUE;
2947 if(g_file_test(filename, G_FILE_TEST_EXISTS))
2948 {
2949 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
2950 if(copy_status) copy_status = g_chmod(backup, S_IRUSR) == 0;
2951 }
2952 else
2953 {
2954 // there is nothing to backup, create an empty file to prevent further backup attempts
2955 int fd = g_open(backup, O_CREAT, S_IRUSR);
2956 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
2957 }
2958 if(!copy_status) fprintf(stderr, "[backup failed] %s -> %s\n", filename, backup);
2959
2960 g_object_unref(src);
2961 g_object_unref(dest);
2962 }
2963
2964 dt_free(version);
2965 dt_free(backup);
2966}
2967
2968int _get_pragma_int_val(sqlite3 *db, const char* pragma)
2969{
2970 gchar* query= g_strdup_printf("PRAGMA %s", pragma);
2971 int val = -1;
2972 sqlite3_stmt *stmt;
2973 const int rc = sqlite3_prepare_v2(db, query,-1, &stmt, NULL);
2974 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
2975 {
2976 val = sqlite3_column_int(stmt, 0);
2977 }
2978 sqlite3_finalize(stmt);
2979 dt_free(query);
2980
2981 return val;
2982}
2983
2984gchar* _get_pragma_string_val(sqlite3 *db, const char* pragma)
2985{
2986 gchar* query= g_strdup_printf("PRAGMA %s", pragma);
2987 sqlite3_stmt *stmt;
2988 gchar* val = NULL;
2989 const int rc = sqlite3_prepare_v2(db, query,-1, &stmt, NULL);
2990 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
2991 {
2992 val = g_strdup((const char *)sqlite3_column_text(stmt, 0));
2993 while(sqlite3_step(stmt) == SQLITE_ROW)
2994 {
2995 gchar* cur_val = g_strdup((const char *)sqlite3_column_text(stmt, 0));
2996 gchar* tmp_val = g_strdup(val);
2997 dt_free(val);
2998 val = g_strconcat(tmp_val, "\n", cur_val, NULL);
2999 dt_free(cur_val);
3000 dt_free(tmp_val);
3001 }
3002 }
3003 sqlite3_finalize(stmt);
3004 dt_free(query);
3005 return val;
3006}
3007
3008dt_database_t *dt_database_init(const char *alternative, const gboolean load_data, const gboolean has_gui)
3009{
3010 /* set the threading mode to Serialized */
3011 sqlite3_config(SQLITE_CONFIG_SERIALIZED);
3012
3013 sqlite3_initialize();
3014
3015start:
3016 if(IS_NULL_PTR(alternative))
3017 {
3018 /* migrate default database location to new default */
3019 _database_migrate_to_xdg_structure();
3020 }
3021
3022 /* delete old mipmaps files */
3023 _database_delete_mipmaps_files();
3024
3025 /* lets construct the db filename */
3026 gchar *dbname = NULL;
3027 gchar dbfilename_library[PATH_MAX] = { 0 };
3028 gchar datadir[PATH_MAX] = { 0 };
3029
3030 dt_loc_get_user_config_dir(datadir, sizeof(datadir));
3031
3032 if(IS_NULL_PTR(alternative))
3033 {
3034 dbname = dt_conf_get_string("database");
3035 if(IS_NULL_PTR(dbname))
3036 dt_concat_path_file(dbfilename_library, datadir, "library.db");
3037 else if(!strcmp(dbname, ":memory:"))
3038 g_strlcpy(dbfilename_library, dbname, sizeof(dbfilename_library));
3039 else if(dbname[0] != '/')
3040 dt_concat_path_file(dbfilename_library, datadir, dbname);
3041 else
3042 g_strlcpy(dbfilename_library, dbname, sizeof(dbfilename_library));
3043 }
3044 else
3045 {
3046 g_strlcpy(dbfilename_library, alternative, sizeof(dbfilename_library));
3047
3048 GFile *galternative = g_file_new_for_path(alternative);
3049 dbname = g_file_get_basename(galternative);
3050 g_object_unref(galternative);
3051 }
3052
3053 /* we also need a 2nd db with permanent data like presets, styles and tags */
3054 char dbfilename_data[PATH_MAX] = { 0 };
3055 if(load_data)
3056 dt_concat_path_file(dbfilename_data, datadir, "data.db");
3057 else
3058 snprintf(dbfilename_data, sizeof(dbfilename_data), ":memory:");
3059
3060 /* create database */
3061 dt_database_t *db = (dt_database_t *)g_malloc0(sizeof(dt_database_t));
3062 db->dbfilename_data = g_strdup(dbfilename_data);
3063 db->dbfilename_library = g_strdup(dbfilename_library);
3064
3065 dt_atomic_set_int(&_trxid, 0);
3066 dt_atomic_set_int(&_trx_batch_level, 0);
3067 g_atomic_pointer_set(&_trx_batch_owner, NULL);
3068
3069 /* make sure the folder exists. this might not be the case for new databases */
3070 /* also check if a database backup is needed */
3071 if(g_strcmp0(dbfilename_data, ":memory:"))
3072 {
3073 char *data_path = g_path_get_dirname(dbfilename_data);
3074 g_mkdir_with_parents(data_path, 0750);
3075 dt_free(data_path);
3076 dt_database_backup(dbfilename_data);
3077 }
3078 if(g_strcmp0(dbfilename_library, ":memory:"))
3079 {
3080 char *library_path = g_path_get_dirname(dbfilename_library);
3081 g_mkdir_with_parents(library_path, 0750);
3082 dt_free(library_path);
3083 dt_database_backup(dbfilename_library);
3084 }
3085
3086 dt_print(DT_DEBUG_SQL, "[init sql] library: %s, data: %s\n", dbfilename_library, dbfilename_data);
3087
3088 /* having more than one instance of darktable using the same database is a bad idea */
3089 /* try to get locks for the databases */
3090 db->lock_acquired = _lock_databases(db);
3091
3092 if(!db->lock_acquired)
3093 {
3094 fprintf(stderr, "[init] database is locked, probably another process is already using it\n");
3095 dt_free(dbname);
3096 return db;
3097 }
3098
3099 /* opening / creating database */
3100 if(sqlite3_open(db->dbfilename_library, &db->handle))
3101 {
3102 fprintf(stderr, "[init] could not find database ");
3103 if(dbname)
3104 fprintf(stderr, "`%s'!\n", dbname);
3105 else
3106 fprintf(stderr, "\n");
3107 fprintf(stderr, "[init] maybe your %s/anselrc is corrupt?\n", datadir);
3108 dt_loc_get_datadir(dbfilename_library, sizeof(dbfilename_library));
3109 fprintf(stderr, "[init] try `cp %s/anselrc %s/anselrc'\n", dbfilename_library, datadir);
3110 sqlite3_close(db->handle);
3111 dt_free(dbname);
3112 dt_free(db->lockfile_data);
3113 dt_free(db->dbfilename_data);
3114 dt_free(db->lockfile_library);
3115 dt_free(db->dbfilename_library);
3116 dt_free(db);
3117 return NULL;
3118 }
3119
3120 dt_print(DT_DEBUG_SQL, "[sql] Opened database: '%s'\n", db->dbfilename_library);
3121
3122 sqlite3_stmt *stmt;
3123 int rc;
3124
3125 if(sqlite3_db_readonly(db->handle, "main") != 0)
3126 {
3127 fprintf(stderr, "%s database is read only. Abort...\n", db->dbfilename_library);
3128
3129 GtkWidget *dialog;
3130 GtkDialogFlags dflags = GTK_DIALOG_MODAL | GTK_DIALOG_DESTROY_WITH_PARENT;
3131
3132 dialog = gtk_dialog_new_with_buttons(_("Ansel - Database is read-only"),
3133 NULL, dflags,
3134 _("Close Ansel"), GTK_RESPONSE_CLOSE, NULL);
3135 gtk_dialog_set_default_response(GTK_DIALOG(dialog), GTK_RESPONSE_CLOSE);
3136 char *label_options
3137 = g_strdup_printf(_("<span weight='bold'>Ansel library database is read-only</span>\n\n"
3138 "This happens if you don't have permissions to write on the filesystem\n"
3139 "or if you have restored a write-protected backup snapshot.\n\n"
3140 "Please change the filesystem access permissions for:\n\n"
3141 "\t<span style='italic'>%s</span>"),
3142 db->dbfilename_library);
3143
3144 GtkWidget *content_area = gtk_dialog_get_content_area(GTK_DIALOG (dialog));
3145 GtkWidget *label = gtk_label_new(NULL);
3146 gtk_label_set_markup(GTK_LABEL(label), label_options);
3147 gtk_container_add(GTK_CONTAINER (content_area), label);
3148 gtk_widget_show_all(content_area);
3149 gtk_dialog_run(GTK_DIALOG(dialog));
3150 gtk_widget_destroy(dialog);
3151 dt_database_destroy(db);
3152 dt_free(dbname);
3153 dt_free(label_options);
3154 db = NULL;
3155 return NULL;
3156 }
3157
3158 dt_print(DT_DEBUG_SQL, "[sql] Checked that we can write in database: '%s'\n", db->dbfilename_library);
3159
3160 /* attach a memory database to db connection for use with temporary tables
3161 used during instance life time, which is discarded on exit.
3162 */
3163 sqlite3_exec(db->handle, "attach database ':memory:' as memory", NULL, NULL, NULL);
3164
3165 // attach the data database which contains presets, styles, tags and similar things not tied to single images
3166 gboolean have_data_db = load_data && g_file_test(dbfilename_data, G_FILE_TEST_EXISTS);
3167 rc = sqlite3_prepare_v2(db->handle, "ATTACH DATABASE ?1 AS data", -1, &stmt, NULL);
3168 sqlite3_bind_text(stmt, 1, dbfilename_data, -1, SQLITE_TRANSIENT);
3169 if(rc != SQLITE_OK || sqlite3_step(stmt) != SQLITE_DONE)
3170 {
3171 sqlite3_finalize(stmt);
3172 fprintf(stderr, "[init] database `%s' couldn't be opened. aborting\n", dbfilename_data);
3173 dt_database_destroy(db);
3174 db = NULL;
3175 goto error;
3176 }
3177 sqlite3_finalize(stmt);
3178
3179 dt_print(DT_DEBUG_SQL, "[sql] Opened database: '%s'\n", dbfilename_data);
3180
3181 // some sqlite3 config
3182 sqlite3_exec(db->handle, "PRAGMA synchronous = OFF", NULL, NULL, NULL);
3183 sqlite3_exec(db->handle, "PRAGMA journal_mode = MEMORY", NULL, NULL, NULL);
3184 sqlite3_exec(db->handle, "PRAGMA page_size = 32768", NULL, NULL, NULL);
3185
3186 // WARNING: the foreign_keys pragma must not be used, the integrity of the
3187 // database rely on it.
3188 sqlite3_exec(db->handle, "PRAGMA foreign_keys = ON", NULL, NULL, NULL);
3189
3190 dt_print(DT_DEBUG_SQL, "[sql] Configured database\n");
3191
3192 /* now that we got functional databases that are locked for us we can make sure that the schema is set up */
3193
3194 // first we update the data database to the latest version so that we can potentially move data from the library
3195 // over when updating that one
3196 if(!have_data_db)
3197 {
3198 _create_data_schema(db); // a brand new db it seems
3199 }
3200 else
3201 {
3202 gchar* data_status = _get_pragma_string_val(db->handle, "data.quick_check");
3203 rc = sqlite3_prepare_v2(db->handle, "select value from data.db_info where key = 'version'", -1, &stmt, NULL);
3204 if(!g_strcmp0(data_status, "ok") && rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3205 {
3206 dt_free(data_status); // status is OK and we don't need to care :)
3207
3208 // compare the version of the db with what is current for this executable
3209 const int db_version = sqlite3_column_int(stmt, 0);
3210 sqlite3_finalize(stmt);
3211 if(db_version < CURRENT_DATABASE_VERSION_DATA)
3212 {
3213 ask_for_upgrade(dbfilename_data, has_gui);
3214
3215 // older: upgrade
3216 if(!_upgrade_data_schema(db, db_version))
3217 {
3218 // we couldn't upgrade the db for some reason. bail out.
3219 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version %d to %d. aborting\n",
3220 dbfilename_data, db_version, CURRENT_DATABASE_VERSION_DATA);
3221 dt_database_destroy(db);
3222 db = NULL;
3223 goto error;
3224 }
3225 }
3226 else if(db_version > CURRENT_DATABASE_VERSION_DATA)
3227 {
3228 // newer: bail out
3229 fprintf(stderr, "[init] database version of `%s' is too new for this build of darktable. aborting\n",
3230 dbfilename_data);
3231 dt_database_destroy(db);
3232 db = NULL;
3233 goto error;
3234 }
3235 // else: the current version, do nothing
3236 }
3237 else
3238 {
3239 // oh, bad situation. the database is corrupt and can't be read!
3240 // we inform the user here and let him decide what to do: exit or delete and try again.
3241
3242 gchar* quick_check_text = NULL;
3243 if(g_strcmp0(data_status, "ok")) // data_status is not ok
3244 {
3245 quick_check_text = g_strdup_printf(_("quick_check said:\n"
3246 "%s \n"), data_status);
3247 }
3248 else
3249 {
3250 quick_check_text = g_strdup(""); // a trick;
3251 }
3252
3253 gchar *data_snap = dt_database_get_most_recent_snap(dbfilename_data);
3254
3255 GtkWidget *dialog;
3256 GtkDialogFlags dflags = GTK_DIALOG_MODAL | GTK_DIALOG_DESTROY_WITH_PARENT;
3257
3258 const char* label_options = NULL;
3259
3260 if(data_snap)
3261 {
3262 dialog = gtk_dialog_new_with_buttons(_("ansel - error opening database"),
3263 NULL,
3264 dflags,
3265 _("close Ansel"),
3266 GTK_RESPONSE_CLOSE,
3267 _("attempt restore"),
3268 GTK_RESPONSE_ACCEPT,
3269 _("delete database"),
3270 GTK_RESPONSE_REJECT,
3271 NULL);
3272 gtk_dialog_set_default_response(GTK_DIALOG(dialog), GTK_RESPONSE_ACCEPT);
3273 label_options = _("do you want to close Ansel now to manually restore\n"
3274 "the database from a backup, attempt an automatic restore\n"
3275 "from the most recent snapshot or delete the corrupted database\n"
3276 "and start with a new one?");
3277 }
3278 else
3279 {
3280 dialog = gtk_dialog_new_with_buttons(_("ansel - error opening database"),
3281 NULL,
3282 dflags,
3283 _("close Ansel"),
3284 GTK_RESPONSE_CLOSE,
3285 _("delete database"),
3286 GTK_RESPONSE_REJECT,
3287 NULL);
3288 gtk_dialog_set_default_response(GTK_DIALOG(dialog), GTK_RESPONSE_CLOSE);
3289 label_options = _("do you want to close Ansel now to manually restore\n"
3290 "the database from a backup or delete the corrupted database\n"
3291 "and start with a new one?");
3292 }
3293
3294
3295
3296 char *label_text = g_markup_printf_escaped(_("an error has occurred while trying to open the database from\n"
3297 "\n"
3298 "<span style='italic'>%s</span>\n"
3299 "\n"
3300 "it seems that the database is corrupted.\n"
3301 "%s"
3302 "%s"),
3303 dbfilename_data, quick_check_text, label_options);
3304
3305 dt_free(quick_check_text);
3306 dt_free(data_status);
3307
3308 GtkWidget *content_area = gtk_dialog_get_content_area(GTK_DIALOG (dialog));
3309 GtkWidget *label = gtk_label_new(NULL);
3310 gtk_label_set_markup(GTK_LABEL(label), label_text);
3311 dt_free(label_text);
3312 gtk_container_add(GTK_CONTAINER (content_area), label);
3313
3314 gtk_widget_show_all(content_area);
3315
3316 const int resp = gtk_dialog_run(GTK_DIALOG(dialog));
3317
3318 gtk_widget_destroy(dialog);
3319
3320 dt_database_destroy(db);
3321 db = NULL;
3322
3323 if(resp != GTK_RESPONSE_ACCEPT && resp != GTK_RESPONSE_REJECT)
3324 {
3325 fprintf(stderr, "[init] database `%s' is corrupt and can't be opened! either replace it from a backup or "
3326 "delete the file so that darktable can create a new one the next time. aborting\n", dbfilename_data);
3327 dt_free(data_snap);
3328 goto error;
3329 }
3330
3331 //here were sure that response is either accept (restore from snap) or reject (just delete the damaged db)
3332
3333 fprintf(stderr, "[init] deleting `%s' on user request", dbfilename_data);
3334
3335 if(g_unlink(dbfilename_data) == 0)
3336 fprintf(stderr, " ... ok\n");
3337 else
3338 fprintf(stderr, " ... failed\n");
3339
3340 if(resp == GTK_RESPONSE_ACCEPT && data_snap)
3341 {
3342 fprintf(stderr, "[init] restoring `%s' from `%s'...", dbfilename_data, data_snap);
3343 GError *gerror = NULL;
3344 if(!g_file_test(dbfilename_data, G_FILE_TEST_EXISTS))
3345 {
3346 GFile *src = g_file_new_for_path(data_snap);
3347 GFile *dest = g_file_new_for_path(dbfilename_data);
3348 gboolean copy_status = TRUE;
3349 if(g_file_test(data_snap, G_FILE_TEST_EXISTS))
3350 {
3351 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
3352 if(copy_status) copy_status = g_chmod(dbfilename_data, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == 0;
3353 }
3354 else
3355 {
3356 // there is nothing to restore, create an empty file
3357 const int fd = g_open(dbfilename_data, O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
3358 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
3359 }
3360 if(copy_status)
3361 fprintf(stderr, " success!\n");
3362 else
3363 fprintf(stderr, " failed!\n");
3364
3365 g_object_unref(src);
3366 g_object_unref(dest);
3367 }
3368 }
3369 dt_free(data_snap);
3370 dt_free(dbname);
3371 goto start;
3372 }
3373 }
3374
3375 dt_print(DT_DEBUG_SQL, "[sql] Checked database schema and migrated if needed\n");
3376
3377 gchar* libdb_status = _get_pragma_string_val(db->handle, "main.quick_check");
3378 // next we are looking at the library database
3379 // does the db contain the new 'db_info' table?
3380 rc = sqlite3_prepare_v2(db->handle, "select value from main.db_info where key = 'version'", -1, &stmt, NULL);
3381 if(!g_strcmp0(libdb_status, "ok") && rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3382 {
3383 dt_free(libdb_status);//it's ok :)
3384
3385 // compare the version of the db with what is current for this executable
3386 const int db_version = sqlite3_column_int(stmt, 0);
3387 sqlite3_finalize(stmt);
3388
3389 if(db_version < CURRENT_DATABASE_VERSION_LIBRARY)
3390 {
3391 ask_for_upgrade(dbfilename_library, has_gui);
3392
3393 // older: upgrade
3394 if(!_upgrade_library_schema(db, db_version))
3395 {
3396 // we couldn't upgrade the db for some reason. bail out.
3397 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version %d to %d. aborting\n", dbname,
3398 db_version, CURRENT_DATABASE_VERSION_LIBRARY);
3399 dt_database_destroy(db);
3400 db = NULL;
3401 goto error;
3402 }
3403 }
3404 else if(db_version > CURRENT_DATABASE_VERSION_LIBRARY)
3405 {
3406 // newer: bail out. it's better than what we did before: delete everything
3407 fprintf(stderr, "[init] database version of `%s' is too new for this build of darktable. aborting\n",
3408 dbname);
3409 dt_database_destroy(db);
3410 db = NULL;
3411 goto error;
3412 }
3413 // else: the current version, do nothing
3414 }
3415 else if(g_strcmp0(libdb_status, "ok") || rc == SQLITE_CORRUPT || rc == SQLITE_NOTADB)
3416 {
3417 // oh, bad situation. the database is corrupt and can't be read!
3418 // we inform the user here and let him decide what to do: exit or delete and try again.
3419
3420 gchar* quick_check_text = NULL;
3421 if(g_strcmp0(libdb_status, "ok")) // data_status is not ok
3422 {
3423 quick_check_text = g_strdup_printf(_("quick_check said:\n"
3424 "%s \n"), libdb_status);
3425 }
3426 else
3427 {
3428 quick_check_text = g_strdup(""); // a trick;
3429 }
3430
3431 gchar *data_snap = dt_database_get_most_recent_snap(dbfilename_library);
3432
3433 GtkWidget *dialog;
3434 GtkDialogFlags dflags = GTK_DIALOG_MODAL | GTK_DIALOG_DESTROY_WITH_PARENT;
3435
3436 const char* label_options = NULL;
3437
3438 if(data_snap)
3439 {
3440 dialog = gtk_dialog_new_with_buttons(_("ansel - error opening database"),
3441 NULL,
3442 dflags,
3443 _("close Ansel"),
3444 GTK_RESPONSE_CLOSE,
3445 _("attempt restore"),
3446 GTK_RESPONSE_ACCEPT,
3447 _("delete database"),
3448 GTK_RESPONSE_REJECT,
3449 NULL);
3450 gtk_dialog_set_default_response(GTK_DIALOG(dialog), GTK_RESPONSE_ACCEPT);
3451 label_options = _("do you want to close Ansel now to manually restore\n"
3452 "the database from a backup, attempt an automatic restore\n"
3453 "from the most recent snapshot or delete the corrupted database\n"
3454 "and start with a new one?");
3455 }
3456 else
3457 {
3458 dialog = gtk_dialog_new_with_buttons(_("ansel - error opening database"),
3459 NULL,
3460 dflags,
3461 _("close Ansel"),
3462 GTK_RESPONSE_CLOSE,
3463 _("delete database"),
3464 GTK_RESPONSE_REJECT,
3465 NULL);
3466 gtk_dialog_set_default_response(GTK_DIALOG(dialog), GTK_RESPONSE_CLOSE);
3467 label_options = _("do you want to close Ansel now to manually restore\n"
3468 "the database from a backup or delete the corrupted database\n"
3469 "and start with a new one?");
3470 }
3471
3472 char *label_text = g_markup_printf_escaped(_("an error has occurred while trying to open the database from\n"
3473 "\n"
3474 "<span style='italic'>%s</span>\n"
3475 "\n"
3476 "it seems that the database is corrupted.\n"
3477 "%s"
3478 "%s"),
3479 dbfilename_data, quick_check_text, label_options);
3480
3481 dt_free(quick_check_text);
3482 dt_free(libdb_status);
3483
3484 GtkWidget *content_area = gtk_dialog_get_content_area(GTK_DIALOG (dialog));
3485 GtkWidget *label = gtk_label_new(NULL);
3486 gtk_label_set_markup(GTK_LABEL(label), label_text);
3487 dt_free(label_text);
3488 gtk_container_add(GTK_CONTAINER (content_area), label);
3489
3490 gtk_widget_show_all(content_area);
3491
3492 const int resp = gtk_dialog_run(GTK_DIALOG(dialog));
3493
3494 gtk_widget_destroy(dialog);
3495
3496 dt_database_destroy(db);
3497 db = NULL;
3498
3499 if(resp != GTK_RESPONSE_ACCEPT && resp != GTK_RESPONSE_REJECT)
3500 {
3501 fprintf(stderr, "[init] database `%s' is corrupt and can't be opened! either replace it from a backup or "
3502 "delete the file so that darktable can create a new one the next time. aborting\n", dbfilename_library);
3503 dt_free(data_snap);
3504 goto error;
3505 }
3506
3507 //here were sure that response is either accept (restore from snap) or reject (just delete the damaged db)
3508
3509 fprintf(stderr, "[init] deleting `%s' on user request", dbfilename_library);
3510
3511 if(g_unlink(dbfilename_library) == 0)
3512 fprintf(stderr, " ... ok\n");
3513 else
3514 fprintf(stderr, " ... failed\n");
3515
3516 if(resp == GTK_RESPONSE_ACCEPT && data_snap)
3517 {
3518 fprintf(stderr, "[init] restoring `%s' from `%s'...", dbfilename_library, data_snap);
3519 GError *gerror = NULL;
3520 if(!g_file_test(dbfilename_library, G_FILE_TEST_EXISTS))
3521 {
3522 GFile *src = g_file_new_for_path(data_snap);
3523 GFile *dest = g_file_new_for_path(dbfilename_library);
3524 gboolean copy_status = TRUE;
3525 if(g_file_test(data_snap, G_FILE_TEST_EXISTS))
3526 {
3527 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
3528 if(copy_status) copy_status = g_chmod(dbfilename_library, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == 0;
3529 }
3530 else
3531 {
3532 // there is nothing to restore, create an empty file to prevent further backup attempts
3533 const int fd = g_open(dbfilename_library, O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
3534 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
3535 }
3536 if(copy_status)
3537 fprintf(stderr, " success!\n");
3538 else
3539 fprintf(stderr, " failed!\n");
3540 g_object_unref(src);
3541 g_object_unref(dest);
3542 }
3543 }
3544 dt_free(data_snap);
3545 dt_free(dbname);
3546 goto start;
3547 }
3548 else
3549 {
3550 // does it contain the legacy 'settings' table?
3551 sqlite3_finalize(stmt);
3552 rc = sqlite3_prepare_v2(db->handle, "select settings from main.settings", -1, &stmt, NULL);
3553 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3554 {
3555 // the old blob had the version as an int in the first place
3556 const void *set = sqlite3_column_blob(stmt, 0);
3557 const int db_version = *(int *)set;
3558 sqlite3_finalize(stmt);
3559 if(!_migrate_schema(db, db_version)) // bring the legacy layout to the first one known to our upgrade
3560 // path ...
3561 {
3562 // we couldn't migrate the db for some reason. bail out.
3563 fprintf(stderr, "[init] database `%s' couldn't be migrated from the legacy version %d. aborting\n",
3564 dbname, db_version);
3565 dt_database_destroy(db);
3566 db = NULL;
3567 goto error;
3568 }
3569 if(!_upgrade_library_schema(db, 1)) // ... and upgrade it
3570 {
3571 // we couldn't upgrade the db for some reason. bail out.
3572 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version 1 to %d. aborting\n", dbname,
3573 CURRENT_DATABASE_VERSION_LIBRARY);
3574 dt_database_destroy(db);
3575 db = NULL;
3576 goto error;
3577 }
3578 }
3579 else
3580 {
3581 sqlite3_finalize(stmt);
3582 _create_library_schema(db); // a brand new db it seems
3583 }
3584 }
3585
3586 dt_print(DT_DEBUG_SQL, "[sql] Checked db_info\n");
3587
3588 // create the in-memory tables
3589 _create_memory_schema(db);
3590
3591 dt_print(DT_DEBUG_SQL, "[sql] Created in-memory DB\n");
3592
3593 // create a table legacy_presets with all the presets from pre-auto-apply-cleanup darktable.
3594 dt_legacy_presets_create(db);
3595
3596 dt_print(DT_DEBUG_SQL, "[sql] Loaded legacy presets\n");
3597
3598 // drop table settings -- we don't want old versions of dt to drop our tables
3599 sqlite3_exec(db->handle, "drop table main.settings", NULL, NULL, NULL);
3600
3601 // take care of potential bad data in the db.
3602 _sanitize_db(db);
3603
3604#ifdef HAVE_ICU
3605 // check if sqlite is already icu enabled
3606 // if not enabled expected error: no such function:icu_load_collation
3607 rc = sqlite3_prepare_v2(db->handle,
3608 "SELECT icu_load_collation('en_US', 'english')",
3609 -1, &stmt, NULL);
3610 sqlite3_finalize(stmt);
3611
3612 if(rc != SQLITE_OK)
3613 {
3614 rc = sqlite3IcuInit(db->handle);
3615 if(rc != SQLITE_OK)
3616 fprintf(stderr, "[sqlite] init icu extension error %d\n", rc);
3617 }
3618#endif
3619
3620error:
3621 dt_free(dbname);
3622
3623 return db;
3624}
3625
3626void dt_database_destroy(const dt_database_t *db)
3627{
3628 sqlite3_close(db->handle);
3629 if (db->lockfile_data)
3630 {
3631 g_unlink(db->lockfile_data);
3632 dt_free(db->lockfile_data);
3633 }
3634 if (db->lockfile_library)
3635 {
3636 g_unlink(db->lockfile_library);
3637 dt_free(db->lockfile_library);
3638 }
3639 dt_free(db->dbfilename_data);
3640 dt_free(db->dbfilename_library);
3641 dt_free(db);
3642
3643 sqlite3_shutdown();
3644}
3645
3646sqlite3 *dt_database_get(const dt_database_t *db)
3647{
3648 return db ? db->handle : NULL;
3649}
3650
3651const gchar *dt_database_get_path(const struct dt_database_t *db)
3652{
3653 return db->dbfilename_library;
3654}
3655
3656static void _database_migrate_to_xdg_structure()
3657{
3658 gchar dbfilename[PATH_MAX] = { 0 };
3659 gchar *conf_db = dt_conf_get_string("database");
3660
3661 gchar datadir[PATH_MAX] = { 0 };
3662 dt_loc_get_datadir(datadir, sizeof(datadir));
3663
3664 if(conf_db && conf_db[0] != '/')
3665 {
3666 const char *homedir = getenv("HOME");
3667 snprintf(dbfilename, sizeof(dbfilename), "%s/%s", homedir, conf_db);
3668 if(g_file_test(dbfilename, G_FILE_TEST_EXISTS))
3669 {
3670 char destdbname[PATH_MAX] = { 0 };
3671 dt_concat_path_file(destdbname, datadir, "library.db");
3672 if(!g_file_test(destdbname, G_FILE_TEST_EXISTS))
3673 {
3674 fprintf(stderr, "[init] moving database into new XDG directory structure\n");
3675 rename(dbfilename, destdbname);
3676 dt_conf_set_string("database", "library.db");
3677 }
3678 }
3679 }
3680
3681 dt_free(conf_db);
3682}
3683
3684/* delete old mipmaps files */
3685static void _database_delete_mipmaps_files()
3686{
3687 /* This migration is intended to be run only from 0.9.x to new cache in 1.0 */
3688
3689 // Directory
3690 char cachedir[PATH_MAX] = { 0 }, mipmapfilename[PATH_MAX] = { 0 };
3691 dt_loc_get_user_cache_dir(cachedir, sizeof(cachedir));
3692 dt_concat_path_file(mipmapfilename, cachedir, "mipmaps");
3693
3694 if(g_access(mipmapfilename, F_OK) != -1)
3695 {
3696 fprintf(stderr, "[mipmap_cache] dropping old version file: %s\n", mipmapfilename);
3697 g_unlink(mipmapfilename);
3698 dt_concat_path_file(mipmapfilename, cachedir, "mipmaps.fallback");
3699
3700 if(g_access(mipmapfilename, F_OK) != -1) g_unlink(mipmapfilename);
3701 }
3702}
3703
3704gboolean dt_database_get_lock_acquired(const dt_database_t *db)
3705{
3706 return db->lock_acquired;
3707}
3708
3709void dt_database_cleanup_busy_statements(const struct dt_database_t *db)
3710{
3711 sqlite3_stmt *stmt = NULL;
3712 while( (stmt = sqlite3_next_stmt(db->handle, NULL)) != NULL)
3713 {
3714 const char* sql = sqlite3_sql(stmt);
3715 if(sqlite3_stmt_busy(stmt))
3716 {
3717 dt_print(DT_DEBUG_SQL, "[db busy stmt] non-finalized nor stepped through statement: '%s'\n",sql);
3718 sqlite3_reset(stmt);
3719 }
3720 else {
3721 dt_print(DT_DEBUG_SQL, "[db busy stmt] non-finalized statement: '%s'\n",sql);
3722 }
3723 sqlite3_finalize(stmt);
3724 }
3725}
3726
3727#define ERRCHECK {if (err!=NULL) {dt_print(DT_DEBUG_SQL, "[db maintenance] maintenance error: '%s'\n",err); sqlite3_free(err); err=NULL;}}
3728void dt_database_perform_maintenance(const struct dt_database_t *db)
3729{
3730 char* err = NULL;
3731
3732 const int main_pre_free_count = _get_pragma_int_val(db->handle, "main.freelist_count");
3733 const int main_page_size = _get_pragma_int_val(db->handle, "main.page_size");
3734 const int data_pre_free_count = _get_pragma_int_val(db->handle, "data.freelist_count");
3735 const int data_page_size = _get_pragma_int_val(db->handle, "data.page_size");
3736
3737 const guint64 calc_pre_size = (main_pre_free_count*main_page_size) + (data_pre_free_count*data_page_size);
3738
3739 if(calc_pre_size == 0)
3740 {
3741 dt_print(DT_DEBUG_SQL, "[db maintenance] maintenance deemed unnecesary, performing only analyze.\n");
3742 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE data", NULL, NULL, &err);
3743 ERRCHECK
3744 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE main", NULL, NULL, &err);
3745 ERRCHECK
3746 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE", NULL, NULL, &err);
3747 ERRCHECK
3748 return;
3749 }
3750
3751 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM data", NULL, NULL, &err);
3752 ERRCHECK
3753 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM main", NULL, NULL, &err);
3754 ERRCHECK
3755 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE data", NULL, NULL, &err);
3756 ERRCHECK
3757 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE main", NULL, NULL, &err);
3758 ERRCHECK
3759
3760 // for some reason this is needed in some cases
3761 // in case above performed vacuum+analyze properly, this is noop.
3762 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM", NULL, NULL, &err);
3763 ERRCHECK
3764 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE", NULL, NULL, &err);
3765 ERRCHECK
3766
3767 const int main_post_free_count = _get_pragma_int_val(db->handle, "main.freelist_count");
3768 const int data_post_free_count = _get_pragma_int_val(db->handle, "data.freelist_count");
3769
3770 const guint64 calc_post_size = (main_post_free_count*main_page_size) + (data_post_free_count*data_page_size);
3771 const gint64 bytes_freed = calc_pre_size - calc_post_size;
3772
3773 dt_print(DT_DEBUG_SQL, "[db maintenance] maintenance done, %" G_GINT64_FORMAT " bytes freed.\n", bytes_freed);
3774
3775 if(calc_post_size >= calc_pre_size)
3776 {
3777 dt_print(DT_DEBUG_SQL, "[db maintenance] maintenance problem. if no errors logged, it should work fine next time.\n");
3778 }
3779}
3780#undef ERRCHECK
3781
3782gboolean _ask_for_maintenance(const gboolean has_gui, const gboolean closing_time, const guint64 size)
3783{
3784 if(!has_gui)
3785 {
3786 return FALSE;
3787 }
3788
3789 char *later_info = NULL;
3790 char *size_info = g_format_size(size);
3791 const char *config = dt_conf_get_string_const("database/maintenance_check");
3792 if((closing_time && (!g_strcmp0(config, "on both"))) || !g_strcmp0(config, "on startup"))
3793 {
3794 later_info = _("click later to be asked on next startup");
3795 }
3796 else if (!closing_time && (!g_strcmp0(config, "on both")))
3797 {
3798 later_info = _("click later to be asked when closing Ansel");
3799 }
3800 else if (!g_strcmp0(config, "on close"))
3801 {
3802 later_info = _("click later to be asked next time when closing Ansel");
3803 }
3804
3805 char *label_text = g_markup_printf_escaped(_("the database could use some maintenance\n"
3806 "\n"
3807 "there's <span style='italic'>%s</span> to be freed"
3808 "\n\n"
3809 "do you want to proceed now?\n\n"
3810 "%s\n"
3811 "you can always change maintenance preferences in core options"),
3812 size_info, later_info);
3813
3814 const gboolean shall_perform_maintenance =
3815 dt_gui_show_standalone_yes_no_dialog(_("ansel - schema maintenance"), label_text,
3816 _("later"), _("yes"));
3817
3818 dt_free(label_text);
3819 dt_free(size_info);
3820
3821 return shall_perform_maintenance;
3822}
3823
3824static inline gboolean _is_mem_db(const struct dt_database_t *db)
3825{
3826 return !g_strcmp0(db->dbfilename_data, ":memory:") || !g_strcmp0(db->dbfilename_library, ":memory:");
3827}
3828
3829gboolean dt_database_maybe_maintenance(const struct dt_database_t *db, const gboolean has_gui, const gboolean closing_time)
3830{
3831 if(_is_mem_db(db))
3832 return FALSE;
3833
3834 const char *config = dt_conf_get_string_const("database/maintenance_check");
3835
3836 if(!g_strcmp0(config, "never"))
3837 {
3838 // early bail out on "never"
3839 dt_print(DT_DEBUG_SQL, "[db maintenance] please consider enabling database maintenance.\n");
3840 return FALSE;
3841 }
3842
3843 gboolean check_for_maintenance = FALSE;
3844 const gboolean force_maintenance = g_str_has_suffix (config, "(don't ask)");
3845
3846 if(config)
3847 {
3848 if((strstr(config, "on both")) // should cover "(don't ask) suffix
3849 || (closing_time && (strstr(config, "on close")))
3850 || (!closing_time && (strstr(config, "on startup"))))
3851 {
3852 // we have "on both/on close/on startup" setting, so - checking!
3853 dt_print(DT_DEBUG_SQL, "[db maintenance] checking for maintenance, due to rule: '%s'.\n", config);
3854 check_for_maintenance = TRUE;
3855 }
3856 // if the config was "never", check_for_vacuum is false.
3857 }
3858
3859 if(!check_for_maintenance)
3860 {
3861 return FALSE;
3862 }
3863
3864 // checking free pages
3865 const int main_free_count = _get_pragma_int_val(db->handle, "main.freelist_count");
3866 const int main_page_count = _get_pragma_int_val(db->handle, "main.page_count");
3867 const int main_page_size = _get_pragma_int_val(db->handle, "main.page_size");
3868
3869 const int data_free_count = _get_pragma_int_val(db->handle, "data.freelist_count");
3870 const int data_page_count = _get_pragma_int_val(db->handle, "data.page_count");
3871 const int data_page_size = _get_pragma_int_val(db->handle, "data.page_size");
3872
3873 dt_print(DT_DEBUG_SQL,
3874 "[db maintenance] main: [%d/%d pages], data: [%d/%d pages].\n",
3875 main_free_count, main_page_count, data_free_count, data_page_count);
3876
3877 if(main_page_count <= 0 || data_page_count <= 0)
3878 {
3879 //something's wrong with PRAGMA page_size returns. early bail.
3880 dt_print(DT_DEBUG_SQL,
3881 "[db maintenance] page_count <= 0 : main.page_count: %d, data.page_count: %d \n",
3882 main_page_count, data_page_count);
3883 return FALSE;
3884 }
3885
3886 // we don't need fine-grained percentages, so let's do ints
3887 const int main_free_percentage = (main_free_count * 100 ) / main_page_count;
3888 const int data_free_percentage = (data_free_count * 100 ) / data_page_count;
3889
3890 const int freepage_ratio = dt_conf_get_int("database/maintenance_freepage_ratio");
3891
3892 if((main_free_percentage >= freepage_ratio)
3893 || (data_free_percentage >= freepage_ratio))
3894 {
3895 const guint64 calc_size = (main_free_count*main_page_size) + (data_free_count*data_page_size);
3896 dt_print(DT_DEBUG_SQL, "[db maintenance] maintenance suggested, %" G_GUINT64_FORMAT " bytes to free.\n", calc_size);
3897
3898 if(force_maintenance || _ask_for_maintenance(has_gui, closing_time, calc_size))
3899 {
3900 return TRUE;
3901 }
3902 }
3903 return FALSE;
3904}
3905
3906void dt_database_optimize(const struct dt_database_t *db)
3907{
3908 if(_is_mem_db(db))
3909 return;
3910 // optimize should in most cases be no-op and have no noticeable downsides
3911 // this should be ran on every exit
3912 // see: https://www.sqlite.org/pragma.html#pragma_optimize
3913 DT_DEBUG_SQLITE3_EXEC(db->handle, "PRAGMA optimize", NULL, NULL, NULL);
3914}
3915
3916static void _print_backup_progress(int remaining, int total)
3917{
3918 // TODO if we have closing splashpage - this can be used to advance progressbar :)
3919 dt_print(DT_DEBUG_SQL, "[db backup] %d out of %d done\n", total - remaining, total);
3920}
3921
3922static int _backup_db(
3923 sqlite3 *src_db, /* Database handle to back up */
3924 const char *src_db_name, /* Database name to back up */
3925 const char *dest_filename, /* Name of file to back up to */
3926 void(*xProgress)(int, int) /* Progress function to invoke */
3927)
3928{
3929 sqlite3 *dest_db; /* Database connection opened on zFilename */
3930 sqlite3_backup *sb_dest; /* Backup handle used to copy data */
3931
3932 /* Open the database file identified by zFilename. */
3933 int rc = sqlite3_open(dest_filename, &dest_db);
3934
3935 if(rc == SQLITE_OK)
3936 {
3937 /* Open the sqlite3_backup object used to accomplish the transfer */
3938 sb_dest = sqlite3_backup_init(dest_db, "main", src_db, src_db_name);
3939 if(sb_dest)
3940 {
3941 dt_print(DT_DEBUG_SQL, "[db backup] %s to %s\n", src_db_name, dest_filename);
3942 gchar *pragma = g_strdup_printf("%s.page_count", src_db_name);
3943 const int spc = _get_pragma_int_val(src_db, pragma);
3944 dt_free(pragma);
3945 const int pc = MIN(spc, MAX(5,spc/100));
3946 do
3947 {
3948 rc = sqlite3_backup_step(sb_dest, pc);
3949 if(xProgress)
3950 xProgress(
3951 sqlite3_backup_remaining(sb_dest),
3952 sqlite3_backup_pagecount(sb_dest)
3953 );
3954 if( rc==SQLITE_OK || rc==SQLITE_BUSY || rc==SQLITE_LOCKED )
3955 {
3956 sqlite3_sleep(25);
3957 }
3958 }
3959 while( rc==SQLITE_OK || rc==SQLITE_BUSY || rc==SQLITE_LOCKED );
3960
3961 /* Release resources allocated by backup_init(). */
3962 (void)sqlite3_backup_finish(sb_dest);
3963 }
3964 rc = sqlite3_errcode(dest_db);
3965 }
3966 /* Close the database connection opened on database file zFilename
3967 ** and return the result of this function. */
3968 (void)sqlite3_close(dest_db);
3969 return rc;
3970}
3971
3972gboolean dt_database_snapshot(const struct dt_database_t *db)
3973{
3974 // backing up memory db is pointelss
3975 if(_is_mem_db(db))
3976 return FALSE;
3977 GDateTime *date_now = g_date_time_new_now_local();
3978 gchar *date_suffix = g_date_time_format(date_now, "%Y%m%d%H%M%S");
3979 g_date_time_unref(date_now);
3980
3981 const char *file_pattern = "%s-snp-%s";
3982 const char *temp_pattern = "%s-tmp-%s";
3983
3984 gchar *lib_backup_file = g_strdup_printf(file_pattern, db->dbfilename_library, date_suffix);
3985 gchar *lib_tmpbackup_file = g_strdup_printf(temp_pattern, db->dbfilename_library, date_suffix);
3986
3987 int rc = _backup_db(db->handle, "main", lib_tmpbackup_file, _print_backup_progress);
3988 if(!(rc==SQLITE_OK))
3989 {
3990 g_unlink(lib_tmpbackup_file);
3991 dt_free(lib_tmpbackup_file);
3992 dt_free(lib_backup_file);
3993 dt_free(date_suffix);
3994 return FALSE;
3995 }
3996 g_rename(lib_tmpbackup_file, lib_backup_file);
3997 g_chmod(lib_backup_file, S_IRUSR);
3998 dt_free(lib_tmpbackup_file);
3999 dt_free(lib_backup_file);
4000
4001 gchar *dat_backup_file = g_strdup_printf(file_pattern, db->dbfilename_data, date_suffix);
4002 gchar *dat_tmpbackup_file = g_strdup_printf(temp_pattern, db->dbfilename_data, date_suffix);
4003
4004 dt_free(date_suffix);
4005
4006 rc = _backup_db(db->handle, "data", dat_tmpbackup_file, _print_backup_progress);
4007 if(!(rc==SQLITE_OK))
4008 {
4009 g_unlink(dat_tmpbackup_file);
4010 dt_free(dat_tmpbackup_file);
4011 dt_free(dat_backup_file);
4012 return FALSE;
4013 }
4014 g_rename(dat_tmpbackup_file, dat_backup_file);
4015 g_chmod(dat_backup_file, S_IRUSR);
4016 dt_free(dat_tmpbackup_file);
4017 dt_free(dat_backup_file);
4018
4019 return TRUE;
4020}
4021
4022gboolean dt_database_maybe_snapshot(const struct dt_database_t *db)
4023{
4024 if(_is_mem_db(db))
4025 return FALSE;
4026
4027 const char *config = dt_conf_get_string_const("database/create_snapshot");
4028 if(!g_strcmp0(config, "never"))
4029 {
4030 // early bail out on "never"
4031 dt_print(DT_DEBUG_SQL, "[db backup] please consider enabling database snapshots.\n");
4032 return FALSE;
4033 }
4034 if(!g_strcmp0(config, "on close"))
4035 {
4036 // early bail out on "on close"
4037 dt_print(DT_DEBUG_SQL, "[db backup] performing unconditional snapshot.\n");
4038 return TRUE;
4039 }
4040
4041 GTimeSpan span_from_last_snap_required;
4042
4043 if(!g_strcmp0(config, "once a day"))
4044 {
4045 span_from_last_snap_required = G_TIME_SPAN_DAY;
4046 }
4047 else if(!g_strcmp0(config, "once a week"))
4048 {
4049 span_from_last_snap_required = G_TIME_SPAN_DAY * 7;
4050 }
4051 else if(!g_strcmp0(config, "once a month"))
4052 {
4053 //average month ;)
4054 span_from_last_snap_required = G_TIME_SPAN_DAY * 30;
4055 }
4056 else
4057 {
4058 // early bail out on "invalid value"
4059 dt_print(DT_DEBUG_SQL, "[db backup] invalid timespan requirement expecting never/on close/once a [day/week/month], got %s.\n", config);
4060 return TRUE;
4061 }
4062
4063 //we're in trouble zone - we have to determine when was the last snapshot done (including version upgrade snapshot) :/
4064 //this could be easy if we wrote date of last successful backup to config, but that's not really an option since
4065 //backup may done as last db operation, way after config file is closed. Plus we might be mixing dates of backups for
4066 //various library.db
4067
4068 dt_print(DT_DEBUG_SQL, "[db backup] checking snapshots existence.\n");
4069 GFile *library = g_file_parse_name(db->dbfilename_library);
4070 GFile *parent = g_file_get_parent(library);
4071
4072 if(IS_NULL_PTR(parent))
4073 {
4074 dt_print(DT_DEBUG_SQL, "[db backup] couldn't get library parent!.\n");
4075 g_object_unref(library);
4076 return FALSE;
4077 }
4078
4079 GError *error = NULL;
4080 GFileEnumerator *library_dir_files = g_file_enumerate_children(parent, G_FILE_ATTRIBUTE_STANDARD_NAME "," G_FILE_ATTRIBUTE_TIME_MODIFIED, G_FILE_QUERY_INFO_NONE, NULL, &error);
4081
4082 if(IS_NULL_PTR(library_dir_files))
4083 {
4084 dt_print(DT_DEBUG_SQL, "[db backup] couldn't enumerate library parent: %s.\n", error->message);
4085 g_object_unref(parent);
4086 g_object_unref(library);
4087 g_error_free(error);
4088 return FALSE;
4089 }
4090
4091 gchar *lib_basename = g_file_get_basename(library);
4092 g_object_unref(library);
4093
4094 gchar *lib_snap_format = g_strdup_printf("%s-snp-", lib_basename);
4095 gchar *lib_backup_format = g_strdup_printf("%s-pre-", lib_basename);
4096 dt_free(lib_basename);
4097
4098 GFileInfo *info = NULL;
4099 guint64 last_snap = 0;
4100
4101 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4102 {
4103 const char* fname = g_file_info_get_name(info);
4104 if(g_str_has_prefix(fname, lib_snap_format) || g_str_has_prefix(fname, lib_backup_format))
4105 {
4106 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4107 if(last_snap == 0)
4108 {
4109 last_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4110 g_object_unref(info);
4111 continue;
4112 }
4113 const guint64 try_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4114 if(try_snap > last_snap)
4115 {
4116 last_snap = try_snap;
4117 }
4118 }
4119 g_object_unref(info);
4120 }
4121 g_object_unref(parent);
4122 dt_free(lib_snap_format);
4123 dt_free(lib_backup_format);
4124
4125 if(error)
4126 {
4127 dt_print(DT_DEBUG_SQL, "[db backup] problem enumerating library parent: %s.\n", error->message);
4128 g_file_enumerator_close(library_dir_files, NULL, NULL);
4129 g_object_unref(library_dir_files);
4130 g_error_free(error);
4131 return FALSE;
4132 }
4133
4134 g_file_enumerator_close(library_dir_files, NULL, NULL);
4135 g_object_unref(library_dir_files);
4136
4137 GDateTime *date_now = g_date_time_new_now_local();
4138
4139 // Even if last_snap is 0 (didn't found any snaps) it produces proper date - unix epoch
4140 GDateTime *date_last_snap = g_date_time_new_from_unix_local(last_snap);
4141
4142 gchar *now_txt = g_date_time_format(date_now, "%Y%m%d%H%M%S");
4143 gchar *ls_txt = g_date_time_format(date_last_snap, "%Y%m%d%H%M%S");
4144 dt_print(DT_DEBUG_SQL, "[db backup] last snap: %s; curr date: %s.\n", ls_txt, now_txt);
4145 dt_free(now_txt);
4146 dt_free(ls_txt);
4147
4148 GTimeSpan span_from_last_snap = g_date_time_difference(date_now, date_last_snap);
4149
4150 g_date_time_unref(date_now);
4151 g_date_time_unref(date_last_snap);
4152
4153 return span_from_last_snap > span_from_last_snap_required;
4154}
4155
4156/* Parse integers in the form d (week days), dd (hours etc), ddd (ordinal days) or dddd (years) */
4157static gboolean _get_iso8601_int (const gchar *text, gsize length, gint *value)
4158{
4159 gsize i;
4160 guint v = 0;
4161
4162 if (length < 1 || length > 4)
4163 return FALSE;
4164
4165 for (i = 0; i < length; i++)
4166 {
4167 const gchar c = text[i];
4168 if (c < '0' || c > '9')
4169 return FALSE;
4170 v = v * 10 + (c - '0');
4171 }
4172
4173 *value = v;
4174 return TRUE;
4175}
4176
4177static gint _db_snap_sort(gconstpointer a, gconstpointer b, gpointer user_data)
4178{
4179 const gchar* e1 = (gchar *) a;
4180 const gchar* e2 = (gchar *) b;
4181
4182 //we assume that both end with date in
4183 //"%Y%m%d%H%M%S" format
4184
4185 gchar *datepos1 = g_strrstr(e1, "-snp-");
4186 gchar *datepos2 = g_strrstr(e2, "-snp-");
4187 if(IS_NULL_PTR(datepos1) || IS_NULL_PTR(datepos2))
4188 return 0;
4189
4190 datepos1 +=5; //skip "-snp-"
4191 datepos2 +=5; //skip "-snp-"
4192
4193 int year,month,day,hour,minute,second;
4194
4195 if(!_get_iso8601_int(datepos1, 4, &year) ||
4196 !_get_iso8601_int(datepos1+4, 2, &month) ||
4197 !_get_iso8601_int(datepos1+6, 2, &day) ||
4198 !_get_iso8601_int(datepos1+8, 2, &hour) ||
4199 !_get_iso8601_int(datepos1+10, 2, &minute) ||
4200 !_get_iso8601_int(datepos1+12, 2, &second)
4201 )
4202 {
4203 return 0;
4204 }
4205
4206 GDateTime *d1=g_date_time_new_local(year, month, day, hour, minute, second);
4207
4208 if(!_get_iso8601_int(datepos2, 4, &year) ||
4209 !_get_iso8601_int(datepos2+4, 2, &month) ||
4210 !_get_iso8601_int(datepos2+6, 2, &day) ||
4211 !_get_iso8601_int(datepos2+8, 2, &hour) ||
4212 !_get_iso8601_int(datepos2+10, 2, &minute) ||
4213 !_get_iso8601_int(datepos2+12, 2, &second)
4214 )
4215 {
4216 g_date_time_unref(d1);
4217 return 0;
4218 }
4219
4220 GDateTime *d2=g_date_time_new_local(year, month, day, hour, minute, second);
4221
4222 const gint ret = g_date_time_compare(d1, d2);
4223
4224 g_date_time_unref(d1);
4225 g_date_time_unref(d2);
4226
4227 return ret;
4228}
4229
4230char **dt_database_snaps_to_remove(const struct dt_database_t *db)
4231{
4232 if(_is_mem_db(db))
4233 return NULL;
4234
4235 const int keep_snaps = dt_conf_get_int("database/keep_snapshots");
4236
4237 if(keep_snaps < 0)
4238 return NULL;
4239
4240 dt_print(DT_DEBUG_SQL, "[db backup] checking snapshots existence.\n");
4241 GFile *lib_file = g_file_parse_name(db->dbfilename_library);
4242 GFile *lib_parent = g_file_get_parent(lib_file);
4243
4244 if(IS_NULL_PTR(lib_parent))
4245 {
4246 dt_print(DT_DEBUG_SQL, "[db backup] couldn't get library parent!.\n");
4247 g_object_unref(lib_file);
4248 return NULL;
4249 }
4250
4251 GFile *dat_file = g_file_parse_name(db->dbfilename_data);
4252 GFile *dat_parent = g_file_get_parent(dat_file);
4253
4254 if(IS_NULL_PTR(dat_parent))
4255 {
4256 dt_print(DT_DEBUG_SQL, "[db backup] couldn't get data parent!.\n");
4257 g_object_unref(dat_file);
4258 g_object_unref(lib_file);
4259 g_object_unref(lib_parent);
4260 }
4261
4262 gchar *lib_basename = g_file_get_basename(lib_file);
4263 g_object_unref(lib_file);
4264 gchar *lib_snap_format = g_strdup_printf("%s-snp-", lib_basename);
4265 gchar *lib_tmp_format = g_strdup_printf("%s-tmp-", lib_basename);
4266 dt_free(lib_basename);
4267
4268 gchar *dat_basename = g_file_get_basename(dat_file);
4269 g_object_unref(dat_file);
4270 gchar *dat_snap_format = g_strdup_printf("%s-snp-", dat_basename);
4271 gchar *dat_tmp_format = g_strdup_printf("%s-tmp-", dat_basename);
4272 dt_free(dat_basename);
4273
4274 GQueue *lib_snaps = g_queue_new();
4275 GQueue *dat_snaps = g_queue_new();
4276 GQueue *tmplib_snaps = g_queue_new();
4277 GQueue *tmpdat_snaps = g_queue_new();
4278
4279 if(g_file_equal(lib_parent, dat_parent))
4280 {
4281 //slight optimization if library and data are in same dir, we only have to scan one
4282 GError *error = NULL;
4283 GFileEnumerator *library_dir_files = g_file_enumerate_children(lib_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4284
4285 if(IS_NULL_PTR(library_dir_files))
4286 {
4287 dt_print(DT_DEBUG_SQL, "[db backup] couldn't enumerate library parent: %s.\n", error->message);
4288 g_object_unref(lib_parent);
4289 g_object_unref(dat_parent);
4290 dt_free(lib_snap_format);
4291 dt_free(dat_snap_format);
4292 dt_free(lib_tmp_format);
4293 dt_free(dat_tmp_format);
4294 g_queue_free(lib_snaps);
4295 g_queue_free(dat_snaps);
4296 g_queue_free(tmplib_snaps);
4297 g_queue_free(tmpdat_snaps);
4298 g_error_free(error);
4299 return NULL;
4300 }
4301
4302 GFileInfo *info = NULL;
4303
4304 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4305 {
4306 const char* fname = g_file_info_get_name(info);
4307 if(g_str_has_prefix(fname, lib_snap_format))
4308 {
4309 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4310 g_queue_insert_sorted(lib_snaps, g_strdup(fname), _db_snap_sort, NULL);
4311 }
4312 else if(g_str_has_prefix(fname, dat_snap_format))
4313 {
4314 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4315 g_queue_insert_sorted(dat_snaps, g_strdup(fname), _db_snap_sort, NULL);
4316 }
4317 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4318 {
4319 //we insert into single queue, since it's just dependent on parent
4320 g_queue_push_head(tmplib_snaps, g_strdup(fname));
4321 }
4322 g_object_unref(info);
4323 }
4324 dt_free(lib_snap_format);
4325 dt_free(dat_snap_format);
4326
4327 if(error)
4328 {
4329 dt_print(DT_DEBUG_SQL, "[db backup] problem enumerating library parent: %s.\n", error->message);
4330 g_object_unref(lib_parent);
4331 g_object_unref(dat_parent);
4332 dt_free(lib_tmp_format);
4333 dt_free(dat_tmp_format);
4334 g_queue_free_full(lib_snaps, g_free);
4335 g_queue_free_full(dat_snaps, g_free);
4336 g_queue_free_full(tmplib_snaps, g_free);
4337 g_queue_free_full(tmpdat_snaps, g_free);
4338 g_file_enumerator_close(library_dir_files, NULL, NULL);
4339 g_object_unref(library_dir_files);
4340 g_error_free(error);
4341 return NULL;
4342 }
4343 g_file_enumerator_close(library_dir_files, NULL, NULL);
4344 g_object_unref(library_dir_files);
4345 }
4346 else
4347 {
4348 //well... fun.
4349
4350 GError *error = NULL;
4351 GFileEnumerator *library_dir_files = g_file_enumerate_children(lib_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4352 if(IS_NULL_PTR(library_dir_files))
4353 {
4354 dt_print(DT_DEBUG_SQL, "[db backup] couldn't enumerate library parent: %s.\n", error->message);
4355 g_object_unref(lib_parent);
4356 g_object_unref(dat_parent);
4357 dt_free(lib_snap_format);
4358 dt_free(dat_snap_format);
4359 dt_free(lib_tmp_format);
4360 dt_free(dat_tmp_format);
4361 g_error_free(error);
4362 g_queue_free(lib_snaps);
4363 g_queue_free(dat_snaps);
4364 g_queue_free(tmplib_snaps);
4365 g_queue_free(tmpdat_snaps);
4366 return NULL;
4367 }
4368
4369 GFileEnumerator *data_dir_files = g_file_enumerate_children(dat_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4370 if(IS_NULL_PTR(data_dir_files))
4371 {
4372 dt_print(DT_DEBUG_SQL, "[db backup] couldn't enumerate data parent: %s.\n", error->message);
4373 g_object_unref(lib_parent);
4374 g_object_unref(dat_parent);
4375 dt_free(lib_snap_format);
4376 dt_free(dat_snap_format);
4377 dt_free(lib_tmp_format);
4378 dt_free(dat_tmp_format);
4379 g_file_enumerator_close(library_dir_files, NULL, NULL);
4380 g_object_unref(library_dir_files);
4381 g_error_free(error);
4382 g_queue_free(lib_snaps);
4383 g_queue_free(dat_snaps);
4384 g_queue_free(tmplib_snaps);
4385 g_queue_free(tmpdat_snaps);
4386 return NULL;
4387 }
4388
4389 GFileInfo *info = NULL;
4390
4391 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4392 {
4393 const char* fname = g_file_info_get_name(info);
4394 if(g_str_has_prefix(fname, lib_snap_format))
4395 {
4396 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4397 g_queue_insert_sorted(lib_snaps, g_strdup(fname), _db_snap_sort, NULL);
4398 }
4399 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4400 {
4401 // we remove all incomplete snaps matching pattern in BOTH dirs
4402 g_queue_push_head(tmplib_snaps, g_strdup(fname));
4403 }
4404 g_object_unref(info);
4405 }
4406 dt_free(lib_snap_format);
4407
4408 if(error)
4409 {
4410 dt_print(DT_DEBUG_SQL, "[db backup] problem enumerating library parent: %s.\n", error->message);
4411 g_object_unref(lib_parent);
4412 g_object_unref(dat_parent);
4413 dt_free(lib_tmp_format);
4414 dt_free(dat_tmp_format);
4415 g_queue_free_full(lib_snaps, g_free);
4416 g_queue_free(dat_snaps);
4417 g_queue_free_full(tmplib_snaps, g_free);
4418 g_queue_free(tmpdat_snaps);
4419 g_file_enumerator_close(library_dir_files, NULL, NULL);
4420 g_object_unref(library_dir_files);
4421 g_file_enumerator_close(data_dir_files, NULL, NULL);
4422 g_object_unref(data_dir_files);
4423 g_error_free(error);
4424 return NULL;
4425 }
4426 g_file_enumerator_close(library_dir_files, NULL, NULL);
4427 g_object_unref(library_dir_files);
4428
4429 while ((info = g_file_enumerator_next_file(data_dir_files, NULL, &error)))
4430 {
4431 const char* fname = g_file_info_get_name(info);
4432 if(g_str_has_prefix(fname, dat_snap_format))
4433 {
4434 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4435 g_queue_insert_sorted(dat_snaps, g_strdup(fname), _db_snap_sort, NULL);
4436 }
4437 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4438 {
4439 //we add to queue both matches - it just depends on parent
4440 g_queue_push_head(tmpdat_snaps, g_strdup(fname));
4441 }
4442 g_object_unref(info);
4443 }
4444 dt_free(dat_snap_format);
4445 dt_free(lib_tmp_format);
4446 dt_free(dat_tmp_format);
4447
4448 if(error)
4449 {
4450 dt_print(DT_DEBUG_SQL, "[db backup] problem enumerating data parent: %s.\n", error->message);
4451 g_object_unref(lib_parent);
4452 g_object_unref(dat_parent);
4453 g_queue_free_full(lib_snaps, g_free);
4454 g_queue_free_full(dat_snaps, g_free);
4455 g_queue_free_full(tmplib_snaps, g_free);
4456 g_queue_free_full(tmpdat_snaps, g_free);
4457 g_file_enumerator_close(data_dir_files, NULL, NULL);
4458 g_object_unref(data_dir_files);
4459 g_error_free(error);
4460 return NULL;
4461 }
4462
4463 g_file_enumerator_close(data_dir_files, NULL, NULL);
4464 g_object_unref(data_dir_files);
4465 }
4466
4467 // here we have list of snaps sorted in date order, now we have to
4468 // create from that list of snaps to be deleted and return that :D
4469
4470 GPtrArray *ret = g_ptr_array_new();
4471
4472 gchar *lib_parent_path = g_file_get_path(lib_parent);
4473 g_object_unref(lib_parent);
4474
4475 while(g_queue_get_length(lib_snaps) > keep_snaps)
4476 {
4477 gchar *head = g_queue_pop_head(lib_snaps);
4478 g_ptr_array_add(ret, g_strconcat(lib_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4479 dt_free(head);
4480 }
4481 while(!g_queue_is_empty(tmplib_snaps))
4482 {
4483 gchar *head = g_queue_pop_head(tmplib_snaps);
4484 g_ptr_array_add(ret, g_strconcat(lib_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4485 dt_free(head);
4486 }
4487 dt_free(lib_parent_path);
4488 g_queue_free_full(lib_snaps, g_free);
4489 g_queue_free_full(tmplib_snaps, g_free); // should be totally freed, but eh - this won't make doublefree
4490
4491 gchar *dat_parent_path = g_file_get_path(dat_parent);
4492 g_object_unref(dat_parent);
4493
4494 while(g_queue_get_length(dat_snaps) > keep_snaps)
4495 {
4496 gchar *head = g_queue_pop_head(dat_snaps);
4497 g_ptr_array_add(ret, g_strconcat(dat_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4498 dt_free(head);
4499 }
4500 while(!g_queue_is_empty(tmpdat_snaps))
4501 {
4502 gchar *head = g_queue_pop_head(tmpdat_snaps);
4503 g_ptr_array_add(ret, g_strconcat(dat_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4504 dt_free(head);
4505 }
4506 dt_free(dat_parent_path);
4507 g_queue_free_full(dat_snaps, g_free);
4508 g_queue_free_full(tmpdat_snaps, g_free);
4509
4510 g_ptr_array_add (ret, NULL);
4511
4512 return (char**)g_ptr_array_free(ret, FALSE);
4513}
4514
4515gchar *dt_database_get_most_recent_snap(const char* db_filename)
4516{
4517 if(!g_strcmp0(db_filename, ":memory:"))
4518 return NULL;
4519
4520 dt_print(DT_DEBUG_SQL, "[db backup] checking snapshots existence.\n");
4521 GFile *db_file = g_file_parse_name(db_filename);
4522 GFile *parent = g_file_get_parent(db_file);
4523
4524 if(IS_NULL_PTR(parent))
4525 {
4526 dt_print(DT_DEBUG_SQL, "[db backup] couldn't get database parent!.\n");
4527 g_object_unref(db_file);
4528 return NULL;
4529 }
4530
4531 GError *error = NULL;
4532 GFileEnumerator *db_dir_files = g_file_enumerate_children(parent, G_FILE_ATTRIBUTE_STANDARD_NAME "," G_FILE_ATTRIBUTE_TIME_MODIFIED, G_FILE_QUERY_INFO_NONE, NULL, &error);
4533
4534 if(IS_NULL_PTR(db_dir_files))
4535 {
4536 dt_print(DT_DEBUG_SQL, "[db backup] couldn't enumerate database parent: %s.\n", error->message);
4537 g_object_unref(parent);
4538 g_object_unref(db_file);
4539 g_error_free(error);
4540 return NULL;
4541 }
4542
4543 gchar *db_basename = g_file_get_basename(db_file);
4544 g_object_unref(db_file);
4545
4546 gchar *db_snap_format = g_strdup_printf("%s-snp-", db_basename);
4547 gchar *db_backup_format = g_strdup_printf("%s-pre-", db_basename);
4548 dt_free(db_basename);
4549
4550 GFileInfo *info = NULL;
4551 guint64 last_snap = 0;
4552 gchar *last_snap_name = NULL;
4553
4554 while ((info = g_file_enumerator_next_file(db_dir_files, NULL, &error)))
4555 {
4556 const char* fname = g_file_info_get_name(info);
4557 if(g_str_has_prefix(fname, db_snap_format) || g_str_has_prefix(fname, db_backup_format))
4558 {
4559 dt_print(DT_DEBUG_SQL, "[db backup] found file: %s.\n", fname);
4560 if(last_snap == 0)
4561 {
4562 last_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4563 last_snap_name = g_strdup(fname);
4564 g_object_unref(info);
4565 continue;
4566 }
4567 guint64 try_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4568 if(try_snap > last_snap)
4569 {
4570 last_snap = try_snap;
4571 dt_free(last_snap_name);
4572 last_snap_name = g_strdup(fname);
4573 }
4574 }
4575 g_object_unref(info);
4576 }
4577 dt_free(db_snap_format);
4578 dt_free(db_backup_format);
4579
4580 if(error)
4581 {
4582 dt_print(DT_DEBUG_SQL, "[db backup] problem enumerating database parent: %s.\n", error->message);
4583 g_file_enumerator_close(db_dir_files, NULL, NULL);
4584 g_object_unref(db_dir_files);
4585 g_error_free(error);
4586 dt_free(last_snap_name);
4587 return NULL;
4588 }
4589
4590 g_file_enumerator_close(db_dir_files, NULL, NULL);
4591 g_object_unref(db_dir_files);
4592
4593 if(IS_NULL_PTR(last_snap_name))
4594 {
4595 g_object_unref(parent);
4596 return NULL;
4597 }
4598
4599 gchar *parent_path = g_file_get_path(parent);
4600 g_object_unref(parent);
4601
4602 gchar *ret = g_strconcat(parent_path, G_DIR_SEPARATOR_S, last_snap_name, NULL);
4603 dt_free(parent_path);
4604 dt_free(last_snap_name);
4605
4606 return ret;
4607}
4608
4609// Nested transactions support
4610//
4611// NOTE: the nested support is actually not activated (see || TRUE below). This current
4612// implementation is just a refactoring of the previous code using:
4613// - dt_database_start_transaction()
4614// - dt_database_release_transaction()
4615// - dt_database_rollback_transaction()
4616//
4617// With this refactoring we can count and check for nested transaction and unmatched
4618// transaction routines. And it has been done to help further implementation for
4619// proper threading and nested transaction support.
4620//
4621void dt_database_start_transaction_debug(const struct dt_database_t *db)
4622{
4623 gpointer const owner = g_thread_self();
4624 const int batch_level = dt_atomic_get_int(&_trx_batch_level);
4625 if(batch_level > 0)
4626 {
4627 // If a batch transaction is active on this thread, suppress nested BEGIN/COMMIT
4628 // to avoid per-module transaction overhead during presets initialization.
4629 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4630 {
4631 dt_atomic_add_int(&_trxid, 1);
4632 return;
4633 }
4634 }
4635
4636 if(g_atomic_pointer_get(&_trx_owner) == owner)
4637 {
4638 dt_atomic_add_int(&_trxid, 1);
4639 return;
4640 }
4641
4642 dt_pthread_rwlock_wrlock(&darktable.database_threadsafe);
4643 g_atomic_pointer_set(&_trx_owner, owner);
4644
4645 const int trxid = dt_atomic_add_int(&_trxid, 1);
4646
4647 // if top level a simple unamed transaction is used BEGIN / COMMIT / ROLLBACK
4648 // otherwise we use a savepoint (named transaction).
4649
4650 if(trxid == 0)
4651 {
4652 // In theads application it may be safer to use an IMMEDIATE transaction:
4653 // "BEGIN IMMEDIATE TRANSACTION"
4654 // This implies "BEGIN DEFERRED TRANSACTION", which means
4655 // no write event is dispatched to DB until the first "COMMIT"
4656 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), "BEGIN TRANSACTION", NULL, NULL, NULL);
4657 }
4658#ifdef USE_NESTED_TRANSACTIONS
4659 else
4660 {
4661 char SQLTRX[32] = { 0 };
4662 g_snprintf(SQLTRX, sizeof(SQLTRX), "SAVEPOINT trx%d", trxid);
4663 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), SQLTRX, NULL, NULL, NULL);
4664 }
4665#endif
4666
4667 if(trxid > MAX_NESTED_TRANSACTIONS)
4668 fprintf(stderr, "[dt_database_start_transaction] more than %d nested transaction\n", MAX_NESTED_TRANSACTIONS);
4669}
4670
4671void dt_database_release_transaction_debug(const struct dt_database_t *db)
4672{
4673 gpointer const owner = g_thread_self();
4674 const int batch_level = dt_atomic_get_int(&_trx_batch_level);
4675 if(batch_level > 0)
4676 {
4677 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4678 {
4679 dt_atomic_sub_int(&_trxid, 1);
4680 return;
4681 }
4682 }
4683
4684 if(g_atomic_pointer_get(&_trx_owner) != owner)
4685 {
4686 fprintf(stderr, "[dt_database_release_transaction] COMMIT from non-owner thread\n");
4687 return;
4688 }
4689
4690 const int trxid = dt_atomic_sub_int(&_trxid, 1);
4691
4692 if(trxid <= 0)
4693 fprintf(stderr, "[dt_database_release_transaction] COMMIT outside a transaction\n");
4694
4695 if(trxid == 1)
4696 {
4697 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), "COMMIT TRANSACTION", NULL, NULL, NULL);
4698 g_atomic_pointer_set(&_trx_owner, NULL);
4699 dt_pthread_rwlock_unlock(&darktable.database_threadsafe);
4700 }
4701#ifdef USE_NESTED_TRANSACTIONS
4702 else
4703 {
4704 char SQLTRX[64] = { 0 };
4705 g_snprintf(SQLTRX, sizeof(SQLTRX), "RELEASE SAVEPOINT trx%d", trxid - 1);
4706 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), SQLTRX, NULL, NULL, NULL);
4707 }
4708#endif
4709}
4710
4711void dt_database_rollback_transaction(const struct dt_database_t *db)
4712{
4713 gpointer const owner = g_thread_self();
4714 if(g_atomic_pointer_get(&_trx_owner) != owner && g_atomic_pointer_get(&_trx_batch_owner) != owner)
4715 {
4716 fprintf(stderr, "[dt_database_rollback_transaction] ROLLBACK from non-owner thread\n");
4717 return;
4718 }
4719
4720 const int trxid = dt_atomic_sub_int(&_trxid, 1);
4721
4722 if(trxid <= 0)
4723 fprintf(stderr, "[dt_database_rollback_transaction] ROLLBACK outside a transaction\n");
4724
4725 if(trxid >= 1)
4726 {
4727 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), "ROLLBACK TRANSACTION", NULL, NULL, NULL);
4728 dt_atomic_set_int(&_trxid, 0);
4729 dt_atomic_set_int(&_trx_batch_level, 0);
4730 g_atomic_pointer_set(&_trx_owner, NULL);
4731 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4732 dt_pthread_rwlock_unlock(&darktable.database_threadsafe);
4733 }
4734#ifdef USE_NESTED_TRANSACTIONS
4735 else
4736 {
4737 char SQLTRX[64] = { 0 };
4738 g_snprintf(SQLTRX, sizeof(SQLTRX), "ROLLBACK TRANSACTION TO SAVEPOINT trx%d", trxid - 1);
4739 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), SQLTRX, NULL, NULL, NULL);
4740 }
4741#endif
4742}
4743
4744void dt_database_begin_transaction_batch(const struct dt_database_t *db)
4745{
4746 gpointer const owner = g_thread_self();
4747 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4748 {
4749 dt_atomic_add_int(&_trx_batch_level, 1);
4750 return;
4751 }
4752
4753 dt_pthread_rwlock_wrlock(&darktable.database_threadsafe);
4754 g_atomic_pointer_set(&_trx_owner, owner);
4755 g_atomic_pointer_set(&_trx_batch_owner, owner);
4756
4757 const int level = dt_atomic_add_int(&_trx_batch_level, 1);
4758 if(level == 0)
4759 {
4760 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), "BEGIN TRANSACTION", NULL, NULL, NULL);
4761 }
4762}
4763
4764void dt_database_end_transaction_batch(const struct dt_database_t *db)
4765{
4766 gpointer const owner = g_thread_self();
4767 if(g_atomic_pointer_get(&_trx_batch_owner) != owner)
4768 {
4769 dt_print(DT_DEBUG_SQL, "[dt_database_end_transaction_batch] COMMIT from non-owner thread\n");
4770 return;
4771 }
4772
4773 const int level = dt_atomic_sub_int(&_trx_batch_level, 1);
4774 if(level <= 0)
4775 {
4776 dt_print(DT_DEBUG_SQL, "[dt_database_end_transaction_batch] COMMIT outside a batch transaction\n");
4777 dt_atomic_set_int(&_trx_batch_level, 0);
4778 g_atomic_pointer_set(&_trx_owner, NULL);
4779 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4780 dt_pthread_rwlock_unlock(&darktable.database_threadsafe);
4781 return;
4782 }
4783
4784 if(level == 1)
4785 {
4786 DT_DEBUG_SQLITE3_EXEC(dt_database_get(db), "COMMIT TRANSACTION", NULL, NULL, NULL);
4787 g_atomic_pointer_set(&_trx_owner, NULL);
4788 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4789 dt_pthread_rwlock_unlock(&darktable.database_threadsafe);
4790 }
4791}
4792
4793// clang-format off
4794// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
4795// vim: shiftwidth=2 expandtab tabstop=2 cindent
4796// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
4797// clang-format on
error
static void error(char *msg)
Definition ashift_lsd.c:202
TRUE
#define TRUE
Definition ashift_lsd.c:162
FALSE
#define FALSE
Definition ashift_lsd.c:158
dt_atomic_set_int
void dt_atomic_set_int(dt_atomic_int *var, int value)
dt_atomic_get_int
int dt_atomic_get_int(dt_atomic_int *var)
dt_atomic_sub_int
int dt_atomic_sub_int(dt_atomic_int *var, int decr)
dt_atomic_add_int
int dt_atomic_add_int(dt_atomic_int *var, int incr)
dt_atomic_int
atomic_int dt_atomic_int
Definition atomic.h:66
kind
static const dt_adaptation_t kind
Definition chromatic_adaptation.h:124
i
const float i
Definition colorspaces_inline_conversions.h:440
n
const float n
Definition colorspaces_inline_conversions.h:678
void
typedef void((*dt_cache_allocate_t)(void *userdata, dt_cache_entry_t *entry))
name
char * name
Definition common/metadata.c:61
darktable_package_version
const char darktable_package_version[]
dt_conf_get_string
gchar * dt_conf_get_string(const char *name)
Definition control/conf.c:371
dt_conf_get_int
int dt_conf_get_int(const char *name)
Definition control/conf.c:241
dt_conf_set_string
void dt_conf_set_string(const char *name, const char *val)
Definition control/conf.c:167
dt_conf_get_string_const
const char * dt_conf_get_string_const(const char *name)
Definition control/conf.c:377
dt_concat_path_file
void dt_concat_path_file(char destination[PATH_MAX], const char path[PATH_MAX], const char *const file)
Definition darktable.c:1889
darktable
darktable_t darktable
Definition darktable.c:181
dt_print
void dt_print(dt_debug_thread_t thread, const char *msg,...)
Definition darktable.c:1542
UNKNOWN_IMAGE
#define UNKNOWN_IMAGE
Definition darktable.h:182
DT_DEBUG_SQL
@ DT_DEBUG_SQL
Definition darktable.h:723
dt_free_gpointer
static void dt_free_gpointer(gpointer ptr)
Definition darktable.h:463
dt_free
#define dt_free(ptr)
Definition darktable.h:456
value
static const dt_aligned_pixel_simd_t value
Definition darktable.h:577
PATH_MAX
#define PATH_MAX
Definition darktable.h:1062
IS_NULL_PTR
#define IS_NULL_PTR(p)
C is way too permissive with !=, == and if(var) checks, which can mean too many things depending on w...
Definition darktable.h:281
_lock_single_database
static gboolean _lock_single_database(dt_database_t *db, const char *dbfilename, char **lockfile)
Definition database.c:2786
_print_backup_progress
static void _print_backup_progress(int remaining, int total)
Definition database.c:3916
dt_database_end_transaction_batch
void dt_database_end_transaction_batch(const struct dt_database_t *db)
Definition database.c:4764
dt_database_backup
void dt_database_backup(const char *filename)
Definition database.c:2924
TRY_PREPARE
#define TRY_PREPARE(_stmt, _query, _message)
Definition database.c:468
dt_database_get_most_recent_snap
gchar * dt_database_get_most_recent_snap(const char *db_filename)
Definition database.c:4515
dt_database_cleanup_busy_statements
void dt_database_cleanup_busy_statements(const struct dt_database_t *db)
Definition database.c:3709
_trx_batch_owner
static gpointer _trx_batch_owner
Definition database.c:96
pid_is_alive
static gboolean pid_is_alive(int pid)
Definition database.c:2739
MAX_NESTED_TRANSACTIONS
#define MAX_NESTED_TRANSACTIONS
Definition database.c:91
ask_for_upgrade
void ask_for_upgrade(const gchar *dbname, const gboolean has_gui)
Definition database.c:2892
dt_database_optimize
void dt_database_optimize(const struct dt_database_t *db)
Definition database.c:3906
_get_pragma_string_val
gchar * _get_pragma_string_val(sqlite3 *db, const char *pragma)
Definition database.c:2984
_trxid
static dt_atomic_int _trxid
Definition database.c:93
_is_mem_db
static gboolean _is_mem_db(const struct dt_database_t *db)
Definition database.c:3824
dt_database_snapshot
gboolean dt_database_snapshot(const struct dt_database_t *db)
Definition database.c:3972
_trx_owner
static gpointer _trx_owner
Definition database.c:95
_sanitize_db
static void _sanitize_db(dt_database_t *db)
Definition database.c:2561
dt_database_perform_maintenance
void dt_database_perform_maintenance(const struct dt_database_t *db)
Definition database.c:3728
FINALIZE
#define FINALIZE
Definition database.c:482
ERRCHECK
#define ERRCHECK
Definition database.c:3727
dt_database_start_transaction_debug
void dt_database_start_transaction_debug(const struct dt_database_t *db)
Definition database.c:4621
_migrate_schema
static gboolean _migrate_schema(dt_database_t *db, int version)
Definition database.c:131
dt_database_destroy
void dt_database_destroy(const dt_database_t *db)
Definition database.c:3626
dt_database_show_error
gboolean dt_database_show_error(const dt_database_t *db)
Definition database.c:2651
dt_database_begin_transaction_batch
void dt_database_begin_transaction_batch(const struct dt_database_t *db)
Definition database.c:4744
dt_database_get
sqlite3 * dt_database_get(const dt_database_t *db)
Definition database.c:3646
_database_delete_mipmaps_files
static void _database_delete_mipmaps_files()
Definition database.c:3685
_upgrade_library_schema_step
static int _upgrade_library_schema_step(dt_database_t *db, int version)
Definition database.c:485
TRY_EXEC
#define TRY_EXEC(_query, _message)
Definition database.c:442
dt_database_snaps_to_remove
char ** dt_database_snaps_to_remove(const struct dt_database_t *db)
Definition database.c:4230
dt_database_maybe_maintenance
gboolean dt_database_maybe_maintenance(const struct dt_database_t *db, const gboolean has_gui, const gboolean closing_time)
Definition database.c:3829
dt_database_get_lock_acquired
gboolean dt_database_get_lock_acquired(const dt_database_t *db)
Definition database.c:3704
_upgrade_data_schema_step
static int _upgrade_data_schema_step(dt_database_t *db, int version)
Definition database.c:2110
_SQLITE3_EXEC
#define _SQLITE3_EXEC(a, b, c, d, e)
Definition database.c:122
dt_database_init
dt_database_t * dt_database_init(const char *alternative, const gboolean load_data, const gboolean has_gui)
Definition database.c:3008
dt_database_release_transaction_debug
void dt_database_release_transaction_debug(const struct dt_database_t *db)
Definition database.c:4671
_get_pragma_int_val
int _get_pragma_int_val(sqlite3 *db, const char *pragma)
Definition database.c:2968
_create_library_schema
static void _create_library_schema(dt_database_t *db)
Definition database.c:2358
_upgrade_library_schema
static gboolean _upgrade_library_schema(dt_database_t *db, int version)
Definition database.c:2329
_backup_db
static int _backup_db(sqlite3 *src_db, const char *src_db_name, const char *dest_filename, void(*xProgress)(int, int))
Definition database.c:3922
_trx_batch_level
static dt_atomic_int _trx_batch_level
Definition database.c:94
_database_migrate_to_xdg_structure
static void _database_migrate_to_xdg_structure()
Definition database.c:3656
dt_database_get_path
const gchar * dt_database_get_path(const struct dt_database_t *db)
Definition database.c:3651
CURRENT_DATABASE_VERSION_LIBRARY
#define CURRENT_DATABASE_VERSION_LIBRARY
Definition database.c:87
dt_database_rollback_transaction
void dt_database_rollback_transaction(const struct dt_database_t *db)
Definition database.c:4711
_upgrade_data_schema
static gboolean _upgrade_data_schema(dt_database_t *db, int version)
Definition database.c:2344
_create_memory_schema
static void _create_memory_schema(dt_database_t *db)
Definition database.c:2517
TRY_STEP
#define TRY_STEP(_stmt, _expected, _message)
Definition database.c:455
_ask_for_maintenance
gboolean _ask_for_maintenance(const gboolean has_gui, const gboolean closing_time, const guint64 size)
Definition database.c:3782
_lock_databases
static gboolean _lock_databases(dt_database_t *db)
Definition database.c:2879
_get_iso8601_int
static gboolean _get_iso8601_int(const gchar *text, gsize length, gint *value)
Definition database.c:4157
dt_database_maybe_snapshot
gboolean dt_database_maybe_snapshot(const struct dt_database_t *db)
Definition database.c:4022
_create_data_schema
static void _create_data_schema(dt_database_t *db)
Definition database.c:2464
_db_snap_sort
static gint _db_snap_sort(gconstpointer a, gconstpointer b, gpointer user_data)
Definition database.c:4177
CURRENT_DATABASE_VERSION_DATA
#define CURRENT_DATABASE_VERSION_DATA
Definition database.c:88
dt_datetime_exif_to_gdatetime
GDateTime * dt_datetime_exif_to_gdatetime(const char *exif, const GTimeZone *tz)
Definition datetime.c:239
dt_datetime_gdatetime_to_gtimespan
GTimeSpan dt_datetime_gdatetime_to_gtimespan(GDateTime *gdt)
Definition datetime.c:438
DT_DEBUG_SQLITE3_EXEC
#define DT_DEBUG_SQLITE3_EXEC(a, b, c, d, e)
Definition debug.h:99
dt_pthread_rwlock_wrlock
#define dt_pthread_rwlock_wrlock
Definition dtpthread.h:394
dt_pthread_rwlock_unlock
#define dt_pthread_rwlock_unlock
Definition dtpthread.h:392
dt_loc_get_user_cache_dir
void dt_loc_get_user_cache_dir(char *cachedir, size_t bufsize)
Definition file_location.c:314
dt_loc_get_datadir
void dt_loc_get_datadir(char *datadir, size_t bufsize)
Definition file_location.c:322
dt_loc_get_user_config_dir
void dt_loc_get_user_config_dir(char *configdir, size_t bufsize)
Definition file_location.c:310
file_location.h
dt_gui_show_standalone_yes_no_dialog
gboolean dt_gui_show_standalone_yes_no_dialog(const char *title, const char *markup, const char *no_text, const char *yes_text)
Definition gtk.c:1855
dt_image_flags_t
dt_image_flags_t
Definition image.h:91
DT_IMAGE_RAW
@ DT_IMAGE_RAW
Definition image.h:111
DT_IMAGE_HDR
@ DT_IMAGE_HDR
Definition image.h:113
DT_IMAGE_LDR
@ DT_IMAGE_LDR
Definition image.h:109
dt_imageio_get_type_from_extension
dt_image_flags_t dt_imageio_get_type_from_extension(const char *extension)
Map Exiv2 preview MIME types to decoder format identifiers.
Definition imageio.c:180
dt_ioppr_get_iop_order_list_version
GList * dt_ioppr_get_iop_order_list_version(dt_iop_order_t version)
Return the built-in order list for a given version.
Definition iop_order.c:1044
dt_sort_iop_list_by_order_f
gint dt_sort_iop_list_by_order_f(gconstpointer a, gconstpointer b)
Compare two list nodes holding modules by iop_order.
Definition iop_order.c:893
dt_ioppr_get_iop_order_list_kind
dt_iop_order_t dt_ioppr_get_iop_order_list_kind(GList *iop_order_list)
Determine the kind of an order list by inspecting its content.
Definition iop_order.c:905
dt_ioppr_serialize_text_iop_order_list
char * dt_ioppr_serialize_text_iop_order_list(GList *iop_order_list)
Serialize an order list to a text representation.
Definition iop_order.c:2533
iop_order.h
dt_iop_order_t
dt_iop_order_t
Definition iop_order.h:143
DT_IOP_ORDER_ANSEL_RAW
@ DT_IOP_ORDER_ANSEL_RAW
Definition iop_order.h:148
DT_IOP_ORDER_LEGACY
@ DT_IOP_ORDER_LEGACY
Definition iop_order.h:145
DT_IOP_ORDER_CUSTOM
@ DT_IOP_ORDER_CUSTOM
Definition iop_order.h:144
v
const float v
Definition iop_profile.h:221
legacy_presets.h
dt_legacy_presets_create
static void dt_legacy_presets_create(const struct dt_database_t *db)
Definition legacy_presets.h:1130
k
float *const restrict const size_t k
Definition luminance_mask.h:78
size
size_t size
Definition mipmap_cache.c:3
flags
dt_mipmap_buffer_dsc_flags flags
Definition mipmap_cache.c:4
GtkWidget
struct _GtkWidget GtkWidget
Definition splash.h:29
sqlite3IcuInit
int sqlite3IcuInit(sqlite3 *db)
Definition sqliteicu.c:525
day
static const char *const day[7]
Definition strptime.c:97
darktable_t::database_threadsafe
dt_pthread_rwlock_t database_threadsafe
Definition darktable.h:816
darktable_t::utc_tz
GTimeZone * utc_tz
Definition darktable.h:832
dt_database_t
Definition database.c:99
dt_database_t::error_other_pid
int error_other_pid
Definition database.c:112
dt_database_t::lockfile_data
gchar * lockfile_data
Definition database.c:103
dt_database_t::dbfilename_library
gchar * dbfilename_library
Definition database.c:106
dt_database_t::handle
sqlite3 * handle
Definition database.c:109
dt_database_t::lockfile_library
gchar * lockfile_library
Definition database.c:106
dt_database_t::error_message
gchar * error_message
Definition database.c:111
dt_database_t::lock_acquired
gboolean lock_acquired
Definition database.c:100
dt_database_t::dbfilename_data
gchar * dbfilename_data
Definition database.c:103
dt_database_t::error_dbfilename
gchar * error_dbfilename
Definition database.c:111
dt_iop_order_entry_t
Definition iop_order.h:154
dt_iop_order_entry_t::operation
char operation[20]
Definition iop_order.h:160
dt_iop_order_entry_t::o
union dt_iop_order_entry_t::@8 o
dt_iop_order_entry_t::iop_order_f
double iop_order_f
Definition iop_order.h:156
dt_iop_order_entry_t::instance
int32_t instance
Definition iop_order.h:161
MIN
#define MIN(a, b)
Definition thinplate.c:32
MAX
#define MAX(a, b)
Definition thinplate.c:29
dt_util_foo_to_utf8
gchar * dt_util_foo_to_utf8(const char *string)
Definition utility.c:392