Ansel 0.0
A darktable fork - bloat + design vision
Loading...
Searching...
No Matches
laplacian.c
Go to the documentation of this file.
1/*
2 This file is part of darktable,
3 Copyright (C) 2010 Bruce Guenter.
4 Copyright (C) 2010-2011 Henrik Andersson.
5 Copyright (C) 2010-2014, 2016 johannes hanika.
6 Copyright (C) 2010 Stuart Henderson.
7 Copyright (C) 2011 Antony Dovgal.
8 Copyright (C) 2011 Robert Bieber.
9 Copyright (C) 2011-2014, 2016, 2019 Tobias Ellinghaus.
10 Copyright (C) 2011-2012, 2014, 2016-2017 Ulrich Pegelow.
11 Copyright (C) 2012, 2015 Edouard Gomez.
12 Copyright (C) 2012 Jérémy Rosen.
13 Copyright (C) 2012 Richard Wonka.
14 Copyright (C) 2013, 2020 Aldric Renaudin.
15 Copyright (C) 2014, 2016 Dan Torop.
16 Copyright (C) 2014-2016 Roman Lebedev.
17 Copyright (C) 2015-2016 Pedro Côrte-Real.
18 Copyright (C) 2017 Heiko Bauke.
19 Copyright (C) 2017 luzpaz.
20 Copyright (C) 2018, 2020-2026 Aurélien PIERRE.
21 Copyright (C) 2018 Edgardo Hoszowski.
22 Copyright (C) 2018 Maurizio Paglia.
23 Copyright (C) 2018-2020, 2022 Pascal Obry.
24 Copyright (C) 2018 rawfiner.
25 Copyright (C) 2019 Andreas Schneider.
26 Copyright (C) 2019 Diederik ter Rahe.
27 Copyright (C) 2019-2020, 2022 Hanno Schwalm.
28 Copyright (C) 2020 Chris Elston.
29 Copyright (C) 2020, 2022 Diederik Ter Rahe.
30 Copyright (C) 2020-2021 Ralf Brown.
31 Copyright (C) 2021 Hubert Kowalski.
32 Copyright (C) 2022 Martin Bařinka.
33 Copyright (C) 2022 Philipp Lutz.
34 Copyright (C) 2022 Victor Forsiuk.
35 Copyright (C) 2023 Alynx Zhou.
36 Copyright (C) 2023, 2026 Guillaume Stutin.
37 Copyright (C) 2023 Luca Zulberti.
38
39 darktable is free software: you can redistribute it and/or modify
40 it under the terms of the GNU General Public License as published by
41 the Free Software Foundation, either version 3 of the License, or
42 (at your option) any later version.
43
44 darktable is distributed in the hope that it will be useful,
45 but WITHOUT ANY WARRANTY; without even the implied warranty of
46 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
47 GNU General Public License for more details.
48
49 You should have received a copy of the GNU General Public License
50 along with darktable. If not, see <http://www.gnu.org/licenses/>.
51 */
52
53// Guided-laplacian (2021 a-trous) highlight reconstruction, CPU + OpenCL. (implementation; see laplacian.h for the
54// public API.)
55
56#include "pixel/box_filters.h"
58#include "pixel/bspline.h"
59#include "common/logging.h"
60#include "system/macros.h"
61#include "system/mem_alloc.h"
62#include "system/openmp.h"
63#include "system/simd.h"
66#include "pixel/dwt.h"
68#include "common/opencl.h"
70#include "iop/noise_generator.h"
73#include <math.h>
74#include <string.h>
75
76static inline __attribute__((always_inline)) uint8_t scale_type(const int s, const int scales)
77{
78 uint8_t scale = ANY_SCALE;
79 if(s == 0) scale |= FIRST_SCALE;
80 if(s == scales - 1) scale |= LAST_SCALE;
81 return scale;
82}
83
85static inline void guide_laplacians(const float *const restrict high_freq, const float *const restrict low_freq,
86 const float *const restrict clipping_mask, float *const restrict output,
87 const size_t width, const size_t height, const int mult,
88 const float noise_level, const int salt, const uint8_t scale,
89 const float radius_sq)
90{
91 float *const restrict out = DT_IS_ALIGNED(output);
92 const float *const restrict LF = DT_IS_ALIGNED(low_freq);
93 const float *const restrict HF = DT_IS_ALIGNED(high_freq);
94 const dt_aligned_pixel_simd_t zero = dt_simd_set1(0.f);
95 const dt_aligned_pixel_simd_t ones = dt_simd_set1(1.f);
96 const dt_aligned_pixel_simd_t inv_patch = dt_simd_set1(1.f / 9.f);
97 const dt_aligned_pixel_simd_t scale_multiplier = dt_simd_set1(1.f / radius_sq);
98 const float eps = 1e-12f;
100 for(size_t row = 0; row < height; ++row)
101 {
102 // interleave the order in which we process the rows so that we minimize cache misses
103 const int i = dwt_interleave_rows(row, height, mult);
104 const float *const row0 = HF + 4 * ((size_t)MAX(i - mult, 0) * width);
105 const float *const row1 = HF + 4 * ((size_t)i * width);
106 const float *const row2 = HF + 4 * ((size_t)MIN(i + mult, (int)height - 1) * width);
107 const float *const rows[3] = { row0, row1, row2 };
108 const int max_col = (int)width - 1;
109
110 for(int j = 0; j < width; ++j)
111 {
112 const size_t idx = (i * width + j);
113 const size_t index = idx * 4;
114 const float alpha = clipping_mask[index + ALPHA];
115 const float alpha_comp = 1.f - alpha;
116 dt_aligned_pixel_simd_t high_frequency = dt_load_simd_aligned(HF + index);
117
118 if(alpha > 0.f) // reconstruct
119 {
120 const int col_offsets[3] = { 4 * MAX(j - mult, 0), 4 * j, 4 * MIN(j + mult, max_col) };
121 dt_aligned_pixel_simd_t sum = zero;
122 dt_aligned_pixel_simd_t sum_sq = zero;
123 dt_aligned_pixel_simd_t prod_r = zero;
124 dt_aligned_pixel_simd_t prod_g = zero;
125 dt_aligned_pixel_simd_t prod_b = zero;
126
127 // Walk the dense 3x3 neighbourhood as counted loops so GCC keeps the
128 // fit as a regular reduction instead of fully unrolling all 9 taps and
129 // spilling the intermediate moments to the stack.
130#if defined(__GNUC__) && !defined(__clang__)
131#pragma GCC unroll 1
132#endif
133 for(int jj = 0; jj < 3; ++jj)
134 {
135 const float *const row_ptr = rows[jj];
136#if defined(__GNUC__) && !defined(__clang__)
137#pragma GCC unroll 1
138#endif
139 for(int ii = 0; ii < 3; ++ii)
140 {
141 const dt_aligned_pixel_simd_t sample = dt_load_simd_aligned(row_ptr + col_offsets[ii]);
142
143 sum += sample;
144 sum_sq += sample * sample;
145 prod_r += sample * dt_simd_set1(sample[RED]);
146 prod_g += sample * dt_simd_set1(sample[GREEN]);
147 prod_b += sample * dt_simd_set1(sample[BLUE]);
148 }
149 }
150
151 dt_aligned_pixel_simd_t means = sum * inv_patch;
152 dt_aligned_pixel_simd_t variance = sum_sq * inv_patch - means * means;
153 variance = dt_simd_max_zero(variance);
154 variance[ALPHA] = 0.f;
155
156 size_t guiding_channel = RED;
157 float guide_variance = variance[RED];
158 if(variance[GREEN] > guide_variance)
159 {
160 guiding_channel = GREEN;
161 guide_variance = variance[GREEN];
162 }
163 if(variance[BLUE] > guide_variance)
164 {
165 guiding_channel = BLUE;
166 guide_variance = variance[BLUE];
167 }
168
169 if(guide_variance > eps)
170 {
171 const float guide_mean = means[guiding_channel];
172 dt_aligned_pixel_simd_t covariance
173 = (guiding_channel == RED ? prod_r : (guiding_channel == GREEN ? prod_g : prod_b)) * inv_patch
174 - means * dt_simd_set1(guide_mean);
175 dt_aligned_pixel_simd_t slope = covariance / dt_simd_set1(guide_variance);
176 slope = dt_simd_max_zero(slope);
177 dt_aligned_pixel_simd_t intercept = means - slope * dt_simd_set1(guide_mean);
178 const dt_aligned_pixel_simd_t blend = dt_load_simd_aligned(clipping_mask + index) * scale_multiplier;
179 const dt_aligned_pixel_simd_t guide = dt_simd_set1(high_frequency[guiding_channel]);
180 high_frequency = blend * (slope * guide + intercept) + (ones - blend) * high_frequency;
181 }
182 }
183
184 dt_aligned_pixel_simd_t out_pixel = high_frequency;
185 if((scale & FIRST_SCALE))
186 {
187 // out is not inited yet
188 }
189 else
190 {
191 // just accumulate HF
192 out_pixel += dt_load_simd_aligned(out + index);
193 }
194
195 if((scale & LAST_SCALE))
196 {
197 // add the residual and clamp
198 out_pixel = dt_simd_max_zero(out_pixel + dt_load_simd_aligned(LF + index));
199 }
200
201 // Last step of RGB reconstruct : add noise
202 if((scale & LAST_SCALE) && salt && alpha > 0.f)
203 {
204 // Init random number generator
205 uint32_t DT_ALIGNED_ARRAY state[4]
206 = { splitmix32(j + 1), splitmix32((j + 1) * (i + 3)), splitmix32(1337), splitmix32(666) };
211
212 dt_aligned_pixel_t noise = { 0.f };
213 dt_aligned_pixel_t sigma = { 0.20f };
214 const int DT_ALIGNED_ARRAY flip[4] = { TRUE, FALSE, TRUE, FALSE };
215
216 sigma[RED] = out_pixel[RED] * noise_level;
217 sigma[GREEN] = out_pixel[GREEN] * noise_level;
218 sigma[BLUE] = out_pixel[BLUE] * noise_level;
219 sigma[ALPHA] = out_pixel[ALPHA] * noise_level;
220
221 // create statistical noise
222 dt_aligned_pixel_t current = { out_pixel[RED], out_pixel[GREEN], out_pixel[BLUE], out_pixel[ALPHA] };
224
225 // Save the noisy interpolated image
226 for_each_channel(c, aligned(noise, current)) noise[c] = current[c] + fabsf(noise[c] - current[c]);
227
228 out_pixel[RED] = fmaxf(alpha * noise[RED] + alpha_comp * current[RED], 0.f);
229 out_pixel[GREEN] = fmaxf(alpha * noise[GREEN] + alpha_comp * current[GREEN], 0.f);
230 out_pixel[BLUE] = fmaxf(alpha * noise[BLUE] + alpha_comp * current[BLUE], 0.f);
231 out_pixel[ALPHA] = fmaxf(alpha * noise[ALPHA] + alpha_comp * current[ALPHA], 0.f);
232 }
233
234 if((scale & LAST_SCALE))
235 {
236 // Break the RGB channels into ratios/norm for the next step of reconstruction
237 const float norm = fmaxf(sqrtf(sqf(out_pixel[RED]) + sqf(out_pixel[GREEN]) + sqf(out_pixel[BLUE])), 1e-6f);
238 out_pixel /= dt_simd_set1(norm);
239 out_pixel[ALPHA] = norm;
240 }
241
242 dt_store_simd_aligned(out + index, out_pixel);
243 }
244 }
245}
246
248static inline void heat_PDE_diffusion(const float *const restrict high_freq, const float *const restrict low_freq,
249 const float *const restrict clipping_mask, float *const restrict output,
250 const size_t width, const size_t height, const int mult, const uint8_t scale,
251 const float first_order_factor)
252{
253 // Simultaneous inpainting for image structure and texture using anisotropic heat transfer model
254 // https://www.researchgate.net/publication/220663968
255 // modified as follow :
256 // * apply it in a multi-scale wavelet setup : we basically solve it twice, on the wavelets LF and HF layers.
257 // * replace the manual texture direction/distance selection by an automatic detection similar to the structure
258 // one,
259 // * generalize the framework for isotropic diffusion and anisotropic weighted on the isophote direction
260 // * add a variance regularization to better avoid edges.
261 // The sharpness setting mimics the contrast equalizer effect by simply multiplying the HF by some gain.
262
263 float *const restrict out = DT_IS_ALIGNED(output);
264 const float *const restrict LF = DT_IS_ALIGNED(low_freq);
265 const float *const restrict HF = DT_IS_ALIGNED(high_freq);
267 for(size_t row = 0; row < height; ++row)
268 {
269 // interleave the order in which we process the rows so that we minimize cache misses
270 const size_t i = dwt_interleave_rows(row, height, mult);
271 // compute the 'above' and 'below' coordinates, clamping them to the image, once for the entire row
272 const size_t i_neighbours[3] = { MAX((int)(i - mult), (int)0) * width, // x - mult
273 i * width, // x
274 MIN((int)(i + mult), (int)height - 1) * width }; // x + mult
275
276 static const float DT_ALIGNED_ARRAY anisotropic_kernel_isophote[9]
277 = { 0.25f, 0.5f, 0.25f, 0.5f, -3.f, 0.5f, 0.25f, 0.5f, 0.25f };
278
279 for(size_t j = 0; j < width; ++j)
280 {
281 const size_t idx = (i * width + j);
282 const size_t index = idx * 4;
283
284 // fetch the clipping mask opacity : opaque (alpha = 100 %) where clipped
285 const dt_aligned_pixel_t alpha = { clipping_mask[index + RED], clipping_mask[index + GREEN],
286 clipping_mask[index + BLUE], clipping_mask[index + ALPHA] };
287
288 dt_aligned_pixel_t high_frequency = { HF[index + 0], HF[index + 1], HF[index + 2], HF[index + 3] };
289
290 // The for_each_channel macro uses 4 floats SIMD instructions or 3 float regular ops,
291 // depending on system. Since we don't want to diffuse the norm, make sure to store and restore it later.
292 // This is not much of an issue when processing image at full-res, but more harmful since
293 // we reconstruct highlights on a downscaled variant
294 const float norm_backup = high_frequency[3];
295
296 if(alpha[ALPHA] > 0.f) // reconstruct
297 {
298 // non-local neighbours coordinates
299 const size_t j_neighbours[3] = { MAX((int)(j - mult), (int)0), // y - mult
300 j, // y
301 MIN((int)(j + mult), (int)width - 1) }; // y + mult
302
303 // fetch non-local pixels and store them locally and contiguously
304 dt_aligned_pixel_t neighbour_pixel_HF[9];
305 for_four_channels(c, aligned(neighbour_pixel_HF, HF : 16))
306 {
307 neighbour_pixel_HF[3 * 0 + 0][c] = HF[4 * (i_neighbours[0] + j_neighbours[0]) + c];
308 neighbour_pixel_HF[3 * 0 + 1][c] = HF[4 * (i_neighbours[0] + j_neighbours[1]) + c];
309 neighbour_pixel_HF[3 * 0 + 2][c] = HF[4 * (i_neighbours[0] + j_neighbours[2]) + c];
310
311 neighbour_pixel_HF[3 * 1 + 0][c] = HF[4 * (i_neighbours[1] + j_neighbours[0]) + c];
312 neighbour_pixel_HF[3 * 1 + 1][c] = HF[4 * (i_neighbours[1] + j_neighbours[1]) + c];
313 neighbour_pixel_HF[3 * 1 + 2][c] = HF[4 * (i_neighbours[1] + j_neighbours[2]) + c];
314
315 neighbour_pixel_HF[3 * 2 + 0][c] = HF[4 * (i_neighbours[2] + j_neighbours[0]) + c];
316 neighbour_pixel_HF[3 * 2 + 1][c] = HF[4 * (i_neighbours[2] + j_neighbours[1]) + c];
317 neighbour_pixel_HF[3 * 2 + 2][c] = HF[4 * (i_neighbours[2] + j_neighbours[2]) + c];
318 }
319
320 // Compute the laplacian in the direction parallel to the steepest gradient on the norm
321 // Convolve the filter to get the laplacian
322 dt_aligned_pixel_t laplacian_HF = { 0.f, 0.f, 0.f, 0.f };
323 for(int k = 0; k < 9; k++)
324 {
325 for_each_channel(c, aligned(laplacian_HF, neighbour_pixel_HF : 16)
326 aligned(anisotropic_kernel_isophote : 64)) laplacian_HF[c]
327 += neighbour_pixel_HF[k][c] * anisotropic_kernel_isophote[k];
328 }
329
330 // Diffuse
331 const dt_aligned_pixel_t multipliers_HF
333 for_each_channel(c, aligned(high_frequency, multipliers_HF, laplacian_HF, alpha)) high_frequency[c]
334 += alpha[c] * multipliers_HF[c] * (laplacian_HF[c] - first_order_factor * high_frequency[c]);
335
336 // Restore. See above.
337 high_frequency[3] = norm_backup;
338 }
339
340 if((scale & FIRST_SCALE))
341 {
342 // out is not inited yet
343 for_each_channel(c, aligned(out, high_frequency : 64)) out[index + c] = high_frequency[c];
344 }
345 else
346 {
347 // just accumulate HF
348 for_each_channel(c, aligned(out, high_frequency : 64)) out[index + c] += high_frequency[c];
349 }
350
351 if((scale & LAST_SCALE))
352 {
353 // add the residual and clamp
354 for_each_channel(c, aligned(out, LF, high_frequency : 64)) out[index + c]
355 = fmaxf(out[index + c] + LF[index + c], 0.f);
356
357 // renormalize ratios
358 if(alpha[ALPHA] > 0.f)
359 {
360 const float norm = sqrtf(sqf(out[index + RED]) + sqf(out[index + GREEN]) + sqf(out[index + BLUE]));
361 for_each_channel(c, aligned(out, LF, high_frequency : 64)) out[index + c]
362 /= (c != ALPHA && norm > 1e-4f) ? norm : 1.f;
363 }
364
365 // Last scale : reconstruct RGB from ratios and norm - norm stays in the 4th channel
366 // we need it to evaluate the gradient
367 for_four_channels(c, aligned(out)) out[index + c]
368 = (c == ALPHA) ? out[index + ALPHA] : out[index + c] * out[index + ALPHA];
369 }
370 }
371 }
372}
373
374static inline int wavelets_process(const float *const restrict in, float *const restrict reconstructed,
375 const float *const restrict clipping_mask, const size_t width,
376 const size_t height, const int scales, float *const restrict HF,
377 float *const restrict LF_odd, float *const restrict LF_even,
378 const diffuse_reconstruct_variant_t variant, const float noise_level,
379 const int salt, const float first_order_factor)
380{
381 // À trous decimated wavelet decompose
382 // there is a paper from a guy we know that explains it : https://jo.dreggn.org/home/2010_atrous.pdf
383 // the wavelets decomposition here is the same as the equalizer/atrous module,
384
385 // allocate a one-row temporary buffer for the decomposition
386 size_t padded_size;
387 float *const tempbuf = dt_pixelpipe_cache_alloc_perthread_float(4 * width, &padded_size); // TODO: alloc in caller
388 if(IS_NULL_PTR(tempbuf)) return 1;
389
390 for(int s = 0; s < scales; ++s)
391 {
392 // fprintf(stderr, "CPU Wavelet decompose : scale %i\n", s);
393 const int mult = 1 << s;
394
395 const float *restrict buffer_in;
396 float *restrict buffer_out;
397
398 if(s == 0)
399 {
400 buffer_in = in;
401 buffer_out = LF_odd;
402 }
403 else if(s % 2 != 0)
404 {
405 buffer_in = LF_odd;
406 buffer_out = LF_even;
407 }
408 else
409 {
410 buffer_in = LF_even;
411 buffer_out = LF_odd;
412 }
413
414 decompose_2D_Bspline(buffer_in, HF, buffer_out, width, height, mult, tempbuf, padded_size);
415
416 uint8_t current_scale_type = scale_type(s, scales);
417 const float radius = sqf(equivalent_sigma_at_step(B_SPLINE_SIGMA, s * DS_FACTOR));
418
419 if(variant == DIFFUSE_RECONSTRUCT_RGB)
420 guide_laplacians(HF, buffer_out, clipping_mask, reconstructed, width, height, mult, noise_level, salt,
421 current_scale_type, radius);
422 else
423 heat_PDE_diffusion(HF, buffer_out, clipping_mask, reconstructed, width, height, mult, current_scale_type,
424 first_order_factor);
425
426 }
428
429 return 0;
430}
431
434 const dt_dev_pixelpipe_iop_t *piece, const void *const restrict ivoid,
435 void *const restrict ovoid, const dt_iop_roi_t *const roi_in,
436 const dt_iop_roi_t *const roi_out, const dt_aligned_pixel_t clips)
437{
439 int err = 0;
440 (void)roi_out;
441
442 // Every CFA helper below (normalization, Bayer gather, remosaic) reads FC(row, col, filters) with
443 // tile-local row/col (0-based within this buffer, no roi offset added), so filters must be
444 // pre-shifted for roi_in's crop position here -- mirrors demosaic.c's tile-local algorithms.
445 // dt_dev_get_roi_filters() returns the shifted Bayer word, 9u unchanged for X-Trans, and 0 for
446 // already-demosaiced (non-raw / sRAW) input. Only the gather and the remosaic branch on `cfa`; the
447 // wavelet reconstruction between them is CFA-agnostic.
448 const uint32_t filters = dt_dev_get_roi_filters(piece, roi_in);
449 const dt_hl_cfa_t cfa = _hl_cfa_strategy(filters);
450 const uint8_t(*const xtrans)[6]
451 = (cfa == HL_CFA_XTRANS) ? (const uint8_t(*const)[6])piece->dsc_in.xtrans : NULL;
452
453 const size_t height = roi_in->height;
454 const size_t width = roi_in->width;
455 const size_t size = roi_in->width * roi_in->height;
456
457 const size_t ds_height = height / DS_FACTOR;
458 const size_t ds_width = width / DS_FACTOR;
459 const size_t ds_size = ds_height * ds_width;
460
461 float *const restrict interpolated
462 = dt_pixelpipe_cache_alloc_align_float(size * 4, pipe); // [R, G, B, norm] for each pixel
463 float *const restrict clipping_mask
464 = dt_pixelpipe_cache_alloc_align_float(size * 4, pipe); // [R, G, B, norm] for each pixel
465
466 // temp buffer for blurs. We will need to cycle between them for memory efficiency
467 float *const restrict LF_odd = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
468 float *const restrict LF_even = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
469 float *const restrict temp = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
470
471 const float scale = DS_FACTOR * dt_dev_get_module_scale(pipe, roi_in);
472 const float final_radius = (float)((int)(1 << data->scales)) / scale;
473 const int scales = CLAMP((int)ceilf(log2f(final_radius)), 1, MAX_NUM_SCALES);
474
475 const float noise_level = data->noise_level / scale;
476
477 // wavelets scales buffers
478 float *restrict HF = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
479 float *restrict ds_interpolated = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
480 float *restrict ds_clipping_mask = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
481
482 if(IS_NULL_PTR(interpolated) || IS_NULL_PTR(clipping_mask) || IS_NULL_PTR(LF_odd) || IS_NULL_PTR(LF_even)
483 || IS_NULL_PTR(temp) || IS_NULL_PTR(HF) || IS_NULL_PTR(ds_interpolated) || IS_NULL_PTR(ds_clipping_mask))
484 {
485 err = 1;
486 goto error;
487 }
488
489 const float *const restrict input = (const float *const restrict)ivoid;
490 float *const restrict output = (float *const restrict)ovoid;
491 dt_aligned_pixel_t normalization = { 1.f, 1.f, 1.f, 1.f };
492 _compute_laplacian_normalization(input, roi_in, filters, xtrans, normalization);
493
494 // FLOW step 1a (gather): the only CFA-branching endpoint before the shared wavelet reconstruction.
495 // Guided laplacians run without the knee, so det_scale is unit and both CFA gathers take `clips` directly.
496 switch(cfa)
497 {
498 case HL_CFA_BAYER:
499 {
500 const dt_aligned_pixel_t det_unit = { 1.f, 1.f, 1.f, 1.f };
501 _interpolate_and_mask(input, interpolated, clipping_mask, clips, det_unit, normalization, filters, width,
502 height);
503 break;
504 }
505 case HL_CFA_XTRANS:
506 {
507 int32_t lookup[6][6][32] = { { { 0 } } };
508 _build_xtrans_bilinear_lookup(lookup, roi_in, xtrans);
509 _interpolate_and_mask_xtrans(input, interpolated, clipping_mask, clips, normalization, roi_in, lookup,
510 xtrans, width, height);
511 break;
512 }
514 // Non-raw / sRAW: no demosaic, just copy the RGB planes through + build masks.
515 _interpolate_and_mask_passthrough(input, interpolated, clipping_mask, clips, normalization, width, height);
516 break;
517 }
518 if(dt_box_mean(clipping_mask, height, width, 4, 2, 1) != 0)
519 {
520 err = 1;
521 goto error;
522 }
523
524 // Downsample
525 interpolate_bilinear(clipping_mask, width, height, ds_clipping_mask, ds_width, ds_height, 4);
526 interpolate_bilinear(interpolated, width, height, ds_interpolated, ds_width, ds_height, 4);
527
528 for(int i = 0; i < data->iterations; i++)
529 {
530 // The pipe abandoned this run: stop between two of up to 512 iterations.
532 {
533 err = 1;
534 goto error;
535 }
536 const int salt = (i == data->iterations - 1); // add noise on the last iteration only
537 if(wavelets_process(ds_interpolated, temp, ds_clipping_mask, ds_width, ds_height, scales, HF, LF_odd, LF_even,
538 DIFFUSE_RECONSTRUCT_RGB, noise_level, salt, data->solid_color))
539 {
540 err = 1;
541 goto error;
542 }
543 if(wavelets_process(temp, ds_interpolated, ds_clipping_mask, ds_width, ds_height, scales, HF, LF_odd, LF_even,
544 DIFFUSE_RECONSTRUCT_CHROMA, noise_level, salt, data->solid_color))
545 {
546 err = 1;
547 goto error;
548 }
549 }
550
551 // Upsample
552 interpolate_bilinear(ds_interpolated, ds_width, ds_height, interpolated, width, height, 4);
553 // FLOW: remosaic + composite -- second and last CFA-branching endpoint (clip_is_floor = FALSE here).
554 switch(cfa)
555 {
556 case HL_CFA_BAYER:
557 _remosaic_and_replace(input, input, interpolated, clipping_mask, output, normalization, clips, FALSE,
558 filters, width, height);
559 break;
560 case HL_CFA_XTRANS:
561 _remosaic_and_replace_xtrans(input, input, interpolated, clipping_mask, output, normalization, clips,
562 FALSE, roi_in, xtrans, width, height);
563 break;
565 // Non-raw / sRAW: composite the reconstructed RGB straight back, per channel (clip_is_floor FALSE).
566 _remosaic_and_replace_passthrough(input, input, interpolated, clipping_mask, output, normalization, clips,
567 FALSE, width, height);
568 break;
569 }
570
571error:;
572 dt_pixelpipe_cache_free_align(interpolated);
573 dt_pixelpipe_cache_free_align(clipping_mask);
578 dt_pixelpipe_cache_free_align(ds_interpolated);
579 dt_pixelpipe_cache_free_align(ds_clipping_mask);
580 return err;
581}
582
583#ifdef HAVE_OPENCL
584static inline cl_int wavelets_process_cl(const int devid, cl_mem in, cl_mem reconstructed,
585 cl_mem reconstructed_scratch, cl_mem clipping_mask, const size_t sizes[3],
586 const int width, const int height,
587 dt_iop_highlights_global_data_t *const gd, const int scales, cl_mem HF,
588 cl_mem LF_odd, cl_mem LF_even,
589 const diffuse_reconstruct_variant_t variant, const float noise_level,
590 const int salt, const float solid_color)
591{
592 cl_int err = DT_OPENCL_DEFAULT_ERROR;
593 cl_mem reconstruct_read = reconstructed_scratch;
594
595 // À trous wavelet decompose
596 // there is a paper from a guy we know that explains it : https://jo.dreggn.org/home/2010_atrous.pdf
597 // the wavelets decomposition here is the same as the equalizer/atrous module,
598 for(int s = 0; s < scales; ++s)
599 {
600 // fprintf(stderr, "GPU Wavelet decompose : scale %i\n", s);
601 const int mult = 1 << s;
602
603 cl_mem buffer_in;
604 cl_mem buffer_out;
605
606 if(s == 0)
607 {
608 buffer_in = in;
609 buffer_out = LF_odd;
610 }
611 else if(s % 2 != 0)
612 {
613 buffer_in = LF_odd;
614 buffer_out = LF_even;
615 }
616 else
617 {
618 buffer_in = LF_even;
619 buffer_out = LF_odd;
620 }
621
622 // Compute wavelets low-frequency scales
623 const int clamp_lf = 1;
624 int hblocksize;
626 .xfactor = 1,
627 .yoffset = 0,
628 .yfactor = 1,
629 .cellsize = 4 * sizeof(float),
630 .overhead = 0,
631 .sizex = 1 << 16,
632 .sizey = 1 };
634 hblocksize = hlocopt.sizex;
635 else
636 hblocksize = 1;
637
638 if(hblocksize > 1)
639 {
640 const size_t horizontal_sizes[3] = { ROUNDUP(width, hblocksize), ROUNDUPDHT(height, devid), 1 };
641 const size_t horizontal_local[3] = { hblocksize, 1, 1 };
643 (void *)&buffer_in);
644 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal_local, 1, sizeof(cl_mem), (void *)&HF);
645 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal_local, 2, sizeof(int), (void *)&width);
646 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal_local, 3, sizeof(int), (void *)&height);
647 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal_local, 4, sizeof(int), (void *)&mult);
649 (void *)&clamp_lf);
651 (hblocksize + 4 * mult) * 4 * sizeof(float), NULL);
653 horizontal_sizes, horizontal_local);
654 }
655 else
656 {
657 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 0, sizeof(cl_mem), (void *)&buffer_in);
658 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 1, sizeof(cl_mem), (void *)&HF);
659 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 2, sizeof(int), (void *)&width);
660 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 3, sizeof(int), (void *)&height);
661 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 4, sizeof(int), (void *)&mult);
662 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 5, sizeof(int), (void *)&clamp_lf);
664 }
665 if(err != CL_SUCCESS) return err;
666
667 int vblocksize;
669 .xfactor = 1,
670 .yoffset = 2 * mult,
671 .yfactor = 1,
672 .cellsize = 4 * sizeof(float),
673 .overhead = 0,
674 .sizex = 1,
675 .sizey = 1 << 16 };
677 vblocksize = vlocopt.sizey;
678 else
679 vblocksize = 1;
680
681 if(vblocksize > 1)
682 {
683 const size_t vertical_sizes[3] = { ROUNDUPDWD(width, devid), ROUNDUP(height, vblocksize), 1 };
684 const size_t vertical_local[3] = { 1, vblocksize, 1 };
685 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical_local, 0, sizeof(cl_mem), (void *)&HF);
687 (void *)&buffer_out);
688 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical_local, 2, sizeof(int), (void *)&width);
689 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical_local, 3, sizeof(int), (void *)&height);
690 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical_local, 4, sizeof(int), (void *)&mult);
691 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical_local, 5, sizeof(int), (void *)&clamp_lf);
693 (vblocksize + 4 * mult) * 4 * sizeof(float), NULL);
695 vertical_local);
696 }
697 else
698 {
699 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 0, sizeof(cl_mem), (void *)&HF);
700 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 1, sizeof(cl_mem), (void *)&buffer_out);
701 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 2, sizeof(int), (void *)&width);
702 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 3, sizeof(int), (void *)&height);
703 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 4, sizeof(int), (void *)&mult);
704 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 5, sizeof(int), (void *)&clamp_lf);
706 }
707 if(err != CL_SUCCESS) return err;
708
709 uint8_t current_scale_type = scale_type(s, scales);
710 const float radius = sqf(equivalent_sigma_at_step(B_SPLINE_SIGMA, s * DS_FACTOR));
711 cl_mem reconstruct_write = (s == scales - 1)
712 ? reconstructed
713 : (reconstruct_read == reconstructed ? reconstructed_scratch : reconstructed);
714
715 // Keep the accumulation image read/write handles distinct at each scale.
716 // Some AMD OpenCL drivers get unstable when the same image is bound for both roles.
717 if(variant == DIFFUSE_RECONSTRUCT_RGB)
718 {
720 (void *)&buffer_in);
722 (void *)&buffer_out);
724 (void *)&clipping_mask);
726 (void *)&reconstruct_read);
728 (void *)&reconstruct_write);
729 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 5, sizeof(int), (void *)&width);
730 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 6, sizeof(int), (void *)&height);
731 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 7, sizeof(int), (void *)&mult);
733 (void *)&noise_level);
734 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 9, sizeof(int), (void *)&salt);
735 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 10, sizeof(uint8_t),
736 (void *)&current_scale_type);
737 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 11, sizeof(float), (void *)&radius);
739 if(err != CL_SUCCESS) return err;
740 }
741 else // DIFFUSE_RECONSTRUCT_CHROMA
742 {
743 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 0, sizeof(cl_mem), (void *)&buffer_in);
744 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 1, sizeof(cl_mem), (void *)&buffer_out);
745 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 2, sizeof(cl_mem),
746 (void *)&clipping_mask);
747 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 3, sizeof(cl_mem),
748 (void *)&reconstruct_read);
749 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 4, sizeof(cl_mem),
750 (void *)&reconstruct_write);
751 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 5, sizeof(int), (void *)&width);
752 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 6, sizeof(int), (void *)&height);
753 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 7, sizeof(int), (void *)&mult);
754 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 8, sizeof(uint8_t),
755 (void *)&current_scale_type);
756 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 9, sizeof(float), (void *)&solid_color);
758 if(err != CL_SUCCESS) return err;
759 }
760
761 reconstruct_read = reconstruct_write;
762 }
763
764 return err;
765}
766#endif // HAVE_OPENCL
767
768#ifdef HAVE_OPENCL
769
771 const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out,
772 const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out,
773 const dt_aligned_pixel_t clips)
774{
777
778 cl_int err = DT_OPENCL_DEFAULT_ERROR;
779
780 const int devid = pipe->devid;
781 const int width = roi_in->width;
782 const int height = roi_in->height;
783
784 const int ds_height = height / DS_FACTOR;
785 const int ds_width = width / DS_FACTOR;
786
787 size_t sizes[] = { ROUNDUPDWD(width, devid), ROUNDUPDHT(height, devid), 1 };
788 size_t ds_sizes[] = { ROUNDUPDWD(ds_width, devid), ROUNDUPDHT(ds_height, devid), 1 };
789
790 // kernel_highlights_normalize_reduce_first is self-correcting: it takes the raw filters
791 // below PLUS roi_in->x/y as separate kernel args and adds them itself. interpolate_and_mask
792 // and remosaic_and_replace have no roi offset args at all -- they need filters pre-shifted
793 // for roi_in's crop position instead (mirrors the CPU process_laplacian fix).
794 const uint32_t filters = piece->dsc_in.filters;
795 const uint32_t filters_shifted = dt_dev_get_roi_filters(piece, roi_in);
796
797 cl_mem interpolated
798 = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4); // [R, G, B, norm] for each pixel
799 cl_mem clipping_mask
800 = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4); // [R, G, B, norm] for each pixel
801 cl_mem normalization = NULL;
802 cl_mem normalization_tmp = NULL;
803 cl_mem normalization_partials = NULL;
804 cl_mem normalization_final = NULL;
805
806 // temp buffer for blurs. We will need to cycle between them for memory efficiency
807 cl_mem LF_odd = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
808 cl_mem LF_even = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
809 cl_mem temp
810 = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4); // need full size here for blurring
811
812 const float scale = DS_FACTOR * dt_dev_get_module_scale(pipe, roi_in);
813 const float final_radius = (float)((int)(1 << data->scales)) / scale;
814 const int scales = CLAMP((int)ceilf(log2f(final_radius)), 1, MAX_NUM_SCALES);
815
816 const float noise_level = data->noise_level / scale;
817
818 // wavelets scales buffers
819 cl_mem HF = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
820 cl_mem ds_interpolated = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
821 cl_mem ds_clipping_mask = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
822 cl_mem reconstructed_scratch = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
823 cl_mem clips_cl = dt_opencl_copy_host_to_device_constant(devid, 4 * sizeof(float), (float *)clips);
824
825 if(IS_NULL_PTR(interpolated) || IS_NULL_PTR(clipping_mask) || IS_NULL_PTR(LF_odd) || IS_NULL_PTR(LF_even)
826 || IS_NULL_PTR(temp) || IS_NULL_PTR(HF) || IS_NULL_PTR(ds_interpolated) || IS_NULL_PTR(ds_clipping_mask)
827 || IS_NULL_PTR(reconstructed_scratch) || IS_NULL_PTR(clips_cl))
828 goto error;
829
830 {
832 .xfactor = 1,
833 .yoffset = 0,
834 .yfactor = 1,
835 .cellsize = 4 * sizeof(float),
836 .overhead = 0,
837 .sizex = 1 << 4,
838 .sizey = 1 << 4 };
839
841
842 const size_t bwidth = ROUNDUP(width, flocopt.sizex);
843 const size_t bheight = ROUNDUP(height, flocopt.sizey);
844 const int bufsize = (int)((bwidth / flocopt.sizex) * (bheight / flocopt.sizey));
845
846 normalization_partials = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * (size_t)bufsize);
847 normalization = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
848 normalization_tmp = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
849 if(!normalization_partials || !normalization || !normalization_tmp) goto error;
850
851 size_t fsizes[3] = { bwidth, bheight, 1 };
852 size_t flocal[3] = { flocopt.sizex, flocopt.sizey, 1 };
853 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_first, 0, sizeof(cl_mem), &dev_in);
857 &normalization_partials);
858 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_first, 4, sizeof(uint32_t), &filters);
859 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_first, 5, sizeof(int), &roi_in->x);
860 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_first, 6, sizeof(int), &roi_in->y);
862 sizeof(float) * 4 * flocopt.sizex * flocopt.sizey, NULL);
864 flocal);
865 if(err != CL_SUCCESS) goto error;
866
868 .xfactor = 1,
869 .yoffset = 0,
870 .yfactor = 1,
871 .cellsize = 4 * sizeof(float),
872 .overhead = 0,
873 .sizex = 1 << 16,
874 .sizey = 1 };
875
877
878 int current_length = bufsize;
879 cl_mem reduce_in = normalization_partials;
880 cl_mem reduce_out = normalization;
881
882 while(TRUE)
883 {
884 const int reducesize = MIN(REDUCESIZE, ROUNDUP(current_length, slocopt.sizex) / slocopt.sizex);
885 size_t ssizes[3] = { (size_t)reducesize * slocopt.sizex, 1, 1 };
886 size_t slocal[3] = { slocopt.sizex, 1, 1 };
887 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_second, 0, sizeof(cl_mem), &reduce_in);
889 &reduce_out);
891 &current_length);
893 sizeof(float) * 4 * slocopt.sizex, NULL);
895 slocal);
896 if(err != CL_SUCCESS) goto error;
897
898 if(reducesize == 1) break;
899 current_length = reducesize;
900 cl_mem swap = reduce_in;
901 reduce_in = reduce_out;
902 reduce_out = (swap == normalization_partials) ? normalization_tmp : normalization;
903 }
904
905 normalization_final = reduce_out;
906 }
907
908 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_bilinear_and_mask, 0, sizeof(cl_mem), (void *)&dev_in);
910 (void *)&interpolated);
911 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_bilinear_and_mask, 2, sizeof(cl_mem), (void *)&temp);
912 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_bilinear_and_mask, 3, sizeof(cl_mem), (void *)&clips_cl);
914 (void *)&normalization_final);
916 (void *)&filters_shifted);
917 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_bilinear_and_mask, 6, sizeof(int), (void *)&roi_out->width);
919 (void *)&roi_out->height);
921 if(err != CL_SUCCESS) goto error;
922
923 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 0, sizeof(cl_mem), (void *)&temp);
924 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 1, sizeof(cl_mem), (void *)&clipping_mask);
925 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 2, sizeof(int), (void *)&roi_out->width);
926 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 3, sizeof(int), (void *)&roi_out->height);
928 if(err != CL_SUCCESS) goto error;
929
930 // Downsample
931 const int RGBa = TRUE;
932 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&clipping_mask);
933 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
934 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
935 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_clipping_mask);
936 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
937 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
938 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
940 if(err != CL_SUCCESS) goto error;
941
942 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&interpolated);
943 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
944 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
945 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_interpolated);
946 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
947 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
948 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
950 if(err != CL_SUCCESS) goto error;
951
952 for(int i = 0; i < data->iterations; i++)
953 {
954 // The pipe abandoned this run: stop between two of up to 512 iterations, once the previous one has
955 // run on the device (the host enqueues them all far ahead of it).
957 {
959 goto error;
960 }
961 const int salt = (i == data->iterations - 1); // add noise on the last iteration only
962 err = wavelets_process_cl(devid, ds_interpolated, temp, reconstructed_scratch, ds_clipping_mask, ds_sizes,
963 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_RGB,
964 noise_level, salt, data->solid_color);
965 if(err != CL_SUCCESS) goto error;
966
967 err = wavelets_process_cl(devid, temp, ds_interpolated, reconstructed_scratch, ds_clipping_mask, ds_sizes,
968 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_CHROMA,
969 noise_level, salt, data->solid_color);
970 if(err != CL_SUCCESS) goto error;
971 }
972
973 // Upsample
974 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&ds_interpolated);
975 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&ds_width);
976 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&ds_height);
977 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&interpolated);
978 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&width);
979 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&height);
981 if(err != CL_SUCCESS) goto error;
982
983 // Remosaic
984 const int clip_floor_off = FALSE; // 2021 mode keeps the historical blend
985 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 0, sizeof(cl_mem), (void *)&dev_in);
986 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 1, sizeof(cl_mem), (void *)&dev_in);
988 (void *)&interpolated);
990 (void *)&clipping_mask);
991 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 4, sizeof(cl_mem), (void *)&dev_out);
993 (void *)&normalization_final);
994 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 6, sizeof(cl_mem), (void *)&clips_cl);
996 (void *)&clip_floor_off);
998 (void *)&filters_shifted);
999 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 9, sizeof(int), (void *)&width);
1000 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 10, sizeof(int), (void *)&height);
1002 if(err != CL_SUCCESS) goto error;
1003
1004 // cleanup and exit on success
1006 dt_opencl_release_mem_object(normalization_partials);
1007 if(normalization_tmp != normalization_final) dt_opencl_release_mem_object(normalization_tmp);
1008 if(normalization != normalization_final) dt_opencl_release_mem_object(normalization);
1009 dt_opencl_release_mem_object(normalization_final);
1010 dt_opencl_release_mem_object(interpolated);
1011 dt_opencl_release_mem_object(clipping_mask);
1016 dt_opencl_release_mem_object(ds_clipping_mask);
1017 dt_opencl_release_mem_object(ds_interpolated);
1018 dt_opencl_release_mem_object(reconstructed_scratch);
1019 return err;
1020
1021error:
1023 dt_opencl_release_mem_object(normalization_partials);
1024 if(normalization_tmp != normalization_final) dt_opencl_release_mem_object(normalization_tmp);
1025 if(normalization != normalization_final) dt_opencl_release_mem_object(normalization);
1026 dt_opencl_release_mem_object(normalization_final);
1027 dt_opencl_release_mem_object(interpolated);
1028 dt_opencl_release_mem_object(clipping_mask);
1033 dt_opencl_release_mem_object(ds_clipping_mask);
1034 dt_opencl_release_mem_object(ds_interpolated);
1035 dt_opencl_release_mem_object(reconstructed_scratch);
1036
1037 dt_print(DT_DEBUG_OPENCL, "[opencl_highlights] couldn't enqueue kernel! %i\n", err);
1038 return err;
1039}
1040#endif // HAVE_OPENCL
1041
1042#ifdef HAVE_OPENCL
1043
1045 const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out,
1046 const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out,
1047 const dt_aligned_pixel_t clips)
1048{
1051
1052 cl_int err = DT_OPENCL_DEFAULT_ERROR;
1053
1054 const uint8_t(*const xtrans)[6] = (const uint8_t(*const)[6])piece->dsc_in.xtrans;
1055 const int devid = pipe->devid;
1056 const int width = roi_in->width;
1057 const int height = roi_in->height;
1058
1059 const int ds_height = height / DS_FACTOR;
1060 const int ds_width = width / DS_FACTOR;
1061
1062 size_t sizes[] = { ROUNDUPDWD(width, devid), ROUNDUPDHT(height, devid), 1 };
1063 size_t ds_sizes[] = { ROUNDUPDWD(ds_width, devid), ROUNDUPDHT(ds_height, devid), 1 };
1064
1065 cl_mem interpolated = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1066 cl_mem clipping_mask = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1067 cl_mem normalization = NULL;
1068 cl_mem normalization_tmp = NULL;
1069 cl_mem normalization_partials = NULL;
1070 cl_mem normalization_final = NULL;
1071 cl_mem LF_odd = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1072 cl_mem LF_even = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1073 cl_mem temp = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1074
1075 const float scale = DS_FACTOR * dt_dev_get_module_scale(pipe, roi_in);
1076 const float final_radius = (float)((int)(1 << data->scales)) / scale;
1077 const int scales = CLAMP((int)ceilf(log2f(final_radius)), 1, MAX_NUM_SCALES);
1078 const float noise_level = data->noise_level / scale;
1079
1080 cl_mem HF = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1081 cl_mem ds_interpolated = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1082 cl_mem ds_clipping_mask = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1083 cl_mem reconstructed_scratch = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1084
1085 cl_mem clips_cl = dt_opencl_copy_host_to_device_constant(devid, 4 * sizeof(float), (float *)clips);
1086 cl_mem dev_xtrans
1087 = dt_opencl_copy_host_to_device_constant(devid, sizeof(piece->dsc_in.xtrans), (void *)piece->dsc_in.xtrans);
1088 int32_t lookup[6][6][32] = { { { 0 } } };
1089 _build_xtrans_bilinear_lookup(lookup, roi_in, xtrans);
1090 cl_mem lookup_cl = dt_opencl_copy_host_to_device_constant(devid, sizeof(lookup), lookup);
1091
1092 if(IS_NULL_PTR(interpolated) || IS_NULL_PTR(clipping_mask) || IS_NULL_PTR(LF_odd) || IS_NULL_PTR(LF_even)
1093 || IS_NULL_PTR(temp) || IS_NULL_PTR(HF) || IS_NULL_PTR(ds_interpolated) || IS_NULL_PTR(ds_clipping_mask)
1094 || IS_NULL_PTR(reconstructed_scratch) || IS_NULL_PTR(clips_cl) || IS_NULL_PTR(dev_xtrans)
1095 || IS_NULL_PTR(lookup_cl))
1096 goto error;
1097
1098 {
1100 .xfactor = 1,
1101 .yoffset = 0,
1102 .yfactor = 1,
1103 .cellsize = 4 * sizeof(float),
1104 .overhead = 0,
1105 .sizex = 1 << 4,
1106 .sizey = 1 << 4 };
1107
1109 goto error;
1110
1111 const size_t bwidth = ROUNDUP(width, flocopt.sizex);
1112 const size_t bheight = ROUNDUP(height, flocopt.sizey);
1113 const int bufsize = (int)((bwidth / flocopt.sizex) * (bheight / flocopt.sizey));
1114
1115 normalization_partials = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * (size_t)bufsize);
1116 normalization = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
1117 normalization_tmp = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
1118 if(!normalization_partials || !normalization || !normalization_tmp) goto error;
1119
1120 size_t fsizes[3] = { bwidth, bheight, 1 };
1121 size_t flocal[3] = { flocopt.sizex, flocopt.sizey, 1 };
1123 &dev_in);
1127 &normalization_partials);
1129 &roi_in->x);
1131 &roi_in->y);
1133 &dev_xtrans);
1135 sizeof(float) * 4 * flocopt.sizex * flocopt.sizey, NULL);
1137 fsizes, flocal);
1138 if(err != CL_SUCCESS) goto error;
1139
1141 .xfactor = 1,
1142 .yoffset = 0,
1143 .yfactor = 1,
1144 .cellsize = 4 * sizeof(float),
1145 .overhead = 0,
1146 .sizex = 1 << 16,
1147 .sizey = 1 };
1148
1150
1151 int current_length = bufsize;
1152 cl_mem reduce_in = normalization_partials;
1153 cl_mem reduce_out = normalization;
1154
1155 while(TRUE)
1156 {
1157 const int reducesize = MIN(REDUCESIZE, ROUNDUP(current_length, slocopt.sizex) / slocopt.sizex);
1158 size_t ssizes[3] = { (size_t)reducesize * slocopt.sizex, 1, 1 };
1159 size_t slocal[3] = { slocopt.sizex, 1, 1 };
1160 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_second, 0, sizeof(cl_mem), &reduce_in);
1162 &reduce_out);
1164 &current_length);
1166 sizeof(float) * 4 * slocopt.sizex, NULL);
1168 slocal);
1169 if(err != CL_SUCCESS) goto error;
1170
1171 if(reducesize == 1) break;
1172 current_length = reducesize;
1173 cl_mem swap = reduce_in;
1174 reduce_in = reduce_out;
1175 reduce_out = (swap == normalization_partials) ? normalization_tmp : normalization;
1176 }
1177
1178 normalization_final = reduce_out;
1179 }
1180
1182 (void *)&dev_in);
1184 (void *)&interpolated);
1185 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_bilinear_and_mask_xtrans, 2, sizeof(cl_mem), (void *)&temp);
1187 (void *)&clips_cl);
1189 (void *)&normalization_final);
1193 (void *)&roi_in->x);
1195 (void *)&roi_in->y);
1197 (void *)&dev_xtrans);
1199 (void *)&lookup_cl);
1201 if(err != CL_SUCCESS) goto error;
1202
1203 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 0, sizeof(cl_mem), (void *)&temp);
1204 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 1, sizeof(cl_mem), (void *)&clipping_mask);
1205 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 2, sizeof(int), (void *)&roi_out->width);
1206 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 3, sizeof(int), (void *)&roi_out->height);
1208 if(err != CL_SUCCESS) goto error;
1209
1210 const int RGBa = TRUE;
1211 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&clipping_mask);
1212 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
1213 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
1214 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_clipping_mask);
1215 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
1216 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
1217 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
1218 err = dt_opencl_enqueue_kernel_2d(devid, gd->kernel_interpolate_bilinear, ds_sizes);
1219 if(err != CL_SUCCESS) goto error;
1220
1221 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&interpolated);
1222 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
1223 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
1224 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_interpolated);
1225 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
1226 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
1227 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
1228 err = dt_opencl_enqueue_kernel_2d(devid, gd->kernel_interpolate_bilinear, ds_sizes);
1229 if(err != CL_SUCCESS) goto error;
1230
1231 for(int i = 0; i < data->iterations; i++)
1232 {
1233 // The pipe abandoned this run: stop between two of up to 512 iterations, once the previous one has
1234 // run on the device (the host enqueues them all far ahead of it).
1236 {
1238 goto error;
1239 }
1240 const int salt = (i == data->iterations - 1);
1241 err = wavelets_process_cl(devid, ds_interpolated, temp, reconstructed_scratch, ds_clipping_mask, ds_sizes,
1242 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_RGB,
1243 noise_level, salt, data->solid_color);
1244 if(err != CL_SUCCESS) goto error;
1245
1246 err = wavelets_process_cl(devid, temp, ds_interpolated, reconstructed_scratch, ds_clipping_mask, ds_sizes,
1247 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_CHROMA,
1248 noise_level, salt, data->solid_color);
1249 if(err != CL_SUCCESS) goto error;
1250 }
1251
1252 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&ds_interpolated);
1253 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&ds_width);
1254 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&ds_height);
1255 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&interpolated);
1256 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&width);
1257 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&height);
1259 if(err != CL_SUCCESS) goto error;
1260
1261 const int clip_floor_off = FALSE; // 2021 mode keeps the historical blend
1263 (void *)&dev_in);
1265 (void *)&dev_in);
1267 (void *)&interpolated);
1269 (void *)&clipping_mask);
1271 (void *)&dev_out);
1273 (void *)&normalization_final);
1275 (void *)&clips_cl);
1277 (void *)&clip_floor_off);
1279 (void *)&width);
1281 (void *)&height);
1283 (void *)&roi_in->x);
1285 (void *)&roi_in->y);
1287 (void *)&dev_xtrans);
1289 if(err != CL_SUCCESS) goto error;
1292 dt_opencl_release_mem_object(dev_xtrans);
1293 dt_opencl_release_mem_object(normalization_partials);
1294 if(normalization_tmp != normalization_final) dt_opencl_release_mem_object(normalization_tmp);
1295 if(normalization != normalization_final) dt_opencl_release_mem_object(normalization);
1296 dt_opencl_release_mem_object(normalization_final);
1297 dt_opencl_release_mem_object(interpolated);
1298 dt_opencl_release_mem_object(clipping_mask);
1303 dt_opencl_release_mem_object(ds_clipping_mask);
1304 dt_opencl_release_mem_object(ds_interpolated);
1305 dt_opencl_release_mem_object(reconstructed_scratch);
1306 return err;
1307
1308error:
1311 dt_opencl_release_mem_object(dev_xtrans);
1312 dt_opencl_release_mem_object(normalization_partials);
1313 if(normalization_tmp != normalization_final) dt_opencl_release_mem_object(normalization_tmp);
1314 if(normalization != normalization_final) dt_opencl_release_mem_object(normalization);
1315 dt_opencl_release_mem_object(normalization_final);
1316 dt_opencl_release_mem_object(interpolated);
1317 dt_opencl_release_mem_object(clipping_mask);
1322 dt_opencl_release_mem_object(ds_clipping_mask);
1323 dt_opencl_release_mem_object(ds_interpolated);
1324 dt_opencl_release_mem_object(reconstructed_scratch);
1325
1326 dt_print(DT_DEBUG_OPENCL, "[opencl_highlights] couldn't enqueue kernel! %i\n", err);
1327 return err;
1328}
1330 const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out,
1331 const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out,
1332 const dt_aligned_pixel_t clips)
1333{
1334 // Non-raw / sRAW guided-laplacian, fully on-device: already-demosaiced RGB input, so the gather is a
1335 // plane copy (interpolate_and_mask_passthrough), the normalization sums all three colours per pixel
1336 // (highlights_normalize_reduce_first_passthrough), and the remosaic is a per-channel composite
1337 // (remosaic_and_replace_passthrough). Between them the downsample -> a-trous wavelets -> upsample is
1338 // the same CFA-agnostic device path as the Bayer/X-Trans drivers. No FC, no xtrans lookup, no roi phase.
1341
1342 cl_int err = DT_OPENCL_DEFAULT_ERROR;
1343
1344 const int devid = pipe->devid;
1345 const int width = roi_in->width;
1346 const int height = roi_in->height;
1347
1348 const int ds_height = height / DS_FACTOR;
1349 const int ds_width = width / DS_FACTOR;
1350
1351 size_t sizes[] = { ROUNDUPDWD(width, devid), ROUNDUPDHT(height, devid), 1 };
1352 size_t ds_sizes[] = { ROUNDUPDWD(ds_width, devid), ROUNDUPDHT(ds_height, devid), 1 };
1353
1354 cl_mem interpolated = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1355 cl_mem clipping_mask = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1356 cl_mem normalization = NULL;
1357 cl_mem normalization_tmp = NULL;
1358 cl_mem normalization_partials = NULL;
1359 cl_mem normalization_final = NULL;
1360
1361 cl_mem LF_odd = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1362 cl_mem LF_even = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1363 cl_mem temp = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1364
1365 const float scale = DS_FACTOR * dt_dev_get_module_scale(pipe, roi_in);
1366 const float final_radius = (float)((int)(1 << data->scales)) / scale;
1367 const int scales = CLAMP((int)ceilf(log2f(final_radius)), 1, MAX_NUM_SCALES);
1368 const float noise_level = data->noise_level / scale;
1369
1370 cl_mem HF = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1371 cl_mem ds_interpolated = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1372 cl_mem ds_clipping_mask = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1373 cl_mem reconstructed_scratch = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1374 cl_mem clips_cl = dt_opencl_copy_host_to_device_constant(devid, 4 * sizeof(float), (float *)clips);
1375
1376 if(IS_NULL_PTR(interpolated) || IS_NULL_PTR(clipping_mask) || IS_NULL_PTR(LF_odd) || IS_NULL_PTR(LF_even)
1377 || IS_NULL_PTR(temp) || IS_NULL_PTR(HF) || IS_NULL_PTR(ds_interpolated) || IS_NULL_PTR(ds_clipping_mask)
1378 || IS_NULL_PTR(reconstructed_scratch) || IS_NULL_PTR(clips_cl))
1379 goto error;
1380
1381 {
1383 .xfactor = 1,
1384 .yoffset = 0,
1385 .yfactor = 1,
1386 .cellsize = 4 * sizeof(float),
1387 .overhead = 0,
1388 .sizex = 1 << 4,
1389 .sizey = 1 << 4 };
1390
1392 goto error;
1393
1394 const size_t bwidth = ROUNDUP(width, flocopt.sizex);
1395 const size_t bheight = ROUNDUP(height, flocopt.sizey);
1396 const int bufsize = (int)((bwidth / flocopt.sizex) * (bheight / flocopt.sizey));
1397
1398 normalization_partials = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * (size_t)bufsize);
1399 normalization = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
1400 normalization_tmp = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
1401 if(!normalization_partials || !normalization || !normalization_tmp) goto error;
1402
1403 size_t fsizes[3] = { bwidth, bheight, 1 };
1404 size_t flocal[3] = { flocopt.sizex, flocopt.sizey, 1 };
1406 &dev_in);
1408 &width);
1410 &height);
1412 &normalization_partials);
1414 sizeof(float) * 4 * flocopt.sizex * flocopt.sizey, NULL);
1416 fsizes, flocal);
1417 if(err != CL_SUCCESS) goto error;
1418
1420 .xfactor = 1,
1421 .yoffset = 0,
1422 .yfactor = 1,
1423 .cellsize = 4 * sizeof(float),
1424 .overhead = 0,
1425 .sizex = 1 << 16,
1426 .sizey = 1 };
1427
1429
1430 int current_length = bufsize;
1431 cl_mem reduce_in = normalization_partials;
1432 cl_mem reduce_out = normalization;
1433
1434 while(TRUE)
1435 {
1436 const int reducesize = MIN(REDUCESIZE, ROUNDUP(current_length, slocopt.sizex) / slocopt.sizex);
1437 size_t ssizes[3] = { (size_t)reducesize * slocopt.sizex, 1, 1 };
1438 size_t slocal[3] = { slocopt.sizex, 1, 1 };
1440 &reduce_in);
1442 &reduce_out);
1444 &current_length);
1446 sizeof(float) * 4 * slocopt.sizex, NULL);
1448 slocal);
1449 if(err != CL_SUCCESS) goto error;
1450
1451 if(reducesize == 1) break;
1452 current_length = reducesize;
1453 cl_mem swap = reduce_in;
1454 reduce_in = reduce_out;
1455 reduce_out = (swap == normalization_partials) ? normalization_tmp : normalization;
1456 }
1457
1458 normalization_final = reduce_out;
1459 }
1460
1461 // gather: plane copy + per-channel clip mask (mask written to `temp`, then 5x5-boxed into clipping_mask)
1463 (void *)&dev_in);
1465 (void *)&interpolated);
1467 (void *)&temp);
1469 (void *)&clips_cl);
1471 (void *)&normalization_final);
1473 (void *)&roi_out->width);
1475 (void *)&roi_out->height);
1477 if(err != CL_SUCCESS) goto error;
1478
1479 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 0, sizeof(cl_mem), (void *)&temp);
1480 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 1, sizeof(cl_mem), (void *)&clipping_mask);
1481 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 2, sizeof(int), (void *)&roi_out->width);
1482 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 3, sizeof(int), (void *)&roi_out->height);
1484 if(err != CL_SUCCESS) goto error;
1485
1486 // Downsample
1487 const int RGBa = TRUE;
1488 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&clipping_mask);
1489 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
1490 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
1491 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_clipping_mask);
1492 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
1493 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
1494 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
1495 err = dt_opencl_enqueue_kernel_2d(devid, gd->kernel_interpolate_bilinear, ds_sizes);
1496 if(err != CL_SUCCESS) goto error;
1497
1498 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&interpolated);
1499 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
1500 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
1501 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_interpolated);
1502 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
1503 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
1504 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
1505 err = dt_opencl_enqueue_kernel_2d(devid, gd->kernel_interpolate_bilinear, ds_sizes);
1506 if(err != CL_SUCCESS) goto error;
1507
1508 for(int i = 0; i < data->iterations; i++)
1509 {
1510 // The pipe abandoned this run: stop between two of up to 512 iterations, once the previous one has
1511 // run on the device (the host enqueues them all far ahead of it).
1513 {
1515 goto error;
1516 }
1517 const int salt = (i == data->iterations - 1); // add noise on the last iteration only
1518 err = wavelets_process_cl(devid, ds_interpolated, temp, reconstructed_scratch, ds_clipping_mask, ds_sizes,
1519 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_RGB,
1520 noise_level, salt, data->solid_color);
1521 if(err != CL_SUCCESS) goto error;
1522
1523 err = wavelets_process_cl(devid, temp, ds_interpolated, reconstructed_scratch, ds_clipping_mask, ds_sizes,
1524 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_CHROMA,
1525 noise_level, salt, data->solid_color);
1526 if(err != CL_SUCCESS) goto error;
1527 }
1528
1529 // Upsample
1530 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&ds_interpolated);
1531 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&ds_width);
1532 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&ds_height);
1533 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&interpolated);
1534 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&width);
1535 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&height);
1537 if(err != CL_SUCCESS) goto error;
1538
1539 // Remosaic: per-channel composite (clip_is_floor FALSE, historical blend, mirrors the CPU path)
1540 const int clip_floor_off = FALSE;
1542 (void *)&dev_in);
1544 (void *)&dev_in);
1546 (void *)&interpolated);
1548 (void *)&clipping_mask);
1550 (void *)&dev_out);
1552 (void *)&normalization_final);
1554 (void *)&clips_cl);
1556 (void *)&clip_floor_off);
1558 (void *)&width);
1560 (void *)&height);
1562 if(err != CL_SUCCESS) goto error;
1563
1564error:
1566 dt_opencl_release_mem_object(normalization_partials);
1567 if(normalization_tmp != normalization_final) dt_opencl_release_mem_object(normalization_tmp);
1568 if(normalization != normalization_final) dt_opencl_release_mem_object(normalization);
1569 dt_opencl_release_mem_object(normalization_final);
1570 dt_opencl_release_mem_object(interpolated);
1571 dt_opencl_release_mem_object(clipping_mask);
1576 dt_opencl_release_mem_object(ds_clipping_mask);
1577 dt_opencl_release_mem_object(ds_interpolated);
1578 dt_opencl_release_mem_object(reconstructed_scratch);
1579 if(err != CL_SUCCESS)
1580 dt_print(DT_DEBUG_OPENCL, "[opencl_highlights] couldn't enqueue kernel! %i\n", err);
1581 return err;
1582}
1583#endif // HAVE_OPENCL
static void error(char *msg)
Definition ashift_lsd.c:202
#define TRUE
Definition ashift_lsd.c:162
#define FALSE
Definition ashift_lsd.c:158
#define MAX_NUM_SCALES
Definition atrous.c:98
int dt_box_mean(float *const buf, const size_t height, const size_t width, const int ch, const int radius, const unsigned iterations)
#define B_SPLINE_SIGMA
Definition bspline.h:38
static float equivalent_sigma_at_step(const float sigma, const unsigned int s)
Definition bspline.h:52
#define B_SPLINE_TO_LAPLACIAN
Definition bspline.h:49
static void decompose_2D_Bspline(const float *const restrict in, float *const restrict HF, float *const restrict LF, const size_t width, const size_t height, const int mult, float *const tempbuf, size_t padded_size)
Definition bspline.h:351
typedef void((*dt_cache_allocate_t)(void *userdata, dt_cache_entry_t *entry))
return vector dt_simd_set1(valid ?(scaling+NORM_MIN) :NORM_MIN)
static float lookup(read_only image2d_t lut, const float x)
const dt_colormatrix_t dt_aligned_pixel_t out
dt_store_simd_aligned(out, dt_mat3x4_mul_vec4(vin, dt_colormatrix_row_to_simd(matrix, 0), dt_colormatrix_row_to_simd(matrix, 1), dt_colormatrix_row_to_simd(matrix, 2)))
static const int row
#define BLUE
#define RED
#define GREEN
#define ALPHA
static float4 dt_noise_generator_simd(const dt_noise_distribution_t distribution, const float4 mu, const float4 param, uint state[4])
static unsigned int splitmix32(const unsigned long seed)
static float xoshiro128plus(uint state[4])
#define REDUCESIZE
Definition demosaic.c:111
static int dwt_interleave_rows(const int rowid, const int height, const int stride)
Definition dwt.h:93
static __DT_CLONE_TARGETS__ void interpolate_bilinear(const float *const restrict in, const size_t width_in, const size_t height_in, float *const restrict out, const size_t width_out, const size_t height_out, const size_t ch)
__DT_CLONE_TARGETS__ void _interpolate_and_mask_passthrough(const float *const restrict input, float *const restrict interpolated, float *const restrict clipping_mask, const dt_aligned_pixel_t clips, const dt_aligned_pixel_t white_balance, const size_t width, const size_t height)
Definition gather.c:424
__DT_CLONE_TARGETS__ void _compute_laplacian_normalization(const float *const restrict input, const dt_iop_roi_t *const roi_in, const uint32_t filters, const uint8_t(*const xtrans)[6], dt_aligned_pixel_t normalization)
Definition gather.c:223
__DT_CLONE_TARGETS__ void _remosaic_and_replace_xtrans(const float *const restrict input, const float *const restrict input_raw, const float *const restrict interpolated, const float *const restrict clipping_mask, float *const restrict output, const dt_aligned_pixel_t white_balance, const dt_aligned_pixel_t clips, const int clip_is_floor, const dt_iop_roi_t *const roi_in, const uint8_t(*const xtrans)[6], const size_t width, const size_t height)
Definition gather.c:488
__DT_CLONE_TARGETS__ void _interpolate_and_mask(const float *const restrict input, float *const restrict interpolated, float *const restrict clipping_mask, const dt_aligned_pixel_t clips_in, const dt_aligned_pixel_t det_scale, const dt_aligned_pixel_t white_balance, const uint32_t filters, const size_t width, const size_t height)
Definition gather.c:67
__DT_CLONE_TARGETS__ void _remosaic_and_replace(const float *const restrict input, const float *const restrict input_raw, const float *const restrict interpolated, const float *const restrict clipping_mask, float *const restrict output, const dt_aligned_pixel_t white_balance, const dt_aligned_pixel_t clips, const int clip_is_floor, const uint32_t filters, const size_t width, const size_t height)
Definition gather.c:457
__DT_CLONE_TARGETS__ void _interpolate_and_mask_xtrans(const float *const restrict input, float *const restrict interpolated, float *const restrict clipping_mask, const dt_aligned_pixel_t clips, const dt_aligned_pixel_t white_balance, const dt_iop_roi_t *const roi_in, const int32_t lookup[6][6][32], const uint8_t(*const xtrans)[6], const size_t width, const size_t height)
Definition gather.c:317
__DT_CLONE_TARGETS__ void _build_xtrans_bilinear_lookup(int32_t lookup[6][6][32], const dt_iop_roi_t *const roi_in, const uint8_t(*const xtrans)[6])
Definition gather.c:277
__DT_CLONE_TARGETS__ void _remosaic_and_replace_passthrough(const float *const restrict input, const float *const restrict input_raw, const float *const restrict interpolated, const float *const restrict clipping_mask, float *const restrict output, const dt_aligned_pixel_t white_balance, const dt_aligned_pixel_t clips, const int clip_is_floor, const size_t width, const size_t height)
Definition gather.c:514
static dt_hl_cfa_t _hl_cfa_strategy(const uint32_t filters)
Definition gather.h:74
dt_hl_cfa_t
Definition gather.h:67
@ HL_CFA_PASSTHROUGH
Definition gather.h:70
@ HL_CFA_BAYER
Definition gather.h:68
@ HL_CFA_XTRANS
Definition gather.h:69
uint32_t dt_dev_get_roi_filters(const dt_dev_pixelpipe_iop_t *const piece, const dt_iop_roi_t *const roi_in)
Definition imageop.c:139
float dt_dev_get_module_scale(const dt_dev_pixelpipe_t *const pipe, const dt_iop_roi_t *const roi_in)
Definition imageop.c:134
void *const ovoid
static __DT_CLONE_TARGETS__ void heat_PDE_diffusion(const float *const restrict high_freq, const float *const restrict low_freq, const float *const restrict clipping_mask, float *const restrict output, const size_t width, const size_t height, const int mult, const uint8_t scale, const float first_order_factor)
Definition laplacian.c:248
static int wavelets_process(const float *const restrict in, float *const restrict reconstructed, const float *const restrict clipping_mask, const size_t width, const size_t height, const int scales, float *const restrict HF, float *const restrict LF_odd, float *const restrict LF_even, const diffuse_reconstruct_variant_t variant, const float noise_level, const int salt, const float first_order_factor)
Definition laplacian.c:374
cl_int process_laplacian_bayer_cl(struct dt_iop_module_t *self, const dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out, const dt_aligned_pixel_t clips)
Definition laplacian.c:770
cl_int process_laplacian_xtrans_cl(struct dt_iop_module_t *self, const dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out, const dt_aligned_pixel_t clips)
Definition laplacian.c:1044
cl_int process_laplacian_passthrough_cl(struct dt_iop_module_t *self, const dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out, const dt_aligned_pixel_t clips)
Definition laplacian.c:1329
__DT_CLONE_TARGETS__ int process_laplacian(struct dt_iop_module_t *self, const dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece, const void *const restrict ivoid, void *const restrict ovoid, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out, const dt_aligned_pixel_t clips)
Definition laplacian.c:433
static __DT_CLONE_TARGETS__ void guide_laplacians(const float *const restrict high_freq, const float *const restrict low_freq, const float *const restrict clipping_mask, float *const restrict output, const size_t width, const size_t height, const int mult, const float noise_level, const int salt, const uint8_t scale, const float radius_sq)
Definition laplacian.c:85
static cl_int wavelets_process_cl(const int devid, cl_mem in, cl_mem reconstructed, cl_mem reconstructed_scratch, cl_mem clipping_mask, const size_t sizes[3], const int width, const int height, dt_iop_highlights_global_data_t *const gd, const int scales, cl_mem HF, cl_mem LF_odd, cl_mem LF_even, const diffuse_reconstruct_variant_t variant, const float noise_level, const int salt, const float solid_color)
Definition laplacian.c:584
static const dt_dev_pixelpipe_t * pipe
Definition layers.h:49
static const dt_dev_pixelpipe_t const dt_dev_pixelpipe_iop_t * piece
Definition layers.h:50
static void swap(float *x, float *y)
Definition lightroom.c:1023
@ DT_DEBUG_OPENCL
Definition logging.h:57
void dt_print(dt_debug_thread_t thread, const char *msg,...) __attribute__((format(printf
Print to stdout when thread is enabled, prefixed with seconds since startup.
float *const restrict const size_t k
#define IS_NULL_PTR(p)
C is way too permissive with !=, == and if(var) checks, which can mean too many things depending on w...
Definition macros.h:96
#define DT_ALIGNED_ARRAY
Align an object on a cacheline boundary, so AVX2 can load it whole.
Definition mem_alloc.h:80
#define DT_IS_ALIGNED(x)
Promise the compiler that x is already cacheline-aligned, and return it.
Definition mem_alloc.h:51
uint32_t width
Definition mipmap_cache.c:0
uint32_t height
Definition mipmap_cache.c:1
size_t size
Definition mipmap_cache.c:3
int dt_opencl_local_buffer_opt(const int devid, const int kernel, dt_opencl_local_buffer_t *factors)
Definition opencl.c:3786
int dt_opencl_enqueue_kernel_2d(const int dev, const int kernel, const size_t *sizes)
Definition opencl.c:2612
void * dt_opencl_alloc_device_buffer(const int devid, const size_t size)
Definition opencl.c:3028
void * dt_opencl_alloc_device(const int devid, const int width, const int height, const int bpp)
Definition opencl.c:2952
void * dt_opencl_copy_host_to_device_constant(const int devid, const size_t size, void *host)
Definition opencl.c:2808
int dt_opencl_set_kernel_arg(const int dev, const int kernel, const int num, const size_t size, const void *arg)
Definition opencl.c:2603
gboolean dt_opencl_finish(const int devid)
Definition opencl.c:1712
int dt_opencl_enqueue_kernel_2d_with_local(const int dev, const int kernel, const size_t *sizes, const size_t *local)
Definition opencl.c:2618
void dt_opencl_release_mem_object(cl_mem mem)
Definition opencl.c:2863
#define DT_OPENCL_DEFAULT_ERROR
Definition opencl.h:61
#define ROUNDUP(a, n)
Definition opencl.h:82
#define ROUNDUPDHT(a, b)
Definition opencl.h:86
#define ROUNDUPDWD(a, b)
Definition opencl.h:85
#define __OMP_PARALLEL_FOR__(...)
Definition openmp.h:95
#define dt_pixelpipe_cache_free_align(mem)
#define dt_pixelpipe_cache_alloc_align_float(pixels, pipe)
#define dt_pixelpipe_cache_alloc_perthread_float(n, padded_size)
static gboolean dt_dev_pixelpipe_has_shutdown(const dt_dev_pixelpipe_t *pipe)
#define eps
Definition rcd.c:87
static const dt_aligned_pixel_simd_t const dt_aligned_pixel_simd_t row1
Definition simd.h:190
DT_ALIGNED_PIXEL float dt_aligned_pixel_t[4]
Definition simd.h:53
#define for_each_channel(_var,...)
Definition simd.h:87
float dt_aligned_pixel_simd_t __attribute__((vector_size(16), aligned(16)))
Apply one channel's tone curve to each of the three colour channels, or pass the channel through unto...
Definition simd.h:55
static const dt_aligned_pixel_simd_t const dt_aligned_pixel_simd_t const dt_aligned_pixel_simd_t row2
Definition simd.h:191
#define for_four_channels(_var,...)
Definition simd.h:89
static const dt_aligned_pixel_simd_t row0
Definition simd.h:189
diffuse_reconstruct_variant_t
@ DIFFUSE_RECONSTRUCT_RGB
@ DIFFUSE_RECONSTRUCT_CHROMA
#define DS_FACTOR
const float uint32_t state[4]
const float sigma
const float const int flip
const float noise
dt_atrous_wavelets_scales_t scales
dt_iop_global_data_t * global_data
Definition imageop.h:337
Region of interest passed through the pixelpipe.
Definition format.h:49
int width
Definition format.h:50
int height
Definition format.h:50
#define __DT_CLONE_TARGETS__
#define MIN(a, b)
Definition thinplate.c:32
#define MAX(a, b)
Definition thinplate.c:29