Ansel 0.0
A darktable fork - bloat + design vision
Loading...
Searching...
No Matches
laplacian.c
Go to the documentation of this file.
1/*
2 This file is part of darktable,
3 Copyright (C) 2010 Bruce Guenter.
4 Copyright (C) 2010-2011 Henrik Andersson.
5 Copyright (C) 2010-2014, 2016 johannes hanika.
6 Copyright (C) 2010 Stuart Henderson.
7 Copyright (C) 2011 Antony Dovgal.
8 Copyright (C) 2011 Robert Bieber.
9 Copyright (C) 2011-2014, 2016, 2019 Tobias Ellinghaus.
10 Copyright (C) 2011-2012, 2014, 2016-2017 Ulrich Pegelow.
11 Copyright (C) 2012, 2015 Edouard Gomez.
12 Copyright (C) 2012 Jérémy Rosen.
13 Copyright (C) 2012 Richard Wonka.
14 Copyright (C) 2013, 2020 Aldric Renaudin.
15 Copyright (C) 2014, 2016 Dan Torop.
16 Copyright (C) 2014-2016 Roman Lebedev.
17 Copyright (C) 2015-2016 Pedro Côrte-Real.
18 Copyright (C) 2017 Heiko Bauke.
19 Copyright (C) 2017 luzpaz.
20 Copyright (C) 2018, 2020-2026 Aurélien PIERRE.
21 Copyright (C) 2018 Edgardo Hoszowski.
22 Copyright (C) 2018 Maurizio Paglia.
23 Copyright (C) 2018-2020, 2022 Pascal Obry.
24 Copyright (C) 2018 rawfiner.
25 Copyright (C) 2019 Andreas Schneider.
26 Copyright (C) 2019 Diederik ter Rahe.
27 Copyright (C) 2019-2020, 2022 Hanno Schwalm.
28 Copyright (C) 2020 Chris Elston.
29 Copyright (C) 2020, 2022 Diederik Ter Rahe.
30 Copyright (C) 2020-2021 Ralf Brown.
31 Copyright (C) 2021 Hubert Kowalski.
32 Copyright (C) 2022 Martin Bařinka.
33 Copyright (C) 2022 Philipp Lutz.
34 Copyright (C) 2022 Victor Forsiuk.
35 Copyright (C) 2023 Alynx Zhou.
36 Copyright (C) 2023 Guillaume Stutin.
37 Copyright (C) 2023 Luca Zulberti.
38
39 darktable is free software: you can redistribute it and/or modify
40 it under the terms of the GNU General Public License as published by
41 the Free Software Foundation, either version 3 of the License, or
42 (at your option) any later version.
43
44 darktable is distributed in the hope that it will be useful,
45 but WITHOUT ANY WARRANTY; without even the implied warranty of
46 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
47 GNU General Public License for more details.
48
49 You should have received a copy of the GNU General Public License
50 along with darktable. If not, see <http://www.gnu.org/licenses/>.
51 */
52
53// Guided-laplacian (2021 a-trous) highlight reconstruction, CPU + OpenCL. (implementation; see laplacian.h for the
54// public API.)
55
56#include "pixel/box_filters.h"
58#include "pixel/bspline.h"
59#include "common/logging.h"
60#include "system/macros.h"
61#include "system/mem_alloc.h"
62#include "system/openmp.h"
63#include "system/simd.h"
66#include "pixel/dwt.h"
68#include "common/opencl.h"
70#include "iop/noise_generator.h"
73#include <math.h>
74#include <string.h>
75
76static inline __attribute__((always_inline)) uint8_t scale_type(const int s, const int scales)
77{
78 uint8_t scale = ANY_SCALE;
79 if(s == 0) scale |= FIRST_SCALE;
80 if(s == scales - 1) scale |= LAST_SCALE;
81 return scale;
82}
83
85static inline void guide_laplacians(const float *const restrict high_freq, const float *const restrict low_freq,
86 const float *const restrict clipping_mask, float *const restrict output,
87 const size_t width, const size_t height, const int mult,
88 const float noise_level, const int salt, const uint8_t scale,
89 const float radius_sq)
90{
91 float *const restrict out = DT_IS_ALIGNED(output);
92 const float *const restrict LF = DT_IS_ALIGNED(low_freq);
93 const float *const restrict HF = DT_IS_ALIGNED(high_freq);
94 const dt_aligned_pixel_simd_t zero = dt_simd_set1(0.f);
95 const dt_aligned_pixel_simd_t ones = dt_simd_set1(1.f);
96 const dt_aligned_pixel_simd_t inv_patch = dt_simd_set1(1.f / 9.f);
97 const dt_aligned_pixel_simd_t scale_multiplier = dt_simd_set1(1.f / radius_sq);
98 const float eps = 1e-12f;
100 for(size_t row = 0; row < height; ++row)
101 {
102 // interleave the order in which we process the rows so that we minimize cache misses
103 const int i = dwt_interleave_rows(row, height, mult);
104 const float *const row0 = HF + 4 * ((size_t)MAX(i - mult, 0) * width);
105 const float *const row1 = HF + 4 * ((size_t)i * width);
106 const float *const row2 = HF + 4 * ((size_t)MIN(i + mult, (int)height - 1) * width);
107 const float *const rows[3] = { row0, row1, row2 };
108 const int max_col = (int)width - 1;
109
110 for(int j = 0; j < width; ++j)
111 {
112 const size_t idx = (i * width + j);
113 const size_t index = idx * 4;
114 const float alpha = clipping_mask[index + ALPHA];
115 const float alpha_comp = 1.f - alpha;
116 dt_aligned_pixel_simd_t high_frequency = dt_load_simd_aligned(HF + index);
117
118 if(alpha > 0.f) // reconstruct
119 {
120 const int col_offsets[3] = { 4 * MAX(j - mult, 0), 4 * j, 4 * MIN(j + mult, max_col) };
121 dt_aligned_pixel_simd_t sum = zero;
122 dt_aligned_pixel_simd_t sum_sq = zero;
123 dt_aligned_pixel_simd_t prod_r = zero;
124 dt_aligned_pixel_simd_t prod_g = zero;
125 dt_aligned_pixel_simd_t prod_b = zero;
126
127 // Walk the dense 3x3 neighbourhood as counted loops so GCC keeps the
128 // fit as a regular reduction instead of fully unrolling all 9 taps and
129 // spilling the intermediate moments to the stack.
130#if defined(__GNUC__) && !defined(__clang__)
131#pragma GCC unroll 1
132#endif
133 for(int jj = 0; jj < 3; ++jj)
134 {
135 const float *const row_ptr = rows[jj];
136#if defined(__GNUC__) && !defined(__clang__)
137#pragma GCC unroll 1
138#endif
139 for(int ii = 0; ii < 3; ++ii)
140 {
141 const dt_aligned_pixel_simd_t sample = dt_load_simd_aligned(row_ptr + col_offsets[ii]);
142
143 sum += sample;
144 sum_sq += sample * sample;
145 prod_r += sample * dt_simd_set1(sample[RED]);
146 prod_g += sample * dt_simd_set1(sample[GREEN]);
147 prod_b += sample * dt_simd_set1(sample[BLUE]);
148 }
149 }
150
151 dt_aligned_pixel_simd_t means = sum * inv_patch;
152 dt_aligned_pixel_simd_t variance = sum_sq * inv_patch - means * means;
153 variance = dt_simd_max_zero(variance);
154 variance[ALPHA] = 0.f;
155
156 size_t guiding_channel = RED;
157 float guide_variance = variance[RED];
158 if(variance[GREEN] > guide_variance)
159 {
160 guiding_channel = GREEN;
161 guide_variance = variance[GREEN];
162 }
163 if(variance[BLUE] > guide_variance)
164 {
165 guiding_channel = BLUE;
166 guide_variance = variance[BLUE];
167 }
168
169 if(guide_variance > eps)
170 {
171 const float guide_mean = means[guiding_channel];
172 dt_aligned_pixel_simd_t covariance
173 = (guiding_channel == RED ? prod_r : (guiding_channel == GREEN ? prod_g : prod_b)) * inv_patch
174 - means * dt_simd_set1(guide_mean);
175 dt_aligned_pixel_simd_t slope = covariance / dt_simd_set1(guide_variance);
176 slope = dt_simd_max_zero(slope);
177 dt_aligned_pixel_simd_t intercept = means - slope * dt_simd_set1(guide_mean);
178 const dt_aligned_pixel_simd_t blend = dt_load_simd_aligned(clipping_mask + index) * scale_multiplier;
179 const dt_aligned_pixel_simd_t guide = dt_simd_set1(high_frequency[guiding_channel]);
180 high_frequency = blend * (slope * guide + intercept) + (ones - blend) * high_frequency;
181 }
182 }
183
184 dt_aligned_pixel_simd_t out_pixel = high_frequency;
185 if((scale & FIRST_SCALE))
186 {
187 // out is not inited yet
188 }
189 else
190 {
191 // just accumulate HF
192 out_pixel += dt_load_simd_aligned(out + index);
193 }
194
195 if((scale & LAST_SCALE))
196 {
197 // add the residual and clamp
198 out_pixel = dt_simd_max_zero(out_pixel + dt_load_simd_aligned(LF + index));
199 }
200
201 // Last step of RGB reconstruct : add noise
202 if((scale & LAST_SCALE) && salt && alpha > 0.f)
203 {
204 // Init random number generator
205 uint32_t DT_ALIGNED_ARRAY state[4]
206 = { splitmix32(j + 1), splitmix32((j + 1) * (i + 3)), splitmix32(1337), splitmix32(666) };
211
212 dt_aligned_pixel_t noise = { 0.f };
213 dt_aligned_pixel_t sigma = { 0.20f };
214 const int DT_ALIGNED_ARRAY flip[4] = { TRUE, FALSE, TRUE, FALSE };
215
216 sigma[RED] = out_pixel[RED] * noise_level;
217 sigma[GREEN] = out_pixel[GREEN] * noise_level;
218 sigma[BLUE] = out_pixel[BLUE] * noise_level;
219 sigma[ALPHA] = out_pixel[ALPHA] * noise_level;
220
221 // create statistical noise
222 dt_aligned_pixel_t current = { out_pixel[RED], out_pixel[GREEN], out_pixel[BLUE], out_pixel[ALPHA] };
224
225 // Save the noisy interpolated image
226 for_each_channel(c, aligned(noise, current)) noise[c] = current[c] + fabsf(noise[c] - current[c]);
227
228 out_pixel[RED] = fmaxf(alpha * noise[RED] + alpha_comp * current[RED], 0.f);
229 out_pixel[GREEN] = fmaxf(alpha * noise[GREEN] + alpha_comp * current[GREEN], 0.f);
230 out_pixel[BLUE] = fmaxf(alpha * noise[BLUE] + alpha_comp * current[BLUE], 0.f);
231 out_pixel[ALPHA] = fmaxf(alpha * noise[ALPHA] + alpha_comp * current[ALPHA], 0.f);
232 }
233
234 if((scale & LAST_SCALE))
235 {
236 // Break the RGB channels into ratios/norm for the next step of reconstruction
237 const float norm = fmaxf(sqrtf(sqf(out_pixel[RED]) + sqf(out_pixel[GREEN]) + sqf(out_pixel[BLUE])), 1e-6f);
238 out_pixel /= dt_simd_set1(norm);
239 out_pixel[ALPHA] = norm;
240 }
241
242 dt_store_simd_aligned(out + index, out_pixel);
243 }
244 }
245}
246
248static inline void heat_PDE_diffusion(const float *const restrict high_freq, const float *const restrict low_freq,
249 const float *const restrict clipping_mask, float *const restrict output,
250 const size_t width, const size_t height, const int mult, const uint8_t scale,
251 const float first_order_factor)
252{
253 // Simultaneous inpainting for image structure and texture using anisotropic heat transfer model
254 // https://www.researchgate.net/publication/220663968
255 // modified as follow :
256 // * apply it in a multi-scale wavelet setup : we basically solve it twice, on the wavelets LF and HF layers.
257 // * replace the manual texture direction/distance selection by an automatic detection similar to the structure
258 // one,
259 // * generalize the framework for isotropic diffusion and anisotropic weighted on the isophote direction
260 // * add a variance regularization to better avoid edges.
261 // The sharpness setting mimics the contrast equalizer effect by simply multiplying the HF by some gain.
262
263 float *const restrict out = DT_IS_ALIGNED(output);
264 const float *const restrict LF = DT_IS_ALIGNED(low_freq);
265 const float *const restrict HF = DT_IS_ALIGNED(high_freq);
267 for(size_t row = 0; row < height; ++row)
268 {
269 // interleave the order in which we process the rows so that we minimize cache misses
270 const size_t i = dwt_interleave_rows(row, height, mult);
271 // compute the 'above' and 'below' coordinates, clamping them to the image, once for the entire row
272 const size_t i_neighbours[3] = { MAX((int)(i - mult), (int)0) * width, // x - mult
273 i * width, // x
274 MIN((int)(i + mult), (int)height - 1) * width }; // x + mult
275
276 static const float DT_ALIGNED_ARRAY anisotropic_kernel_isophote[9]
277 = { 0.25f, 0.5f, 0.25f, 0.5f, -3.f, 0.5f, 0.25f, 0.5f, 0.25f };
278
279 for(size_t j = 0; j < width; ++j)
280 {
281 const size_t idx = (i * width + j);
282 const size_t index = idx * 4;
283
284 // fetch the clipping mask opacity : opaque (alpha = 100 %) where clipped
285 const dt_aligned_pixel_t alpha = { clipping_mask[index + RED], clipping_mask[index + GREEN],
286 clipping_mask[index + BLUE], clipping_mask[index + ALPHA] };
287
288 dt_aligned_pixel_t high_frequency = { HF[index + 0], HF[index + 1], HF[index + 2], HF[index + 3] };
289
290 // The for_each_channel macro uses 4 floats SIMD instructions or 3 float regular ops,
291 // depending on system. Since we don't want to diffuse the norm, make sure to store and restore it later.
292 // This is not much of an issue when processing image at full-res, but more harmful since
293 // we reconstruct highlights on a downscaled variant
294 const float norm_backup = high_frequency[3];
295
296 if(alpha[ALPHA] > 0.f) // reconstruct
297 {
298 // non-local neighbours coordinates
299 const size_t j_neighbours[3] = { MAX((int)(j - mult), (int)0), // y - mult
300 j, // y
301 MIN((int)(j + mult), (int)width - 1) }; // y + mult
302
303 // fetch non-local pixels and store them locally and contiguously
304 dt_aligned_pixel_t neighbour_pixel_HF[9];
305 for_four_channels(c, aligned(neighbour_pixel_HF, HF : 16))
306 {
307 neighbour_pixel_HF[3 * 0 + 0][c] = HF[4 * (i_neighbours[0] + j_neighbours[0]) + c];
308 neighbour_pixel_HF[3 * 0 + 1][c] = HF[4 * (i_neighbours[0] + j_neighbours[1]) + c];
309 neighbour_pixel_HF[3 * 0 + 2][c] = HF[4 * (i_neighbours[0] + j_neighbours[2]) + c];
310
311 neighbour_pixel_HF[3 * 1 + 0][c] = HF[4 * (i_neighbours[1] + j_neighbours[0]) + c];
312 neighbour_pixel_HF[3 * 1 + 1][c] = HF[4 * (i_neighbours[1] + j_neighbours[1]) + c];
313 neighbour_pixel_HF[3 * 1 + 2][c] = HF[4 * (i_neighbours[1] + j_neighbours[2]) + c];
314
315 neighbour_pixel_HF[3 * 2 + 0][c] = HF[4 * (i_neighbours[2] + j_neighbours[0]) + c];
316 neighbour_pixel_HF[3 * 2 + 1][c] = HF[4 * (i_neighbours[2] + j_neighbours[1]) + c];
317 neighbour_pixel_HF[3 * 2 + 2][c] = HF[4 * (i_neighbours[2] + j_neighbours[2]) + c];
318 }
319
320 // Compute the laplacian in the direction parallel to the steepest gradient on the norm
321 // Convolve the filter to get the laplacian
322 dt_aligned_pixel_t laplacian_HF = { 0.f, 0.f, 0.f, 0.f };
323 for(int k = 0; k < 9; k++)
324 {
325 for_each_channel(c, aligned(laplacian_HF, neighbour_pixel_HF : 16)
326 aligned(anisotropic_kernel_isophote : 64)) laplacian_HF[c]
327 += neighbour_pixel_HF[k][c] * anisotropic_kernel_isophote[k];
328 }
329
330 // Diffuse
331 const dt_aligned_pixel_t multipliers_HF
333 for_each_channel(c, aligned(high_frequency, multipliers_HF, laplacian_HF, alpha)) high_frequency[c]
334 += alpha[c] * multipliers_HF[c] * (laplacian_HF[c] - first_order_factor * high_frequency[c]);
335
336 // Restore. See above.
337 high_frequency[3] = norm_backup;
338 }
339
340 if((scale & FIRST_SCALE))
341 {
342 // out is not inited yet
343 for_each_channel(c, aligned(out, high_frequency : 64)) out[index + c] = high_frequency[c];
344 }
345 else
346 {
347 // just accumulate HF
348 for_each_channel(c, aligned(out, high_frequency : 64)) out[index + c] += high_frequency[c];
349 }
350
351 if((scale & LAST_SCALE))
352 {
353 // add the residual and clamp
354 for_each_channel(c, aligned(out, LF, high_frequency : 64)) out[index + c]
355 = fmaxf(out[index + c] + LF[index + c], 0.f);
356
357 // renormalize ratios
358 if(alpha[ALPHA] > 0.f)
359 {
360 const float norm = sqrtf(sqf(out[index + RED]) + sqf(out[index + GREEN]) + sqf(out[index + BLUE]));
361 for_each_channel(c, aligned(out, LF, high_frequency : 64)) out[index + c]
362 /= (c != ALPHA && norm > 1e-4f) ? norm : 1.f;
363 }
364
365 // Last scale : reconstruct RGB from ratios and norm - norm stays in the 4th channel
366 // we need it to evaluate the gradient
367 for_four_channels(c, aligned(out)) out[index + c]
368 = (c == ALPHA) ? out[index + ALPHA] : out[index + c] * out[index + ALPHA];
369 }
370 }
371 }
372}
373
374static inline int wavelets_process(const float *const restrict in, float *const restrict reconstructed,
375 const float *const restrict clipping_mask, const size_t width,
376 const size_t height, const int scales, float *const restrict HF,
377 float *const restrict LF_odd, float *const restrict LF_even,
378 const diffuse_reconstruct_variant_t variant, const float noise_level,
379 const int salt, const float first_order_factor)
380{
381 // À trous decimated wavelet decompose
382 // there is a paper from a guy we know that explains it : https://jo.dreggn.org/home/2010_atrous.pdf
383 // the wavelets decomposition here is the same as the equalizer/atrous module,
384
385 // allocate a one-row temporary buffer for the decomposition
386 size_t padded_size;
387 float *const tempbuf = dt_pixelpipe_cache_alloc_perthread_float(4 * width, &padded_size); // TODO: alloc in caller
388 if(IS_NULL_PTR(tempbuf)) return 1;
389
390 for(int s = 0; s < scales; ++s)
391 {
392 // fprintf(stderr, "CPU Wavelet decompose : scale %i\n", s);
393 const int mult = 1 << s;
394
395 const float *restrict buffer_in;
396 float *restrict buffer_out;
397
398 if(s == 0)
399 {
400 buffer_in = in;
401 buffer_out = LF_odd;
402 }
403 else if(s % 2 != 0)
404 {
405 buffer_in = LF_odd;
406 buffer_out = LF_even;
407 }
408 else
409 {
410 buffer_in = LF_even;
411 buffer_out = LF_odd;
412 }
413
414 decompose_2D_Bspline(buffer_in, HF, buffer_out, width, height, mult, tempbuf, padded_size);
415
416 uint8_t current_scale_type = scale_type(s, scales);
417 const float radius = sqf(equivalent_sigma_at_step(B_SPLINE_SIGMA, s * DS_FACTOR));
418
419 if(variant == DIFFUSE_RECONSTRUCT_RGB)
420 guide_laplacians(HF, buffer_out, clipping_mask, reconstructed, width, height, mult, noise_level, salt,
421 current_scale_type, radius);
422 else
423 heat_PDE_diffusion(HF, buffer_out, clipping_mask, reconstructed, width, height, mult, current_scale_type,
424 first_order_factor);
425
426 }
428
429 return 0;
430}
431
434 const dt_dev_pixelpipe_iop_t *piece, const void *const restrict ivoid,
435 void *const restrict ovoid, const dt_iop_roi_t *const roi_in,
436 const dt_iop_roi_t *const roi_out, const dt_aligned_pixel_t clips)
437{
439 int err = 0;
440 (void)roi_out;
441
442 // Every CFA helper below (normalization, Bayer gather, remosaic) reads FC(row, col, filters) with
443 // tile-local row/col (0-based within this buffer, no roi offset added), so filters must be
444 // pre-shifted for roi_in's crop position here -- mirrors demosaic.c's tile-local algorithms.
445 // dt_dev_get_roi_filters() returns the shifted Bayer word, 9u unchanged for X-Trans, and 0 for
446 // already-demosaiced (non-raw / sRAW) input. Only the gather and the remosaic branch on `cfa`; the
447 // wavelet reconstruction between them is CFA-agnostic.
448 const uint32_t filters = dt_dev_get_roi_filters(piece, roi_in);
449 const dt_hl_cfa_t cfa = _hl_cfa_strategy(filters);
450 const uint8_t(*const xtrans)[6]
451 = (cfa == HL_CFA_XTRANS) ? (const uint8_t(*const)[6])piece->dsc_in.xtrans : NULL;
452
453 const size_t height = roi_in->height;
454 const size_t width = roi_in->width;
455 const size_t size = roi_in->width * roi_in->height;
456
457 const size_t ds_height = height / DS_FACTOR;
458 const size_t ds_width = width / DS_FACTOR;
459 const size_t ds_size = ds_height * ds_width;
460
461 float *const restrict interpolated
462 = dt_pixelpipe_cache_alloc_align_float(size * 4, pipe); // [R, G, B, norm] for each pixel
463 float *const restrict clipping_mask
464 = dt_pixelpipe_cache_alloc_align_float(size * 4, pipe); // [R, G, B, norm] for each pixel
465
466 // temp buffer for blurs. We will need to cycle between them for memory efficiency
467 float *const restrict LF_odd = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
468 float *const restrict LF_even = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
469 float *const restrict temp = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
470
471 const float scale = DS_FACTOR * dt_dev_get_module_scale(pipe, roi_in);
472 const float final_radius = (float)((int)(1 << data->scales)) / scale;
473 const int scales = CLAMP((int)ceilf(log2f(final_radius)), 1, MAX_NUM_SCALES);
474
475 const float noise_level = data->noise_level / scale;
476
477 // wavelets scales buffers
478 float *restrict HF = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
479 float *restrict ds_interpolated = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
480 float *restrict ds_clipping_mask = dt_pixelpipe_cache_alloc_align_float(ds_size * 4, pipe);
481
482 if(IS_NULL_PTR(interpolated) || IS_NULL_PTR(clipping_mask) || IS_NULL_PTR(LF_odd) || IS_NULL_PTR(LF_even)
483 || IS_NULL_PTR(temp) || IS_NULL_PTR(HF) || IS_NULL_PTR(ds_interpolated) || IS_NULL_PTR(ds_clipping_mask))
484 {
485 err = 1;
486 goto error;
487 }
488
489 const float *const restrict input = (const float *const restrict)ivoid;
490 float *const restrict output = (float *const restrict)ovoid;
491 dt_aligned_pixel_t normalization = { 1.f, 1.f, 1.f, 1.f };
492 _compute_laplacian_normalization(input, roi_in, filters, xtrans, normalization);
493
494 // FLOW step 1a (gather): the only CFA-branching endpoint before the shared wavelet reconstruction.
495 // Guided laplacians run without the knee, so det_scale is unit and both CFA gathers take `clips` directly.
496 switch(cfa)
497 {
498 case HL_CFA_BAYER:
499 {
500 const dt_aligned_pixel_t det_unit = { 1.f, 1.f, 1.f, 1.f };
501 _interpolate_and_mask(input, interpolated, clipping_mask, clips, det_unit, normalization, filters, width,
502 height);
503 break;
504 }
505 case HL_CFA_XTRANS:
506 {
507 int32_t lookup[6][6][32] = { { { 0 } } };
508 _build_xtrans_bilinear_lookup(lookup, roi_in, xtrans);
509 _interpolate_and_mask_xtrans(input, interpolated, clipping_mask, clips, normalization, roi_in, lookup,
510 xtrans, width, height);
511 break;
512 }
514 // Non-raw / sRAW: no demosaic, just copy the RGB planes through + build masks.
515 _interpolate_and_mask_passthrough(input, interpolated, clipping_mask, clips, normalization, width, height);
516 break;
517 }
518 if(dt_box_mean(clipping_mask, height, width, 4, 2, 1) != 0)
519 {
520 err = 1;
521 goto error;
522 }
523
524 // Downsample
525 interpolate_bilinear(clipping_mask, width, height, ds_clipping_mask, ds_width, ds_height, 4);
526 interpolate_bilinear(interpolated, width, height, ds_interpolated, ds_width, ds_height, 4);
527
528 for(int i = 0; i < data->iterations; i++)
529 {
530 const int salt = (i == data->iterations - 1); // add noise on the last iteration only
531 if(wavelets_process(ds_interpolated, temp, ds_clipping_mask, ds_width, ds_height, scales, HF, LF_odd, LF_even,
532 DIFFUSE_RECONSTRUCT_RGB, noise_level, salt, data->solid_color))
533 {
534 err = 1;
535 goto error;
536 }
537 if(wavelets_process(temp, ds_interpolated, ds_clipping_mask, ds_width, ds_height, scales, HF, LF_odd, LF_even,
538 DIFFUSE_RECONSTRUCT_CHROMA, noise_level, salt, data->solid_color))
539 {
540 err = 1;
541 goto error;
542 }
543 }
544
545 // Upsample
546 interpolate_bilinear(ds_interpolated, ds_width, ds_height, interpolated, width, height, 4);
547 // FLOW: remosaic + composite -- second and last CFA-branching endpoint (clip_is_floor = FALSE here).
548 switch(cfa)
549 {
550 case HL_CFA_BAYER:
551 _remosaic_and_replace(input, input, interpolated, clipping_mask, output, normalization, clips, FALSE,
552 filters, width, height);
553 break;
554 case HL_CFA_XTRANS:
555 _remosaic_and_replace_xtrans(input, input, interpolated, clipping_mask, output, normalization, clips,
556 FALSE, roi_in, xtrans, width, height);
557 break;
559 // Non-raw / sRAW: composite the reconstructed RGB straight back, per channel (clip_is_floor FALSE).
560 _remosaic_and_replace_passthrough(input, input, interpolated, clipping_mask, output, normalization, clips,
561 FALSE, width, height);
562 break;
563 }
564
565error:;
566 dt_pixelpipe_cache_free_align(interpolated);
567 dt_pixelpipe_cache_free_align(clipping_mask);
572 dt_pixelpipe_cache_free_align(ds_interpolated);
573 dt_pixelpipe_cache_free_align(ds_clipping_mask);
574 return err;
575}
576
577#ifdef HAVE_OPENCL
578static inline cl_int wavelets_process_cl(const int devid, cl_mem in, cl_mem reconstructed,
579 cl_mem reconstructed_scratch, cl_mem clipping_mask, const size_t sizes[3],
580 const int width, const int height,
581 dt_iop_highlights_global_data_t *const gd, const int scales, cl_mem HF,
582 cl_mem LF_odd, cl_mem LF_even,
583 const diffuse_reconstruct_variant_t variant, const float noise_level,
584 const int salt, const float solid_color)
585{
586 cl_int err = DT_OPENCL_DEFAULT_ERROR;
587 cl_mem reconstruct_read = reconstructed_scratch;
588
589 // À trous wavelet decompose
590 // there is a paper from a guy we know that explains it : https://jo.dreggn.org/home/2010_atrous.pdf
591 // the wavelets decomposition here is the same as the equalizer/atrous module,
592 for(int s = 0; s < scales; ++s)
593 {
594 // fprintf(stderr, "GPU Wavelet decompose : scale %i\n", s);
595 const int mult = 1 << s;
596
597 cl_mem buffer_in;
598 cl_mem buffer_out;
599
600 if(s == 0)
601 {
602 buffer_in = in;
603 buffer_out = LF_odd;
604 }
605 else if(s % 2 != 0)
606 {
607 buffer_in = LF_odd;
608 buffer_out = LF_even;
609 }
610 else
611 {
612 buffer_in = LF_even;
613 buffer_out = LF_odd;
614 }
615
616 // Compute wavelets low-frequency scales
617 const int clamp_lf = 1;
618 int hblocksize;
620 .xfactor = 1,
621 .yoffset = 0,
622 .yfactor = 1,
623 .cellsize = 4 * sizeof(float),
624 .overhead = 0,
625 .sizex = 1 << 16,
626 .sizey = 1 };
628 hblocksize = hlocopt.sizex;
629 else
630 hblocksize = 1;
631
632 if(hblocksize > 1)
633 {
634 const size_t horizontal_sizes[3] = { ROUNDUP(width, hblocksize), ROUNDUPDHT(height, devid), 1 };
635 const size_t horizontal_local[3] = { hblocksize, 1, 1 };
637 (void *)&buffer_in);
638 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal_local, 1, sizeof(cl_mem), (void *)&HF);
639 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal_local, 2, sizeof(int), (void *)&width);
640 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal_local, 3, sizeof(int), (void *)&height);
641 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal_local, 4, sizeof(int), (void *)&mult);
643 (void *)&clamp_lf);
645 (hblocksize + 4 * mult) * 4 * sizeof(float), NULL);
647 horizontal_sizes, horizontal_local);
648 }
649 else
650 {
651 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 0, sizeof(cl_mem), (void *)&buffer_in);
652 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 1, sizeof(cl_mem), (void *)&HF);
653 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 2, sizeof(int), (void *)&width);
654 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 3, sizeof(int), (void *)&height);
655 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 4, sizeof(int), (void *)&mult);
656 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_horizontal, 5, sizeof(int), (void *)&clamp_lf);
658 }
659 if(err != CL_SUCCESS) return err;
660
661 int vblocksize;
663 .xfactor = 1,
664 .yoffset = 2 * mult,
665 .yfactor = 1,
666 .cellsize = 4 * sizeof(float),
667 .overhead = 0,
668 .sizex = 1,
669 .sizey = 1 << 16 };
671 vblocksize = vlocopt.sizey;
672 else
673 vblocksize = 1;
674
675 if(vblocksize > 1)
676 {
677 const size_t vertical_sizes[3] = { ROUNDUPDWD(width, devid), ROUNDUP(height, vblocksize), 1 };
678 const size_t vertical_local[3] = { 1, vblocksize, 1 };
679 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical_local, 0, sizeof(cl_mem), (void *)&HF);
681 (void *)&buffer_out);
682 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical_local, 2, sizeof(int), (void *)&width);
683 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical_local, 3, sizeof(int), (void *)&height);
684 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical_local, 4, sizeof(int), (void *)&mult);
685 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical_local, 5, sizeof(int), (void *)&clamp_lf);
687 (vblocksize + 4 * mult) * 4 * sizeof(float), NULL);
689 vertical_local);
690 }
691 else
692 {
693 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 0, sizeof(cl_mem), (void *)&HF);
694 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 1, sizeof(cl_mem), (void *)&buffer_out);
695 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 2, sizeof(int), (void *)&width);
696 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 3, sizeof(int), (void *)&height);
697 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 4, sizeof(int), (void *)&mult);
698 dt_opencl_set_kernel_arg(devid, gd->kernel_filmic_bspline_vertical, 5, sizeof(int), (void *)&clamp_lf);
700 }
701 if(err != CL_SUCCESS) return err;
702
703 uint8_t current_scale_type = scale_type(s, scales);
704 const float radius = sqf(equivalent_sigma_at_step(B_SPLINE_SIGMA, s * DS_FACTOR));
705 cl_mem reconstruct_write = (s == scales - 1)
706 ? reconstructed
707 : (reconstruct_read == reconstructed ? reconstructed_scratch : reconstructed);
708
709 // Keep the accumulation image read/write handles distinct at each scale.
710 // Some AMD OpenCL drivers get unstable when the same image is bound for both roles.
711 if(variant == DIFFUSE_RECONSTRUCT_RGB)
712 {
714 (void *)&buffer_in);
716 (void *)&buffer_out);
718 (void *)&clipping_mask);
720 (void *)&reconstruct_read);
722 (void *)&reconstruct_write);
723 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 5, sizeof(int), (void *)&width);
724 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 6, sizeof(int), (void *)&height);
725 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 7, sizeof(int), (void *)&mult);
727 (void *)&noise_level);
728 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 9, sizeof(int), (void *)&salt);
729 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 10, sizeof(uint8_t),
730 (void *)&current_scale_type);
731 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_guide_laplacians, 11, sizeof(float), (void *)&radius);
733 if(err != CL_SUCCESS) return err;
734 }
735 else // DIFFUSE_RECONSTRUCT_CHROMA
736 {
737 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 0, sizeof(cl_mem), (void *)&buffer_in);
738 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 1, sizeof(cl_mem), (void *)&buffer_out);
739 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 2, sizeof(cl_mem),
740 (void *)&clipping_mask);
741 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 3, sizeof(cl_mem),
742 (void *)&reconstruct_read);
743 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 4, sizeof(cl_mem),
744 (void *)&reconstruct_write);
745 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 5, sizeof(int), (void *)&width);
746 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 6, sizeof(int), (void *)&height);
747 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 7, sizeof(int), (void *)&mult);
748 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 8, sizeof(uint8_t),
749 (void *)&current_scale_type);
750 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_diffuse_color, 9, sizeof(float), (void *)&solid_color);
752 if(err != CL_SUCCESS) return err;
753 }
754
755 reconstruct_read = reconstruct_write;
756 }
757
758 return err;
759}
760#endif // HAVE_OPENCL
761
762#ifdef HAVE_OPENCL
763
765 const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out,
766 const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out,
767 const dt_aligned_pixel_t clips)
768{
771
772 cl_int err = DT_OPENCL_DEFAULT_ERROR;
773
774 const int devid = pipe->devid;
775 const int width = roi_in->width;
776 const int height = roi_in->height;
777
778 const int ds_height = height / DS_FACTOR;
779 const int ds_width = width / DS_FACTOR;
780
781 size_t sizes[] = { ROUNDUPDWD(width, devid), ROUNDUPDHT(height, devid), 1 };
782 size_t ds_sizes[] = { ROUNDUPDWD(ds_width, devid), ROUNDUPDHT(ds_height, devid), 1 };
783
784 // kernel_highlights_normalize_reduce_first is self-correcting: it takes the raw filters
785 // below PLUS roi_in->x/y as separate kernel args and adds them itself. interpolate_and_mask
786 // and remosaic_and_replace have no roi offset args at all -- they need filters pre-shifted
787 // for roi_in's crop position instead (mirrors the CPU process_laplacian fix).
788 const uint32_t filters = piece->dsc_in.filters;
789 const uint32_t filters_shifted = dt_dev_get_roi_filters(piece, roi_in);
790
791 cl_mem interpolated
792 = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4); // [R, G, B, norm] for each pixel
793 cl_mem clipping_mask
794 = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4); // [R, G, B, norm] for each pixel
795 cl_mem normalization = NULL;
796 cl_mem normalization_tmp = NULL;
797 cl_mem normalization_partials = NULL;
798 cl_mem normalization_final = NULL;
799
800 // temp buffer for blurs. We will need to cycle between them for memory efficiency
801 cl_mem LF_odd = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
802 cl_mem LF_even = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
803 cl_mem temp
804 = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4); // need full size here for blurring
805
806 const float scale = DS_FACTOR * dt_dev_get_module_scale(pipe, roi_in);
807 const float final_radius = (float)((int)(1 << data->scales)) / scale;
808 const int scales = CLAMP((int)ceilf(log2f(final_radius)), 1, MAX_NUM_SCALES);
809
810 const float noise_level = data->noise_level / scale;
811
812 // wavelets scales buffers
813 cl_mem HF = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
814 cl_mem ds_interpolated = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
815 cl_mem ds_clipping_mask = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
816 cl_mem reconstructed_scratch = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
817 cl_mem clips_cl = dt_opencl_copy_host_to_device_constant(devid, 4 * sizeof(float), (float *)clips);
818
819 if(IS_NULL_PTR(interpolated) || IS_NULL_PTR(clipping_mask) || IS_NULL_PTR(LF_odd) || IS_NULL_PTR(LF_even)
820 || IS_NULL_PTR(temp) || IS_NULL_PTR(HF) || IS_NULL_PTR(ds_interpolated) || IS_NULL_PTR(ds_clipping_mask)
821 || IS_NULL_PTR(reconstructed_scratch) || IS_NULL_PTR(clips_cl))
822 goto error;
823
824 {
826 .xfactor = 1,
827 .yoffset = 0,
828 .yfactor = 1,
829 .cellsize = 4 * sizeof(float),
830 .overhead = 0,
831 .sizex = 1 << 4,
832 .sizey = 1 << 4 };
833
835
836 const size_t bwidth = ROUNDUP(width, flocopt.sizex);
837 const size_t bheight = ROUNDUP(height, flocopt.sizey);
838 const int bufsize = (int)((bwidth / flocopt.sizex) * (bheight / flocopt.sizey));
839
840 normalization_partials = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * (size_t)bufsize);
841 normalization = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
842 normalization_tmp = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
843 if(!normalization_partials || !normalization || !normalization_tmp) goto error;
844
845 size_t fsizes[3] = { bwidth, bheight, 1 };
846 size_t flocal[3] = { flocopt.sizex, flocopt.sizey, 1 };
847 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_first, 0, sizeof(cl_mem), &dev_in);
851 &normalization_partials);
852 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_first, 4, sizeof(uint32_t), &filters);
853 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_first, 5, sizeof(int), &roi_in->x);
854 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_first, 6, sizeof(int), &roi_in->y);
856 sizeof(float) * 4 * flocopt.sizex * flocopt.sizey, NULL);
858 flocal);
859 if(err != CL_SUCCESS) goto error;
860
862 .xfactor = 1,
863 .yoffset = 0,
864 .yfactor = 1,
865 .cellsize = 4 * sizeof(float),
866 .overhead = 0,
867 .sizex = 1 << 16,
868 .sizey = 1 };
869
871
872 int current_length = bufsize;
873 cl_mem reduce_in = normalization_partials;
874 cl_mem reduce_out = normalization;
875
876 while(TRUE)
877 {
878 const int reducesize = MIN(REDUCESIZE, ROUNDUP(current_length, slocopt.sizex) / slocopt.sizex);
879 size_t ssizes[3] = { (size_t)reducesize * slocopt.sizex, 1, 1 };
880 size_t slocal[3] = { slocopt.sizex, 1, 1 };
881 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_second, 0, sizeof(cl_mem), &reduce_in);
883 &reduce_out);
885 &current_length);
887 sizeof(float) * 4 * slocopt.sizex, NULL);
889 slocal);
890 if(err != CL_SUCCESS) goto error;
891
892 if(reducesize == 1) break;
893 current_length = reducesize;
894 cl_mem swap = reduce_in;
895 reduce_in = reduce_out;
896 reduce_out = (swap == normalization_partials) ? normalization_tmp : normalization;
897 }
898
899 normalization_final = reduce_out;
900 }
901
902 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_bilinear_and_mask, 0, sizeof(cl_mem), (void *)&dev_in);
904 (void *)&interpolated);
905 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_bilinear_and_mask, 2, sizeof(cl_mem), (void *)&temp);
906 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_bilinear_and_mask, 3, sizeof(cl_mem), (void *)&clips_cl);
908 (void *)&normalization_final);
910 (void *)&filters_shifted);
911 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_bilinear_and_mask, 6, sizeof(int), (void *)&roi_out->width);
913 (void *)&roi_out->height);
915 if(err != CL_SUCCESS) goto error;
916
917 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 0, sizeof(cl_mem), (void *)&temp);
918 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 1, sizeof(cl_mem), (void *)&clipping_mask);
919 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 2, sizeof(int), (void *)&roi_out->width);
920 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 3, sizeof(int), (void *)&roi_out->height);
922 if(err != CL_SUCCESS) goto error;
923
924 // Downsample
925 const int RGBa = TRUE;
926 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&clipping_mask);
927 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
928 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
929 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_clipping_mask);
930 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
931 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
932 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
934 if(err != CL_SUCCESS) goto error;
935
936 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&interpolated);
937 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
938 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
939 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_interpolated);
940 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
941 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
942 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
944 if(err != CL_SUCCESS) goto error;
945
946 for(int i = 0; i < data->iterations; i++)
947 {
948 const int salt = (i == data->iterations - 1); // add noise on the last iteration only
949 err = wavelets_process_cl(devid, ds_interpolated, temp, reconstructed_scratch, ds_clipping_mask, ds_sizes,
950 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_RGB,
951 noise_level, salt, data->solid_color);
952 if(err != CL_SUCCESS) goto error;
953
954 err = wavelets_process_cl(devid, temp, ds_interpolated, reconstructed_scratch, ds_clipping_mask, ds_sizes,
955 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_CHROMA,
956 noise_level, salt, data->solid_color);
957 if(err != CL_SUCCESS) goto error;
958 }
959
960 // Upsample
961 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&ds_interpolated);
962 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&ds_width);
963 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&ds_height);
964 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&interpolated);
965 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&width);
966 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&height);
968 if(err != CL_SUCCESS) goto error;
969
970 // Remosaic
971 const int clip_floor_off = FALSE; // 2021 mode keeps the historical blend
972 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 0, sizeof(cl_mem), (void *)&dev_in);
973 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 1, sizeof(cl_mem), (void *)&dev_in);
975 (void *)&interpolated);
977 (void *)&clipping_mask);
978 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 4, sizeof(cl_mem), (void *)&dev_out);
980 (void *)&normalization_final);
981 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 6, sizeof(cl_mem), (void *)&clips_cl);
983 (void *)&clip_floor_off);
985 (void *)&filters_shifted);
986 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 9, sizeof(int), (void *)&width);
987 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_remosaic_and_replace, 10, sizeof(int), (void *)&height);
989 if(err != CL_SUCCESS) goto error;
990
991 // cleanup and exit on success
993 dt_opencl_release_mem_object(normalization_partials);
994 if(normalization_tmp != normalization_final) dt_opencl_release_mem_object(normalization_tmp);
995 if(normalization != normalization_final) dt_opencl_release_mem_object(normalization);
996 dt_opencl_release_mem_object(normalization_final);
997 dt_opencl_release_mem_object(interpolated);
998 dt_opencl_release_mem_object(clipping_mask);
1003 dt_opencl_release_mem_object(ds_clipping_mask);
1004 dt_opencl_release_mem_object(ds_interpolated);
1005 dt_opencl_release_mem_object(reconstructed_scratch);
1006 return err;
1007
1008error:
1010 dt_opencl_release_mem_object(normalization_partials);
1011 if(normalization_tmp != normalization_final) dt_opencl_release_mem_object(normalization_tmp);
1012 if(normalization != normalization_final) dt_opencl_release_mem_object(normalization);
1013 dt_opencl_release_mem_object(normalization_final);
1014 dt_opencl_release_mem_object(interpolated);
1015 dt_opencl_release_mem_object(clipping_mask);
1020 dt_opencl_release_mem_object(ds_clipping_mask);
1021 dt_opencl_release_mem_object(ds_interpolated);
1022 dt_opencl_release_mem_object(reconstructed_scratch);
1023
1024 dt_print(DT_DEBUG_OPENCL, "[opencl_highlights] couldn't enqueue kernel! %i\n", err);
1025 return err;
1026}
1027#endif // HAVE_OPENCL
1028
1029#ifdef HAVE_OPENCL
1030
1032 const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out,
1033 const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out,
1034 const dt_aligned_pixel_t clips)
1035{
1038
1039 cl_int err = DT_OPENCL_DEFAULT_ERROR;
1040
1041 const uint8_t(*const xtrans)[6] = (const uint8_t(*const)[6])piece->dsc_in.xtrans;
1042 const int devid = pipe->devid;
1043 const int width = roi_in->width;
1044 const int height = roi_in->height;
1045
1046 const int ds_height = height / DS_FACTOR;
1047 const int ds_width = width / DS_FACTOR;
1048
1049 size_t sizes[] = { ROUNDUPDWD(width, devid), ROUNDUPDHT(height, devid), 1 };
1050 size_t ds_sizes[] = { ROUNDUPDWD(ds_width, devid), ROUNDUPDHT(ds_height, devid), 1 };
1051
1052 cl_mem interpolated = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1053 cl_mem clipping_mask = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1054 cl_mem normalization = NULL;
1055 cl_mem normalization_tmp = NULL;
1056 cl_mem normalization_partials = NULL;
1057 cl_mem normalization_final = NULL;
1058 cl_mem LF_odd = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1059 cl_mem LF_even = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1060 cl_mem temp = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1061
1062 const float scale = DS_FACTOR * dt_dev_get_module_scale(pipe, roi_in);
1063 const float final_radius = (float)((int)(1 << data->scales)) / scale;
1064 const int scales = CLAMP((int)ceilf(log2f(final_radius)), 1, MAX_NUM_SCALES);
1065 const float noise_level = data->noise_level / scale;
1066
1067 cl_mem HF = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1068 cl_mem ds_interpolated = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1069 cl_mem ds_clipping_mask = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1070 cl_mem reconstructed_scratch = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1071
1072 cl_mem clips_cl = dt_opencl_copy_host_to_device_constant(devid, 4 * sizeof(float), (float *)clips);
1073 cl_mem dev_xtrans
1074 = dt_opencl_copy_host_to_device_constant(devid, sizeof(piece->dsc_in.xtrans), (void *)piece->dsc_in.xtrans);
1075 int32_t lookup[6][6][32] = { { { 0 } } };
1076 _build_xtrans_bilinear_lookup(lookup, roi_in, xtrans);
1077 cl_mem lookup_cl = dt_opencl_copy_host_to_device_constant(devid, sizeof(lookup), lookup);
1078
1079 if(IS_NULL_PTR(interpolated) || IS_NULL_PTR(clipping_mask) || IS_NULL_PTR(LF_odd) || IS_NULL_PTR(LF_even)
1080 || IS_NULL_PTR(temp) || IS_NULL_PTR(HF) || IS_NULL_PTR(ds_interpolated) || IS_NULL_PTR(ds_clipping_mask)
1081 || IS_NULL_PTR(reconstructed_scratch) || IS_NULL_PTR(clips_cl) || IS_NULL_PTR(dev_xtrans)
1082 || IS_NULL_PTR(lookup_cl))
1083 goto error;
1084
1085 {
1087 .xfactor = 1,
1088 .yoffset = 0,
1089 .yfactor = 1,
1090 .cellsize = 4 * sizeof(float),
1091 .overhead = 0,
1092 .sizex = 1 << 4,
1093 .sizey = 1 << 4 };
1094
1096 goto error;
1097
1098 const size_t bwidth = ROUNDUP(width, flocopt.sizex);
1099 const size_t bheight = ROUNDUP(height, flocopt.sizey);
1100 const int bufsize = (int)((bwidth / flocopt.sizex) * (bheight / flocopt.sizey));
1101
1102 normalization_partials = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * (size_t)bufsize);
1103 normalization = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
1104 normalization_tmp = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
1105 if(!normalization_partials || !normalization || !normalization_tmp) goto error;
1106
1107 size_t fsizes[3] = { bwidth, bheight, 1 };
1108 size_t flocal[3] = { flocopt.sizex, flocopt.sizey, 1 };
1110 &dev_in);
1114 &normalization_partials);
1116 &roi_in->x);
1118 &roi_in->y);
1120 &dev_xtrans);
1122 sizeof(float) * 4 * flocopt.sizex * flocopt.sizey, NULL);
1124 fsizes, flocal);
1125 if(err != CL_SUCCESS) goto error;
1126
1128 .xfactor = 1,
1129 .yoffset = 0,
1130 .yfactor = 1,
1131 .cellsize = 4 * sizeof(float),
1132 .overhead = 0,
1133 .sizex = 1 << 16,
1134 .sizey = 1 };
1135
1137
1138 int current_length = bufsize;
1139 cl_mem reduce_in = normalization_partials;
1140 cl_mem reduce_out = normalization;
1141
1142 while(TRUE)
1143 {
1144 const int reducesize = MIN(REDUCESIZE, ROUNDUP(current_length, slocopt.sizex) / slocopt.sizex);
1145 size_t ssizes[3] = { (size_t)reducesize * slocopt.sizex, 1, 1 };
1146 size_t slocal[3] = { slocopt.sizex, 1, 1 };
1147 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_normalize_reduce_second, 0, sizeof(cl_mem), &reduce_in);
1149 &reduce_out);
1151 &current_length);
1153 sizeof(float) * 4 * slocopt.sizex, NULL);
1155 slocal);
1156 if(err != CL_SUCCESS) goto error;
1157
1158 if(reducesize == 1) break;
1159 current_length = reducesize;
1160 cl_mem swap = reduce_in;
1161 reduce_in = reduce_out;
1162 reduce_out = (swap == normalization_partials) ? normalization_tmp : normalization;
1163 }
1164
1165 normalization_final = reduce_out;
1166 }
1167
1169 (void *)&dev_in);
1171 (void *)&interpolated);
1172 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_bilinear_and_mask_xtrans, 2, sizeof(cl_mem), (void *)&temp);
1174 (void *)&clips_cl);
1176 (void *)&normalization_final);
1180 (void *)&roi_in->x);
1182 (void *)&roi_in->y);
1184 (void *)&dev_xtrans);
1186 (void *)&lookup_cl);
1188 if(err != CL_SUCCESS) goto error;
1189
1190 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 0, sizeof(cl_mem), (void *)&temp);
1191 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 1, sizeof(cl_mem), (void *)&clipping_mask);
1192 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 2, sizeof(int), (void *)&roi_out->width);
1193 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 3, sizeof(int), (void *)&roi_out->height);
1195 if(err != CL_SUCCESS) goto error;
1196
1197 const int RGBa = TRUE;
1198 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&clipping_mask);
1199 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
1200 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
1201 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_clipping_mask);
1202 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
1203 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
1204 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
1205 err = dt_opencl_enqueue_kernel_2d(devid, gd->kernel_interpolate_bilinear, ds_sizes);
1206 if(err != CL_SUCCESS) goto error;
1207
1208 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&interpolated);
1209 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
1210 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
1211 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_interpolated);
1212 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
1213 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
1214 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
1215 err = dt_opencl_enqueue_kernel_2d(devid, gd->kernel_interpolate_bilinear, ds_sizes);
1216 if(err != CL_SUCCESS) goto error;
1217
1218 for(int i = 0; i < data->iterations; i++)
1219 {
1220 const int salt = (i == data->iterations - 1);
1221 err = wavelets_process_cl(devid, ds_interpolated, temp, reconstructed_scratch, ds_clipping_mask, ds_sizes,
1222 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_RGB,
1223 noise_level, salt, data->solid_color);
1224 if(err != CL_SUCCESS) goto error;
1225
1226 err = wavelets_process_cl(devid, temp, ds_interpolated, reconstructed_scratch, ds_clipping_mask, ds_sizes,
1227 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_CHROMA,
1228 noise_level, salt, data->solid_color);
1229 if(err != CL_SUCCESS) goto error;
1230 }
1231
1232 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&ds_interpolated);
1233 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&ds_width);
1234 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&ds_height);
1235 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&interpolated);
1236 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&width);
1237 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&height);
1239 if(err != CL_SUCCESS) goto error;
1240
1241 const int clip_floor_off = FALSE; // 2021 mode keeps the historical blend
1243 (void *)&dev_in);
1245 (void *)&dev_in);
1247 (void *)&interpolated);
1249 (void *)&clipping_mask);
1251 (void *)&dev_out);
1253 (void *)&normalization_final);
1255 (void *)&clips_cl);
1257 (void *)&clip_floor_off);
1259 (void *)&width);
1261 (void *)&height);
1263 (void *)&roi_in->x);
1265 (void *)&roi_in->y);
1267 (void *)&dev_xtrans);
1269 if(err != CL_SUCCESS) goto error;
1272 dt_opencl_release_mem_object(dev_xtrans);
1273 dt_opencl_release_mem_object(normalization_partials);
1274 if(normalization_tmp != normalization_final) dt_opencl_release_mem_object(normalization_tmp);
1275 if(normalization != normalization_final) dt_opencl_release_mem_object(normalization);
1276 dt_opencl_release_mem_object(normalization_final);
1277 dt_opencl_release_mem_object(interpolated);
1278 dt_opencl_release_mem_object(clipping_mask);
1283 dt_opencl_release_mem_object(ds_clipping_mask);
1284 dt_opencl_release_mem_object(ds_interpolated);
1285 dt_opencl_release_mem_object(reconstructed_scratch);
1286 return err;
1287
1288error:
1291 dt_opencl_release_mem_object(dev_xtrans);
1292 dt_opencl_release_mem_object(normalization_partials);
1293 if(normalization_tmp != normalization_final) dt_opencl_release_mem_object(normalization_tmp);
1294 if(normalization != normalization_final) dt_opencl_release_mem_object(normalization);
1295 dt_opencl_release_mem_object(normalization_final);
1296 dt_opencl_release_mem_object(interpolated);
1297 dt_opencl_release_mem_object(clipping_mask);
1302 dt_opencl_release_mem_object(ds_clipping_mask);
1303 dt_opencl_release_mem_object(ds_interpolated);
1304 dt_opencl_release_mem_object(reconstructed_scratch);
1305
1306 dt_print(DT_DEBUG_OPENCL, "[opencl_highlights] couldn't enqueue kernel! %i\n", err);
1307 return err;
1308}
1310 const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out,
1311 const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out,
1312 const dt_aligned_pixel_t clips)
1313{
1314 // Non-raw / sRAW guided-laplacian, fully on-device: already-demosaiced RGB input, so the gather is a
1315 // plane copy (interpolate_and_mask_passthrough), the normalization sums all three colours per pixel
1316 // (highlights_normalize_reduce_first_passthrough), and the remosaic is a per-channel composite
1317 // (remosaic_and_replace_passthrough). Between them the downsample -> a-trous wavelets -> upsample is
1318 // the same CFA-agnostic device path as the Bayer/X-Trans drivers. No FC, no xtrans lookup, no roi phase.
1321
1322 cl_int err = DT_OPENCL_DEFAULT_ERROR;
1323
1324 const int devid = pipe->devid;
1325 const int width = roi_in->width;
1326 const int height = roi_in->height;
1327
1328 const int ds_height = height / DS_FACTOR;
1329 const int ds_width = width / DS_FACTOR;
1330
1331 size_t sizes[] = { ROUNDUPDWD(width, devid), ROUNDUPDHT(height, devid), 1 };
1332 size_t ds_sizes[] = { ROUNDUPDWD(ds_width, devid), ROUNDUPDHT(ds_height, devid), 1 };
1333
1334 cl_mem interpolated = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1335 cl_mem clipping_mask = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1336 cl_mem normalization = NULL;
1337 cl_mem normalization_tmp = NULL;
1338 cl_mem normalization_partials = NULL;
1339 cl_mem normalization_final = NULL;
1340
1341 cl_mem LF_odd = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1342 cl_mem LF_even = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1343 cl_mem temp = dt_opencl_alloc_device(devid, sizes[0], sizes[1], sizeof(float) * 4);
1344
1345 const float scale = DS_FACTOR * dt_dev_get_module_scale(pipe, roi_in);
1346 const float final_radius = (float)((int)(1 << data->scales)) / scale;
1347 const int scales = CLAMP((int)ceilf(log2f(final_radius)), 1, MAX_NUM_SCALES);
1348 const float noise_level = data->noise_level / scale;
1349
1350 cl_mem HF = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1351 cl_mem ds_interpolated = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1352 cl_mem ds_clipping_mask = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1353 cl_mem reconstructed_scratch = dt_opencl_alloc_device(devid, ds_sizes[0], ds_sizes[1], sizeof(float) * 4);
1354 cl_mem clips_cl = dt_opencl_copy_host_to_device_constant(devid, 4 * sizeof(float), (float *)clips);
1355
1356 if(IS_NULL_PTR(interpolated) || IS_NULL_PTR(clipping_mask) || IS_NULL_PTR(LF_odd) || IS_NULL_PTR(LF_even)
1357 || IS_NULL_PTR(temp) || IS_NULL_PTR(HF) || IS_NULL_PTR(ds_interpolated) || IS_NULL_PTR(ds_clipping_mask)
1358 || IS_NULL_PTR(reconstructed_scratch) || IS_NULL_PTR(clips_cl))
1359 goto error;
1360
1361 {
1363 .xfactor = 1,
1364 .yoffset = 0,
1365 .yfactor = 1,
1366 .cellsize = 4 * sizeof(float),
1367 .overhead = 0,
1368 .sizex = 1 << 4,
1369 .sizey = 1 << 4 };
1370
1372 goto error;
1373
1374 const size_t bwidth = ROUNDUP(width, flocopt.sizex);
1375 const size_t bheight = ROUNDUP(height, flocopt.sizey);
1376 const int bufsize = (int)((bwidth / flocopt.sizex) * (bheight / flocopt.sizey));
1377
1378 normalization_partials = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * (size_t)bufsize);
1379 normalization = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
1380 normalization_tmp = dt_opencl_alloc_device_buffer(devid, sizeof(float) * 4 * REDUCESIZE);
1381 if(!normalization_partials || !normalization || !normalization_tmp) goto error;
1382
1383 size_t fsizes[3] = { bwidth, bheight, 1 };
1384 size_t flocal[3] = { flocopt.sizex, flocopt.sizey, 1 };
1386 &dev_in);
1388 &width);
1390 &height);
1392 &normalization_partials);
1394 sizeof(float) * 4 * flocopt.sizex * flocopt.sizey, NULL);
1396 fsizes, flocal);
1397 if(err != CL_SUCCESS) goto error;
1398
1400 .xfactor = 1,
1401 .yoffset = 0,
1402 .yfactor = 1,
1403 .cellsize = 4 * sizeof(float),
1404 .overhead = 0,
1405 .sizex = 1 << 16,
1406 .sizey = 1 };
1407
1409
1410 int current_length = bufsize;
1411 cl_mem reduce_in = normalization_partials;
1412 cl_mem reduce_out = normalization;
1413
1414 while(TRUE)
1415 {
1416 const int reducesize = MIN(REDUCESIZE, ROUNDUP(current_length, slocopt.sizex) / slocopt.sizex);
1417 size_t ssizes[3] = { (size_t)reducesize * slocopt.sizex, 1, 1 };
1418 size_t slocal[3] = { slocopt.sizex, 1, 1 };
1420 &reduce_in);
1422 &reduce_out);
1424 &current_length);
1426 sizeof(float) * 4 * slocopt.sizex, NULL);
1428 slocal);
1429 if(err != CL_SUCCESS) goto error;
1430
1431 if(reducesize == 1) break;
1432 current_length = reducesize;
1433 cl_mem swap = reduce_in;
1434 reduce_in = reduce_out;
1435 reduce_out = (swap == normalization_partials) ? normalization_tmp : normalization;
1436 }
1437
1438 normalization_final = reduce_out;
1439 }
1440
1441 // gather: plane copy + per-channel clip mask (mask written to `temp`, then 5x5-boxed into clipping_mask)
1443 (void *)&dev_in);
1445 (void *)&interpolated);
1447 (void *)&temp);
1449 (void *)&clips_cl);
1451 (void *)&normalization_final);
1453 (void *)&roi_out->width);
1455 (void *)&roi_out->height);
1457 if(err != CL_SUCCESS) goto error;
1458
1459 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 0, sizeof(cl_mem), (void *)&temp);
1460 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 1, sizeof(cl_mem), (void *)&clipping_mask);
1461 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 2, sizeof(int), (void *)&roi_out->width);
1462 dt_opencl_set_kernel_arg(devid, gd->kernel_highlights_box_blur, 3, sizeof(int), (void *)&roi_out->height);
1464 if(err != CL_SUCCESS) goto error;
1465
1466 // Downsample
1467 const int RGBa = TRUE;
1468 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&clipping_mask);
1469 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
1470 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
1471 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_clipping_mask);
1472 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
1473 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
1474 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
1475 err = dt_opencl_enqueue_kernel_2d(devid, gd->kernel_interpolate_bilinear, ds_sizes);
1476 if(err != CL_SUCCESS) goto error;
1477
1478 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&interpolated);
1479 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&width);
1480 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&height);
1481 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&ds_interpolated);
1482 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&ds_width);
1483 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&ds_height);
1484 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 6, sizeof(int), (void *)&RGBa);
1485 err = dt_opencl_enqueue_kernel_2d(devid, gd->kernel_interpolate_bilinear, ds_sizes);
1486 if(err != CL_SUCCESS) goto error;
1487
1488 for(int i = 0; i < data->iterations; i++)
1489 {
1490 const int salt = (i == data->iterations - 1); // add noise on the last iteration only
1491 err = wavelets_process_cl(devid, ds_interpolated, temp, reconstructed_scratch, ds_clipping_mask, ds_sizes,
1492 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_RGB,
1493 noise_level, salt, data->solid_color);
1494 if(err != CL_SUCCESS) goto error;
1495
1496 err = wavelets_process_cl(devid, temp, ds_interpolated, reconstructed_scratch, ds_clipping_mask, ds_sizes,
1497 ds_width, ds_height, gd, scales, HF, LF_odd, LF_even, DIFFUSE_RECONSTRUCT_CHROMA,
1498 noise_level, salt, data->solid_color);
1499 if(err != CL_SUCCESS) goto error;
1500 }
1501
1502 // Upsample
1503 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 0, sizeof(cl_mem), (void *)&ds_interpolated);
1504 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 1, sizeof(int), (void *)&ds_width);
1505 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 2, sizeof(int), (void *)&ds_height);
1506 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 3, sizeof(cl_mem), (void *)&interpolated);
1507 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 4, sizeof(int), (void *)&width);
1508 dt_opencl_set_kernel_arg(devid, gd->kernel_interpolate_bilinear, 5, sizeof(int), (void *)&height);
1510 if(err != CL_SUCCESS) goto error;
1511
1512 // Remosaic: per-channel composite (clip_is_floor FALSE, historical blend, mirrors the CPU path)
1513 const int clip_floor_off = FALSE;
1515 (void *)&dev_in);
1517 (void *)&dev_in);
1519 (void *)&interpolated);
1521 (void *)&clipping_mask);
1523 (void *)&dev_out);
1525 (void *)&normalization_final);
1527 (void *)&clips_cl);
1529 (void *)&clip_floor_off);
1531 (void *)&width);
1533 (void *)&height);
1535 if(err != CL_SUCCESS) goto error;
1536
1537error:
1539 dt_opencl_release_mem_object(normalization_partials);
1540 if(normalization_tmp != normalization_final) dt_opencl_release_mem_object(normalization_tmp);
1541 if(normalization != normalization_final) dt_opencl_release_mem_object(normalization);
1542 dt_opencl_release_mem_object(normalization_final);
1543 dt_opencl_release_mem_object(interpolated);
1544 dt_opencl_release_mem_object(clipping_mask);
1549 dt_opencl_release_mem_object(ds_clipping_mask);
1550 dt_opencl_release_mem_object(ds_interpolated);
1551 dt_opencl_release_mem_object(reconstructed_scratch);
1552 if(err != CL_SUCCESS)
1553 dt_print(DT_DEBUG_OPENCL, "[opencl_highlights] couldn't enqueue kernel! %i\n", err);
1554 return err;
1555}
1556#endif // HAVE_OPENCL
static void error(char *msg)
Definition ashift_lsd.c:202
#define TRUE
Definition ashift_lsd.c:162
#define FALSE
Definition ashift_lsd.c:158
#define MAX_NUM_SCALES
Definition atrous.c:97
int dt_box_mean(float *const buf, const size_t height, const size_t width, const int ch, const int radius, const unsigned iterations)
#define B_SPLINE_SIGMA
Definition bspline.h:38
static float equivalent_sigma_at_step(const float sigma, const unsigned int s)
Definition bspline.h:52
#define B_SPLINE_TO_LAPLACIAN
Definition bspline.h:49
static void decompose_2D_Bspline(const float *const restrict in, float *const restrict HF, float *const restrict LF, const size_t width, const size_t height, const int mult, float *const tempbuf, size_t padded_size)
Definition bspline.h:351
typedef void((*dt_cache_allocate_t)(void *userdata, dt_cache_entry_t *entry))
return vector dt_simd_set1(valid ?(scaling+NORM_MIN) :NORM_MIN)
static float lookup(read_only image2d_t lut, const float x)
const dt_colormatrix_t dt_aligned_pixel_t out
dt_store_simd_aligned(out, dt_mat3x4_mul_vec4(vin, dt_colormatrix_row_to_simd(matrix, 0), dt_colormatrix_row_to_simd(matrix, 1), dt_colormatrix_row_to_simd(matrix, 2)))
static const int row
#define BLUE
#define RED
#define GREEN
#define ALPHA
static float4 dt_noise_generator_simd(const dt_noise_distribution_t distribution, const float4 mu, const float4 param, uint state[4])
static unsigned int splitmix32(const unsigned long seed)
static float xoshiro128plus(uint state[4])
#define REDUCESIZE
Definition demosaic.c:111
static int dwt_interleave_rows(const int rowid, const int height, const int stride)
Definition dwt.h:93
static __DT_CLONE_TARGETS__ void interpolate_bilinear(const float *const restrict in, const size_t width_in, const size_t height_in, float *const restrict out, const size_t width_out, const size_t height_out, const size_t ch)
__DT_CLONE_TARGETS__ void _interpolate_and_mask_passthrough(const float *const restrict input, float *const restrict interpolated, float *const restrict clipping_mask, const dt_aligned_pixel_t clips, const dt_aligned_pixel_t white_balance, const size_t width, const size_t height)
Definition gather.c:424
__DT_CLONE_TARGETS__ void _compute_laplacian_normalization(const float *const restrict input, const dt_iop_roi_t *const roi_in, const uint32_t filters, const uint8_t(*const xtrans)[6], dt_aligned_pixel_t normalization)
Definition gather.c:223
__DT_CLONE_TARGETS__ void _remosaic_and_replace_xtrans(const float *const restrict input, const float *const restrict input_raw, const float *const restrict interpolated, const float *const restrict clipping_mask, float *const restrict output, const dt_aligned_pixel_t white_balance, const dt_aligned_pixel_t clips, const int clip_is_floor, const dt_iop_roi_t *const roi_in, const uint8_t(*const xtrans)[6], const size_t width, const size_t height)
Definition gather.c:488
__DT_CLONE_TARGETS__ void _interpolate_and_mask(const float *const restrict input, float *const restrict interpolated, float *const restrict clipping_mask, const dt_aligned_pixel_t clips_in, const dt_aligned_pixel_t det_scale, const dt_aligned_pixel_t white_balance, const uint32_t filters, const size_t width, const size_t height)
Definition gather.c:67
__DT_CLONE_TARGETS__ void _remosaic_and_replace(const float *const restrict input, const float *const restrict input_raw, const float *const restrict interpolated, const float *const restrict clipping_mask, float *const restrict output, const dt_aligned_pixel_t white_balance, const dt_aligned_pixel_t clips, const int clip_is_floor, const uint32_t filters, const size_t width, const size_t height)
Definition gather.c:457
__DT_CLONE_TARGETS__ void _interpolate_and_mask_xtrans(const float *const restrict input, float *const restrict interpolated, float *const restrict clipping_mask, const dt_aligned_pixel_t clips, const dt_aligned_pixel_t white_balance, const dt_iop_roi_t *const roi_in, const int32_t lookup[6][6][32], const uint8_t(*const xtrans)[6], const size_t width, const size_t height)
Definition gather.c:317
__DT_CLONE_TARGETS__ void _build_xtrans_bilinear_lookup(int32_t lookup[6][6][32], const dt_iop_roi_t *const roi_in, const uint8_t(*const xtrans)[6])
Definition gather.c:277
__DT_CLONE_TARGETS__ void _remosaic_and_replace_passthrough(const float *const restrict input, const float *const restrict input_raw, const float *const restrict interpolated, const float *const restrict clipping_mask, float *const restrict output, const dt_aligned_pixel_t white_balance, const dt_aligned_pixel_t clips, const int clip_is_floor, const size_t width, const size_t height)
Definition gather.c:514
static dt_hl_cfa_t _hl_cfa_strategy(const uint32_t filters)
Definition gather.h:74
dt_hl_cfa_t
Definition gather.h:67
@ HL_CFA_PASSTHROUGH
Definition gather.h:70
@ HL_CFA_BAYER
Definition gather.h:68
@ HL_CFA_XTRANS
Definition gather.h:69
uint32_t dt_dev_get_roi_filters(const dt_dev_pixelpipe_iop_t *const piece, const dt_iop_roi_t *const roi_in)
Definition imageop.c:139
float dt_dev_get_module_scale(const dt_dev_pixelpipe_t *const pipe, const dt_iop_roi_t *const roi_in)
Definition imageop.c:134
void *const ovoid
static __DT_CLONE_TARGETS__ void heat_PDE_diffusion(const float *const restrict high_freq, const float *const restrict low_freq, const float *const restrict clipping_mask, float *const restrict output, const size_t width, const size_t height, const int mult, const uint8_t scale, const float first_order_factor)
Definition laplacian.c:248
static int wavelets_process(const float *const restrict in, float *const restrict reconstructed, const float *const restrict clipping_mask, const size_t width, const size_t height, const int scales, float *const restrict HF, float *const restrict LF_odd, float *const restrict LF_even, const diffuse_reconstruct_variant_t variant, const float noise_level, const int salt, const float first_order_factor)
Definition laplacian.c:374
cl_int process_laplacian_bayer_cl(struct dt_iop_module_t *self, const dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out, const dt_aligned_pixel_t clips)
Definition laplacian.c:764
cl_int process_laplacian_xtrans_cl(struct dt_iop_module_t *self, const dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out, const dt_aligned_pixel_t clips)
Definition laplacian.c:1031
cl_int process_laplacian_passthrough_cl(struct dt_iop_module_t *self, const dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece, cl_mem dev_in, cl_mem dev_out, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out, const dt_aligned_pixel_t clips)
Definition laplacian.c:1309
__DT_CLONE_TARGETS__ int process_laplacian(struct dt_iop_module_t *self, const dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece, const void *const restrict ivoid, void *const restrict ovoid, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out, const dt_aligned_pixel_t clips)
Definition laplacian.c:433
static __DT_CLONE_TARGETS__ void guide_laplacians(const float *const restrict high_freq, const float *const restrict low_freq, const float *const restrict clipping_mask, float *const restrict output, const size_t width, const size_t height, const int mult, const float noise_level, const int salt, const uint8_t scale, const float radius_sq)
Definition laplacian.c:85
static cl_int wavelets_process_cl(const int devid, cl_mem in, cl_mem reconstructed, cl_mem reconstructed_scratch, cl_mem clipping_mask, const size_t sizes[3], const int width, const int height, dt_iop_highlights_global_data_t *const gd, const int scales, cl_mem HF, cl_mem LF_odd, cl_mem LF_even, const diffuse_reconstruct_variant_t variant, const float noise_level, const int salt, const float solid_color)
Definition laplacian.c:578
static void swap(float *x, float *y)
Definition lightroom.c:1023
@ DT_DEBUG_OPENCL
Definition logging.h:57
void dt_print(dt_debug_thread_t thread, const char *msg,...) __attribute__((format(printf
Print to stdout when thread is enabled, prefixed with seconds since startup.
float *const restrict const size_t k
#define IS_NULL_PTR(p)
C is way too permissive with !=, == and if(var) checks, which can mean too many things depending on w...
Definition macros.h:96
#define DT_ALIGNED_ARRAY
Align an object on a cacheline boundary, so AVX2 can load it whole.
Definition mem_alloc.h:80
#define DT_IS_ALIGNED(x)
Promise the compiler that x is already cacheline-aligned, and return it.
Definition mem_alloc.h:51
uint32_t width
Definition mipmap_cache.c:0
uint32_t height
Definition mipmap_cache.c:1
size_t size
Definition mipmap_cache.c:3
int dt_opencl_local_buffer_opt(const int devid, const int kernel, dt_opencl_local_buffer_t *factors)
Definition opencl.c:3713
int dt_opencl_enqueue_kernel_2d(const int dev, const int kernel, const size_t *sizes)
Definition opencl.c:2554
void * dt_opencl_alloc_device_buffer(const int devid, const size_t size)
Definition opencl.c:2970
void * dt_opencl_alloc_device(const int devid, const int width, const int height, const int bpp)
Definition opencl.c:2894
void * dt_opencl_copy_host_to_device_constant(const int devid, const size_t size, void *host)
Definition opencl.c:2750
int dt_opencl_set_kernel_arg(const int dev, const int kernel, const int num, const size_t size, const void *arg)
Definition opencl.c:2545
int dt_opencl_enqueue_kernel_2d_with_local(const int dev, const int kernel, const size_t *sizes, const size_t *local)
Definition opencl.c:2560
void dt_opencl_release_mem_object(cl_mem mem)
Definition opencl.c:2805
#define DT_OPENCL_DEFAULT_ERROR
Definition opencl.h:61
#define ROUNDUP(a, n)
Definition opencl.h:82
#define ROUNDUPDHT(a, b)
Definition opencl.h:86
#define ROUNDUPDWD(a, b)
Definition opencl.h:85
#define __OMP_PARALLEL_FOR__(...)
Definition openmp.h:95
#define dt_pixelpipe_cache_free_align(mem)
#define dt_pixelpipe_cache_alloc_align_float(pixels, pipe)
#define dt_pixelpipe_cache_alloc_perthread_float(n, padded_size)
#define eps
Definition rcd.c:81
static const dt_aligned_pixel_simd_t const dt_aligned_pixel_simd_t row1
Definition simd.h:190
DT_ALIGNED_PIXEL float dt_aligned_pixel_t[4]
Definition simd.h:53
#define for_each_channel(_var,...)
Definition simd.h:87
float dt_aligned_pixel_simd_t __attribute__((vector_size(16), aligned(16)))
Apply one channel's tone curve to each of the three colour channels, or pass the channel through unto...
Definition simd.h:55
static const dt_aligned_pixel_simd_t const dt_aligned_pixel_simd_t const dt_aligned_pixel_simd_t row2
Definition simd.h:191
#define for_four_channels(_var,...)
Definition simd.h:89
static const dt_aligned_pixel_simd_t row0
Definition simd.h:189
diffuse_reconstruct_variant_t
@ DIFFUSE_RECONSTRUCT_RGB
@ DIFFUSE_RECONSTRUCT_CHROMA
#define DS_FACTOR
const float uint32_t state[4]
const float sigma
const float const int flip
const float noise
dt_iop_buffer_dsc_t dsc_in
struct dt_iop_module_t *void * data
uint32_t filters
Definition format.h:89
uint8_t xtrans[6][6]
Definition format.h:99
dt_atrous_wavelets_scales_t scales
dt_iop_global_data_t * global_data
Definition imageop.h:337
Region of interest passed through the pixelpipe.
Definition format.h:49
int width
Definition format.h:50
int height
Definition format.h:50
#define __DT_CLONE_TARGETS__
#define MIN(a, b)
Definition thinplate.c:32
#define MAX(a, b)
Definition thinplate.c:29