Ansel 0.0
A darktable fork - bloat + design vision
Loading...
Searching...
No Matches
nn_model.c
Go to the documentation of this file.
1/*
2 This file is part of Ansel,
3 Copyright (C) 2026 Aurélien PIERRE.
4
5 Ansel is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, either version 3 of the License, or
8 (at your option) any later version.
9
10 Ansel is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with Ansel. If not, see <http://www.gnu.org/licenses/>.
17*/
18#include "common/nn_model.h"
19
20// the only dependency this unit takes beyond libm/glib/json-glib: the
21// multi-versioning attribute, duplicated there so darktable.h stays out
23
24#include <glib/gstdio.h>
25#include <json-glib/json-glib.h>
26#include <limits.h>
27#include <math.h>
28#include <stdint.h>
29#include <stdio.h>
30#include <stdlib.h>
31#include <string.h>
32
33#define NN_MAX_DEPTH 8
34#define NN_MIN(a, b) ((a) < (b) ? (a) : (b))
35
36typedef struct nn_conv_t
37{
38 const float *w; // (out_ch, in_ch, k, k), row-major
39 const float *b; // (out_ch)
40 int out_ch, in_ch, k;
42
43#ifdef HAVE_OPENCL
44#define NN_MAX_DEVICES 16
45#endif
46
47// one wired U-Net (weights point into the model's shared blob)
56
58{
59 nn_unet_t fine; // stage 0: the mosaic net (the only net for arch "unet")
60 nn_unet_t coarse; // stage 1: the superpixel-RGB net (arch "unet-ms" only)
62 int bin_bayer, bin_xtrans; // superpixel bin factors per CFA family
63 int anchor; // low-band anchor scale in sensor px (0 = none)
64 float *blob; // whole payload, tensors point into it
65 size_t blob_floats; // number of floats in blob (for device upload)
66#ifdef HAVE_OPENCL
67 cl_mem dev_weights[NN_MAX_DEVICES]; // blob uploaded per device, lazily
68 dt_pthread_mutex_t cl_lock;
69#endif
70};
71
72/* injected pixel-buffer allocator (see nn_model.h); malloc fallback keeps the
73 * standalone fixture test dependency-free */
76
78{
79 _nn_alloc_fn = alloc_fn;
80 _nn_free_fn = free_fn;
81}
82
83/* injected cancellation test (see nn_model.h); unset, nothing cancels */
85
87{
88 _nn_cancel_fn = cancel_fn;
89}
90
91/* Pixel buffers come from the injected arena — the pixelpipe cache memory
92 * arena in the application — and ONLY from it: the arena is the application's
93 * memory-budget control, and a malloc escape hatch would simply move the
94 * failure to the OS OOM killer, which kills the process instead of one tile.
95 * An arena refusal therefore fails the forward, and the caller falls back to
96 * an unprocessed tile — the tiling engine's job is to plan tiles small enough
97 * that this cannot happen (see dt_nn_unet_scratch_per_px and the module's
98 * tiling_callback). The malloc path below exists solely for the standalone
99 * fixture test, which runs outside the application with no allocator set. */
100
101static void *_nn_alloc(size_t floats, int long_lived)
102{
103 const size_t bytes = floats * sizeof(float);
104 return _nn_alloc_fn ? _nn_alloc_fn(bytes, long_lived) : malloc(bytes);
105}
106
107static void _nn_free(void *p)
108{
109 if(!p) return;
110 if(_nn_free_fn)
111 _nn_free_fn(p);
112 else
113 free(p);
114}
115
116static void _err(char *err, size_t err_len, const char *msg)
117{
118 if(err && err_len) snprintf(err, err_len, "%s", msg);
119}
120
121/* ------------------------------------------------------------------------
122 * loader
123 * ------------------------------------------------------------------------ */
124
125typedef struct nn_header_t
126{
127 JsonArray *tensors;
128 const float *payload;
131
132// resolve one conv layer's weight+bias by pytorch state-dict name prefix,
133// validating dimensions against the expectation
134static int _wire_conv(const nn_header_t *h, const char *prefix, int out_ch, int in_ch, int k, nn_conv_t *cv,
135 char *err, size_t err_len)
136{
137 char name[128];
138 const float *w = NULL, *b = NULL;
139 const guint n = json_array_get_length(h->tensors);
140 for(int part = 0; part < 2; part++)
141 {
142 snprintf(name, sizeof(name), "%s.%s", prefix, part == 0 ? "weight" : "bias");
143 const size_t want = (part == 0 ? (size_t)out_ch * in_ch * k * k : (size_t)out_ch) * sizeof(float);
144 const float *found = NULL;
145 for(guint i = 0; i < n; i++)
146 {
147 // the file comes from the user's config dir: never trust its structure.
148 // A non-object element or a missing/typeless member must be a clean
149 // reject, not a NULL dereference.
150 JsonNode *node = json_array_get_element(h->tensors, i);
151 if(!node || !JSON_NODE_HOLDS_OBJECT(node)) continue;
152 JsonObject *t = json_node_get_object(node);
153 if(!json_object_has_member(t, "name") || !json_object_has_member(t, "offset")
154 || !json_object_has_member(t, "size"))
155 continue;
156 if(g_strcmp0(json_object_get_string_member(t, "name"), name)) continue;
157 const gint64 offset = json_object_get_int_member(t, "offset");
158 const gint64 size = json_object_get_int_member(t, "size");
159 if(size != (gint64)want || offset < 0 || (size_t)(offset + size) > h->payload_size)
160 {
161 _err(err, err_len, "tensor with unexpected size or offset");
162 return 1;
163 }
164 found = (const float *)((const uint8_t *)h->payload + offset);
165 break;
166 }
167 if(!found)
168 {
169 if(err && err_len) snprintf(err, err_len, "missing tensor %s", name);
170 return 1;
171 }
172 if(part == 0)
173 w = found;
174 else
175 b = found;
176 }
177 cv->w = w;
178 cv->b = b;
179 cv->out_ch = out_ch;
180 cv->in_ch = in_ch;
181 cv->k = k;
182 return 0;
183}
184
185// wire one full U-Net from the tensor table; stage_prefix is "" for arch
186// "unet" and "coarse." / "fine." for arch "unet-ms" (pytorch submodule names)
187static int _wire_unet(const nn_header_t *h, const char *stage_prefix, int base, int depth, int in_ch, int out_ch,
188 nn_unet_t *u, char *err, size_t err_len)
189{
190 u->base = base;
191 u->depth = depth;
192 u->in_ch = in_ch;
193 u->out_ch = out_ch;
194 char prefix[96];
195 int bad = 0, cin = in_ch;
196 for(int l = 0; l < depth && !bad; l++)
197 {
198 const int w = base << l;
199 snprintf(prefix, sizeof(prefix), "%senc.%d.0", stage_prefix, l);
200 bad |= _wire_conv(h, prefix, w, cin, 3, &u->enc1[l], err, err_len);
201 snprintf(prefix, sizeof(prefix), "%senc.%d.2", stage_prefix, l);
202 bad |= _wire_conv(h, prefix, w, w, 3, &u->enc2[l], err, err_len);
203 snprintf(prefix, sizeof(prefix), "%sdown.%d", stage_prefix, l);
204 bad |= _wire_conv(h, prefix, w, w, 2, &u->down[l], err, err_len);
205 cin = w;
206 }
207 const int wb = base << depth;
208 snprintf(prefix, sizeof(prefix), "%sbottleneck.0", stage_prefix);
209 bad |= _wire_conv(h, prefix, wb, base << (depth - 1), 3, &u->bot1, err, err_len);
210 snprintf(prefix, sizeof(prefix), "%sbottleneck.2", stage_prefix);
211 bad |= _wire_conv(h, prefix, wb, wb, 3, &u->bot2, err, err_len);
212 // decoder ModuleLists were built from the deepest level down: up.0/dec.0
213 // operate on the bottleneck output, up.(depth-1)/dec.(depth-1) on level 0
214 for(int i = 0; i < depth && !bad; i++)
215 {
216 const int w_skip = base << (depth - 1 - i);
217 const int w_in = w_skip << 1;
218 snprintf(prefix, sizeof(prefix), "%sup.%d", stage_prefix, i);
219 bad |= _wire_conv(h, prefix, w_skip, w_in, 1, &u->up[i], err, err_len);
220 snprintf(prefix, sizeof(prefix), "%sdec.%d.0", stage_prefix, i);
221 bad |= _wire_conv(h, prefix, w_skip, 2 * w_skip, 3, &u->dec1[i], err, err_len);
222 snprintf(prefix, sizeof(prefix), "%sdec.%d.2", stage_prefix, i);
223 bad |= _wire_conv(h, prefix, w_skip, w_skip, 3, &u->dec2[i], err, err_len);
224 }
225 snprintf(prefix, sizeof(prefix), "%shead", stage_prefix);
226 bad |= _wire_conv(h, prefix, out_ch, base, 3, &u->head, err, err_len);
227 return bad;
228}
229
230// read {base, depth, in_channels, out_channels} from a (sub-)cfg object with
231// range validation; out_ch_max 1 keeps the historic fine-stage contract
232static int _read_net_cfg(JsonObject *cfg, int out_ch_max, int *base, int *depth, int *in_ch, int *out_ch)
233{
234 if(!cfg || !json_object_has_member(cfg, "base") || !json_object_has_member(cfg, "depth")
235 || !json_object_has_member(cfg, "in_channels") || !json_object_has_member(cfg, "out_channels"))
236 return 1;
237 *base = (int)json_object_get_int_member(cfg, "base");
238 *depth = (int)json_object_get_int_member(cfg, "depth");
239 *in_ch = (int)json_object_get_int_member(cfg, "in_channels");
240 *out_ch = (int)json_object_get_int_member(cfg, "out_channels");
241 return *base < 1 || *base > 512 || *depth < 1 || *depth > NN_MAX_DEPTH || *in_ch < 1 || *in_ch > 16
242 || *out_ch < 1 || *out_ch > out_ch_max;
243}
244
245dt_nn_model_t *dt_nn_model_load(const char *path, char *err, size_t err_len)
246{
247 FILE *f = g_fopen(path, "rb");
248 if(!f)
249 {
250 _err(err, err_len, "cannot open model file");
251 return NULL;
252 }
253 uint8_t magic[8];
254 uint32_t header_len = 0;
255 if(fread(magic, 1, 8, f) != 8 || memcmp(magic, "ANSELDN1", 8) || fread(&header_len, 4, 1, f) != 1
256 || header_len == 0 || header_len > (64u << 20))
257 {
258 _err(err, err_len, "not an ANSELDN1 model file");
259 fclose(f);
260 return NULL;
261 }
262 char *header = malloc(header_len + 1);
263 if(!header || fread(header, 1, header_len, f) != header_len)
264 {
265 _err(err, err_len, "truncated model header");
266 free(header);
267 fclose(f);
268 return NULL;
269 }
270 header[header_len] = '\0';
271
272 fseek(f, 0, SEEK_END);
273 const long file_size = ftell(f);
274 if(file_size < 12 + (long)header_len)
275 {
276 _err(err, err_len, "truncated model file");
277 free(header);
278 fclose(f);
279 return NULL;
280 }
281 const size_t payload_size = (size_t)file_size - 12 - header_len;
282 float *blob = malloc(payload_size);
283 fseek(f, 12 + (long)header_len, SEEK_SET);
284 const int payload_ok = blob && fread(blob, 1, payload_size, f) == payload_size;
285 fclose(f);
286 if(!payload_ok)
287 {
288 _err(err, err_len, "truncated model payload");
289 free(header);
290 free(blob);
291 return NULL;
292 }
293
294 dt_nn_model_t *m = NULL;
295 JsonParser *parser = json_parser_new();
296 if(!json_parser_load_from_data(parser, header, header_len, NULL))
297 {
298 _err(err, err_len, "invalid model header JSON");
299 goto out;
300 }
301 JsonObject *root = json_node_get_object(json_parser_get_root(parser));
302 if(!root || !json_object_has_member(root, "cfg") || !json_object_has_member(root, "tensors"))
303 {
304 _err(err, err_len, "model header missing cfg or tensors");
305 goto out;
306 }
307 JsonObject *cfg = json_object_get_object_member(root, "cfg");
308 const char *arch = cfg ? json_object_get_string_member(cfg, "arch") : NULL;
309 const int is_ms = !g_strcmp0(arch, "unet-ms");
310 if(!is_ms && g_strcmp0(arch, "unet"))
311 {
312 _err(err, err_len, "unsupported model architecture");
313 goto out;
314 }
315 int f_base = 0, f_depth = 0, f_in = 0, f_out = 0;
316 int c_base = 0, c_depth = 0, c_in = 0, c_out = 0;
317 int bin_bayer = 1, bin_xtrans = 1;
318 if(is_ms)
319 {
320 if(!json_object_has_member(cfg, "coarse") || !json_object_has_member(cfg, "fine")
321 || !json_object_has_member(cfg, "bin")
322 || _read_net_cfg(json_object_get_object_member(cfg, "fine"), 1, &f_base, &f_depth, &f_in, &f_out)
323 || _read_net_cfg(json_object_get_object_member(cfg, "coarse"), 8, &c_base, &c_depth, &c_in, &c_out))
324 {
325 _err(err, err_len, "model config out of supported range");
326 goto out;
327 }
328 JsonObject *bin = json_object_get_object_member(cfg, "bin");
329 if(!bin || !json_object_has_member(bin, "bayer") || !json_object_has_member(bin, "xtrans"))
330 {
331 _err(err, err_len, "model config missing bin factors");
332 goto out;
333 }
334 bin_bayer = (int)json_object_get_int_member(bin, "bayer");
335 bin_xtrans = (int)json_object_get_int_member(bin, "xtrans");
336 if(bin_bayer < 2 || bin_bayer > 16 || bin_xtrans < 2 || bin_xtrans > 16)
337 {
338 _err(err, err_len, "model bin factors out of supported range");
339 goto out;
340 }
341 }
342 else if(_read_net_cfg(cfg, 1, &f_base, &f_depth, &f_in, &f_out))
343 {
344 _err(err, err_len, "model config out of supported range");
345 goto out;
346 }
347 // the OpenCL path indexes weights with int offsets into the float blob
348 if(payload_size / sizeof(float) > (size_t)INT_MAX)
349 {
350 _err(err, err_len, "model payload too large");
351 goto out;
352 }
353 JsonNode *tensors_node = json_object_get_member(root, "tensors");
354 if(!tensors_node || !JSON_NODE_HOLDS_ARRAY(tensors_node))
355 {
356 _err(err, err_len, "model header tensors is not an array");
357 goto out;
358 }
359
360 m = calloc(1, sizeof(dt_nn_model_t));
361 if(!m) goto out;
362 m->has_coarse = is_ms;
363 m->bin_bayer = bin_bayer;
364 m->bin_xtrans = bin_xtrans;
365 if(is_ms && json_object_has_member(cfg, "anchor"))
366 {
367 const int anchor = (int)json_object_get_int_member(cfg, "anchor");
368 if(anchor >= 8 && anchor <= 256) m->anchor = anchor;
369 }
370 m->blob = blob;
371 m->blob_floats = payload_size / sizeof(float);
372 // cfg "sigma_calibration" documents the noise convention the weights were
373 // trained under; it is deliberately NOT read here — the module's sigma
374 // conditioning is carried entirely by user-visible GUI values, never by a
375 // factor hidden inside the model file.
376#ifdef HAVE_OPENCL
377 dt_pthread_mutex_init(&m->cl_lock, NULL);
378#endif
379
380 const nn_header_t h
381 = { .tensors = json_object_get_array_member(root, "tensors"), .payload = blob, .payload_size = payload_size };
382 int bad = _wire_unet(&h, is_ms ? "fine." : "", f_base, f_depth, f_in, f_out, &m->fine, err, err_len);
383 if(is_ms && !bad) bad = _wire_unet(&h, "coarse.", c_base, c_depth, c_in, c_out, &m->coarse, err, err_len);
384
385 if(bad)
386 {
387#ifdef HAVE_OPENCL
388 dt_pthread_mutex_destroy(&m->cl_lock);
389#endif
390 free(m); // blob freed below through the common error path
391 m = NULL;
392 }
393
394out:
395 if(!m) free(blob);
396 free(header);
397 g_object_unref(parser);
398 return m;
399}
400
401#ifdef HAVE_OPENCL
403{
404 for(int d = 0; d < NN_MAX_DEVICES; d++)
405 if(m->dev_weights[d])
406 {
407 dt_opencl_release_mem_object(m->dev_weights[d]);
408 m->dev_weights[d] = NULL;
409 }
410}
411#endif
412
414{
415 if(!m) return;
416#ifdef HAVE_OPENCL
418 dt_pthread_mutex_destroy(&m->cl_lock);
419#endif
420 free(m->blob);
421 free(m);
422}
423
425{
426 return m->fine.in_ch;
427}
428
430{
431 return m->fine.out_ch;
432}
433
434int dt_nn_model_bin(const dt_nn_model_t *m, const int is_xtrans)
435{
436 if(!m->has_coarse) return 1;
437 return is_xtrans ? m->bin_xtrans : m->bin_bayer;
438}
439
441{
442 return m->has_coarse ? m->coarse.in_ch : 0;
443}
444
446{
447 return m->has_coarse ? m->coarse.out_ch : 0;
448}
449
451{
452 return m->anchor;
453}
454
455static int _lcm(int a, int b)
456{
457 if(a <= 0 || b <= 0) return a > b ? a : b; // degenerate inputs: no zero division
458 int x = a, y = b;
459 while(y)
460 {
461 const int t = x % y;
462 x = y;
463 y = t;
464 }
465 return a / x * b;
466}
467
469{
470 // a padded tile must divide by the fine net's stride pyramid AND, for a
471 // multi-scale model, its binned version must divide by the coarse net's —
472 // for either CFA family, since the model file is CFA-agnostic
473 int align = 1 << m->fine.depth;
474 if(m->has_coarse)
475 {
476 align = _lcm(align, m->bin_bayer << m->coarse.depth);
477 align = _lcm(align, m->bin_xtrans << m->coarse.depth);
478 }
479 // ...and, when the model asks for the low-band fusion, by that pyramid too.
480 // The fusion runs at 16/32/64 sensor px (DT_NN_FUSION_COARSEST); a tile the
481 // coarsest band does not divide would silently fall back to a two-level
482 // pyramid, which is NOT what the model was fused against at training time
483 // and — since the tile grid depends on free RAM/vRAM — makes the rendered
484 // result depend on the machine and on whether the pipe tiled at all.
485 if(m->anchor > 0) align = _lcm(align, DT_NN_FUSION_COARSEST);
486 return align;
487}
488
489/* ------------------------------------------------------------------------
490 * executor
491 * ------------------------------------------------------------------------ */
492
493// Number of output channels computed together in the conv inner loop. Each
494// input value is loaded once and reused across this many weight-broadcast FMAs
495// (register-blocking), turning the memory-bound single-channel saxpy into
496// arithmetic-bound work. 4 keeps the accumulators + broadcasts within the SIMD
497// register file on AVX2/NEON. All layer widths here are multiples of 4 except
498// the 1-channel head, which falls through to the scalar-remainder path.
499#define NN_OC_BLOCK 4
500
501/* NOTE on a measured dead end: an 8-wide-strip row-pair microkernel (2 output
502 * rows x 4 output channels, the CPU translation of the GPU quad kernel) was
503 * implemented and benchmarked 2.6x SLOWER than the long-row formulation below
504 * (12.5 vs 4.9 s/MP at 512x512): the short strips defeat the compiler's
505 * full-width row vectorization and the 64-float accumulator block spills.
506 * With compiler-driven SIMD, long streaming rows win; beating them would take
507 * explicit intrinsics, not restructuring. */
508
509// out[oc] = bias[oc] + sum_ic conv(in[ic]); zero padding, any (k, stride).
510//
511// Output channels are blocked NN_OC_BLOCK at a time so each input load feeds
512// that many accumulators (one per output channel) before moving on; the input
513// plane is thus streamed out_ch/NN_OC_BLOCK times instead of out_ch times. The
514// (oy, oc-block) collapse keeps the k input rows hot across the block. The
515// per-output-element accumulation order (ic, ky, kx) is unchanged, so results
516// match the reference bit-for-bit under identical FP settings. The naive
517// formulation (shifted-plane saxpy, output revisited in_ch*k*k times) is
518// memory-bound and was measured ~20x slower at 512x512 tiles.
520static void _conv2d(const nn_conv_t *cv, const float *in, int w, int h, int stride, int pad, float *out)
521{
522 const int k = cv->k;
523 const int ow = (w + 2 * pad - k) / stride + 1;
524 const int oh = (h + 2 * pad - k) / stride + 1;
525 const size_t inhw = (size_t)w * h;
526 const size_t wstride = (size_t)cv->in_ch * k * k; // weight step between output channels
527#ifdef _OPENMP
528#pragma omp parallel
529#endif
530 {
531 float *const acc = malloc(sizeof(float) * NN_OC_BLOCK * ow);
532#ifdef _OPENMP
533#pragma omp for collapse(2) schedule(static)
534#endif
535 for(int oy = 0; oy < oh; oy++)
536 for(int ocb = 0; ocb < cv->out_ch; ocb += NN_OC_BLOCK)
537 {
538 const int nb = NN_MIN(NN_OC_BLOCK, cv->out_ch - ocb);
539 for(int r = 0; r < nb; r++)
540 {
541 float *const ar = acc + (size_t)r * ow;
542 const float bias = cv->b[ocb + r];
543 for(int ox = 0; ox < ow; ox++) ar[ox] = bias;
544 }
545 for(int ic = 0; ic < cv->in_ch; ic++)
546 {
547 const float *const ip = in + (size_t)ic * inhw;
548 for(int ky = 0; ky < k; ky++)
549 {
550 const int iy = oy * stride + ky - pad;
551 if(iy < 0 || iy >= h) continue;
552 const float *const irow = ip + (size_t)iy * w;
553 for(int kx = 0; kx < k; kx++)
554 {
555 const int shift = kx - pad;
556 int ox0 = 0, ox1 = ow;
557 while(ox0 < ow && ox0 * stride + shift < 0) ox0++;
558 while(ox1 > ox0 && (ox1 - 1) * stride + shift >= w) ox1--;
559 const float *const wbase = cv->w + ((size_t)ocb * cv->in_ch + ic) * k * k + ky * k + kx;
560
561 if(nb == NN_OC_BLOCK)
562 {
563 const float w0 = wbase[0], w1 = wbase[wstride];
564 const float w2 = wbase[2 * wstride], w3 = wbase[3 * wstride];
565 float *const a0 = acc, *const a1 = acc + ow;
566 float *const a2 = acc + 2 * ow, *const a3 = acc + 3 * ow;
567 if(stride == 1)
568 {
569 const float *const is = irow + shift;
570#ifdef _OPENMP
571#pragma omp simd
572#endif
573 for(int ox = ox0; ox < ox1; ox++)
574 {
575 const float xv = is[ox];
576 a0[ox] += w0 * xv;
577 a1[ox] += w1 * xv;
578 a2[ox] += w2 * xv;
579 a3[ox] += w3 * xv;
580 }
581 }
582 else
583 for(int ox = ox0; ox < ox1; ox++)
584 {
585 const float xv = irow[ox * stride + shift];
586 a0[ox] += w0 * xv;
587 a1[ox] += w1 * xv;
588 a2[ox] += w2 * xv;
589 a3[ox] += w3 * xv;
590 }
591 }
592 else // remainder block (out_ch not a multiple of NN_OC_BLOCK, e.g. the head)
593 for(int r = 0; r < nb; r++)
594 {
595 const float wv = wbase[(size_t)r * wstride];
596 float *const ar = acc + (size_t)r * ow;
597 if(stride == 1)
598 {
599 const float *const is = irow + shift;
600#ifdef _OPENMP
601#pragma omp simd
602#endif
603 for(int ox = ox0; ox < ox1; ox++) ar[ox] += wv * is[ox];
604 }
605 else
606 for(int ox = ox0; ox < ox1; ox++) ar[ox] += wv * irow[ox * stride + shift];
607 }
608 }
609 }
610 }
611 for(int r = 0; r < nb; r++)
612 memcpy(out + ((size_t)(ocb + r) * oh + oy) * ow, acc + (size_t)r * ow, sizeof(float) * ow);
613 }
614 free(acc);
615 }
616}
617
618// exact GELU, matching pytorch nn.GELU(approximate='none')
620static void _gelu(float *x, size_t n)
621{
622#ifdef _OPENMP
623#pragma omp parallel for schedule(static)
624#endif
625 for(size_t i = 0; i < n; i++) x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * (float)M_SQRT1_2));
626}
627
628
629/* Peak live scratch of one net's forward, in floats: the ledger of the EXACT
630 * allocate/free sequence of the forwards. cl_variant selects which one:
631 * the CPU path (cl_variant 0) reads dec1's concat in place via _conv2d_cat2
632 * and allocates neither the physical concat nor an upsample staging buffer;
633 * the CL path (cl_variant 1) materializes both (cat + us). Any edit to either
634 * forward must be reflected here, or the tiling engine plans against the
635 * wrong number. */
636static size_t _unet_peak_floats(const nn_unet_t *u, size_t wh, int cl_variant)
637{
638 const size_t base = (size_t)u->base;
639 size_t live = 0, peak = 0, cur = 0;
640#define NN_LEDGER(delta) \
641 do \
642 { \
643 live += (delta); \
644 if(live > peak) peak = live; \
645 } while(0)
646 // encoder
647 for(int l = 0; l < u->depth; l++)
648 {
649 const size_t lvl = base * wh >> l;
650 NN_LEDGER(lvl); // tmp
651 NN_LEDGER(lvl); // skips[l] — stays live until its decoder concat
652 NN_LEDGER(lvl >> 2); // next
653 live -= lvl; // tmp freed
654 live -= cur; // previous level's input freed
655 cur = lvl >> 2;
656 }
657 // bottleneck
658 const size_t bot = base * wh >> u->depth;
659 NN_LEDGER(bot);
660 NN_LEDGER(bot);
661 live -= bot;
662 live -= cur;
663 cur = bot;
664 // decoder
665 for(int i = 0; i < u->depth; i++)
666 {
667 const int l = u->depth - 1 - i;
668 const size_t half = base * wh >> l;
669 NN_LEDGER(half >> 2); // v (1x1 output on the coarse grid)
670 live -= cur; // cur freed
671 if(cl_variant)
672 {
673 NN_LEDGER(2 * half); // cat
674 live -= half; // skips[l] freed after its copy
675 NN_LEDGER(half); // us
676 live -= half >> 2; // v freed
677 live -= half; // us freed
678 NN_LEDGER(half); // d1
679 live -= 2 * half; // cat freed
680 }
681 else
682 {
683 NN_LEDGER(half); // d1 (dec1 reads skip + v in place)
684 live -= half >> 2; // v freed
685 live -= half; // skips[l] freed
686 }
687 NN_LEDGER(half); // new cur (dec2 output)
688 live -= half; // d1 freed
689 cur = half;
690 }
691 NN_LEDGER((size_t)u->out_ch * wh); // head
692#undef NN_LEDGER
693 return peak;
694}
695
696static float _scratch_per_px(const dt_nn_model_t *m, int cl_variant)
697{
698 /* A large reference area keeps the integer ledger exact (every term is
699 * base*wh >> k); the result is the dimensionless factor of the input image
700 * size the tiling engine wants — never absolute bytes. The coarse and fine
701 * stages never run concurrently (the caller frees the coarse buffers before
702 * the fine forward), so the model peak is the MAX of the two, not the sum. */
703 const size_t ref = (size_t)1 << 24;
704 float per_px = (float)_unet_peak_floats(&m->fine, ref, cl_variant) / (float)ref;
705 if(m->has_coarse)
706 {
707 const int bin = NN_MIN(m->bin_bayer, m->bin_xtrans); // smaller bin = larger coarse buffer
708 const float coarse
709 = (float)_unet_peak_floats(&m->coarse, ref / ((size_t)bin * bin), cl_variant) / (float)ref;
710 if(coarse > per_px) per_px = coarse;
711 }
712 return per_px;
713}
714
716{
717 return _scratch_per_px(m, 0);
718}
719
721{
722 return _scratch_per_px(m, 1);
723}
724
726{
727 /* Largest SINGLE scratch tensor per input pixel, host path: one base*wh
728 * plane (dec1's output / a skip / an encoder tmp — all equal at level 0).
729 * The pixelpipe arena serves contiguous runs and its address space is
730 * partitioned by entries pinned during the pipe recursion, so the largest
731 * tensor — not the total — is what an allocation can actually get; the
732 * module's tiling_callback uses this to keep it comfortably below the
733 * planned budget. */
734 float per_px = (float)m->fine.base;
735 if(m->has_coarse)
736 {
737 const int bin = NN_MIN(m->bin_bayer, m->bin_xtrans);
738 const float coarse = (float)m->coarse.base / (float)(bin * bin);
739 if(coarse > per_px) per_px = coarse;
740 }
741 return per_px;
742}
743
745{
746 const size_t wh = (size_t)width * height;
747 size_t floats = _unet_peak_floats(&m->fine, wh, 0);
748 if(m->has_coarse)
749 {
750 const int bin = NN_MIN(m->bin_bayer, m->bin_xtrans);
751 const size_t coarse = _unet_peak_floats(&m->coarse, wh / ((size_t)bin * bin), 0);
752 if(coarse > floats) floats = coarse;
753 }
754 return floats * sizeof(float);
755}
756
757/* dec1 variant of _conv2d for k=3, stride=1, pad=1: the input is the channel
758 * concat [a (in_ch_a channels, full res) | b (cv->in_ch - in_ch_a channels,
759 * HALF resolution, read through a nearest-x2 upsample view)]. Reading b in
760 * place is bit-identical to materializing upsample(b) and running _conv2d on
761 * a physical concat — nearest upsampling replicates values, so every tap
762 * reads the same number in the same accumulation order — but the 2*base*wh
763 * concat tensor, the module's largest single allocation and therefore the
764 * arena's contiguity bottleneck, never exists. Keep the accumulation order
765 * identical to _conv2d: same (ic, ky, kx) nesting, same OC blocking. */
767static void _conv2d_cat2(const nn_conv_t *cv, const float *a, int in_ch_a, const float *b, int w, int h,
768 float *out)
769{
770 const int k = cv->k; // always 3 here, kept general for the tap arithmetic
771 const int pad = 1;
772 const int ow = w, oh = h;
773 const int bw = w / 2;
774 const size_t inhw = (size_t)w * h;
775 const size_t bhw = (size_t)bw * (h / 2);
776 const size_t wstride = (size_t)cv->in_ch * k * k;
777#ifdef _OPENMP
778#pragma omp parallel
779#endif
780 {
781 float *const acc = malloc(sizeof(float) * NN_OC_BLOCK * ow);
782#ifdef _OPENMP
783#pragma omp for collapse(2) schedule(static)
784#endif
785 for(int oy = 0; oy < oh; oy++)
786 for(int ocb = 0; ocb < cv->out_ch; ocb += NN_OC_BLOCK)
787 {
788 const int nb = NN_MIN(NN_OC_BLOCK, cv->out_ch - ocb);
789 for(int r = 0; r < nb; r++)
790 {
791 float *const ar = acc + (size_t)r * ow;
792 const float bias = cv->b[ocb + r];
793 for(int ox = 0; ox < ow; ox++) ar[ox] = bias;
794 }
795 for(int ic = 0; ic < cv->in_ch; ic++)
796 {
797 const int from_b = ic >= in_ch_a;
798 const float *const ip = from_b ? b + (size_t)(ic - in_ch_a) * bhw : a + (size_t)ic * inhw;
799 for(int ky = 0; ky < k; ky++)
800 {
801 const int iy = oy + ky - pad;
802 if(iy < 0 || iy >= h) continue;
803 const float *const irow = from_b ? ip + (size_t)(iy >> 1) * bw : ip + (size_t)iy * w;
804 for(int kx = 0; kx < k; kx++)
805 {
806 const int shift = kx - pad;
807 int ox0 = 0, ox1 = ow;
808 while(ox0 < ow && ox0 + shift < 0) ox0++;
809 while(ox1 > ox0 && (ox1 - 1) + shift >= w) ox1--;
810 const float *const wbase = cv->w + ((size_t)ocb * cv->in_ch + ic) * k * k + ky * k + kx;
811 if(nb == NN_OC_BLOCK)
812 {
813 // 4-way output-channel blocking, mirroring _conv2d's fast
814 // path: one pass over the row feeds four accumulators
815 const float w0 = wbase[0], w1 = wbase[wstride];
816 const float w2 = wbase[2 * wstride], w3 = wbase[3 * wstride];
817 float *const a0 = acc, *const a1 = acc + ow;
818 float *const a2 = acc + 2 * ow, *const a3 = acc + 3 * ow;
819 if(from_b)
820 for(int ox = ox0; ox < ox1; ox++)
821 {
822 const float xv = irow[(ox + shift) >> 1];
823 a0[ox] += w0 * xv;
824 a1[ox] += w1 * xv;
825 a2[ox] += w2 * xv;
826 a3[ox] += w3 * xv;
827 }
828 else
829 {
830 const float *const is = irow + shift;
831#ifdef _OPENMP
832#pragma omp simd
833#endif
834 for(int ox = ox0; ox < ox1; ox++)
835 {
836 const float xv = is[ox];
837 a0[ox] += w0 * xv;
838 a1[ox] += w1 * xv;
839 a2[ox] += w2 * xv;
840 a3[ox] += w3 * xv;
841 }
842 }
843 }
844 else
845 for(int r = 0; r < nb; r++)
846 {
847 const float wr = wbase[(size_t)r * wstride];
848 float *const ar = acc + (size_t)r * ow;
849 if(from_b)
850 for(int ox = ox0; ox < ox1; ox++) ar[ox] += wr * irow[(ox + shift) >> 1];
851 else
852 {
853 const float *const is = irow + shift;
854#ifdef _OPENMP
855#pragma omp simd
856#endif
857 for(int ox = ox0; ox < ox1; ox++) ar[ox] += wr * is[ox];
858 }
859 }
860 }
861 }
862 }
863 for(int r = 0; r < nb; r++)
864 memcpy(out + ((size_t)(ocb + r)) * inhw + (size_t)oy * ow, acc + (size_t)r * ow,
865 sizeof(float) * ow);
866 }
867 free(acc);
868 }
869}
870
871/* Full forward pass of one U-Net.
872 *
873 * Memory discipline — this is where the module's RAM peak lives, so every
874 * tensor is allocated at its EXACT size the moment it is needed and released
875 * on its last use, instead of the former three worst-case ping-pong arenas
876 * plus all skips held to the end (7.9*base*wh floats live). The live-set peak
877 * is now dec1 at level 0: ~2.25*base*wh floats, with the largest single
878 * allocation ONE base*wh plane — the binding constraint for the pixelpipe
879 * arena, which serves contiguous runs. _unet_peak_floats()
880 * is the ledger of this exact sequence — KEEP THE TWO IN SYNC.
881 *
882 * The 1x1 up-convs are applied BEFORE their nearest x2 upsample: a 1x1 conv
883 * is per-pixel and nearest upsampling replicates pixels, so conv(up(x)) and
884 * up(conv(x)) are the same values from the same FP operations — but computed
885 * on 4x fewer pixels, and the (2*w_skip)@full-res tensor never exists.
886 *
887 * residual_ch > 0 subtracts the head from the input's first residual_ch
888 * planes; residual_ch == 0 writes the raw head output. */
889static int _unet_forward(const nn_unet_t *u, const float *in, float *out, int width, int height, int residual_ch)
890{
891 const int align = 1 << u->depth;
892 if(width % align || height % align || width <= 0 || height <= 0) return 1;
893
894 const size_t wh = (size_t)width * height;
895 const size_t base = (size_t)u->base;
896 float *skips[NN_MAX_DEPTH] = { NULL };
897
898 // encoder: skip[l] = (base<<l) channels at (wh >> 2l) px = base*wh >> l floats
899 const float *src = in;
900 int cw = width, chh = height;
901 float *cur = NULL;
902 int ok = 1;
903 int cancelled = 0;
904 /* The cancel hook is asked once per level: a level quarters the pixels and
905 * doubles the channels, so each costs about the same and the forward is cut
906 * into 2 * depth + 2 roughly equal steps. */
907 for(int l = 0; l < u->depth && ok; l++)
908 {
910 {
911 cancelled = 1;
912 ok = 0;
913 break;
914 }
915 const size_t lvl = base * wh >> l;
916 float *tmp = _nn_alloc(lvl, 0);
917 skips[l] = _nn_alloc(lvl, 1);
918 float *next = _nn_alloc(lvl >> 2, 0);
919 if(!tmp || !skips[l] || !next)
920 {
921 _nn_free(tmp);
922 _nn_free(next);
923 ok = 0;
924 break;
925 }
926 _conv2d(&u->enc1[l], src, cw, chh, 1, 1, tmp);
927 _gelu(tmp, (size_t)u->enc1[l].out_ch * cw * chh);
928 _conv2d(&u->enc2[l], tmp, cw, chh, 1, 1, skips[l]);
929 _gelu(skips[l], (size_t)u->enc2[l].out_ch * cw * chh);
930 _nn_free(tmp);
931 _conv2d(&u->down[l], skips[l], cw, chh, 2, 0, next);
932 _nn_free(cur); // level l's input, dead now (never frees `in`: cur is NULL then)
933 cur = next;
934 cw /= 2;
935 chh /= 2;
936 src = cur;
937 }
938
939 // bottleneck: (base<<depth) channels at wh >> 2*depth px
940 if(ok && _nn_cancel_fn && _nn_cancel_fn())
941 {
942 cancelled = 1;
943 ok = 0;
944 }
945 if(ok)
946 {
947 const size_t bot = base * wh >> u->depth;
948 float *tmp = _nn_alloc(bot, 0);
949 float *bout = _nn_alloc(bot, 0);
950 if(!tmp || !bout)
951 {
952 _nn_free(tmp);
953 _nn_free(bout);
954 ok = 0;
955 }
956 else
957 {
958 _conv2d(&u->bot1, src, cw, chh, 1, 1, tmp);
959 _gelu(tmp, (size_t)u->bot1.out_ch * cw * chh);
960 _conv2d(&u->bot2, tmp, cw, chh, 1, 1, bout);
961 _gelu(bout, (size_t)u->bot2.out_ch * cw * chh);
962 _nn_free(tmp);
963 _nn_free(cur);
964 cur = bout;
965 }
966 }
967
968 // decoder: up.i / dec.i #i pairs with encoder level (depth-1-i). The 1x1
969 // up-conv runs on the coarse grid (see the doc comment), and dec1 reads its
970 // concat input IN PLACE via _conv2d_cat2 — skip at full resolution, the 1x1
971 // output through a nearest-upsample view — so the module's former largest
972 // tensor (the physical 2*w_skip concat) is never allocated at all.
973 for(int i = 0; i < u->depth && ok; i++)
974 {
976 {
977 cancelled = 1;
978 ok = 0;
979 break;
980 }
981 const int l = u->depth - 1 - i;
982 const size_t w_skip = base << l;
983 const size_t half = w_skip * (size_t)(2 * cw) * (size_t)(2 * chh); // one concat half
984 float *v = _nn_alloc(half >> 2, 1); // top end: must not split the big-tensor churn area
985 if(!v) { ok = 0; break; }
986 _conv2d(&u->up[i], cur, cw, chh, 1, 0, v);
987 _nn_free(cur);
988 cur = NULL;
989 cw *= 2;
990 chh *= 2;
991 float *d1 = _nn_alloc(half, 0);
992 if(!d1) { _nn_free(v); ok = 0; break; }
993 _conv2d_cat2(&u->dec1[i], skips[l], (int)w_skip, v, cw, chh, d1);
994 _gelu(d1, w_skip * (size_t)cw * chh);
995 _nn_free(v);
996 _nn_free(skips[l]);
997 skips[l] = NULL;
998 float *d2 = _nn_alloc(half, 0);
999 if(!d2) { _nn_free(d1); ok = 0; break; }
1000 _conv2d(&u->dec2[i], d1, cw, chh, 1, 1, d2);
1001 _gelu(d2, w_skip * (size_t)cw * chh);
1002 _nn_free(d1);
1003 cur = d2;
1004 }
1005
1006 if(ok && _nn_cancel_fn && _nn_cancel_fn())
1007 {
1008 cancelled = 1;
1009 ok = 0;
1010 }
1011 if(ok)
1012 {
1013 float *head = _nn_alloc((size_t)u->out_ch * wh, 0);
1014 if(!head)
1015 ok = 0;
1016 else
1017 {
1018 _conv2d(&u->head, cur, width, height, 1, 1, head);
1019 if(residual_ch > 0)
1020 {
1021 // residual head: out = input planes - predicted noise
1022#ifdef _OPENMP
1023#pragma omp parallel for schedule(static)
1024#endif
1025 for(size_t i = 0; i < (size_t)residual_ch * wh; i++) out[i] = in[i] - head[i];
1026 }
1027 else
1028 memcpy(out, head, (size_t)u->out_ch * wh * sizeof(float));
1029 _nn_free(head);
1030 }
1031 }
1032
1033 for(int l = 0; l < u->depth; l++) _nn_free(skips[l]);
1034 _nn_free(cur);
1035 if(cancelled) return DT_NN_CANCELLED;
1036 return ok ? 0 : 1;
1037}
1038
1039int dt_nn_unet_apply(const dt_nn_model_t *m, const float *in, float *out, int width, int height)
1040{
1041 return _unet_forward(&m->fine, in, out, width, height, m->fine.out_ch);
1042}
1043
1044int dt_nn_unet_apply_stage(const dt_nn_model_t *m, int stage, const float *in, float *out, int width,
1045 int height, int apply_residual)
1046{
1047 if(stage == 1)
1048 {
1049 if(!m->has_coarse) return 1;
1050 // coarse stage: the head predicts a correction to its RGB planes
1051 return _unet_forward(&m->coarse, in, out, width, height, apply_residual ? m->coarse.out_ch : 0);
1052 }
1053 return _unet_forward(&m->fine, in, out, width, height, apply_residual ? m->fine.out_ch : 0);
1054}
1055
1057void dt_nn_bin_planes(const float *planes, int pw, int ph, int bin, float *out_rgb, float *out_cnt)
1058{
1059 // planes = [mosaic, onehot_R, onehot_G, onehot_B, ...] as assembled for the
1060 // fine net. Each coarse pixel is the count-weighted mean of the block's
1061 // same-channel sensels — the exact contract of cfa.bin_mosaic_torch in the
1062 // training repo. With bin 4 (Bayer) / 6 (X-Trans) every count is > 0 by
1063 // construction; the max() is a numerical guard, not a fallback.
1064 const size_t plane = (size_t)pw * ph;
1065 const int cw = pw / bin, chh = ph / bin;
1066 const float *const mosaic = planes;
1067#ifdef _OPENMP
1068#pragma omp parallel for schedule(static) collapse(2)
1069#endif
1070 for(int c = 0; c < 3; c++)
1071 for(int cy = 0; cy < chh; cy++)
1072 {
1073 const float *const onehot = planes + (size_t)(1 + c) * plane;
1074 float *const orow = out_rgb + (size_t)c * cw * chh + (size_t)cy * cw;
1075 float *const crow = out_cnt + (size_t)c * cw * chh + (size_t)cy * cw;
1076 for(int cx = 0; cx < cw; cx++)
1077 {
1078 float sum = 0.0f, cnt = 0.0f;
1079 for(int y = cy * bin; y < (cy + 1) * bin; y++)
1080 for(int x = cx * bin; x < (cx + 1) * bin; x++)
1081 {
1082 const size_t i = (size_t)y * pw + x;
1083 sum += mosaic[i] * onehot[i];
1084 cnt += onehot[i];
1085 }
1086 crow[cx] = cnt;
1087 orow[cx] = sum / (cnt > 1.0f ? cnt : 1.0f);
1088 }
1089 }
1090}
1091
1093void dt_nn_upsample_nearest(const float *in, int ch, int w, int h, int factor, float *out)
1094{
1095#ifdef _OPENMP
1096#pragma omp parallel for schedule(static)
1097#endif
1098 for(int c = 0; c < ch; c++)
1099 {
1100 const float *const ip = in + (size_t)c * w * h;
1101 float *const op = out + (size_t)c * w * h * factor * factor;
1102 for(int y = 0; y < h * factor; y++)
1103 {
1104 const float *const irow = ip + (size_t)(y / factor) * w;
1105 float *const orow = op + (size_t)y * w * factor;
1106 for(int x = 0; x < w * factor; x++) orow[x] = irow[x / factor];
1107 }
1108 }
1109}
1110
1111/* ------------------------------------------------------------------------
1112 * OpenCL executor
1113 * ------------------------------------------------------------------------ */
1114#ifdef HAVE_OPENCL
1115
1122
1124{
1125 dt_nn_cl_t *cl = calloc(1, sizeof(dt_nn_cl_t));
1126 if(!cl) return NULL;
1127 cl->kernel_conv = dt_opencl_create_kernel(program, "nn_conv");
1128 cl->kernel_conv3x3 = dt_opencl_create_kernel(program, "nn_conv3x3");
1129 cl->kernel_upsample = dt_opencl_create_kernel(program, "nn_upsample");
1130 return cl;
1131}
1132
1134{
1135 if(!cl) return;
1139 free(cl);
1140}
1141
1142
1143// upload the weight blob to the device once, cached per device
1144static cl_mem _weights_cl(const dt_nn_model_t *m, int devid)
1145{
1146 if(devid < 0 || devid >= NN_MAX_DEVICES) return NULL;
1147 dt_nn_model_t *mm = (dt_nn_model_t *)m; // cache mutation on a logically const model
1149 if(!mm->dev_weights[devid])
1150 {
1151 const size_t bytes = m->blob_floats * sizeof(float);
1152 cl_mem buf = dt_opencl_alloc_device_buffer(devid, bytes);
1153 if(buf && dt_opencl_write_buffer_to_device(devid, m->blob, buf, 0, bytes, CL_TRUE) == CL_SUCCESS)
1154 mm->dev_weights[devid] = buf;
1155 else if(buf)
1157 }
1158 cl_mem w = mm->dev_weights[devid];
1160 return w;
1161}
1162
1163// enqueue one convolution (+ optional GELU) reading from `in`, writing `out`
1164static int _conv_cl(dt_nn_cl_t *cl, int devid, cl_mem weights, const float *blob_base, cl_mem in, cl_mem out,
1165 int w, int h, const nn_conv_t *cv, int stride, int pad, int do_gelu)
1166{
1167 const int ow = (w + 2 * pad - cv->k) / stride + 1;
1168 const int oh = (h + 2 * pad - cv->k) / stride + 1;
1169 const int weight_off = (int)(cv->w - blob_base);
1170 const int bias_off = (int)(cv->b - blob_base);
1171 const size_t slice_bytes = sizeof(float) * 4 * cv->in_ch * cv->k * cv->k;
1172 const int use_local = slice_bytes <= (30 << 10);
1173 const size_t lx = 128;
1174
1175 if(cv->k == 3 && stride == 1)
1176 {
1177 // fast quad kernel: 2x2 output pixels per work-item (see rawdenoiseai.cl).
1178 // Weights are staged in local memory `chunk` input channels at a time so
1179 // even the in_ch 256/512 layers never read weights per-item from global
1180 // (~24 KB of local keeps 2 work-groups per SM on common GPUs).
1181 const int chunk = NN_MIN(cv->in_ch, (24 << 10) / (int)(sizeof(float) * 4 * 9));
1182 const int K3 = cl->kernel_conv3x3;
1183 dt_opencl_set_kernel_arg(devid, K3, 0, sizeof(cl_mem), &in);
1184 dt_opencl_set_kernel_arg(devid, K3, 1, sizeof(cl_mem), &weights);
1185 dt_opencl_set_kernel_arg(devid, K3, 2, sizeof(cl_mem), &out);
1186 dt_opencl_set_kernel_arg(devid, K3, 3, sizeof(int), &w);
1187 dt_opencl_set_kernel_arg(devid, K3, 4, sizeof(int), &h);
1188 dt_opencl_set_kernel_arg(devid, K3, 5, sizeof(int), &cv->in_ch);
1189 dt_opencl_set_kernel_arg(devid, K3, 6, sizeof(int), &cv->out_ch);
1190 dt_opencl_set_kernel_arg(devid, K3, 7, sizeof(int), &weight_off);
1191 dt_opencl_set_kernel_arg(devid, K3, 8, sizeof(int), &bias_off);
1192 dt_opencl_set_kernel_arg(devid, K3, 9, sizeof(int), &do_gelu);
1193 dt_opencl_set_kernel_arg(devid, K3, 10, sizeof(int), &chunk);
1194 dt_opencl_set_kernel_arg(devid, K3, 11, sizeof(float) * 4 * 9 * chunk, NULL);
1195 const size_t quads = (size_t)((w + 1) / 2) * ((h + 1) / 2);
1196 size_t sizes3[3] = { (quads + lx - 1) / lx * lx, ((size_t)cv->out_ch + 3) / 4, 1 };
1197 size_t local3[3] = { lx, 1, 1 };
1198 return dt_opencl_enqueue_kernel_2d_with_local(devid, K3, sizes3, local3);
1199 }
1200
1201 const int K = cl->kernel_conv;
1202 dt_opencl_set_kernel_arg(devid, K, 0, sizeof(cl_mem), &in);
1203 dt_opencl_set_kernel_arg(devid, K, 1, sizeof(cl_mem), &weights);
1204 dt_opencl_set_kernel_arg(devid, K, 2, sizeof(cl_mem), &out);
1205 dt_opencl_set_kernel_arg(devid, K, 3, sizeof(int), &w);
1206 dt_opencl_set_kernel_arg(devid, K, 4, sizeof(int), &h);
1207 dt_opencl_set_kernel_arg(devid, K, 5, sizeof(int), &ow);
1208 dt_opencl_set_kernel_arg(devid, K, 6, sizeof(int), &oh);
1209 dt_opencl_set_kernel_arg(devid, K, 7, sizeof(int), &cv->in_ch);
1210 dt_opencl_set_kernel_arg(devid, K, 8, sizeof(int), &cv->out_ch);
1211 dt_opencl_set_kernel_arg(devid, K, 9, sizeof(int), &cv->k);
1212 dt_opencl_set_kernel_arg(devid, K, 10, sizeof(int), &stride);
1213 dt_opencl_set_kernel_arg(devid, K, 11, sizeof(int), &pad);
1214 dt_opencl_set_kernel_arg(devid, K, 12, sizeof(int), &weight_off);
1215 dt_opencl_set_kernel_arg(devid, K, 13, sizeof(int), &bias_off);
1216 dt_opencl_set_kernel_arg(devid, K, 14, sizeof(int), &do_gelu);
1217 // 4 output channels per work-item (NN_OCB in the kernel); their weight
1218 // slices are staged in local memory when they fit the conservative 30 KB
1219 // budget — always true for the wide shallow layers where weight traffic
1220 // matters. (Staging the INPUT rows in local memory was tried and measured
1221 // slower: with the output-channel blocking in work dim 1, every oc-block
1222 // group re-stages the same rows and the per-channel barriers serialize,
1223 // while the L2 already absorbs the 3x3 neighbourhood overlap.)
1224 dt_opencl_set_kernel_arg(devid, K, 15, sizeof(int), &use_local);
1225 dt_opencl_set_kernel_arg(devid, K, 16, use_local ? slice_bytes : sizeof(float), NULL);
1226 size_t sizes[3] = { ((size_t)ow * oh + lx - 1) / lx * lx, ((size_t)cv->out_ch + 3) / 4, 1 };
1227 size_t local[3] = { lx, 1, 1 };
1228 return dt_opencl_enqueue_kernel_2d_with_local(devid, K, sizes, local);
1229}
1230
1231static int _upsample_cl(dt_nn_cl_t *cl, int devid, cl_mem in, cl_mem out, int w, int h, int ch)
1232{
1233 const int K = cl->kernel_upsample;
1234 dt_opencl_set_kernel_arg(devid, K, 0, sizeof(cl_mem), &in);
1235 dt_opencl_set_kernel_arg(devid, K, 1, sizeof(cl_mem), &out);
1236 dt_opencl_set_kernel_arg(devid, K, 2, sizeof(int), &w);
1237 dt_opencl_set_kernel_arg(devid, K, 3, sizeof(int), &h);
1238 dt_opencl_set_kernel_arg(devid, K, 4, sizeof(int), &ch);
1239 size_t sizes[3] = { ROUNDUPDWD((size_t)4 * w * h, devid), ROUNDUPDHT(ch, devid), 1 };
1240 return dt_opencl_enqueue_kernel_2d(devid, K, sizes);
1241}
1242
1243// forward one U-Net on the device. Writes the RAW head output (no residual):
1244// the fine stage's residual is applied by the caller after readback (the
1245// historic CPU/CL asymmetry), and the coarse stage's residual is applied
1246// host-side too so both paths share the same subtraction code bit-for-bit.
1247/* Device twin of _unet_forward: the same live-set discipline — every buffer
1248 * allocated at its exact size when needed and released on last use, the same
1249 * 1x1-before-upsample commutation — so VRAM peaks at the level-0 concat
1250 * (~3.25*base*wh floats) instead of holding three worst-case arenas plus
1251 * every skip to the end. All temporaries live at function scope and are released in
1252 * cleanup, so a device-OOM mid-sequence (the very case this path exists for:
1253 * fall back to CPU on a full card) cannot leak. Releases happen at enqueue
1254 * time and the runtime defers destruction until in-flight commands finish, so
1255 * on drivers that commit at clCreateBuffer the instantaneous footprint can
1256 * transiently exceed the ledger by the pending-release set; the factor_cl
1257 * headroom absorbs that, and the failure mode is the graceful CPU fallback. _unet_peak_floats() is the
1258 * shared ledger of this sequence: KEEP ALL THREE IN SYNC. */
1259static int _unet_forward_cl(const dt_nn_model_t *m, const nn_unet_t *u, dt_nn_cl_t *cl, int devid, cl_mem dev_in,
1260 cl_mem dev_out, int width, int height)
1261{
1262 const int align = 1 << u->depth;
1263 if(width % align || height % align || width <= 0 || height <= 0) return -1;
1264
1265 cl_mem weights = _weights_cl(m, devid);
1266 if(!weights) return -1;
1267
1268 const size_t wh = (size_t)width * height;
1269 const size_t base = (size_t)u->base;
1270 int err = CL_SUCCESS;
1271 cl_mem skips[NN_MAX_DEPTH] = { NULL };
1272 cl_mem cur = NULL, tmp = NULL, v = NULL, cat = NULL, us = NULL, d1 = NULL;
1273
1274#define NN_CL_ALLOC(var, floats) \
1275 do \
1276 { \
1277 var = dt_opencl_alloc_device_buffer(devid, (floats) * sizeof(float)); \
1278 if(!var) \
1279 { \
1280 err = -1; \
1281 goto cleanup; \
1282 } \
1283 } while(0)
1284#define NN_CL_FREE(var) \
1285 do \
1286 { \
1287 if(var) dt_opencl_release_mem_object(var); \
1288 var = NULL; \
1289 } while(0)
1290
1291 /* Cancellation, as in _unet_forward and at the same points. The host enqueues
1292 * the whole forward in milliseconds and only waits once it is all queued, so
1293 * a check at enqueue time would always come before the device started:
1294 * each check first waits for what is already queued (one level's work), and
1295 * OpenCL cannot abort enqueued kernels anyway. */
1296 // encoder
1297 cl_mem src = dev_in;
1298 int cw = width, chh = height;
1299 for(int l = 0; l < u->depth && err == CL_SUCCESS; l++)
1300 {
1302 {
1303 err = DT_NN_CANCELLED;
1304 goto cleanup;
1305 }
1306 const size_t lvl = base * wh >> l;
1307 cl_mem next = NULL;
1308 NN_CL_ALLOC(tmp, lvl);
1309 NN_CL_ALLOC(skips[l], lvl);
1310 NN_CL_ALLOC(next, lvl >> 2);
1311 err |= _conv_cl(cl, devid, weights, m->blob, src, tmp, cw, chh, &u->enc1[l], 1, 1, 1);
1312 err |= _conv_cl(cl, devid, weights, m->blob, tmp, skips[l], cw, chh, &u->enc2[l], 1, 1, 1);
1313 NN_CL_FREE(tmp);
1314 err |= _conv_cl(cl, devid, weights, m->blob, skips[l], next, cw, chh, &u->down[l], 2, 0, 0);
1315 NN_CL_FREE(cur); // level l's input; never dev_in (cur is NULL on l == 0)
1316 cur = next;
1317 cw /= 2;
1318 chh /= 2;
1319 src = cur;
1320 }
1321
1322 // bottleneck (bout reuses the `v` slot so cleanup covers it)
1323 if(err == CL_SUCCESS)
1324 {
1326 {
1327 err = DT_NN_CANCELLED;
1328 goto cleanup;
1329 }
1330 const size_t bot = base * wh >> u->depth;
1331 NN_CL_ALLOC(tmp, bot);
1332 NN_CL_ALLOC(v, bot);
1333 err |= _conv_cl(cl, devid, weights, m->blob, src, tmp, cw, chh, &u->bot1, 1, 1, 1);
1334 err |= _conv_cl(cl, devid, weights, m->blob, tmp, v, cw, chh, &u->bot2, 1, 1, 1);
1335 NN_CL_FREE(tmp);
1336 NN_CL_FREE(cur);
1337 cur = v;
1338 v = NULL;
1339 }
1340
1341 // decoder: up.i / dec.i pair with encoder level l = depth-1-i; the 1x1
1342 // up-conv runs on the coarse grid, then upsamples (see the doc comment)
1343 for(int i = 0; i < u->depth && err == CL_SUCCESS; i++)
1344 {
1346 {
1347 err = DT_NN_CANCELLED;
1348 goto cleanup;
1349 }
1350 const int l = u->depth - 1 - i;
1351 const size_t w_skip = base << l;
1352 const size_t half = w_skip * (size_t)(2 * cw) * (size_t)(2 * chh);
1353 NN_CL_ALLOC(v, half >> 2);
1354 err |= _conv_cl(cl, devid, weights, m->blob, cur, v, cw, chh, &u->up[i], 1, 0, 0);
1355 NN_CL_FREE(cur);
1356 NN_CL_ALLOC(cat, 2 * half);
1357 err |= dt_opencl_enqueue_copy_buffer_to_buffer(devid, skips[l], cat, 0, 0, half * sizeof(float));
1358 NN_CL_FREE(skips[l]);
1359 NN_CL_ALLOC(us, half);
1360 err |= _upsample_cl(cl, devid, v, us, cw, chh, u->up[i].out_ch);
1361 NN_CL_FREE(v);
1362 err |= dt_opencl_enqueue_copy_buffer_to_buffer(devid, us, cat, 0, half * sizeof(float),
1363 half * sizeof(float));
1364 NN_CL_FREE(us);
1365 cw *= 2;
1366 chh *= 2;
1367 NN_CL_ALLOC(d1, half);
1368 err |= _conv_cl(cl, devid, weights, m->blob, cat, d1, cw, chh, &u->dec1[i], 1, 1, 1);
1369 NN_CL_FREE(cat);
1370 NN_CL_ALLOC(cur, half);
1371 err |= _conv_cl(cl, devid, weights, m->blob, d1, cur, cw, chh, &u->dec2[i], 1, 1, 1);
1372 NN_CL_FREE(d1);
1373 }
1374
1375 // head: raw prediction (no activation) into dev_out
1376 if(err == CL_SUCCESS && _nn_cancel_fn && dt_opencl_finish(devid) && _nn_cancel_fn())
1377 err = DT_NN_CANCELLED;
1378 if(err == CL_SUCCESS)
1379 err |= _conv_cl(cl, devid, weights, m->blob, cur, dev_out, width, height, &u->head, 1, 1, 0);
1380
1381cleanup:
1382 for(int l = 0; l < u->depth; l++)
1383 if(skips[l]) dt_opencl_release_mem_object(skips[l]);
1384 NN_CL_FREE(cur);
1385 NN_CL_FREE(tmp);
1386 NN_CL_FREE(v);
1387 NN_CL_FREE(cat);
1388 NN_CL_FREE(us);
1389 NN_CL_FREE(d1);
1390 return err;
1391#undef NN_CL_ALLOC
1392#undef NN_CL_FREE
1393}
1394
1395int dt_nn_unet_apply_stage_cl(const dt_nn_model_t *m, int stage, dt_nn_cl_t *cl, int devid, cl_mem dev_in,
1396 cl_mem dev_out, int width, int height)
1397{
1398 if(stage == 1)
1399 {
1400 if(!m->has_coarse) return -1;
1401 return _unet_forward_cl(m, &m->coarse, cl, devid, dev_in, dev_out, width, height);
1402 }
1403 return _unet_forward_cl(m, &m->fine, cl, devid, dev_in, dev_out, width, height);
1404}
1405
1406#endif // HAVE_OPENCL
void cleanup(dt_imageio_module_format_t *self)
Definition avif.c:170
#define m
Definition basecurve.c:283
static const float x
const float f
const int t
const float v
const dt_colormatrix_t dt_aligned_pixel_t out
static int dt_pthread_mutex_unlock(dt_pthread_mutex_t *mutex) RELEASE(mutex) NO_THREAD_SAFETY_ANALYSIS
Definition dtpthread.h:127
static int dt_pthread_mutex_init(dt_pthread_mutex_t *mutex, const pthread_mutexattr_t *mutexattr)
Initialise a mutex. With mutexattr NULL – which is how 54 of the 56 call sites in this tree spell it ...
Definition dtpthread.h:104
static int dt_pthread_mutex_destroy(dt_pthread_mutex_t *mutex)
Definition dtpthread.h:132
static int dt_pthread_mutex_lock(dt_pthread_mutex_t *mutex) ACQUIRE(mutex) NO_THREAD_SAFETY_ANALYSIS
Definition dtpthread.h:117
memcpy(roundtrip, origin, sizeof(roundtrip))
dt_iop_roi_t in
Definition geometry.h:117
#define w2
Definition lmmse.c:60
#define w1
Definition lmmse.c:59
#define w3
Definition lmmse.c:61
float *const restrict const size_t k
float *const restrict const size_t const size_t ch
uint32_t width
Definition mipmap_cache.c:0
uint32_t height
Definition mipmap_cache.c:1
size_t size
Definition mipmap_cache.c:3
void dt_nn_cl_destroy(dt_nn_cl_t *cl)
Definition nn_model.c:1133
int dt_nn_unet_apply_stage_cl(const dt_nn_model_t *m, int stage, dt_nn_cl_t *cl, int devid, cl_mem dev_in, cl_mem dev_out, int width, int height)
Definition nn_model.c:1395
int dt_nn_model_anchor(const dt_nn_model_t *m)
Definition nn_model.c:450
#define NN_CL_ALLOC(var, floats)
#define NN_MIN(a, b)
Definition nn_model.c:34
void dt_nn_model_free(dt_nn_model_t *m)
Definition nn_model.c:413
static dt_nn_alloc_f _nn_alloc_fn
Definition nn_model.c:74
#define NN_LEDGER(delta)
float dt_nn_unet_scratch_maxblock_per_px(const dt_nn_model_t *m)
Definition nn_model.c:725
static size_t _unet_peak_floats(const nn_unet_t *u, size_t wh, int cl_variant)
Definition nn_model.c:636
static cl_mem _weights_cl(const dt_nn_model_t *m, int devid)
Definition nn_model.c:1144
int dt_nn_model_in_channels(const dt_nn_model_t *m)
Definition nn_model.c:424
#define NN_OC_BLOCK
Definition nn_model.c:499
static void _err(char *err, size_t err_len, const char *msg)
Definition nn_model.c:116
#define NN_MAX_DEPTH
Definition nn_model.c:33
int dt_nn_model_coarse_in_channels(const dt_nn_model_t *m)
Definition nn_model.c:440
dt_nn_cl_t * dt_nn_cl_create(int program)
Definition nn_model.c:1123
static int _conv_cl(dt_nn_cl_t *cl, int devid, cl_mem weights, const float *blob_base, cl_mem in, cl_mem out, int w, int h, const nn_conv_t *cv, int stride, int pad, int do_gelu)
Definition nn_model.c:1164
static int _wire_conv(const nn_header_t *h, const char *prefix, int out_ch, int in_ch, int k, nn_conv_t *cv, char *err, size_t err_len)
Definition nn_model.c:134
static int _unet_forward(const nn_unet_t *u, const float *in, float *out, int width, int height, int residual_ch)
Definition nn_model.c:889
static int _unet_forward_cl(const dt_nn_model_t *m, const nn_unet_t *u, dt_nn_cl_t *cl, int devid, cl_mem dev_in, cl_mem dev_out, int width, int height)
Definition nn_model.c:1259
size_t dt_nn_unet_scratch_bytes(const dt_nn_model_t *m, int width, int height)
Definition nn_model.c:744
int dt_nn_unet_apply_stage(const dt_nn_model_t *m, int stage, const float *in, float *out, int width, int height, int apply_residual)
Definition nn_model.c:1044
static __DT_CLONE_TARGETS__ void _conv2d_cat2(const nn_conv_t *cv, const float *a, int in_ch_a, const float *b, int w, int h, float *out)
Definition nn_model.c:767
static __DT_CLONE_TARGETS__ void _conv2d(const nn_conv_t *cv, const float *in, int w, int h, int stride, int pad, float *out)
Definition nn_model.c:520
#define NN_MAX_DEVICES
Definition nn_model.c:44
static void dt_nn_model_free_cl(dt_nn_model_t *m)
Definition nn_model.c:402
static void * _nn_alloc(size_t floats, int long_lived)
Definition nn_model.c:101
static dt_nn_free_f _nn_free_fn
Definition nn_model.c:75
void dt_nn_set_cancel(dt_nn_cancel_f cancel_fn)
Definition nn_model.c:86
static void _nn_free(void *p)
Definition nn_model.c:107
float dt_nn_unet_scratch_per_px(const dt_nn_model_t *m)
Definition nn_model.c:715
dt_nn_model_t * dt_nn_model_load(const char *path, char *err, size_t err_len)
Definition nn_model.c:245
__DT_CLONE_TARGETS__ void dt_nn_upsample_nearest(const float *in, int ch, int w, int h, int factor, float *out)
Definition nn_model.c:1093
static __DT_CLONE_TARGETS__ void _gelu(float *x, size_t n)
Definition nn_model.c:620
int dt_nn_model_coarse_out_channels(const dt_nn_model_t *m)
Definition nn_model.c:445
static int _lcm(int a, int b)
Definition nn_model.c:455
float dt_nn_unet_scratch_per_px_cl(const dt_nn_model_t *m)
Definition nn_model.c:720
int dt_nn_model_alignment(const dt_nn_model_t *m)
Definition nn_model.c:468
int dt_nn_unet_apply(const dt_nn_model_t *m, const float *in, float *out, int width, int height)
Definition nn_model.c:1039
static dt_nn_cancel_f _nn_cancel_fn
Definition nn_model.c:84
static int _read_net_cfg(JsonObject *cfg, int out_ch_max, int *base, int *depth, int *in_ch, int *out_ch)
Definition nn_model.c:232
__DT_CLONE_TARGETS__ void dt_nn_bin_planes(const float *planes, int pw, int ph, int bin, float *out_rgb, float *out_cnt)
Definition nn_model.c:1057
#define NN_CL_FREE(var)
int dt_nn_model_bin(const dt_nn_model_t *m, const int is_xtrans)
Definition nn_model.c:434
static int _wire_unet(const nn_header_t *h, const char *stage_prefix, int base, int depth, int in_ch, int out_ch, nn_unet_t *u, char *err, size_t err_len)
Definition nn_model.c:187
static float _scratch_per_px(const dt_nn_model_t *m, int cl_variant)
Definition nn_model.c:696
static int _upsample_cl(dt_nn_cl_t *cl, int devid, cl_mem in, cl_mem out, int w, int h, int ch)
Definition nn_model.c:1231
int dt_nn_model_out_channels(const dt_nn_model_t *m)
Definition nn_model.c:429
void dt_nn_set_allocator(dt_nn_alloc_f alloc_fn, dt_nn_free_f free_fn)
Definition nn_model.c:77
void *(* dt_nn_alloc_f)(size_t bytes, int long_lived)
Definition nn_model.h:65
void(* dt_nn_free_f)(void *ptr)
Definition nn_model.h:66
#define DT_NN_CANCELLED
Definition nn_model.h:78
#define DT_NN_FUSION_COARSEST
Definition nn_model.h:111
int(* dt_nn_cancel_f)(void)
Definition nn_model.h:79
int dt_opencl_enqueue_kernel_2d(const int dev, const int kernel, const size_t *sizes)
Definition opencl.c:2644
void * dt_opencl_alloc_device_buffer(const int devid, const size_t size)
Definition opencl.c:3060
int dt_opencl_create_kernel(const int prog, const char *name)
Definition opencl.c:2538
int dt_opencl_write_buffer_to_device(const int devid, void *host, void *device, const size_t offset, const size_t size, const int blocking)
Definition opencl.c:2828
void dt_opencl_free_kernel(const int kernel)
Definition opencl.c:2581
int dt_opencl_set_kernel_arg(const int dev, const int kernel, const int num, const size_t size, const void *arg)
Definition opencl.c:2635
gboolean dt_opencl_finish(const int devid)
Definition opencl.c:1744
int dt_opencl_enqueue_kernel_2d_with_local(const int dev, const int kernel, const size_t *sizes, const size_t *local)
Definition opencl.c:2650
int dt_opencl_enqueue_copy_buffer_to_buffer(const int devid, cl_mem src_buffer, cl_mem dst_buffer, size_t srcoffset, size_t dstoffset, size_t size)
Definition opencl.c:2804
void dt_opencl_release_mem_object(cl_mem mem)
Definition opencl.c:2895
#define ROUNDUPDHT(a, b)
Definition opencl.h:87
#define ROUNDUPDWD(a, b)
Definition opencl.h:86
const float factor
Definition pdf.h:91
const char * name
Definition pdf.h:90
const float r
int kernel_upsample
Definition nn_model.c:1120
int kernel_conv
Definition nn_model.c:1118
int kernel_conv3x3
Definition nn_model.c:1119
dt_pthread_mutex_t cl_lock
Definition nn_model.c:68
cl_mem dev_weights[16]
Definition nn_model.c:67
nn_unet_t coarse
Definition nn_model.c:60
float * blob
Definition nn_model.c:64
size_t blob_floats
Definition nn_model.c:65
nn_unet_t fine
Definition nn_model.c:59
int in_ch
Definition nn_model.c:40
const float * b
Definition nn_model.c:39
int out_ch
Definition nn_model.c:40
const float * w
Definition nn_model.c:38
JsonArray * tensors
Definition nn_model.c:127
size_t payload_size
Definition nn_model.c:129
const float * payload
Definition nn_model.c:128
int depth
Definition nn_model.c:50
int base
Definition nn_model.c:50
nn_conv_t head
Definition nn_model.c:54
int in_ch
Definition nn_model.c:50
int out_ch
Definition nn_model.c:50
nn_conv_t dec2[8]
Definition nn_model.c:53
nn_conv_t bot2
Definition nn_model.c:52
nn_conv_t enc2[8]
Definition nn_model.c:51
nn_conv_t dec1[8]
Definition nn_model.c:53
nn_conv_t down[8]
Definition nn_model.c:51
nn_conv_t enc1[8]
Definition nn_model.c:51
nn_conv_t up[8]
Definition nn_model.c:53
nn_conv_t bot1
Definition nn_model.c:52
#define __DT_CLONE_TARGETS__