Ansel 0.0
A darktable fork - bloat + design vision
Loading...
Searching...
No Matches
nn_model.h
Go to the documentation of this file.
1/*
2 This file is part of Ansel,
3 Copyright (C) 2026 Aurélien PIERRE.
4
5 Ansel is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, either version 3 of the License, or
8 (at your option) any later version.
9
10 Ansel is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with Ansel. If not, see <http://www.gnu.org/licenses/>.
17*/
18#ifndef DT_COMMON_NN_MODEL_H
19#define DT_COMMON_NN_MODEL_H
20
21#include <stddef.h>
22
23/*
24 * Loader and CPU executor for the .anselnn neural denoising models trained by
25 * https://github.com/aurelienpierreeng/ansel-denoise.
26 *
27 * Deliberately self-contained: no darktable.h, no pipeline types. The only
28 * dependencies are json-glib (header parsing), libm and OpenMP. The pixelpipe
29 * integration lives in the IOP module; this translation unit owns the model
30 * data and the math, nothing else.
31 *
32 * File format (little-endian):
33 * 8 bytes magic "ANSELDN1"
34 * 4 bytes uint32 JSON header length N
35 * N bytes JSON: {"cfg": {"arch": "unet", "base", "depth",
36 * "in_channels", "out_channels"},
37 * "tensors": [{"name", "shape", "offset", "size"}, ...]}
38 * payload float32 tensor data, offsets relative to payload start
39 *
40 * Fixed topology ("unet"): depth encoder levels of two 3x3 conv + GELU with a
41 * 2x2 stride-2 downsampling conv between levels, a two-conv bottleneck, and a
42 * decoder of nearest x2 upsampling, 1x1 channel-reduction conv, skip
43 * concatenation and two 3x3 conv + GELU per level; a final 3x3 conv predicts
44 * the noise, subtracted from the input mosaic plane (residual output).
45 */
46
48
49/* Load a .anselnn file. Returns NULL on failure and, if err is non-NULL,
50 * writes a human-readable reason into err[0..err_len-1]. */
51/* Pixel-buffer allocator injection. The executors allocate their scratch
52 * (feature maps, skip connections) through these hooks so the buffers come
53 * from the application's pixelpipe cache arena — per the project rule that
54 * pixel buffers never come from bare malloc — WITHOUT this unit importing any
55 * pipeline header. Unset, they fall back to malloc/free (the standalone
56 * fixture test). Set once at module init, before any pipeline runs; not
57 * thread-safe against concurrent forwards. The model WEIGHTS blob stays on
58 * malloc: it is model data with session lifetime, not a pixel buffer. */
59/* long_lived: the block either survives across pipeline stages (a U-Net skip
60 * connection, held from its encoder level to its decoder level) or is a small
61 * helper that coexists with the largest tensors (the decoder's 1x1 output).
62 * A region-based arena places these at the opposite end from the big-tensor
63 * churn — simulated over the full forward, that layout reaches the ledger's
64 * live-set peak exactly, with no fragmentation slack. */
65typedef void *(*dt_nn_alloc_f)(size_t bytes, int long_lived);
66typedef void (*dt_nn_free_f)(void *ptr);
67void dt_nn_set_allocator(dt_nn_alloc_f alloc_fn, dt_nn_free_f free_fn);
68
69/* Cancellation injection, on the same terms as the allocator: a forward can
70 * take tens of seconds on CPU, and the pipeline's kill switch is only read
71 * between modules, so without this a stale forward always runs to the end.
72 * The executor asks the hook between U-Net levels, on the calling thread
73 * (never inside an OpenMP region); a non-zero answer abandons the forward,
74 * which then returns DT_NN_CANCELLED. The OpenCL executor asks at the same
75 * points but first waits for the device queue, which costs one sync per
76 * level while a hook is set. Unset, a forward is never cancelled
77 * (the standalone fixture test). */
78#define DT_NN_CANCELLED 2
79typedef int (*dt_nn_cancel_f)(void);
80void dt_nn_set_cancel(dt_nn_cancel_f cancel_fn);
81
82dt_nn_model_t *dt_nn_model_load(const char *path, char *err, size_t err_len);
83
85
86/* Number of input planes the FINE net expects (5 for arch "unet": mosaic,
87 * R, G, B one-hot, sigma; 5 + coarse_out for arch "unet-ms") and output
88 * planes (1: denoised mosaic). */
91
92/* Multi-scale ("unet-ms") accessors. bin: the superpixel factor for the
93 * image's CFA family, 1 for a single-scale model (no coarse stage).
94 * coarse_in/out: the coarse net's plane counts, 0 without a coarse stage. */
95int dt_nn_model_bin(const dt_nn_model_t *model, const int is_xtrans);
98
99/* Low-band anchor scale (sensor px) carried in the model cfg: below it the
100 * module replaces the output's per-channel means with the noisy input's —
101 * the n-averaged measurement is the true diluted estimate there, while a
102 * denoiser's low band accumulates model error. 0 = no anchoring. */
104
105/* Scales of the low-band fusion pyramid, in sensor px — fixed by the training
106 * reference (cfa.fuse_low_bands, scales=(16, 32, 64)), not by the model file.
107 * A padded tile must divide by the coarsest one, or the pyramid loses a level
108 * and the fusion no longer matches what the model was trained against; see
109 * dt_nn_model_alignment(). */
110#define DT_NN_FUSION_FINEST 16
111#define DT_NN_FUSION_COARSEST 64
112
113/* Width and height passed to dt_nn_unet_apply must be multiples of this:
114 * 2^depth for arch "unet"; for "unet-ms" also every bin << coarse_depth so
115 * the binned planes stay aligned for either CFA. The caller pads (reflect)
116 * and crops. */
118
119/* Peak executor scratch as a DIMENSIONLESS factor of the input image size:
120 * floats of scratch per input pixel, i.e. multiples of a 1-float-per-pixel
121 * buffer. This is the form a tiling factor wants — never feed it absolute
122 * bytes. Includes the coarse net's share for a multi-scale model. */
124
125/* Same, for the OpenCL executor, whose sequence differs (it materializes the
126 * decoder concat and an upsample staging buffer where the CPU reads both in
127 * place): feed this one to factor_cl, the CPU one to factor. */
129
130/* Largest single host-side scratch tensor per input pixel (the contiguity
131 * requirement an arena allocation must satisfy, as opposed to the total). */
133
134/* Peak scratch in absolute bytes for a w x h input (diagnostics, messages). */
136
137/* Run the network. in: in_channels planar w*h float32 planes; out: one w*h
138 * plane (may not alias in). Returns 0 on success, DT_NN_CANCELLED when the
139 * cancel hook asked to stop, other non-zero on allocation failure or
140 * misaligned dimensions. Thread-safe for concurrent calls on the
141 * same model (weights are read-only; scratch is per-call). */
142int dt_nn_unet_apply(const dt_nn_model_t *model, const float *in, float *out, int width, int height);
143
144/* Run one stage of a multi-scale model. stage 0 = fine (identical to
145 * dt_nn_unet_apply); stage 1 = coarse: in is the 6-plane binned input
146 * [R, G, B, sigmaR, sigmaG, sigmaB] at (width, height) COARSE resolution,
147 * out receives the denoised coarse RGB (residual applied). */
148/* apply_residual: non-zero subtracts the head prediction from the matching
149 * input planes (what the torch reference does inside its forward); zero writes
150 * the RAW head output — which is what dt_nn_unet_apply_stage_cl always does,
151 * and what a caller wants when it applies the residual itself. */
152int dt_nn_unet_apply_stage(const dt_nn_model_t *model, int stage, const float *in, float *out, int width,
153 int height, int apply_residual);
154
155/* Superpixel-bin the assembled fine planes [mosaic, R, G, B one-hot, ...]:
156 * out_rgb (3 planes at pw/bin x ph/bin) receives the count-weighted mean of
157 * each block's same-channel sensels, out_cnt the per-channel sensel counts
158 * (for the analytic coarse sigma). The exact contract of the training
159 * repo's cfa.bin_mosaic_torch. */
160void dt_nn_bin_planes(const float *planes, int pw, int ph, int bin, float *out_rgb, float *out_cnt);
161
162/* Nearest-neighbour upsample of ch planar w*h planes by an integer factor
163 * (the coarse guide injection). */
164void dt_nn_upsample_nearest(const float *in, int ch, int w, int h, int factor, float *out);
165
166#ifdef HAVE_OPENCL
167#include "common/opencl.h"
168
169/* OpenCL kernel handles for the U-Net, created once per session from the
170 * rawdenoiseai.cl program. Opaque; owned by dt_nn_cl_create/destroy. */
171typedef struct dt_nn_cl_t dt_nn_cl_t;
172
173dt_nn_cl_t *dt_nn_cl_create(int program);
175
176/* GPU forward of one stage. Writes the RAW head output — stage 0 the predicted
177 * noise, stage 1 the coarse head — never the residual: subtracting it is the
178 * caller's business, on both devices, because "the net predicts what to
179 * remove" is the consumer's convention and not this runtime's. The CPU twin
180 * dt_nn_unet_apply_stage() takes apply_residual = 0 for the same contract. */
181int dt_nn_unet_apply_stage_cl(const dt_nn_model_t *model, int stage, dt_nn_cl_t *cl, int devid, cl_mem dev_in,
182 cl_mem dev_out, int width, int height);
183
184#endif
185#endif // DT_COMMON_NN_MODEL_H
typedef void((*dt_cache_allocate_t)(void *userdata, dt_cache_entry_t *entry))
const dt_colormatrix_t dt_aligned_pixel_t out
dt_iop_roi_t in
Definition geometry.h:117
const char * model
float *const restrict const size_t const size_t ch
uint32_t width
Definition mipmap_cache.c:0
uint32_t height
Definition mipmap_cache.c:1
void dt_nn_cl_destroy(dt_nn_cl_t *cl)
Definition nn_model.c:1133
void *(* dt_nn_alloc_f)(size_t bytes, int long_lived)
Definition nn_model.h:65
int dt_nn_model_in_channels(const dt_nn_model_t *model)
Definition nn_model.c:424
float dt_nn_unet_scratch_maxblock_per_px(const dt_nn_model_t *model)
Definition nn_model.c:725
int dt_nn_model_out_channels(const dt_nn_model_t *model)
Definition nn_model.c:429
int dt_nn_model_bin(const dt_nn_model_t *model, const int is_xtrans)
Definition nn_model.c:434
void dt_nn_model_free(dt_nn_model_t *model)
Definition nn_model.c:413
int dt_nn_model_alignment(const dt_nn_model_t *model)
Definition nn_model.c:468
float dt_nn_unet_scratch_per_px(const dt_nn_model_t *model)
Definition nn_model.c:715
size_t dt_nn_unet_scratch_bytes(const dt_nn_model_t *model, int width, int height)
Definition nn_model.c:744
int dt_nn_model_anchor(const dt_nn_model_t *model)
Definition nn_model.c:450
void(* dt_nn_free_f)(void *ptr)
Definition nn_model.h:66
dt_nn_cl_t * dt_nn_cl_create(int program)
Definition nn_model.c:1123
int dt_nn_model_coarse_in_channels(const dt_nn_model_t *model)
Definition nn_model.c:440
int dt_nn_unet_apply_stage(const dt_nn_model_t *model, int stage, const float *in, float *out, int width, int height, int apply_residual)
Definition nn_model.c:1044
void dt_nn_bin_planes(const float *planes, int pw, int ph, int bin, float *out_rgb, float *out_cnt)
Definition nn_model.c:1057
int(* dt_nn_cancel_f)(void)
Definition nn_model.h:79
void dt_nn_set_cancel(dt_nn_cancel_f cancel_fn)
Definition nn_model.c:86
float dt_nn_unet_scratch_per_px_cl(const dt_nn_model_t *model)
Definition nn_model.c:720
dt_nn_model_t * dt_nn_model_load(const char *path, char *err, size_t err_len)
Definition nn_model.c:245
int dt_nn_model_coarse_out_channels(const dt_nn_model_t *model)
Definition nn_model.c:445
int dt_nn_unet_apply_stage_cl(const dt_nn_model_t *model, int stage, dt_nn_cl_t *cl, int devid, cl_mem dev_in, cl_mem dev_out, int width, int height)
Definition nn_model.c:1395
int dt_nn_unet_apply(const dt_nn_model_t *model, const float *in, float *out, int width, int height)
Definition nn_model.c:1039
void dt_nn_upsample_nearest(const float *in, int ch, int w, int h, int factor, float *out)
Definition nn_model.c:1093
void dt_nn_set_allocator(dt_nn_alloc_f alloc_fn, dt_nn_free_f free_fn)
Definition nn_model.c:77
const float factor
Definition pdf.h:91