24#include <glib/gstdio.h>
25#include <json-glib/json-glib.h>
34#define NN_MIN(a, b) ((a) < (b) ? (a) : (b))
44#define NN_MAX_DEVICES 16
103 const size_t bytes = floats *
sizeof(float);
116static void _err(
char *err,
size_t err_len,
const char *msg)
118 if(err && err_len) snprintf(err, err_len,
"%s", msg);
135 char *err,
size_t err_len)
138 const float *w = NULL, *b = NULL;
139 const guint
n = json_array_get_length(h->
tensors);
140 for(
int part = 0; part < 2; part++)
142 snprintf(
name,
sizeof(
name),
"%s.%s", prefix, part == 0 ?
"weight" :
"bias");
143 const size_t want = (part == 0 ? (size_t)out_ch * in_ch *
k *
k : (size_t)out_ch) *
sizeof(
float);
144 const float *found = NULL;
145 for(guint
i = 0;
i <
n;
i++)
150 JsonNode *node = json_array_get_element(h->
tensors,
i);
151 if(!node || !JSON_NODE_HOLDS_OBJECT(node))
continue;
152 JsonObject *
t = json_node_get_object(node);
153 if(!json_object_has_member(
t,
"name") || !json_object_has_member(
t,
"offset")
154 || !json_object_has_member(
t,
"size"))
156 if(g_strcmp0(json_object_get_string_member(
t,
"name"),
name))
continue;
157 const gint64 offset = json_object_get_int_member(
t,
"offset");
158 const gint64
size = json_object_get_int_member(
t,
"size");
161 _err(err, err_len,
"tensor with unexpected size or offset");
164 found = (
const float *)((
const uint8_t *)h->
payload + offset);
169 if(err && err_len) snprintf(err, err_len,
"missing tensor %s",
name);
195 int bad = 0, cin = in_ch;
196 for(
int l = 0; l < depth && !bad; l++)
198 const int w = base << l;
199 snprintf(prefix,
sizeof(prefix),
"%senc.%d.0", stage_prefix, l);
200 bad |=
_wire_conv(h, prefix, w, cin, 3, &u->
enc1[l], err, err_len);
201 snprintf(prefix,
sizeof(prefix),
"%senc.%d.2", stage_prefix, l);
203 snprintf(prefix,
sizeof(prefix),
"%sdown.%d", stage_prefix, l);
207 const int wb = base << depth;
208 snprintf(prefix,
sizeof(prefix),
"%sbottleneck.0", stage_prefix);
209 bad |=
_wire_conv(h, prefix, wb, base << (depth - 1), 3, &u->
bot1, err, err_len);
210 snprintf(prefix,
sizeof(prefix),
"%sbottleneck.2", stage_prefix);
214 for(
int i = 0;
i < depth && !bad;
i++)
216 const int w_skip = base << (depth - 1 -
i);
217 const int w_in = w_skip << 1;
218 snprintf(prefix,
sizeof(prefix),
"%sup.%d", stage_prefix,
i);
219 bad |=
_wire_conv(h, prefix, w_skip, w_in, 1, &u->
up[
i], err, err_len);
220 snprintf(prefix,
sizeof(prefix),
"%sdec.%d.0", stage_prefix,
i);
221 bad |=
_wire_conv(h, prefix, w_skip, 2 * w_skip, 3, &u->
dec1[
i], err, err_len);
222 snprintf(prefix,
sizeof(prefix),
"%sdec.%d.2", stage_prefix,
i);
223 bad |=
_wire_conv(h, prefix, w_skip, w_skip, 3, &u->
dec2[
i], err, err_len);
225 snprintf(prefix,
sizeof(prefix),
"%shead", stage_prefix);
226 bad |=
_wire_conv(h, prefix, out_ch, base, 3, &u->
head, err, err_len);
232static int _read_net_cfg(JsonObject *cfg,
int out_ch_max,
int *base,
int *depth,
int *in_ch,
int *out_ch)
234 if(!cfg || !json_object_has_member(cfg,
"base") || !json_object_has_member(cfg,
"depth")
235 || !json_object_has_member(cfg,
"in_channels") || !json_object_has_member(cfg,
"out_channels"))
237 *base = (int)json_object_get_int_member(cfg,
"base");
238 *depth = (int)json_object_get_int_member(cfg,
"depth");
239 *in_ch = (int)json_object_get_int_member(cfg,
"in_channels");
240 *out_ch = (int)json_object_get_int_member(cfg,
"out_channels");
241 return *base < 1 || *base > 512 || *depth < 1 || *depth >
NN_MAX_DEPTH || *in_ch < 1 || *in_ch > 16
242 || *out_ch < 1 || *out_ch > out_ch_max;
247 FILE *
f = g_fopen(path,
"rb");
250 _err(err, err_len,
"cannot open model file");
254 uint32_t header_len = 0;
255 if(fread(magic, 1, 8,
f) != 8 || memcmp(magic,
"ANSELDN1", 8) || fread(&header_len, 4, 1,
f) != 1
256 || header_len == 0 || header_len > (64u << 20))
258 _err(err, err_len,
"not an ANSELDN1 model file");
262 char *header = malloc(header_len + 1);
263 if(!header || fread(header, 1, header_len,
f) != header_len)
265 _err(err, err_len,
"truncated model header");
270 header[header_len] =
'\0';
272 fseek(
f, 0, SEEK_END);
273 const long file_size = ftell(
f);
274 if(file_size < 12 + (
long)header_len)
276 _err(err, err_len,
"truncated model file");
281 const size_t payload_size = (size_t)file_size - 12 - header_len;
282 float *blob = malloc(payload_size);
283 fseek(
f, 12 + (
long)header_len, SEEK_SET);
284 const int payload_ok = blob && fread(blob, 1, payload_size,
f) == payload_size;
288 _err(err, err_len,
"truncated model payload");
295 JsonParser *parser = json_parser_new();
296 if(!json_parser_load_from_data(parser, header, header_len, NULL))
298 _err(err, err_len,
"invalid model header JSON");
301 JsonObject *root = json_node_get_object(json_parser_get_root(parser));
302 if(!root || !json_object_has_member(root,
"cfg") || !json_object_has_member(root,
"tensors"))
304 _err(err, err_len,
"model header missing cfg or tensors");
307 JsonObject *cfg = json_object_get_object_member(root,
"cfg");
308 const char *arch = cfg ? json_object_get_string_member(cfg,
"arch") : NULL;
309 const int is_ms = !g_strcmp0(arch,
"unet-ms");
310 if(!is_ms && g_strcmp0(arch,
"unet"))
312 _err(err, err_len,
"unsupported model architecture");
315 int f_base = 0, f_depth = 0, f_in = 0, f_out = 0;
316 int c_base = 0, c_depth = 0, c_in = 0, c_out = 0;
317 int bin_bayer = 1, bin_xtrans = 1;
320 if(!json_object_has_member(cfg,
"coarse") || !json_object_has_member(cfg,
"fine")
321 || !json_object_has_member(cfg,
"bin")
322 ||
_read_net_cfg(json_object_get_object_member(cfg,
"fine"), 1, &f_base, &f_depth, &f_in, &f_out)
323 ||
_read_net_cfg(json_object_get_object_member(cfg,
"coarse"), 8, &c_base, &c_depth, &c_in, &c_out))
325 _err(err, err_len,
"model config out of supported range");
328 JsonObject *bin = json_object_get_object_member(cfg,
"bin");
329 if(!bin || !json_object_has_member(bin,
"bayer") || !json_object_has_member(bin,
"xtrans"))
331 _err(err, err_len,
"model config missing bin factors");
334 bin_bayer = (int)json_object_get_int_member(bin,
"bayer");
335 bin_xtrans = (int)json_object_get_int_member(bin,
"xtrans");
336 if(bin_bayer < 2 || bin_bayer > 16 || bin_xtrans < 2 || bin_xtrans > 16)
338 _err(err, err_len,
"model bin factors out of supported range");
342 else if(
_read_net_cfg(cfg, 1, &f_base, &f_depth, &f_in, &f_out))
344 _err(err, err_len,
"model config out of supported range");
348 if(payload_size /
sizeof(
float) > (
size_t)INT_MAX)
350 _err(err, err_len,
"model payload too large");
353 JsonNode *tensors_node = json_object_get_member(root,
"tensors");
354 if(!tensors_node || !JSON_NODE_HOLDS_ARRAY(tensors_node))
356 _err(err, err_len,
"model header tensors is not an array");
362 m->has_coarse = is_ms;
363 m->bin_bayer = bin_bayer;
364 m->bin_xtrans = bin_xtrans;
365 if(is_ms && json_object_has_member(cfg,
"anchor"))
367 const int anchor = (int)json_object_get_int_member(cfg,
"anchor");
368 if(anchor >= 8 && anchor <= 256)
m->anchor = anchor;
371 m->blob_floats = payload_size /
sizeof(float);
381 = { .tensors = json_object_get_array_member(root,
"tensors"), .payload = blob, .payload_size = payload_size };
382 int bad =
_wire_unet(&h, is_ms ?
"fine." :
"", f_base, f_depth, f_in, f_out, &
m->fine, err, err_len);
383 if(is_ms && !bad) bad =
_wire_unet(&h,
"coarse.", c_base, c_depth, c_in, c_out, &
m->coarse, err, err_len);
397 g_object_unref(parser);
405 if(
m->dev_weights[
d])
408 m->dev_weights[
d] = NULL;
426 return m->fine.in_ch;
431 return m->fine.out_ch;
436 if(!
m->has_coarse)
return 1;
437 return is_xtrans ?
m->bin_xtrans :
m->bin_bayer;
442 return m->has_coarse ?
m->coarse.in_ch : 0;
447 return m->has_coarse ?
m->coarse.out_ch : 0;
457 if(a <= 0 || b <= 0)
return a > b ? a : b;
473 int align = 1 <<
m->fine.depth;
476 align =
_lcm(align,
m->bin_bayer <<
m->coarse.depth);
477 align =
_lcm(align,
m->bin_xtrans <<
m->coarse.depth);
523 const int ow = (w + 2 * pad -
k) / stride + 1;
524 const int oh = (h + 2 * pad -
k) / stride + 1;
525 const size_t inhw = (size_t)w * h;
526 const size_t wstride = (size_t)cv->
in_ch *
k *
k;
531 float *
const acc = malloc(
sizeof(
float) *
NN_OC_BLOCK * ow);
533#pragma omp for collapse(2) schedule(static)
535 for(
int oy = 0; oy < oh; oy++)
539 for(
int r = 0;
r < nb;
r++)
541 float *
const ar = acc + (size_t)
r * ow;
542 const float bias = cv->
b[ocb +
r];
543 for(
int ox = 0; ox < ow; ox++) ar[ox] = bias;
545 for(
int ic = 0; ic < cv->
in_ch; ic++)
547 const float *
const ip =
in + (size_t)ic * inhw;
548 for(
int ky = 0; ky <
k; ky++)
550 const int iy = oy * stride + ky - pad;
551 if(iy < 0 || iy >= h)
continue;
552 const float *
const irow = ip + (size_t)iy * w;
553 for(
int kx = 0; kx <
k; kx++)
555 const int shift = kx - pad;
556 int ox0 = 0, ox1 = ow;
557 while(ox0 < ow && ox0 * stride + shift < 0) ox0++;
558 while(ox1 > ox0 && (ox1 - 1) * stride + shift >= w) ox1--;
559 const float *
const wbase = cv->
w + ((size_t)ocb * cv->
in_ch + ic) *
k *
k + ky *
k + kx;
563 const float w0 = wbase[0],
w1 = wbase[wstride];
564 const float w2 = wbase[2 * wstride],
w3 = wbase[3 * wstride];
565 float *
const a0 = acc, *
const a1 = acc + ow;
566 float *
const a2 = acc + 2 * ow, *
const a3 = acc + 3 * ow;
569 const float *
const is = irow + shift;
573 for(
int ox = ox0; ox < ox1; ox++)
575 const float xv = is[ox];
583 for(
int ox = ox0; ox < ox1; ox++)
585 const float xv = irow[ox * stride + shift];
593 for(
int r = 0;
r < nb;
r++)
595 const float wv = wbase[(size_t)
r * wstride];
596 float *
const ar = acc + (size_t)
r * ow;
599 const float *
const is = irow + shift;
603 for(
int ox = ox0; ox < ox1; ox++) ar[ox] += wv * is[ox];
606 for(
int ox = ox0; ox < ox1; ox++) ar[ox] += wv * irow[ox * stride + shift];
611 for(
int r = 0;
r < nb;
r++)
612 memcpy(
out + ((
size_t)(ocb +
r) * oh + oy) * ow, acc + (
size_t)
r * ow,
sizeof(
float) * ow);
623#pragma omp parallel for schedule(static)
625 for(
size_t i = 0;
i <
n;
i++)
x[
i] = 0.5f *
x[
i] * (1.0f + erff(
x[
i] * (
float)M_SQRT1_2));
638 const size_t base = (size_t)u->
base;
639 size_t live = 0, peak = 0, cur = 0;
640#define NN_LEDGER(delta) \
644 if(live > peak) peak = live; \
647 for(
int l = 0; l < u->
depth; l++)
649 const size_t lvl = base * wh >> l;
658 const size_t bot = base * wh >> u->
depth;
667 const int l = u->
depth - 1 -
i;
668 const size_t half = base * wh >> l;
703 const size_t ref = (size_t)1 << 24;
707 const int bin =
NN_MIN(
m->bin_bayer,
m->bin_xtrans);
709 = (float)
_unet_peak_floats(&
m->coarse, ref / ((
size_t)bin * bin), cl_variant) / (float)ref;
710 if(coarse > per_px) per_px = coarse;
734 float per_px = (float)
m->fine.base;
737 const int bin =
NN_MIN(
m->bin_bayer,
m->bin_xtrans);
738 const float coarse = (float)
m->coarse.base / (
float)(bin * bin);
739 if(coarse > per_px) per_px = coarse;
750 const int bin =
NN_MIN(
m->bin_bayer,
m->bin_xtrans);
752 if(coarse > floats) floats = coarse;
754 return floats *
sizeof(float);
772 const int ow = w, oh = h;
773 const int bw = w / 2;
774 const size_t inhw = (size_t)w * h;
775 const size_t bhw = (size_t)bw * (h / 2);
776 const size_t wstride = (size_t)cv->
in_ch *
k *
k;
781 float *
const acc = malloc(
sizeof(
float) *
NN_OC_BLOCK * ow);
783#pragma omp for collapse(2) schedule(static)
785 for(
int oy = 0; oy < oh; oy++)
789 for(
int r = 0;
r < nb;
r++)
791 float *
const ar = acc + (size_t)
r * ow;
792 const float bias = cv->
b[ocb +
r];
793 for(
int ox = 0; ox < ow; ox++) ar[ox] = bias;
795 for(
int ic = 0; ic < cv->
in_ch; ic++)
797 const int from_b = ic >= in_ch_a;
798 const float *
const ip = from_b ? b + (size_t)(ic - in_ch_a) * bhw : a + (size_t)ic * inhw;
799 for(
int ky = 0; ky <
k; ky++)
801 const int iy = oy + ky - pad;
802 if(iy < 0 || iy >= h)
continue;
803 const float *
const irow = from_b ? ip + (size_t)(iy >> 1) * bw : ip + (size_t)iy * w;
804 for(
int kx = 0; kx <
k; kx++)
806 const int shift = kx - pad;
807 int ox0 = 0, ox1 = ow;
808 while(ox0 < ow && ox0 + shift < 0) ox0++;
809 while(ox1 > ox0 && (ox1 - 1) + shift >= w) ox1--;
810 const float *
const wbase = cv->
w + ((size_t)ocb * cv->
in_ch + ic) *
k *
k + ky *
k + kx;
815 const float w0 = wbase[0],
w1 = wbase[wstride];
816 const float w2 = wbase[2 * wstride],
w3 = wbase[3 * wstride];
817 float *
const a0 = acc, *
const a1 = acc + ow;
818 float *
const a2 = acc + 2 * ow, *
const a3 = acc + 3 * ow;
820 for(
int ox = ox0; ox < ox1; ox++)
822 const float xv = irow[(ox + shift) >> 1];
830 const float *
const is = irow + shift;
834 for(
int ox = ox0; ox < ox1; ox++)
836 const float xv = is[ox];
845 for(
int r = 0;
r < nb;
r++)
847 const float wr = wbase[(size_t)
r * wstride];
848 float *
const ar = acc + (size_t)
r * ow;
850 for(
int ox = ox0; ox < ox1; ox++) ar[ox] += wr * irow[(ox + shift) >> 1];
853 const float *
const is = irow + shift;
857 for(
int ox = ox0; ox < ox1; ox++) ar[ox] += wr * is[ox];
863 for(
int r = 0;
r < nb;
r++)
864 memcpy(
out + ((
size_t)(ocb +
r)) * inhw + (
size_t)oy * ow, acc + (
size_t)
r * ow,
891 const int align = 1 << u->
depth;
895 const size_t base = (size_t)u->
base;
899 const float *src =
in;
907 for(
int l = 0; l < u->
depth && ok; l++)
915 const size_t lvl = base * wh >> l;
919 if(!tmp || !skips[l] || !next)
928 _conv2d(&u->
enc2[l], tmp, cw, chh, 1, 1, skips[l]);
931 _conv2d(&u->
down[l], skips[l], cw, chh, 2, 0, next);
947 const size_t bot = base * wh >> u->
depth;
973 for(
int i = 0;
i < u->
depth && ok;
i++)
981 const int l = u->
depth - 1 -
i;
982 const size_t w_skip = base << l;
983 const size_t half = w_skip * (size_t)(2 * cw) * (size_t)(2 * chh);
985 if(!
v) { ok = 0;
break; }
994 _gelu(d1, w_skip * (
size_t)cw * chh);
999 if(!d2) {
_nn_free(d1); ok = 0;
break; }
1001 _gelu(d2, w_skip * (
size_t)cw * chh);
1023#pragma omp parallel for schedule(static)
1025 for(
size_t i = 0;
i < (size_t)residual_ch * wh;
i++)
out[
i] =
in[
i] - head[
i];
1045 int height,
int apply_residual)
1049 if(!
m->has_coarse)
return 1;
1057void dt_nn_bin_planes(
const float *planes,
int pw,
int ph,
int bin,
float *out_rgb,
float *out_cnt)
1064 const size_t plane = (size_t)pw * ph;
1065 const int cw = pw / bin, chh = ph / bin;
1066 const float *
const mosaic = planes;
1068#pragma omp parallel for schedule(static) collapse(2)
1070 for(
int c = 0; c < 3; c++)
1071 for(
int cy = 0; cy < chh; cy++)
1073 const float *
const onehot = planes + (size_t)(1 + c) * plane;
1074 float *
const orow = out_rgb + (size_t)c * cw * chh + (
size_t)cy * cw;
1075 float *
const crow = out_cnt + (size_t)c * cw * chh + (
size_t)cy * cw;
1076 for(
int cx = 0; cx < cw; cx++)
1078 float sum = 0.0f, cnt = 0.0f;
1079 for(
int y = cy * bin; y < (cy + 1) * bin; y++)
1080 for(
int x = cx * bin;
x < (cx + 1) * bin;
x++)
1082 const size_t i = (size_t)y * pw +
x;
1083 sum += mosaic[
i] * onehot[
i];
1087 orow[cx] = sum / (cnt > 1.0f ? cnt : 1.0f);
1096#pragma omp parallel for schedule(static)
1098 for(
int c = 0; c <
ch; c++)
1100 const float *
const ip =
in + (size_t)c * w * h;
1102 for(
int y = 0; y < h *
factor; y++)
1104 const float *
const irow = ip + (size_t)(y /
factor) * w;
1105 float *
const orow = op + (size_t)y * w *
factor;
1126 if(!cl)
return NULL;
1151 const size_t bytes =
m->blob_floats *
sizeof(float);
1165 int w,
int h,
const nn_conv_t *cv,
int stride,
int pad,
int do_gelu)
1167 const int ow = (w + 2 * pad - cv->
k) / stride + 1;
1168 const int oh = (h + 2 * pad - cv->
k) / stride + 1;
1169 const int weight_off = (int)(cv->
w - blob_base);
1170 const int bias_off = (int)(cv->
b - blob_base);
1171 const size_t slice_bytes =
sizeof(float) * 4 * cv->
in_ch * cv->
k * cv->
k;
1172 const int use_local = slice_bytes <= (30 << 10);
1173 const size_t lx = 128;
1175 if(cv->
k == 3 && stride == 1)
1181 const int chunk =
NN_MIN(cv->
in_ch, (24 << 10) / (
int)(
sizeof(
float) * 4 * 9));
1195 const size_t quads = (size_t)((w + 1) / 2) * ((h + 1) / 2);
1196 size_t sizes3[3] = { (quads + lx - 1) / lx * lx, ((
size_t)cv->
out_ch + 3) / 4, 1 };
1197 size_t local3[3] = { lx, 1, 1 };
1226 size_t sizes[3] = { ((size_t)ow * oh + lx - 1) / lx * lx, ((
size_t)cv->
out_ch + 3) / 4, 1 };
1227 size_t local[3] = { lx, 1, 1 };
1262 const int align = 1 << u->
depth;
1266 if(!weights)
return -1;
1269 const size_t base = (size_t)u->
base;
1270 int err = CL_SUCCESS;
1272 cl_mem cur = NULL, tmp = NULL,
v = NULL, cat = NULL, us = NULL, d1 = NULL;
1274#define NN_CL_ALLOC(var, floats) \
1277 var = dt_opencl_alloc_device_buffer(devid, (floats) * sizeof(float)); \
1284#define NN_CL_FREE(var) \
1287 if(var) dt_opencl_release_mem_object(var); \
1297 cl_mem src = dev_in;
1299 for(
int l = 0; l < u->
depth && err == CL_SUCCESS; l++)
1306 const size_t lvl = base * wh >> l;
1311 err |=
_conv_cl(cl, devid, weights,
m->blob, src, tmp, cw, chh, &u->
enc1[l], 1, 1, 1);
1312 err |=
_conv_cl(cl, devid, weights,
m->blob, tmp, skips[l], cw, chh, &u->
enc2[l], 1, 1, 1);
1314 err |=
_conv_cl(cl, devid, weights,
m->blob, skips[l], next, cw, chh, &u->
down[l], 2, 0, 0);
1323 if(err == CL_SUCCESS)
1330 const size_t bot = base * wh >> u->
depth;
1333 err |=
_conv_cl(cl, devid, weights,
m->blob, src, tmp, cw, chh, &u->
bot1, 1, 1, 1);
1334 err |=
_conv_cl(cl, devid, weights,
m->blob, tmp,
v, cw, chh, &u->
bot2, 1, 1, 1);
1343 for(
int i = 0;
i < u->
depth && err == CL_SUCCESS;
i++)
1350 const int l = u->
depth - 1 -
i;
1351 const size_t w_skip = base << l;
1352 const size_t half = w_skip * (size_t)(2 * cw) * (size_t)(2 * chh);
1354 err |=
_conv_cl(cl, devid, weights,
m->blob, cur,
v, cw, chh, &u->
up[
i], 1, 0, 0);
1363 half *
sizeof(
float));
1368 err |=
_conv_cl(cl, devid, weights,
m->blob, cat, d1, cw, chh, &u->
dec1[
i], 1, 1, 1);
1371 err |=
_conv_cl(cl, devid, weights,
m->blob, d1, cur, cw, chh, &u->
dec2[
i], 1, 1, 1);
1378 if(err == CL_SUCCESS)
1379 err |=
_conv_cl(cl, devid, weights,
m->blob, cur, dev_out,
width,
height, &u->
head, 1, 1, 0);
1382 for(
int l = 0; l < u->
depth; l++)
1400 if(!
m->has_coarse)
return -1;
void cleanup(dt_imageio_module_format_t *self)
const dt_colormatrix_t dt_aligned_pixel_t out
static int dt_pthread_mutex_unlock(dt_pthread_mutex_t *mutex) RELEASE(mutex) NO_THREAD_SAFETY_ANALYSIS
static int dt_pthread_mutex_init(dt_pthread_mutex_t *mutex, const pthread_mutexattr_t *mutexattr)
Initialise a mutex. With mutexattr NULL – which is how 54 of the 56 call sites in this tree spell it ...
static int dt_pthread_mutex_destroy(dt_pthread_mutex_t *mutex)
static int dt_pthread_mutex_lock(dt_pthread_mutex_t *mutex) ACQUIRE(mutex) NO_THREAD_SAFETY_ANALYSIS
memcpy(roundtrip, origin, sizeof(roundtrip))
float *const restrict const size_t k
float *const restrict const size_t const size_t ch
void dt_nn_cl_destroy(dt_nn_cl_t *cl)
int dt_nn_unet_apply_stage_cl(const dt_nn_model_t *m, int stage, dt_nn_cl_t *cl, int devid, cl_mem dev_in, cl_mem dev_out, int width, int height)
int dt_nn_model_anchor(const dt_nn_model_t *m)
#define NN_CL_ALLOC(var, floats)
void dt_nn_model_free(dt_nn_model_t *m)
static dt_nn_alloc_f _nn_alloc_fn
float dt_nn_unet_scratch_maxblock_per_px(const dt_nn_model_t *m)
static size_t _unet_peak_floats(const nn_unet_t *u, size_t wh, int cl_variant)
static cl_mem _weights_cl(const dt_nn_model_t *m, int devid)
int dt_nn_model_in_channels(const dt_nn_model_t *m)
static void _err(char *err, size_t err_len, const char *msg)
int dt_nn_model_coarse_in_channels(const dt_nn_model_t *m)
dt_nn_cl_t * dt_nn_cl_create(int program)
static int _conv_cl(dt_nn_cl_t *cl, int devid, cl_mem weights, const float *blob_base, cl_mem in, cl_mem out, int w, int h, const nn_conv_t *cv, int stride, int pad, int do_gelu)
static int _wire_conv(const nn_header_t *h, const char *prefix, int out_ch, int in_ch, int k, nn_conv_t *cv, char *err, size_t err_len)
static int _unet_forward(const nn_unet_t *u, const float *in, float *out, int width, int height, int residual_ch)
static int _unet_forward_cl(const dt_nn_model_t *m, const nn_unet_t *u, dt_nn_cl_t *cl, int devid, cl_mem dev_in, cl_mem dev_out, int width, int height)
size_t dt_nn_unet_scratch_bytes(const dt_nn_model_t *m, int width, int height)
int dt_nn_unet_apply_stage(const dt_nn_model_t *m, int stage, const float *in, float *out, int width, int height, int apply_residual)
static __DT_CLONE_TARGETS__ void _conv2d_cat2(const nn_conv_t *cv, const float *a, int in_ch_a, const float *b, int w, int h, float *out)
static __DT_CLONE_TARGETS__ void _conv2d(const nn_conv_t *cv, const float *in, int w, int h, int stride, int pad, float *out)
static void dt_nn_model_free_cl(dt_nn_model_t *m)
static void * _nn_alloc(size_t floats, int long_lived)
static dt_nn_free_f _nn_free_fn
void dt_nn_set_cancel(dt_nn_cancel_f cancel_fn)
static void _nn_free(void *p)
float dt_nn_unet_scratch_per_px(const dt_nn_model_t *m)
dt_nn_model_t * dt_nn_model_load(const char *path, char *err, size_t err_len)
__DT_CLONE_TARGETS__ void dt_nn_upsample_nearest(const float *in, int ch, int w, int h, int factor, float *out)
static __DT_CLONE_TARGETS__ void _gelu(float *x, size_t n)
int dt_nn_model_coarse_out_channels(const dt_nn_model_t *m)
static int _lcm(int a, int b)
float dt_nn_unet_scratch_per_px_cl(const dt_nn_model_t *m)
int dt_nn_model_alignment(const dt_nn_model_t *m)
int dt_nn_unet_apply(const dt_nn_model_t *m, const float *in, float *out, int width, int height)
static dt_nn_cancel_f _nn_cancel_fn
static int _read_net_cfg(JsonObject *cfg, int out_ch_max, int *base, int *depth, int *in_ch, int *out_ch)
__DT_CLONE_TARGETS__ void dt_nn_bin_planes(const float *planes, int pw, int ph, int bin, float *out_rgb, float *out_cnt)
int dt_nn_model_bin(const dt_nn_model_t *m, const int is_xtrans)
static int _wire_unet(const nn_header_t *h, const char *stage_prefix, int base, int depth, int in_ch, int out_ch, nn_unet_t *u, char *err, size_t err_len)
static float _scratch_per_px(const dt_nn_model_t *m, int cl_variant)
static int _upsample_cl(dt_nn_cl_t *cl, int devid, cl_mem in, cl_mem out, int w, int h, int ch)
int dt_nn_model_out_channels(const dt_nn_model_t *m)
void dt_nn_set_allocator(dt_nn_alloc_f alloc_fn, dt_nn_free_f free_fn)
void *(* dt_nn_alloc_f)(size_t bytes, int long_lived)
void(* dt_nn_free_f)(void *ptr)
#define DT_NN_FUSION_COARSEST
int(* dt_nn_cancel_f)(void)
int dt_opencl_enqueue_kernel_2d(const int dev, const int kernel, const size_t *sizes)
void * dt_opencl_alloc_device_buffer(const int devid, const size_t size)
int dt_opencl_create_kernel(const int prog, const char *name)
int dt_opencl_write_buffer_to_device(const int devid, void *host, void *device, const size_t offset, const size_t size, const int blocking)
void dt_opencl_free_kernel(const int kernel)
int dt_opencl_set_kernel_arg(const int dev, const int kernel, const int num, const size_t size, const void *arg)
gboolean dt_opencl_finish(const int devid)
int dt_opencl_enqueue_kernel_2d_with_local(const int dev, const int kernel, const size_t *sizes, const size_t *local)
int dt_opencl_enqueue_copy_buffer_to_buffer(const int devid, cl_mem src_buffer, cl_mem dst_buffer, size_t srcoffset, size_t dstoffset, size_t size)
void dt_opencl_release_mem_object(cl_mem mem)
dt_pthread_mutex_t cl_lock
#define __DT_CLONE_TARGETS__