62#define RESAMPLING_BORDER_MODE BORDER_REPLICATE
66#define INTERPOLATION_BORDER_MODE BORDER_MIRROR
70#define MAX_HALF_FILTER_WIDTH 3
73#define DEBUG_PRINT_VERBOSE 0
131 if(i < min || i >
max)
144static inline __attribute__((always_inline))
void _prepare_tap_boundaries(
int *tap_first,
147 const int filterwidth,
160 *tap_last = filterwidth;
176 const size_t num_taps,
178 const float first_tap,
179 const float interval)
181 static const dt_aligned_pixel_simd_t bootstrap = { 0.0f, 1.0f, 2.0f, 3.0f };
182 const dt_aligned_pixel_simd_t interval_v =
dt_simd_set1(interval);
183 const dt_aligned_pixel_simd_t iter =
dt_simd_set1(4.0f * interval);
184 dt_aligned_pixel_simd_t vt =
dt_simd_set1(first_tap) + bootstrap * interval_v;
186 const int runs = (num_taps + 3) / 4;
188 for(
size_t i = 0;
i < runs;
i++)
201 const size_t num_taps,
203 const float first_tap,
204 const float interval)
206 static const dt_aligned_pixel_simd_t bootstrap = { 0.0f, 1.0f, 2.0f, 3.0f };
207 const dt_aligned_pixel_simd_t half =
dt_simd_set1(0.5f);
209 const dt_aligned_pixel_simd_t three =
dt_simd_set1(3.0f);
210 const dt_aligned_pixel_simd_t four =
dt_simd_set1(4.0f);
211 const dt_aligned_pixel_simd_t five =
dt_simd_set1(5.0f);
212 const dt_aligned_pixel_simd_t eight =
dt_simd_set1(8.0f);
213 const dt_aligned_pixel_simd_t interval_v =
dt_simd_set1(interval);
214 const dt_aligned_pixel_simd_t iter =
dt_simd_set1(4.0f * interval);
215 dt_aligned_pixel_simd_t vt =
dt_simd_set1(first_tap) + bootstrap * interval_v;
217 const int runs = (num_taps + 3) / 4;
219 for(
size_t i = 0;
i < runs;
i++)
221 const dt_aligned_pixel_simd_t vt_abs = dt_simd_abs(vt);
222 const dt_aligned_pixel_simd_t t2 = vt * vt;
223 const dt_aligned_pixel_simd_t t5 = five * vt_abs;
224 const dt_aligned_pixel_simd_t r12 = (vt_abs * (t5 - eight - t2) + four) * half;
225 const dt_aligned_pixel_simd_t r01 = ((three * t2 - t5) * vt_abs + two) * half;
226 dt_aligned_pixel_simd_t taps4 = r12;
228 taps4[c] = (vt_abs[c] <= 1.0f) ? r01[c] : r12[c];
254 const size_t num_taps,
256 const float first_tap,
257 const float interval)
259 static const dt_aligned_pixel_simd_t bootstrap = { 0.0f, 1.0f, 2.0f, 3.0f };
260 const dt_aligned_pixel_simd_t c7_6 =
dt_simd_set1(7.0f / 6.0f);
262 const dt_aligned_pixel_simd_t c8_9 =
dt_simd_set1(8.0f / 9.0f);
263 const dt_aligned_pixel_simd_t c7_18 =
dt_simd_set1(7.0f / 18.0f);
264 const dt_aligned_pixel_simd_t c10_3 =
dt_simd_set1(10.0f / 3.0f);
265 const dt_aligned_pixel_simd_t c16_9 =
dt_simd_set1(16.0f / 9.0f);
266 const dt_aligned_pixel_simd_t interval_v =
dt_simd_set1(interval);
267 const dt_aligned_pixel_simd_t iter =
dt_simd_set1(4.0f * interval);
268 dt_aligned_pixel_simd_t vt =
dt_simd_set1(first_tap) + bootstrap * interval_v;
270 const int runs = (num_taps + 3) / 4;
272 for(
size_t i = 0;
i < runs;
i++)
274 const dt_aligned_pixel_simd_t a = dt_simd_abs(vt);
275 const dt_aligned_pixel_simd_t a2 = a * a;
276 const dt_aligned_pixel_simd_t a3 = a2 * a;
278 const dt_aligned_pixel_simd_t r01 = c7_6 * a3 -
c2 * a2 + c8_9;
279 const dt_aligned_pixel_simd_t r12 =
c2 * a2 - c7_18 * a3 - c10_3 * a + c16_9;
280 dt_aligned_pixel_simd_t taps4 = r12;
282 taps4[c] = (a[c] <= 1.0f) ? r01[c] : r12[c];
330 int f = (int)floorf(
t) - itor->
width + 1;
359 const float outoinratio,
363 const float w = (float)itor->
width;
367 const float xin =
ceil_fast(((
float)xout - w) / outoinratio);
374 float t = xin * outoinratio - (float)xout;
377 int num_taps = *taps = (int)((w -
t) / outoinratio);
383 for(
size_t i = 0;
i < num_taps;
i++)
393#define MAX_KERNEL_REQ ((2 * (MAX_HALF_FILTER_WIDTH) + 3) & (~3))
402 const int samplestride,
403 const int linestride)
411 float normh = _compute_upsampling_kernel(itor, kernelh, NULL,
x);
412 float normv = _compute_upsampling_kernel(itor, kernelv, NULL, y);
428 in = (
float *)in + linestride * iy + ix * samplestride;
429 in = in - (itor->
width - 1) * (samplestride + linestride);
433 for(
int i = 0;
i < 2 * itor->
width;
i++)
436 for(
int j = 0; j < 2 * itor->
width; j++)
438 h += kernelh[j] * in[j * samplestride];
443 r = fmaxf(0.0f, s / (normh * normv));
445 else if(ix >= 0 && iy >= 0 && ix <
width && iy <
height)
450 iy -= itor->
width - 1;
451 ix -= itor->
width - 1;
458 _prepare_tap_boundaries(&xtap_first, &xtap_last,
463 _prepare_tap_boundaries(&ytap_first, &ytap_last,
468 for(ssize_t
i = ytap_first;
i < ytap_last;
i++)
470 const ssize_t clip_y = _clip(iy +
i, 0,
height - 1, bordermode);
472 for(ssize_t j = xtap_first; j < xtap_last; j++)
474 const ssize_t clip_x = _clip(ix + j, 0,
width - 1, bordermode);
475 const float *ipixel = in + clip_y * linestride + clip_x * samplestride;
476 h += kernelh[j] * ipixel[0];
481 r = fmaxf(0.0f, s / (normh * normv));
503 const int linestride)
512 float normh = _compute_upsampling_kernel(itor, kernelh, NULL,
x);
513 float normv = _compute_upsampling_kernel(itor, kernelv, NULL, y);
516 const float oonorm = (1.f / (normh * normv));
525 if(ix >= (itor->
width - 1)
526 && iy >= (itor->
width - 1)
533 in = (
float *)in + linestride * iy + ix * 4;
534 in = in - (itor->
width - 1) * (4 + linestride);
536 const size_t itor_width = 2 * itor->
width;
540 for(
size_t i = 0;
i < itor_width;
i++)
543 for(
size_t j = 0; j < itor_width; j++)
544 h += dt_load_simd_aligned(in + 4 * j) *
dt_simd_set1(kernelh[j]);
551 else if(ix >= 0 && iy >= 0 && ix <
width && iy <
height)
556 iy -= itor->
width - 1;
557 ix -= itor->
width - 1;
564 _prepare_tap_boundaries(&xtap_first, &xtap_last,
569 _prepare_tap_boundaries(&ytap_first, &ytap_last,
574 for(ssize_t
i = ytap_first;
i < ytap_last;
i++)
576 const ssize_t clip_y = _clip(iy +
i, 0,
height - 1, bordermode);
578 const float *ipixel = in + clip_y * linestride;
579 for(ssize_t j = xtap_first; j < xtap_last; j++)
581 const ssize_t clip_x = _clip(ix + j, 0,
width - 1, bordermode);
582 h += dt_load_simd_aligned(ipixel + 4 * clip_x) *
dt_simd_set1(kernelh[j]);
742 maxtapsapixel = 2 * itor->
width;
751 const int nindex = maxtapsapixel *
out;
752 const int nkernel = maxtapsapixel *
out;
760 const size_t totalreq = kernelreq + lengthreq + indexreq + scratchreq + metareq;
764 int *lengths = (
int *)blob;
765 blob = (
char *)blob + lengthreq;
766 int *index = (
int *)blob;
767 blob = (
char *)blob + indexreq;
768 float *
kernel = (
float *)blob;
769 blob = (
char *)blob + kernelreq;
775 float *scratchpad = (
float *)blob;
776 blob = (
char *)blob + scratchreq;
777 int *
meta = metareq ? (
int *)blob : NULL;
793 for(
int x = 0;
x <
out;
x++)
803 float fx = (float)(out_x0 +
x) / scale - in_x0;
807 (
void)_compute_upsampling_kernel(itor, scratchpad, &first,
fx);
813 _prepare_tap_boundaries(&tap_first, &tap_last, bordermode, 2 * itor->
width, first, in);
816 lengths[lidx++] = tap_last - tap_first;
820 for(
int tap = tap_first; tap < tap_last; tap++)
822 norm += scratchpad[tap];
831 for(
int tap = tap_first; tap < tap_last; tap++)
833 kernel[kidx++] = scratchpad[tap] * norm;
834 index[iidx++] = _clip(first++, 0, in - 1, bordermode);
844 for(
int x = 0;
x <
out;
x++)
862 _prepare_tap_boundaries(&tap_first, &tap_last, bordermode, taps, first, in);
865 lengths[lidx++] = tap_last - tap_first;
869 for(
int tap = tap_first; tap < tap_last; tap++)
871 norm += scratchpad[tap];
880 for(
int tap = tap_first; tap < tap_last; tap++)
882 kernel[kidx++] = scratchpad[tap] * norm;
883 index[iidx++] = _clip(first++, 0, in - 1, bordermode);
904 float *
const restrict
out,
906 const float *
const restrict in,
911 float *hkernel = NULL;
914 float *vkernel = NULL;
917 const int32_t in_stride_floats = roi_in->
width * 4;
918 const int32_t out_stride_floats = roi_out->
width * 4;
923 const size_t x0 = (roi_out->
x - roi_in->
x) * 4 *
sizeof(
float);
924 const size_t y0 = (roi_out->
y - roi_in->
y);
929 for(
int y = yt; y < y_end; y++)
930 memcpy((
char *)__builtin_assume_aligned(
out, 64) + (
size_t)out_stride_floats *
sizeof(
float) * y,
931 (
char *)__builtin_assume_aligned(in, 64) + (
size_t)in_stride_floats *
sizeof(
float) * (y + y0) + x0,
932 out_stride_floats *
sizeof(
float));
944 const float resample_scale = roi_out->
scale / roi_in->
scale;
947 roi_out->
width, roi_out->
x, resample_scale,
948 &hlength, &hkernel, &hindex, NULL))
952 roi_out->
height, roi_out->
y, resample_scale,
953 &vlength, &vkernel, &vindex, &vmeta))
961 for(
size_t oy = 0; oy <
height; oy++)
964 int vlidx = vmeta[3 * oy + 0];
965 int vkidx = vmeta[3 * oy + 1];
966 int viidx = vmeta[3 * oy + 2];
973 int vl = vlength[vlidx++];
976 for(
size_t ox = 0; ox <
width; ox++)
982 const int hl = hlength[hlidx++];
983 const int *
const column_hindex = hindex + hkidx;
984 const float *
const column_hkernel = hkernel + hkidx;
985 const int *
const column_vindex = vindex + viidx;
986 const float *
const column_vkernel = vkernel + vkidx;
988 for(
size_t iy = 0; iy < vl; iy++)
991 const size_t baseidx_vindex = (size_t)column_vindex[iy] * in_stride_floats;
995 for(
size_t ix = 0; ix < hl; ix++)
998 const size_t baseidx = baseidx_vindex + (size_t)column_hindex[ix] * 4;
999 const float htap = column_hkernel[ix];
1000 vhs += dt_load_simd_aligned(in + baseidx) *
dt_simd_set1(htap);
1004 const float vtap = column_vkernel[iy];
1009 const size_t baseidx = (size_t)oy * out_stride_floats + (
size_t)ox * 4;
1040 const float *
const in,
1054 const float *
const in,
1079 const int program = 2;
1118 int *hlength = NULL;
1119 float *hkernel = NULL;
1122 int *vlength = NULL;
1123 float *vkernel = NULL;
1128 cl_mem dev_hindex = NULL;
1129 cl_mem dev_hlength = NULL;
1130 cl_mem dev_hkernel = NULL;
1131 cl_mem dev_hmeta = NULL;
1132 cl_mem dev_vindex = NULL;
1133 cl_mem dev_vlength = NULL;
1134 cl_mem dev_vkernel = NULL;
1135 cl_mem dev_vmeta = NULL;
1140 size_t iorigin[] = { roi_out->
x - roi_in->
x, roi_out->
y - roi_in->
y, 0 };
1141 size_t oorigin[] = { 0, 0, 0 };
1142 size_t region[] = { roi_out->
width, roi_out->
height, 1 };
1146 if(err != CL_SUCCESS)
goto error;
1156 const float resample_scale = roi_out->
scale / roi_in->
scale;
1159 roi_out->
width, roi_out->
x, resample_scale,
1160 &hlength, &hkernel, &hindex, &hmeta))
1164 roi_out->
height, roi_out->
y, resample_scale,
1165 &vlength, &vkernel, &vindex, &vmeta))
1168 int hmaxtaps = -1, vmaxtaps = -1;
1169 for(
int k = 0;
k < roi_out->
width;
k++) hmaxtaps =
MAX(hmaxtaps, hlength[
k]);
1170 for(
int k = 0;
k < roi_out->
height;
k++) vmaxtaps =
MAX(vmaxtaps, vlength[
k]);
1193 .cellsize = 4 *
sizeof(float),
1194 .overhead = hmaxtaps *
sizeof(
float) + hmaxtaps *
sizeof(int),
1196 .sizey = (1 << 16) * taps };
1199 vblocksize = locopt.
sizey;
1203 if(vblocksize < taps)
1209 err = CL_INVALID_WORK_GROUP_SIZE;
1214 size_t local[3] = { 1, vblocksize, 1 };
1304 const float *
const in,
1308 int *hlength = NULL;
1309 float *hkernel = NULL;
1311 int *vlength = NULL;
1312 float *vkernel = NULL;
1316 const size_t out_stride = roi_out->
width *
sizeof(float);
1317 const size_t in_stride = roi_in->
width *
sizeof(float);
1322 const size_t x0 = (roi_out->
x - roi_in->
x) *
sizeof(
float);
1323 const size_t y0 = (roi_out->
y - roi_in->
y);
1325 for(
int y = 0; y < roi_out->
height; y++)
1327 float *
i = (
float *)((
char *)in + in_stride * (y + y0) + x0);
1328 float *o = (
float *)((
char *)
out + out_stride * y);
1329 memcpy(o,
i, out_stride);
1340 &hlength, &hkernel, &hindex, NULL))
1345 &vlength, &vkernel, &vindex, &vmeta))
1350 for(
int oy = 0; oy < roi_out->
height; oy++)
1353 int vlidx = vmeta[3 * oy + 0];
1354 int vkidx = vmeta[3 * oy + 1];
1355 int viidx = vmeta[3 * oy + 2];
1363 int vl = vlength[vlidx++];
1366 for(
int ox = 0; ox < roi_out->
width; ox++)
1372 const int hl = hlength[hlidx++];
1373 for(
int iy = 0; iy < vl; iy++)
1376 const float *
i = (
float *)((
char *)in + in_stride * vindex[viidx++]);
1380 for(
int ix = 0; ix < hl; ix++)
1383 const size_t baseidx = (size_t)hindex[hiidx++];
1384 const float htap = hkernel[hkidx++];
1385 vhs +=
i[baseidx] * htap;
1389 const float vtap = vkernel[vkidx++];
1398 float *o = (
float *)((
char *)
out + (size_t)oy * out_stride
1399 + (
size_t)ox *
sizeof(float));
1426 const float *
const in,
1439 const float *
const in,
static void error(char *msg)
typedef void((*dt_cache_allocate_t)(void *userdata, dt_cache_entry_t *entry))
return vector dt_simd_set1(valid ?(scaling+NORM_MIN) :NORM_MIN)
static const float const float const float min
const dt_colormatrix_t dt_aligned_pixel_t out
dt_store_simd_aligned(out, dt_mat3x4_mul_vec4(vin, dt_colormatrix_row_to_simd(matrix, 0), dt_colormatrix_row_to_simd(matrix, 1), dt_colormatrix_row_to_simd(matrix, 2)))
const char * dt_conf_get_string_const(const char *name)
Borrow the stored string for name without copying it.
static CameraMetaData * meta
const struct dt_interpolation * dt_interpolation_new(enum dt_interpolation_type type)
#define MAX_HALF_FILTER_WIDTH
__DT_CLONE_TARGETS__ void dt_interpolation_compute_pixel4c(const struct dt_interpolation *itor, const float *in, float *out, const float x, const float y, const int width, const int height, const int linestride)
static void _interpolation_resample_1c_plain(const struct dt_interpolation *itor, float *out, const dt_iop_roi_t *const roi_out, const float *const in, const dt_iop_roi_t *const roi_in)
static float _maketaps_mitchell(float *taps, const size_t num_taps, const float width, const float first_tap, const float interval)
static uint32_t roundToNextPowerOfTwo(uint32_t x)
#define RESAMPLING_BORDER_MODE
void dt_interpolation_resample_roi(const struct dt_interpolation *itor, float *out, const dt_iop_roi_t *const roi_out, const float *const in, const dt_iop_roi_t *const roi_in)
static __DT_CLONE_TARGETS__ gboolean _prepare_resampling_plan(const struct dt_interpolation *itor, const int in, const int in_x0, const int out, const int out_x0, const float scale, int **plength, float **pkernel, int **pindex, int **pmeta)
void dt_interpolation_resample_roi_1c(const struct dt_interpolation *itor, float *out, const dt_iop_roi_t *const roi_out, const float *const in, const dt_iop_roi_t *const roi_in)
void dt_interpolation_resample(const struct dt_interpolation *itor, float *out, const dt_iop_roi_t *const roi_out, const float *const in, const dt_iop_roi_t *const roi_in)
int dt_interpolation_resample_cl(const struct dt_interpolation *itor, const int devid, cl_mem dev_out, const dt_iop_roi_t *const roi_out, cl_mem dev_in, const dt_iop_roi_t *const roi_in)
void dt_interpolation_resample_1c(const struct dt_interpolation *itor, float *out, const dt_iop_roi_t *const roi_out, const float *const in, const dt_iop_roi_t *const roi_in)
static dt_interpolation_cl_global_t * _interpolation_cl_global
void dt_interpolation_free_cl_global(void)
static float _maketaps_bilinear(float *taps, const size_t num_taps, const float width, const float first_tap, const float interval)
int dt_interpolation_resample_roi_cl(const struct dt_interpolation *itor, const int devid, cl_mem dev_out, const dt_iop_roi_t *const roi_out, cl_mem dev_in, const dt_iop_roi_t *const roi_in)
void dt_interpolation_init_cl_global(void)
static const struct dt_interpolation dt_interpolator[]
__DT_CLONE_TARGETS__ float dt_interpolation_compute_sample(const struct dt_interpolation *itor, const float *in, const float x, const float y, const int width, const int height, const int samplestride, const int linestride)
#define INTERPOLATION_BORDER_MODE
static __DT_CLONE_TARGETS__ void _interpolation_resample_plain(const struct dt_interpolation *itor, float *const restrict out, const dt_iop_roi_t *const roi_out, const float *const restrict in, const dt_iop_roi_t *const roi_in)
static void _compute_downsampling_kernel(const struct dt_interpolation *itor, int *taps, int *first, float *kernel, float *norm, const float outoinratio, const int xout)
static float _maketaps_bicubic(float *taps, const size_t num_taps, const float width, const float first_tap, const float interval)
@ DT_INTERPOLATION_BICUBIC
@ DT_INTERPOLATION_BILINEAR
@ DT_INTERPOLATION_DEFAULT
@ DT_INTERPOLATION_MITCHELL
@ DT_INTERPOLATION_USERPREF
@ DT_INTERPOLATION_DEFAULT_WARP
@ DT_INTERPOLATION_USERPREF_WARP
static float kernel(const float *x, const float *y)
_lib_location_type_t type
float *const restrict const size_t k
#define IS_NULL_PTR(p)
C is way too permissive with !=, == and if(var) checks, which can mean too many things depending on w...
static float ceil_fast(float x)
#define DT_ALIGNED_ARRAY
Align an object on a cacheline boundary, so AVX2 can load it whole.
#define dt_free(ptr)
g_free() ptr and set it to NULL, skipping both if it is already NULL.
static size_t dt_round_size(const size_t size, const size_t alignment)
Round size UP to the next multiple of alignment.
#define DT_CACHELINE_BYTES
int dt_opencl_local_buffer_opt(const int devid, const int kernel, dt_opencl_local_buffer_t *factors)
int dt_opencl_create_kernel(const int prog, const char *name)
void * dt_opencl_copy_host_to_device_constant(const int devid, const size_t size, void *host)
int dt_opencl_enqueue_copy_image(const int devid, cl_mem src, cl_mem dst, size_t *orig_src, size_t *orig_dst, size_t *region)
void dt_opencl_free_kernel(const int kernel)
int dt_opencl_set_kernel_arg(const int dev, const int kernel, const int num, const size_t size, const void *arg)
int dt_opencl_enqueue_kernel_2d_with_local(const int dev, const int kernel, const size_t *sizes, const size_t *local)
void dt_opencl_release_mem_object(cl_mem mem)
#define DT_OPENCL_DEFAULT_ERROR
#define dt_omploop_sfence()
#define __OMP_PARALLEL_FOR__(...)
#define dt_pixelpipe_cache_alloc_align_cache(size, id)
#define dt_pixelpipe_cache_free_align(mem)
dt_store_simd(out, value)
DT_ALIGNED_PIXEL float dt_aligned_pixel_t[4]
float dt_aligned_pixel_simd_t __attribute__((vector_size(16), aligned(16)))
Apply one channel's tone curve to each of the three colour channels, or pass the channel through unto...
static void copy_pixel_nontemporal(float *const __restrict__ out, const float *const __restrict__ in)
#define for_four_channels(_var,...)
int kernel_interpolation_resample
dt_interpolation_func maketaps
enum dt_interpolation_type id
Region of interest passed through the pixelpipe.
#define __DT_CLONE_TARGETS__