18#ifndef DT_SYSTEM_SIMD_H
19#define DT_SYSTEM_SIMD_H
31#if defined(__aarch64__)
35#if defined(__x86_64__) || defined(__i386__)
55typedef float dt_aligned_pixel_simd_t
__attribute__((vector_size(16), aligned(16)));
60#ifdef DT_NO_VECTORIZATION
61#define DT_PIXEL_SIMD_CHANNELS 3
63#define DT_PIXEL_SIMD_CHANNELS 4
76#if defined(_OPENMP) && defined(OPENMP_SIMD_) && !defined(DT_NO_SIMD_HINTS)
78#define _DT_Pragma_(x) _Pragma(#x)
79#define _DT_Pragma(x) _DT_Pragma_(x)
80#define for_each_channel(_var, ...) \
81 _DT_Pragma(omp simd __VA_ARGS__) \
82 for (size_t _var = 0; _var < DT_PIXEL_SIMD_CHANNELS; _var++)
83#define for_four_channels(_var, ...) \
84 _DT_Pragma(omp simd __VA_ARGS__) \
85 for (size_t _var = 0; _var < 4; _var++)
87#define for_each_channel(_var, ...) \
88 for (size_t _var = 0; _var < DT_PIXEL_SIMD_CHANNELS; _var++)
89#define for_four_channels(_var, ...) \
90 for (size_t _var = 0; _var < 4; _var++)
98static inline __attribute__((always_inline)) dt_aligned_pixel_simd_t
99dt_simd_abs(
const dt_aligned_pixel_simd_t
value)
101 dt_aligned_pixel_simd_t
out =
value;
102 for(
int c = 0;
c < 4;
c++)
107static inline __attribute__((always_inline)) dt_aligned_pixel_simd_t
108dt_simd_max_zero(
const dt_aligned_pixel_simd_t
value)
110 dt_aligned_pixel_simd_t
out =
value;
111 for(
int c = 0;
c < 4;
c++)
116static inline __attribute__((always_inline)) dt_aligned_pixel_simd_t
117dt_simd_copysign(
const dt_aligned_pixel_simd_t magnitude,
const dt_aligned_pixel_simd_t
sign)
119 dt_aligned_pixel_simd_t
out = magnitude;
120 for(
int c = 0; c < 4; c++)
121 out[c] = copysignf(magnitude[c],
sign[c]);
125static inline __attribute__((always_inline)) dt_aligned_pixel_simd_t
126dt_simd_pow(
const dt_aligned_pixel_simd_t base,
const dt_aligned_pixel_simd_t
exponent)
128 dt_aligned_pixel_simd_t
out = base;
129 for(
int c = 0; c < 4; c++)
134static inline __attribute__((always_inline)) dt_aligned_pixel_simd_t
135dt_load_simd(
const float *
const pixel)
137 dt_aligned_pixel_simd_t
out;
138 __builtin_memcpy(&
out, pixel,
sizeof(
out));
145 __builtin_memcpy(pixel, &
value,
sizeof(
value));
148static inline __attribute__((always_inline)) dt_aligned_pixel_simd_t
149dt_load_simd_aligned(
const float *
const pixel)
151 const float *
const in = (
const float *
const)__builtin_assume_aligned(pixel, 16);
152 return dt_load_simd(
in);
158 float *
const out = (
float *
const)__builtin_assume_aligned(pixel, 16);
163dt_store_simd_nontemporal(
float *
const pixel,
const dt_aligned_pixel_simd_t
value)
165 float *
const out = (
float *
const)__builtin_assume_aligned(pixel, 16);
167#if defined(__x86_64__) || defined(__i386__)
170 dt_aligned_pixel_simd_t simd;
172 } cast = { .simd =
value };
173 _mm_stream_ps(
out, cast.sse);
174#elif defined(__aarch64__)
177 dt_aligned_pixel_simd_t simd;
179 } cast = { .simd =
value };
180 vst1q_f32(
out, cast.neon);
181#elif (__clang__+0 > 7) && (__clang__+0 < 10)
188static inline __attribute__((always_inline)) dt_aligned_pixel_simd_t
189dt_mat3x4_mul_vec4(
const dt_aligned_pixel_simd_t
in,
const dt_aligned_pixel_simd_t
row0,
190 const dt_aligned_pixel_simd_t
row1,
const dt_aligned_pixel_simd_t
row2)
194 dt_aligned_pixel_simd_t
out =
row0 *
in[0];
208 float *
const __restrict__
out,
209 const float *
const __restrict__
in)
211 dt_store_simd_nontemporal(
out, dt_load_simd(
in));
217static inline void copy_pixel(
float *
const __restrict__
out,
const float *
const __restrict__
in)
return vector dt_simd_set1(valid ?(scaling+NORM_MIN) :NORM_MIN)
const dt_colormatrix_t dt_aligned_pixel_t out
dt_store_simd_aligned(out, dt_mat3x4_mul_vec4(vin, dt_colormatrix_row_to_simd(matrix, 0), dt_colormatrix_row_to_simd(matrix, 1), dt_colormatrix_row_to_simd(matrix, 2)))
float *const restrict const size_t k
#define DT_ALIGNED_PIXEL
Align a 4-float pixel on 16 bytes, enough for SSE. Same struct-member caveat as DT_ALIGNED_ARRAY,...
static const dt_aligned_pixel_simd_t const dt_aligned_pixel_simd_t row1
dt_store_simd(out, value)
static void copy_pixel(float *const __restrict__ out, const float *const __restrict__ in)
DT_ALIGNED_PIXEL float dt_aligned_pixel_t[4]
#define for_each_channel(_var,...)
float dt_aligned_pixel_simd_t __attribute__((vector_size(16), aligned(16)))
Apply one channel's tone curve to each of the three colour channels, or pass the channel through unto...
static void copy_pixel_nontemporal(float *const __restrict__ out, const float *const __restrict__ in)
static const dt_aligned_pixel_simd_t sign
static const dt_aligned_pixel_simd_t const dt_aligned_pixel_simd_t const dt_aligned_pixel_simd_t row2
static const dt_aligned_pixel_simd_t exponent
static const dt_aligned_pixel_simd_t row0