opencl.h

Go to the documentation of this file.

/*
    This file is part of darktable,
    Copyright (C) 2010-2022 darktable developers.
 
    darktable is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
 
    darktable is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
 
    You should have received a copy of the GNU General Public License
    along with darktable.  If not, see <http://www.gnu.org/licenses/>.
*/
 
#pragma once
 
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
 
#define DT_OPENCL_MAX_PLATFORMS 5
#define DT_OPENCL_MAX_PROGRAMS 256
#define DT_OPENCL_MAX_KERNELS 512
#define DT_OPENCL_EVENTLISTSIZE 256
#define DT_OPENCL_EVENTNAMELENGTH 64
#define DT_OPENCL_MAX_ERRORS 5
#define DT_OPENCL_MAX_INCLUDES 7
#define DT_OPENCL_VENDOR_AMD 4098
#define DT_OPENCL_VENDOR_NVIDIA 4318
#define DT_OPENCL_VENDOR_INTEL 0x8086u
#define DT_OPENCL_CBUFFSIZE 1024
 
// some pseudo error codes in dt opencl usage
#define DT_OPENCL_DEFAULT_ERROR -999
#define DT_OPENCL_SYSMEM_ALLOCATION -998
 
#include "common/darktable.h"
 
#ifdef HAVE_OPENCL
 
#include "common/dlopencl.h"
#include "common/dtpthread.h"
#include "common/iop_profile.h"
#include "control/conf.h"
 
// #pragma GCC diagnostic push
// #pragma GCC diagnostic ignored "-Wcomment"
#include <CL/cl.h>
// #pragma GCC diagnostic
 
#ifdef __cplusplus
extern "C" {
#endif
 
#define ROUNDUP(a, n) ((a) % (n) == 0 ? (a) : ((a) / (n)+1) * (n))
 
// use per device roundups here
#define ROUNDUPDWD(a, b) dt_opencl_dev_roundup_width(a, b)
#define ROUNDUPDHT(a, b) dt_opencl_dev_roundup_height(a, b)
 
#define DT_OPENCL_DEFAULT_COMPILE_INTEL ("-cl-fast-relaxed-math")
#define DT_OPENCL_DEFAULT_COMPILE_AMD ("-cl-fast-relaxed-math")
#define DT_OPENCL_DEFAULT_COMPILE_NVIDIA ("-cl-fast-relaxed-math")
#define DT_OPENCL_DEFAULT_COMPILE ("-cl-fast-relaxed-math")
#define DT_CLDEVICE_HEAD ("cldevice_v4_")
 
typedef enum dt_opencl_memory_t
{
  OPENCL_MEMORY_ADD,
  OPENCL_MEMORY_SUB
} dt_opencl_memory_t;
 
typedef struct dt_opencl_eventtag_t
{
  cl_int retval;
  cl_ulong timelapsed;
  char tag[DT_OPENCL_EVENTNAMELENGTH];
} dt_opencl_eventtag_t;
 
typedef enum dt_opencl_pinmode_t
{
  DT_OPENCL_PINNING_OFF = 0,
  DT_OPENCL_PINNING_ON = 1,
  DT_OPENCL_PINNING_DISABLED = 2
} dt_opencl_pinmode_t;
 
typedef struct dt_opencl_device_t
{
  dt_pthread_mutex_t lock;
  cl_device_id devid;
  cl_context context;
  cl_command_queue cmd_queue;
  size_t max_image_width;
  size_t max_image_height;
  cl_ulong max_mem_alloc;
  cl_ulong max_global_mem;
  cl_ulong used_global_mem;
  cl_program program[DT_OPENCL_MAX_PROGRAMS];
  cl_kernel kernel[DT_OPENCL_MAX_KERNELS];
  int program_used[DT_OPENCL_MAX_PROGRAMS];
  int kernel_used[DT_OPENCL_MAX_KERNELS];
  cl_event *eventlist;
  dt_opencl_eventtag_t *eventtags;
  int numevents;
  int eventsconsolidated;
  int maxevents;
  int lostevents;
  int totalevents;
  int totalsuccess;
  int totallost;
  int maxeventslot;
  int nvidia_sm_20;
  const char *vendor;
  const char *name;
  const char *cname;
  const char *options;
  cl_int summary;
  // the benchmark value must not be changed by the user
  float benchmark;
  size_t memory_in_use;
  size_t peak_memory;
  size_t used_available;
 
  // flags detected errors
  int runtime_error;
  // if set to TRUE darktable will not use OpenCL kernels which contain atomic operations (example bilateral).
  // pixelpipe processing will be done on CPU for the affected modules.
  // useful (only for very old devices) if your OpenCL implementation freezes/crashes on atomics or if
  // they are processed with a bad performance.
  int avoid_atomics;
 
  // pause OpenCL processing for this number of microseconds from time to time
  int micro_nap;
 
  // During tiling huge amounts of memory need to be transferred between host and device.
  // For some OpenCL implementations direct memory transfers give a drastic performance penalty,
  // this can often be avoided by using indirect transfers via pinned memory,
  // other devices have more efficient direct memory transfer implementations.
  // We can't predict on solid grounds if a device belongs to the first or second group,
  // also pinned mem transfer requires slightly more ram.
  // this holds a bitmask defined by dt_opencl_pinmode_t
  // the device specific conf key might hold
  // 0 -> disabled by default; might be switched on by tune for performance
  // 1 -> enabled by default
  // 2 -> disabled under all circumstances. This could/should be used if we give away / ship specific keys for buggy systems
  int pinned_memory;
 
  // in OpenCL processing round width/height of global work groups to a multiple of these values.
  // reasonable values are powers of 2. this parameter can have high impact on OpenCL performance.
  int clroundup_wd;
  int clroundup_ht;
 
  // A bitfield that identifies the type of OpenCL device required to test for on-CPU and more.
  unsigned int cltype;
 
  // This defines how often should dt_opencl_events_get_slot do a dt_opencl_events_flush.
  // It should definitely le lower than the number of events that can be handled by the device/driver.
  // FIXME we should be able to test for that with using >= OpenCl 2.0
  int event_handles;
 
  // opencl_events enabled for the device, set internally via event_handles
  int use_events;
 
  // a device might be turned off by force by setting this value to 1
  // also used for blacklisted drivers
  int disabled;
 
  // Some devices are known to be unused by other apps so there is no need to test for available memory at all.
  // Also some devices might behave badly with the checking code, in this case we could enforce a headroom here.
  size_t forced_headroom;
} dt_opencl_device_t;
 
struct dt_bilateral_cl_global_t;
struct dt_local_laplacian_cl_global_t;
struct dt_dwt_cl_global_t; // wavelet decompose
struct dt_heal_cl_global_t; // healing
struct dt_colorspaces_cl_global_t; // colorspaces transform
struct dt_guided_filter_cl_global_t;
 
typedef struct dt_opencl_t
{
  dt_pthread_mutex_t lock;
  int inited;
  int print_statistics;
  int enabled;
  int stopped;
  int num_devs;
  int error_count;
  int opencl_synchronization_timeout;
  uint32_t crc;
  int mandatory[5];
  int *dev_priority_image;
  int *dev_priority_preview;
  int *dev_priority_export;
  int *dev_priority_thumbnail;
  dt_opencl_device_t *dev;
  dt_dlopencl_t *dlocl;
 
  // Benchmarking GUI to make user wait while we run expensive stuff in background
  GtkWidget *dialog;
  GtkWidget *progress;
  GtkWidget *label;
  char progress_label[256];
  double steps;
  double step;
 
  // we want the cpu benchmark to be available
  float cpubenchmark;
  // global kernels for blending operations.
  struct dt_blendop_cl_global_t *blendop;
 
  // global kernels for bilateral filtering, to be reused by a few plugins.
  struct dt_bilateral_cl_global_t *bilateral;
 
  // global kernels for gaussian filtering, to be reused by a few plugins.
  struct dt_gaussian_cl_global_t *gaussian;
 
  // global kernels for interpolation resampling.
  struct dt_interpolation_cl_global_t *interpolation;
 
  // global kernels for local laplacian filter.
  struct dt_local_laplacian_cl_global_t *local_laplacian;
 
  // global kernels for dwt filter.
  struct dt_dwt_cl_global_t *dwt;
 
  // global kernels for heal filter.
  struct dt_heal_cl_global_t *heal;
 
  // global kernels for colorspaces filter.
  struct dt_colorspaces_cl_global_t *colorspaces;
 
  // global kernels for guided filter.
  struct dt_guided_filter_cl_global_t *guided_filter;
} dt_opencl_t;
 
typedef struct dt_opencl_local_buffer_t
{
  const int xoffset;
  const int xfactor;
  const int yoffset;
  const int yfactor;
  const size_t cellsize;
  const size_t overhead;
  int sizex;  // initial value and final values after optimization
  int sizey;  // initial value and final values after optimization
} dt_opencl_local_buffer_t;
 
int dt_opencl_get_device_info(dt_opencl_t *cl, cl_device_id device, cl_device_info param_name, void **param_value,
                              size_t *param_value_size);
 
void dt_opencl_init(dt_opencl_t *cl, const gboolean exclude_opencl, const gboolean print_statistics);
 
void dt_opencl_cleanup(dt_opencl_t *cl);
 
void dt_opencl_cleanup_device(dt_opencl_t *cl, int i);
 
int dt_opencl_finish(const int devid);
 
int dt_opencl_enqueue_barrier(const int devid);
 
int dt_opencl_lock_device(const int pipetype);
 
void dt_opencl_unlock_device(const int dev);
 
void dt_opencl_md5sum(const char **files, char **md5sums);
 
int dt_opencl_load_program(const int dev, const int prog, const char *filename, const char *binname,
                           const char *cachedir, char *md5sum, char **includemd5, int *loaded_cached);
 
int dt_opencl_build_program(const int dev, const int prog, const char *binname, const char *cachedir,
                            char *md5sum, int loaded_cached);
 
int dt_opencl_create_kernel(const int program, const char *name);
 
void dt_opencl_free_kernel(const int kernel);
 
int dt_opencl_get_max_work_item_sizes(const int dev, size_t *sizes);
 
int dt_opencl_get_work_group_limits(const int dev, size_t *sizes, size_t *workgroupsize,
                                    unsigned long *localmemsize);
 
int dt_opencl_get_kernel_work_group_size(const int dev, const int kernel, size_t *kernelworkgroupsize);
 
int dt_opencl_set_kernel_arg(const int dev, const int kernel, const int num, const size_t size,
                             const void *arg);
 
int dt_opencl_enqueue_kernel_2d(const int dev, const int kernel, const size_t *sizes);
 
int dt_opencl_enqueue_kernel_2d_with_local(const int dev, const int kernel, const size_t *sizes,
                                           const size_t *local);
 
int dt_opencl_is_inited(void);
 
int dt_opencl_is_enabled(void);
 
void dt_opencl_disable(void);
 
int dt_opencl_update_settings(void);
 
int dt_opencl_copy_device_to_host(const int devid, void *host, void *device, const int width,
                                  const int height, const int bpp);
 
int dt_opencl_read_host_from_device(const int devid, void *host, void *device, const int width,
                                    const int height, const int bpp);
 
int dt_opencl_read_host_from_device_rowpitch(const int devid, void *host, void *device, const int width,
                                             const int height, const int rowpitch);
 
int dt_opencl_read_host_from_device_non_blocking(const int devid, void *host, void *device, const int width,
                                                 const int height, const int bpp);
 
int dt_opencl_read_host_from_device_rowpitch_non_blocking(const int devid, void *host, void *device,
                                                          const int width, const int height,
                                                          const int rowpitch);
 
int dt_opencl_read_host_from_device_raw(const int devid, void *host, void *device, const size_t *origin,
                                        const size_t *region, const int rowpitch, const int blocking);
 
int dt_opencl_write_host_to_device(const int devid, void *host, void *device, const int width,
                                   const int height, const int bpp);
 
int dt_opencl_write_host_to_device_rowpitch(const int devid, void *host, void *device, const int width,
                                            const int height, const int rowpitch);
 
int dt_opencl_write_host_to_device_non_blocking(const int devid, void *host, void *device, const int width,
                                                const int height, const int bpp);
 
int dt_opencl_write_host_to_device_rowpitch_non_blocking(const int devid, void *host, void *device,
                                                         const int width, const int height,
                                                         const int rowpitch);
 
int dt_opencl_write_host_to_device_raw(const int devid, void *host, void *device, const size_t *origin,
                                       const size_t *region, const int rowpitch, const int blocking);
 
void *dt_opencl_copy_host_to_device(const int devid, void *host, const int width, const int height,
                                    const int bpp);
 
void *dt_opencl_copy_host_to_device_rowpitch(const int devid, void *host, const int width, const int height,
                                             const int bpp, const int rowpitch);
 
void *dt_opencl_copy_host_to_device_constant(const int devid, const size_t size, void *host);
 
int dt_opencl_enqueue_copy_image(const int devid, cl_mem src, cl_mem dst, size_t *orig_src, size_t *orig_dst,
                                 size_t *region);
 
void *dt_opencl_alloc_device(const int devid, const int width, const int height, const int bpp);
 
void *dt_opencl_alloc_device_use_host_pointer(const int devid, const int width, const int height,
                                              const int bpp, void *host, const int flags);
 
int dt_opencl_enqueue_copy_image_to_buffer(const int devid, cl_mem src_image, cl_mem dst_buffer,
                                           size_t *origin, size_t *region, size_t offset);
 
int dt_opencl_enqueue_copy_buffer_to_image(const int devid, cl_mem src_buffer, cl_mem dst_image,
                                           size_t offset, size_t *origin, size_t *region);
 
int dt_opencl_enqueue_copy_buffer_to_buffer(const int devid, cl_mem src_buffer, cl_mem dst_buffer,
                                            size_t srcoffset, size_t dstoffset, size_t size);
 
int dt_opencl_read_buffer_from_device(const int devid, void *host, void *device, const size_t offset,
                                      const size_t size, const int blocking);
 
int dt_opencl_write_buffer_to_device(const int devid, void *host, void *device, const size_t offset,
                                     const size_t size, const int blocking);
 
void *dt_opencl_alloc_device_buffer(const int devid, const size_t size);
 
void *dt_opencl_alloc_device_buffer_with_flags(const int devid, const size_t size, const int flags);
 
void dt_opencl_release_mem_object(cl_mem mem);
 
void *dt_opencl_map_buffer(const int devid, cl_mem buffer, const int blocking, const int flags, size_t offset,
                           size_t size);
 
void *dt_opencl_map_image(const int devid, cl_mem buffer, const int blocking, const int flags, size_t width, size_t height, int bpp);
 
int dt_opencl_unmap_mem_object(const int devid, cl_mem mem_object, void *mapped_ptr);
 
size_t dt_opencl_get_mem_object_size(cl_mem mem);
 
int dt_opencl_get_image_width(cl_mem mem);
 
int dt_opencl_get_image_height(cl_mem mem);
 
int dt_opencl_get_image_element_size(cl_mem mem);
 
int dt_opencl_get_mem_context_id(cl_mem mem);
 
void dt_opencl_memory_statistics(int devid, cl_mem mem, dt_opencl_memory_t action);
 
gboolean dt_opencl_image_fits_device(const int devid, const size_t width, const size_t height, const unsigned bpp,
                                const float factor, const size_t overhead);
cl_ulong dt_opencl_get_device_available(const int devid);
 
void dt_opencl_check_tuning(const int devid);
 
cl_ulong dt_opencl_get_device_memalloc(const int devid);
 
int dt_opencl_dev_roundup_width(int size, const int devid);
int dt_opencl_dev_roundup_height(int size, const int devid);
 
cl_event *dt_opencl_events_get_slot(const int devid, const char *tag);
 
void dt_opencl_events_reset(const int devid);
 
void dt_opencl_events_wait_for(const int devid);
 
cl_int dt_opencl_events_flush(const int devid, const int reset);
 
void dt_opencl_events_profiling(const int devid, const int aggregated);
 
int dt_opencl_local_buffer_opt(const int devid, const int kernel, dt_opencl_local_buffer_t *factors);
 
void dt_opencl_write_device_config(const int devid);
gboolean dt_opencl_read_device_config(const int devid);
int dt_opencl_avoid_atomics(const int devid);
int dt_opencl_micro_nap(const int devid);
gboolean dt_opencl_use_pinned_memory(const int devid);
 
#ifdef __cplusplus
}
#endif
 
#else
 
#include "control/conf.h"
#include <stdlib.h>
 
#ifdef __cplusplus
extern "C" {
#endif
 
typedef struct dt_opencl_t
{
  int inited;
  int enabled;
  int stopped;
  int error_count;
} dt_opencl_t;
static inline void dt_opencl_init(dt_opencl_t *cl, const gboolean exclude_opencl, const gboolean print_statistics)
{
  cl->inited = 0;
  cl->enabled = 0;
  cl->stopped = 0;
  cl->error_count = 0;
  dt_conf_set_bool("opencl", FALSE);
  dt_print(DT_DEBUG_OPENCL, "[opencl_init] this version of darktable was built without opencl support\n");
}
static inline void dt_opencl_cleanup(dt_opencl_t *cl)
{
}
static inline gboolean dt_opencl_finish(const int devid)
{
  return -1;
}
static inline int dt_opencl_enqueue_barrier(const int devid)
{
  return -1;
}
static inline int dt_opencl_lock_device(const int dev)
{
  return -1;
}
static inline void dt_opencl_unlock_device(const int dev)
{
}
static inline int dt_opencl_load_program(const int dev, const char *filename)
{
  return -1;
}
static inline int dt_opencl_build_program(const int dev, const int program)
{
  return -1;
}
static inline int dt_opencl_create_kernel(const int program, const char *name)
{
  return -1;
}
static inline void dt_opencl_free_kernel(const int kernel)
{
}
static inline int dt_opencl_get_max_work_item_sizes(const int dev, size_t *sizes)
{
  return -1;
}
static inline int dt_opencl_get_work_group_limits(const int dev, size_t *sizes, size_t *workgroupsize,
                                                  unsigned long *localmemsize)
{
  return -1;
}
static inline int dt_opencl_get_kernel_work_group_size(const int dev, const int kernel,
                                                       size_t *kernelworkgroupsize)
{
  return -1;
}
static inline int dt_opencl_set_kernel_arg(const int dev, const int kernel, const size_t size, const void *arg)
{
  return -1;
}
static inline int dt_opencl_enqueue_kernel_2d(const int dev, const int kernel, const size_t *sizes)
{
  return -1;
}
static inline int dt_opencl_enqueue_kernel_2d_with_local(const int dev, const int kernel, const size_t *sizes,
                                                         const size_t *local)
{
  return -1;
}
static inline int dt_opencl_is_inited(void)
{
  return 0;
}
static inline int dt_opencl_is_enabled(void)
{
  return 0;
}
static inline void dt_opencl_disable(void)
{
}
static inline int dt_opencl_update_settings(void)
{
  return 0;
}
static inline gboolean dt_opencl_image_fits_device(const int devid, const size_t width, const size_t height,
                                              const unsigned bpp, const float factor, const size_t overhead)
{
  return FALSE;
}
static inline size_t dt_opencl_get_device_available(const int devid)
{
  return 0;
}
static inline void dt_opencl_check_tuning(const int devid)
{
  return;
}
static inline size_t dt_opencl_get_device_memalloc(const int devid)
{
  return 0;
}
static inline void dt_opencl_release_mem_object(void *mem)
{
}
static inline void *dt_opencl_events_get_slot(const int devid, const char *tag)
{
  return NULL;
}
static inline void dt_opencl_events_reset(const int devid)
{
}
static inline void dt_opencl_events_wait_for(const int devid)
{
}
static inline int dt_opencl_events_flush(const int devid, const int reset)
{
  return 0;
}
static inline void dt_opencl_events_profiling(const int devid, const int aggregated)
{
}
 
#ifdef __cplusplus
}
#endif
 
#endif
 
// clang-format off
// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
// vim: shiftwidth=2 expandtab tabstop=2 cindent
// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
// clang-format on