diff --git a/.github/workflows/release_src_artifact.yml b/.github/workflows/release_src_artifact.yml index c616c8db5b..41b01d4f72 100644 --- a/.github/workflows/release_src_artifact.yml +++ b/.github/workflows/release_src_artifact.yml @@ -9,7 +9,7 @@ name: ci jobs: upload_src_tarball: name: Upload release source tarball - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - name: Fetch Repo Info run: | @@ -40,7 +40,7 @@ jobs: libopenblas-dev \ ocl-icd-opencl-dev \ nvidia-cuda-toolkit \ - libboost1.68-dev + libboost-dev - name: CMake Configure run: | diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml index 3146358772..07ffba36f7 100644 --- a/.github/workflows/unix_cpu_build.yml +++ b/.github/workflows/unix_cpu_build.yml @@ -63,7 +63,7 @@ jobs: needs: [clang-format, documentation] env: NINJA_VER: 1.10.2 - CMAKE_VER: 3.10.2 + CMAKE_VER: 3.16.3 strategy: fail-fast: false matrix: @@ -93,7 +93,7 @@ jobs: chmod +x ninja ${GITHUB_WORKSPACE}/ninja --version - - name: Download CMake 3.10.2 for Linux + - name: Download CMake 3.16.3 for Linux if: matrix.os != 'macos-latest' env: OS_NAME: ${{ matrix.os }} @@ -151,7 +151,7 @@ jobs: sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB sudo sh -c 'echo deb https://apt.repos.intel.com/oneapi all main > /etc/apt/sources.list.d/oneAPI.list' sudo apt-get -qq update - sudo apt-get install -y intel-oneapi-mkl-devel + sudo apt-get install -y intel-oneapi-mkl-devel intel-oneapi-tbb-devel if [ "$CC" == 'icx' ]; then sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp; fi echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> ${GITHUB_ENV} @@ -171,10 +171,10 @@ jobs: branch=$(git rev-parse --abbrev-ref HEAD) buildname=$(if [ -z "$prnum" ]; then echo "$branch"; else echo "PR-$prnum"; fi) dashboard=$(if [ -z "$prnum" ]; then echo "Continuous"; else echo "Experimental"; fi) - backend=$(if [ "$USE_MKL" == 1 ]; then echo "Intel-MKL"; else echo "FFTW/LAPACK/BLAS"; fi) + backend=$(if [ "$USE_MKL" == true ]; then echo "Intel-MKL"; else echo "FFTW/LAPACK/BLAS"; fi) buildname="$buildname-cpu-$BLAS_BACKEND" cmake_rpath=$(if [ $OS_NAME == 'macos-latest' ]; then echo "-DCMAKE_INSTALL_RPATH=/opt/arrayfire/lib"; fi) - if [ "$CC" == 'icx' ]; then source /opt/intel/oneapi/setvars.sh intel64; fi + if [ "$CC" == 'icx' ] || [ "$USE_MKL" == true ]; then source /opt/intel/oneapi/setvars.sh; fi mkdir build && cd build && unset VCPKG_ROOT ${CMAKE_PROGRAM} -G Ninja \ -DCMAKE_MAKE_PROGRAM:FILEPATH=${GITHUB_WORKSPACE}/ninja \ @@ -189,7 +189,8 @@ jobs: - name: Build and Test env: CC: ${{ matrix.compiler }} + USE_MKL: ${{ matrix.blas_backend == 'MKL' }} run: | cd ${GITHUB_WORKSPACE}/build - if [ "$CC" == 'icx' ]; then source /opt/intel/oneapi/setvars.sh intel64; fi + if [ "$CC" == 'icx' ] || [ "$USE_MKL" == true ]; then source /opt/intel/oneapi/setvars.sh; fi ctest -D Experimental --track ${CTEST_DASHBOARD} -T Test -T Submit -R cpu -j2 diff --git a/.github/workflows/win_cpu_build.yml b/.github/workflows/win_cpu_build.yml index 8564bd03b8..d42450f103 100644 --- a/.github/workflows/win_cpu_build.yml +++ b/.github/workflows/win_cpu_build.yml @@ -13,7 +13,7 @@ jobs: name: CPU (fftw, OpenBLAS, windows-latest) runs-on: windows-latest env: - VCPKG_HASH: f14984af3738e69f197bf0e647a8dca12de92996 + VCPKG_HASH: 9d47b24eacbd1cd94f139457ef6cd35e5d92cc84 VCPKG_DEFAULT_TRIPLET: x64-windows steps: - name: Checkout Repository diff --git a/.gitignore b/.gitignore index d56dd8ccf0..933736dba0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ #CMakeCache.txt #./CMakeFiles/ +CMakeUserPresets.json build*/ Release/ #Makefile diff --git a/CMakeLists.txt b/CMakeLists.txt index a4c3eef645..21bc48d39e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,14 +5,17 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.10.2) +if(AF_BUILD_ONEAPI) + cmake_minimum_required(VERSION 3.20) +else() + cmake_minimum_required(VERSION 3.16.3) +endif() include(CheckLanguage) include(CMakeModules/AF_vcpkg_options.cmake) -project(ArrayFire VERSION 3.9.0 LANGUAGES C CXX) - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") +project(ArrayFire VERSION 3.10.0 LANGUAGES C CXX) include(AFconfigure_deps_vars) include(AFBuildConfigurations) @@ -38,17 +41,22 @@ set_policies( CMP0074 CMP0077 CMP0079) +if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.27") + cmake_policy(SET CMP0146 OLD) +endif() arrayfire_set_cmake_default_variables() option(AF_WITH_EXTERNAL_PACKAGES_ONLY "Build ArrayFire with External packages only" OFF) if(AF_WITH_EXTERNAL_PACKAGES_ONLY) set(AF_REQUIRED REQUIRED) endif() - -get_filename_component(CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) -if(CXX_COMPILER_NAME STREQUAL "dpcpp" OR CXX_COMPILER_NAME STREQUAL "dpcpp.exe" - OR CXX_COMPILER_NAME STREQUAL "icpx" OR CXX_COMPILER_NAME STREQUAL "icx.exe") +if(CMAKE_SYCL_COMPILER) + get_filename_component(SYCL_COMPILER_NAME ${CMAKE_SYCL_COMPILER} NAME) +endif() +if(SYCL_COMPILER_NAME STREQUAL "dpcpp" OR SYCL_COMPILER_NAME STREQUAL "dpcpp.exe" + OR SYCL_COMPILER_NAME STREQUAL "icpx" OR SYCL_COMPILER_NAME STREQUAL "icx.exe") set(MKL_THREAD_LAYER "TBB" CACHE STRING "The thread layer to choose for MKL") + set(TBB_ROOT "$ENV{TBBROOT}") set(MKL_INTERFACE "ilp64") set(MKL_INTERFACE_INTEGER_SIZE 8) else() @@ -103,6 +111,7 @@ option(AF_WITH_SPDLOG_HEADER_ONLY "Build ArrayFire with header only version of s option(AF_WITH_FMT_HEADER_ONLY "Build ArrayFire with header only version of fmt" OFF) option(AF_WITH_FAST_MATH "Use lower precision but high performance numeric optimizations" OFF) option(AF_CTEST_SEPARATED "Run tests separately when called from ctest(increases test times)" OFF) +option(AF_SKIP_UNSUPPORTED_TESTS "Skip tests where functions are unsupported by the backend instead of failing" OFF) if(AF_WITH_STATIC_CUDA_NUMERIC_LIBS) option(AF_WITH_PRUNE_STATIC_CUDA_NUMERIC_LIBS "Prune CUDA static libraries to reduce binary size.(WARNING: May break some libs on older CUDA toolkits for some compute arch)" OFF) @@ -134,7 +143,8 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13) if(DEFINED ENV{MKLROOT} AND NOT DEFINED MKL_ROOT) set(MKL_ROOT "$ENV{MKLROOT}") endif() - find_package(MKL 2023.1) + set(SYCL_COMPILER ON) + find_package(MKL) endif() af_multiple_option(NAME AF_COMPUTE_LIBRARY @@ -272,11 +282,15 @@ else() if(AF_WITH_SPDLOG_HEADER_ONLY) set_target_properties(af_spdlog PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "FMT_HEADER_ONLY=1" INTERFACE_LINK_LIBRARIES "spdlog_header_only") else() target_compile_options(spdlog PRIVATE $<$:-fp-model precise>) + install(TARGETS spdlog + COMPONENT common_backend_dependencies + DESTINATION ${AF_INSTALL_BIN_DIR}) set_target_properties(af_spdlog PROPERTIES INTERFACE_LINK_LIBRARIES "spdlog") @@ -302,7 +316,7 @@ if(NOT TARGET nonstd::span-lite) URI https://github.com/martinmoene/span-lite REF "ccf2351" ) - add_subdirectory(${span-lite_SOURCE_DIR} EXCLUDE_FROM_ALL) + add_subdirectory(${span-lite_SOURCE_DIR} ${span-lite_BINARY_DIR} EXCLUDE_FROM_ALL) get_property(span_include_dir TARGET span-lite PROPERTY INTERFACE_INCLUDE_DIRECTORIES) @@ -519,6 +533,14 @@ install(FILES ${ArrayFire_BINARY_DIR}/cmake/install/ArrayFireConfig.cmake DESTINATION ${AF_INSTALL_CMAKE_DIR} COMPONENT cmake) +if(WIN32 AND AF_INSTALL_STANDALONE) + find_program(MSVC_REDIST NAMES vc_redist.x64.exe + PATHS "$ENV{VCINSTALLDIR}Redist\\MSVC\\v${MSVC_TOOLSET_VERSION}") + get_filename_component(MSVC_REDIST_INSTALLER ${MSVC_REDIST} NAME) + install(PROGRAMS ${MSVC_REDIST} COMPONENT common_backend_dependencies + DESTINATION ${AF_INSTALL_BIN_DIR}) +endif() + if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE) if(TARGET MKL::ThreadingLibrary) get_filename_component(mkl_tl ${MKL_ThreadingLibrary_LINK_LIBRARY} REALPATH) @@ -537,8 +559,18 @@ if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE) ${mkl_int} DESTINATION ${AF_INSTALL_LIB_DIR} COMPONENT mkl_dependencies) + + # LP64 library is required for the CPU and OpenCL back ends, so install it too + if(MKL_INTERFACE_INTEGER_SIZE EQUAL 8) + get_filename_component(mkl_int_lp ${MKL_InterfaceLP_LINK_LIBRARY} REALPATH) + install(FILES + ${mkl_int_lp} + DESTINATION ${AF_INSTALL_LIB_DIR} + COMPONENT mkl_dependencies) + endif() endif() + if(UNIX) get_filename_component(mkl_rnt ${MKL_RT_LINK_LIBRARY} REALPATH) get_filename_component(mkl_shd ${MKL_Core_LINK_LIBRARY} REALPATH) get_filename_component(mkl_tly ${MKL_ThreadLayer_LINK_LIBRARY} REALPATH) @@ -546,6 +578,11 @@ if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE) ${mkl_rnt} ${mkl_shd} ${mkl_tly} + DESTINATION ${AF_INSTALL_LIB_DIR} + COMPONENT mkl_dependencies) + endif() + + install(FILES $ $ $ @@ -557,6 +594,38 @@ if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE) ${AF_ADDITIONAL_MKL_LIBRARIES} DESTINATION ${AF_INSTALL_LIB_DIR} COMPONENT mkl_dependencies) + if(AF_BUILD_ONEAPI) + if(WIN32) + get_filename_component(mkl_sycl_lapack ${MKL_SyclLapack_DLL_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_dft ${MKL_SyclDft_DLL_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_blas ${MKL_SyclBlas_DLL_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_sparse ${MKL_SyclSparse_DLL_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_data ${MKL_SyclDataFitting_DLL_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_rng ${MKL_SyclRNG_DLL_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_stats ${MKL_SyclStats_DLL_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_vm ${MKL_SyclVM_DLL_LIBRARY} REALPATH) + else() + get_filename_component(mkl_sycl_lapack ${MKL_SyclLapack_LINK_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_dft ${MKL_SyclDft_LINK_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_blas ${MKL_SyclBlas_LINK_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_sparse ${MKL_SyclSparse_LINK_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_data ${MKL_SyclDataFitting_LINK_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_rng ${MKL_SyclRNG_LINK_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_stats ${MKL_SyclStats_LINK_LIBRARY} REALPATH) + get_filename_component(mkl_sycl_vm ${MKL_SyclVM_LINK_LIBRARY} REALPATH) + endif() + install(FILES + ${mkl_sycl_lapack} + ${mkl_sycl_dft} + ${mkl_sycl_blas} + ${mkl_sycl_sparse} + ${mkl_sycl_data} + ${mkl_sycl_rng} + ${mkl_sycl_stats} + ${mkl_sycl_vm} + DESTINATION ${AF_INSTALL_LIB_DIR} + COMPONENT mkl_dependencies) + endif() endif() endif() diff --git a/CMakeModules/AF_vcpkg_options.cmake b/CMakeModules/AF_vcpkg_options.cmake index 09701af274..c84adcee82 100644 --- a/CMakeModules/AF_vcpkg_options.cmake +++ b/CMakeModules/AF_vcpkg_options.cmake @@ -6,7 +6,6 @@ # http://arrayfire.com/licenses/BSD-3-Clause set(ENV{VCPKG_FEATURE_FLAGS} "versions") -set(ENV{VCPKG_KEEP_ENV_VARS} "MKLROOT") set(VCPKG_MANIFEST_NO_DEFAULT_FEATURES ON) set(VCPKG_OVERLAY_TRIPLETS ${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/vcpkg/vcpkg-triplets) @@ -28,9 +27,7 @@ if(BUILD_TESTING) list(APPEND VCPKG_MANIFEST_FEATURES "tests") endif() -if(AF_COMPUTE_LIBRARY STREQUAL "Intel-MKL") - list(APPEND VCPKG_MANIFEST_FEATURES "mkl") -else() +if(NOT AF_COMPUTE_LIBRARY STREQUAL "Intel-MKL") list(APPEND VCPKG_MANIFEST_FEATURES "openblasfftw") endif() diff --git a/CMakeModules/CMakeCompilerABI.h b/CMakeModules/CMakeCompilerABI.h new file mode 100644 index 0000000000..c5ce4dd9ab --- /dev/null +++ b/CMakeModules/CMakeCompilerABI.h @@ -0,0 +1,45 @@ + +/* Size of a pointer-to-data in bytes. */ +#define SIZEOF_DPTR (sizeof(void*)) +const char info_sizeof_dptr[] = { + /* clang-format off */ + 'I', 'N', 'F', 'O', ':', 's', 'i', 'z', 'e', 'o', 'f', '_', 'd', 'p', 't', + 'r', '[', ('0' + ((SIZEOF_DPTR / 10) % 10)), ('0' + (SIZEOF_DPTR % 10)), ']', + '\0' + /* clang-format on */ +}; + +/* Byte order. Only one of these will have bytes in the right order. */ +static unsigned short const info_byte_order_big_endian[] = { + /* INFO:byte_order string for BIG_ENDIAN */ + 0x494E, 0x464F, 0x3A62, 0x7974, 0x655F, 0x6F72, 0x6465, 0x725B, + 0x4249, 0x475F, 0x454E, 0x4449, 0x414E, 0x5D00, 0x0000 +}; +static unsigned short const info_byte_order_little_endian[] = { + /* INFO:byte_order string for LITTLE_ENDIAN */ + 0x4E49, 0x4F46, 0x623A, 0x7479, 0x5F65, 0x726F, 0x6564, 0x5B72, + 0x494C, 0x5454, 0x454C, 0x455F, 0x444E, 0x4149, 0x5D4E, 0x0000 +}; + +/* Application Binary Interface. */ + +/* Check for (some) ARM ABIs. + * See e.g. http://wiki.debian.org/ArmEabiPort for some information on this. */ +#if defined(__GNU__) && defined(__ELF__) && defined(__ARM_EABI__) +# define ABI_ID "ELF ARMEABI" +#elif defined(__GNU__) && defined(__ELF__) && defined(__ARMEB__) +# define ABI_ID "ELF ARM" +#elif defined(__GNU__) && defined(__ELF__) && defined(__ARMEL__) +# define ABI_ID "ELF ARM" + +#elif defined(__linux__) && defined(__ELF__) && defined(__amd64__) && \ + defined(__ILP32__) +# define ABI_ID "ELF X32" + +#elif defined(__ELF__) +# define ABI_ID "ELF" +#endif + +#if defined(ABI_ID) +static char const info_abi[] = "INFO:abi[" ABI_ID "]"; +#endif diff --git a/CMakeModules/CMakeDetermineSYCLCompiler.cmake b/CMakeModules/CMakeDetermineSYCLCompiler.cmake new file mode 100644 index 0000000000..669e8a79e3 --- /dev/null +++ b/CMakeModules/CMakeDetermineSYCLCompiler.cmake @@ -0,0 +1,239 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + + +# determine the compiler to use for C++ programs +# NOTE, a generator may set CMAKE_SYCL_COMPILER before +# loading this file to force a compiler. +# use environment variable SYCL first if defined by user, next use +# the cmake variable CMAKE_GENERATOR_SYCL which can be defined by a generator +# as a default compiler +# If the internal cmake variable _CMAKE_TOOLCHAIN_PREFIX is set, this is used +# as prefix for the tools (e.g. arm-elf-g++, arm-elf-ar etc.) +# +# Sets the following variables: +# CMAKE_SYCL_COMPILER +# CMAKE_COMPILER_IS_GNUSYCL +# CMAKE_AR +# CMAKE_RANLIB +# +# If not already set before, it also sets +# _CMAKE_TOOLCHAIN_PREFIX + +#list(APPEND CMAKE_MODULE_PATH ${CMAKE_ROOT}) +include(CMakeDetermineCompiler) + +# Load system-specific compiler preferences for this language. +#include(Platform/${CMAKE_SYSTEM_NAME}-Determine-SYCL OPTIONAL) +#include(Platform/${CMAKE_SYSTEM_NAME}-SYCL OPTIONAL) +if(NOT CMAKE_SYCL_COMPILER_NAMES) + set(CMAKE_SYCL_COMPILER_NAMES icpx) +endif() + +if(${CMAKE_GENERATOR} MATCHES "Visual Studio") +elseif("${CMAKE_GENERATOR}" MATCHES "Green Hills MULTI") +elseif("${CMAKE_GENERATOR}" MATCHES "Xcode") + set(CMAKE_SYCL_COMPILER_XCODE_TYPE sourcecode.cpp.cpp) + _cmake_find_compiler_path(SYCL) +else() + if(NOT CMAKE_SYCL_COMPILER) + set(CMAKE_SYCL_COMPILER_INIT NOTFOUND) + + # prefer the environment variable SYCL + if(NOT $ENV{SYCL} STREQUAL "") + get_filename_component(CMAKE_SYCL_COMPILER_INIT $ENV{SYCL} PROGRAM PROGRAM_ARGS CMAKE_SYCL_FLAGS_ENV_INIT) + if(CMAKE_SYCL_FLAGS_ENV_INIT) + set(CMAKE_SYCL_COMPILER_ARG1 "${CMAKE_SYCL_FLAGS_ENV_INIT}" CACHE STRING "Arguments to SYCL compiler") + endif() + if(NOT EXISTS ${CMAKE_SYCL_COMPILER_INIT}) + message(FATAL_ERROR "Could not find compiler set in environment variable SYCL:\n$ENV{SYCL}.\n${CMAKE_SYCL_COMPILER_INIT}") + endif() + endif() + + # next prefer the generator specified compiler + if(CMAKE_GENERATOR_SYCL) + if(NOT CMAKE_SYCL_COMPILER_INIT) + set(CMAKE_SYCL_COMPILER_INIT ${CMAKE_GENERATOR_SYCL}) + endif() + endif() + + # finally list compilers to try + if(NOT CMAKE_SYCL_COMPILER_INIT) + set(CMAKE_SYCL_COMPILER_LIST icpx icx) + if(NOT CMAKE_HOST_WIN32) + # FIXME(#24314): Add support for the GNU-like icpx compiler driver + # on Windows, first introduced by Intel oneAPI 2023.0. + list(APPEND CMAKE_SYCL_COMPILER_LIST icpx) + endif() + endif() + + _cmake_find_compiler(SYCL) + else() + _cmake_find_compiler_path(SYCL) + endif() + mark_as_advanced(CMAKE_SYCL_COMPILER) + + # Each entry in this list is a set of extra flags to try + # adding to the compile line to see if it helps produce + # a valid identification file. + set(CMAKE_SYCL_COMPILER_ID_TEST_FLAGS_FIRST) + set(CMAKE_SYCL_COMPILER_ID_TEST_FLAGS + "-fsycl" + # Try compiling to an object file only. + "-c" + # IAR does not detect language automatically + "--c++" + "--ec++" + + # ARMClang need target options + "--target=arm-arm-none-eabi -mcpu=cortex-m3" + + # MSVC needs at least one include directory for __has_include to function, + # but custom toolchains may run MSVC with no INCLUDE env var and no -I flags. + # Also avoid linking so this works with no LIB env var. + "-c -I__does_not_exist__" + ) +endif() + +if(CMAKE_SYCL_COMPILER_TARGET) + set(CMAKE_SYCL_COMPILER_ID_TEST_FLAGS_FIRST "-c --target=${CMAKE_SYCL_COMPILER_TARGET}") +endif() + +# Build a small source file to identify the compiler. +if(NOT CMAKE_SYCL_COMPILER_ID_RUN) + set(CMAKE_SYCL_COMPILER_ID_RUN 1) + + # Try to identify the compiler. + set(CMAKE_SYCL_COMPILER_ID) + set(CMAKE_SYCL_PLATFORM_ID) + file(READ ${CMAKE_ROOT}/Modules/CMakePlatformId.h.in + CMAKE_SYCL_COMPILER_ID_PLATFORM_CONTENT) + + # The IAR compiler produces weird output. + # See https://gitlab.kitware.com/cmake/cmake/-/issues/10176#note_153591 + list(APPEND CMAKE_SYCL_COMPILER_ID_VENDORS IAR) + set(CMAKE_SYCL_COMPILER_ID_VENDOR_FLAGS_IAR ) + set(CMAKE_SYCL_COMPILER_ID_VENDOR_REGEX_IAR "IAR .+ Compiler") + + # Match the link line from xcodebuild output of the form + # Ld ... + # ... + # /path/to/cc ...CompilerIdSYCL/... + # to extract the compiler front-end for the language. + set(CMAKE_SYCL_COMPILER_ID_TOOL_MATCH_REGEX "\nLd[^\n]*(\n[ \t]+[^\n]*)*\n[ \t]+([^ \t\r\n]+)[^\r\n]*-o[^\r\n]*CompilerIdSYCL/(\\./)?(CompilerIdSYCL.(framework|xctest|build/[^ \t\r\n]+)/)?CompilerIdSYCL[ \t\n\\\"]") + set(CMAKE_SYCL_COMPILER_ID_TOOL_MATCH_INDEX 2) + + include(${CMAKE_ROOT}/Modules/CMakeDetermineCompilerId.cmake) + set(SYCLFLAGS "-fsycl -Werror") + CMAKE_DETERMINE_COMPILER_ID(SYCL SYCLFLAGS CMakeSYCLCompilerId.cpp) + + _cmake_find_compiler_sysroot(SYCL) + + # Set old compiler and platform id variables. + if(CMAKE_SYCL_COMPILER_ID STREQUAL "GNU") + set(CMAKE_COMPILER_IS_GNUSYCL 1) + endif() +else() + if(NOT DEFINED CMAKE_SYCL_COMPILER_FRONTEND_VARIANT) + # Some toolchain files set our internal CMAKE_SYCL_COMPILER_ID_RUN + # variable but are not aware of CMAKE_SYCL_COMPILER_FRONTEND_VARIANT. + # They pre-date our support for the GNU-like variant targeting the + # MSVC ABI so we do not consider that here. + if(CMAKE_SYCL_COMPILER_ID STREQUAL "Clang" + OR "x${CMAKE_SYCL_COMPILER_ID}" STREQUAL "xIntelLLVM") + if("x${CMAKE_SYCL_SIMULATE_ID}" STREQUAL "xMSVC") + set(CMAKE_SYCL_COMPILER_FRONTEND_VARIANT "MSVC") + else() + set(CMAKE_SYCL_COMPILER_FRONTEND_VARIANT "GNU") + endif() + else() + set(CMAKE_SYCL_COMPILER_FRONTEND_VARIANT "") + endif() + endif() +endif() + +if (NOT _CMAKE_TOOLCHAIN_LOCATION) + get_filename_component(_CMAKE_TOOLCHAIN_LOCATION "${CMAKE_SYCL_COMPILER}" PATH) +endif () + +# if we have a g++ cross compiler, they have usually some prefix, like +# e.g. powerpc-linux-g++, arm-elf-g++ or i586-mingw32msvc-g++ , optionally +# with a 3-component version number at the end (e.g. arm-eabi-gcc-4.5.2). +# The other tools of the toolchain usually have the same prefix +# NAME_WE cannot be used since then this test will fail for names like +# "arm-unknown-nto-qnx6.3.0-gcc.exe", where BASENAME would be +# "arm-unknown-nto-qnx6" instead of the correct "arm-unknown-nto-qnx6.3.0-" + + +if (NOT _CMAKE_TOOLCHAIN_PREFIX) + + if("${CMAKE_SYCL_COMPILER_ID}" MATCHES "GNU|Clang|QCC|LCC") + get_filename_component(COMPILER_BASENAME "${CMAKE_SYCL_COMPILER}" NAME) + if (COMPILER_BASENAME MATCHES "^(.+-)?(clang\\+\\+|[gc]\\+\\+|clang-cl)(-[0-9]+(\\.[0-9]+)*)?(-[^.]+)?(\\.exe)?$") + set(_CMAKE_TOOLCHAIN_PREFIX ${CMAKE_MATCH_1}) + set(_CMAKE_TOOLCHAIN_SUFFIX ${CMAKE_MATCH_3}) + set(_CMAKE_COMPILER_SUFFIX ${CMAKE_MATCH_5}) + elseif("${CMAKE_SYCL_COMPILER_ID}" MATCHES "Clang") + if(CMAKE_SYCL_COMPILER_TARGET) + set(_CMAKE_TOOLCHAIN_PREFIX ${CMAKE_SYCL_COMPILER_TARGET}-) + endif() + elseif(COMPILER_BASENAME MATCHES "QCC(\\.exe)?$") + if(CMAKE_SYCL_COMPILER_TARGET MATCHES "gcc_nto([a-z0-9]+_[0-9]+|[^_le]+)(le)") + set(_CMAKE_TOOLCHAIN_PREFIX nto${CMAKE_MATCH_1}-) + endif() + endif () + + # if "llvm-" is part of the prefix, remove it, since llvm doesn't have its own binutils + # but uses the regular ar, objcopy, etc. (instead of llvm-objcopy etc.) + if ("${_CMAKE_TOOLCHAIN_PREFIX}" MATCHES "(.+-)?llvm-$") + set(_CMAKE_TOOLCHAIN_PREFIX ${CMAKE_MATCH_1}) + endif () + elseif("${CMAKE_SYCL_COMPILER_ID}" MATCHES "TI") + # TI compilers are named e.g. cl6x, cl470 or armcl.exe + get_filename_component(COMPILER_BASENAME "${CMAKE_SYCL_COMPILER}" NAME) + if (COMPILER_BASENAME MATCHES "^(.+)?cl([^.]+)?(\\.exe)?$") + set(_CMAKE_TOOLCHAIN_PREFIX "${CMAKE_MATCH_1}") + set(_CMAKE_TOOLCHAIN_SUFFIX "${CMAKE_MATCH_2}") + endif () + + endif() + +endif () + +set(_CMAKE_PROCESSING_LANGUAGE "SYCL") +include(CMakeFindBinUtils) +include(Compiler/${CMAKE_SYCL_COMPILER_ID}-FindBinUtils OPTIONAL) +unset(_CMAKE_PROCESSING_LANGUAGE) + +if(CMAKE_SYCL_COMPILER_SYSROOT) + string(CONCAT _SET_CMAKE_SYCL_COMPILER_SYSROOT + "set(CMAKE_SYCL_COMPILER_SYSROOT \"${CMAKE_SYCL_COMPILER_SYSROOT}\")\n" + "set(CMAKE_COMPILER_SYSROOT \"${CMAKE_SYCL_COMPILER_SYSROOT}\")") +else() + set(_SET_CMAKE_SYCL_COMPILER_SYSROOT "") +endif() + +if(CMAKE_SYCL_COMPILER_ARCHITECTURE_ID) + set(_SET_CMAKE_SYCL_COMPILER_ARCHITECTURE_ID + "set(CMAKE_SYCL_COMPILER_ARCHITECTURE_ID ${CMAKE_SYCL_COMPILER_ARCHITECTURE_ID})") +else() + set(_SET_CMAKE_SYCL_COMPILER_ARCHITECTURE_ID "") +endif() + +if(MSVC_SYCL_ARCHITECTURE_ID) + set(SET_MSVC_SYCL_ARCHITECTURE_ID + "set(MSVC_SYCL_ARCHITECTURE_ID ${MSVC_SYCL_ARCHITECTURE_ID})") +endif() + +if(CMAKE_SYCL_XCODE_ARCHS) + set(SET_CMAKE_XCODE_ARCHS + "set(CMAKE_XCODE_ARCHS \"${CMAKE_SYCL_XCODE_ARCHS}\")") +endif() + +# configure all variables set in this file +configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/CMakeSYCLCompiler.cmake.in + ${CMAKE_PLATFORM_INFO_DIR}/CMakeSYCLCompiler.cmake + @ONLY + ) + +set(CMAKE_SYCL_COMPILER_ENV_VAR "SYCL") diff --git a/CMakeModules/CMakeSYCLCompiler.cmake.in b/CMakeModules/CMakeSYCLCompiler.cmake.in new file mode 100644 index 0000000000..e0193afb13 --- /dev/null +++ b/CMakeModules/CMakeSYCLCompiler.cmake.in @@ -0,0 +1,83 @@ +set(CMAKE_SYCL_COMPILER "@CMAKE_SYCL_COMPILER@") +set(CMAKE_SYCL_COMPILER_ARG1 "@CMAKE_SYCL_COMPILER_ARG1@") +set(CMAKE_SYCL_COMPILER_ID "@CMAKE_SYCL_COMPILER_ID@") +set(CMAKE_SYCL_COMPILER_VERSION "@CMAKE_SYCL_COMPILER_VERSION@") +set(CMAKE_SYCL_COMPILER_VERSION_INTERNAL "@CMAKE_SYCL_COMPILER_VERSION_INTERNAL@") +set(CMAKE_SYCL_COMPILER_WRAPPER "@CMAKE_SYCL_COMPILER_WRAPPER@") +set(CMAKE_SYCL_STANDARD_COMPUTED_DEFAULT "@CMAKE_SYCL_STANDARD_COMPUTED_DEFAULT@") +set(CMAKE_SYCL_EXTENSIONS_COMPUTED_DEFAULT "@CMAKE_SYCL_EXTENSIONS_COMPUTED_DEFAULT@") +set(CMAKE_SYCL_COMPILE_FEATURES "@CMAKE_SYCL_COMPILE_FEATURES@") +set(CMAKE_SYCL98_COMPILE_FEATURES "@CMAKE_SYCL98_COMPILE_FEATURES@") +set(CMAKE_SYCL11_COMPILE_FEATURES "@CMAKE_SYCL11_COMPILE_FEATURES@") +set(CMAKE_SYCL14_COMPILE_FEATURES "@CMAKE_SYCL14_COMPILE_FEATURES@") +set(CMAKE_SYCL17_COMPILE_FEATURES "@CMAKE_SYCL17_COMPILE_FEATURES@") +set(CMAKE_SYCL20_COMPILE_FEATURES "@CMAKE_SYCL20_COMPILE_FEATURES@") +set(CMAKE_SYCL23_COMPILE_FEATURES "@CMAKE_SYCL23_COMPILE_FEATURES@") + +set(CMAKE_SYCL_PLATFORM_ID "@CMAKE_SYCL_PLATFORM_ID@") +set(CMAKE_SYCL_SIMULATE_ID "@CMAKE_SYCL_SIMULATE_ID@") +set(CMAKE_SYCL_COMPILER_FRONTEND_VARIANT "@CMAKE_SYCL_COMPILER_FRONTEND_VARIANT@") +set(CMAKE_SYCL_SIMULATE_VERSION "@CMAKE_SYCL_SIMULATE_VERSION@") +@_SET_CMAKE_SYCL_COMPILER_ARCHITECTURE_ID@ +@_SET_CMAKE_SYCL_COMPILER_SYSROOT@ +@SET_MSVC_SYCL_ARCHITECTURE_ID@ +@SET_CMAKE_XCODE_ARCHS@ +set(CMAKE_AR "@CMAKE_AR@") +set(CMAKE_SYCL_COMPILER_AR "@CMAKE_SYCL_COMPILER_AR@") +set(CMAKE_RANLIB "@CMAKE_RANLIB@") +set(CMAKE_SYCL_COMPILER_RANLIB "@CMAKE_SYCL_COMPILER_RANLIB@") +set(CMAKE_LINKER "@CMAKE_LINKER@") +set(CMAKE_MT "@CMAKE_MT@") +set(CMAKE_COMPILER_IS_GNUSYCL @CMAKE_COMPILER_IS_GNUSYCL@) +set(CMAKE_SYCL_COMPILER_LOADED 1) +set(CMAKE_SYCL_COMPILER_WORKS @CMAKE_SYCL_COMPILER_WORKS@) +set(CMAKE_SYCL_ABI_COMPILED @CMAKE_SYCL_ABI_COMPILED@) + +set(CMAKE_SYCL_COMPILER_ENV_VAR "SYCL") + +set(CMAKE_SYCL_COMPILER_ID_RUN 1) +set(CMAKE_SYCL_SOURCE_FILE_EXTENSIONS C;M;c++;cc;cpp;cxx;m;mm;mpp;CPP;ixx;cppm) +set(CMAKE_SYCL_IGNORE_EXTENSIONS inl;h;hpp;HPP;H;o;O;obj;OBJ;def;DEF;rc;RC) + +foreach (lang SYCL) + if (CMAKE_${lang}_COMPILER_ID_RUN) + foreach(extension IN LISTS CMAKE_${lang}_SOURCE_FILE_EXTENSIONS) + list(REMOVE_ITEM CMAKE_SYCL_SOURCE_FILE_EXTENSIONS ${extension}) + endforeach() + endif() +endforeach() + +set(CMAKE_SYCL_LINKER_PREFERENCE 30) +set(CMAKE_SYCL_LINKER_PREFERENCE_PROPAGATES 1) + +# Save compiler ABI information. +set(CMAKE_SYCL_SIZEOF_DATA_PTR "@CMAKE_SYCL_SIZEOF_DATA_PTR@") +set(CMAKE_SYCL_COMPILER_ABI "@CMAKE_SYCL_COMPILER_ABI@") +set(CMAKE_SYCL_BYTE_ORDER "@CMAKE_SYCL_BYTE_ORDER@") +set(CMAKE_SYCL_LIBRARY_ARCHITECTURE "@CMAKE_SYCL_LIBRARY_ARCHITECTURE@") + +if(CMAKE_SYCL_SIZEOF_DATA_PTR) + set(CMAKE_SIZEOF_VOID_P "${CMAKE_SYCL_SIZEOF_DATA_PTR}") +endif() + +if(CMAKE_SYCL_COMPILER_ABI) + set(CMAKE_INTERNAL_PLATFORM_ABI "${CMAKE_SYCL_COMPILER_ABI}") +endif() + +if(CMAKE_SYCL_LIBRARY_ARCHITECTURE) + set(CMAKE_LIBRARY_ARCHITECTURE "@CMAKE_SYCL_LIBRARY_ARCHITECTURE@") +endif() + +set(CMAKE_SYCL_CL_SHOWINCLUDES_PREFIX "@CMAKE_SYCL_CL_SHOWINCLUDES_PREFIX@") +if(CMAKE_SYCL_CL_SHOWINCLUDES_PREFIX) + set(CMAKE_CL_SHOWINCLUDES_PREFIX "${CMAKE_SYCL_CL_SHOWINCLUDES_PREFIX}") +endif() + +@CMAKE_SYCL_COMPILER_CUSTOM_CODE@ +@CMAKE_SYCL_SYSROOT_FLAG_CODE@ +@CMAKE_SYCL_OSX_DEPLOYMENT_TARGET_FLAG_CODE@ + +set(CMAKE_SYCL_IMPLICIT_INCLUDE_DIRECTORIES "@CMAKE_SYCL_IMPLICIT_INCLUDE_DIRECTORIES@") +set(CMAKE_SYCL_IMPLICIT_LINK_LIBRARIES "@CMAKE_SYCL_IMPLICIT_LINK_LIBRARIES@") +set(CMAKE_SYCL_IMPLICIT_LINK_DIRECTORIES "@CMAKE_SYCL_IMPLICIT_LINK_DIRECTORIES@") +set(CMAKE_SYCL_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES "@CMAKE_SYCL_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES@") diff --git a/CMakeModules/CMakeSYCLCompilerABI.cpp b/CMakeModules/CMakeSYCLCompilerABI.cpp new file mode 100644 index 0000000000..cac613b114 --- /dev/null +++ b/CMakeModules/CMakeSYCLCompilerABI.cpp @@ -0,0 +1,19 @@ +#ifndef __cplusplus +# error "A C compiler has been selected for C++." +#endif + +#include "CMakeCompilerABI.h" + +int main(int argc, char* argv[]) +{ + int require = 0; + require += info_sizeof_dptr[argc]; + require += info_byte_order_big_endian[argc]; + require += info_byte_order_little_endian[argc]; +#if defined(ABI_ID) + require += info_abi[argc]; +#endif + static_cast(argv); + + return require; +} diff --git a/CMakeModules/CMakeSYCLCompilerId.cpp.in b/CMakeModules/CMakeSYCLCompilerId.cpp.in new file mode 100644 index 0000000000..913dbc7932 --- /dev/null +++ b/CMakeModules/CMakeSYCLCompilerId.cpp.in @@ -0,0 +1,105 @@ +/* This source file must have a .cpp extension so that all C++ compilers + recognize the extension without flags. Borland does not know .cxx for + example. */ +#ifndef __cplusplus +# error "A C compiler has been selected for C++." +#endif + +#if !defined(__has_include) +/* If the compiler does not have __has_include, pretend the answer is + always no. */ +# define __has_include(x) 0 +#endif + +@CMAKE_SYCL_COMPILER_ID_CONTENT@ + +/* Construct the string literal in pieces to prevent the source from + getting matched. Store it in a pointer rather than an array + because some compilers will just produce instructions to fill the + array rather than assigning a pointer to a static array. */ +char const* info_compiler = "INFO" ":" "compiler[" COMPILER_ID "]"; +#ifdef SIMULATE_ID +char const* info_simulate = "INFO" ":" "simulate[" SIMULATE_ID "]"; +#endif + +#ifdef __QNXNTO__ +char const* qnxnto = "INFO" ":" "qnxnto[]"; +#endif + +#if defined(__CRAYXT_COMPUTE_LINUX_TARGET) +char const *info_cray = "INFO" ":" "compiler_wrapper[CrayPrgEnv]"; +#endif + +@CMAKE_SYCL_COMPILER_ID_PLATFORM_CONTENT@ +@CMAKE_SYCL_COMPILER_ID_ERROR_FOR_TEST@ + +#if defined(__INTEL_COMPILER) && defined(_MSVC_LANG) && _MSVC_LANG < 201403L +# if defined(__INTEL_CXX11_MODE__) +# if defined(__cpp_aggregate_nsdmi) +# define CXX_STD 201402L +# else +# define CXX_STD 201103L +# endif +# else +# define CXX_STD 199711L +# endif +#elif defined(_MSC_VER) && defined(_MSVC_LANG) +# define CXX_STD _MSVC_LANG +#else +# define CXX_STD __cplusplus +#endif + +const char* info_language_standard_default = "INFO" ":" "standard_default[" +#if CXX_STD > 202002L + "23" +#elif CXX_STD > 201703L + "20" +#elif CXX_STD >= 201703L + "17" +#elif CXX_STD >= 201402L + "14" +#elif CXX_STD >= 201103L + "11" +#else + "98" +#endif +"]"; + +const char* info_language_extensions_default = "INFO" ":" "extensions_default[" +#if (defined(__clang__) || defined(__GNUC__) || defined(__xlC__) || \ + defined(__TI_COMPILER_VERSION__)) && \ + !defined(__STRICT_ANSI__) + "ON" +#else + "OFF" +#endif +"]"; + +/*--------------------------------------------------------------------------*/ + +int main(int argc, char* argv[]) +{ + int require = 0; + require += info_compiler[argc]; + require += info_platform[argc]; + require += info_arch[argc]; +#ifdef COMPILER_VERSION_MAJOR + require += info_version[argc]; +#endif +#ifdef COMPILER_VERSION_INTERNAL + require += info_version_internal[argc]; +#endif +#ifdef SIMULATE_ID + require += info_simulate[argc]; +#endif +#ifdef SIMULATE_VERSION_MAJOR + require += info_simulate_version[argc]; +#endif +#if defined(__CRAYXT_COMPUTE_LINUX_TARGET) + require += info_cray[argc]; +#endif + require += info_language_standard_default[argc]; + require += info_language_extensions_default[argc]; + (void)argv; + return require; +} diff --git a/CMakeModules/CMakeSYCLInformation.cmake b/CMakeModules/CMakeSYCLInformation.cmake new file mode 100644 index 0000000000..b5ec7876db --- /dev/null +++ b/CMakeModules/CMakeSYCLInformation.cmake @@ -0,0 +1,381 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +# make sure default modules are accesible +list(APPEND CMAKE_MODULE_PATH ${CMAKE_ROOT}/Modules) +message(${CMAKE_MODULE_PATH}) + +set(CMAKE_SYCL_COMPILER_ID IntelLLVM) + +# This file sets the basic flags for the C++ language in CMake. +# It also loads the available platform file for the system-compiler +# if it exists. +# It also loads a system - compiler - processor (or target hardware) +# specific file, which is mainly useful for crosscompiling and embedded systems. + +include(CMakeLanguageInformation) + +# some compilers use different extensions (e.g. sdcc uses .rel) +# so set the extension here first so it can be overridden by the compiler specific file +if(UNIX) + set(CMAKE_SYCL_OUTPUT_EXTENSION .o) +else() + set(CMAKE_SYCL_OUTPUT_EXTENSION .obj) +endif() + +set(_INCLUDED_FILE 0) + +# Load compiler-specific information. +if(CMAKE_SYCL_COMPILER_ID) + #include(Compiler/${CMAKE_SYCL_COMPILER_ID}-CXX OPTIONAL) +endif() + +set(CMAKE_BASE_NAME) +get_filename_component(CMAKE_BASE_NAME "${CMAKE_SYCL_COMPILER}" NAME_WE) +# since the gnu compiler has several names force g++ +if(CMAKE_COMPILER_IS_GNUSYCL) + set(CMAKE_BASE_NAME g++) +endif() + +include(Compiler/${CMAKE_SYCL_COMPILER_ID} OPTIONAL) +__compiler_intel_llvm(SYCL) + +if("x${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "xMSVC") + string(APPEND CMAKE_SYCL_FLAGS_INIT " /DWIN32 /D_WINDOWS") + string(APPEND CMAKE_SYCL_FLAGS_DEBUG_INIT " /Zi /Ob0 /Od /RTC1") + string(APPEND CMAKE_SYCL_FLAGS_MINSIZEREL_INIT " /O1 /Ob1 /DNDEBUG") + string(APPEND CMAKE_SYCL_FLAGS_RELEASE_INIT " /O2 /Ob2 /DNDEBUG") + string(APPEND CMAKE_SYCL_FLAGS_RELWITHDEBINFO_INIT " /Zi /O2 /Ob1 /DNDEBUG") + set(CMAKE_SYCL_COMPILE_OPTIONS_EXPLICIT_LANGUAGE -TP) + set(CMAKE_SYCL_CLANG_TIDY_DRIVER_MODE "cl") + set(CMAKE_SYCL_INCLUDE_WHAT_YOU_USE_DRIVER_MODE "cl") + if((NOT DEFINED CMAKE_DEPENDS_USE_COMPILER OR CMAKE_DEPENDS_USE_COMPILER) + AND CMAKE_GENERATOR MATCHES "Makefiles|WMake" + AND CMAKE_DEPFILE_FLAGS_SYCL) + set(CMAKE_SYCL_DEPENDS_USE_COMPILER TRUE) + endif() +else() + set(CMAKE_SYCL_COMPILE_OPTIONS_EXPLICIT_LANGUAGE -x c++) + if((NOT DEFINED CMAKE_DEPENDS_USE_COMPILER OR CMAKE_DEPENDS_USE_COMPILER) + AND CMAKE_GENERATOR MATCHES "Makefiles|WMake" + AND CMAKE_DEPFILE_FLAGS_SYCL) + # dependencies are computed by the compiler itself + set(CMAKE_SYCL_DEPFILE_FORMAT gcc) + set(CMAKE_SYCL_DEPENDS_USE_COMPILER TRUE) + endif() + + set(CMAKE_SYCL_COMPILE_OPTIONS_VISIBILITY_INLINES_HIDDEN "-fvisibility-inlines-hidden") + + string(APPEND CMAKE_SYCL_FLAGS_MINSIZEREL_INIT " -DNDEBUG") + string(APPEND CMAKE_SYCL_FLAGS_RELEASE_INIT " -DNDEBUG") + string(APPEND CMAKE_SYCL_FLAGS_RELWITHDEBINFO_INIT " -DNDEBUG") +endif() + +set(CMAKE_SYCL98_STANDARD__HAS_FULL_SUPPORT ON) +set(CMAKE_SYCL11_STANDARD__HAS_FULL_SUPPORT ON) +set(CMAKE_SYCL14_STANDARD__HAS_FULL_SUPPORT ON) + +if(NOT "x${CMAKE_SYCL_SIMULATE_ID}" STREQUAL "xMSVC") + set(CMAKE_SYCL98_STANDARD_COMPILE_OPTION "-std=c++98") + set(CMAKE_SYCL98_EXTENSION_COMPILE_OPTION "-std=gnu++98") + + set(CMAKE_SYCL11_STANDARD_COMPILE_OPTION "-std=c++11") + set(CMAKE_SYCL11_EXTENSION_COMPILE_OPTION "-std=gnu++11") + + set(CMAKE_SYCL14_STANDARD_COMPILE_OPTION "-std=c++14") + set(CMAKE_SYCL14_EXTENSION_COMPILE_OPTION "-std=gnu++14") + + set(CMAKE_SYCL17_STANDARD_COMPILE_OPTION "-std=c++17") + set(CMAKE_SYCL17_EXTENSION_COMPILE_OPTION "-std=gnu++17") + + set(CMAKE_SYCL20_STANDARD_COMPILE_OPTION "-std=c++20") + set(CMAKE_SYCL20_EXTENSION_COMPILE_OPTION "-std=gnu++20") + + set(CMAKE_SYCL23_STANDARD_COMPILE_OPTION "-std=c++2b") + set(CMAKE_SYCL23_EXTENSION_COMPILE_OPTION "-std=gnu++2b") +else() + set(CMAKE_SYCL98_STANDARD_COMPILE_OPTION "") + set(CMAKE_SYCL98_EXTENSION_COMPILE_OPTION "") + + set(CMAKE_SYCL11_STANDARD_COMPILE_OPTION "") + set(CMAKE_SYCL11_EXTENSION_COMPILE_OPTION "") + + set(CMAKE_SYCL14_STANDARD_COMPILE_OPTION "-Qstd:c++14") + set(CMAKE_SYCL14_EXTENSION_COMPILE_OPTION "-Qstd:c++14") + + set(CMAKE_SYCL17_STANDARD_COMPILE_OPTION "-Qstd:c++17") + set(CMAKE_SYCL17_EXTENSION_COMPILE_OPTION "-Qstd:c++17") + + set(CMAKE_SYCL20_STANDARD_COMPILE_OPTION "-Qstd:c++20") + set(CMAKE_SYCL20_EXTENSION_COMPILE_OPTION "-Qstd:c++20") + + set(CMAKE_SYCL23_STANDARD_COMPILE_OPTION "-Qstd:c++2b") + set(CMAKE_SYCL23_EXTENSION_COMPILE_OPTION "-Qstd:c++2b") +endif() + +include(Platform/${CMAKE_EFFECTIVE_SYSTEM_NAME}-${CMAKE_SYCL_COMPILER_ID} OPTIONAL RESULT_VARIABLE _INCLUDED_FILE) + +if(WIN32) + set(_COMPILE_CXX " /TP") + __windows_compiler_intel(SYCL) +elseif(UNIX AND NOT APPLE) + __linux_compiler_intel_llvm(SYCL) + # This should be -isystem but icpx throws an error on Ubuntu + # when you include /usr/include as a system header + set(CMAKE_INCLUDE_SYSTEM_FLAG_SYCL "-I ") +else() + __apple_compiler_intel_llvm(SYCL) +endif() + +# We specify the compiler information in the system file for some +# platforms, but this language may not have been enabled when the file +# was first included. Include it again to get the language info. +# Remove this when all compiler info is removed from system files. +if (NOT _INCLUDED_FILE) + include(Platform/${CMAKE_SYSTEM_NAME} OPTIONAL) +endif () + +if(CMAKE_SYCL_SIZEOF_DATA_PTR) + foreach(f ${CMAKE_SYCL_ABI_FILES}) + include(${f}) + endforeach() + unset(CMAKE_SYCL_ABI_FILES) +endif() + +# This should be included before the _INIT variables are +# used to initialize the cache. Since the rule variables +# have if blocks on them, users can still define them here. +# But, it should still be after the platform file so changes can +# be made to those values. + +if(CMAKE_USER_MAKE_RULES_OVERRIDE) + # Save the full path of the file so try_compile can use it. + include(${CMAKE_USER_MAKE_RULES_OVERRIDE} RESULT_VARIABLE _override) + set(CMAKE_USER_MAKE_RULES_OVERRIDE "${_override}") +endif() + +if(CMAKE_USER_MAKE_RULES_OVERRIDE_SYCL) + # Save the full path of the file so try_compile can use it. + include(${CMAKE_USER_MAKE_RULES_OVERRIDE_SYCL} RESULT_VARIABLE _override) + set(CMAKE_USER_MAKE_RULES_OVERRIDE_SYCL "${_override}") +endif() + + +# Create a set of shared library variable specific to C++ +# For 90% of the systems, these are the same flags as the C versions +# so if these are not set just copy the flags from the c version +if(NOT CMAKE_SHARED_LIBRARY_CREATE_SYCL_FLAGS) + set(CMAKE_SHARED_LIBRARY_CREATE_SYCL_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS}) +endif() + +if(NOT CMAKE_SYCL_COMPILE_OPTIONS_PIC) + set(CMAKE_SYCL_COMPILE_OPTIONS_PIC ${CMAKE_CXX_COMPILE_OPTIONS_PIC}) +endif() + +if(NOT CMAKE_SYCL_COMPILE_OPTIONS_PIE) + set(CMAKE_SYCL_COMPILE_OPTIONS_PIE ${CMAKE_CXX_COMPILE_OPTIONS_PIE}) +endif() +if(NOT CMAKE_SYCL_LINK_OPTIONS_PIE) + set(CMAKE_SYCL_LINK_OPTIONS_PIE ${CMAKE_CXX_LINK_OPTIONS_PIE}) +endif() +if(NOT CMAKE_SYCL_LINK_OPTIONS_NO_PIE) + set(CMAKE_SYCL_LINK_OPTIONS_NO_PIE ${CMAKE_CXX_LINK_OPTIONS_NO_PIE}) +endif() + +if(NOT CMAKE_SYCL_COMPILE_OPTIONS_DLL) + set(CMAKE_SYCL_COMPILE_OPTIONS_DLL ${CMAKE_CXX_COMPILE_OPTIONS_DLL}) +endif() + +if(NOT CMAKE_SHARED_LIBRARY_SYCL_FLAGS) + set(CMAKE_SHARED_LIBRARY_SYCL_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS}) +endif() + +if(NOT DEFINED CMAKE_SHARED_LIBRARY_LINK_SYCL_FLAGS) + set(CMAKE_SHARED_LIBRARY_LINK_SYCL_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS}) +endif() + +if(NOT CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG) + set(CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG}) +endif() + +if(NOT CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG_SEP) + set(CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP}) +endif() + +if(NOT CMAKE_SHARED_LIBRARY_RPATH_LINK_SYCL_FLAG) + set(CMAKE_SHARED_LIBRARY_RPATH_LINK_SYCL_FLAG ${CMAKE_SHARED_LIBRARY_RPATH_LINK_CXX_FLAG}) +endif() + +if(NOT DEFINED CMAKE_EXE_EXPORTS_SYCL_FLAG) + set(CMAKE_EXE_EXPORTS_SYCL_FLAG ${CMAKE_EXE_EXPORTS_CXX_FLAG}) +endif() + +if(NOT DEFINED CMAKE_SHARED_LIBRARY_SONAME_SYCL_FLAG) + set(CMAKE_SHARED_LIBRARY_SONAME_SYCL_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG}) +endif() + +if(NOT CMAKE_EXECUTABLE_RUNTIME_SYCL_FLAG) + set(CMAKE_EXECUTABLE_RUNTIME_SYCL_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG}) +endif() + +if(NOT CMAKE_EXECUTABLE_RUNTIME_SYCL_FLAG_SEP) + set(CMAKE_EXECUTABLE_RUNTIME_SYCL_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG_SEP}) +endif() + +if(NOT CMAKE_EXECUTABLE_RPATH_LINK_SYCL_FLAG) + set(CMAKE_EXECUTABLE_RPATH_LINK_SYCL_FLAG ${CMAKE_SHARED_LIBRARY_RPATH_LINK_SYCL_FLAG}) +endif() + +if(NOT DEFINED CMAKE_SHARED_LIBRARY_LINK_SYCL_WITH_RUNTIME_PATH) + set(CMAKE_SHARED_LIBRARY_LINK_SYCL_WITH_RUNTIME_PATH ${CMAKE_SHARED_LIBRARY_LINK_CXX_WITH_RUNTIME_PATH}) +endif() + +if(NOT CMAKE_INCLUDE_FLAG_SYCL) + set(CMAKE_INCLUDE_FLAG_SYCL ${CMAKE_INCLUDE_FLAG_C}) +endif() + +# for most systems a module is the same as a shared library +# so unless the variable CMAKE_MODULE_EXISTS is set just +# copy the values from the LIBRARY variables +if(NOT CMAKE_MODULE_EXISTS) + set(CMAKE_SHARED_MODULE_SYCL_FLAGS ${CMAKE_SHARED_LIBRARY_SYCL_FLAGS}) + set(CMAKE_SHARED_MODULE_CREATE_SYCL_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_SYCL_FLAGS}) +endif() + +# repeat for modules +if(NOT CMAKE_SHARED_MODULE_CREATE_SYCL_FLAGS) + set(CMAKE_SHARED_MODULE_CREATE_SYCL_FLAGS ${CMAKE_SHARED_MODULE_CREATE_CXX_FLAGS}) +endif() + +if(NOT CMAKE_SHARED_MODULE_SYCL_FLAGS) + set(CMAKE_SHARED_MODULE_SYCL_FLAGS ${CMAKE_SHARED_MODULE_CXX_FLAGS}) +endif() + +# Initialize SYCL link type selection flags from C versions. +foreach(type SHARED_LIBRARY SHARED_MODULE EXE) + if(NOT CMAKE_${type}_LINK_STATIC_SYCL_FLAGS) + set(CMAKE_${type}_LINK_STATIC_SYCL_FLAGS + ${CMAKE_${type}_LINK_STATIC_CXX_FLAGS}) + endif() + if(NOT CMAKE_${type}_LINK_DYNAMIC_SYCL_FLAGS) + set(CMAKE_${type}_LINK_DYNAMIC_SYCL_FLAGS + ${CMAKE_${type}_LINK_DYNAMIC_CXX_FLAGS}) + endif() +endforeach() + +if(CMAKE_EXECUTABLE_FORMAT STREQUAL "ELF") + if(NOT DEFINED CMAKE_SYCL_LINK_WHAT_YOU_USE_FLAG) + set(CMAKE_SYCL_LINK_WHAT_YOU_USE_FLAG "LINKER:--no-as-needed") + endif() + if(NOT DEFINED CMAKE_LINK_WHAT_YOU_USE_CHECK) + set(CMAKE_LINK_WHAT_YOU_USE_CHECK ldd -u -r) + endif() +endif() + +# add the flags to the cache based +# on the initial values computed in the platform/*.cmake files +# use _INIT variables so that this only happens the first time +# and you can set these flags in the cmake cache +set(CMAKE_SYCL_FLAGS_INIT "-fsycl $ENV{SYCLFLAGS} ${CMAKE_SYCL_FLAGS_INIT}") + +cmake_initialize_per_config_variable(CMAKE_SYCL_FLAGS "Flags used by the SYCL compiler") + +if(CMAKE_SYCL_STANDARD_LIBRARIES_INIT) + set(CMAKE_SYCL_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES_INIT}" + CACHE STRING "Libraries linked by default with all C++ applications.") + mark_as_advanced(CMAKE_SYCL_STANDARD_LIBRARIES) +endif() + +if(NOT CMAKE_SYCL_COMPILER_LAUNCHER AND DEFINED ENV{CMAKE_SYCL_COMPILER_LAUNCHER}) + set(CMAKE_SYCL_COMPILER_LAUNCHER "$ENV{CMAKE_SYCL_COMPILER_LAUNCHER}" + CACHE STRING "Compiler launcher for SYCL.") +endif() + +if(NOT CMAKE_SYCL_LINKER_LAUNCHER AND DEFINED ENV{CMAKE_SYCL_LINKER_LAUNCHER}) + set(CMAKE_SYCL_LINKER_LAUNCHER "$ENV{CMAKE_SYCL_LINKER_LAUNCHER}" + CACHE STRING "Linker launcher for SYCL.") +endif() + +include(CMakeCommonLanguageInclude) + +# now define the following rules: +# CMAKE_SYCL_CREATE_SHARED_LIBRARY +# CMAKE_SYCL_CREATE_SHARED_MODULE +# CMAKE_SYCL_COMPILE_OBJECT +# CMAKE_SYCL_LINK_EXECUTABLE + +# variables supplied by the generator at use time +# +# the target without the suffix +# +# +# +# +# + +# SYCL compiler information +# +# +# +# + +# Static library tools +# +# + +# create a shared C++ library +if(NOT CMAKE_SYCL_CREATE_SHARED_LIBRARY) + set(CMAKE_SYCL_CREATE_SHARED_LIBRARY + " -o ") +endif() + +# create a c++ shared module copy the shared library rule by default +if(NOT CMAKE_SYCL_CREATE_SHARED_MODULE) + set(CMAKE_SYCL_CREATE_SHARED_MODULE ${CMAKE_SYCL_CREATE_SHARED_LIBRARY}) +endif() + + +# Create a static archive incrementally for large object file counts. +# If CMAKE_SYCL_CREATE_STATIC_LIBRARY is set it will override these. +if(NOT DEFINED CMAKE_SYCL_ARCHIVE_CREATE) + set(CMAKE_SYCL_ARCHIVE_CREATE " qc ") +endif() +if(NOT DEFINED CMAKE_SYCL_ARCHIVE_APPEND) + set(CMAKE_SYCL_ARCHIVE_APPEND " q ") +endif() +if(NOT DEFINED CMAKE_SYCL_ARCHIVE_FINISH) + set(CMAKE_SYCL_ARCHIVE_FINISH " ") +endif() + +# compile a C++ file into an object file +if(NOT CMAKE_SYCL_COMPILE_OBJECT) + set(CMAKE_SYCL_COMPILE_OBJECT + " -o -c ") +endif() + +if(NOT CMAKE_SYCL_LINK_EXECUTABLE) + set(CMAKE_SYCL_LINK_EXECUTABLE + " -o ") +endif() + +if(CMAKE_HOST_WIN32) + set(MSVC_RUNTIME "") + if("${CMAKE_MSVC_RUNTIME_LIBRARY}" STREQUAL "MultiThreaded") + set(MSVC_RUNTIME "-MT") + elseif("${CMAKE_MSVC_RUNTIME_LIBRARY}" STREQUAL "MultiThreadedDLL") + set(MSVC_RUNTIME "-MD") + elseif("${CMAKE_MSVC_RUNTIME_LIBRARY}" STREQUAL "MultiThreadedDebug") + set(MSVC_RUNTIME "-MTd") + elseif("${CMAKE_MSVC_RUNTIME_LIBRARY}" STREQUAL "MultiThreadedDebugDLL") + set(MSVC_RUNTIME "-MDd") + else() + set(MSVC_RUNTIME "-MD$<$:d>") + endif() + set(CMAKE_MSVC_RUNTIME_LIBRARY "") +endif() + +mark_as_advanced( +CMAKE_VERBOSE_MAKEFILE +) + +set(CMAKE_SYCL_INFORMATION_LOADED 1) diff --git a/CMakeModules/CMakeTestSYCLCompiler.cmake b/CMakeModules/CMakeTestSYCLCompiler.cmake new file mode 100644 index 0000000000..ef38081b37 --- /dev/null +++ b/CMakeModules/CMakeTestSYCLCompiler.cmake @@ -0,0 +1,95 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + + +if(CMAKE_SYCL_COMPILER_FORCED) + # The compiler configuration was forced by the user. + # Assume the user has configured all compiler information. + set(CMAKE_SYCL_COMPILER_WORKS TRUE) + return() +endif() + +include(CMakeTestCompilerCommon) + +# work around enforced code signing and / or missing executable target type +set(__CMAKE_SAVED_TRY_COMPILE_TARGET_TYPE ${CMAKE_TRY_COMPILE_TARGET_TYPE}) +if(_CMAKE_FEATURE_DETECTION_TARGET_TYPE) + set(CMAKE_TRY_COMPILE_TARGET_TYPE ${_CMAKE_FEATURE_DETECTION_TARGET_TYPE}) +endif() + +# Remove any cached result from an older CMake version. +# We now store this in CMakeSYCLCompiler.cmake. +unset(CMAKE_SYCL_COMPILER_WORKS CACHE) + +# Try to identify the ABI and configure it into CMakeSYCLCompiler.cmake +include(CMakeDetermineCompilerABI) +CMAKE_DETERMINE_COMPILER_ABI(SYCL ${ArrayFire_SOURCE_DIR}/CMakeModules/CMakeSYCLCompilerABI.cpp) +if(CMAKE_SYCL_ABI_COMPILED) + # The compiler worked so skip dedicated test below. + set(CMAKE_SYCL_COMPILER_WORKS TRUE) + message(STATUS "Check for working SYCL compiler: ${CMAKE_SYCL_COMPILER} - skipped") +endif() + +# This file is used by EnableLanguage in cmGlobalGenerator to +# determine that the selected C++ compiler can actually compile +# and link the most basic of programs. If not, a fatal error +# is set and cmake stops processing commands and will not generate +# any makefiles or projects. +if(NOT CMAKE_SYCL_COMPILER_WORKS) + PrintTestCompilerStatus("SYCL") + __TestCompiler_setTryCompileTargetType() + file(WRITE ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testSYCLCompiler.cxx + "#ifndef __cplusplus\n" + "# error \"The CMAKE_SYCL_COMPILER is set to a C compiler\"\n" + "#endif\n" + "int main(){return 0;}\n") + # Clear result from normal variable. + unset(CMAKE_SYCL_COMPILER_WORKS) + # Puts test result in cache variable. + try_compile(CMAKE_SYCL_COMPILER_WORKS ${CMAKE_BINARY_DIR} + ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testSYCLCompiler.cxx + OUTPUT_VARIABLE __CMAKE_SYCL_COMPILER_OUTPUT) + unset(__TestCompiler_testSYCLCompilerSource) + # Move result from cache to normal variable. + set(CMAKE_SYCL_COMPILER_WORKS ${CMAKE_SYCL_COMPILER_WORKS}) + unset(CMAKE_SYCL_COMPILER_WORKS CACHE) + __TestCompiler_restoreTryCompileTargetType() + if(NOT CMAKE_SYCL_COMPILER_WORKS) + PrintTestCompilerResult(CHECK_FAIL "broken") + string(REPLACE "\n" "\n " _output "${__CMAKE_SYCL_COMPILER_OUTPUT}") + message(FATAL_ERROR "The C++ compiler\n \"${CMAKE_SYCL_COMPILER}\"\n" + "is not able to compile a simple test program.\nIt fails " + "with the following output:\n ${_output}\n\n" + "CMake will not be able to correctly generate this project.") + endif() + PrintTestCompilerResult(CHECK_PASS "works") +endif() + +# Try to identify the compiler features +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.30.0) + include(CMakeDetermineCompilerSupport) + CMAKE_DETERMINE_COMPILER_SUPPORT(CXX) +else() + include(CMakeDetermineCompileFeatures) + CMAKE_DETERMINE_COMPILE_FEATURES(CXX) +endif() + +set(CMAKE_TRY_COMPILE_CONFIGURATION "") +# Re-configure to save learned information. +configure_file( + ${ArrayFire_SOURCE_DIR}/CMakeModules/CMakeSYCLCompiler.cmake.in + ${CMAKE_PLATFORM_INFO_DIR}/CMakeSYCLCompiler.cmake + @ONLY +) +include(${CMAKE_PLATFORM_INFO_DIR}/CMakeSYCLCompiler.cmake) + +if(CMAKE_SYCL_SIZEOF_DATA_PTR) + foreach(f ${CMAKE_SYCL_ABI_FILES}) + include(${f}) + endforeach() + unset(CMAKE_SYCL_ABI_FILES) +endif() + +set(CMAKE_TRY_COMPILE_TARGET_TYPE ${__CMAKE_SAVED_TRY_COMPILE_TARGET_TYPE}) +unset(__CMAKE_SAVED_TRY_COMPILE_TARGET_TYPE) +unset(__CMAKE_SYCL_COMPILER_OUTPUT) diff --git a/CMakeModules/CPackConfig.cmake b/CMakeModules/CPackConfig.cmake index 6cd13a1d71..8cf0880faa 100644 --- a/CMakeModules/CPackConfig.cmake +++ b/CMakeModules/CPackConfig.cmake @@ -43,9 +43,9 @@ set(CPACK_PACKAGE_NAME "${LIBRARY_NAME}") set(CPACK_PACKAGE_VENDOR "${VENDOR_NAME}") set(CPACK_PACKAGE_INSTALL_REGISTRY_KEY ${LIBRARY_NAME}) set(CPACK_PACKAGE_CONTACT "ArrayFire ") -set(MY_CPACK_PACKAGE_ICON "${CMAKE_SOURCE_DIR}/assets/${APP_LOW_NAME}.ico") +set(MY_CPACK_PACKAGE_ICON "${ASSETS_DIR}/${APP_LOW_NAME}.ico") -file(TO_NATIVE_PATH "${CMAKE_SOURCE_DIR}/assets/" NATIVE_ASSETS_PATH) +file(TO_NATIVE_PATH "${ASSETS_DIR}/" NATIVE_ASSETS_PATH) string(REPLACE "\\" "\\\\" NATIVE_ASSETS_PATH ${NATIVE_ASSETS_PATH}) set(CPACK_AF_ASSETS_DIR "${NATIVE_ASSETS_PATH}") @@ -137,6 +137,9 @@ elseif(WIN32) else (CMAKE_CL_64) set(CPACK_NSIS_INSTALL_ROOT "$PROGRAMFILES") endif (CMAKE_CL_64) + configure_file( + ${PROJECT_SOURCE_DIR}/CMakeModules/nsis/NSIS.definitions.nsh.in + ${CMAKE_CURRENT_BINARY_DIR}/NSIS.definitions.nsh) else() set(CPACK_RESOURCE_FILE_LICENSE "${ArrayFire_SOURCE_DIR}/LICENSE") set(CPACK_RESOURCE_FILE_README "${ArrayFire_SOURCE_DIR}/README.md") diff --git a/CMakeModules/CPackProjectConfig.cmake b/CMakeModules/CPackProjectConfig.cmake index 6cd6e20088..f75591f8bb 100644 --- a/CMakeModules/CPackProjectConfig.cmake +++ b/CMakeModules/CPackProjectConfig.cmake @@ -161,9 +161,12 @@ if(NOT CPACK_GENERATOR MATCHES "DEB") DESCRIPTION "ArrayFire development files including headers and configuration files" EXPANDED) - cpack_add_component_group(debug - DISPLAY_NAME "ArrayFire Debug Symbols" - DESCRIPTION "ArrayFire Debug symbols") + if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR + CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + cpack_add_component_group(debug + DISPLAY_NAME "ArrayFire Debug Symbols" + DESCRIPTION "ArrayFire Debug symbols") + endif() endif() set(arrayfire_cuda_runtime_name "CUDA Runtime(${CPACK_CUDA_VERSION_MAJOR}.${CPACK_CUDA_VERSION_MINOR})") @@ -287,7 +290,7 @@ af_component( DEB_USE_SHLIBDEPS DEB_PROVIDES "arrayfire-cuda (= ${CPACK_PACKAGE_VERSION}), arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION}), libarrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION})" DEB_REPLACES "arrayfire-cuda (<< ${CPACK_PACKAGE_VERSION}), arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION})" - DEB_OPTIONAL libcudnn8 forge libfreeimage3 + DEB_OPTIONAL cudnn9-cuda-${CPACK_CUDA_VERSION_MAJOR}-${CPACK_CUDA_VERSION_MINOR} forge libfreeimage3 ) af_component( @@ -343,6 +346,42 @@ af_component( DEB_OPTIONAL "cmake (>= 3.0)" ) +af_component( + COMPONENT oneapi + DISPLAY_NAME "oneAPI Runtime" + SUMMARY "ArrayFire oneAPI backend shared libraries" + DESCRIPTION "ArrayFire oneAPI backend shared libraries" + REQUIRES ${oneapi_deps_comps} licenses + OPTIONAL forge + GROUP afruntime + INSTALL_TYPES All Runtime + + DEB_PACKAGE_NAME ${deb_oneapi_runtime_package_name} + DEB_PROVIDES "arrayfire-oneapi (= ${CPACK_PACKAGE_VERSION}), arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION}), libarrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION})" + DEB_REPLACES "arrayfire-oneapi (<< ${CPACK_PACKAGE_VERSION}), arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION})" + DEB_REQUIRES ${deb_oneapi_runtime_requirements} + DEB_USE_SHLIBDEPS + DEB_ADD_POSTINST + DEB_OPTIONAL forge libfreeimage3 +) + +af_component( + COMPONENT oneapi_dev + DISPLAY_NAME "oneAPI Dev" + SUMMARY "ArrayFire oneAPI backend development files" + DESCRIPTION "ArrayFire oneAPI backend development files" + REQUIRES oneapi headers cmake + GROUP afdevelopment + INSTALL_TYPES All Development + + DEB_PACKAGE_NAME arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR}-dev + DEB_PROVIDES "arrayfire-oneapi-dev (= ${CPACK_PACKAGE_VERSION}), arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR}-dev (= ${CPACK_PACKAGE_VERSION}), libarrayfire-oneapi-dev (= ${CPACK_PACKAGE_VERSION})" + DEB_REPLACES "arrayfire-oneapi-dev (<< ${CPACK_PACKAGE_VERSION}), arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR}-dev (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-oneapi-dev (<< ${CPACK_PACKAGE_VERSION})" + DEB_REQUIRES "arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR} (>= ${CPACK_PACKAGE_VERSION}), arrayfire-headers (>= ${CPACK_PACKAGE_VERSION})" + DEB_RECOMMENDS "arrayfire-cmake (>= ${CPACK_PACKAGE_VERSION})" + DEB_OPTIONAL "cmake (>= 3.0)" +) + af_component( COMPONENT unified DISPLAY_NAME "Unified Runtime" @@ -437,37 +476,48 @@ endif() # Debug symbols in debian installers are created using the DEBINFO property if(NOT APPLE AND NOT CPACK_GENERATOR MATCHES "DEB") - af_component( - COMPONENT afopencl_debug_symbols - DISPLAY_NAME "OpenCL Debug Symbols" - DESCRIPTION "Debug symbols for the OpenCL backend." - GROUP debug - DISABLED - INSTALL_TYPES Development) - - af_component( - COMPONENT afcuda_debug_symbols - DISPLAY_NAME "CUDA Debug Symbols" - DESCRIPTION "Debug symbols for CUDA backend backend." - GROUP debug - DISABLED - INSTALL_TYPES Development) - - af_component( - COMPONENT afcpu_debug_symbols - DISPLAY_NAME "CPU Debug Symbols" - DESCRIPTION "Debug symbols for CPU backend backend." - GROUP debug - DISABLED - INSTALL_TYPES Development) - - af_component( - COMPONENT af_debug_symbols - DISPLAY_NAME "Unified Debug Symbols" - DESCRIPTION "Debug symbols for the Unified backend." - GROUP debug - DISABLED - INSTALL_TYPES Development) + if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR + CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + af_component( + COMPONENT afoneapi_debug_symbols + DISPLAY_NAME "oneAPI Debug Symbols" + DESCRIPTION "Debug symbols for the oneAPI backend." + GROUP debug + DISABLED + INSTALL_TYPES Development) + + af_component( + COMPONENT afopencl_debug_symbols + DISPLAY_NAME "OpenCL Debug Symbols" + DESCRIPTION "Debug symbols for the OpenCL backend." + GROUP debug + DISABLED + INSTALL_TYPES Development) + + af_component( + COMPONENT afcuda_debug_symbols + DISPLAY_NAME "CUDA Debug Symbols" + DESCRIPTION "Debug symbols for CUDA backend backend." + GROUP debug + DISABLED + INSTALL_TYPES Development) + + af_component( + COMPONENT afcpu_debug_symbols + DISPLAY_NAME "CPU Debug Symbols" + DESCRIPTION "Debug symbols for CPU backend backend." + GROUP debug + DISABLED + INSTALL_TYPES Development) + + af_component( + COMPONENT af_debug_symbols + DISPLAY_NAME "Unified Debug Symbols" + DESCRIPTION "Debug symbols for the Unified backend." + GROUP debug + DISABLED + INSTALL_TYPES Development) + endif() endif() # if (AF_INSTALL_FORGE_DEV) diff --git a/CMakeModules/FindAF_MKL.cmake b/CMakeModules/FindAF_MKL.cmake index 662f0046da..2da1ed4584 100644 --- a/CMakeModules/FindAF_MKL.cmake +++ b/CMakeModules/FindAF_MKL.cmake @@ -74,8 +74,12 @@ include(CheckTypeSize) include(FindPackageHandleStandardArgs) -check_type_size("int" INT_SIZE - BUILTIN_TYPES_ONLY LANGUAGE C) +if(DEFINED MKL_INTERFACE_INTEGER_SIZE) + set(INT_SIZE ${MKL_INTERFACE_INTEGER_SIZE}) +else() + check_type_size("int" INT_SIZE + BUILTIN_TYPES_ONLY LANGUAGE C) +endif() set(MKL_THREAD_LAYER "TBB" CACHE STRING "The thread layer to choose for MKL") set_property(CACHE MKL_THREAD_LAYER PROPERTY STRINGS "TBB" "GNU OpenMP" "Intel OpenMP" "Sequential") @@ -221,7 +225,7 @@ function(find_mkl_library) add_library(MKL::${mkl_args_NAME}_STATIC STATIC IMPORTED) if(NOT (WIN32 AND mkl_args_DLL_ONLY)) - list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".so.1") + list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".so.1;.so.2;.so.3;.so.4;.so.12") find_library(MKL_${mkl_args_NAME}_LINK_LIBRARY NAMES ${mkl_args_LIBRARY_NAME}${shared_suffix} @@ -299,9 +303,16 @@ function(find_mkl_library) NAMES ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}${md_suffix}${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}.2${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}.5${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}12${CMAKE_SHARED_LIBRARY_SUFFIX} lib${mkl_args_LIBRARY_NAME}${md_suffix}${CMAKE_SHARED_LIBRARY_SUFFIX} $ENV{LIB} $ENV{LIBRARY_PATH} + PATHS + $ENV{MKLROOT}/bin + $ENV{TBBROOT}/bin + $ENV{ONEAPI_ROOT}/compiler/latest/bin PATH_SUFFIXES IntelSWTools/compilers_and_libraries/windows/redist/intel64/mkl IntelSWTools/compilers_and_libraries/windows/redist/intel64/compiler @@ -321,6 +332,18 @@ endfunction() find_mkl_library(NAME Core LIBRARY_NAME mkl_core SEARCH_STATIC) find_mkl_library(NAME RT LIBRARY_NAME mkl_rt) +if(AF_BUILD_ONEAPI) + find_mkl_library(NAME Sycl LIBRARY_NAME sycl DLL_ONLY) + find_mkl_library(NAME SyclLapack LIBRARY_NAME mkl_sycl_lapack DLL_ONLY) + find_mkl_library(NAME SyclDft LIBRARY_NAME mkl_sycl_dft DLL_ONLY) + find_mkl_library(NAME SyclBlas LIBRARY_NAME mkl_sycl_blas DLL_ONLY) + find_mkl_library(NAME SyclSparse LIBRARY_NAME mkl_sycl_sparse DLL_ONLY) + find_mkl_library(NAME SyclDataFitting LIBRARY_NAME mkl_sycl_data_fitting DLL_ONLY) + find_mkl_library(NAME SyclRNG LIBRARY_NAME mkl_sycl_rng DLL_ONLY) + find_mkl_library(NAME SyclStats LIBRARY_NAME mkl_sycl_stats DLL_ONLY) + find_mkl_library(NAME SyclVM LIBRARY_NAME mkl_sycl_vm DLL_ONLY) +endif() + # MKL can link against Intel OpenMP, GNU OpenMP, TBB, and Sequential if(MKL_THREAD_LAYER STREQUAL "Intel OpenMP") find_mkl_library(NAME ThreadLayer LIBRARY_NAME mkl_intel_thread SEARCH_STATIC) @@ -348,10 +371,13 @@ endif() if("${INT_SIZE}" EQUAL 4) set(MKL_INTERFACE_INTEGER_SIZE 4) + set(MKL_INTERFACE "lp64") find_mkl_library(NAME Interface LIBRARY_NAME mkl_intel_lp64 SEARCH_STATIC) else() set(MKL_INTERFACE_INTEGER_SIZE 8) + set(MKL_INTERFACE "ilp64") find_mkl_library(NAME Interface LIBRARY_NAME mkl_intel_ilp64 SEARCH_STATIC) + find_mkl_library(NAME InterfaceLP LIBRARY_NAME mkl_intel_lp64 SEARCH_STATIC) endif() set(MKL_KernelLibraries "mkl_def;mkl_mc;mkl_mc3;mkl_avx;mkl_avx2;mkl_avx512") diff --git a/CMakeModules/FindFreeImage.cmake b/CMakeModules/FindFreeImage.cmake index b049ec06a3..3b2d3fca29 100644 --- a/CMakeModules/FindFreeImage.cmake +++ b/CMakeModules/FindFreeImage.cmake @@ -75,12 +75,14 @@ find_library(FreeImage_STATIC_LIBRARY DOC "The FreeImage static library") if (WIN32) + get_filename_component(FreeImage_LIB_PATH ${FreeImage_LINK_LIBRARY} DIRECTORY) find_file(FreeImage_DLL_LIBRARY NAMES ${CMAKE_SHARED_LIBRARY_PREFIX}FreeImage${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_SHARED_LIBRARY_PREFIX}freeimage${CMAKE_SHARED_LIBRARY_SUFFIX} PATHS ${FreeImage_ROOT} + ${FreeImage_LIB_PATH}/../bin DOC "The FreeImage dll") mark_as_advanced(FreeImage_DLL_LIBRARY) endif () diff --git a/CMakeModules/FindcuDNN.cmake b/CMakeModules/FindcuDNN.cmake index 4c28d3c854..98641f4198 100644 --- a/CMakeModules/FindcuDNN.cmake +++ b/CMakeModules/FindcuDNN.cmake @@ -169,13 +169,17 @@ if(cuDNN_INCLUDE_DIRS) endmacro() af_find_cudnn_libs("") # gets base cudnn shared library - if(cuDNN_VERSION_MAJOR VERSION_GREATER 8 OR cuDNN_VERSION_MAJOR VERSION_EQUAL 8) + if(cuDNN_VERSION_MAJOR VERSION_EQUAL 8) af_find_cudnn_libs("_adv_infer") af_find_cudnn_libs("_adv_train") af_find_cudnn_libs("_cnn_infer") af_find_cudnn_libs("_cnn_train") af_find_cudnn_libs("_ops_infer") af_find_cudnn_libs("_ops_train") + elseif(cuDNN_VERSION_MAJOR VERSION_GREATER_EQUAL 9) + af_find_cudnn_libs("_adv") + af_find_cudnn_libs("_cnn") + af_find_cudnn_libs("_ops") endif() endif() diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake index 863cbaed22..8d29718365 100644 --- a/CMakeModules/InternalUtils.cmake +++ b/CMakeModules/InternalUtils.cmake @@ -39,6 +39,37 @@ check_cxx_compiler_flag(-Rno-debug-disables-optimization has_cxx_debug-disables- function(arrayfire_set_default_cxx_flags target) target_compile_options(${target} PRIVATE + + $<$: + $<$: + # OpenCL targets need this flag to avoid + # ignored attribute warnings in the OpenCL + # headers + -Wno-ignored-attributes + -Wall + -Wno-unqualified-std-cast-call + -Werror=reorder-ctor + #-fp-model precise + $<$: -ffast-math -fno-errno-math -fno-trapping-math -fno-signed-zeros -mno-ieee-fp> + $<$>: $,/fp=precise,-fp-model=precise>> + $<$:-Rno-debug-disables-optimization> + + $<$: /wd4251 + /wd4068 + /wd4275 + /wd4668 + /wd4710 + /wd4505 + /we5038 + /bigobj + /EHsc + /nologo + # MSVC incorrectly sets the cplusplus to 199711L even if the compiler supports + # c++11 features. This flag sets it to the correct standard supported by the + # compiler + $<$:/Zc:__cplusplus> + $<$:/permissive-> > + >> $<$: # C4068: Warnings about unknown pragmas # C4668: Warnings about unknown defintions @@ -53,6 +84,7 @@ function(arrayfire_set_default_cxx_flags target) /we5038 /bigobj /EHsc + /nologo # MSVC incorrectly sets the cplusplus to 199711L even if the compiler supports # c++11 features. This flag sets it to the correct standard supported by the # compiler diff --git a/CMakeModules/build_CLBlast.cmake b/CMakeModules/build_CLBlast.cmake index 0f67d3fdee..7ea0b43256 100644 --- a/CMakeModules/build_CLBlast.cmake +++ b/CMakeModules/build_CLBlast.cmake @@ -24,6 +24,7 @@ if(TARGET clblast OR AF_WITH_EXTERNAL_PACKAGES_ONLY) message(ERROR "CLBlast now found") endif() else() + # This specific reference passes tests af_dep_check_and_populate(${clblast_prefix} URI https://github.com/cnugteren/CLBlast.git REF 4500a03440e2cc54998c0edab366babf5e504d67 @@ -69,12 +70,14 @@ else() BUILD_BYPRODUCTS ${CLBlast_location} CONFIGURE_COMMAND ${CMAKE_COMMAND} ${extproj_gen_opts} -Wno-dev + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER} "-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}" -DOVERRIDE_MSVC_FLAGS_TO_MT:BOOL=OFF -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER} "-DCMAKE_C_FLAGS:STRING=${CMAKE_C_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DOPENCL_LIBRARIES="${OPENCL_LIBRARIES}" ${extproj_build_type_option} -DCMAKE_INSTALL_PREFIX:PATH= -DCMAKE_INSTALL_LIBDIR:PATH=lib diff --git a/CMakeModules/build_cl2hpp.cmake b/CMakeModules/build_cl2hpp.cmake index 0a3fef2de0..b38c4bc1d1 100644 --- a/CMakeModules/build_cl2hpp.cmake +++ b/CMakeModules/build_cl2hpp.cmake @@ -27,7 +27,7 @@ if(NOT TARGET OpenCL::cl2hpp) elseif (NOT TARGET OpenCL::cl2hpp OR NOT TARGET cl2hpp) af_dep_check_and_populate(${cl2hpp_prefix} URI https://github.com/KhronosGroup/OpenCL-CLHPP.git - REF v2022.09.30) + REF v2024.10.24) find_path(cl2hpp_var NAMES CL/cl2.hpp diff --git a/CMakeModules/nsis/NSIS.InstallOptions.ini.in b/CMakeModules/nsis/NSIS.InstallOptions.ini.in index d92d77959c..cc17d8268a 100644 --- a/CMakeModules/nsis/NSIS.InstallOptions.ini.in +++ b/CMakeModules/nsis/NSIS.InstallOptions.ini.in @@ -3,7 +3,7 @@ NumFields=5 [Field 1] Type=label -Text=By default @CPACK_PACKAGE_INSTALL_DIRECTORY@ does not add its directory to the system PATH. +Text=By default @CPACK_PACKAGE_INSTALL_DIRECTORY@ will add its directory to the system PATH. This will make the dynamic libraries available to all users and software on the system. Left=0 Right=-1 Top=0 @@ -16,7 +16,7 @@ Left=0 Right=-1 Top=30 Bottom=40 -State=1 +State=0 [Field 3] Type=radiobutton @@ -25,7 +25,7 @@ Left=0 Right=-1 Top=40 Bottom=50 -State=0 +State=1 [Field 4] Type=radiobutton diff --git a/CMakeModules/nsis/NSIS.definitions.nsh.in b/CMakeModules/nsis/NSIS.definitions.nsh.in index 4c6e8998b7..1062271940 100644 --- a/CMakeModules/nsis/NSIS.definitions.nsh.in +++ b/CMakeModules/nsis/NSIS.definitions.nsh.in @@ -3,23 +3,23 @@ !define MUI_WELCOMEPAGE_TEXT \ "ArrayFire is a high performance software library for parallel computing with an easy-to-use API.\r\n\r\n\ Its array based function set makes parallel programming simple.\r\n\r\n\ -ArrayFire's multiple backends (CUDA, OpenCL and native CPU) make it platform independent and highly portable.\r\n\r\n\ +ArrayFire's multiple backends (CUDA, OneAPI, OpenCL, and native CPU) make it platform independent and highly portable.\r\n\r\n\ A few lines of code in ArrayFire can replace dozens of lines of parallel compute code, \ saving you valuable time and lowering development costs.\r\n\r\n\ Follow these steps to install the ArrayFire libraries." -!define MUI_ICON "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@.ico" -!define MUI_UNICON "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@.ico" +!define MUI_ICON "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@.ico" +!define MUI_UNICON "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@.ico" -!define MUI_WELCOMEFINISHPAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@_sym.bmp" -!define MUI_UNWELCOMEFINISHPAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@_sym.bmp" +!define MUI_WELCOMEFINISHPAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@_sym.bmp" +!define MUI_UNWELCOMEFINISHPAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@_sym.bmp" !define MUI_WELCOMEFINISHPAGE_UNBITMAP_NOSTRETCH !define MUI_UNWELCOMEFINISHPAGE_BITMAP_NOSTRETCH !define MUI_HEADERIMAGE !define MUI_HEADERIMAGE_RIGHT -!define MUI_HEADERIMAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@_logo.bmp" -!define MUI_HEADERIMAGE_UNBITMAP "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@_logo.bmp" +!define MUI_HEADERIMAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@_logo.bmp" +!define MUI_HEADERIMAGE_UNBITMAP "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@_logo.bmp" !define MUI_HEADERIMAGE_BITMAP_NOSTRETCH !define MUI_HEADERIMAGE_UNBITMAP_NOSTRETCH !define MUI_ABORTWARNING diff --git a/CMakeModules/nsis/NSIS.template.in b/CMakeModules/nsis/NSIS.template.in index bc3a44f233..c46274518c 100644 --- a/CMakeModules/nsis/NSIS.template.in +++ b/CMakeModules/nsis/NSIS.template.in @@ -714,7 +714,7 @@ Section "-Core installation" ; make sure windows knows about the change SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000 - MessageBox MB_OK "Added AF_PATH environment variable for all users.$\n$\nIf you chose not to modify PATH in the installer, please manually add $\"%AF_PATH%\lib$\" to the user or system PATH variable for running applications using ArrayFire." + MessageBox MB_OK "Added AF_PATH environment variable for all users.$\n$\nIf you chose not to modify PATH in the installer, please manually add $\"%AF_PATH%\lib$\" to the user or system PATH variable for running applications using ArrayFire." /SD IDOK ; Write special uninstall registry entries @@ -740,6 +740,11 @@ Section "-Core installation" SectionEnd +Section "-Visual C++ installation" + ExecWait "$INSTDIR\lib\vc_redist.x64.exe /install /passive /norestart" + Delete "$INSTDIR\lib\vc_redist.x64.exe" +SectionEnd + Section "-Add to path" Push $INSTDIR\lib StrCmp "@CPACK_NSIS_MODIFY_PATH@" "ON" 0 doNotAddToPath diff --git a/CMakeModules/select_compute_arch.cmake b/CMakeModules/select_compute_arch.cmake index 16abb8e6cd..e09490a7e5 100644 --- a/CMakeModules/select_compute_arch.cmake +++ b/CMakeModules/select_compute_arch.cmake @@ -7,7 +7,7 @@ # ARCH_AND_PTX : NAME | NUM.NUM | NUM.NUM(NUM.NUM) | NUM.NUM+PTX # NAME: Fermi Kepler Maxwell Kepler+Tegra Kepler+Tesla Maxwell+Tegra Pascal Volta Turing Ampere # NUM: Any number. Only those pairs are currently accepted by NVCC though: -# 2.0 2.1 3.0 3.2 3.5 3.7 5.0 5.2 5.3 6.0 6.2 7.0 7.2 7.5 8.0 8.6 +# 2.0 2.1 3.0 3.2 3.5 3.7 5.0 5.2 5.3 6.0 6.2 7.0 7.2 7.5 8.0 8.6 9.0 # Returns LIST of flags to be added to CUDA_NVCC_FLAGS in ${out_variable} # Additionally, sets ${out_variable}_readable to the resulting numeric list # Example: @@ -92,6 +92,25 @@ if(CUDA_VERSION VERSION_GREATER_EQUAL "11.1") set(CUDA_LIMIT_GPU_ARCHITECTURE "9.0") endif() +if(CUDA_VERSION VERSION_GREATER_EQUAL "11.8") + list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "8.9") + list(APPEND CUDA_ALL_GPU_ARCHITECTURES "8.9") + + set(_CUDA_MAX_COMMON_ARCHITECTURE "8.9+PTX") + set(CUDA_LIMIT_GPU_ARCHITECTURE "9.0") +endif() + +if(CUDA_VERSION VERSION_GREATER_EQUAL "12.0") + list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Hopper") + list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "9.0") + list(APPEND CUDA_ALL_GPU_ARCHITECTURES "9.0") + + set(_CUDA_MAX_COMMON_ARCHITECTURE "9.0+PTX") + set(CUDA_LIMIT_GPU_ARCHITECTURE "9.0") + + list(REMOVE_ITEM CUDA_ALL_GPU_ARCHITECTURES "3.5" "3.7") +endif() + list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "${_CUDA_MAX_COMMON_ARCHITECTURE}") # Check with: cmake -DCUDA_VERSION=7.0 -P select_compute_arch.cmake @@ -246,6 +265,9 @@ function(CUDA_SELECT_NVCC_ARCH_FLAGS out_variable) elseif(${arch_name} STREQUAL "Ampere") set(arch_bin 8.0) set(arch_ptx 8.0) + elseif(${arch_name} STREQUAL "Hopper") + set(arch_bin 9.0) + set(arch_ptx 9.0) else() message(SEND_ERROR "Unknown CUDA Architecture Name ${arch_name} in CUDA_SELECT_NVCC_ARCH_FLAGS") endif() diff --git a/LICENSE b/LICENSE index 8f4c645ca1..d63051d62b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2014-2022, ArrayFire +Copyright (c) 2014-2025, ArrayFire All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index c56f29623f..eb6dc6a5f6 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,15 @@ -

-ArrayFire is a general-purpose tensor library that simplifies the process of -software development for the parallel architectures found in CPUs, GPUs, and -other hardware acceleration devices. The library serves users in every technical -computing market. +ArrayFire is a general-purpose tensor library that simplifies the software +development process for the parallel architectures found in CPUs, GPUs, and +other hardware acceleration devices. The library serves users in every +technical computing market. Several of ArrayFire's benefits include: -* Hundreds of accelerated [tensor computing functions](https://arrayfire.org/docs/group__arrayfire__func.htm), in the following areas: +* Hundreds of accelerated [tensor computing + functions](https://arrayfire.org/docs/group__arrayfire__func.htm), in the + following areas: * Array handling * Computer vision * Image processing @@ -21,8 +22,10 @@ Several of ArrayFire's benefits include: * [Easy to use](http://arrayfire.org/docs/gettingstarted.htm), stable, [well-documented](http://arrayfire.org/docs) API * Rigorous benchmarks and tests ensuring top performance and numerical accuracy -* Cross-platform compatibility with support for CUDA, OpenCL, and native CPU on Windows, Mac, and Linux -* Built-in visualization functions through [Forge](https://github.com/arrayfire/forge) +* Cross-platform compatibility with support for CUDA, oneAPI, OpenCL, and + native CPU on Windows, Mac, and Linux +* Built-in visualization functions through + [Forge](https://github.com/arrayfire/forge) * Commercially friendly open-source licensing * Enterprise support from [ArrayFire](http://arrayfire.com) @@ -33,19 +36,22 @@ translated into near-optimal kernels that execute on the computational device. ArrayFire runs on devices ranging from low-power mobile phones to high-power GPU-enabled supercomputers. ArrayFire runs on CPUs from all major vendors -(Intel, AMD, ARM), GPUs from the prominent manufacturers (NVIDIA, AMD, and -Qualcomm), as well as a variety of other accelerator devices on Windows, Mac, -and Linux. +(Intel, AMD, ARM), GPUs from the prominent manufacturers (AMD, Intel, NVIDIA, +and Qualcomm), as well as a variety of other accelerator devices on Windows, +Mac, and Linux. # Getting ArrayFire -Instructions to [install][32] or to build ArrayFire from source can be found on the [wiki][1]. +Instructions to [install][32] or to build ArrayFire from source can be found on +the [wiki][1]. ### Conway's Game of Life Using ArrayFire Visit the [Wikipedia page][2] for a description of Conway's Game of Life. -Conway's Game of Life + ```cpp static const float h_kernel[] = { 1, 1, 1, 1, 0, 1, 1, 1, 1 }; @@ -65,7 +71,9 @@ The complete source code can be found [here][3]. ### Perceptron -Perceptron + ```cpp array predict(const array &X, const array &W) { @@ -131,9 +139,10 @@ Mission](https://github.com/arrayfire/arrayfire/wiki/The-ArrayFire-Mission-State for fast scientific computing for all. Contributions of any kind are welcome! Please refer to [the -wiki](https://github.com/arrayfire/arrayfire/wiki) and our [Code of Conduct](33) -to learn more about how you can get involved with the ArrayFire Community -through [Sponsorship](https://github.com/arrayfire/arrayfire/wiki/Sponsorship), +wiki](https://github.com/arrayfire/arrayfire/wiki) and our [Code of +Conduct](33) to learn more about how you can get involved with the ArrayFire +Community through +[Sponsorship](https://github.com/arrayfire/arrayfire/wiki/Sponsorship), [Developer Commits](https://github.com/arrayfire/arrayfire/wiki/Contributing-Code-to-ArrayFire), or [Governance](https://github.com/arrayfire/arrayfire/wiki/Governance). @@ -145,8 +154,8 @@ license](LICENSE). If you wish to cite ArrayFire in an academic publication, please use the following [citation document](.github/CITATION.md). ArrayFire development is funded by AccelerEyes LLC and several third parties, -please see the list of [acknowledgements](ACKNOWLEDGEMENTS.md) for an expression -of our gratitude. +please see the list of [acknowledgements](ACKNOWLEDGEMENTS.md) for an +expression of our gratitude. # Support and Contact Info @@ -156,10 +165,10 @@ of our gratitude. # Trademark Policy -The literal mark "ArrayFire" and ArrayFire logos are trademarks of -AccelerEyes LLC (dba ArrayFire). -If you wish to use either of these marks in your own project, please consult -[ArrayFire's Trademark Policy](http://arrayfire.com/trademark-policy/) +The literal mark "ArrayFire" and ArrayFire logos are trademarks of AccelerEyes +LLC (dba ArrayFire). If you wish to use either of these marks in your own +project, please consult [ArrayFire's Trademark +Policy](http://arrayfire.com/trademark-policy/) [1]: https://github.com/arrayfire/arrayfire/wiki [2]: https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life diff --git a/docs/details/algorithm.dox b/docs/details/algorithm.dox index 38b3c26d5a..69633524e2 100644 --- a/docs/details/algorithm.dox +++ b/docs/details/algorithm.dox @@ -1,312 +1,321 @@ /*! \page batch_detail_algo algorithm - -This function performs the operation across all batches present in the input simultaneously. - +This function runs across all batches in the input simultaneously. */ + /** \addtogroup arrayfire_func @{ -\defgroup reduce_func_sum sum + + +\defgroup reduce_func_sum sum \ingroup reduce_mat -Find the sum of values in the input +Sum array elements over a given dimension. -This table defines the return value types for the corresponding input types +This table defines output types for corresponding input types: Input Type | Output Type --------------------|--------------------- f32, f64, c32, c64 | same as input -s32, u32, s64, u64 | same as input -s16 | s32 +s32, s64, u32, u64 | same as input +s16, s8 | s32 u16, u8, b8 | u32 \copydoc batch_detail_algo -\defgroup reduce_func_sum_by_key sumByKey + +\defgroup reduce_func_sum_by_key sumByKey \ingroup reduce_mat -Finds the sum of an input array according to an array of keys. +Sum array elements over a given dimension, according to an array of keys. + The values corresponding to each group of consecutive equal keys will be summed -together. Keys can repeat, however only consecutive key values will be +together. Keys can repeat; however, only consecutive key values will be considered for each reduction. If a key value is repeated somewhere else in the -keys array it will be considered the start of a new reduction. There are two +keys array it will be considered the start of a new reduction. There are two outputs: the reduced set of consecutive keys and the corresponding final -reduced values. An example demonstrating the reduction behavior can be seen in -the following snippet. +set of reduced values. + +An example demonstrating the reduction behavior can be seen in the following +snippet. \snippet test/reduce.cpp ex_reduce_sum_by_key -The keys input type must be an integer type(s32 or u32). -This table defines the return types for the corresponding values type +The keys' input type must be integer (s32 or u32). + +This table defines output types for corresponding input types: Input Type | Output Type --------------------|--------------------- f32, f64, c32, c64 | same as input -s32, u32, s64, u64 | same as input -s16 | s32 +s32, s64, u32, u64 | same as input +s16, s8 | s32 u16, u8, b8 | u32 f16 | f32 -The input keys must be a 1-D vector matching the size of the reduced dimension. -In the case of multiple dimensions in the input values array, the dim parameter -specifies which dimension to reduce along. An example of multi-dimensional -reduce by key can be seen below: +The keys array must be 1-dimensional matching the size of the reduced +dimension. An example of multi-dimensional reduce-by-key can be seen below: \snippet test/reduce.cpp ex_reduce_sum_by_key_dim - \defgroup reduce_func_product product - \ingroup reduce_mat -Find the product of values in the input +Multiply array elements over a given dimension. -This table defines the return value types for the corresponding input types +This table defines output types for corresponding input types: Input Type | Output Type --------------------|--------------------- f32, f64, c32, c64 | same as input s32, u32, s64, u64 | same as input -s16 | s32 +s16, s8 | s32 u16, u8, b8 | u32 \copydoc batch_detail_algo -\defgroup reduce_func_product_by_key productByKey + +\defgroup reduce_func_product_by_key productByKey \ingroup reduce_mat -Finds the product of an input array according to an array of keys. +Multiply array elements over a given dimension, according to an array of keys. + The values corresponding to each group of consecutive equal keys will be -multiplied together. Keys can repeat, however only consecutive key values will +multiplied together. Keys can repeat; however, only consecutive key values will be considered for each reduction. If a key value is repeated somewhere else in -the keys array it will be considered the start of a new reduction. There are +the keys array it will be considered the start of a new reduction. There are two outputs: the reduced set of consecutive keys and the corresponding final -reduced values. An example demonstrating the reduction behavior can be seen in -the following snippet. +set of reduced values. + +An example demonstrating the reduction behavior can be seen in the following +snippet. \snippet test/reduce.cpp ex_reduce_product_by_key -The keys input type must be an integer type(s32 or u32). -This table defines the return types for the corresponding values type +The keys' input type must be integer (s32 or u32). + +This table defines output types for corresponding input types: Input Type | Output Type --------------------|--------------------- f32, f64, c32, c64 | same as input s32, u32, s64, u64 | same as input -s16 | s32 +s16, s8 | s32 u16, u8, b8 | u32 f16 | f32 -The input keys must be a 1-D vector matching the size of the reduced dimension. -In the case of multiple dimensions in the input values array, the dim parameter -specifies which dimension to reduce along. An example of multi-dimensional -reduce by key can be seen below: +The keys array must be 1-dimenstional matching the size of the reduced +dimension. An example of multi-dimensional reduce-by-key can be seen below: \snippet test/reduce.cpp ex_reduce_product_by_key_dim - \defgroup reduce_func_min min - \ingroup reduce_mat -Find the minimum values and their locations +Return the minimum along a given dimension. \copydoc batch_detail_algo -\defgroup reduce_func_min_by_key minByKey + +\defgroup reduce_func_min_by_key minByKey \ingroup reduce_mat -Finds the min of an input array according to an array of keys. The minimum -will be found of all values corresponding to each group of consecutive equal -keys. Keys can repeat, however only consecutive key values will be considered -for each reduction. If a key value is repeated somewhere else in the keys array -it will be considered the start of a new reduction. There are two outputs: -the reduced set of consecutive keys and the corresponding final reduced -values. An example demonstrating the reduction behavior can be seen in the -following snippet. +Return the minimum along a given dimension, according to an array of keys. + +The minimum is returned from the values corresponding to each group of +consecutive equal keys. Keys can repeat; however, only consecutive key values +will be considered for each reduction. If a key value is repeated somewhere +else in the keys array it will be considered the start of a new reduction. +There are two outputs: the reduced set of consecutive keys and the +corresponding final set of reduced values. + +An example demonstrating the reduction behavior can be seen in the following +snippet. \snippet test/reduce.cpp ex_reduce_min_by_key -The keys input type must be an integer type(s32 or u32). -The values return type will be the same as the values input type. +The keys' input type must be integer (s32 or u32). -The input keys must be a 1-D vector matching the size of the reduced dimension. -In the case of multiple dimensions in the input values array, the dim parameter -specifies which dimension to reduce along. An example of multi-dimensional -reduce by key can be seen below: +The output type is the same as input type. + +The keys array must be 1-dimenstional matching the size of the reduced +dimension. An example of multi-dimensional reduce-by-key can be seen below: \snippet test/reduce.cpp ex_reduce_min_by_key_dim -\defgroup reduce_func_max max +\defgroup reduce_func_max max \ingroup reduce_mat -Find the maximum values and their locations +Return the maximum along a given dimension. \copydoc batch_detail_algo -\defgroup reduce_func_max_by_key maxByKey +\defgroup reduce_func_max_by_key maxByKey \ingroup reduce_mat -Finds the max of an input array according to an array of keys. The maximum -will be found of all values corresponding to each group of consecutive equal -keys. Keys can repeat, however only consecutive key values will be considered -for each reduction. If a key value is repeated somewhere else in the keys array -it will be considered the start of a new reduction. There are two outputs: -the reduced set of consecutive keys and the corresponding final reduced -values. An example demonstrating the reduction behavior can be seen in the -following snippet. +Return the maximum along a given dimension, according to an array of keys. + +The maximum is returned from the values corresponding to each group of +consecutive equal keys. Keys can repeat; however, only consecutive key values +will be considered for each reduction. If a key value is repeated somewhere +else in the keys array it will be considered the start of a new reduction. +There are two outputs: the reduced set of consecutive keys and the +corresponding final set of reduced values. + +An example demonstrating the reduction behavior can be seen in the following +snippet. \snippet test/reduce.cpp ex_reduce_max_by_key -The keys input type must be an integer type(s32 or u32). -The values return type will be the same as the values input type. +The keys' input type must be integer (s32 or u32). + +The output type is the same as input type. -The input keys must be a 1-D vector matching the size of the reduced dimension. -In the case of multiple dimensions in the input values array, the dim parameter -specifies which dimension to reduce along. An example of multi-dimensional -reduce by key can be seen below: +The keys array must be 1-dimenstional matching the size of the reduced +dimension. An example of multi-dimensional reduce-by-key can be seen below: \snippet test/reduce.cpp ex_reduce_max_by_key_dim \defgroup reduce_func_all_true allTrue -\brief Test if all values in an array are true - \ingroup reduce_mat -Find if of all of the values in input are true +Check if all values along a given dimension are true. -Return type is b8 for all input types +Return type is `b8` for all input types. \copydoc batch_detail_algo -\defgroup reduce_func_all_true_by_key allTrueByKey -\brief Calculate if all values that share the same consecutive keys are true + +\defgroup reduce_func_all_true_by_key allTrueByKey \ingroup reduce_mat -Finds if all of the values of an input array are true according to an array of -keys. All values corresponding to each group of consecutive equal keys will be -tested to make sure all are true. Keys can repeat, however only consecutive -key values will be considered for each reduction. If a key value is repeated +Check if all values along a given dimension are true, according to an array of +keys. + +All values corresponding to each group of consecutive equal keys will be tested +to make sure all are true. Keys can repeat; however, only consecutive key +values will be considered for each reduction. If a key value is repeated somewhere else in the keys array it will be considered the start of a new -reduction. There are two outputs: the reduced set of consecutive keys and the -corresponding final reduced values. An example demonstrating the reduction -behavior can be seen in the following snippet. +reduction. There are two outputs: the reduced set of consecutive keys and the +corresponding final set of reduced values. + +An example demonstrating the reduction behavior can be seen in the following +snippet. \snippet test/reduce.cpp ex_reduce_alltrue_by_key -The keys input type must be an integer type(s32 or u32). -The values return type will be of type b8. +The keys' input type must be integer (s32 or u32). -The input keys must be a 1-D vector matching the size of the reduced dimension. -In the case of multiple dimensions in the input values array, the dim parameter -specifies which dimension to reduce along. An example of multi-dimensional -reduce by key can be seen below: +The output type is `b8`. -\snippet test/reduce.cpp ex_reduce_alltrue_by_key_dim +The keys array must be 1-dimenstional matching the size of the reduced +dimension. An example of multi-dimensional reduce-by-key can be seen below: +\snippet test/reduce.cpp ex_reduce_alltrue_by_key_dim \defgroup reduce_func_any_true anytrue -\brief Calculate if any values in an array are true - \ingroup reduce_mat -Find if of any of the values in input are true +Check if any values along a given dimension are true. -Return type is b8 for all input types +The output type is `b8`. \copydoc batch_detail_algo -\defgroup reduce_func_anytrue_by_key anyTrueByKey -\brief Calculate if any values that share the same consecutive keys are true + +\defgroup reduce_func_anytrue_by_key anyTrueByKey \ingroup reduce_mat -Finds if any of the values of an input array are true according to an array of -keys. All values corresponding to each group of consecutive equal keys will be -tested to make sure any are true. Keys can repeat, however only consecutive -key values will be considered for each reduction. If a key value is repeated +Check if any values along a given dimension are true, according to an array of +keys. + +Values corresponding to each group of consecutive equal keys will be tested to +check if any are true. Keys can repeat; however, only consecutive key +values will be considered for each reduction. If a key value is repeated somewhere else in the keys array it will be considered the start of a new -reduction. There are two outputs: the reduced set of consecutive keys and the -corresponding final reduced values. An example demonstrating the reduction -behavior can be seen in the following snippet. +reduction. There are two outputs: the reduced set of consecutive keys and the +corresponding final set of reduced values. + +An example demonstrating the reduction behavior can be seen in the following +snippet. \snippet test/reduce.cpp ex_reduce_anytrue_by_key -The keys input type must be an integer type(s32 or u32). -The values return type will be of type u8. +The keys' input type must be integer (s32 or u32). -The input keys must be a 1-D vector matching the size of the reduced dimension. -In the case of multiple dimensions in the input values array, the dim parameter -specifies which dimension to reduce along. An example of multi-dimensional -reduce by key can be seen below: +The output type is `b8`. + +The keys array must be 1-dimenstional matching the size of the reduced +dimension. An example of multi-dimensional reduce-by-key can be seen below: \snippet test/reduce.cpp ex_reduce_anytrue_by_key_dim -\defgroup reduce_func_count count +\defgroup reduce_func_count count \ingroup reduce_mat -Count the number of non-zero elements in the input +Count non-zero values in an array along a given dimension. -Return type is u32 for all input types +The output type is `u32`. \copydoc batch_detail_algo -\defgroup reduce_func_count_by_key countByKey + +\defgroup reduce_func_count_by_key countByKey \ingroup reduce_mat -Counts the non-zero values of an input array according to an array of keys. +Count non-zero values in an array, according to an array of keys. + All non-zero values corresponding to each group of consecutive equal keys will -be counted. Keys can repeat, however only consecutive key values will be +be counted. Keys can repeat; however, only consecutive key values will be considered for each reduction. If a key value is repeated somewhere else in the -keys array it will be considered the start of a new reduction. There are two -outputs: the reduced set of consecutive keys and the corresponding final -reduced values. An example demonstrating the reduction behavior can be seen in -the following snippet. +keys array it will be considered the start of a new reduction. There are two +outputs: the reduced set of consecutive keys and the corresponding final set of +reduced values. + +An example demonstrating the reduction behavior can be seen in the following +snippet. \snippet test/reduce.cpp ex_reduce_count_by_key -The keys input type must be an integer type(s32 or u32). -The values return type will be of type u32. +The keys' input type must be integer (s32 or u32). -The input keys must be a 1-D vector matching the size of the reduced dimension. -In the case of multiple dimensions in the input values array, the dim parameter -specifies which dimension to reduce along. An example of multi-dimensional -reduce by key can be seen below: +The output type is `u32`. -\snippet test/reduce.cpp ex_reduce_count_by_key_dim +The keys array must be 1-dimenstional matching the size of the reduced +dimension. An example of multi-dimensional reduce-by-key can be seen below: +\snippet test/reduce.cpp ex_reduce_count_by_key_dim \defgroup scan_func_accum accum -\brief Cumulative sum (inclusive). Also known as a scan - \ingroup scan_mat -Calculate the cumulative sum (inclusive) along the specified dimension +Evaluate the cumulative sum (inclusive) along a given dimension. For a 1D array \f$X\f$, the inclusive cumulative sum calculates \f$x_i = \sum_{p=0}^{i}x_p\f$ for every \f$x \in X\f$. Here is a simple example for the @@ -314,7 +323,7 @@ For a 1D array \f$X\f$, the inclusive cumulative sum calculates \f$x_i = \snippet test/scan.cpp ex_accum_1D -For 2D arrays (and higher dimensions), you can specify the dimension along which +For 2D arrays and higher dimensions, you can specify the dimension along which the cumulative sum will be calculated. Thus, the formula above will be calculated for all array slices along the specified dimension (in the 2D case for example, this looks like \f$x_{i,j} = \sum_{p=0}^{j}x_{i,p}\f$ if the second @@ -325,164 +334,160 @@ required to be specified in the C API): \snippet test/scan.cpp ex_accum_2D The output array type may be different from the input array type. The following -table defines the corresponding output types for each input type: +table defines corresponding output types for each input type: Input Type | Output Type --------------------|--------------------- f32, f64, c32, c64 | same as input -s32, u32, s64, u64 | same as input -s16 | s32 +s32, s64, u32, u64 | same as input +s16, s8 | s32 u16, u8, b8 | u32 \copydoc batch_detail_algo -\defgroup scan_func_where where - +\defgroup scan_func_scan scan \ingroup scan_mat -Locate the indices of non-zero elements - -Return type is u32 for all input types +Scan an array (generalized) over a given dimension. -The locations are provided by flattening the input into a linear array. +Perform inclusive or exclusive scan using a given binary operation along a +given dimension. +Binary operations can be [add](\ref AF_BINARY_ADD), [mul](\ref AF_BINARY_MUL), +[min](\ref AF_BINARY_MIN), [max](\ref AF_BINARY_MAX) as defined by \ref +af_binary_op. -\defgroup scan_func_scan scan +\defgroup scan_func_scanbykey scanByKey \ingroup scan_mat -Inclusive or exclusive scan of an array +Scan an array (generalized) over a given dimension, according to an array of +keys. Perform inclusive or exclusive scan using a given binary operation along a -given dimension. +given dimension using a key. Binary operations can be [add](\ref AF_BINARY_ADD), [mul](\ref AF_BINARY_MUL), -[min](\ref AF_BINARY_MIN), [max](\ref AF_BINARY_MAX) as defined by \ref af_binary_op. - +[min](\ref AF_BINARY_MIN), [max](\ref AF_BINARY_MAX) as defined by \ref +af_binary_op. -\defgroup scan_func_scanbykey scanByKey +\defgroup scan_func_where where \ingroup scan_mat -Inclusive or exclusive scan of an array by key +Locate the indices of the non-zero values in an array. -Perform inclusive or exclusive scan using a given binary operation along a -given dimension using a key. +Output type is `u32`. -Binary operations can be [add](\ref AF_BINARY_ADD), [mul](\ref AF_BINARY_MUL), -[min](\ref AF_BINARY_MIN), [max](\ref AF_BINARY_MAX) as defined by \ref af_binary_op. +The locations are provided by flattening the input into a linear array. \defgroup calc_func_diff1 diff1 - \ingroup calc_mat -First order numerical difference along specified dimension +Calculate the first order difference in an array over a given dimension. \copydoc batch_detail_algo \defgroup calc_func_diff2 diff2 - \ingroup calc_mat -Second order numerical difference along specified dimension +Calculate the second order difference in an array over a given dimension. \copydoc batch_detail_algo \defgroup sort_func_sort sort - \ingroup sort_mat -Sort input arrays - -Sort an multi dimensional array +Sort an array over a given dimension. \defgroup sort_func_sort_index sortIndex - \ingroup sort_mat -Sort input arrays get the sorted indices +Sort an array over a given dimension and return the original indices. -Sort a multi dimensional array and return sorted indices. Index array is of -type u32. +Output type is `u32`. \defgroup sort_func_sort_keys sortByKey - \ingroup sort_mat -Sort input arrays based on keys - -Sort a multi dimensional array based on keys +Sort an array over a given dimension, according to an array of keys. \defgroup set_func_unique setunique - \ingroup set_mat -Finds unique values from an input set. The input must be a one-dimensional array. Batching is not currently supported. +Return the unique values in an array. + +The input must be a one-dimensional array. Batching is not currently supported. -A simple example of finding the unique values of a set using setUnique() can be seen below: +An example, unsorted: \snippet test/set.cpp ex_set_unique_simple The function can be sped up if it is known that the inputs are sorted. +An example, sorted (ascending): + \snippet test/set.cpp ex_set_unique_sorted The inputs can be sorted in ascending or descending order. -\snippet test/set.cpp ex_set_unique_desc - - +An example, sorted (descending): +\snippet test/set.cpp ex_set_unique_desc \defgroup set_func_union setunion - \ingroup set_mat -Find the union of two sets. The inputs must be one-dimensional arrays. Batching is not currently supported. +Evaluate the union of two arrays. + +The inputs must be one-dimensional arrays. Batching is not currently supported. -A simple example of finding the union of two sets using setUnion() can be seen below: +An example: \snippet test/set.cpp ex_set_union_simple -The function can be sped up if it is known that each input is sorted in increasing order and its values are unique. +The function can be sped up if the input is sorted in increasing order and its +values are unique. \snippet test/set.cpp ex_set_union - \defgroup set_func_intersect setintersect - \ingroup set_mat -Find the intersection of two sets. The inputs must be one-dimensional arrays. Batching is not currently supported. +Evaluate the intersection of two arrays. + +The inputs must be one-dimensional arrays. Batching is not currently supported. -A simple example of finding the intersection of two sets using setIntersect() can be seen below: +An example: \snippet test/set.cpp ex_set_intersect_simple -The function can be sped up if it is known that each input is sorted in increasing order and its values are unique. +The function can be sped up if the input is sorted in increasing order and its +values are unique. \snippet test/set.cpp ex_set_intersect + @} */ diff --git a/docs/details/arith.dox b/docs/details/arith.dox index 2e123f7ba8..3a118bc890 100644 --- a/docs/details/arith.dox +++ b/docs/details/arith.dox @@ -1,6 +1,7 @@ /*! \page arith_real_only arith_real -\note This function supports real inputs only. Complex inputs are not yet supported. +\note This function only supports real inputs; complex inputs are not yet +supported. */ /*! @@ -19,28 +20,28 @@ \defgroup arith_func_add add \ingroup arith_mat -Elementwise addition +Elementwise addition. \defgroup arith_func_sub sub \ingroup arith_mat -Elementwise subtraction +Elementwise subtraction. \defgroup arith_func_mul mul \ingroup arith_mat -Elementwise multiply +Elementwise multiply. \defgroup arith_func_div div \ingroup arith_mat -Elementwise division +Elementwise division. @@ -67,7 +68,8 @@ Check if the elements of one array are greater than those of another array. Less than or equal to, an elementwise comparison of two arrays. -Check if the elements of one array are less than or equal to those of another array. +Check if the elements of one array are less than or equal to those of another +array. \defgroup arith_func_ge ge @@ -75,14 +77,15 @@ Check if the elements of one array are less than or equal to those of another ar Greater than or equal to, an elementwise comparison of two arrays. -Check if the elements of one array are greater than or equal to those of another array. +Check if the elements of one array are greater than or equal to those of +another array. \defgroup arith_func_eq eq \ingroup logic_mat -\brief Equal to, an elementwise comparison of two arrays. +Equal to, an elementwise comparison of two arrays. Check if the elements of one array are equal to those of another array. @@ -91,7 +94,7 @@ Check if the elements of one array are equal to those of another array. \defgroup arith_func_neq neq \ingroup logic_mat -\brief Not equal to, an elementwise comparison of two arrays. +Not equal to, an elementwise comparison of two arrays. Check if the elements of one array are not equal to those of another array. @@ -384,10 +387,14 @@ Create complex arrays. Complex arrays are created from any of the following four inputs: -1. a single real array, returning zeros for the imaginary component. See `array b` in the example. -2. two real arrays, one for the real component and one for the imaginary component. See `array c` in the example. -3. a single real array for the real component and a single scalar for each imaginary component. See `array d` in the example. -4. a single scalar for each real component and a single real array for the imaginary component. See `array e` in the example. +1. a single real array, returning zeros for the imaginary component. See + `array b` in the example. +2. two real arrays, one for the real component and one for the imaginary + component. See `array c` in the example. +3. a single real array for the real component and a single scalar for each + imaginary component. See `array d` in the example. +4. a single scalar for each real component and a single real array for the + imaginary component. See `array e` in the example. __Examples:__ diff --git a/docs/details/blas.dox b/docs/details/blas.dox index b8757d81fb..ac0aa99673 100644 --- a/docs/details/blas.dox +++ b/docs/details/blas.dox @@ -1,29 +1,18 @@ /** \addtogroup arrayfire_func @{ -\defgroup blas_func_dot dot - -\ingroup blas_mat - -\brief Calculate the dot product of a vector - -Scalar dot product between two vectors. Also referred to as the inner -product. - -======================================================================= \defgroup blas_func_matmul matmul -\ingroup blas_mat -\brief Matrix multiplication using array +Matrix multiplication. Performs a matrix multiplication on the two input arrays after performing the operations specified in the options. The operations are done while reading the data from memory. This results in no additional memory being used for temporary buffers. -Batched matrix multiplications are supported. Given below are the supported -types of batch operations for any given set of two matrices A and B. +Batched matrix multiplications are supported. The supported types of batch +operations for any given set of two matrices A and B are given below, | Size of Input Matrix A | Size of Input Matrix B | Output Matrix Size | |:--------------------------:|:--------------------------:|:---------------------------:| @@ -32,8 +21,8 @@ types of batch operations for any given set of two matrices A and B. | \f$ \{ M, K, 1, 1 \} \f$ | \f$ \{ K, N, b2, b3 \} \f$ | \f$ \{ M, N, b2, b3 \} \f$ | | \f$ \{ M, K, b2, b3 \} \f$ | \f$ \{ K, N, 1, 1 \} \f$ | \f$ \{ M, N, b2, b3 \} \f$ | -where M, K, N are dimensions of the matrix and b2, b3 indicate batch size along the -respective dimension. +where `M`, `K`, `N` are dimensions of the matrix and `b2`, `b3` indicate batch +size along the respective dimension. For the last two entries in the above table, the 2D matrix is broadcasted to match the dimensions of 3D/4D array. This broadcast doesn't involve any additional @@ -43,14 +32,28 @@ memory allocations either on host or device. for Sparse-Dense matrix multiplication. See the notes of the function for usage and restrictions. +\par +\note Limited support for \ref s8 was added to the CUDA backend in ArrayFire +v3.10.0. See \ref af_gemm "s8 Support" notes for details. + +\ingroup blas_mat ======================================================================= -\defgroup blas_func_transpose transpose +\defgroup blas_func_dot dot + +Compute the dot product. + +Scalar dot product between two vectors, also referred to as the inner +product. + \ingroup blas_mat -\ingroup manip_mat -\brief Transpose a matrix. +======================================================================= + +\defgroup blas_func_transpose transpose + +Transpose a matrix. Reverse or permute the dimensions of an array; returns the modified array. For an array a with two dimensions, `transpose(a)` gives the matrix transpose. @@ -70,6 +73,9 @@ __Examples:__ \snippet test/transpose.cpp ex_blas_func_transpose +\ingroup blas_mat +\ingroup manip_mat + ======================================================================= @} diff --git a/docs/details/data.dox b/docs/details/data.dox index 99a94f1202..bb96a4c61f 100644 --- a/docs/details/data.dox +++ b/docs/details/data.dox @@ -4,20 +4,9 @@ \defgroup data_func_constant constant -\brief Create a array from a scalar input value +Create an array from a scalar input value. -The array created has the same value at all locations - -\ingroup data_mat -\ingroup arrayfire_func - -======================================================================= - -\defgroup data_func_pad pad - -\brief Pad an array - -Pad the input array using a constant or values from input along border +Generate an array with elements set to a specified value. \ingroup data_mat \ingroup arrayfire_func @@ -26,7 +15,7 @@ Pad the input array using a constant or values from input along border \defgroup data_func_identity identity -\brief Create an identity array with diagonal values 1 +Generate an identity matrix. \code array a = identity(5, 3); @@ -45,7 +34,8 @@ array a = identity(5, 3); \defgroup data_func_range range -\brief Create an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions. +Generate an array with `[0, n-1]` values along the a specified dimension and +tiled across other dimensions. __Examples:__ @@ -58,7 +48,8 @@ __Examples:__ \defgroup data_func_iota iota -\brief Create an sequence [0, dims.elements() - 1] and modify to specified dimensions dims and then tile it according to tile_dims +Generate an array with `[0, n-1]` values modified to specified dimensions and +tiling. \code // Generate [0, 5x3 - 1] in dimensions 5, 3 @@ -87,7 +78,12 @@ array b = iota(dim4(5, 3), dim4(1, 2)) ======================================================================= \defgroup data_func_diag diag -\brief Extract diagonal from a matrix when \p extract is set to true. Create a diagonal matrix from input array when \p extract is set to false + +Extract the diagonal from an array. + +If `extract` is true, an array is extracted containing diagonal of the matrix, +while a false condition returns a diagonal matrix. + \code // Extraction @@ -140,9 +136,10 @@ array b = diag(a, -1, false); \defgroup manip_func_join join -\brief Join up to 4 arrays along specified dimension. +Join up to 4 arrays along specified dimension. -Requires that all dimensions except the join dimension must be the same for all arrays. +Requires that all dimensions except the join dimension must be the same for all +arrays. \ingroup manip_mat \ingroup arrayfire_func @@ -151,13 +148,14 @@ Requires that all dimensions except the join dimension must be the same for all \defgroup manip_func_tile tile -\brief Repeat the contents of the input array along the specified dimensions +Generate a tiled array by repeating an array's contents along a specified +dimension. Creates copies of the input array and concatenates them with each other, such that the output array will have as many copies of the input array as the user -specifies, along each dimension. In this sense, the output array is essentially -a set of "tiles", where each copy of the input array (including the original) is -a "tile" (hence the name of this function). +specifies along each dimension. In this sense, the output array is a set of +"tiles" where each copy of the input array, including the original, is +a "tile". Given below are some examples. The input array looks like this: @@ -184,7 +182,7 @@ dimension: \defgroup manip_func_reorder reorder -\brief Reorder an array according to the specified dimensions. +Reorder an array. Exchanges data of an array such that the requested change in dimension is satisfied. The linear ordering of data within the array is preserved. @@ -201,7 +199,7 @@ a [2 2 3 1] 2.0000 4.0000 -reorder(a, 1, 0, 2) [2 2 3 1] //equivalent to a transpose +reorder(a, 1, 0, 2) [2 2 3 1] // equivalent to a transpose 1.0000 2.0000 3.0000 4.0000 @@ -229,9 +227,9 @@ reorder(a, 2, 0, 1) [3 2 2 1] \defgroup manip_func_shift shift -\brief Circular shift slong specified dimensions +Shift an array. -Shifts the values in a circular fashion along the specified dimesion. +Circular shift array values along a specified dimesion. \ingroup manip_mat \ingroup arrayfire_func @@ -240,9 +238,10 @@ Shifts the values in a circular fashion along the specified dimesion. \defgroup manip_func_moddims moddims -\brief Modify the dimensions of an array without changing the order of its elements. +Modify the dimensions of an array without changing the order of its elements. -This function only modifies array metadata and requires no computation. It is a NOOP. +This function only modifies array metadata and requires no computation. It is a +NOOP. __Examples:__ @@ -255,9 +254,9 @@ __Examples:__ \defgroup manip_func_flat flat -\brief Flatten the input to a single dimension +Flatten an array. -Simply returns the array as a vector. This is a noop. +Simply returns the array as a vector. This is a NOOP. \ingroup manip_mat \ingroup arrayfire_func @@ -266,9 +265,9 @@ Simply returns the array as a vector. This is a noop. \defgroup manip_func_flip flip -\brief Flip the input along specified dimension +Flip the input along a specified dimension. -Mirrors the array along the specified dimensions. +Mirrors the array along the specified dimension. \ingroup manip_mat \ingroup arrayfire_func @@ -277,7 +276,7 @@ Mirrors the array along the specified dimensions. \defgroup data_func_lower lower -\brief Create a lower triangular matrix from input array +Return the lower triangular matrix from an input array. \ingroup data_mat \ingroup arrayfire_func @@ -286,7 +285,7 @@ Mirrors the array along the specified dimensions. \defgroup data_func_upper upper -\brief Create a upper triangular matrix from input array +Return the upper triangular matrix from an input array. \ingroup data_mat \ingroup arrayfire_func @@ -295,13 +294,12 @@ Mirrors the array along the specified dimensions. \defgroup data_func_select select -\brief Selects elements from two arrays based on the values of a binary - conditional array. +Select elements based on a conditional array. -Creates a new array that is composed of values either from array \p a or array -\p b, based on a third conditional array. For all non-zero elements in the -conditional array, the output array will contain values from \p a. Otherwise the -output will contain values from \p b. +Creates a new array that is composed of values either from array `a` or array +`b`, based on a third conditional array. For all non-zero elements in the +conditional array, the output array will contain values from `a`. Otherwise the +output will contain values from `b`. \snippet test/select.cpp ex_data_select @@ -309,7 +307,7 @@ is equivalent to: \snippet test/select.cpp ex_data_select_c -The conditional array must be a b8 typed array. +The conditional array must be a \ref b8 typed array. The select function can perform batched operations based on the size of each of the inputs. The following table describes the input and output sizes for @@ -330,15 +328,27 @@ supported batched configurations. \defgroup data_func_replace replace -\brief Replace elements of an array based on a conditional array +Replace elements of an array with elements of another array. -- Input values are retained when corresponding elements from condition array are true. -- Input values are replaced when corresponding elements from condition array are false. +Input values are retained when corresponding elements from the conditional +array are true. Input values are replaced when corresponding elements from the +conditional array are false. \ingroup manip_mat \ingroup arrayfire_func ======================================================================= +\defgroup data_func_pad pad + +Pad an array. + +Pad the input array using a constant or values from input along the border. + +\ingroup data_mat +\ingroup arrayfire_func + +======================================================================= + @} */ diff --git a/docs/details/image.dox b/docs/details/image.dox index a93f1ebaed..312b88c880 100644 --- a/docs/details/image.dox +++ b/docs/details/image.dox @@ -1007,6 +1007,7 @@ Iterative deconvolution function excepts \ref af::array of the following types o - \ref f32 - \ref s16 - \ref u16 + - \ref s8 - \ref u8 \note The type of output \ref af::array from deconvolution will be double if @@ -1044,6 +1045,7 @@ Inverse deconvolution function excepts \ref af::array of the following types onl - \ref f32 - \ref s16 - \ref u16 + - \ref s8 - \ref u8 \note The type of output \ref af::array from deconvolution will be double diff --git a/docs/details/lapack.dox b/docs/details/lapack.dox index bf977b0c0c..995d47129b 100644 --- a/docs/details/lapack.dox +++ b/docs/details/lapack.dox @@ -1,25 +1,47 @@ /** \addtogroup arrayfire_func @{ -\defgroup lapack_factor_func_lu lu + +\defgroup lapack_factor_func_svd svd + +Perform singular value decomposition. + +This function factorizes a matrix \f$A\f$ into two unitary matrices, \f$U\f$ +and \f$V^T\f$, and a diagonal matrix \f$S\f$, such that \f$A = USV^T\f$. If +\f$A\f$ has \f$M\f$ rows and \f$N\f$ columns (\f$M \times N\f$), then \f$U\f$ +will be \f$M \times M\f$, \f$V\f$ will be \f$N \times N\f$, and \f$S\f$ will be +\f$M \times N\f$. However, for \f$S\f$, this function only returns the non-zero +diagonal elements as a sorted (in descending order) 1D array. + +To reconstruct the original matrix \f$A\f$ from the individual factors, the +following code snippet can be used: + +\snippet test/svd_dense.cpp ex_svd_reg + +When memory is a concern, and \f$A\f$ is dispensable, \ref af::svdInPlace() can +be used. However, this in-place version is currently limited to input arrays +where \f$M \geq N\f$. \ingroup lapack_factor_mat -\brief Perform LU decomposition +=============================================================================== -This function decomposes input matrix **A** into a lower triangle **L**, an upper triangle **U** such that +\defgroup lapack_factor_func_lu lu - \f$A = L * U\f$ +Perform LU decomposition. -For stability, a permutation array **P** is also used to modify the formula in the following manner. +This function decomposes input matrix \f$A\f$ into a lower triangle \f$L\f$, an +upper triangle \f$U\f$ such that \f$A = L * U\f$. - \f$A(P, span) = L * U\f$ +For stability, a permutation array \f$P\f$ is also used to modify the formula +in the following manner, \f$A(P, span) = L * U\f$. -This operation can be performed in ArrayFire using the following code snippet. +This operation can be performed in ArrayFire, using the following code snippet. \snippet test/lu_dense.cpp ex_lu_unpacked -The permuted version of the original matrix can be reconstructed using the following snippet. +The permuted version of the original matrix can be reconstructed, using the +following snippet. \snippet test/lu_dense.cpp ex_lu_recon @@ -57,115 +79,98 @@ a_perm [3 3 1 1] 1.0000 4.0000 7.0000 \endcode -When memory is a concern, users can perform the LU decomposition in place as shown below. +When memory is a concern, users can perform the LU decomposition in place as +shown below. \snippet test/lu_dense.cpp ex_lu_packed -The lower and upper triangle matrices can be obtained if necessary in the following manner. +The lower and upper triangle matrices can be obtained if necessary in the +following manner. \snippet test/lu_dense.cpp ex_lu_extract -LU decompositions has many applications including solving a system of linear equations. Check \ref af::solveLU fore more information. - -======================================================================= - -\defgroup lapack_factor_func_qr qr +LU decompositions have many applications including + +solving a system of linear equations. Check \ref af::solveLU for more +information. \ingroup lapack_factor_mat -\brief Perform QR decomposition - -This function decomposes input matrix **A** into an orthogonal matrix **Q** and an upper triangular matrix **R** such that +=============================================================================== - \f$A = Q * R\f$ +\defgroup lapack_factor_func_qr qr - \f$Q * Q^T = I\f$ +Perform QR decomposition. -Where **I** is an identity matrix. The matrix **Q** is a square matrix of size **max(M, N)** where **M** and **N** are rows and columns of **A** respectively. The matrix **R** is the same size as **A*. +This function decomposes input matrix \f$A\f$ into an orthogonal matrix \f$Q\f$ +and an upper triangular matrix \f$R\f$ such that, \f$A = Q * R\f$ and +\f$Q * Q^T = I\f$, where \f$I\f$ is an identity matrix. The matrix \f$Q\f$ is a +square matrix of size \f$max(M, N)\f$ where \f$M\f$ and \f$N\f$ are rows and +columns of \f$A\f$ respectively. The matrix \f$R\f$ is the same size as +\f$A\f$. This operation can be performed in ArrayFire using the following code snippet. \snippet test/qr_dense.cpp ex_qr_unpacked -The additional parameter **Tau** can be used to speed up solving over and under determined system of equations. +The additional parameter `tau` can be used to speed up solving over- and +under-determined systems of equations. The original matrix can be reconstructed using the following code snippet. \snippet test/qr_dense.cpp ex_qr_recon -When memory is a concern, users can perform QR decomposition in place as shown below. +When memory is a concern, users can perform QR decomposition in place as shown +below. \snippet test/qr_dense.cpp ex_qr_packed -======================================================================= - -\defgroup lapack_factor_func_cholesky cholesky - \ingroup lapack_factor_mat -\brief Perform Cholesky decomposition +=============================================================================== -This function decomposes a positive definite matrix **A** into two triangular matrices such that +\defgroup lapack_factor_func_cholesky cholesky - \f$A = L * U\f$ +Perform Cholesky decomposition. - \f$L = U^T\f$ +This function decomposes a +positive +definite matrix \f$A\f$ into two triangular matrices such that, +\f$A = L * U\f$ and \f$L = U^T\f$. -Only one of **L** and **U** is stored to conserve space when solving linear equations. +Only one of \f$L\f$ and \f$U\f$ is stored to conserve space when solving linear +equations. This operation can be performed in ArrayFire using the following code snippet. \snippet test/cholesky_dense.cpp ex_chol_reg -When memory is a concern, users can perform Cholesky decomposition in place as shown below. +When memory is a concern, users can perform Cholesky decomposition in place as +shown below. \snippet test/cholesky_dense.cpp ex_chol_inplace -======================================================================= - -\defgroup lapack_factor_func_svd svd - \ingroup lapack_factor_mat -\brief Computes the singular value decomposition of a matrix - -This function factorizes a matrix \f$A\f$ into two unitary matrices, \f$U\f$ and -\f$V^T\f$, and a diagonal matrix \f$S\f$, such that \f$A = USV^T\f$. If \f$A\f$ -has \f$M\f$ rows and \f$N\f$ columns (\f$M \times N\f$), then \f$U\f$ will be -\f$M \times M\f$, \f$V\f$ will be \f$N \times N\f$, and \f$S\f$ will be -\f$M \times N\f$. However, for \f$S\f$, this function only returns the non-zero -diagonal elements as a sorted (in descending order) 1D array. - -To reconstruct the original matrix \f$A\f$ from the individual factors, the -following code snippet can be used: - -\snippet test/svd_dense.cpp ex_svd_reg - -When memory is a concern, and \f$A\f$ is dispensable, \ref af::svdInPlace() can be -used. However, this in-place version is currently limited to input arrays where -\f$M \geq N\f$. - -======================================================================= +=============================================================================== \defgroup lapack_solve_func_gen solve -\ingroup lapack_solve_mat - -\brief Solve a system of equations +Solve a system of equations. -This function takes a co-efficient matrix **A** and an output matrix **B** as inputs to solve the following equation for **X** - - \f$A * X = B\f$ +This function takes a co-efficient matrix \f$A\f$ and an output matrix \f$B\f$ +as inputs to solve the following equation for \f$X\f$, \f$A * X = B\f$. This operation can be done in ArrayFire using the following code snippet. \snippet test/solve_common.hpp ex_solve -The results can be verified by reconstructing the output matrix using \ref af::matmul in the following manner. +The results can be verified by reconstructing the output matrix using \ref +af::matmul in the following manner, \snippet test/solve_common.hpp ex_solve_recon -The sample output can be seen below +The sample output can be seen below. \code A [3 3 1 1] @@ -189,52 +194,57 @@ B1 [3 1 1 1] 39.0000 \endcode -If the coefficient matrix is known to be a triangular matrix, \ref AF_MAT_LOWER or \ref AF_MAT_UPPER can be passed to make solve faster. +If the coefficient matrix is known to be a triangular matrix, \ref AF_MAT_LOWER +or \ref AF_MAT_UPPER can be passed to make solve faster. -The sample code snippets for solving a lower triangular matrix can be seen below. +The sample code snippets for solving a lower triangular matrix can be seen +below. \snippet test/solve_common.hpp ex_solve_lower -Similarily, the code snippet for solving an upper triangular matrix can be seen below. +Similarily, the code snippet for solving an upper triangular matrix can be seen +below. \snippet test/solve_common.hpp ex_solve_upper See also: \ref af::solveLU -======================================================================= - -\defgroup lapack_solve_lu_func_gen solveLU - \ingroup lapack_solve_mat -\brief Solve a system of equations +=============================================================================== + +\defgroup lapack_solve_lu_func_gen solveLU -This function takes a co-efficient matrix **A** and an output matrix **B** as inputs to solve the following equation for **X** +Solve a system of equations. - \f$A * X = B\f$ +This function takes a co-efficient matrix \f$A\f$ and an output matrix \f$B\f$ +as inputs to solve the following equation for \f$X\f$, \f$A * X = B\f$. This operation can be done in ArrayFire using the following code snippet. \snippet test/solve_common.hpp ex_solve_lu -This function along with \ref af::lu split up the task af::solve performs for square matrices. +This function, along with \ref af::lu, split up the task af::solve performs for +square matrices. -\note This function is beneficial over \ref af::solve only in long running application where the coefficient matrix **A** stays the same, but the observed variables keep changing. +This function is beneficial over \ref af::solve only in long running +application where the coefficient matrix \f$A\f$ stays the same, but the +observed variables keep changing. +\ingroup lapack_solve_mat -======================================================================= +=============================================================================== \defgroup lapack_ops_func_inv inverse -\ingroup lapack_ops_mat - -\brief Invert a matrix +Invert a matrix. -This function inverts a square matrix **A**. The code snippet to demonstrate this can be seen below. +This function inverts a square matrix \f$A\f$. The code snippet to demonstrate +this can be seen below. \snippet test/inverse_dense.cpp ex_inverse -The sample output can be seen below +The sample output can be seen below. \code A [3 3 1 1] @@ -254,71 +264,74 @@ I [3 3 1 1] \endcode -======================================================================= +\ingroup lapack_ops_mat -\defgroup lapack_ops_func_pinv pinverse +=============================================================================== -\ingroup lapack_ops_mat +\defgroup lapack_ops_func_pinv pinverse -\brief Pseudo-invert a matrix +Pseudo-invert (Moore-Penrose) a matrix. This function calculates the Moore-Penrose pseudoinverse of a matrix \f$A\f$, -using \ref af::svd at its core. If \f$A\f$ is of size \f$M \times N\f$, then its -pseudoinverse \f$A^+\f$ will be of size \f$N \times M\f$. +using \ref af::svd at its core. If \f$A\f$ is of size \f$M \times N\f$, then +its pseudoinverse \f$A^+\f$ will be of size \f$N \times M\f$. This calculation can be batched if the input array is three or four-dimensional \f$(M \times N \times P \times Q\f$, with \f$Q=1\f$ for only three dimensions -\f$)\f$. Each \f$M \times N\f$ slice along the third dimension will have its own -pseudoinverse, for a total of \f$P \times Q\f$ pseudoinverses in the output array -\f$(N \times M \times P \times Q)\f$. +\f$)\f$. Each \f$M \times N\f$ slice along the third dimension will have its +own pseudoinverse, for a total of \f$P \times Q\f$ pseudoinverses in the output +array \f$(N \times M \times P \times Q)\f$. -Here's an example snippet of its usage. In this example, we have a matrix \f$A\f$ -and we compute its pseudoinverse \f$A^+\f$. This condition must hold: +Below is an example snippet of its usage. In this example, we have a matrix +\f$A\f$ and compute its pseudoinverse \f$A^+\f$. This condition must hold: \f$AA^+A=A\f$, given that the two matrices are pseudoinverses of each other (in fact, this is one of the Moore-Penrose conditions): \snippet test/pinverse.cpp ex_pinverse -================================================================================== +\ingroup lapack_ops_mat + +=============================================================================== \defgroup lapack_ops_func_rank rank -\ingroup lapack_ops_mat +Find the rank of a matrix. -\brief Find the rank of the input matrix. +This function uses \ref af::qr to find the rank of the input matrix within the +given tolerance. -This function uses \ref af::qr to find the rank of the input matrix within the given tolerance. +\ingroup lapack_ops_mat -===================================================================================== +=============================================================================== \defgroup lapack_ops_func_det det -\ingroup lapack_ops_mat +Find the determinant of a matrix. -\brief Find the determinant of the input matrix. +This function requires scratch space equal to the input array. - -\note This function requires scratch space equal to the input array +\ingroup lapack_ops_mat =============================================================================== \defgroup lapack_ops_func_norm norm -\ingroup lapack_ops_mat +Find the norm of a matrix -\brief Find the norm of the input matrix +This function can return the norm using various metrics based on the `type` +parameter. -This function can return the norm using various metrics based on the type paramter. +\ref AF_NORM_MATRIX_2 is currently not supported. -\note \ref AF_NORM_MATRIX_2 is currently not supported. +\ingroup lapack_ops_mat =============================================================================== \defgroup lapack_helper_func_available isLAPACKAvailable -\ingroup lapack_helper +\brief Returns true if ArrayFire is compiled with LAPACK support -\brief Returns true is ArrayFire is compiled with LAPACK support +\ingroup lapack_helper =============================================================================== diff --git a/docs/details/random.dox b/docs/details/random.dox index 63ca846106..d2400fcbbe 100644 --- a/docs/details/random.dox +++ b/docs/details/random.dox @@ -5,7 +5,7 @@ \brief Random Number Generation Functions -Functions to generate and manage random numbers and random number engines +Functions to generate and manage random numbers and random number engines. \ingroup data_mat @@ -16,7 +16,7 @@ Functions to generate and manage random numbers and random number engines \defgroup random_func_random_engine randomEngine -\brief Functions to create, modify, use, and destroy randomEngine objects +\brief Functions to create, modify, use, and destroy randomEngine objects. A \ref af::randomEngine object can be used to generate psuedo random numbers using various types of random number generation algorithms defined by \ref @@ -76,7 +76,7 @@ returned by \ref af_get_default_random_engine. \defgroup random_func_set_seed setSeed -\brief Set the seed for random number generation +\brief Set the seed for random number generation. Sets the seed for the current default random engine. @@ -86,7 +86,7 @@ Sets the seed for the current default random engine. \defgroup random_func_get_seed getSeed -\brief Returns the seed for random number generation +\brief Returns the seed for random number generation. Returns the seed for the current default random engine. diff --git a/docs/doxygen.mk b/docs/doxygen.mk index 914ebb35b4..9f46a1e37b 100644 --- a/docs/doxygen.mk +++ b/docs/doxygen.mk @@ -1,4 +1,4 @@ -# Doxyfile 1.9.6 +# Doxyfile 1.9.7 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -377,6 +377,17 @@ MARKDOWN_SUPPORT = YES TOC_INCLUDE_HEADINGS = 0 +# The MARKDOWN_ID_STYLE tag can be used to specify the algorithm used to +# generate identifiers for the Markdown headings. Note: Every identifier is +# unique. +# Possible values are: DOXYGEN Use a fixed 'autotoc_md' string followed by a +# sequence number starting at 0. and GITHUB Use the lower case version of title +# with any whitespace replaced by '-' and punctations characters removed.. +# The default value is: DOXYGEN. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +MARKDOWN_ID_STYLE = DOXYGEN + # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by putting a % sign in front of the word or @@ -501,6 +512,14 @@ LOOKUP_CACHE_SIZE = 0 NUM_PROC_THREADS = 0 +# If the TIMESTAMP tag is set different from NO then each generated page will +# contain the date or date and time when the page was generated. Setting this to +# NO can help when comparing the output of multiple runs. +# Possible values are: YES, NO, DATETIME and DATE. +# The default value is: NO. + +TIMESTAMP = YES + #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- @@ -886,7 +905,14 @@ WARN_IF_UNDOC_ENUM_VAL = NO # a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS # then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but # at the end of the doxygen process doxygen will return with a non-zero status. -# Possible values are: NO, YES and FAIL_ON_WARNINGS. +# If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS_PRINT then doxygen behaves +# like FAIL_ON_WARNINGS but in case no WARN_LOGFILE is defined doxygen will not +# write the warning messages in between other messages but write them at the end +# of a run, in case a WARN_LOGFILE is defined the warning messages will be +# besides being in the defined file also be shown at the end of a run, unless +# the WARN_LOGFILE is defined as - i.e. standard output (stdout) in that case +# the behavior will remain as with the setting FAIL_ON_WARNINGS. +# Possible values are: NO, YES, FAIL_ON_WARNINGS and FAIL_ON_WARNINGS_PRINT. # The default value is: NO. WARN_AS_ERROR = NO @@ -1012,9 +1038,6 @@ EXCLUDE_PATTERNS = *.cpp # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, # ANamespace::AClass, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* EXCLUDE_SYMBOLS = @@ -1405,15 +1428,6 @@ HTML_COLORSTYLE_SAT = 219 HTML_COLORSTYLE_GAMMA = 70 -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = YES - # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML # documentation will contain a main index with vertical navigation menus that # are dynamically created via JavaScript. If disabled, the navigation index will @@ -1563,6 +1577,16 @@ BINARY_TOC = NO TOC_EXPAND = NO +# The SITEMAP_URL tag is used to specify the full URL of the place where the +# generated documentation will be placed on the server by the user during the +# deployment of the documentation. The generated sitemap is called sitemap.xml +# and placed on the directory specified by HTML_OUTPUT. In case no SITEMAP_URL +# is specified no sitemap is generated. For information about the sitemap +# protocol see https://www.sitemaps.org +# This tag requires that the tag GENERATE_HTML is set to YES. + +SITEMAP_URL = + # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help @@ -2051,9 +2075,16 @@ PDF_HYPERLINKS = YES USE_PDFLATEX = YES -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode -# command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. +# The LATEX_BATCHMODE tag ignals the behavior of LaTeX in case of an error. +# Possible values are: NO same as ERROR_STOP, YES same as BATCH, BATCH In batch +# mode nothing is printed on the terminal, errors are scrolled as if is +# hit at every error; missing files that TeX tries to input or request from +# keyboard input (\read on a not open input stream) cause the job to abort, +# NON_STOP In nonstop mode the diagnostic message will appear on the terminal, +# but there is no possibility of user interaction just like in batch mode, +# SCROLL In scroll mode, TeX will stop only for missing files to input or if +# keyboard input is necessary and ERROR_STOP In errorstop mode, TeX will stop at +# each error, asking for user intervention. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -2074,14 +2105,6 @@ LATEX_HIDE_INDICES = NO LATEX_BIB_STYLE = plain -# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated -# page will contain the date and time when the page was generated. Setting this -# to NO can help when comparing the output of multiple runs. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_TIMESTAMP = NO - # The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute) # path from which the emoji images will be read. If a relative path is entered, # it will be relative to the LATEX_OUTPUT directory. If left blank the @@ -2247,7 +2270,7 @@ DOCBOOK_OUTPUT = docbook #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an -# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures +# AutoGen Definitions (see https://autogen.sourceforge.net/) file that captures # the structure of the code including all documentation. Note that this feature # is still experimental and incomplete at the moment. # The default value is: NO. @@ -2422,16 +2445,9 @@ EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES #--------------------------------------------------------------------------- -# Configuration options related to the dot tool +# Configuration options related to diagram generator tools #--------------------------------------------------------------------------- -# You can include diagrams made with dia in doxygen documentation. Doxygen will -# then run dia to produce the diagram and insert it in the documentation. The -# DIA_PATH tag allows you to specify the directory where the dia binary resides. -# If left empty dia is assumed to be found in the default search path. - -DIA_PATH = - # If set to YES the inheritance and collaboration graphs will hide inheritance # and usage relations if the target is undocumented or is not a class. # The default value is: YES. @@ -2440,7 +2456,7 @@ HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz (see: -# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# https://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent # Bell Labs. The other options in this section have no effect if this option is # set to NO # The default value is: NO. @@ -2493,13 +2509,15 @@ DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" DOT_FONTPATH = -# If the CLASS_GRAPH tag is set to YES (or GRAPH) then doxygen will generate a -# graph for each documented class showing the direct and indirect inheritance -# relations. In case HAVE_DOT is set as well dot will be used to draw the graph, -# otherwise the built-in generator will be used. If the CLASS_GRAPH tag is set -# to TEXT the direct and indirect inheritance relations will be shown as texts / -# links. -# Possible values are: NO, YES, TEXT and GRAPH. +# If the CLASS_GRAPH tag is set to YES or GRAPH or BUILTIN then doxygen will +# generate a graph for each documented class showing the direct and indirect +# inheritance relations. In case the CLASS_GRAPH tag is set to YES or GRAPH and +# HAVE_DOT is enabled as well, then dot will be used to draw the graph. In case +# the CLASS_GRAPH tag is set to YES and HAVE_DOT is disabled or if the +# CLASS_GRAPH tag is set to BUILTIN, then the built-in generator will be used. +# If the CLASS_GRAPH tag is set to TEXT the direct and indirect inheritance +# relations will be shown as texts / links. +# Possible values are: NO, YES, TEXT, GRAPH and BUILTIN. # The default value is: YES. CLASS_GRAPH = YES @@ -2640,7 +2658,7 @@ DIR_GRAPH_MAX_DEPTH = 1 # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. For an explanation of the image formats see the section # output formats in the documentation of the dot tool (Graphviz (see: -# http://www.graphviz.org/)). +# https://www.graphviz.org/)). # Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order # to make the SVG files visible in IE 9+ (other browsers do not have this # requirement). @@ -2677,11 +2695,12 @@ DOT_PATH = DOTFILE_DIRS = -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the \mscfile -# command). +# You can include diagrams made with dia in doxygen documentation. Doxygen will +# then run dia to produce the diagram and insert it in the documentation. The +# DIA_PATH tag allows you to specify the directory where the dia binary resides. +# If left empty dia is assumed to be found in the default search path. -MSCFILE_DIRS = +DIA_PATH = # The DIAFILE_DIRS tag can be used to specify one or more directories that # contain dia files that are included in the documentation (see the \diafile @@ -2758,3 +2777,19 @@ GENERATE_LEGEND = YES # The default value is: YES. DOT_CLEANUP = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. If the MSCGEN_TOOL tag is left empty (the default), then doxygen will +# use a built-in version of mscgen tool to produce the charts. Alternatively, +# the MSCGEN_TOOL tag can also specify the name an external tool. For instance, +# specifying prog as the value, doxygen will call the tool as prog -T +# -o . The external tool should support +# output file formats "png", "eps", "svg", and "ismap". + +MSCGEN_TOOL = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the \mscfile +# command). + +MSCFILE_DIRS = diff --git a/docs/pages/README.md b/docs/pages/README.md index d20dc6b246..6ecb68ce4e 100644 --- a/docs/pages/README.md +++ b/docs/pages/README.md @@ -5,12 +5,14 @@ Overview {#mainpage} ## About ArrayFire -ArrayFire is a high performance software library for parallel computing with an easy-to-use API. Its array based function set makes parallel programming more accessible. +ArrayFire is a high performance software library for parallel computing with +an easy-to-use API. Its array based function set makes parallel programming +more accessible. ## Installing ArrayFire -You can install ArrayFire using either a binary installer for Windows, OSX, -or Linux or download it from source: +Install ArrayFire using either a binary installer for Windows, OSX, or Linux +or download it from source: * [Binary installers for Windows, OSX, and Linux](\ref installing) * [Build from source](https://github.com/arrayfire/arrayfire) @@ -20,18 +22,18 @@ or Linux or download it from source: The [array](\ref af::array) object is beautifully simple. Array-based notation effectively expresses computational algorithms in -readable math-resembling notation. You _do not_ need expertise in -parallel programming to use ArrayFire. +readable math-resembling notation. Expertise in parallel programming _is not_ +required to use ArrayFire. -A few lines of ArrayFire code -accomplishes what can take 100s of complicated lines in CUDA or OpenCL -kernels. +A few lines of ArrayFire code accomplishes what can take 100s of complicated +lines in CUDA, oneAPI, or OpenCL kernels. ## ArrayFire is extensive! #### Support for multiple domains -ArrayFire contains [hundreds of functions](\ref arrayfire_func) across various domains including: +ArrayFire contains [hundreds of functions](\ref arrayfire_func) across various +domains including: - [Vector Algorithms](\ref vector_mat) - [Image Processing](\ref image_mat) - [Computer Vision](\ref cv_mat) @@ -40,61 +42,65 @@ ArrayFire contains [hundreds of functions](\ref arrayfire_func) across various d - [Statistics](\ref stats_mat) - and more. -Each function is hand-tuned by ArrayFire -developers with all possible low-level optimizations. +Each function is hand-tuned by ArrayFire developers with all possible +low-level optimizations. #### Support for various data types and sizes -ArrayFire operates on common [data shapes and sizes](\ref indexing), -including vectors, matrices, volumes, and +ArrayFire operates on common [data shapes and sizes](\ref indexing), including +vectors, matrices, volumes, and -It supports common [data types](\ref gettingstarted_datatypes), -including single and double precision floating -point values, complex numbers, booleans, and 32-bit signed and -unsigned integers. +It supports common [data types](\ref gettingstarted_datatypes), including +single and double precision floating point values, complex numbers, booleans, +and 8/16/32-bit signed and unsigned integers. #### Extending ArrayFire -ArrayFire can be used as a stand-alone application or integrated with -existing CUDA or OpenCL code. All ArrayFire `arrays` can be -interchanged with other CUDA or OpenCL data structures. +ArrayFire can be used as a stand-alone application or integrated with existing +CUDA, oneAPI, or OpenCL code. ## Code once, run anywhere! -With support for x86, ARM, CUDA, and OpenCL devices, ArrayFire supports for a comprehensive list of devices. +With support for x86, ARM, CUDA, oneAPI, and OpenCL devices, ArrayFire +supports for a comprehensive list of devices. Each ArrayFire installation comes with: - - a CUDA version (named 'libafcuda') for [NVIDIA - GPUs](https://developer.nvidia.com/cuda-gpus), - - an OpenCL version (named 'libafopencl') for [OpenCL devices](http://www.khronos.org/conformance/adopters/conformant-products#opencl) - - a CPU version (named 'libafcpu') to fall back to when CUDA or OpenCL devices are not available. +- a CUDA backend (named 'libafcuda') for [NVIDIA + GPUs](https://developer.nvidia.com/cuda-gpus), +- a oneAPI backend (named 'libafoneapi') for [oneAPI + devices](https://www.intel.com/content/www/us/en/developer/articles/system-requirements/intel-oneapi-base-toolkit-system-requirements.html), +- an OpenCL backend (named 'libafopencl') for [OpenCL + devices](http://www.khronos.org/conformance/adopters/conformant-products#opencl), +- a CPU backend (named 'libafcpu') to fall back to when CUDA, oneAPI, or + OpenCL devices are unavailable. ## ArrayFire is highly efficient #### Vectorized and Batched Operations -ArrayFire supports batched operations on N-dimensional arrays. -Batch operations in ArrayFire are run in parallel ensuring an optimal usage of your CUDA or OpenCL device. +ArrayFire supports batched operations on N-dimensional arrays. Batch +operations in ArrayFire are run in parallel ensuring an optimal usage of CUDA, +oneAPI, or OpenCL devices. -You can get the best performance out of ArrayFire using [vectorization techniques](\ref vectorization). +Best performance with ArrayFire is achieved using +[vectorization techniques](\ref vectorization). ArrayFire can also execute loop iterations in parallel with [the gfor function](\ref gfor). #### Just in Time compilation -ArrayFire performs run-time analysis of your code to increase -arithmetic intensity and memory throughput, while avoiding unnecessary -temporary allocations. It has an awesome internal JIT compiler to make -optimizations for you. +ArrayFire performs run-time analysis of code to increase arithmetic intensity +and memory throughput, while avoiding unnecessary temporary allocations. It +has an awesome internal JIT compiler to make important optimizations. -Read more about how [ArrayFire JIT](http://arrayfire.com/performance-of-arrayfire-jit-code-generation/) can improve the performance in your application. +Read more about how [ArrayFire JIT](\ref jit). can improve the performance in +your application. ## Simple Example -Here's a live example to let you see ArrayFire code. You create [arrays](\ref af::array) -which reside on CUDA or OpenCL devices. Then you can use -[ArrayFire functions](modules.htm) on those [arrays](\ref af::array). +Here is an example of ArrayFire code that performs a Monte Carlo estimation of +PI. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} // sample 40 million points on the GPU @@ -111,17 +117,19 @@ af_print(pi); #### Free Community Options -* [ArrayFire mailing list](https://groups.google.com/forum/#!forum/arrayfire-users) (recommended) +* [ArrayFire mailing + list](https://groups.google.com/forum/#!forum/arrayfire-users) (recommended) * [StackOverflow](http://stackoverflow.com/questions/tagged/arrayfire) #### Premium Support -* Phone Support - available for purchase ([request a quote](mailto:sales@arrayfire.com)) +* Phone Support - available for purchase ([request a + quote](mailto:sales@arrayfire.com)) #### Contact Us -* If you need to contact us, visit our -[contact us page](http://arrayfire.com/company/#contact). +* If you need to contact us, visit our [contact us + page](http://arrayfire.com/company/#contact). #### Email @@ -130,9 +138,10 @@ af_print(pi); ## Citations and Acknowledgements -If you redistribute ArrayFire, please follow the terms established in the license. -If you wish to cite ArrayFire in an academic publication, please use the -following reference: +If you redistribute ArrayFire, please follow the terms established in the +license. If you wish to cite ArrayFire in an academic publication, please +use the following reference: Formatted: @@ -153,4 +162,6 @@ BibTeX: year = {2015} } -ArrayFire development is funded by ArrayFire LLC and several third parties, please see the list of acknowledgements. +ArrayFire development is funded by AccelerEyes LLC (dba ArrayFire) and several +third parties, please see the list of acknowledgements. diff --git a/docs/pages/configuring_arrayfire_environment.md b/docs/pages/configuring_arrayfire_environment.md index fd11628105..7b20be9b4a 100644 --- a/docs/pages/configuring_arrayfire_environment.md +++ b/docs/pages/configuring_arrayfire_environment.md @@ -38,6 +38,16 @@ variable are the device identifiers shown when af::info is run. AF_CUDA_DEFAULT_DEVICE=1 ./myprogram_cuda ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +AF_ONEAPI_DEFAULT_DEVICE {#af_oneapi_default_device} +------------------------------------------------------------------------------- + +Use this variable to set the default oneAPI device. Valid values for this +variable are the device identifiers shown when af::info is run. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +AF_ONEAPI_DEFAULT_DEVICE=1 ./myprogram_oneapi +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Note: af::setDevice call in the source code will take precedence over this variable. diff --git a/docs/pages/getting_started.md b/docs/pages/getting_started.md index d958892c2e..2bd3b4d1f6 100644 --- a/docs/pages/getting_started.md +++ b/docs/pages/getting_started.md @@ -24,10 +24,12 @@ can represent one of many different [basic data types](\ref af_dtype): * [c32](\ref c32) complex single-precision (`cfloat`) * [f64](\ref f64) real double-precision (`double`) * [c64](\ref c64) complex double-precision (`cdouble`) +* [f16](\ref f16) real half-precision (`half_float::half`) * [b8](\ref b8) 8-bit boolean values (`bool`) * [s32](\ref s32) 32-bit signed integer (`int`) * [u32](\ref u32) 32-bit unsigned integer (`unsigned`) -* [u8](\ref u8) 8-bit unsigned values (`unsigned char`) +* [s8](\ref s8) 8-bit signed integer (`signed char`) +* [u8](\ref u8) 8-bit unsigned integer (`unsigned char`) * [s64](\ref s64) 64-bit signed integer (`intl`) * [u64](\ref u64) 64-bit unsigned integer (`uintl`) * [s16](\ref s16) 16-bit signed integer (`short`) @@ -153,11 +155,11 @@ using the `af::` namespace. # Indexing {#getting_started_indexing} -Like all functions in ArrayFire, indexing is also executed in parallel on -the OpenCL/CUDA device. -Because of this, indexing becomes part of a JIT operation and is accomplished -using parentheses instead of square brackets (i.e. as `A(0)` instead of `A[0]`). -To index `af::array`s you may use one or a combination of the following functions: +Like all functions in ArrayFire, indexing is also executed in parallel on the +OpenCL/CUDA devices. Because of this, indexing becomes part of a JIT operation +and is accomplished using parentheses instead of square brackets (i.e. as `A(0)` +instead of `A[0]`). To index `af::array`s you may use one or a combination of +the following functions: * integer scalars * [seq()](\ref af::seq) representing a linear sequence @@ -223,7 +225,7 @@ simply include the `arrayfire.h` header file and start coding! double result; af_sum_all(&result, 0, a); printf("sum: %g\n", result); - + return 0; } ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/pages/gfor.md b/docs/pages/gfor.md index e6886b5bb4..bbced5d14b 100644 --- a/docs/pages/gfor.md +++ b/docs/pages/gfor.md @@ -8,18 +8,17 @@ Run many independent loops simultaneously on the GPU or device. Introduction {#gfor_intro} ============ -The gfor-loop construct may be used to simultaneously launch all of -the iterations of a for-loop on the GPU or device, as long as the -iterations are independent. While the standard for-loop performs each -iteration sequentially, ArrayFire's gfor-loop performs each iteration -at the same time (in parallel). ArrayFire does this by tiling out the -values of all loop iterations and then performing computation on those -tiles in one pass. - -You can think of `gfor` as performing auto-vectorization of your -code, e.g. you write a gfor-loop that increments every element of a -vector but behind the scenes ArrayFire rewrites it to operate on -the entire vector in parallel. +The gfor-loop construct may be used to simultaneously launch all of the +iterations of a for-loop on the GPU or device, as long as the iterations are +independent. While the standard for-loop performs each iteration sequentially, +ArrayFire's gfor-loop performs each iteration at the same time (in +parallel). ArrayFire does this by tiling out the values of all loop iterations +and then performing computation on those tiles in one pass. + +You can think of `gfor` as performing auto-vectorization of your code, +e.g. you write a gfor-loop that increments every element of a vector but +behind the scenes ArrayFire rewrites it to operate on the entire vector in +parallel. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} for (int i = 0; i < n; ++i) @@ -29,19 +28,19 @@ gfor (seq i, n) A(i) = A(i) + 1; ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Behind the scenes, ArrayFire rewrites your code into this -equivalent and faster version: +Behind the scenes, ArrayFire rewrites your code into this equivalent and +faster version: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} A = A + 1; ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -It is best to vectorize computation as much as possible to avoid -the overhead in both for-loops and gfor-loops. +It is best to vectorize computation as much as possible to avoid the overhead +in both for-loops and gfor-loops. -To see another example, you could run an FFT on every 2D slice of a -volume in a for-loop, or you could "vectorize" and simply do it all -in one gfor-loop operation: +To see another example, you could run an FFT on every 2D slice of a volume in +a for-loop, or you could "vectorize" and simply do it all in one gfor-loop +operation: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} for (int i = 0; i < N; ++i) @@ -89,11 +88,11 @@ User Functions called within GFOR {#gfor_user_functions} --------------------------------- If you have defined a function that you want to call within a GFOR loop, then -that function has to meet all the conditions described in this page in -order to be able to work as expected. +that function has to meet all the conditions described in this page in order +to be able to work as expected. -Consider the (trivial) example below. The function compute() has to satisfy all -requirements for GFOR Usage, so you cannot use if-else conditions inside +Consider the (trivial) example below. The function compute() has to satisfy +all requirements for GFOR Usage, so you cannot use if-else conditions inside it. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} @@ -384,7 +383,8 @@ gfor (seq i, n) { } ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The problem is that every GFOR tile has a different number of elements, something which GFOR cannot yet handle. +The problem is that every GFOR tile has a different number of elements, +something which GFOR cannot yet handle. Similar to the workaround for conditional statements, it might work to use masked arithmetic: @@ -410,14 +410,13 @@ gfor (seq i, n) { Memory considerations {#gfor_memory} ===================== -Since each computation is done in parallel for all iterator values, -you need to have enough card memory available to do all iterations -simultaneously. If the problem exceeds memory, it will trigger "out of -memory" errors. +Since each computation is done in parallel for all iterator values, you need +to have enough card memory available to do all iterations simultaneously. If +the problem exceeds memory, it will trigger "out of memory" errors. -You can work around the memory limitations of your GPU or device by -breaking the GFOR loop up into segments; however, you might want to -consider using a larger memory GPU or device. +You can work around the memory limitations of your GPU or device by breaking +the GFOR loop up into segments; however, you might want to consider using a +larger memory GPU or device. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} // BEFORE diff --git a/docs/pages/install.md b/docs/pages/install.md index 7a78b95f71..01b268af34 100644 --- a/docs/pages/install.md +++ b/docs/pages/install.md @@ -1,24 +1,14 @@ # ArrayFire Installer {#installing} Installing ArrayFire couldn't be easier. Navigate to -https://arrayfire.com/download and download the installer for your architecture -and operating system. Although you could [build ArrayFire from -source](https://github.com/arrayfire/arrayfire), we recommend using our -installers as we have packaged together all of the necessary dependencies to -give you the best performance. - -We provide installers for Windows, Linux, and macOS. There are two installers -for each operating system: one with graphics support and the other without -graphics support. Download the installer with graphics support if you would like -to be able to do high performance visualizations using our -[Forge](https://github.com/arrayfire/forge) library. Otherwise, download the -installer without graphics support. - -Make sure you have the latest device drivers installed on your system before -using ArrayFire. If you are going to be targeting the CPU using ArrayFire’s -OpenCL backend, you will need to have the OpenCL **runtime** installed on your -system. Drivers and runtimes should be downloaded and installed from your device -vendor’s website. +https://arrayfire.com/download and download the appropriate installer for the +target architecture and operating system. Although ArrayFire can be [built +from source](https://github.com/arrayfire/arrayfire), the installers +conveniently package necessary dependencies. + +Install the latest device drivers before using ArrayFire. If you target the +CPU using ArrayFire’s OpenCL backend, install the OpenCL runtime. Drivers and +runtimes should be downloaded and installed from each device vendor's website. # Install Instructions {#InstallInstructions} @@ -28,15 +18,11 @@ vendor’s website. ## Windows {#Windows} -Prior to installing ArrayFire on Windows, -[download](https://www.microsoft.com/en-in/download/details.aspx?id=48145) -install the Visual Studio 2015 (x64) runtime libraries. +Once the ArrayFire has been downloaded, run the installer. -Once you have downloaded the ArrayFire installer, execute the installer as you -normally would on Windows. If you choose not to modify the path during the -installation procedure, you'll need to manually add ArrayFire to the path for -all users. Simply append `%%AF_PATH%/lib` to the PATH variable so that the loader -can find ArrayFire DLLs. +The installer offers the option to automatically add ArrayFire to the path for +all users. If the installer did not do this, simply append `%%AF_PATH%/lib` to +the PATH variable so that the loader can find ArrayFire DLLs. For more information on using ArrayFire on Windows, visit the following [page](http://arrayfire.org/docs/using_on_windows.htm). @@ -45,41 +31,42 @@ For more information on using ArrayFire on Windows, visit the following There are two ways to install ArrayFire on Linux. 1. Package Manager -2. Using ArrayFire Linux Installer +2. Using the ArrayFire Linux Installer -As of today, approach (1) is only supported for Ubuntu 18.04 and 20.04. Please go -through [our GitHub wiki page](https://github.com/arrayfire/arrayfire/wiki/Install-ArrayFire-From-Linux-Package-Managers) -for the detailed instructions. +As of today, approach (1) is only supported for Ubuntu 18.04 and 20.04. Please +go through [the GitHub +wiki[page](https://github.com/arrayfire/arrayfire/wiki/Install-ArrayFire-From-Linux-Package-Managers) +for detailed instructions. -For approach (2), once you have downloaded the ArrayFire installer, execute the +For approach (2), once the ArrayFire installer is downloaded, execute the installer from the terminal as shown below. Set the `--prefix` argument to the -directory you would like to install ArrayFire to - we recommend `/opt`. +target install directory; we recommend `/opt`. - ./Arrayfire_*_Linux_x86_64.sh --include-subdir --prefix=/opt + ./ArrayFire_*_Linux_x86_64.sh --include-subdir --prefix=/opt -Given sudo permissions, you can add the ArrayFire libraries via `ldconfig` like -so: +Given sudo permissions, the ArrayFire libraries can be added to the path via +`ldconfig` like so: echo /opt/arrayfire/lib64 > /etc/ld.so.conf.d/arrayfire.conf sudo ldconfig -Otherwise, you will need to set the `LD_LIBRARY_PATH` environment variable in -order to let your shared library loader find the ArrayFire libraries. +Otherwise, the `LD_LIBRARY_PATH` environment variable can be set so that the +shared library loader can find the ArrayFire libraries. For more information on using ArrayFire on Linux, visit the following [page](http://arrayfire.org/docs/using_on_linux.htm). ### Graphics support -ArrayFire allows you to do high performance visualizations via our +ArrayFire enables high-performance visualizations via the [Forge](https://github.com/arrayfire/forge) library. On Linux, there are a few -dependencies you will need to install to enable graphics support: +dependencies to install to enable graphics support: -FreeImage -Fontconfig -GLU (OpenGL Utility Library) +* FreeImage +* Fontconfig +* GLU (OpenGL Utility Library) -We show how to install these dependencies on common Linux distributions: +To install these dependencies on common Linux distributions: __Debian, Ubuntu (14.04 and above), and other Debian derivatives__ @@ -92,9 +79,9 @@ __Fedora, Redhat, CentOS__ ## macOS {#macOS} -Once you have downloaded the ArrayFire installer, execute the installer by -either double clicking on the ArrayFire `pkg` file or running the following -command from your terminal: +Once the ArrayFire installer has been downloaded, execute the installer by +either double-clicking on the ArrayFire `pkg` file or running the following +command: sudo installer -pkg Arrayfire-*_OSX.pkg -target / @@ -103,11 +90,10 @@ For more information on using ArrayFire on macOS, visit the following ## NVIDIA Tegra devices -ArrayFire is capable of running on TX1 and TX2 devices. The TK1 is no longer -supported. +ArrayFire is capable of running TX2 devices. -Prior to installing ArrayFire, make sure you have the latest version of JetPack -(v2.3 and above) or L4T (v24.2 and above) on your device. +Before installing ArrayFire, make sure the latest version of JetPack (v2.3 and +above) or L4T (v24.2 and above) is installed. ### Tegra prerequisites @@ -117,26 +103,25 @@ The following dependencies are required for Tegra devices: ## Testing installation -After ArrayFire is finished installing, we recommend building and running a few -of the provided examples to verify things are working as expected. +After ArrayFire is finished installing, we recommend building and running a +few of the provided examples to verify things are working as expected. + +On Windows, open the CMakeLists.txt file from CMake-GUI. Once the project is +configured and generated, build and run the examples from Visual Studio. -On Unix-like systems: +On Linux, run the following commands: cp -r /opt/arrayfire/share/ArrayFire/examples /tmp/examples cd /tmp/examples mkdir build cd build - cmake -DASSETS_DIR:PATH=/tmp .. + cmake .. make - ./helloworld/helloworld_{cpu,cuda,opencl} - -On Windows, open the CMakeLists.txt file from CMake-GUI and set `ASSETS_DIR` -variable to the parent folder of examples folder. Once the project is configured -and generated, you can build and run the examples from Visual Studio. + ./helloworld/helloworld_{cpu,cuda,oneapi,opencl} ## Getting help * Google Groups: https://groups.google.com/forum/#!forum/arrayfire-users -* ArrayFire Services: [Consulting](https://arrayfire.com/consulting/) | [Support](https://arrayfire.com/support/) | [Training](https://arrayfire.com/training/) +* ArrayFire Services: [Consulting](https://arrayfire.com/consulting/) | [Training](https://arrayfire.com/training/) * ArrayFire Blogs: http://arrayfire.com/blog/ -* Email: +* Email: diff --git a/docs/pages/interop_cuda.md b/docs/pages/interop_cuda.md index dae46ae027..2132dfcb2c 100644 --- a/docs/pages/interop_cuda.md +++ b/docs/pages/interop_cuda.md @@ -80,8 +80,7 @@ int main() { // 5. Determine ArrayFire's CUDA stream int af_id = af::getDevice(); - int cuda_id = afcu::getNativeId(af_id); - cudaStream_t af_cuda_stream = afcu::getStream(cuda_id); + cudaStream_t af_cuda_stream = afcu::getStream(af_id); // 6. Set arguments and run your kernel in ArrayFire's stream // Here launch with 1 block of 10 threads diff --git a/docs/pages/jit.md b/docs/pages/jit.md new file mode 100644 index 0000000000..8b5c783755 --- /dev/null +++ b/docs/pages/jit.md @@ -0,0 +1,102 @@ +ArrayFire JIT Code Generation {#jit} +================ + +The ArrayFire library offers JIT (Just In Time) compiling for elementwise +arithmetic operations. This includes trigonometric functions, comparisons, and +element-wise operations. + +At runtime, ArrayFire aggregates these function calls using an Abstract Syntax +Tree (AST) data structure such that whenever a JIT-supported function is +called, it is added into the AST for a given variable instance. The AST of the +variable is computed if one of the following conditions is met: + +* an explication evaluation is required by the programmer using the + [eval](\ref af::eval) function, or +* the variable is required to compute a different variable that is not + JIT-supported. + +When the above occurs, and the variable needs to be evaluated, the functions +and variables in the AST data structure are used to create a single +kernel. This is done by creating a customized kernel on-the-fly that is made +up of all the functions in the AST. The customized function is then executed. + +This JIT compilation technique has multiple benefits: + +* A reduced number of kernel calls – a kernel call can be a significant + overhead for small data sets. +* Better cache performance – there are many instances in which the memory + required by a single element in the array can be reused multiple times, or + the temporary value of a computation can be stored in the cache and reused + by future computations. +* Temporary memory allocation and write-back can be reduced – when multiple + expressions are evaluated and stored into temporary arrays, these arrays + need to be allocated and the results written back to main memory. +* Avoid computing elements that are not used – there are cases in which the + AST is created for a variable; however, the expression is not used later in + the computation. Thus, its evaluation can be avoided. +* Better performance – all the above can help reduce the total execution time. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} +// As JIT is automatically enabled in ArrayFire, this version of the function +// forces each expression to be evaluated. If the eval() function calls are +// removed, then the execution of this code would be equivalent to the +// following function. + +static double pi_no_jit(array x, array y, array temp, int samples) { + temp = x * x; + temp.eval(); + temp += y * y; + temp.eval(); + temp = sqrt(temp); + temp.eval(); + temp = temp < 1; + temp.eval(); + return 4.0 sum(temp)/samples; +} + +static double pi_jit(array x, array y, array temp,int samples){ + temp = sqrt(x*x + y*y) < 1; + temp.eval(); + return 4.0 * sum(temp) / samples; +} +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The above code computes the value of π using a Monte-Carlo simulation where +points are randomly generated within the unit square. Each point is tested to +see if it is within the unit circle. The ratio of points within the circle and +square approximate the value π. The accuracy of π improves as the number of +samples is increased, which motivates using additional samples. + +There are two implementations above: +1. an implementation that does not benefit from the JIT (pi\_no\_jit), and +2. an implementation that takes advantage of the JIT feature (pi\_jit). + +Specifically, as JIT is an integral feature of the ArrayFire library, it +cannot simply be turned on and off. The only way for a programmer to sidestep +the JIT operations is to manually force the evaluation of expressions. This is +done in the non-JIT-supported implementation. + +Timing these two implementations results in the following performance +benchmark: + +Performance of JIT and Non-JIT implementations + + +The above figure depicts the execution time (abscissa) as a function of the +number of samples (ordinate) for the two implementations discussed above. + +When the number of samples is small, the execution time of pi\_no\_jit is +dominated by the launch of multiple kernels and the execution time pi\_jit is +dominated by on-the-fly compilation of the JIT code required to launch a +single kernel. Even with this JIT compilation time, pi\_jit outperforms +pi_no_jit by 1.4-2.0X for smaller sample sizes. + +When the number of samples is large, both the kernel launch overhead and the +JIT code creation are no longer the limiting factors – the kernel’s +computational load dominates the execution time. Here, the pi\_jit outperforms +pi\_no\_jit by 2.0-2.7X. + +The number of applications that benefit from the JIT code generation is +significant. The actual performance benefits are also application-dependent. + diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index bc40f2a7b7..525542246f 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -1,6 +1,125 @@ Release Notes {#releasenotes} ============== +v3.10.0 +====== + +## Improvements +- Added signed int8 support \PR{3661} \PR{3508} \PR{3507} \PR{3503} +- Increased support for half (fp16) \PR{3680} \PR{3258} \PR{3561} \PR{3627} \PR{3561} \PR{3627} \PR{3559} +- Updated oneAPI to use Intel oneAPI (R) 2025.1 \PR{3643} \PR{3573} +- Updated cl2hpp dependency \PR{3651} \pr{3562} +- Add support for CUDA 12.3, 12.4, 12.5, 12.6, 12.8, and 12.9 \PR{3657} \PR{3645} \PR{3641} \PR{3636} \PR{3588} \PR{3552} \PR{3586} \PR{3541} +- Added minimum driver version check for CUDA GPUs \PR{3648} +- Add more examples \PR{3530} \PR{3455} \PR{3375} \PR{3612} \PR{3584} \PR{3577} +- Updated documentation \PR{3496} \PR{3613} +- Improved performance of matrix multiplication of sparse matrices on the OpenCL backend \PR{3608} +- Improved cmake configure \PR{3581} \PR{3569} \PR{3567} \PR{3564} \PR{3554} +- Loosen indexing assertions for assignments \PR{3514} + +## Fixes +- Fix jit tree when doing operations containing moddims and original array \PR{3671} +- Fix incorrect behavior of sub-arrays with multiple functions \PR{3679} \PR{3668} \PR{3666} \PR{3665} \PR{3664} \PR{3663} \PR{3658} \PR{3659} \PR{3650} \PR{3611} \PR{3633} \PR{3602} +- Fix half precision operations in multiple backends \PR{3676} \PR{3662} +- Fix for join not always respecting the order of parameters \PR{3667} \PR{3513} +- Fix for cmake building as an external project (needed by arrayfire python wheels) \PR{3669} +- Fix for cmake build in Windows (including with vcpkg) \PR{3655} \PR{3646} \PR{3644} \PR{3512} \PR{3626} \PR{3566} \PR{3557} \pr{3591} \PR{3592} +- Fix race condition in OpenCL flood fill \PR{3535} +- Fix indexing array using sequences `af_seq` that have non-unit steps \PR{3587} +- Fix padding issue convolve2GradientNN \PR{3519} +- Fix incorrect axis values for histogram \PR{3590} +- Fix unified exceptions errors \PR{3617} +- Fix OpenCL memory migration on devices with different contexts \PR{3510} +- Fix conversion of COO Sparse to Dense matrix \PR{3589} \PR{3579} +- Fix `AF_JIT_KERNEL_TRACE` on Windows \PR{3517} +- Fix cmake build with CUDNN \PR{3521} +- Fix cmake build with `AF_DISABLE_CPU_ASYNC` \PR{3551} + + +## Contributions + +Special thanks to our contributors: +[Willy Born](https://github.com/willyborn) +[verstatx](https://github.com/verstatx) +[Filip Matzner](https://github.com/FloopCZ) +[Fraser Cormack](https://github.com/frasercrmck) +[errata-c](https://github.com/errata-c) +[Tyler Hilbert](https://github.com/Tyler-Hilbert) + +v3.9.0 +====== + +## Improvements +- Add oneAPI backend \PR{3296} +- Add support to directly access arrays on other devices \PR{3447} +- Add broadcast support \PR{2871} +- Improve OpenCL CPU JIT performance \PR{3257} \PR{3392} +- Optimize thread/block calculations of several kernels \PR{3144} +- Add support for fast math compiliation when building ArrayFire \PR{3334} + \PR{3337} +- Optimize performance of fftconvolve when using floats \PR{3338} +- Add support for CUDA 12.1 and 12.2 +- Better handling of empty arrays \PR{3398} +- Better handling of memory in linear algebra functions in OpenCL \PR{3423} +- Better logging with JIT kernels \PR{3468} +- Optimize memory manager/JIT interactions for small number of buffers + \PR{3468} +- Documentation improvements \PR{3485} +- Optimize reorder function \PR{3488} + +## Fixes +- Improve Errors when creating OpenCL contexts from devices \PR{3257} +- Improvements to vcpkg builds \PR{3376} \PR{3476} +- Fix reduce by key when nan's are present \PR{3261} +- Fix error in convolve where the ndims parameter was forced to be equal to 2 + \PR{3277} +- Make constructors that accept dim_t to be explicit to avoid invalid + conversions \PR{3259} +- Fix error in randu when compiling against clang 14 \PR{3333} +- Fix bug in OpenCL linear algebra functions \PR{3398} +- Fix bug with thread local variables when device was changed \PR{3420} + \PR{3421} +- Fix bug in qr related to uninitialized memory \PR{3422} +- Fix bug in shift where the array had an empty middle dimension \PR{3488} + +## Contributions + +Special thanks to our contributors: +[Willy Born](https://github.com/willyborn) +[Mike Mullen](https://github.com/mfzmullen) + + +v3.8.3 +====== + +## Improvements + +- Add support for CUDA 12 \PR{3352} +- Modernize documentation style and content \PR{3351} +- memcpy performance improvements \PR{3144} +- JIT performance improvements \PR{3144} +- join performance improvements \PR{3144} +- Improve support for Intel and newer Clang compilers \PR{3334} +- CCache support on Windows \PR{3257} + +## Fixes + +- Fix issue with some locales with OpenCL kernel generation \PR{3294} +- Internal improvements +- Fix leak in clfft on exit. +- Fix some cases where ndims was incorrectly used ot calculate shape \PR{3277} +- Fix issue when setDevice was not called in new threads \PR{3269} +- Restrict initializer list to just fundamental types \PR{3264} + +## Contributions + +Special thanks to our contributors: +[Carlo Cabrera](https://github.com/carlocab) +[Guillaume Schmid](https://github.com/GuillaumeSchmid) +[Willy Born](https://github.com/willyborn) +[ktdq](https://github.com/ktdq) + + v3.8.2 ====== @@ -32,6 +151,7 @@ Special thanks to our contributors: [Jacob Kahn](https://github.com/jacobkahn) [Willy Born](https://github.com/willyborn) + v3.8.1 ====== diff --git a/docs/pages/timing.md b/docs/pages/timing.md index fc9b1a725f..8c43808a5c 100644 --- a/docs/pages/timing.md +++ b/docs/pages/timing.md @@ -1,64 +1,153 @@ -Timing Your Code {#timing} +Timing ArrayFire Code {#timing} ================ -timer() : A platform-independent timer with microsecond accuracy: -* [timer::start()](\ref af::timer::start) starts a timer +In performance-sensitive applications, it is vital to profile and measure the +execution time of operations. ArrayFire provides mechanisms to achieve this. -* [timer::start()](\ref af::timer::stop) seconds since last \ref af::timer::start "start" +ArrayFire employs an asynchronous evaluation model for all of its +functions. This means that operations are queued to execute but do not +necessarily complete prior to function return. Hence, directly measuring the +time taken for an ArrayFire function could be misleading. To accurately +measure time, one must ensure the operations are evaluated and synchronize the +ArrayFire stream. -* \ref af::timer::stop(af::timer start) "timer::stop(timer start)" seconds since 'start' +ArrayFire also employs a lazy evaluation model for its elementwise arithmetic +operations. This means operations are not queued for execution until the +result is needed by downstream operations blocking until the operations are +complete. -Example: single timer +The following describes how to time ArrayFire code using the eval and sync +functions along with the timer and timeit functions. A final note on kernel +caching also provides helpful details about ArrayFire runtimes. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} - // start timer - timer::start(); - // run your code - printf("elapsed seconds: %g\n", timer::stop()); -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +## Using ArrayFire eval and sync functions -Example: multiple timers +ArrayFire provides functions to force the evaluation of lazy functions and to +block until all asynchoronous operations complete. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} - // start timers - timer start1 = timer::start(); - timer start2 = timer::start(); - // run some code - printf("elapsed seconds: %g\n", timer::stop(start1)); - // run more code - printf("elapsed seconds: %g\n", timer::stop(start2)); -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +1. The [eval](\ref af::eval) function: -Accurate and reliable measurement of performance involves several factors: -* Executing enough iterations to achieve peak performance. -* Executing enough repetitions to amortize any overhead from system timers. + Forces the evaluation of an ArrayFire array. It ensures the execution of + operations queued up for a specific array. -To take care of much of this boilerplate, [timeit](\ref af::timeit) provides -accurate and reliable estimates of both CPU or GPU code. + It is only required for timing purposes if elementwise arithmetic functions + are called on the array, since these are handled by the ArrayFire JIT. -Here`s a stripped down example of -[Monte-Carlo estimation of PI](\ref benchmarks/pi.cpp) making use -of [timeit](\ref af::timeit). Notice how it expects a `void` function pointer. + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} + af::array A = af::randu(1000, 1000); + af::array B = A + A; // Elementwise arithmetic operation. + B.eval(); // Forces evaluation of B. + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} -#include -#include -using namespace af; + The function initializes the evaluation of the JIT-tree for that array and + may return prior to the completion of those operations. To ensure proper + timing, combine with a [sync](\ref af::sync) function. -void pi_function() { - int n = 20e6; // 20 million random samples - array x = randu(n,f32), y = randu(n,f32); - // how many fell inside unit circle? - float pi = 4.0 * sum(sqrt(x*x + y*y)) < 1) / n; -} +2. The [sync](\ref af::sync) function: -int main() { - printf("pi_function took %g seconds\n", timeit(pi_function)); - return 0; -} -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Synchronizes the ArrayFire stream. It waits for all the previous operations + in the stream to finish. It is often used after [eval](\ref af::eval) to + ensure that operations have indeed been completed. -This produces: + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} + af::sync(); // Waits for all previous operations to complete. + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - pi_function took 0.007252 seconds - (test machine: Core i7 920 @ 2.67GHz with a Tesla C2070) +## Using ArrayFire timer and timeit functions + +ArrayFire provides a simple timer functions that returns the current time in +seconds. + +1. The [timer](\ref af::timer) function: + + timer() : A platform-independent timer with microsecond accuracy: + * [timer::start()](\ref af::timer::start) starts a timer + + * [timer::start()](\ref af::timer::stop) seconds since last \ref + af::timer::start "start" + + * \ref af::timer::stop(af::timer start) "timer::stop(timer start)" seconds + since 'start' + + Example: single timer + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} + // start timer + // - be sure to use the eval and sync functions so that previous code + // does not get timed as part of the execution segment being measured + timer::start(); + // run a code segment + // - be sure to use the eval and sync functions to ensure the code + // segment operations have been completed + // stop timer + printf("elapsed seconds: %g\n", timer::stop()); + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Example: multiple timers + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} + // start timers + // - be sure to use the eval and sync functions so that previous code + // does not get timed as part of the execution segment being measured + timer start1 = timer::start(); + timer start2 = timer::start(); + // run a code segment + // - be sure to use the eval and sync functions to ensure the code + // segment operations have been completed + // stop timer1 + printf("elapsed seconds: %g\n", timer::stop(start1)); + // run another code segment + // - be sure to use the eval and sync functions to ensure the code + // segment operations have been completed + // stop timer2 + printf("elapsed seconds: %g\n", timer::stop(start2)); + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Accurate and reliable measurement of performance involves several factors: + * Executing enough iterations to achieve peak performance. + * Executing enough repetitions to amortize any overhead from system timers. + +2. The [timeit](\ref af::timeit) function: + + To take care of much of this boilerplate, [timeit](\ref af::timeit) provides + accurate and reliable estimates of both CPU or GPU code. + + Here is a stripped down example of [Monte-Carlo estimation of PI](\ref + benchmarks/pi.cpp) making use of [timeit](\ref af::timeit). Notice how it + expects a `void` function pointer. + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp} + #include + #include + using namespace af; + + void pi_function() { + int n = 20e6; // 20 million random samples + array x = randu(n, f32), y = randu(n, f32); + // how many fell inside unit circle? + float pi = 4.0 * sum(sqrt(x*x + y*y)) < 1) / n; + } + + int main() { + printf("pi_function took %g seconds\n", timeit(pi_function)); + return 0; + } + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This produces: + + pi_function took 0.007252 seconds + (test machine: Core i7 920 @ 2.67GHz with a Tesla C2070) + + +## A note on kernel caching + +The first run of ArrayFire code exercises any JIT compilation in the +application, automatically saving a cache of the compilation to +disk. Subsequent runs load the cache from disk, executing without +compilation. Therefore, it is typically best to "warm up" the code with one +run to initiate the application's kernel cache. Afterwards, subsequent runs do +not include the compile time and are tend to be faster than the first run. + +Averaging the time taken is always the best approach and one reason why the +[timeit](\ref af::timeit) function is helpful. diff --git a/docs/pages/tutorials.md b/docs/pages/tutorials.md index f6056b8e19..34b65be12c 100644 --- a/docs/pages/tutorials.md +++ b/docs/pages/tutorials.md @@ -15,4 +15,5 @@ * [Timing ArrayFire](\ref timing) * [Configuring ArrayFire Environment](\ref configuring_environment) * [Debugging ArrayFire Code](\ref debugging) +* [ArrayFire JIT Code Generation](\ref jit) * [GFOR Usage](\ref page_gfor) diff --git a/docs/pages/unified_backend.md b/docs/pages/unified_backend.md index 6924f92707..5a99bff8f4 100644 --- a/docs/pages/unified_backend.md +++ b/docs/pages/unified_backend.md @@ -7,7 +7,7 @@ Unified Backend {#unifiedbackend} The Unified backend was introduced in ArrayFire with version 3.2. While this is not an independent backend, it allows the user to switch between -the different ArrayFire backends (CPU, CUDA and OpenCL) at runtime. +the different ArrayFire backends (CPU, CUDA, oneAPI and OpenCL) at runtime. # Compiling with Unified @@ -24,7 +24,7 @@ To use with CMake, use the __ArrayFire_Unified_LIBRARIES__ variable. # Using the Unified Backend The Unified backend will try to dynamically load the backend libraries. The -priority of backends is __CUDA -> OpenCL -> CPU__ +priority of backends is __CUDA -> oneAPI -> OpenCL -> CPU__ The most important aspect to note here is that all the libraries the ArrayFire libs depend on need to be in the environment paths @@ -78,6 +78,15 @@ int main() fprintf(stderr, "%s\n", e.what()); } + try { + printf("Trying oneAPI Backend\n"); + af::setBackend(AF_BACKEND_ONEAPI); + testBackend(); + } catch (af::exception& e) { + printf("Caught exception when trying oneAPI backend\n"); + fprintf(stderr, "%s\n", e.what()); + } + try { printf("Trying CUDA Backend\n"); af::setBackend(AF_BACKEND_CUDA); @@ -103,39 +112,53 @@ int main() This output would be: Trying CPU Backend - ArrayFire v3.2.0 (CPU, 64-bit Linux, build fc7630f) - [0] Intel: Intel(R) Core(TM) i7-4770K CPU @ 3.50GHz Max threads(8) + ArrayFire v3.9.0 (CPU, 64-bit Linux, build 23ee0650e) + [0] AMD: AMD Ryzen Threadripper PRO 3955WX 16-Cores af::randu(5, 4) + [5 4 1 1] + 0.6010 0.5497 0.1583 0.3636 + 0.0278 0.2864 0.3712 0.4165 + 0.9806 0.3410 0.3543 0.5814 + 0.2126 0.7509 0.6450 0.8962 + 0.0655 0.4105 0.9675 0.3712 + + Trying oneAPI Backend + ArrayFire v3.9.0 (oneAPI, 64-bit Linux, build 23ee0650e) + [0] Intel(R) OpenCL: AMD Ryzen Threadripper PRO 3955WX 16-Cores , 128650 MB (fp64) af::randu(5, 4) [5 4 1 1] - 0.0000 0.2190 0.3835 0.5297 - 0.1315 0.0470 0.5194 0.6711 - 0.7556 0.6789 0.8310 0.0077 - 0.4587 0.6793 0.0346 0.3834 - 0.5328 0.9347 0.0535 0.0668 + 0.6010 0.5497 0.1583 0.3636 + 0.0278 0.2864 0.3712 0.4165 + 0.9806 0.3410 0.3543 0.5814 + 0.2126 0.7509 0.6450 0.8962 + 0.0655 0.4105 0.9675 0.3712 Trying CUDA Backend - ArrayFire v3.2.0 (CUDA, 64-bit Linux, build fc7630f) - Platform: CUDA Toolkit 7.5, Driver: 355.11 - [0] Quadro K5000, 4093 MB, CUDA Compute 3.0 + ArrayFire v3.9.0 (CUDA, 64-bit Linux, build 23ee0650e) + Platform: CUDA Runtime 12.2, Driver: 535.104.05 + [0] NVIDIA RTX A5500, 22721 MB, CUDA Compute 8.6 + -1- NVIDIA RTX A5500, 22719 MB, CUDA Compute 8.6 af::randu(5, 4) [5 4 1 1] - 0.7402 0.4464 0.7762 0.2920 - 0.9210 0.6673 0.2948 0.3194 - 0.0390 0.1099 0.7140 0.8109 - 0.9690 0.4702 0.3585 0.1541 - 0.9251 0.5132 0.6814 0.4452 + 0.6010 0.5497 0.1583 0.3636 + 0.0278 0.2864 0.3712 0.4165 + 0.9806 0.3410 0.3543 0.5814 + 0.2126 0.7509 0.6450 0.8962 + 0.0655 0.4105 0.9675 0.3712 Trying OpenCL Backend - ArrayFire v3.2.0 (OpenCL, 64-bit Linux, build fc7630f) - [0] NVIDIA : Quadro K5000 - -1- INTEL : Intel(R) Core(TM) i7-4770K CPU @ 3.50GHz + ArrayFire v3.9.0 (OpenCL, 64-bit Linux, build 23ee0650e) + [0] NVIDIA: NVIDIA RTX A5500, 22720 MB + -1- NVIDIA: NVIDIA RTX A5500, 22718 MB + -2- Intel(R) FPGA Emulation Platform for OpenCL(TM): Intel(R) FPGA Emulation Device, 128650 MB + -3- INTEL: AMD Ryzen Threadripper PRO 3955WX 16-Cores , 128650 MB af::randu(5, 4) [5 4 1 1] - 0.4107 0.0081 0.6600 0.1046 - 0.8224 0.3775 0.0764 0.8827 - 0.9518 0.3027 0.0901 0.1647 - 0.1794 0.6456 0.5933 0.8060 - 0.4198 0.5591 0.1098 0.5938 + 0.6010 0.5497 0.1583 0.3636 + 0.0278 0.2864 0.3712 0.4165 + 0.9806 0.3410 0.3543 0.5814 + 0.2126 0.7509 0.6450 0.8962 + 0.0655 0.4105 0.9675 0.3712 + # Dos and Don'ts diff --git a/docs/pages/using_on_linux.md b/docs/pages/using_on_linux.md index 0fcd23bba1..91035426c5 100644 --- a/docs/pages/using_on_linux.md +++ b/docs/pages/using_on_linux.md @@ -4,9 +4,9 @@ Using ArrayFire on Linux {#using_on_linux} Once you have [installed](\ref installing) ArrayFire on your system, the next thing to do is set up your build system. On Linux, you can create ArrayFire projects using almost any editor, compiler, or build system. The only -requirements are that you include the ArrayFire header directories and link with -the ArrayFire library you intend to use i.e. CUDA, OpenCL, CPU, or Unified -backends. +requirements are that you include the ArrayFire header directories and link +with the ArrayFire library you intend to use i.e. CUDA, OpenCL, oneAPI, CPU, +or Unified backends. ## The big picture {#big-picture-linux} @@ -15,17 +15,18 @@ installer will populate files in the following sub-directories: include/arrayfire.h - Primary ArrayFire include file include/af/*.h - Additional include files - lib/libaf* - CPU, CUDA, and OpenCL libraries (.a, .so) + lib/libaf* - CPU, CUDA, oneAPI, and OpenCL libraries (.a, .so) lib/libforge* - Visualization library lib/libcu* - CUDA backend dependencies lib/libOpenCL.so - OpenCL ICD Loader library share/ArrayFire/cmake/* - CMake config (find) scripts share/ArrayFire/examples/* - All ArrayFire examples -Because ArrayFire follows standard installation practices, you can use basically -any build system to create and compile projects that use ArrayFire. Among the -many possible build systems on Linux we suggest using ArrayFire with either -CMake or Makefiles with CMake being our preferred build system. +Because ArrayFire follows standard installation practices, you can use +basically any build system to create and compile projects that use +ArrayFire. Among the many possible build systems on Linux we suggest using +ArrayFire with either CMake or Makefiles with CMake being our preferred build +system. ## Prerequisite software @@ -57,8 +58,8 @@ apt install build-essential cmake cmake-curses-gui ## CMake We recommend that the CMake build system be used to create ArrayFire projects. -As [discussed above](#big-picture-linux), ArrayFire ships with a series of CMake -scripts to make finding and using our library easy. +As [discussed above](#big-picture-linux), ArrayFire ships with a series of +CMake scripts to make finding and using our library easy. First create a file called `CMakeLists.txt` in your project directory: @@ -74,18 +75,19 @@ and populate it with the following code: # Unified backend lets you choose the backend at runtime target_link_libraries( ArrayFire::af) -where `my_executable` is the name of the executable you wish to create. See the -[CMake documentation](https://cmake.org/documentation/) for more information on -how to use CMake. To link with a specific backend directly, replace the -`ArrayFire::af` with the following for their respective backends. +where `my_executable` is the name of the executable you wish to create. See +the [CMake documentation](https://cmake.org/documentation/) for more +information on how to use CMake. To link with a specific backend directly, +replace the `ArrayFire::af` with the following for their respective backends. * `ArrayFire::afcpu` for CPU backend. * `ArrayFire::afcuda` for CUDA backend. +* `ArrayFire::afoneapi` for oneAPI backend. * `ArrayFire::afopencl` for OpenCL backend. -Next we need to instruct CMake to create build instructions and then compile. We -suggest using CMake's out-of-source build functionality to keep your build and -source files cleanly separated. To do this open the CMake GUI. +Next we need to instruct CMake to create build instructions and then +compile. We suggest using CMake's out-of-source build functionality to keep +your build and source files cleanly separated. To do this open the CMake GUI. cd your-project-directory mkdir build @@ -97,8 +99,9 @@ source files cleanly separated. To do this open the CMake GUI. still help you out. When you execute CMake specify the path to ArrayFire installation root as `ArrayFire_DIR` variable. -For example, if ArrayFire were installed locally to `/home/user/ArrayFire` then -you would modify the `cmake` command above to contain the following definition: +For example, if ArrayFire were installed locally to `/home/user/ArrayFire` +then you would modify the `cmake` command above to contain the following +definition: cmake -DArrayFire_DIR=/home/user/ArrayFire .. @@ -106,18 +109,18 @@ You can also specify this information in the `ccmake` command-line interface. ## Makefiles -Building ArrayFire projects with Makefiles is fairly similar to CMake except you -must specify all paths and libraries manually. +Building ArrayFire projects with Makefiles is fairly similar to CMake except +you must specify all paths and libraries manually. As with any `make` project, you need to specify the include path to the directory containing `arrayfire.h` file. This should be `-I /opt/arrayfire/include` if you followed our installation instructions. -Similarly, you will need to specify the path to the ArrayFire library using the -`-L` option (e.g. `-L/opt/arrayfire/lib`) followed by the specific ArrayFire -library you wish to use using the `-l` option (for example `-lafcpu`, -`-lafopencl`, `-lafcuda`, or `-laf` for the CPU, OpenCL, CUDA, and unified -backends, respectively. +Similarly, you will need to specify the path to the ArrayFire library using +the `-L` option (e.g. `-L/opt/arrayfire/lib`) followed by the specific +ArrayFire library you wish to use using the `-l` option (for example +`-lafcpu`, `-lafopencl`, `-lafoneapi`, `-lafcuda`, or `-laf` for the CPU, +OpenCL, oneAPI, and CUDA, and unified backends, respectively. Here is a minimal example Makefile which uses ArrayFire's CPU backend: diff --git a/docs/pages/using_on_windows.md b/docs/pages/using_on_windows.md index b178ad9c86..b9084723d1 100644 --- a/docs/pages/using_on_windows.md +++ b/docs/pages/using_on_windows.md @@ -2,7 +2,8 @@ Using ArrayFire with Microsoft Windows and Visual Studio {#using_on_windows} ============================================================================ If you have not already done so, please make sure you have installed, -configured, and tested ArrayFire following the [installation instructions](#installing). +configured, and tested ArrayFire following the [installation +instructions](#installing). # The big picture {#big-picture-windows} @@ -10,70 +11,60 @@ The ArrayFire Windows installer creates the following: 1. **AF_PATH** environment variable to point to the installation location. The default install location is `C:\Program Files\ArrayFire\v3` 2. **AF_PATH/include** : Header files for ArrayFire (include directory) -3. **AF_PATH/lib** : All ArrayFire backends libraries, dlls and dependency dlls - (library directory) -4. **AF_PATH/examples** : Examples to get started. +3. **AF_PATH/lib** : All ArrayFire backend libraries, dlls, and dependency + dlls (library directory) +4. **AF_PATH/examples** : Examples to get started 5. **AF_PATH/cmake** : CMake config files 6. **AF_PATH/uninstall.exe** : Uninstaller -The installer will prompt the user for following three options. -* Do not add **%%AF_PATH%/lib** to PATH -* Add **%%AF_PATH%/lib** to PATH environment variable of current user -* Add **%%AF_PATH%/lib** to PATH environment variable for all users - -If you chose not to modify PATH during installation please make sure to do so -manually so that all applications using ArrayFire libraries will be able to find -the required DLLs. - # Build and Run Helloworld {#section1} This can be done in two ways either by using CMake build tool or using Visual Studio directly. ## Using CMake {#section1part1} -1. Download and install [CMake](https://cmake.org/download/), preferrably the +1. Download and install [CMake](https://cmake.org/download/), preferably the latest version. 2. Open CMake-GUI and set the field __Where is the source code__ to the root directory of examples. 3. Set the field __Where to build the binaries__ to - **path_to_examples_root_dir/build** and click the `Configure` button towards - the lower left bottom. -4. CMake will prompt you asking if it has to create the `build` directory if - it's not already present. Click yes to create the build directory. -5. Before the configuration begins, CMake will show you a list(drop-down menu) - of available Visual Studio versions on your system to chose from. Select one - and check the radio button that says **Use default native compilers** and - click finish button in the bottom right corner. -6. CMake will show you errors in red text if any once configuration is finished. - Ideally, you wouldn't need to do anything and CMake should be able to find - ArrayFire automatically. Please let us know if it didn't on your machine. + **path_to_examples_root_dir/build** and click the `Configure` button. +4. CMake will prompt you to create the `build` directory if not already + present. Click "yes" to create the build directory. +5. Before the configuration begins, CMake will show you a list (drop-down + menu) of available Visual Studio versions. Select one and check the radio + button that says **Use default native compilers** and click finish. +6. CMake will show you errors in red text, if any, once configuration is + finished. Sometimes a second configuration is necessary. 7. Click **Generate** button to generate the Visual Studio solution files for the examples. 8. Click **Open Project** button that is right next to **Generate** button to open the solution file. -9. You will see a bunch of examples segregated into three sets named after the - compute backends of ArrayFire: cpu, cuda & opencl if you have installed all - backends. Select the helloworld project from any of the installed backends - and mark it as startup project and hit `F5`. +9. You will see the examples segregated into four sets named after the compute + backends of ArrayFire: cpu, cuda, oneapi, & opencl, if you installed all + backends. Select the helloworld project from any of the installed backends, + mark it as startup project, and hit `F5`. 10. Once the helloworld example builds, you will see a console window with the output from helloworld program. ## Using Visual Studio {#section1part2} -1. Open Visual Studio of your choice and create an empty C++ project. -2. Right click the project and add an existing source file +1. Open Visual Studio and create an empty C++ project. +2. Right-click the project and add an existing source file `examples/helloworld/helloworld.cpp` to this project. 3. Add `"$(AF_PATH)/include;"` to _Project Properties -> C/C++ -> General -> Additional Include Directories_. 4. Add `"$(AF_PATH)/lib;"` to _Project Properties -> Linker -> General -> Additional Library Directories_. -5. Add `afcpu.lib` or `afcuda.lib` or `afopencl.lib` to _Project Properties -> - Linker -> Input -> Additional Dependencies_. based on your preferred backend. -6. (Optional) You may choose to define `NOMINMAX`, `AF_` and/or - `AF_` in your projects. This can be added to _Project - Properties -> C/C++ -> General -> Preprocessor-> Preprocessory definitions_. -7. Build and run the project. You will see a console window with the output from - helloworld program. +5. Add `afcpu.lib`, `afcuda.lib`, `afoneapi.lib`, or `afopencl.lib` to + _Project Properties -> Linker -> Input -> Additional Dependencies_. based + on your preferred backend. +6. (Optional) You may choose to define `NOMINMAX`, + `AF_`, or `AF_` in your + projects. This can be added to _Project Properties -> C/C++ -> General -> + Preprocessor-> Preprocessory definitions_. +7. Build and run the project. You will see a console window with the output + from helloworld program. # Using ArrayFire within Existing Visual Studio Projects {#section2} This is divided into three parts: @@ -83,10 +74,10 @@ This is divided into three parts: ## Part A: Adding ArrayFire to an existing solution (Single Backend) {#section2partA} -Note: If you plan on using Native CUDA code in the project, use the steps under -[Part B](#section2partB). +Note: If you plan on using Native CUDA code in the project, use the steps +under [Part B](#section2partB). -Adding a single backend to an existing project is quite simple. +Adding a single backend to an existing project is quite simple: 1. Add `"$(AF_PATH)/include;"` to _Project Properties -> C/C++ -> General -> Additional Include Directories_. @@ -97,8 +88,9 @@ Adding a single backend to an existing project is quite simple. preferred backend. ## Part B: Adding ArrayFire CUDA to a new/existing CUDA project {#section2partB} -Lastly, if your project contains custom CUDA code, the instructions are slightly -different as it requires using a CUDA NVCC Project: + +Lastly, if your project contains custom CUDA code, the instructions are +slightly different as it requires using a CUDA NVCC Project: 1. Create a custom "CUDA NVCC project" in Visual Studio 2. Add `"$(AF_PATH)/include;"` to _Project Properties -> CUDA C/C++ -> General @@ -108,7 +100,8 @@ different as it requires using a CUDA NVCC Project: 4. Add `afcpu.lib`, `afcuda.lib`, `afopencl.lib`, or `af.lib` to _Project Properties -> Linker -> Input -> Additional Dependencies_. based on your preferred backend. -### Part C: Project with all ArrayFire backends {#section2partC} +## Part C: Project with all ArrayFire backends {#section2partC} + If you wish to create a project that allows you to use all the ArrayFire backends with ease, you should use `af.lib` in step 3 from [Part A](#section2partA). @@ -116,11 +109,12 @@ A](#section2partA). You can alternately download the template project from [ArrayFire Template Projects](https://github.com/arrayfire/arrayfire-project-templates) -# Using ArrayFire with CMake -ArrayFire ships with a series of CMake scripts to make finding and using our +# Using ArrayFire with CMake + +ArrayFire ships with a series of CMake scripts to make finding and using the library easy. -First create a file called `CMakeLists.txt` in your project directory: +First, create a file called `CMakeLists.txt` in your project directory: cd your-project-directory touch CMakeLists.txt @@ -130,26 +124,27 @@ and populate it with the following code: find_package(ArrayFire) add_executable( [list your source files here]) - # To use Unified backend, do the following. - # Unified backend lets you choose the backend at runtime + # The Unified backend lets you choose the backend at runtime. + # To use the Unified backend, do the following: target_link_libraries( ArrayFire::af) -where `` is the name of the executable you wish to create. See the -[CMake documentation](https://cmake.org/documentation/) for more information on -how to use CMake. To link with a specific backend directly, replace the +, where `` is the name of the executable to create. See the +[CMake documentation](https://cmake.org/documentation/) for more information +on how to use CMake. To link with a specific backend directly, replace the `ArrayFire::af` with the following for their respective backends. * `ArrayFire::afcpu` for CPU backend. * `ArrayFire::afcuda` for CUDA backend. +* `ArrayFire::afoneapi` for oneAPI backend. * `ArrayFire::afopencl` for OpenCL backend. -Next we need to instruct CMake to create build instructions and then compile. We -suggest using CMake's out-of-source build functionality to keep your build and -source files cleanly separated. To do this open the CMake GUI. +Next, instruct CMake to create build instructions and compile them. We suggest +using CMake's out-of-source build functionality to keep your build and source +files cleanly separated. To do this, open the CMake GUI. -* Under source directory, add the path to your project -* Under build directory, add the path to your project and append /build -* Click configure and choose a 64 bit Visual Studio generator. -* If configuration was successful, click generate. This will create a - my-project.sln file under build. Click `Open Project` in CMake-GUI to open the - solution and compile the ALL_BUILD project. +* Under "source directory", add the path to your project. +* Under "build directory", add the path to your project and append /build. +* Click "configure" and choose a 64-bit Visual Studio generator. +* If the configuration was successful, click "generate". This will create a + my-project.sln file under build. Click `Open Project` in CMake-GUI to open + the solution and compile the ALL_BUILD project. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index f69eff6e1f..91280e485e 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -5,7 +5,7 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) cmake_policy(VERSION 3.5) project(ArrayFire-Examples VERSION 3.7.0 diff --git a/examples/benchmarks/CMakeLists.txt b/examples/benchmarks/CMakeLists.txt index d5ece4b562..4fd0853e58 100644 --- a/examples/benchmarks/CMakeLists.txt +++ b/examples/benchmarks/CMakeLists.txt @@ -5,12 +5,12 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-Benchmarks VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) if(ArrayFire_CPU_FOUND) add_executable(blas_cpu blas.cpp) diff --git a/examples/computer_vision/CMakeLists.txt b/examples/computer_vision/CMakeLists.txt index 7314d29148..2683eb1931 100644 --- a/examples/computer_vision/CMakeLists.txt +++ b/examples/computer_vision/CMakeLists.txt @@ -5,12 +5,12 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-Computer-Vision VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) add_definitions("-DASSETS_DIR=\"${ASSETS_DIR}\"") diff --git a/examples/financial/CMakeLists.txt b/examples/financial/CMakeLists.txt index 9cc2435b25..f365f88b47 100644 --- a/examples/financial/CMakeLists.txt +++ b/examples/financial/CMakeLists.txt @@ -5,12 +5,12 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-Financial VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) if(ArrayFire_CPU_FOUND) # Black-Scholes Options diff --git a/examples/getting_started/CMakeLists.txt b/examples/getting_started/CMakeLists.txt index f0ee51249a..a9d1ce4bcb 100644 --- a/examples/getting_started/CMakeLists.txt +++ b/examples/getting_started/CMakeLists.txt @@ -5,12 +5,12 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-Getting-Started VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) if(ArrayFire_CPU_FOUND) # Convolve examples diff --git a/examples/graphics/CMakeLists.txt b/examples/graphics/CMakeLists.txt index d59a506278..6140142343 100644 --- a/examples/graphics/CMakeLists.txt +++ b/examples/graphics/CMakeLists.txt @@ -5,12 +5,12 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-Graphics VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) add_definitions("-DASSETS_DIR=\"${ASSETS_DIR}\"") diff --git a/examples/graphics/field.cpp b/examples/graphics/field.cpp index a723791fc8..f493c7ecd6 100644 --- a/examples/graphics/field.cpp +++ b/examples/graphics/field.cpp @@ -22,7 +22,7 @@ int main(int, char**) { af::info(); af::Window myWindow(1024, 1024, "2D Vector Field example: ArrayFire"); - myWindow.grid(1, 2); + myWindow.grid(2, 2); array dataRange = seq(MINIMUM, MAXIMUM, STEP); @@ -38,12 +38,21 @@ int main(int, char**) { array saddle = join(1, flat(x), -1.0f * flat(y)); array bvals = sin(scale * (x * x + y * y)); - array hbowl = join(1, constant(1, x.elements()), flat(bvals)); + array hbowl = join(1, constant(1., x.elements()), flat(bvals)); hbowl.eval(); + // 2D points myWindow(0, 0).vectorField(points, saddle, "Saddle point"); myWindow(0, 1).vectorField( points, hbowl, "hilly bowl (in a loop with varying amplitude)"); + + // 2D coordinates + myWindow(1, 0).vectorField(2.0 * flat(x), flat(y), flat(x), + -flat(y), "Saddle point"); + myWindow(1, 1).vectorField( + 2.0 * flat(x), flat(y), constant(1., x.elements()), flat(bvals), + "hilly bowl (in a loop with varying amplitude)"); + myWindow.show(); scale -= 0.0010f; diff --git a/examples/helloworld/CMakeLists.txt b/examples/helloworld/CMakeLists.txt index 3567873958..b3a02e9fc6 100644 --- a/examples/helloworld/CMakeLists.txt +++ b/examples/helloworld/CMakeLists.txt @@ -5,12 +5,12 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-HelloWorld VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) if(ArrayFire_CPU_FOUND) # Hello World example diff --git a/examples/image_processing/CMakeLists.txt b/examples/image_processing/CMakeLists.txt index 12307b679f..e4ab1d3d8a 100644 --- a/examples/image_processing/CMakeLists.txt +++ b/examples/image_processing/CMakeLists.txt @@ -5,12 +5,12 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-Image-Processing VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) add_definitions("-DASSETS_DIR=\"${ASSETS_DIR}\"") diff --git a/examples/lin_algebra/CMakeLists.txt b/examples/lin_algebra/CMakeLists.txt index baba1a4181..89b9c89600 100644 --- a/examples/lin_algebra/CMakeLists.txt +++ b/examples/lin_algebra/CMakeLists.txt @@ -5,12 +5,12 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-Linear-Algebra VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) if(ArrayFire_CPU_FOUND) # Cholesky example diff --git a/examples/machine_learning/CMakeLists.txt b/examples/machine_learning/CMakeLists.txt index 9c2c3ade6c..480f3f7f12 100644 --- a/examples/machine_learning/CMakeLists.txt +++ b/examples/machine_learning/CMakeLists.txt @@ -5,12 +5,12 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-Linear-Algebra VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) add_definitions("-DASSETS_DIR=\"${ASSETS_DIR}\"") diff --git a/examples/machine_learning/kmeans.cpp b/examples/machine_learning/kmeans.cpp index e40cc34368..963d6a609f 100644 --- a/examples/machine_learning/kmeans.cpp +++ b/examples/machine_learning/kmeans.cpp @@ -17,7 +17,7 @@ using namespace af; array distance(array data, array means) { - int n = data.dims(0); // Number of features + int n = data.dims(0); // Number of data points int k = means.dims(1); // Number of means array data2 = tile(data, 1, k, 1); @@ -60,8 +60,8 @@ array new_means(array data, array clusters, int k) { // means: output, vector of means void kmeans(array &means, array &clusters, const array in, int k, int iter = 100) { - unsigned n = in.dims(0); // Num features - unsigned d = in.dims(2); // feature length + unsigned n = in.dims(0); // Num of data points + unsigned d = in.dims(2); // Num of features (will only be 1 in spider image example) // reshape input array data = in * 0; diff --git a/examples/machine_learning/mnist_common.h b/examples/machine_learning/mnist_common.h index a32d21932c..8d079df75a 100644 --- a/examples/machine_learning/mnist_common.h +++ b/examples/machine_learning/mnist_common.h @@ -13,7 +13,7 @@ #include "../common/idxio.h" bool compare(const std::pair l, const std::pair r) { - return l.first >= r.first; + return l.first > r.first; } typedef std::pair sort_type; @@ -145,7 +145,7 @@ static void display_results(const af::array &test_images, (test_images(span, span, i) > 0.1f).as(u8).host(); for (int j = 0; j < 28; j++) { for (int k = 0; k < 28; k++) { - std::cout << (img[j * 28 + k] ? "\u2588" : " ") << " "; + std::cout << (img[k * 28 + j] ? "\u2588" : " ") << " "; } std::cout << std::endl; } diff --git a/examples/machine_learning/naive_bayes.cpp b/examples/machine_learning/naive_bayes.cpp index 9fe6456f0e..aadca32bc0 100644 --- a/examples/machine_learning/naive_bayes.cpp +++ b/examples/machine_learning/naive_bayes.cpp @@ -135,8 +135,8 @@ void naive_bayes_demo(bool console, int perc) { if (!console) { test_images = test_images.T(); test_labels = test_labels.T(); - // FIXME: Crashing in mnist_common.h::classify - // display_results(test_images, res_labels, test_labels , 20); + + display_results(test_images, res_labels, test_labels, 20); } } diff --git a/examples/pde/CMakeLists.txt b/examples/pde/CMakeLists.txt index 0b74e6165f..57f689a9e9 100644 --- a/examples/pde/CMakeLists.txt +++ b/examples/pde/CMakeLists.txt @@ -5,30 +5,54 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-PDE VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) + +add_definitions("-DASSETS_DIR=\"${ASSETS_DIR}\"") if(ArrayFire_CPU_FOUND) # Shallow Water simulation example add_executable(swe_cpu swe.cpp) target_link_libraries(swe_cpu ArrayFire::afcpu) + + # Black Hole Raytracing example + add_executable(bhrt_cpu bhrt.cpp) + target_link_libraries(bhrt_cpu ArrayFire::afcpu) + + add_executable(boltzmann_cfd_cpu boltzmann_cfd.cpp) + target_link_libraries(boltzmann_cfd_cpu ArrayFire::afcpu) endif() if(ArrayFire_CUDA_FOUND) add_executable(swe_cuda swe.cpp) target_link_libraries(swe_cuda ArrayFire::afcuda) + + add_executable(bhrt_cuda bhrt.cpp) + target_link_libraries(bhrt_cuda ArrayFire::afcuda) + + add_executable(boltzmann_cfd_cuda boltzmann_cfd.cpp) + target_link_libraries(boltzmann_cfd_cuda ArrayFire::afcuda) endif() if(ArrayFire_OpenCL_FOUND) add_executable(swe_opencl swe.cpp) target_link_libraries(swe_opencl ArrayFire::afopencl) + + add_executable(bhrt_opencl bhrt.cpp) + target_link_libraries(bhrt_opencl ArrayFire::afopencl) + + add_executable(boltzmann_cfd_opencl boltzmann_cfd.cpp) + target_link_libraries(boltzmann_cfd_opencl ArrayFire::afopencl) endif() if(ArrayFire_oneAPI_FOUND) add_executable(swe_oneapi swe.cpp) target_link_libraries(swe_oneapi ArrayFire::afoneapi) + + add_executable(boltzmann_cfd_oneapi boltzmann_cfd.cpp) + target_link_libraries(boltzmann_cfd_oneapi ArrayFire::afoneapi) endif() diff --git a/examples/pde/bhrt.cpp b/examples/pde/bhrt.cpp new file mode 100644 index 0000000000..55e116a330 --- /dev/null +++ b/examples/pde/bhrt.cpp @@ -0,0 +1,1139 @@ +/******************************************************* + * Copyright (c) 2024, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +/* + This is a Black Hole Raytracer. + For this raytracer we are using backwards path tracing to compute the + resulting image The path of the rays shot from the camera are simulated step + by step from the null geodesics light follows in spacetime. The geodesics are + computed from the spacetime metric of the space. This project has three + metrics that can be used: Schwarzchild, Kerr, and Ellis. + + For more information on the black hole raytracing, check out + Riazuelo, A. (2015). Seeing relativity -- I. Ray tracing in a Schwarzschild + metric to explore the maximal analytic extension of the metric and making a + proper rendering of the stars. ArXiv. + https://doi.org/10.1142/S0218271819500421 + + For more information on raytracing, check out + Raytracing in a Weekend Series, https://raytracing.github.io/ + + Image being used for the background is Westerlund 2 from + NASA, ESA, the Hubble Heritage Team (STScI/AURA), A. Nota (ESA/STScI), and + the Westerlund 2 Science Team See + http://www.spacetelescope.org/images/heic1509a/ for details. + + The default scene is the rotating black hole using the Kerr metric set by + the global variable 'scene' The parameters of the blackholes/wormholes may be + changed at the top with the simulation constants The parameters of the image + may be changed in the 'raytracing' function. +*/ +#include + +#include +#include +#include +#include +#include +#include + +enum class Scene { ROTATE_BH, STATIC_BH, WORMHOLE }; + +// Scene being computed +static constexpr Scene scene = Scene::ROTATE_BH; + +// **** Simulation Constants **** +static constexpr double M = 0.5; // Black Hole Mass +static constexpr double J = 0.249; // Black Hole Rotation (J < M^2) +static constexpr double b = 3.0; // Wormhole drainhole parameter + +/** + * @brief Generates a string progress bar + * + * @param current current job + * @param total total number of jobs + * @param start_info progress bar prior info + */ +void status_bar(int64_t current, int64_t total, const std::string& start_info) { + auto precision = std::cout.precision(); + static auto prev_time = std::chrono::high_resolution_clock::now(); + static auto prev = current - 1; + static auto prev2 = prev; + static auto prev2_time = prev_time; + + auto curr_time = std::chrono::high_resolution_clock::now(); + + double percent = 100.0 * (double)(current + 1) / (double)total; + std::string str = "["; + for (int i = 0; i < 50; ++i) { + if (percent >= i * 2) + str += "="; + else + str += " "; + } + str += "]"; + + auto time = + current != prev + ? (total - current) * (curr_time - prev_time) / (current - prev) + : (total - current) * (curr_time - prev2_time) / (current - prev2); + + if (current != prev && prev != prev2) { + prev2 = prev; + prev2_time = prev_time; + } + prev = current; + prev_time = curr_time; + + if (current != total) { + using namespace std::chrono_literals; + std::cout << start_info << " " << std::fixed << std::setprecision(1) + << percent << "% " << str << " Time Remaining: "; + if (std::chrono::duration_cast(time).count() > + 300) + std::cout << std::chrono::duration_cast(time) + .count() + << " min"; + else + std::cout << std::chrono::duration_cast(time) + .count() + << " s"; + + std::cout << std::string(5, ' ') << '\r'; + } else + std::cout << "\rDone!" << std::string(120, ' ') << std::endl; + + std::cout << std::setprecision(precision) << std::defaultfloat; +} + +/** + * @brief Returns the euclidean dot product for two cartesian vectors with 3 + * coords + * + * @param lhs + * @param rhs + * @return af::array + */ +af::array dot3(const af::array& lhs, const af::array& rhs) { + return af::sum(lhs * rhs, 0); +} + +/** + * @brief Returns the euclidean norm for a cartesian vector with 3 coords + * + * @param vector + * @return af::array + */ +af::array norm3(const af::array& vector) { + return af::sqrt(dot3(vector, vector)); +} + +/** + * @brief Returns the normalized vector for a cartesian vector with 3 coords + * + * @param vector + * @return af::array + */ +af::array normalize3(const af::array& vector) { return vector / norm3(vector); } + +af::exception make_error(const char* string) { + std::cout << string << std::endl; + return af::exception(string); +} + +/** + * @brief Transforms degrees to radians + * + * @param degrees + * @return double + */ +double radians(double degrees) { return degrees * af::Pi / 180.0; } + +/** + * @brief Computes the cross_product of two euclidean vectors + * + * @param lhs + * @param rhs + * @return af::array + */ +af::array cross_product(const af::array& lhs, const af::array& rhs) { + if (lhs.dims() != rhs.dims()) + throw make_error("Arrays must have the same dimensions"); + else if (lhs.dims()[0] != 3) + throw make_error("Arrays must have 3 principal coordintes"); + + return af::join( + 0, + lhs(1, af::span, af::span) * rhs(2, af::span, af::span) - + lhs(2, af::span, af::span) * rhs(1, af::span, af::span), + lhs(2, af::span, af::span) * rhs(0, af::span, af::span) - + lhs(0, af::span, af::span) * rhs(2, af::span, af::span), + lhs(0, af::span, af::span) * rhs(1, af::span, af::span) - + lhs(1, af::span, af::span) * rhs(0, af::span, af::span)); +} + +/** + * @brief Transform the position vectors from cartesian to spherical coordinates + * + * @param pos + * @return af::array + */ +af::array cart_to_sph_position(const af::array& pos) { + if (pos.dims()[0] != 3) + throw make_error("Arrays must have 3 principal coordintes"); + + af::array x = pos(0, af::span); + af::array y = pos(1, af::span); + af::array z = pos(2, af::span); + + af::array r = af::sqrt(x * x + y * y + z * z); + af::array o = af::acos(z / r); + af::array p = af::atan2(y, x); + + af::array transformed_pos = af::join(0, r, o, p); + + return transformed_pos; +} + +/** + * @brief Transform the velocity vectors from cartesian to spherical coordinates + * + * @param vel + * @param pos + * @return af::array + */ +af::array cart_to_sph_velocity(const af::array& vel, const af::array& pos) { + if (vel.dims() != pos.dims()) + throw make_error("Arrays must have the same dimensions"); + else if (pos.dims()[0] != 3) + throw make_error("Arrays must have 3 principal coordintes"); + + af::array x = pos(0, af::span); + af::array y = pos(1, af::span); + af::array z = pos(2, af::span); + + af::array r = af::sqrt(x * x + y * y + z * z); + af::array o = af::acos(z / r); + af::array p = af::atan2(y, x); + + af::array ux = vel(0, af::span); + af::array uy = vel(1, af::span); + af::array uz = vel(2, af::span); + + af::array ur = (ux * x + uy * y + uz * z) / r; + af::array up = (uy * af::cos(p) - ux * af::sin(p)) / (r * af::sin(o)); + af::array uo = + (af::cos(o) * (ux * af::cos(p) + uy * af::sin(p)) - uz * af::sin(o)) / + r; + af::array transformed_vel = af::join(0, ur, uo, up); + + return transformed_vel; +} + +/** + * @brief Transform the velocity vectors from cartesian to spherical coordinates + * + * @param vel + * @param pos + * @return af::array + */ +af::array sph_to_cart_velocity(const af::array& vel, const af::array& pos) { + if (vel.dims() != pos.dims()) + throw make_error("Arrays must have the same dimensions"); + else if (pos.dims()[0] != 3) + throw make_error("Arrays must have 3 principal coordintes"); + + af::array r = pos(0, af::span); + af::array o = pos(1, af::span); + af::array p = pos(2, af::span); + + af::array ur = vel(0, af::span); + af::array uo = vel(1, af::span); + af::array up = vel(2, af::span); + + af::array ux = (ur * af::sin(o) + uo * r * af::cos(o)) * af::cos(p) - + up * r * af::sin(o) * af::sin(p); + af::array uy = (ur * af::sin(o) + uo * r * af::cos(o)) * af::sin(p) + + up * r * af::sin(o) * af::cos(p); + af::array uz = ur * af::cos(o) - uo * r * af::sin(o); + af::array transformed_vel = af::join(0, ux, uy, uz); + + return transformed_vel; +} + +/** + * @brief Transform the position vectors from cartesian to oblate coordinates + * + * @param vel + * @param pos + * @return af::array + */ +af::array cart_to_oblate_position(const af::array& pos) { + if (pos.dims()[0] != 3) + throw make_error("Arrays must have 3 principal coordintes"); + + af::array x = pos(0, af::span); + af::array y = pos(1, af::span); + af::array z = pos(2, af::span); + auto a = J / M; + auto diff = x * x + y * y + z * z - a * a; + + af::array r = + af::sqrt((diff + af::sqrt(diff * diff + z * z * a * a * 4.0)) / 2.0); + af::array o = af::acos(z / r); + af::array p = af::atan2(y, x); + + af::array transformed_pos = af::join(0, r, o, p); + + return transformed_pos; +} + +/** + * @brief Transform the position vectors from oblate to cartesian coordinates + * + * @param vel + * @param pos + * @return af::array + */ +af::array oblate_to_cart_position(const af::array& pos) { + if (pos.dims()[0] != 3) + throw make_error("Arrays must have 3 principal coordintes"); + + af::array r = pos(0, af::span); + af::array o = pos(1, af::span); + af::array p = pos(2, af::span); + auto a = J / M; + auto R = af::sqrt(r * r + a * a); + + af::array x = R * af::sin(o) * af::cos(p); + af::array y = R * af::sin(o) * af::sin(p); + af::array z = r * af::cos(o); + + af::array transformed_pos = af::join(0, x, y, z); + + return transformed_pos; +} + +/** + * @brief Transform the velocity vectors from oblate to cartesian coordinates + * + * @param vel + * @param pos + * @return af::array + */ +af::array oblate_to_cart_velocity(const af::array& vel, const af::array& pos) { + if (vel.dims() != pos.dims()) + throw make_error("Arrays must have the same dimensions"); + else if (pos.dims()[0] != 3) + throw make_error("Arrays must have 3 principal coordintes"); + + af::array r = pos(0, af::span); + af::array o = pos(1, af::span); + af::array p = pos(2, af::span); + + af::array ur = vel(0, af::span); + af::array uo = vel(1, af::span); + af::array up = vel(2, af::span); + + double a = J / M; + af::array ra = af::sqrt(r * r + a * a); + + af::array ux = + (ur * r * af::sin(o) / ra + uo * ra * af::cos(o)) * af::cos(p) - + up * r * af::sin(o) * af::sin(p); + af::array uy = + (ur * r * af::sin(o) / ra + uo * ra * af::cos(o)) * af::sin(p) + + up * r * af::sin(o) * af::cos(p); + af::array uz = ur * af::cos(o) - uo * r * af::sin(o); + af::array transformed_vel = af::join(0, ux, uy, uz); + + return transformed_vel; +} + +/** + * @brief Transform the velocity vectors from cartesian to oblate coordinates + * + * @param vel + * @param pos + * @return af::array + */ +af::array cart_to_oblate_velocity(const af::array& vel, const af::array& pos) { + if (vel.dims() != pos.dims()) + throw make_error("Arrays must have the same dimensions"); + else if (pos.dims()[0] != 3) + throw make_error("Arrays must have 3 principal coordintes"); + + af::array x = pos(0, af::span); + af::array y = pos(1, af::span); + af::array z = pos(2, af::span); + + auto a = J / M; + auto diff = x * x + y * y + z * z - a * a; + + af::array r = + af::sqrt((diff + af::sqrt(diff * diff + z * z * a * a * 4.0)) / 2.0); + af::array o = af::acos(z / r); + af::array p = af::atan2(y, x); + + af::array ux = vel(0, af::span); + af::array uy = vel(1, af::span); + af::array uz = vel(2, af::span); + + af::array ra = r * r + a * a; + af::array ur = ((ux * x + uy * y) * r + uz * ra * z / r) / + (r * r + af::pow(a * af::cos(o), 2.0)); + af::array up = (uy * x - ux * y) / (x * x + y * y); + af::array uo = ((ux * x + uy * y) / af::tan(o) - uz * z * af::tan(o)) / + (r * r + af::pow(a * af::cos(o), 2.0)); + af::array transformed_vel = af::join(0, ur, uo, up); + + return transformed_vel; +} + +/** + * @brief Transform the position vectors from spherical to cartesian coordinates + * + * @param pos + * @return af::array + */ +af::array sph_to_cart_position(const af::array& pos) { + af::array r = pos(0, af::span); + af::array o = pos(1, af::span); + af::array p = pos(2, af::span); + + af::array x = r * af::sin(o) * af::cos(p); + af::array y = r * af::sin(o) * af::sin(p); + af::array z = r * af::cos(o); + + af::array transformed_pos = af::join(0, x, y, z); + + return transformed_pos; +} + +/** + * @brief Computes the inverse of a 4x4 matrix with the layout + * [ a 0 0 b ] + * [ 0 c 0 0 ] + * [ 0 0 d 0 ] + * [ b 0 0 e ] + * + * @param metric af::array with the shape af::dims4(4, 4, M, N) + * + * @return af::array with the shape af::dims4(4, 4, M, N) + */ +af::array inv_metric(const af::array& metric) { + af::array a = metric(0, 0, af::span); + af::array b = metric(3, 0, af::span); + af::array c = metric(1, 1, af::span); + af::array d = metric(2, 2, af::span); + af::array e = metric(3, 3, af::span); + + af::array det = b * b - a * e; + + auto res = af::constant(0, 4, 4, metric.dims()[2], metric.dims()[3], f64); + + res(0, 0, af::span) = -e / det; + res(0, 3, af::span) = b / det; + res(3, 0, af::span) = b / det; + res(1, 1, af::span) = 1.0 / c; + res(2, 2, af::span) = 1.0 / d; + res(3, 3, af::span) = -a / det; + + return res; +} + +/** + * @brief Computes the 4x4 metric matrix for the given 4-vector positions + * + * @param pos af::dim4(4, N) + * @return af::array af::dim4(4, 4, 1, N) + */ +af::array metric4(const af::array& pos) { + if (pos.dims()[0] != 4) + throw make_error("Arrays must have 4 principal coordinates"); + + auto dims = pos.dims(); + + af::array t = af::moddims(pos(0, af::span), 1, 1, dims[1]); + af::array r = af::moddims(pos(1, af::span), 1, 1, dims[1]); + af::array o = af::moddims(pos(2, af::span), 1, 1, dims[1]); + af::array p = af::moddims(pos(3, af::span), 1, 1, dims[1]); + + af::array gtt, gtr, gto, gtp, grt, grr, gro, grp, got, gor, goo, gop, gpt, + gpr, gpo, gpp; + + switch (scene) { + // ******* Kerr Black Hole Metric ******* + case Scene::ROTATE_BH: { + auto rs = 2.0 * M; + auto a = J / M; + auto delta = (r - rs) * r + a * a; + auto sigma = r * r + af::pow(a * af::cos(o), 2); + + gtt = 1.0 - r * rs / sigma; + gtr = af::constant(0.0, 1, 1, dims[1], f64); + gto = af::constant(0.0, 1, 1, dims[1], f64); + gtp = rs * r * a * af::pow(af::sin(o), 2.0) / sigma; + grr = -sigma / delta; + gro = af::constant(0.0, 1, 1, dims[1], f64); + grp = af::constant(0.0, 1, 1, dims[1], f64); + goo = -sigma; + gop = af::constant(0.0, 1, 1, dims[1], f64); + gpp = + -(r * r + a * a + rs * r * af::pow(a * af::sin(o), 2) / sigma) * + af::pow(af::sin(o), 2); + + break; + } + + // ******* Schwarzchild Black Hole Metric ******* + case Scene::STATIC_BH: { + gtt = 1.0 - 2.0 * M / r; + gtr = af::constant(0.0, 1, 1, dims[1], f64); + gto = af::constant(0.0, 1, 1, dims[1], f64); + gtp = af::constant(0.0, 1, 1, dims[1], f64); + grr = -1.0 / (1.0 - 2.0 * M / r); + gro = af::constant(0.0, 1, 1, dims[1], f64); + grp = af::constant(0.0, 1, 1, dims[1], f64); + goo = -r * r; + gop = af::constant(0.0, 1, 1, dims[1], f64); + gpp = -af::pow(r * af::sin(o), 2); + + break; + } + + // ******* Ellis Wormhole Metric ******* + case Scene::WORMHOLE: { + gtt = af::constant(1.0, 1, 1, dims[1], f64); + gtr = af::constant(0.0, 1, 1, dims[1], f64); + gto = af::constant(0.0, 1, 1, dims[1], f64); + gtp = af::constant(0.0, 1, 1, dims[1], f64); + grr = -af::constant(1.0, 1, 1, dims[1], f64); + gro = af::constant(0.0, 1, 1, dims[1], f64); + grp = af::constant(0.0, 1, 1, dims[1], f64); + goo = -(r * r + b * b); + gop = af::constant(0.0, 1, 1, dims[1], f64); + gpp = -(r * r + b * b) * af::pow(af::sin(o), 2); + + break; + } + + default: throw; + } + + auto res = af::join( + 0, af::join(1, gtt, gtr, gto, gtp), af::join(1, gtr, grr, gro, grp), + af::join(1, gto, gro, goo, gop), af::join(1, gtp, grp, gop, gpp)); + + return res; +} + +/** + * @brief Computes the dot product as defined by a metric between two 4-vector + * velocities + * + * @param pos + * @param lhs + * @param rhs + * @return af::array + */ +af::array dot_product(const af::array& pos, const af::array& lhs, + const af::array& rhs) { + if (pos.dims() != lhs.dims()) + throw make_error( + "Position and lhs velocity must have the same dimensions"); + else if (lhs.dims() != rhs.dims()) + throw make_error( + "Position and rhs velocity must have the same dimensions"); + else if (rhs.dims()[0] != 4) + throw make_error("Arrays must have 4 principal coordinates"); + + return af::matmul(af::moddims(lhs, 1, 4, lhs.dims()[1]), metric4(pos), + af::moddims(rhs, 4, 1, rhs.dims()[1])); +} + +af::array norm4(const af::array& pos, const af::array& vel) { + return dot_product(pos, vel, vel); +} + +af::array partials(const af::array& pos4, uint32_t index, double rel_diff, + double abs_diff) { + double arr[4] = {0.0}; + arr[index] = 1.0; + + auto pos_diff = pos4 * rel_diff + abs_diff; + auto h4 = pos_diff * af::array(af::dim4(4, 1), arr); + af::array h = + af::moddims(pos_diff(index, af::span), af::dim4(1, 1, pos4.dims()[1])); + + return (-metric4(pos4 + h4 * 2.0) + metric4(pos4 + h4) * 8.0 - + metric4(pos4 - h4) * 8.0 + metric4(pos4 - h4 * 2.0)) / + (h * 12.0); +} + +/** + * @brief Computes the geodesics from the established metric, 4-vector positions + * and velocities + * + * @param pos4 + * @param vel4 + * @return af::array + */ +af::array geodesics(const af::array& pos4, const af::array& vel4) { + auto N = vel4.dims()[1]; + + af::array uu = af::matmul(af::moddims(vel4, af::dim4(4, 1, N)), + af::moddims(vel4, af::dim4(1, 4, N))); + uu = af::moddims(uu, af::dim4(1, 4, 4, N)); + + af::array metric = metric4(pos4); + af::array invmetric = af::moddims(inv_metric(metric), af::dim4(4, 4, 1, N)); + + // Compute the partials of the metric with respect to coordinates indices + af::array dt = af::constant(0, 4, 4, 1, N, f64); + + auto dr = partials(pos4, 1, 1e-6, 1e-12); + auto dtheta = partials(pos4, 2, 1e-6, 1e-12); + auto dphi = partials(pos4, 3, 1e-6, 1e-12); + + dr = af::moddims(dr, af::dim4(4, 4, 1, N)); + dtheta = af::moddims(dtheta, af::dim4(4, 4, 1, N)); + dphi = af::moddims(dphi, af::dim4(4, 4, 1, N)); + + // Compute the einsum for each of the christoffel terms + af::array partials = af::join(2, dt, dr, dtheta, dphi); + af::array p1 = af::matmul(invmetric, partials); + af::array p2 = af::reorder(p1, 0, 2, 1, 3); + af::array p3 = af::matmul(invmetric, af::reorder(partials, 2, 0, 1, 3)); + + auto christoffels = -0.5 * (p1 + p2 - p3); + + // Use the geodesics equation to find the 4-vector acceleration + return af::moddims(af::sum(af::sum(christoffels * uu, 1), 2), + af::dim4(4, N)); +} + +/** + * @brief Camera struct + * + * Contains all the data pertaining to the parameters for the image as seen from + * the camera + * + */ +struct Camera { + af::array position; + af::array lookat; + double fov; + double focal_length; + uint32_t width; + uint32_t height; + + af::array direction; + af::array vertical; + af::array horizontal; + double aspect_ratio; + + Camera(const af::array& position_, const af::array& lookat_, double fov_, + double focal_length_, uint32_t viewport_width_, + uint32_t viewport_height_) + : position(position_) + , lookat(lookat_) + , fov(fov_) + , focal_length(focal_length_) + , width(viewport_width_) + , height(viewport_height_) { + auto global_vertical = af::array(3, {0.0, 0.0, 1.0}); + + // Compute the camera three main axes + direction = normalize3(lookat - position); + horizontal = normalize3(cross_product(direction, global_vertical)); + vertical = normalize3(cross_product(direction, horizontal)); + + aspect_ratio = (double)width / (double)height; + } + + /** + * @brief Generates the initial rays 4-vector position and velocities + * (direction) for the simulation + * + * @return std::pair (pos4, vel4) + */ + std::pair generate_viewport_4rays() { + auto& camera_direction = direction; + auto& camera_horizontal = horizontal; + auto& camera_vertical = vertical; + auto& camera_position = position; + auto vfov = fov; + + double viewport_height = 2.0 * focal_length * std::tan(vfov / 2.0); + double viewport_width = aspect_ratio * viewport_height; + + // Create rays in equally spaced directions of the viewport + af::array viewport_rays = af::constant(0, 3, width, height, f64); + viewport_rays += + (af::iota(af::dim4(1, width, 1), af::dim4(1, 1, height), f64) / + (width - 1) - + 0.5) * + viewport_width * camera_horizontal; + viewport_rays += + (af::iota(af::dim4(1, 1, height), af::dim4(1, width, 1), f64) / + (height - 1) - + 0.5) * + viewport_height * camera_vertical; + viewport_rays += focal_length * camera_direction; + viewport_rays = af::moddims(af::reorder(viewport_rays, 1, 2, 0), + af::dim4(width * height, 3)) + .T(); + + // Compute the initial position from which the rays are launched + af::array viewport_position = viewport_rays + camera_position; + af::array viewport_sph_pos; + if (scene != Scene::ROTATE_BH) + viewport_sph_pos = cart_to_sph_position(viewport_position); + else + viewport_sph_pos = cart_to_oblate_position(viewport_position); + + // Normalize the ray directions + viewport_rays = normalize3(viewport_rays); + + // Generate the position 4-vector + af::array camera_sph_pos; + if (scene != Scene::ROTATE_BH) + camera_sph_pos = cart_to_sph_position(camera_position); + else + camera_sph_pos = cart_to_oblate_position(camera_position); + + af::array camera_pos4 = + af::join(0, af::constant(0.0, 1, f64), camera_sph_pos); + double camera_velocity = + 1.0 / + af::sqrt(norm4(camera_pos4, af::array(4, {1.0, 0.0, 0.0, 0.0}))) + .scalar(); + af::array camera_vel4 = af::array(4, {camera_velocity, 0.0, 0.0, 0.0}); + + af::array viewport_rays_pos4 = af::join( + 0, af::constant(0.0, 1, width * height, f64), viewport_sph_pos); + + // Generate the velocity 4-vector by setting the camera to be stationary + // with respect to an observer at infinity + af::array vv; + if (scene != Scene::ROTATE_BH) + vv = cart_to_sph_velocity(viewport_rays, viewport_position); + else + vv = cart_to_oblate_velocity(viewport_rays, viewport_position); + + af::array vvr = vv(0, af::span); + af::array vvo = vv(1, af::span); + af::array vvp = vv(2, af::span); + auto viewport_sph_rays4 = + af::join(0, af::constant(1, 1, width * height, f64), vvr, vvo, vvp); + + af::array dot = af::moddims( + af::matmul(metric4(viewport_rays_pos4), + af::moddims(viewport_sph_rays4 * viewport_sph_rays4, + af::dim4(4, 1, width * height))), + af::dim4(4, width * height)); + + // Normalize the 4-velocity vectors + af::array viewport_vel = + af::sqrt(-af::array(dot(0, af::span)) / + (dot(1, af::span) + dot(2, af::span) + dot(3, af::span))); + af::array viewport_rays_vel4 = + af::join(0, af::constant(camera_velocity, 1, width * height, f64), + vv * viewport_vel * camera_velocity); + + return {viewport_rays_pos4, viewport_rays_vel4}; + } +}; + +/** + * @brief Object struct + * + * Contains the methods for testing if a ray has collided with the object + * + */ +struct Object { + using HasHit = af::array; + using HitPos = af::array; + + /** + * @brief Gets the color of the pixel that correspond to the ray that has + * intersected with the object + * + * @param ray_begin begining + * @param ray_end + * @return af::array + */ + virtual af::array get_color(const af::array& ray_begin, + const af::array& ray_end) const = 0; + + /** + * @brief Returns a bool array if the rays have hit the object and the + * correspoding position where the ray has hit + * + * @param ray_begin + * @param ray_end + * @return std::pair + */ + virtual std::pair intersect( + const af::array& ray_begin, const af::array& ray_end) const = 0; +}; + +struct AccretionDisk : public Object { + af::array disk_color; + af::array center; + af::array normal; + double inner_radius; + double outter_radius; + + AccretionDisk(const af::array& center, const af::array& normal, + double inner_radius, double outter_radius) + : disk_color(af::array(3, {209.f, 77.f, 0.f})) + , center(center) + , normal(normal) + , inner_radius(inner_radius) + , outter_radius(outter_radius) { + // disk_color = af::array(3, {254.f, 168.f, 29.f}); + } + + std::pair intersect( + const af::array& ray_begin, const af::array& ray_end) const override { + uint32_t count = ray_begin.dims()[1]; + + // Compute intersection of ray with a plane + af::array has_hit = af::constant(0, count).as(b8); + af::array hit_pos = ray_end; + af::array a = dot3(normal, center - ray_begin); + af::array b = dot3(normal, ray_end - ray_begin); + af::array t = af::select(b != 0.0, a / b, (double)0.0); + + af::array plane_intersect = (ray_end - ray_begin) * t + ray_begin; + af::array dist = norm3(plane_intersect - center); + + t = af::abs(t); + + // Determine if the intersection falls inside the disk radius and occurs + // with the current ray segment + has_hit = af::moddims((dist < outter_radius) && (t <= 1.0) && + (t > 0.0) && (dist > inner_radius), + af::dim4(count)); + hit_pos = plane_intersect; + + return {has_hit, hit_pos}; + } + + af::array get_color(const af::array& ray_begin, + const af::array& ray_end) const override { + auto pair = intersect(ray_begin, ray_end); + af::array hit = pair.first; + af::array pos = pair.second; + + auto val = 1.f - (norm3(pos - center).T() - inner_radius) / + (outter_radius - inner_radius); + + af::array color = + disk_color.T() * 1.5f * (val * val * (val * -2.f + 3.f)).as(f32); + + return af::select(af::tile(hit, af::dim4(1, 3)), color, 0.f); + } +}; +/** + * @brief Background struct + * + * Contains the methods for getting the color of background image + * + */ +struct Background { + af::array image; + + Background(const af::array& image_) { image = image_; } + + af::array get_color(const af::array& ray_dir) const { + auto spherical_dir = cart_to_sph_position(ray_dir); + + auto img_height = image.dims()[0]; + auto img_width = image.dims()[1]; + auto count = ray_dir.dims()[1]; + + // Spherical mapping of the direction to a pixel of the image + af::array o = spherical_dir(1, af::span); + af::array p = spherical_dir(2, af::span); + + auto x = (p / af::Pi + 1.0) * img_width / 2.0; + auto y = (o / af::Pi) * img_height; + + // Interpolate the colors of the image from the calculated pixel + // positions + af::array colors = af::approx2(image, af::moddims(y.as(f32), count), + af::moddims(x.as(f32), count), + af::interpType::AF_INTERP_CUBIC_SPLINE); + + // Zero out the color of any null rays + colors = af::moddims(colors, af::dim4(count, 3)); + af::replace(colors, !af::isNaN(colors), 0.f); + + return colors; + } +}; + +/** + * @brief Transform the array of pixels to the correct image format to display + * + * @param image + * @param width + * @param height + * @return af::array + */ +af::array rearrange_image(const af::array& image, uint32_t width, + uint32_t height) { + return af::clamp(af::moddims(image, af::dim4(width, height, 3)).T(), 0.0, + 255.0) + .as(f32) / + 255.f; +} + +/** + * @brief Returns an rgb image containing the raytraced black hole from the + * camera rays, spacetime metric, objects living in the space, and background + * + * @param initial_pos initial position from where the rays are launched + * @param initial_vel initial velocities (directions) the rays have + * @param objects the objects the rays can collide with + * @param background the background of the scene + * @param time how long are the rays traced through space + * @param steps how many steps should be taken to trace the rays path + * @param width width of the image the camera produces + * @param height height of the image the camera produces + * @param checks the intervals between steps to check if the rays have collided + * with an object + * @return af::array + */ +af::array generate_image(const af::array& initial_pos, + const af::array& initial_vel, + const std::vector >& objects, + const Background& background, uint32_t width, + uint32_t height, double time, double tol, + uint32_t checks = 10) { + uint32_t lines = initial_pos.dims()[1]; + + auto def_step = 0.5 * pow(tol, 0.25); + auto dt = af::constant(def_step, 1, lines, f64); + auto t = af::constant(0.0, 1, lines, f64); + auto index = af::iota(lines); + auto selected = t < time; + + auto result = af::constant(0, lines, 3, f32); + + auto pos = initial_pos; + auto vel = initial_vel; + + af::Window window{(int)width, (int)height, "Black Hole Raytracing"}; + + af::array bg_col = af::constant(0.f, lines, 3); + af::array begin_pos, end_pos; + af::array bh_nohit; + + if (scene != Scene::ROTATE_BH) + begin_pos = sph_to_cart_position(pos(af::seq(1, 3), af::span)); + else + begin_pos = oblate_to_cart_position(pos(af::seq(1, 3), af::span)); + end_pos = begin_pos; + + int i = 0; + + while (t.dims()[1] != 0 && af::anyTrue(t < time) && + af::anyTrue(dt != 0.0)) { + // Displays the current progress and approximate time needed to finish + // it + status_bar((lines - t.dims()[1]) * time + + af::sum(af::clamp(t, 0.0, time)), + time * lines, "Progress:"); + + // RK34 method for second order differential equation + auto dt2 = dt * dt; + auto k1 = geodesics(pos, vel); + auto k2 = geodesics(pos + vel * dt / 4.0 + k1 * dt2 / 32.0, + vel + k1 * dt / 4.0); + auto k3 = geodesics(pos + vel * dt / 2.0 + (k1 + k2) * dt2 / 16.0, + vel + k2 * dt / 2.0); + auto k4 = geodesics(pos + vel * dt + (k1 - k2 + k3 * 2.0) * dt2 / 4.0, + vel + (k1 - k2 * 2.0 + 2.0 * k3) * dt); + + auto diff4 = (k1 + k2 * 8.0 + k3 * 2.0 + k4) / 24.0; + auto diff3 = (k2 * 8.0 + k4) / 18.0; + + auto err = (af::max)(af::abs(diff4 - diff3), 0) * dt2; + auto maxerr = tol * (1.0 + (af::max)(af::abs(pos), 0)); + + auto rdt = af::constant(0, 1, dt.dims()[1], f64); + af::replace(rdt, err > maxerr, dt); + + auto rdt2 = rdt * rdt; + + pos += vel * rdt + (k1 + k2 * 8.0 + k3 * 2.0 + k4) * rdt2 / 24.0; + vel += (k1 + k3 * 4.0 + k4) * rdt / 6.0; + t += rdt; + + auto q = af::clamp(0.8 * af::pow(maxerr / err, 0.25), 0.0, 5.0); + + // Select the next time step + dt = af::select(q * dt < (time - t), q * dt, af::abs(time - t)); + + // Update image + if (i % checks == (checks - 1)) { + af::array ray_dir; + if (scene != Scene::ROTATE_BH) { + end_pos(af::span, index) = + sph_to_cart_position(pos(af::seq(1, 3), af::span)); + ray_dir = sph_to_cart_velocity(vel(af::seq(1, 3), af::span), + pos(af::seq(1, 3), af::span)); + } else { + end_pos(af::span, index) = + oblate_to_cart_position(pos(af::seq(1, 3), af::span)); + ray_dir = oblate_to_cart_velocity(vel(af::seq(1, 3), af::span), + pos(af::seq(1, 3), af::span)); + } + + af::array s_begin_pos = begin_pos(af::span, index); + af::array s_end_pos = end_pos(af::span, index); + + // Check if light ray intersect an object + for (const auto& obj : objects) { + result(index, af::span) += + obj->get_color(s_begin_pos, s_end_pos); + } + + // Update background colors from rays + bg_col(index, af::span) = background.get_color(ray_dir); + + // Display image + window.image(rearrange_image(result + bg_col, width, height)); + + begin_pos = end_pos; + } + + // Stop rays entering the event horizon + switch (scene) { + case Scene::ROTATE_BH: { + auto a = J / M; + bh_nohit = + (pos(1, af::span) > 1.01 * (M + std::sqrt(M * M - a * a))); + selected = bh_nohit && (t < time); + + break; + } + + case Scene::STATIC_BH: { + bh_nohit = pos(1, af::span) > 2.0 * M * 1.01; + selected = bh_nohit && (t < time); + + break; + } + + case Scene::WORMHOLE: { + selected = (t < time); + } + default: break; + } + + // Remove finished rays from computation + if (af::sum(selected.as(f32)) / (float)index.dims()[0] < 0.75) { + if (scene == Scene::STATIC_BH || scene == Scene::ROTATE_BH) + bg_col(af::array(index(!bh_nohit)), af::span) = 0.f; + + index = index(selected); + pos = pos(af::span, selected); + vel = vel(af::span, selected); + dt = dt(af::span, selected); + t = t(af::span, selected); + + // Free finished rays memory + af::deviceGC(); + } + + ++i; + } + + result += bg_col; + + return rearrange_image(result, width, height); +} + +void raytracing(uint32_t width, uint32_t height) { + // Set the parameters of the raytraced image + double vfov = radians(90.0); + double focal_length = 0.01; + + // Set the parameters of the camera + af::array global_vertical = af::array(3, {0.0, 0.0, 1.0}); + af::array camera_position = af::array(3, {-7.0, 6.0, 2.0}); + af::array camera_lookat = af::array(3, {0.0, 0.0, 0.0}); + double accretion_inner_radius = M * 3.0; + double accretion_outter_radius = M * 8.0; + double simulation_tolerance = 1e-6; + double max_simulation_time = 12.; + uint32_t num_steps_per_collide_check = 1; + + // Set the background of the scene + auto bg_image = + af::loadimage(ASSETS_DIR "/examples/images/westerlund.jpg", true); + auto background = Background(bg_image); + + // Set the objects living in the scene + std::vector > objects; + if (scene != Scene::WORMHOLE) + objects.push_back(std::make_unique( + af::array(3, {0.0, 0.0, 0.0}), af::array(3, {0.0, 0.0, 1.0}), + accretion_inner_radius, accretion_outter_radius)); + + // Generate rays from the camera + auto camera = Camera(camera_position, camera_lookat, vfov, focal_length, + width, height); + auto pair = camera.generate_viewport_4rays(); + + auto ray4_pos = pair.first; + auto ray4_vel = pair.second; + + auto begin = std::chrono::high_resolution_clock::now(); + // Generate raytraced image + auto image = generate_image( + ray4_pos, ray4_vel, objects, background, width, height, + max_simulation_time, simulation_tolerance, num_steps_per_collide_check); + + auto end = std::chrono::high_resolution_clock::now(); + + std::cout + << "\nSimulation took: " + << std::chrono::duration_cast(end - begin).count() + << " s" << std::endl; + + // Save image + af::saveImage("result.png", image); +} + +int main(int argc, char** argv) { + int device = argc > 1 ? std::atoi(argv[1]) : 0; + + int width = argc > 2 ? std::atoi(argv[2]) : 200; + int height = argc > 3 ? std::atoi(argv[3]) : 200; + + try { + af::setDevice(device); + af::info(); + + std::cout << "** ArrayFire Black Hole Raytracing Demo\n\n"; + + raytracing(width, height); + } catch (const af::exception& e) { + std::cerr << e.what() << std::endl; + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/examples/pde/boltzmann_cfd.cpp b/examples/pde/boltzmann_cfd.cpp new file mode 100644 index 0000000000..38882f3c5c --- /dev/null +++ b/examples/pde/boltzmann_cfd.cpp @@ -0,0 +1,570 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +/* + This is a Computational Fluid Dynamics Simulation using the Lattice + Boltzmann Method For this simulation we are using D2N9 (2 dimensions, 9 + neighbors) with bounce-back boundary conditions For more information on the + simulation equations, check out + https://en.wikipedia.org/wiki/Lattice_Boltzmann_methods#Mathematical_equations_for_simulations + + The initial conditions of the fluid are obtained from three images that + specify their properties using the function read_initial_condition_arrays. + These images can be modified to simulate different cases +*/ + +#include +#include +#include +#include + +/* + Values of the D2N9 grid follow the following order structure: + + + -1 0 1 + * ----------------------> x + -1 | 6 3 0 + | + 0 | 7 4 1 + | + 1 | 8 5 2 + | + v + y + + The (-1, 0, 1) refer to the x and y offsets with respect to a single cell + and the (0-8) refer to indices of each cell in the 3x3 grid + + Eg. Element with index 4 is the center of the grid which has an x-offset = + ex_vals[4] = 0 and y-offset = ey_vals[4] = 0 with its quantities being + weighted with weight wt_vals[4] = 16/36 +*/ + +static const float ex_vals[] = {1.0, 1.0, 1.0, 0.0, 0.0, 0.0, -1.0, -1.0, -1.0}; + +static const float ey_vals[] = {1.0, 0.0, -1.0, 1.0, 0.0, -1.0, 1.0, 0.0, -1.0}; + +static const float wt_vals[] = {1.0f / 36.0f, 4.0f / 36.0f, 1.0f / 36.0f, + 4.0f / 36.0f, 16.0f / 36.0f, 4.0f / 36.0f, + 1.0f / 36.0f, 4.0f / 36.0f, 1.0f / 36.0f}; + +static const int opposite_indices[] = {8, 7, 6, 5, 4, 3, 2, 1, 0}; + +struct Simulation { + // Fluid quantities + af::array ux; + af::array uy; + af::array rho; + af::array sigma; + af::array f; + af::array feq; + + // Constant velocity boundary conditions positions + af::array set_boundaries; + + // Simulation Parameters + size_t grid_width; + size_t grid_height; + float density; + float velocity; + float reynolds; + + // Helper arrays stored for computation + af::array ex; + af::array ey; + af::array wt; + + af::array ex_T; + af::array ey_T; + af::array wt_T; + + af::array ex_; + af::array ey_; +}; + +/** + * @brief Create a simulation object containing all the initial parameters and + * condition of the simulation + * + * @details + * For the ux, uy, and boundary images, we use RGB values for to define the + * specific quantites for each grid cell/pixel + * + * /// R & B for ux & uy + * + * For ux and uy, Red means positive value while Blue means negative value. The + * speed value for both ux and uy is computed as $(R - B) * velocity / 255$. + * + * For example, for the same pixel in the two images if we had ux = RGB(255,0,0) + * and uy = RGB(0,0,255) means that cell's fluid has an x-velocity of +v and + * y-velocity of -v where v is the velocity quantity pass to this function. + * + * Note that having the same value in the R and B components will cancel each + * other out, i.e., have the fluid has 0 velocity in that direction similar to + * having it be 0. + * + * /// G for ux & uy + * + * The G component is reserved for an object or obstacle. Any non-zero value for + * the green component represents a hard boundary in the simulation + * + * /// RGB for boundary + * + * Any non-zero value for any of the components in the RGB value of the pixel + * means that the initial values passed for ux and uy will remain constant + * throught the simulation + * + */ +Simulation create_simulation(uint32_t grid_width, uint32_t grid_height, + float density, float velocity, float reynolds, + const char* ux_image_filename, + const char* uy_image_filename, + const char* boundaries_filename) { + Simulation sim; + + sim.grid_width = grid_width; + sim.grid_height = grid_height; + sim.velocity = velocity; + sim.density = density; + sim.reynolds = reynolds; + + try { + sim.ux = af::loadImage(ux_image_filename, true); + } catch (const af::exception& e) { + std::cerr << e.what() << std::endl; + sim.ux = af::constant(0, grid_width, grid_height, 3); + } + + auto ux_dim = sim.ux.dims(); + if (ux_dim[0] != grid_width || ux_dim[1] != grid_height) { + std::cerr + << "Fluid flow ux image has dimensions different to the simulation" + << std::endl; + throw std::runtime_error{ + "Fluid flow ux image has dimensions different to the simulation"}; + } + + try { + sim.uy = af::loadImage(uy_image_filename, true); + } catch (const af::exception& e) { + std::cerr << e.what() << std::endl; + sim.uy = af::constant(0, grid_width, grid_height, 3); + } + + auto uy_dim = sim.uy.dims(); + if (uy_dim[0] != grid_width || uy_dim[1] != grid_height) { + std::cerr + << "Fluid flow uy image has dimensions different to the simulation" + << std::endl; + throw std::runtime_error{ + "Fluid flow uy image has dimensions different to the simulation"}; + } + + try { + sim.set_boundaries = af::loadImage(boundaries_filename, false); + } catch (const af::exception& e) { + std::cerr << e.what() << std::endl; + sim.set_boundaries = af::constant(0, grid_width, grid_height); + } + + auto b_dim = sim.set_boundaries.dims(); + if (b_dim[0] != grid_width || b_dim[1] != grid_height) { + std::cerr + << "Fluid boundary image has dimensions different to the simulation" + << std::endl; + throw std::runtime_error{ + "Fluid boundary image has dimensions different to the simulation"}; + } + + sim.ux = (sim.ux(af::span, af::span, 0).T() - + sim.ux(af::span, af::span, 2).T()) * + velocity / 255.f; + sim.uy = (sim.uy(af::span, af::span, 0).T() - + sim.uy(af::span, af::span, 2).T()) * + velocity / 255.f; + sim.set_boundaries = sim.set_boundaries.T() > 0; + + return sim; +} + +/** + * @brief Initializes internal values used for computation + * + */ +void initialize(Simulation& sim) { + auto& ux = sim.ux; + auto& uy = sim.uy; + auto& rho = sim.rho; + auto& sigma = sim.sigma; + auto& f = sim.f; + auto& feq = sim.feq; + + auto& ex = sim.ex; + auto& ey = sim.ey; + auto& wt = sim.wt; + auto& ex_ = sim.ex_; + auto& ey_ = sim.ey_; + auto& ex_T = sim.ex_T; + auto& ey_T = sim.ey_T; + auto& wt_T = sim.wt_T; + + auto density = sim.density; + auto velocity = sim.velocity; + auto xcount = sim.grid_width; + auto ycount = sim.grid_height; + + ex = af::array(1, 1, 9, ex_vals); + ey = af::array(1, 1, 9, ey_vals); + wt = af::array(1, 1, 9, wt_vals); + + ex_T = af::array(1, 9, ex_vals); + ey_T = af::array(1, 9, ey_vals); + wt_T = af::moddims(wt, af::dim4(1, 9)); + + rho = af::constant(density, xcount, ycount, f32); + sigma = af::constant(0, xcount, ycount, f32); + + f = af::constant(0, xcount, ycount, 9, f32); + + ex_ = af::tile(ex, xcount, ycount, 1); + ey_ = af::tile(ey, xcount, ycount, 1); + + // Initialization of the distribution function + auto edotu = ex_ * ux + ey_ * uy; + auto udotu = ux * ux + uy * uy; + + feq = rho * wt * + ((edotu * edotu * 4.5f) - (udotu * 1.5f) + (edotu * 3.0f) + 1.0f); + f = feq; +} + +/** + * @brief Updates the particle distribution functions for the new simulation + * frame + * + */ +void collide_stream(Simulation& sim) { + auto& ux = sim.ux; + auto& uy = sim.uy; + auto& rho = sim.rho; + auto& sigma = sim.sigma; + auto& f = sim.f; + auto& feq = sim.feq; + auto& set_boundaries = sim.set_boundaries; + + auto& ex = sim.ex; + auto& ey = sim.ey; + auto& wt = sim.wt; + auto& ex_ = sim.ex_; + auto& ey_ = sim.ey_; + auto& ex_T = sim.ex_T; + auto& ey_T = sim.ey_T; + auto& wt_T = sim.wt_T; + + auto density = sim.density; + auto velocity = sim.velocity; + auto reynolds = sim.reynolds; + auto xcount = sim.grid_width; + auto ycount = sim.grid_height; + + const float viscosity = + velocity * std::sqrt(static_cast(xcount * ycount)) / reynolds; + const float tau = 0.5f + 3.0f * viscosity; + const float csky = 0.16f; + + auto edotu = ex_ * ux + ey_ * uy; + auto udotu = ux * ux + uy * uy; + + // Compute the new distribution function + feq = + rho * wt * (edotu * edotu * 4.5f - udotu * 1.5f + edotu * 3.0f + 1.0f); + + auto taut = + af::sqrt(sigma * (csky * csky * 18.0f * 0.25f) + (tau * tau * 0.25f)) - + (tau * 0.5f); + + // Compute the shifted distribution functions + auto fplus = f - (f - feq) / (taut + tau); + + // Compute new particle distribution according to the corresponding D2N9 + // weights + for (int i = 0; i < 9; ++i) { + int xshift = static_cast(ex_vals[i]); + int yshift = static_cast(ey_vals[i]); + + fplus(af::span, af::span, i) = + af::shift(fplus(af::span, af::span, i), xshift, yshift); + } + + // Keep the boundary conditions at the borders the same + af::replace(fplus, af::tile(!set_boundaries, af::dim4(1, 1, 9)), f); + + // Update the particle distribution + f = fplus; + + // Computing u dot e at the each of the boundaries + af::array ux_top = ux.rows(0, 2); + ux_top = + af::moddims(af::tile(ux_top, af::dim4(1, 3)).T(), af::dim4(ycount, 9)); + af::array ux_bot = ux.rows(xcount - 3, xcount - 1); + ux_bot = + af::moddims(af::tile(ux_bot, af::dim4(1, 3)).T(), af::dim4(ycount, 9)); + + af::array uy_top = uy.rows(0, 2); + uy_top = + af::moddims(af::tile(uy_top, af::dim4(1, 3)).T(), af::dim4(ycount, 9)); + af::array uy_bot = uy.rows(xcount - 3, xcount - 1); + uy_bot = + af::moddims(af::tile(uy_bot, af::dim4(1, 3)).T(), af::dim4(ycount, 9)); + + auto ux_lft = af::tile(ux.cols(0, 2), af::dim4(1, 3)); + auto uy_lft = af::tile(uy.cols(0, 2), af::dim4(1, 3)); + auto ux_rht = af::tile(ux.cols(ycount - 3, ycount - 1), af::dim4(1, 3)); + auto uy_rht = af::tile(uy.cols(ycount - 3, ycount - 1), af::dim4(1, 3)); + + auto ubdoute_top = ux_top * ex_T + uy_top * ey_T; + auto ubdoute_bot = ux_bot * ex_T + uy_bot * ey_T; + auto ubdoute_lft = ux_lft * ex_T + uy_lft * ey_T; + auto ubdoute_rht = ux_rht * ex_T + uy_rht * ey_T; + + // Computing bounce-back boundary conditions + auto fnew_top = af::moddims(fplus.row(1), af::dim4(ycount, 9)) - + 6.0 * density * wt_T * ubdoute_top; + auto fnew_bot = af::moddims(fplus.row(xcount - 2), af::dim4(ycount, 9)) - + 6.0 * density * wt_T * ubdoute_bot; + auto fnew_lft = af::moddims(fplus.col(1), af::dim4(xcount, 9)) - + 6.0 * density * wt_T * ubdoute_lft; + auto fnew_rht = af::moddims(fplus.col(ycount - 2), af::dim4(xcount, 9)) - + 6.0 * density * wt_T * ubdoute_rht; + + // Update the values near the boundaries with the correct bounce-back + // boundary + for (int i = 0; i < 9; ++i) { + int xshift = static_cast(ex_vals[i]); + int yshift = static_cast(ey_vals[i]); + if (xshift == 1) + f(1, af::span, opposite_indices[i]) = fnew_top(af::span, i); + if (xshift == -1) + f(xcount - 2, af::span, opposite_indices[i]) = + fnew_bot(af::span, i); + if (yshift == 1) + f(af::span, 1, opposite_indices[i]) = fnew_lft(af::span, i); + if (yshift == -1) + f(af::span, ycount - 2, opposite_indices[i]) = + fnew_rht(af::span, i); + } +} + +/** + * @brief Updates the velocity field, density and strain at each point in the + * grid + * + */ +void update(Simulation& sim) { + auto& ux = sim.ux; + auto& uy = sim.uy; + auto& rho = sim.rho; + auto& sigma = sim.sigma; + auto& f = sim.f; + auto& feq = sim.feq; + auto& ex = sim.ex; + auto& ey = sim.ey; + + auto e_tile = af::join(3, af::constant(1, 1, 1, 9), ex, ey); + auto result = af::sum(f * e_tile, 2); + + rho = result(af::span, af::span, af::span, 0); + result /= rho; + ux = result(af::span, af::span, af::span, 1); + uy = result(af::span, af::span, af::span, 2); + + // Above code equivalent to + // rho = af::sum(f, 2); + // ux = af::sum(f * ex, 2) / rho; + // uy = af::sum(f * ey, 2) / rho; + + auto product = f - feq; + auto e_product = af::join(3, ex * ex, ex * ey * std::sqrt(2), ey * ey); + + sigma = af::sqrt(af::sum(af::pow(af::sum(product * e_product, 2), 2), 3)); + + // Above code equivalent to + + // auto xx = af::sum(product * ex * ex, 2); + // auto xy = af::sum(product * ex * ey, 2); + // auto yy = af::sum(product * ey * ey, 2); + + // sigma = af::sqrt(xx * xx + xy * xy * 2 + yy * yy); +} + +af::array generate_image(size_t width, size_t height, const Simulation& sim) { + const auto& ux = sim.ux; + const auto& uy = sim.uy; + const auto& boundaries = sim.set_boundaries; + auto velocity = sim.velocity; + + float image_scale = + static_cast(width) / static_cast(sim.grid_width - 1); + + // Relative Flow speed at each cell + auto val = af::sqrt(ux * ux + uy * uy) / velocity; + + af::replace(val, val != 0 || !boundaries, -1.0); + + // Scaling and interpolating flow speed to the window size + if (width != sim.grid_width || height != sim.grid_height) + val = + af::approx2(val, af::iota(width, af::dim4(1, height)) / image_scale, + af::iota(height, af::dim4(1, width)).T() / image_scale); + + // Flip image + val = val.T(); + + auto image = af::constant(0, height, width, 3); + auto image2 = image; + + // Add custom coloring + image(af::span, af::span, 0) = val * 2; + image(af::span, af::span, 1) = val * 2; + image(af::span, af::span, 2) = 1.0 - val * 2; + + image2(af::span, af::span, 0) = 1; + image2(af::span, af::span, 1) = -2 * val + 2; + image2(af::span, af::span, 2) = 0; + + auto tile_val = af::tile(val, 1, 1, 3); + af::replace(image, tile_val < 0.5, image2); + af::replace(image, tile_val >= 0, 0.0); + + return image; +} + +void lattice_boltzmann_cfd_demo() { + // Define the lattice for the simulation + const size_t len = 128; + const size_t grid_width = len; + const size_t grid_height = len; + + // Specify the image scaling displayed + float scale = 4.0f; + + // Forge window initialization + int height = static_cast(grid_width * scale); + int width = static_cast(grid_height * scale); + af::Window window(height, width, "Driven Cavity Flow"); + + int frame_count = 0; + int max_frames = 20000; + int simulation_frames = 100; + float total_time = 0; + float total_time2 = 0; + + // CFD fluid parameters + const float density = 2.7f; + const float velocity = 0.35f; + const float reynolds = 1e5f; + + const char* ux_image = ASSETS_DIR "/examples/images/default_ux.bmp"; + const char* uy_image = ASSETS_DIR "/examples/images/default_uy.bmp"; + const char* set_boundary_image = + ASSETS_DIR "/examples/images/default_boundary.bmp"; + + // Tesla Valve Fluid Simulation - entering from constricted side + { + // ux_image = ASSETS_DIR "/examples/images/left_tesla_ux.bmp"; + // uy_image = ASSETS_DIR "/examples/images/left_tesla_uy.bmp"; + // set_boundary_image = ASSETS_DIR + // "/examples/images/left_tesla_boundary.bmp"; + } + + // Tesla Valve Fluid Simulation - entering from transfer side + { + // ux_image = ASSETS_DIR + // "/examples/images/right_tesla_ux.bmp"; uy_image = + // ASSETS_DIR "/examples/images/right_tesla_uy.bmp"; + // set_boundary_image = ASSETS_DIR + // "/examples/images/right_tesla_boundary.bmp"; + } + + // Reads the initial values of fluid quantites and simulation parameters + Simulation sim = + create_simulation(grid_width, grid_height, density, velocity, reynolds, + ux_image, uy_image, set_boundary_image); + + // Initializes the simulation quantites + initialize(sim); + + while (!window.close() && frame_count != max_frames) { + af::sync(); + auto begin = std::chrono::high_resolution_clock::now(); + + // Computes the new particle distribution functions for the new + // simulation frame + collide_stream(sim); + + // Updates the velocity, density, and stress fields + update(sim); + + af::sync(); + auto end = std::chrono::high_resolution_clock::now(); + + // Calculate computation time of 1 simulation frame + auto duration = + std::chrono::duration_cast(end - begin) + .count(); + + // Used for computing the distribution of frame computation time + total_time += duration; + total_time2 += duration * duration; + + // Every number of `simulation_frames` display the last computed frame + // to the screen + if (frame_count % simulation_frames == 0) { + auto image = generate_image(width, height, sim); + + // Display colored image + window.image(image); + + float avg_time = total_time / (float)simulation_frames; + float stdv_time = std::sqrt(total_time2 * simulation_frames - + total_time * total_time) / + (float)simulation_frames; + + std::cout << "Average Simulation Step Time: (" << avg_time + << " +/- " << stdv_time + << ") us; Total simulation time: " << total_time + << " us; Simulation Frames: " << simulation_frames + << std::endl; + + total_time = 0; + total_time2 = 0; + } + + frame_count++; + } +} + +int main(int argc, char** argv) { + int device = argc > 1 ? std::atoi(argv[1]) : 0; + + try { + af::setDevice(device); + af::info(); + + std::cout << "** ArrayFire CFD Simulation Demo\n\n"; + + lattice_boltzmann_cfd_demo(); + } catch (const af::exception& e) { + std::cerr << e.what() << std::endl; + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/examples/unified/CMakeLists.txt b/examples/unified/CMakeLists.txt index 330a9c4af7..a399f58c00 100644 --- a/examples/unified/CMakeLists.txt +++ b/examples/unified/CMakeLists.txt @@ -5,12 +5,12 @@ # The complete license agreement can be obtained at: # http://arrayfire.com/licenses/BSD-3-Clause -cmake_minimum_required(VERSION 3.0) +cmake_minimum_required(VERSION 3.5) project(ArrayFire-Example-Unified VERSION 3.5.0 LANGUAGES CXX) -find_package(ArrayFire) +find_package(ArrayFire REQUIRED) if(ArrayFire_Unified_FOUND) # Simple unified backend example diff --git a/include/af/algorithm.h b/include/af/algorithm.h index 801792a32a..4949d0894d 100644 --- a/include/af/algorithm.h +++ b/include/af/algorithm.h @@ -16,62 +16,60 @@ namespace af class array; /** - C++ Interface for sum of elements in an array + C++ Interface to sum array elements over a given dimension. - \param[in] in is the input array - \param[in] dim The dimension along which the add operation occurs - \return result of sum all values along dimension \p dim + \param[in] in input array + \param[in] dim dimension along which the summation occurs, -1 denotes + the first non-singleton dimension + \return sum \ingroup reduce_func_sum - - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. */ AFAPI array sum(const array &in, const int dim = -1); #if AF_API_VERSION >= 31 /** - C++ Interface for sum of elements in an array while replacing nan values + C++ Interface to sum array elements over a given dimension, replacing + any NaNs with a specified value. - \param[in] in is the input array - \param[in] dim The dimension along which the add operation occurs - \param[in] nanval The value that will replace the NaNs in \p in - \return result of sum all values along dimension \p dim + \param[in] in input array + \param[in] dim dimension along which the summation occurs + \param[in] nanval value that replaces NaNs + \return sum \ingroup reduce_func_sum - */ AFAPI array sum(const array &in, const int dim, const double nanval); #endif #if AF_API_VERSION >= 37 /** - C++ Interface for sum of elements along given dimension by key + C++ Interface to sum array elements over a given dimension, according to + an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the sum of all values in \p vals along - \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the add operation occurs + \param[out] keys_out reduced keys + \param[out] vals_out sum + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the summation occurs, -1 + denotes the first non-singleton dimension \ingroup reduce_func_sum_by_key - - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. */ AFAPI void sumByKey(array &keys_out, array &vals_out, const array &keys, const array &vals, - const int dim=-1); + const int dim = -1); /** - C++ Interface for sum of elements along given dimension by key while replacing nan values + C++ Interface to sum array elements over a given dimension, replacing + any NaNs with a specified value, according to an array of keys. - \param[out] keys_out Will contain the reduced keys in \p vals along \p dim - \param[out] vals_out Will contain the sum of all values in \p vals along - \p dim according to \p keys - \param[in] keys Is the key array - \param[in] vals Is the array containing the values to be reduced - \param[in] dim The dimension along which the add operation occurs - \param[in] nanval The value that will replace the NaNs in \p vals + \param[out] keys_out reduced keys + \param[out] vals_out sum + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the summation occurs + \param[in] nanval value that replaces NaNs \ingroup reduce_func_sum_by_key */ @@ -81,27 +79,26 @@ namespace af #endif /** - C++ Interface for product of elements in an array + C++ Interface to multiply array elements over a given dimension. - \param[in] in The input array - \param[in] dim The dimension along which the multiply operation occurs - \return result of product all values along dimension \p dim + \param[in] in input array + \param[in] dim dimension along which the product occurs, -1 denotes the + first non-singleton dimension + \return product \ingroup reduce_func_product - - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. */ AFAPI array product(const array &in, const int dim = -1); #if AF_API_VERSION >= 31 /** - C++ Interface for product of elements in an array while replacing nan - values + C++ Interface to multiply array elements over a given dimension, + replacing any NaNs with a specified value. - \param[in] in The input array - \param[in] dim The dimension along which the multiply operation occurs - \param[in] nanval The value that will replace the NaNs in \p in - \return result of product all values along dimension \p dim + \param[in] in input array + \param[in] dim dimension along which the product occurs + \param[in] nanval value that replaces NaNs + \return product \ingroup reduce_func_product */ @@ -110,35 +107,33 @@ namespace af #if AF_API_VERSION >= 37 /** - C++ Interface for product of elements in an array according to a key + C++ Interface to multiply array elements over a given dimension, + according to an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the product of all values in \p vals - along \p dim according to \p keys - \param[in] keys The key array - \param[in] vals The array containing the values to be reduced - \param[in] dim The dimension along which the product operation occurs + \param[out] keys_out reduced keys + \param[out] vals_out product + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the product occurs, -1 + denotes the first non-singleton dimension \ingroup reduce_func_product_by_key - - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. */ AFAPI void productByKey(array &keys_out, array &vals_out, const array &keys, const array &vals, const int dim = -1); /** - C++ Interface for product of elements in an array according to a key - while replacing nan values + C++ Interface to multiply array elements over a given dimension, + replacing any NaNs with a specified value, according to an array of + keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p - dim - \param[out] vals_out will contain the product of all values in \p - vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the product operation occurs - \param[in] nanval The value that will replace the NaNs in \p vals + \param[out] keys_out reduced keys + \param[out] vals_out product + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the product occurs + \param[in] nanval value that replaces NaNs \ingroup reduce_func_product_by_key @@ -149,33 +144,34 @@ namespace af #endif /** - C++ Interface for minimum values in an array + C++ Interface to return the minimum along a given dimension. - \param[in] in is the input array - \param[in] dim The dimension along which the minimum value needs to be extracted - \return result of minimum all values along dimension \p dim + NaN values are ignored. - \ingroup reduce_func_min + \param[in] in input array + \param[in] dim dimension along which the minimum is found, -1 denotes + the first non-singleton dimension + \return minimum - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. - \note NaN values are ignored + \ingroup reduce_func_min */ AFAPI array min(const array &in, const int dim = -1); #if AF_API_VERSION >= 37 /** - C++ Interface for minimum values in an array according to a key + C++ Interface to return the minimum along a given dimension, according + to an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the minimum of all values in \p vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the min operation occurs + NaN values are ignored. - \ingroup reduce_func_min_by_key + \param[out] keys_out reduced keys + \param[out] vals_out minimum + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the minimum is found, -1 + denotes the first non-singleton dimension - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. - \note NaN values are ignored + \ingroup reduce_func_min_by_key */ AFAPI void minByKey(array &keys_out, array &vals_out, const array &keys, const array &vals, @@ -183,33 +179,34 @@ namespace af #endif /** - C++ Interface for maximum values in an array + C++ Interface to return the maximum along a given dimension. - \param[in] in is the input array - \param[in] dim The dimension along which the maximum value needs to be extracted - \return result of maximum all values along dimension \p dim + NaN values are ignored. - \ingroup reduce_func_max + \param[in] in input array + \param[in] dim dimension along which the maximum is found, -1 denotes + the first non-singleton dimension + \return maximum - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. - \note NaN values are ignored + \ingroup reduce_func_max */ AFAPI array max(const array &in, const int dim = -1); #if AF_API_VERSION >= 37 /** - C++ Interface for maximum values in an array according to a key + C++ Interface to return the maximum along a given dimension, according + to an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the maximum of all values in \p vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the max operation occurs + NaN values are ignored. - \ingroup reduce_func_max_by_key + \param[out] keys_out reduced keys + \param[out] vals_out maximum + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the maximum is found, -1 + denotes the first non-singleton dimension - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. - \note NaN values are ignored + \ingroup reduce_func_max_by_key */ AFAPI void maxByKey(array &keys_out, array &vals_out, const array &keys, const array &vals, @@ -218,50 +215,51 @@ namespace af #if AF_API_VERSION >= 38 /** - C++ Interface for ragged max values in an array - Uses an additional input array to determine the number of elements to use along the reduction axis. + C++ Interface to return the ragged maximum along a given dimension. - \param[out] val will contain the maximum ragged values in \p in along \p dim according to \p ragged_len - \param[out] idx will contain the locations of the maximum ragged values in \p in along \p dim according to \p ragged_len - \param[in] in contains the input values to be reduced - \param[in] ragged_len array containing number of elements to use when reducing along \p dim - \param[in] dim The dimension along which the max operation occurs + Input parameter `ragged_len` sets the number of elements to consider. - \ingroup reduce_func_max + NaN values are ignored. + + \param[out] val ragged maximum + \param[out] idx locations of the maximum ragged values + \param[in] in input array + \param[in] ragged_len array containing the number of elements to use + \param[in] dim dimension along which the maximum is found - \note NaN values are ignored + \ingroup reduce_func_max */ AFAPI void max(array &val, array &idx, const array &in, const array &ragged_len, const int dim); #endif /** - C++ Interface for checking all true values in an array + C++ Interface to check if all values along a given dimension are true. - \param[in] in is the input array - \param[in] dim The dimension along which the values are checked to be all true - \return result of checking if values along dimension \p dim are all true + NaN values are ignored. - \ingroup reduce_func_all_true + \param[in] in input array + \param[in] dim dimension along which the check occurs, -1 denotes the + first non-singleton dimension + \return array containing 1's if all true; 0's otherwise - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. - \note NaN values are ignored + \ingroup reduce_func_all_true */ AFAPI array allTrue(const array &in, const int dim = -1); #if AF_API_VERSION >= 37 /** - C++ Interface for checking all true values in an array according to a key + C++ Interface to check if all values along a given dimension are true, + according to an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the reduced and of all values in \p vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the all true operation occurs + NaN values are ignored. - \ingroup reduce_func_alltrue_by_key + \param[out] keys_out reduced keys + \param[out] vals_out array containing 1's if all true; 0's otherwise + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the check occurs - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. - \note NaN values are ignored + \ingroup reduce_func_alltrue_by_key */ AFAPI void allTrueByKey(array &keys_out, array &vals_out, const array &keys, const array &vals, @@ -269,33 +267,33 @@ namespace af #endif /** - C++ Interface for checking any true values in an array + C++ Interface to check if any values along a given dimension are true. - \param[in] in is the input array - \param[in] dim The dimension along which the values are checked to be any true - \return result of checking if values along dimension \p dim are any true + NaN values are ignored. - \ingroup reduce_func_any_true + \param[in] in input array + \param[in] dim dimension along which the check occurs, -1 denotes the + first non-singleton dimension + \return array containing 1's if any true; 0's otherwise - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. - \note NaN values are ignored + \ingroup reduce_func_any_true */ AFAPI array anyTrue(const array &in, const int dim = -1); #if AF_API_VERSION >= 37 /** - C++ Interface for checking any true values in an array according to a key + C++ Interface to check if any values along a given dimension are true, + according to an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the reduced or of all values in \p vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the any true operation occurs + NaN values are ignored. - \ingroup reduce_func_anytrue_by_key + \param[out] keys_out reduced keys + \param[out] vals_out array containing 1's if any true; 0's otherwise + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the check occurs - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. - \note NaN values are ignored + \ingroup reduce_func_anytrue_by_key */ AFAPI void anyTrueByKey(array &keys_out, array &vals_out, const array &keys, const array &vals, @@ -303,33 +301,35 @@ namespace af #endif /** - C++ Interface for counting non-zero values in an array + C++ Interface to count non-zero values in an array along a given + dimension. - \param[in] in is the input array - \param[in] dim The dimension along which the the number of non-zero values are counted - \return the number of non-zero values along dimension \p dim + NaN values are treated as non-zero. - \ingroup reduce_func_count + \param[in] in input array + \param[in] dim dimension along which the count occurs, -1 denotes the + first non-singleton dimension + \return count - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. - \note NaN values are treated as non zero. + \ingroup reduce_func_count */ AFAPI array count(const array &in, const int dim = -1); #if AF_API_VERSION >= 37 /** - C++ Interface for counting non-zero values in an array according to a key + C++ Interface to count non-zero values in an array, according to an + array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the count of all values in \p vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the count operation occurs + NaN values are treated as non-zero. - \ingroup reduce_func_count_by_key + \param[out] keys_out reduced keys + \param[out] vals_out count + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the count occurs, -1 denotes + the first non-singleton dimension - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. - \note NaN values are treated as non zero. + \ingroup reduce_func_count_by_key */ AFAPI void countByKey(array &keys_out, array &vals_out, const array &keys, const array &vals, @@ -337,10 +337,13 @@ namespace af #endif /** - C++ Interface for sum of all elements in an array + C++ Interface to sum array elements over all dimensions. - \param[in] in is the input array - \return the sum of all values of \p in + Results in a single value as an output, which may be a single element + `af::array`. + + \param[in] in input array + \return sum \ingroup reduce_func_sum */ @@ -348,12 +351,15 @@ namespace af #if AF_API_VERSION >= 31 /** - C++ Interface for sum of all elements in an array while replacing nan - values + C++ Interface to sum array elements over all dimensions, replacing any + NaNs with a specified value. + + Results in a single value as an output, which may be a single element + `af::array`. - \param[in] in is the input array - \param[in] nanval The value that will replace the NaNs in \p in - \return the sum of all values of \p in + \param[in] in input array + \param[in] nanval value that replaces NaNs + \return sum \ingroup reduce_func_sum */ @@ -361,10 +367,11 @@ namespace af #endif /** - C++ Interface for product of all elements in an array + C++ Interface to multiply array elements over the first non-singleton + dimension. - \param[in] in is the input array - \return the product of all values of \p in + \param[in] in input array + \return product \ingroup reduce_func_product */ @@ -372,143 +379,155 @@ namespace af #if AF_API_VERSION >= 31 /** - C++ Interface for product of all elements in an array while replacing nan - values + C++ Interface to multiply array elements over the first non-singleton + dimension, replacing any NaNs with a specified value. - \param[in] in is the input array - \param[in] nanval The value that will replace the NaNs in \p in - \return the product of all values of \p in + \param[in] in input array + \param[in] nanval value that replaces NaNs + \return product \ingroup reduce_func_product */ template T product(const array &in, double nanval); #endif - /** - C++ Interface for getting minimum value of an array + C++ Interface to return the minimum along the first non-singleton + dimension. - \param[in] in is the input array - \return the minimum of all values of \p in + NaN values are ignored. - \ingroup reduce_func_min + \param[in] in input array + \return minimum - \note NaN values are ignored + \ingroup reduce_func_min */ template T min(const array &in); /** - C++ Interface for getting maximum value of an array + C++ Interface to return the maximum along the first non-singleton + dimension. - \param[in] in is the input array - \return the maximum of all values of \p in + NaN values are ignored. - \ingroup reduce_func_max + \param[in] in input array + \return maximum - \note NaN values are ignored + \ingroup reduce_func_max */ template T max(const array &in); /** - C++ Interface for checking if all values in an array are true + C++ Interface to check if all values along the first non-singleton + dimension are true. - \param[in] in is the input array - \return true if all values of \p in are true, false otherwise + NaN values are ignored. - \ingroup reduce_func_all_true + \param[in] in input array + \return array containing 1's if all true; 0's otherwise - \note NaN values are ignored + \ingroup reduce_func_all_true */ template T allTrue(const array &in); /** - C++ Interface for checking if any values in an array are true + C++ Interface to check if any values along the first non-singleton + dimension are true. - \param[in] in is the input array - \return true if any values of \p in are true, false otherwise + NaN values are ignored. - \ingroup reduce_func_any_true + \param[in] in input array + \return array containing 1's if any true; 0's otherwise - \note NaN values are ignored + \ingroup reduce_func_any_true */ template T anyTrue(const array &in); /** - C++ Interface for counting total number of non-zero values in an array + C++ Interface to count non-zero values along the first non-singleton + dimension. - \param[in] in is the input array - \return the number of non-zero values in \p in + NaN values are treated as non-zero. - \ingroup reduce_func_count + \param[in] in input array + \return count - \note NaN values are treated as non zero + \ingroup reduce_func_count */ template T count(const array &in); /** - C++ Interface for getting minimum values and their locations in an array - - \param[out] val will contain the minimum values along dimension \p dim - \param[out] idx will contain the locations of minimum all values along dimension \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the minimum value needs to be extracted + C++ Interface to return the minimum and its location along a given + dimension. - \ingroup reduce_func_min + NaN values are ignored. - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. + \param[out] val minimum + \param[out] idx location + \param[in] in input array + \param[in] dim dimension along which the minimum is found, -1 denotes + the first non-singleton dimension - \note NaN values are ignored + \ingroup reduce_func_min */ AFAPI void min(array &val, array &idx, const array &in, const int dim = -1); /** - C++ Interface for getting maximum values and their locations in an array - - \param[out] val will contain the maximum values along dimension \p dim - \param[out] idx will contain the locations of maximum all values along dimension \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the maximum value needs to be extracted + C++ Interface to return the maximum and its location along a given + dimension. - \ingroup reduce_func_max + NaN values are ignored. - \note \p dim is -1 by default. -1 denotes the first non-singleton dimension. + \param[out] val maximum + \param[out] idx location + \param[in] in input array + \param[in] dim dimension along which the maximum is found, -1 denotes + the first non-singleton dimension - \note NaN values are ignored + \ingroup reduce_func_max */ AFAPI void max(array &val, array &idx, const array &in, const int dim = -1); /** - C++ Interface for getting minimum value and its location from the entire array + C++ Interface to return the minimum and its location over all + dimensions. - \param[out] val will contain the minimum values in the input - \param[out] idx will contain the locations of minimum all values in the input - \param[in] in is the input array + NaN values are ignored. - \ingroup reduce_func_min + Often used to return values directly to the host. + + \param[out] val minimum + \param[out] idx location + \param[in] in input array - \note NaN values are ignored + \ingroup reduce_func_min */ template void min(T *val, unsigned *idx, const array &in); /** - C++ Interface for getting maximum value and its location from the entire array + C++ Interface to return the maximum and its location over all + dimensions. - \param[out] val contains the maximum values in the input - \param[out] idx contains the locations of maximum all values in the input - \param[in] in is the input array + NaN values are ignored. - \ingroup reduce_func_max + Often used to return values directly to the host. - \note NaN values are ignored + \param[out] val maximum + \param[out] idx location + \param[in] in input array + + \ingroup reduce_func_max */ template void max(T *val, unsigned *idx, const array &in); /** - C++ Interface for computing the cumulative sum (inclusive) of an array + C++ Interface to evaluate the cumulative sum (inclusive) along a given + dimension. - \param[in] in is the input array - \param[in] dim is the dimension along which the inclusive sum is calculated - \return the output containing inclusive sums of the input + \param[in] in input array + \param[in] dim dimension along which the sum is accumulated, 0 denotes + the first non-singleton dimension + \return cumulative sum \ingroup scan_func_accum */ @@ -516,13 +535,14 @@ namespace af #if AF_API_VERSION >=34 /** - C++ Interface generalized scan of an array + C++ Interface to scan an array (generalized) over a given dimension. - \param[in] in is the input array - \param[in] dim The dimension along which scan is performed - \param[in] op is the type of binary operation used - \param[in] inclusive_scan is flag specifying whether scan is inclusive - \return the output containing scan of the input + \param[in] in input array + \param[in] dim dimension along which the scan occurs, 0 + denotes the first non-singleton dimension + \param[in] op type of binary operation used + \param[in] inclusive_scan flag specifying whether the scan is inclusive + \return scan \ingroup scan_func_scan */ @@ -530,14 +550,16 @@ namespace af binaryOp op = AF_BINARY_ADD, bool inclusive_scan = true); /** - C++ Interface generalized scan by key of an array + C++ Interface to scan an array (generalized) over a given dimension, + according to an array of keys. - \param[in] key is the key array - \param[in] in is the input array - \param[in] dim The dimension along which scan is performed - \param[in] op is the type of binary operations used - \param[in] inclusive_scan is flag specifying whether scan is inclusive - \return the output containing scan of the input + \param[in] key keys array + \param[in] in input array + \param[in] dim dimension along which the scan occurs, 0 + denotes the first non-singleton dimension + \param[in] op type of binary operation used + \param[in] inclusive_scan flag specifying whether the scan is inclusive + \return scan \ingroup scan_func_scanbykey */ @@ -546,44 +568,49 @@ namespace af #endif /** - C++ Interface for finding the locations of non-zero values in an array + C++ Interface to locate the indices of the non-zero values in an array. - \param[in] in is the input array. - \return linear indices where \p in is non-zero + \param[in] in input array + \return linear indices where `in` is non-zero \ingroup scan_func_where */ AFAPI array where(const array &in); /** - C++ Interface for calculating first order differences in an array + C++ Interface to calculate the first order difference in an array over a + given dimension. - \param[in] in is the input array - \param[in] dim The dimension along which numerical difference is performed - \return array of first order numerical difference + \param[in] in input array + \param[in] dim dimension along which the difference occurs, 0 + denotes the first non-singleton dimension + \return first order numerical difference \ingroup calc_func_diff1 */ AFAPI array diff1(const array &in, const int dim = 0); /** - C++ Interface for calculating second order differences in an array + C++ Interface to calculate the second order difference in an array over + a given dimension. - \param[in] in is the input array - \param[in] dim The dimension along which numerical difference is performed - \return array of second order numerical difference + \param[in] in input array + \param[in] dim dimension along which the difference occurs, 0 + denotes the first non-singleton dimension + \return second order numerical difference \ingroup calc_func_diff2 */ AFAPI array diff2(const array &in, const int dim = 0); /** - C++ Interface for sorting an array + C++ Interface to sort an array over a given dimension. - \param[in] in is the input array - \param[in] dim The dimension along which numerical difference is performed + \param[in] in input array + \param[in] dim dimension along which the sort occurs, 0 denotes + the first non-singleton dimension \param[in] isAscending specifies the sorting order - \return the sorted output + \return sorted output \ingroup sort_func_sort */ @@ -591,27 +618,32 @@ namespace af const bool isAscending = true); /** - C++ Interface for sorting an array and getting original indices + C++ Interface to sort an array over a given dimension and to return the + original indices. - \param[out] out will contain the sorted output - \param[out] indices will contain the indices in the original input - \param[in] in is the input array - \param[in] dim The dimension along which numerical difference is performed - \param[in] isAscending specifies the sorting order + \param[out] out sorted output + \param[out] indices indices from the input + \param[in] in input array + \param[in] dim dimension along which the sort occurs, 0 denotes + the first non-singleton dimension + \param[in] isAscending specifies the sorting order \ingroup sort_func_sort_index */ AFAPI void sort(array &out, array &indices, const array &in, const unsigned dim = 0, const bool isAscending = true); + /** - C++ Interface for sorting an array based on keys + C++ Interface to sort an array over a given dimension, according to an + array of keys. - \param[out] out_keys will contain the keys based on sorted values - \param[out] out_values will contain the sorted values - \param[in] keys is the input array - \param[in] values The dimension along which numerical difference is performed - \param[in] dim The dimension along which numerical difference is performed - \param[in] isAscending specifies the sorting order + \param[out] out_keys sorted keys + \param[out] out_values sorted output + \param[in] keys keys array + \param[in] values input array + \param[in] dim dimension along which the sort occurs, 0 denotes + the first non-singleton dimension + \param[in] isAscending specifies the sorting order \ingroup sort_func_sort_keys */ @@ -620,23 +652,23 @@ namespace af const bool isAscending = true); /** - C++ Interface for getting unique values + C++ Interface to return the unique values in an array. - \param[in] in is the input array - \param[in] is_sorted if true, skips the sorting steps internally - \return the unique values from \p in + \param[in] in input array + \param[in] is_sorted if true, skip the sorting steps internally + \return unique values \ingroup set_func_unique */ AFAPI array setUnique(const array &in, const bool is_sorted=false); /** - C++ Interface for finding the union of two arrays + C++ Interface to evaluate the union of two arrays. - \param[in] first is the first input array - \param[in] second is the second input array - \param[in] is_unique if true, skips calling unique internally - \return all unique values present in \p first and \p second (union) in increasing order + \param[in] first input array + \param[in] second input array + \param[in] is_unique if true, skip calling setUnique internally + \return union, values in increasing order \ingroup set_func_union */ @@ -644,12 +676,12 @@ namespace af const bool is_unique=false); /** - C++ Interface for finding the intersection of two arrays + C++ Interface to evaluate the intersection of two arrays. - \param[in] first is the first input array - \param[in] second is the second input array - \param[in] is_unique if true, skips calling unique internally - \return unique values that are present in both \p first and \p second(intersection) in increasing order + \param[in] first input array + \param[in] second input array + \param[in] is_unique if true, skip calling setUnique internally + \return intersection, values in increasing order \ingroup set_func_intersect */ @@ -663,12 +695,13 @@ extern "C" { #endif /** - C Interface for sum of elements in an array + C Interface to sum array elements over a given dimension. - \param[out] out will contain the sum of all values in \p in along \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the add operation occurs - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out sum + \param[in] in input array + \param[in] dim dimension along which the summation occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_sum */ @@ -676,11 +709,14 @@ extern "C" { #if AF_API_VERSION >= 39 /** - C Interface for sum of all elements in an array, resulting in an array + C Interface to sum array elements over all dimensions. - \param[out] out will contain the sum of all values in \p in - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + Results in a single element `af::array`. + + \param[out] out sum + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_sum */ @@ -689,13 +725,15 @@ extern "C" { #if AF_API_VERSION >= 31 /** - C Interface for sum of elements in an array while replacing nans + C Interface to sum array elements over a given dimension, replacing any + NaNs with a specified value. - \param[out] out will contain the sum of all values in \p in along \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the add operation occurs - \param[in] nanval The value that will replace the NaNs in \p in - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out sum + \param[in] in input array + \param[in] dim dimension along which the summation occurs + \param[in] nanval value that replaces NaNs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_sum */ @@ -705,13 +743,16 @@ extern "C" { #if AF_API_VERSION >= 39 /** - C Interface for sum of all elements in an array, resulting in an array with - nan substitution + C Interface to sum array elements over all dimensions, replacing any + NaNs with a specified value. + + Results in a single element `af::array`. - \param[out] out will contain the sum of all values in \p in - \param[in] in is the input array - \param[in] nanval The value that will replace the NaNs in \p in - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out sum + \param[in] in input array + \param[in] nanval value that replaces NaNs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_sum */ @@ -720,14 +761,16 @@ extern "C" { #if AF_API_VERSION >= 37 /** - C Interface for sum of elements in an array according to key + C Interface to sum array elements over a given dimension, according to + an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the sum of all values in \p vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the add operation occurs - \return \ref AF_SUCCESS if the execution completes properly + \param[out] keys_out reduced keys + \param[out] vals_out sum + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the summation occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_sum_by_key */ @@ -735,20 +778,17 @@ extern "C" { const af_array keys, const af_array vals, const int dim); /** - C Interface for sum of elements in an array according to key while - replacing nans - - \param[out] keys_out will contain the reduced keys in \p vals along \p - dim - \param[out] vals_out will contain the sum of all values in \p vals - along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the add operation occurs - \param[in] nanval The value that will replace the NaNs in \p vals + C Interface to sum array elements over a given dimension, replacing any + NaNs with a specified value, according to an array of keys. - - \return \ref AF_SUCCESS if the execution completes properly + \param[out] keys_out reduced keys + \param[out] vals_out sum + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the summation occurs + \param[in] nanval value that replaces NaNs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_sum_by_key */ @@ -758,12 +798,13 @@ extern "C" { #endif /** - C Interface for product of elements in an array + C Interface to multiply array elements over a given dimension. - \param[out] out will contain the product of all values in \p in along \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the multiply operation occurs - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out product + \param[in] in input array + \param[in] dim dimension along which the product occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_product */ @@ -771,11 +812,14 @@ extern "C" { #if AF_API_VERSION >= 39 /** - C Interface for product of elements in an array, resulting in an array + C Interface to multiply array elements over all dimensions. + + Results in a single element `af::array`. - \param[out] out will contain the product of all values in \p in - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out product + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_product */ @@ -784,14 +828,15 @@ extern "C" { #if AF_API_VERSION >= 31 /** - C Interface for product of elements in an array while replacing nans + C Interface to multiply array elements over a given dimension, replacing + any NaNs with a specified value. - \param[out] out will contain the product of all values in \p in along \p - dim - \param[in] in is the input array - \param[in] dim The dimension along which the product operation occurs - \param[in] nanval The value that will replace the NaNs in \p in - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out product + \param[in] in input array + \param[in] dim dimension along with the product occurs + \param[in] nanval value that replaces NaNs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_product */ @@ -800,13 +845,14 @@ extern "C" { #if AF_API_VERSION >= 39 /** - C Interface for product of elements in an array, resulting in an array - while replacing nans + C Interface to multiply array elements over all dimensions, replacing + any NaNs with a specified value. - \param[out] out will contain the product of all values in \p in - \param[in] in is the input array - \param[in] nanval The value that will replace the NaNs in \p in - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out product + \param[in] in input array + \param[in] nanval value that replaces NaNs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_product */ @@ -815,14 +861,16 @@ extern "C" { #if AF_API_VERSION >= 37 /** - C Interface for product of elements in an array according to key + C Interface to multiply array elements over a given dimension, according + to an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the product of all values in \p vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the product operation occurs - \return \ref AF_SUCCESS if the execution completes properly + \param[out] keys_out reduced keys + \param[out] vals_out product + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the product occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_product_by_key */ @@ -830,18 +878,17 @@ extern "C" { const af_array keys, const af_array vals, const int dim); /** - C Interface for product of elements in an array according to key while - replacing nans + C Interface to multiply array elements over a given dimension, replacing + any NaNs with a specified value, according to an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p - dim - \param[out] vals_out will contain the product of all values in \p - vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the product operation occurs - \param[in] nanval The value that will replace the NaNs in \p vals - \return \ref AF_SUCCESS if the execution completes properly + \param[out] keys_out reduced keys + \param[out] vals_out product + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the product occurs + \param[in] nanval value that replaces NaNs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_product_by_key */ @@ -851,12 +898,13 @@ extern "C" { #endif /** - C Interface for minimum values in an array + C Interface to return the minimum along a given dimension. - \param[out] out will contain the minimum of all values in \p in along \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the minimum value is extracted - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out minimum + \param[in] in input array + \param[in] dim dimension along which the minimum is found + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_min */ @@ -864,14 +912,16 @@ extern "C" { #if AF_API_VERSION >= 37 /** - C Interface for minimum values in an array according to key + C Interface to return the minimum along a given dimension, according to + an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the minimum of all values in \p vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the minimum value is extracted - \return \ref AF_SUCCESS if the execution completes properly + \param[out] keys_out reduced keys + \param[out] vals_out minimum + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the minimum is found + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_min_by_key */ @@ -881,12 +931,13 @@ extern "C" { #endif /** - C Interface for maximum values in an array + C Interface to return the maximum along a given dimension. - \param[out] out will contain the maximum of all values in \p in along \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the maximum value is extracted - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out maximum + \param[in] in input array + \param[in] dim dimension along which the maximum is found + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_max */ @@ -894,16 +945,16 @@ extern "C" { #if AF_API_VERSION >= 37 /** - C Interface for maximum values in an array according to key + C Interface to return the maximum along a given dimension, according to + an array of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p - dim - \param[out] vals_out will contain the maximum of all values in \p - vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the maximum value is extracted - \return \ref AF_SUCCESS if the execution completes properly + \param[out] keys_out reduced keys + \param[out] vals_out maximum + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the maximum is found + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_max_by_key */ @@ -914,30 +965,35 @@ extern "C" { #if AF_API_VERSION >= 38 /** - C Interface for finding ragged max values in an array - Uses an additional input array to determine the number of elements to use along the reduction axis. + C Interface to return the ragged maximum over a given dimension. - \param[out] val will contain the maximum ragged values in \p in along \p dim according to \p ragged_len - \param[out] idx will contain the locations of the maximum ragged values in \p in along \p dim according to \p ragged_len - \param[in] in contains the input values to be reduced - \param[in] ragged_len array containing number of elements to use when reducing along \p dim - \param[in] dim The dimension along which the max operation occurs - \return \ref AF_SUCCESS if the execution completes properly + Input parameter `ragged_len` sets the number of elements to consider. - \ingroup reduce_func_max + NaN values are ignored. + + \param[out] val ragged maximum + \param[out] idx locations of the maximum ragged values + \param[in] in input array + \param[in] ragged_len array containing the number of elements to use + \param[in] dim dimension along which the maximum is found + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \note NaN values are ignored + \ingroup reduce_func_max */ AFAPI af_err af_max_ragged(af_array *val, af_array *idx, const af_array in, const af_array ragged_len, const int dim); #endif /** - C Interface for checking all true values in an array + C Interface to check if all values along a given dimension are true. - \param[out] out will contain the result of "and" operation all values in \p in along \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the "and" operation occurs - \return \ref AF_SUCCESS if the execution completes properly + NaN values are ignored. + + \param[out] out array containing 1's if all true; 0's otherwise + \param[in] in input array + \param[in] dim dimention along which the check occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_all_true */ @@ -945,15 +1001,18 @@ extern "C" { #if AF_API_VERSION >= 37 /** - C Interface for checking all true values in an array according to key + C Interface to check if all values along a given dimension are true, + according to an array of keys. + + NaN values are ignored. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the the reduced and of all values in - \p vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the "and" operation occurs - \return \ref AF_SUCCESS if the execution completes properly + \param[out] keys_out reduced keys + \param[out] vals_out array containing 1's if all true; 0's otherwise + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the check occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_alltrue_by_key */ @@ -963,12 +1022,15 @@ extern "C" { #endif /** - C Interface for checking any true values in an array + C Interface to check if any values along a given dimension are true. - \param[out] out will contain the result of "or" operation all values in \p in along \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the "or" operation occurs - \return \ref AF_SUCCESS if the execution completes properly + NaN values are ignored. + + \param[out] out array containing 1's if any true; 0's otherwise + \param[in] in input array + \param[in] dim dimension along which the check occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_any_true */ @@ -976,15 +1038,17 @@ extern "C" { #if AF_API_VERSION >= 37 /** - C Interface for checking any true values in an array according to key + C Interface to check if any values along a given dimension are true. + + NaN values are ignored. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the reduced or of all values in - \p vals along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the "or" operation occurs - \return \ref AF_SUCCESS if the execution completes properly + \param[out] keys_out reduced keys + \param[out] vals_out array containing 1's if any true; 0's otherwise + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimensions along which the check occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_anytrue_by_key */ @@ -994,12 +1058,16 @@ extern "C" { #endif /** - C Interface for counting non-zero values in an array + C Interface to count non-zero values in an array along a given + dimension. + + NaN values are treated as non-zero. - \param[out] out will contain the number of non-zero values in \p in along \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the non-zero values are counted - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out count + \param[in] in input array + \param[in] dim dimension along which the count occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_count */ @@ -1007,15 +1075,18 @@ extern "C" { #if AF_API_VERSION >= 37 /** - C Interface for counting non-zero values in an array according to key + C Interface to count non-zero values in an array, according to an array + of keys. - \param[out] keys_out will contain the reduced keys in \p vals along \p dim - \param[out] vals_out will contain the count of all values in \p vals - along \p dim according to \p keys - \param[in] keys is the key array - \param[in] vals is the array containing the values to be reduced - \param[in] dim The dimension along which the non-zero values are counted - \return \ref AF_SUCCESS if the execution completes properly + NaN values are treated as non-zero. + + \param[out] keys_out reduced keys + \param[out] vals_out count + \param[in] keys keys array + \param[in] vals input array + \param[in] dim dimension along which the count occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_count_by_key */ @@ -1025,16 +1096,15 @@ extern "C" { #endif /** - C Interface for sum of all elements in an array + C Interface to sum array elements over all dimensions. - \param[out] real will contain the real part of adding all elements in - input \p in - \param[out] imag will contain the imaginary part of adding all elements - in input \p in - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + If `in` is real, `imag` will be set to zeros. - \note \p imag is always set to 0 when \p in is real + \param[out] real sum of all real components + \param[out] imag sum of all imaginary components + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_sum */ @@ -1042,17 +1112,17 @@ extern "C" { #if AF_API_VERSION >= 31 /** - C Interface for sum of all elements in an array while replacing nans + C Interface to sum array elements over all dimensions, replacing any + NaNs with a specified value. - \param[out] real will contain the real part of adding all elements in - input \p in - \param[out] imag will contain the imaginary part of adding all elements - in input \p in - \param[in] in is the input array - \param[in] nanval is the value which replaces nan - \return \ref AF_SUCCESS if the execution completes properly + If `in` is real, `imag` will be set to zeros. - \note \p imag is always set to 0 when \p in is real + \param[out] real sum of all real components + \param[out] imag sum of all imaginary components + \param[in] in input array + \param[in] nanval value that replaces NaNs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_sum */ @@ -1061,14 +1131,15 @@ extern "C" { #endif /** - C Interface for product of all elements in an array + C Interface to multiply array elements over all dimensions. - \param[out] real will contain the real part of multiplying all elements in input \p in - \param[out] imag will contain the imaginary part of multiplying all elements in input \p in - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + If `in` is real, `imag` will be set to zeros. - \note \p imag is always set to 0 when \p in is real + \param[out] real product of all real components + \param[out] imag product of all imaginary components + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_product */ @@ -1076,17 +1147,17 @@ extern "C" { #if AF_API_VERSION >= 31 /** - C Interface for product of all elements in an array while replacing nans + C Interface to multiply array elements over all dimensions, replacing + any NaNs with a specified value. - \param[out] real will contain the real part of multiplication of all - elements in input \p in - \param[out] imag will contain the imaginary part of multiplication of - all elements in input \p in - \param[in] in is the input array - \param[in] nanval is the value which replaces nan - \return \ref AF_SUCCESS if the execution completes properly + If `in` is real, `imag` will be set to zeros. - \note \p imag is always set to 0 when \p in is real + \param[out] real product of all real components + \param[out] imag product of all imaginary components + \param[in] in input array + \param[in] nanval value that replaces NaNs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_product */ @@ -1095,14 +1166,15 @@ extern "C" { #endif /** - C Interface for getting minimum value of an array + C Interface to return the minimum over all dimensions. - \param[out] real will contain the real part of minimum value of all elements in input \p in - \param[out] imag will contain the imaginary part of minimum value of all elements in input \p in - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + If `in` is real, `imag` will be set to zeros. - \note \p imag is always set to 0 when \p in is real. + \param[out] real real component of the minimum + \param[out] imag imaginary component of the minimum + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_min */ @@ -1110,11 +1182,12 @@ extern "C" { #if AF_API_VERSION >= 39 /** - C Interface for minimum values in an array, returning an array + C Interface to return the minimum over all dimensions. - \param[out] out will contain the minimum of all values in \p in - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out minimum + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_min */ @@ -1122,14 +1195,15 @@ extern "C" { #endif /** - C Interface for getting maximum value of an array + C Interface to return the maximum over all dimensions. - \param[out] real will contain the real part of maximum value of all elements in input \p in - \param[out] imag will contain the imaginary part of maximum value of all elements in input \p in - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + If `in` is real, `imag` will be set to zeros. - \note \p imag is always set to 0 when \p in is real. + \param[out] real real component of the maximum + \param[out] imag imaginary component of the maximum + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_max */ @@ -1137,13 +1211,12 @@ extern "C" { #if AF_API_VERSION >= 39 /** - C Interface for getting maximum value of an array, returning an array - - \param[out] out will contain the maximum of all values in \p in - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + C Interface to return the maximum over all dimensions. - \note \p imag is always set to 0 when \p in is real. + \param[out] out maximum + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_max */ @@ -1151,14 +1224,13 @@ extern "C" { #endif /** - C Interface for checking if all values in an array are true - - \param[out] real is 1 if all values of input \p in are true, 0 otherwise. - \param[out] imag is always set to 0. - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly - - \note \p imag is always set to 0. + C Interface to check if all values over all dimensions are true. + + \param[out] real 1 if all true; 0 otherwise + \param[out] imag 0 + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_all_true */ @@ -1166,14 +1238,12 @@ extern "C" { #if AF_API_VERSION >= 39 /** - C Interface for checking if all values in an array are true, - while returning an af_array - - \param[out] out will contain 1 if all values of input \p in are true, 0 otherwise - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly - - \note \p imag is always set to 0. + C Interface to check if all values over all dimensions are true. + + \param[out] out 1 if all true; 0 otherwise + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_all_true */ @@ -1181,14 +1251,13 @@ extern "C" { #endif /** - C Interface for checking if any values in an array are true - - \param[out] real is 1 if any value of input \p in is true, 0 otherwise. - \param[out] imag is always set to 0. - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + C Interface to check if any values over all dimensions are true. - \note \p imag is always set to 0. + \param[out] real 1 if any true; 0 otherwise + \param[out] imag 0 + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_any_true */ @@ -1196,14 +1265,12 @@ extern "C" { #if AF_API_VERSION >= 39 /** - C Interface for checking if any values in an array are true, - while returning an af_array + C Interface to check if any values over all dimensions are true. - \param[out] out will contain 1 if any value of input \p in is true, 0 otherwise - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly - - \note \p imag is always set to 0. + \param[out] out 1 if any true; 0 otherwise + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_any_true */ @@ -1211,14 +1278,13 @@ extern "C" { #endif /** - C Interface for counting total number of non-zero values in an array - - \param[out] real will contain the number of non-zero values in \p in. - \param[out] imag is always set to 0. - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + C Interface to count non-zero values over all dimensions. - \note \p imag is always set to 0. + \param[out] real count + \param[out] imag 0 + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_count */ @@ -1226,12 +1292,12 @@ extern "C" { #if AF_API_VERSION >= 39 /** - C Interface for counting total number of non-zero values in an array, - while returning an af_array + C Interface to count non-zero values over all dimensions. - \param[out] out contain the number of non-zero values in \p in. - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out count + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_count */ @@ -1239,13 +1305,15 @@ extern "C" { #endif /** - C Interface for getting minimum values and their locations in an array + C Interface to return the minimum and its location along a given + dimension. - \param[out] out will contain the minimum of all values in \p in along \p dim - \param[out] idx will contain the location of minimum of all values in \p in along \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the minimum value is extracted - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out minimum + \param[out] idx location + \param[in] in input array + \param[in] dim dimension along which the minimum is found + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_min */ @@ -1253,13 +1321,15 @@ extern "C" { const int dim); /** - C Interface for getting maximum values and their locations in an array + C Interface to return the maximum and its location along a given + dimension. - \param[out] out will contain the maximum of all values in \p in along \p dim - \param[out] idx will contain the location of maximum of all values in \p in along \p dim - \param[in] in is the input array - \param[in] dim The dimension along which the maximum value is extracted - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out maximum + \param[out] idx location + \param[in] in input array + \param[in] dim dimension along which the maximum is found + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_max */ @@ -1267,15 +1337,16 @@ extern "C" { const int dim); /** - C Interface for getting minimum value and its location from the entire array + C Interface to return the minimum and its location over all dimensions. - \param[out] real will contain the real part of minimum value of all elements in input \p in - \param[out] imag will contain the imaginary part of minimum value of all elements in input \p in - \param[out] idx will contain the location of minimum of all values in \p in - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + NaN values are ignored. - \note \p imag is always set to 0 when \p in is real. + \param[out] real real component of the minimum + \param[out] imag imaginary component of the minimum; 0 if `idx` is real + \param[out] idx location + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_min */ @@ -1283,27 +1354,30 @@ extern "C" { const af_array in); /** - C Interface for getting maximum value and it's location from the entire array + C Interface to return the maximum and its location over all dimensions. - \param[out] real will contain the real part of maximum value of all elements in input \p in - \param[out] imag will contain the imaginary part of maximum value of all elements in input \p in - \param[out] idx will contain the location of maximum of all values in \p in - \param[in] in is the input array - \return \ref AF_SUCCESS if the execution completes properly + NaN values are ignored. - \note \p imag is always set to 0 when \p in is real. + \param[out] real real component of the maximum + \param[out] imag imaginary component of the maximum; 0 if `idx` is real + \param[out] idx location + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup reduce_func_max */ AFAPI af_err af_imax_all(double *real, double *imag, unsigned *idx, const af_array in); /** - C Interface for computing the cumulative sum (inclusive) of an array + C Interface to evaluate the cumulative sum (inclusive) along a given + dimension. - \param[out] out will contain inclusive sums of the input - \param[in] in is the input array - \param[in] dim is the dimension along which the inclusive sum is calculated - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out cumulative sum + \param[in] in input array + \param[in] dim dimension along which the sum is accumulated + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup scan_func_accum */ @@ -1311,14 +1385,15 @@ extern "C" { #if AF_API_VERSION >=34 /** - C Interface generalized scan of an array + C Interface to scan an array (generalized) over a given dimension. - \param[out] out will contain scan of the input - \param[in] in is the input array - \param[in] dim The dimension along which scan is performed - \param[in] op is the type of binary operations used - \param[in] inclusive_scan is flag specifying whether scan is inclusive - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out scan + \param[in] in input array + \param[in] dim dimension along which the scan occurs + \param[in] op type of binary operation used + \param[in] inclusive_scan flag specifying whether the scan is inclusive + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup scan_func_scan */ @@ -1326,15 +1401,17 @@ extern "C" { af_binary_op op, bool inclusive_scan); /** - C Interface generalized scan by key of an array + C Interface to scan an array (generalized) over a given dimension, + according to an array of keys. - \param[out] out will contain scan of the input - \param[in] key is the key array - \param[in] in is the input array - \param[in] dim The dimension along which scan is performed - \param[in] op is the type of binary operations used - \param[in] inclusive_scan is flag specifying whether scan is inclusive - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out scan + \param[in] key keys array + \param[in] in input array + \param[in] dim dimension along which the scan occurs + \param[in] op type of binary operation used + \param[in] inclusive_scan flag specifying whether the scan is inclusive + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup scan_func_scanbykey */ @@ -1345,48 +1422,54 @@ extern "C" { #endif /** - C Interface for finding the locations of non-zero values in an array + C Interface to locate the indices of the non-zero values in an array. - \param[out] idx will contain indices where \p in is non-zero - \param[in] in is the input array. - \return \ref AF_SUCCESS if the execution completes properly + \param[out] idx linear indices where `in` is non-zero + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup scan_func_where */ AFAPI af_err af_where(af_array *idx, const af_array in); /** - C Interface for calculating first order differences in an array + C Interface to calculate the first order difference in an array over a + given dimension. - \param[out] out will contain the first order numerical differences of \p in - \param[in] in is the input array - \param[in] dim The dimension along which numerical difference is performed - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out first order numerical difference + \param[in] in input array + \param[in] dim dimension along which the difference occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup calc_func_diff1 */ AFAPI af_err af_diff1(af_array *out, const af_array in, const int dim); /** - C Interface for calculating second order differences in an array + C Interface to calculate the second order difference in an array over a + given dimension. - \param[out] out will contain the second order numerical differences of \p in - \param[in] in is the input array - \param[in] dim The dimension along which numerical difference is performed - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out second order numerical difference + \param[in] in input array + \param[in] dim dimension along which the difference occurs + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup calc_func_diff2 */ AFAPI af_err af_diff2(af_array *out, const af_array in, const int dim); /** - C Interface for sorting an array + C Interface to sort an array over a given dimension. - \param[out] out will contain the sorted output - \param[in] in is the input array - \param[in] dim The dimension along which numerical difference is performed - \param[in] isAscending specifies the sorting order - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out sorted output + \param[in] in input array + \param[in] dim dimension along which the sort occurs + \param[in] isAscending specifies the sorting order + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup sort_func_sort */ @@ -1394,29 +1477,33 @@ extern "C" { const bool isAscending); /** - C Interface for sorting an array and getting original indices + C Interface to sort an array over a given dimension and to return the + original indices. - \param[out] out will contain the sorted output - \param[out] indices will contain the indices in the original input - \param[in] in is the input array - \param[in] dim The dimension along which numerical difference is performed - \param[in] isAscending specifies the sorting order - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out sorted output + \param[out] indices indices from the input + \param[in] in input array + \param[in] dim dimension along which the sort occurs + \param[in] isAscending specifies the sorting order + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup sort_func_sort_index */ AFAPI af_err af_sort_index(af_array *out, af_array *indices, const af_array in, const unsigned dim, const bool isAscending); /** - C Interface for sorting an array based on keys + C Interface to sort an array over a given dimension, according to an + array of keys. - \param[out] out_keys will contain the keys based on sorted values - \param[out] out_values will contain the sorted values - \param[in] keys is the input array - \param[in] values The dimension along which numerical difference is performed - \param[in] dim The dimension along which numerical difference is performed - \param[in] isAscending specifies the sorting order - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out_keys sorted keys + \param[out] out_values sorted output + \param[in] keys keys array + \param[in] values input array + \param[in] dim dimension along which the sort occurs + \param[in] isAscending specifies the sorting order + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup sort_func_sort_keys */ @@ -1425,25 +1512,27 @@ extern "C" { const unsigned dim, const bool isAscending); /** - C Interface for getting unique values + C Interface to return the unique values in an array. - \param[out] out will contain the unique values from \p in - \param[in] in is the input array - \param[in] is_sorted if true, skips the sorting steps internally - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out unique values + \param[in] in input array + \param[in] is_sorted if true, skip the sorting steps internally + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup set_func_unique */ AFAPI af_err af_set_unique(af_array *out, const af_array in, const bool is_sorted); /** - C Interface for finding the union of two arrays + C Interface to evaluate the union of two arrays. - \param[out] out will contain the union of \p first and \p second - \param[in] first is the first input array - \param[in] second is the second input array - \param[in] is_unique if true, skips calling unique internally - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out union, values in increasing order + \param[in] first input array + \param[in] second input array + \param[in] is_unique if true, skip calling unique internally + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup set_func_union */ @@ -1451,13 +1540,14 @@ extern "C" { const af_array second, const bool is_unique); /** - C Interface for finding the intersection of two arrays + C Interface to evaluate the intersection of two arrays. - \param[out] out will contain the intersection of \p first and \p second - \param[in] first is the first input array - \param[in] second is the second input array - \param[in] is_unique if true, skips calling unique internally - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out intersection, values in increasing order + \param[in] first input array + \param[in] second input array + \param[in] is_unique if true, skip calling unique internally + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup set_func_intersect */ diff --git a/include/af/arith.h b/include/af/arith.h index ea9be6c328..c75544a5ab 100644 --- a/include/af/arith.h +++ b/include/af/arith.h @@ -1,5 +1,5 @@ /******************************************************* - * Copyright (c) 2014, ArrayFire + * Copyright (c) 2025, ArrayFire * All rights reserved. * * This file is distributed under 3-clause BSD license. @@ -18,25 +18,27 @@ namespace af /// /// \param[in] lhs input array /// \param[in] rhs input array - /// \return minimum of \p lhs and \p rhs + /// \return minimum /// /// \ingroup arith_func_min AFAPI array min (const array &lhs, const array &rhs); - /// C++ Interface to find the elementwise minimum between an array and a scalar value. + /// C++ Interface to find the elementwise minimum between an array and a + /// scalar value. /// /// \param[in] lhs input array /// \param[in] rhs scalar value - /// \return minimum of \p lhs and \p rhs + /// \return minimum /// /// \ingroup arith_func_min AFAPI array min (const array &lhs, const double rhs); - /// C++ Interface to find the elementwise minimum between an array and a scalar value. + /// C++ Interface to find the elementwise minimum between an array and a + /// scalar value. /// /// \param[in] lhs scalar value /// \param[in] rhs input array - /// \return minimum of \p lhs and \p rhs + /// \return minimum /// /// \ingroup arith_func_min AFAPI array min (const double lhs, const array &rhs); @@ -45,25 +47,27 @@ namespace af /// /// \param[in] lhs input array /// \param[in] rhs input array - /// \return maximum of \p lhs and \p rhs + /// \return maximum /// /// \ingroup arith_func_max AFAPI array max (const array &lhs, const array &rhs); - /// C++ Interface to find the elementwise maximum between an array and a scalar value. + /// C++ Interface to find the elementwise maximum between an array and a + /// scalar value. /// /// \param[in] lhs input array /// \param[in] rhs scalar value - /// \return maximum of \p lhs and \p rhs + /// \return maximum /// /// \ingroup arith_func_max AFAPI array max (const array &lhs, const double rhs); - /// C++ Interface to find the elementwise maximum between an array and a scalar value. + /// C++ Interface to find the elementwise maximum between an array and a + /// scalar value. /// /// \param[in] lhs input array /// \param[in] rhs scalar value - /// \return maximum of \p lhs and \p rhs + /// \return maximum /// /// \ingroup arith_func_max AFAPI array max (const double lhs, const array &rhs); @@ -75,8 +79,8 @@ namespace af /// \param[in] in input array /// \param[in] lo lower limit; can be an array or a scalar /// \param[in] hi upper limit; can be an array or a scalar - /// \return array containing values from \p in clamped between \p lo and \p hi - /// + /// \return clamped array + /// /// \ingroup arith_func_clamp AFAPI array clamp(const array &in, const array &lo, const array &hi); #endif @@ -100,10 +104,13 @@ namespace af /// @{ /// C++ Interface to calculate the remainder. /// + /// For integers, it returns the same output as modulus (% operator) + /// For floating point numbers, it returns the same as std::remainder from + /// /// \param[in] lhs numerator; can be an array or a scalar /// \param[in] rhs denominator; can be an array or a scalar - /// \return remainder of \p lhs divided by \p rhs - /// + /// \return remainder + /// /// \ingroup arith_func_rem AFAPI array rem (const array &lhs, const array &rhs); @@ -117,10 +124,13 @@ namespace af /// @{ /// C++ Interface to calculate the modulus. /// + /// For integers, it returns the same output as modulus (% operator) + /// For floating point numbers, it returns the same as std::fmod from + /// /// \param[in] lhs dividend; can be an array or a scalar /// \param[in] rhs divisor; can be an array or a scalar - /// \return \p lhs modulo \p rhs - /// + /// \return modulus + /// /// \ingroup arith_func_mod AFAPI array mod (const array &lhs, const array &rhs); @@ -134,31 +144,32 @@ namespace af /// C++ Interface to calculate the absolute value. /// /// \param[in] in input array - /// \return absolute value + /// \return absolute value /// /// \ingroup arith_func_abs AFAPI array abs (const array &in); - /// C++ Interface to calculate the phase angle (in radians) of a complex array. + /// C++ Interface to calculate the phase angle (in radians) of a complex + /// array. /// /// \param[in] in input array, typically complex - /// \return phase angle (in radians) - /// + /// \return phase angle (in radians) + /// /// \ingroup arith_func_arg AFAPI array arg (const array &in); /// C++ Interface to return the sign of elements in an array. /// /// \param[in] in input array - /// \return array containing 1's for negative values; 0's otherwise - /// + /// \return array containing 1's for negative values; 0's otherwise + /// /// \ingroup arith_func_sign AFAPI array sign (const array &in); /// C++ Interface to round numbers. /// /// \param[in] in input array - /// \return numbers rounded to nearest integer + /// \return nearest integer /// /// \ingroup arith_func_round AFAPI array round (const array &in); @@ -166,15 +177,15 @@ namespace af /// C++ Interface to truncate numbers. /// /// \param[in] in input array - /// \return nearest integer not greater in magnitude than \p in - /// + /// \return nearest integer not greater in magnitude than `in` + /// /// \ingroup arith_func_trunc AFAPI array trunc (const array &in); /// C++ Interface to floor numbers. /// /// \param[in] in input array - /// \return values rounded to nearest integer less than or equal to current value + /// \return nearest integer less than or equal to `in` /// /// \ingroup arith_func_floor AFAPI array floor (const array &in); @@ -182,7 +193,7 @@ namespace af /// C++ Interface to ceil numbers. /// /// \param[in] in input array - /// \return values rounded to nearest integer greater than or equal to current value + /// \return nearest integer greater than or equal to `in` /// /// \ingroup arith_func_ceil AFAPI array ceil (const array &in); @@ -192,11 +203,11 @@ namespace af /// C++ Interface to calculate the length of the hypotenuse of two inputs. /// /// Calculates the hypotenuse of two inputs. The inputs can be both arrays - /// or an array and a scalar. + /// or can be an array and a scalar. /// /// \param[in] lhs length of first side /// \param[in] rhs length of second side - /// \return length of the hypotenuse + /// \return length of the hypotenuse AFAPI array hypot (const array &lhs, const array &rhs); /// \copydoc hypot(const array&, const array&) @@ -209,7 +220,7 @@ namespace af /// C++ Interface to evaluate the sine function. /// /// \param[in] in input array - /// \return sine + /// \return sine /// /// \ingroup arith_func_sin AFAPI array sin (const array &in); @@ -217,7 +228,7 @@ namespace af /// C++ Interface to evaluate the cosine function. /// /// \param[in] in input array - /// \return cosine + /// \return cosine /// /// \ingroup arith_func_cos AFAPI array cos (const array &in); @@ -225,7 +236,7 @@ namespace af /// C++ Interface to evaluate the tangent function. /// /// \param[in] in input array - /// \return tangent + /// \return tangent /// /// \ingroup arith_func_tan AFAPI array tan (const array &in); @@ -233,7 +244,7 @@ namespace af /// C++ Interface to evaluate the inverse sine function. /// /// \param[in] in input array - /// \return inverse sine + /// \return inverse sine /// /// \ingroup arith_func_asin AFAPI array asin (const array &in); @@ -241,7 +252,7 @@ namespace af /// C++ Interface to evaluate the inverse cosine function. /// /// \param[in] in input array - /// \return inverse cosine + /// \return inverse cosine /// /// \ingroup arith_func_acos AFAPI array acos (const array &in); @@ -249,7 +260,7 @@ namespace af /// C++ Interface to evaluate the inverse tangent function. /// /// \param[in] in input array - /// \return inverse tangent + /// \return inverse tangent /// /// \ingroup arith_func_atan AFAPI array atan (const array &in); @@ -260,7 +271,7 @@ namespace af /// /// \param[in] lhs value of numerator /// \param[in] rhs value of denominator - /// \return inverse tangent of the inputs + /// \return inverse tangent of the inputs AFAPI array atan2 (const array &lhs, const array &rhs); /// \copydoc atan2(const array&, const array&) @@ -273,7 +284,7 @@ namespace af /// C++ Interface to evaluate the hyperbolic sine function. /// /// \param[in] in input array - /// \return hyperbolic sine + /// \return hyperbolic sine /// /// \ingroup arith_func_sinh AFAPI array sinh(const array& in); @@ -281,7 +292,7 @@ namespace af /// C++ Interface to evaluate the hyperbolic cosine function. /// /// \param[in] in input array - /// \return hyperbolic cosine + /// \return hyperbolic cosine /// /// \ingroup arith_func_cosh AFAPI array cosh(const array& in); @@ -289,7 +300,7 @@ namespace af /// C++ Interface to evaluate the hyperbolic tangent function. /// /// \param[in] in input array - /// \return hyperbolic tangent + /// \return hyperbolic tangent /// /// \ingroup arith_func_tanh AFAPI array tanh(const array& in); @@ -297,7 +308,7 @@ namespace af /// C++ Interface to evaluate the inverse hyperbolic sine function. /// /// \param[in] in input array - /// \return inverse hyperbolic sine + /// \return inverse hyperbolic sine /// /// \ingroup arith_func_asinh AFAPI array asinh(const array& in); @@ -305,7 +316,7 @@ namespace af /// C++ Interface to evaluate the inverse hyperbolic cosine function. /// /// \param[in] in input array - /// \return inverse hyperbolic cosine + /// \return inverse hyperbolic cosine /// /// \ingroup arith_func_acosh AFAPI array acosh(const array& in); @@ -313,7 +324,7 @@ namespace af /// C++ Interface to evaluate the inverse hyperbolic tangent function. /// /// \param[in] in input array - /// \return inverse hyperbolic tangent + /// \return inverse hyperbolic tangent /// /// \ingroup arith_func_atanh AFAPI array atanh(const array& in); @@ -322,36 +333,44 @@ namespace af /// @{ /// C++ Interface to create a complex array from a single real array. /// - /// \param[in] in a real array - /// \return the returned complex array + /// \param[in] in input array + /// \return complex array AFAPI array complex(const array& in); - + /// C++ Interface to create a complex array from two real arrays. /// - /// \param[in] real_ a real array to be assigned as the real component of the returned complex array - /// \param[in] imag_ a real array to be assigned as the imaginary component of the returned complex array - /// \return the returned complex array + /// \param[in] real_ input array to be assigned as the real component of + /// the returned complex array + /// \param[in] imag_ input array to be assigned as the imaginary component + /// of the returned complex array + /// \return complex array AFAPI array complex(const array &real_, const array &imag_); - /// C++ Interface to create a complex array from a single real array for the real component and a single scalar for each imaginary component. + /// C++ Interface to create a complex array from a single real array for + /// the real component and a single scalar for each imaginary component. /// - /// \param[in] real_ a real array to be assigned as the real component of the returned complex array - /// \param[in] imag_ a single scalar to be assigned as the imaginary component of each value of the returned complex array - /// \return the returned complex array + /// \param[in] real_ input array to be assigned as the real component of + /// the returned complex array + /// \param[in] imag_ single scalar to be assigned as the imaginary + /// component of each value of the returned complex array + /// \return complex array AFAPI array complex(const array &real_, const double imag_); - /// C++ Interface to create a complex array from a single scalar for each real component and a single real array for the imaginary component. + /// C++ Interface to create a complex array from a single scalar for each + /// real component and a single real array for the imaginary component. /// - /// \param[in] real_ a single scalar to be assigned as the real component of each value of the returned complex array - /// \param[in] imag_ a real array to be assigned as the imaginary component of the returned complex array - /// \return the returned complex array + /// \param[in] real_ single scalar to be assigned as the real component of + /// each value of the returned complex array + /// \param[in] imag_ input array to be assigned as the imaginary component + /// of the returned complex array + /// \return complex array AFAPI array complex(const double real_, const array &imag_); /// @} /// C++ Interface to return the real part of a complex array. /// /// \param[in] in input complex array - /// \return real part + /// \return real part /// /// \ingroup arith_func_real AFAPI array real (const array &in); @@ -359,7 +378,7 @@ namespace af /// C++ Interface to return the imaginary part of a complex array. /// /// \param[in] in input complex array - /// \return imaginary part + /// \return imaginary part /// /// \ingroup arith_func_imag AFAPI array imag (const array &in); @@ -367,7 +386,7 @@ namespace af /// C++ Interface to calculate the complex conjugate of an input array. /// /// \param[in] in input complex array - /// \return complex conjugate + /// \return complex conjugate /// /// \ingroup arith_func_conjg AFAPI array conjg (const array &in); @@ -375,8 +394,8 @@ namespace af /// C++ Interface to evaluate the nth root. /// /// \param[in] nth_root nth root - /// \param[in] value value - /// \return \p nth_root th root of \p value + /// \param[in] value value + /// \return `nth_root` th root of `value` /// /// \ingroup arith_func_root AFAPI array root (const array &nth_root, const array &value); @@ -384,8 +403,8 @@ namespace af /// C++ Interface to evaluate the nth root. /// /// \param[in] nth_root nth root - /// \param[in] value value - /// \return \p nth_root th root of \p value + /// \param[in] value value + /// \return `nth_root` th root of `value` /// /// \ingroup arith_func_root AFAPI array root (const array &nth_root, const double value); @@ -393,22 +412,22 @@ namespace af /// C++ Interface to evaluate the nth root. /// /// \param[in] nth_root nth root - /// \param[in] value value - /// \return \p nth_root th root of \p value + /// \param[in] value value + /// \return `nth_root` th root of `value` /// /// \ingroup arith_func_root AFAPI array root (const double nth_root, const array &value); - /// \ingroup arith_func_pow /// @{ /// C++ Interface to raise a base to a power (or exponent). /// - /// Computes the value of \p base raised to the power of \p exponent. The inputs can be two arrays or an array and a scalar. + /// Computes the value of `base` raised to the power of `exponent`. The + /// inputs can be two arrays or an array and a scalar. /// - /// \param[in] base base + /// \param[in] base base /// \param[in] exponent exponent - /// \return \p base raised to the power of \p exponent + /// \return `base` raised to the power of `exponent` AFAPI array pow (const array &base, const array &exponent); /// \copydoc pow(const array&, const array&) @@ -419,19 +438,18 @@ namespace af /// C++ Interface to raise 2 to a power (or exponent). /// - /// \param[in] in exponent - /// \return 2 raised to the power - /// + /// \param[in] in power + /// \return 2 raised to the power AFAPI array pow2 (const array &in); /// @} #if AF_API_VERSION >= 31 /// C++ Interface to evaluate the logistical sigmoid function. /// + /// Computes \f$\frac{1}{1+e^{-x}}\f$. + /// /// \param[in] in input - /// \return sigmoid - /// - /// \note Computes `1/(1+e^-x)`. + /// \return sigmoid /// /// \ingroup arith_func_sigmoid AFAPI array sigmoid (const array &in); @@ -440,57 +458,61 @@ namespace af /// C++ Interface to evaluate the exponential. /// /// \param[in] in exponent - /// \return exponential + /// \return exponential /// /// \ingroup arith_func_exp AFAPI array exp (const array &in); - /// C++ Interface to evaluate the exponential of an array minus 1, `exp(in) - 1`. + /// C++ Interface to evaluate the exponential of an array minus 1, + /// `exp(in) - 1`. + /// + /// This function is useful when `in` is small. /// /// \param[in] in exponent - /// \return the exponential minus 1 + /// \return exponential minus 1 /// - /// \note This function is useful when \p in is small /// \ingroup arith_func_expm1 AFAPI array expm1 (const array &in); /// C++ Interface to evaluate the error function. /// - /// \param[in] in input - /// \return error function + /// \param[in] in input array + /// \return error function /// /// \ingroup arith_func_erf AFAPI array erf (const array &in); /// C++ Interface to evaluate the complementary error function. /// - /// \param[in] in input - /// \return complementary error function + /// \param[in] in input array + /// \return complementary error function /// /// \ingroup arith_func_erfc AFAPI array erfc (const array &in); /// C++ Interface to evaluate the natural logarithm. /// - /// \param[in] in input - /// \return natural logarithm + /// \param[in] in input array + /// \return natural logarithm /// /// \ingroup arith_func_log AFAPI array log (const array &in); - /// C++ Interface to evaluate the natural logarithm of 1 + input, `ln(1+in)`. + /// C++ Interface to evaluate the natural logarithm of 1 + input, + /// `ln(1+in)`. + /// + /// This function is useful when `in` is small. /// /// \param[in] in input /// \return natural logarithm of `1 + input` /// - /// \note This function is useful when \p in is small /// \ingroup arith_func_log1p AFAPI array log1p (const array &in); /// C++ Interface to evaluate the base 10 logarithm. /// /// \param[in] in input - /// \return base 10 logarithm + /// \return base 10 logarithm /// /// \ingroup arith_func_log10 AFAPI array log10 (const array &in); @@ -498,7 +520,7 @@ namespace af /// C++ Interface to evaluate the base 2 logarithm. /// /// \param[in] in input - /// \return base 2 logarithm + /// \return base 2 logarithm /// /// \ingroup explog_func_log2 AFAPI array log2 (const array &in); @@ -506,7 +528,7 @@ namespace af /// C++ Interface to evaluate the square root. /// /// \param[in] in input - /// \return square root + /// \return square root /// /// \ingroup arith_func_sqrt AFAPI array sqrt (const array &in); @@ -515,7 +537,7 @@ namespace af /// C++ Interface to evaluate the reciprocal square root. /// /// \param[in] in input - /// \return reciprocal square root + /// \return reciprocal square root /// /// \ingroup arith_func_rsqrt AFAPI array rsqrt (const array &in); @@ -524,7 +546,7 @@ namespace af /// C++ Interface to evaluate the cube root. /// /// \param[in] in input - /// \return cube root + /// \return cube root /// /// \ingroup arith_func_cbrt AFAPI array cbrt (const array &in); @@ -532,7 +554,7 @@ namespace af /// C++ Interface to calculate the factorial. /// /// \param[in] in input - /// \return the factorial function + /// \return factorial /// /// \ingroup arith_func_factorial AFAPI array factorial (const array &in); @@ -540,15 +562,16 @@ namespace af /// C++ Interface to evaluate the gamma function. /// /// \param[in] in input - /// \return gamma function + /// \return gamma function /// /// \ingroup arith_func_tgamma AFAPI array tgamma (const array &in); - /// C++ Interface to evaluate the logarithm of the absolute value of the gamma function. + /// C++ Interface to evaluate the logarithm of the absolute value of the + /// gamma function. /// /// \param[in] in input - /// \return logarithm of the absolute value of the gamma function + /// \return logarithm of the absolute value of the gamma function /// /// \ingroup arith_func_lgamma AFAPI array lgamma (const array &in); @@ -556,7 +579,7 @@ namespace af /// C++ Interface to check which values are zero. /// /// \param[in] in input - /// \return array containing 1's where input is 0; 0's otherwise + /// \return array containing 1's where input is 0; 0's otherwise /// /// \ingroup arith_func_iszero AFAPI array iszero (const array &in); @@ -564,7 +587,8 @@ namespace af /// C++ Interface to check if values are infinite. /// /// \param[in] in input - /// \return array containing 1's where input is Inf or -Inf; 0's otherwise + /// \return array containing 1's where input is Inf or -Inf; 0's + /// otherwise /// /// \ingroup arith_func_isinf AFAPI array isInf (const array &in); @@ -572,7 +596,7 @@ namespace af /// C++ Interface to check if values are NaN. /// /// \param[in] in input - /// \return array containing 1's where input is NaN; 0's otherwise + /// \return array containing 1's where input is NaN; 0's otherwise /// /// \ingroup arith_func_isnan AFAPI array isNaN (const array &in); @@ -586,11 +610,12 @@ extern "C" { /** C Interface to add two arrays. - \param[out] out sum of \p lhs and \p rhs - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out + + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_add */ @@ -599,11 +624,12 @@ extern "C" { /** C Interface to subtract one array from another array. - \param[out] out subtraction of \p lhs - \p rhs - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out - + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_sub */ @@ -612,11 +638,12 @@ extern "C" { /** C Interface to multiply two arrays. - \param[out] out product of \p lhs and \p rhs - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out * + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_mul */ @@ -625,89 +652,113 @@ extern "C" { /** C Interface to divide one array by another array. - \param[out] out result of \p lhs / \p rhs. - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out \ + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_div */ AFAPI af_err af_div (af_array *out, const af_array lhs, const af_array rhs, const bool batch); /** - C Interface to perform a less-than comparison between corresponding elements of two arrays. + C Interface to perform a less-than comparison between corresponding + elements of two arrays. + + Output type is b8. - \param[out] out result of \p lhs < \p rhs; type is b8 - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out 1's where `lhs < rhs`, else 0's + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup logic_func_lt */ AFAPI af_err af_lt (af_array *out, const af_array lhs, const af_array rhs, const bool batch); /** - C Interface to perform a greater-than comparison between corresponding elements of two arrays. + C Interface to perform a greater-than comparison between corresponding + elements of two arrays. - \param[out] out result of \p lhs > \p rhs; type is b8 - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + Output type is b8. + + \param[out] out 1's where `lhs > rhs`, else 0's + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_gt */ AFAPI af_err af_gt (af_array *out, const af_array lhs, const af_array rhs, const bool batch); /** - C Interface to perform a less-than-or-equal comparison between corresponding elements of two arrays. + C Interface to perform a less-than-or-equal comparison between + corresponding elements of two arrays. + + Output type is b8. - \param[out] out result of \p lhs <= \p rhs; type is b8 - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out 1's where `lhs <= rhs`, else 0's + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_le */ AFAPI af_err af_le (af_array *out, const af_array lhs, const af_array rhs, const bool batch); /** - C Interface to perform a greater-than-or-equal comparison between corresponding elements of two arrays. + C Interface to perform a greater-than-or-equal comparison between + corresponding elements of two arrays. - \param[out] out result of \p lhs >= \p rhs; type is b8 - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + Output type is b8. + + \param[out] out 1's where `lhs >= rhs`, else 0's + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_ge */ AFAPI af_err af_ge (af_array *out, const af_array lhs, const af_array rhs, const bool batch); /** - C Interface to check if corresponding elements of two arrays are equal + C Interface to check if corresponding elements of two arrays are equal. + + Output type is b8. - \param[out] out result of `lhs == rhs`; type is b8 - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out 1's where `lhs == rhs`, else 0's + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_eq */ AFAPI af_err af_eq (af_array *out, const af_array lhs, const af_array rhs, const bool batch); /** - C Interface to check if corresponding elements of two arrays are not equal + C Interface to check if corresponding elements of two arrays are not + equal. + + Output type is b8. - \param[out] out result of `lhs != rhs`; type is b8 - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out 1's where `lhs != rhs`, else 0's + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_neq */ @@ -716,11 +767,14 @@ extern "C" { /** C Interface to evaluate the logical AND of two arrays. - \param[out] out result of \p lhs && \p rhs; type is b8 - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + Output type is b8. + + \param[out] out 1's where `lhs && rhs`, else 0's + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_and */ @@ -729,11 +783,14 @@ extern "C" { /** C Interface the evaluate the logical OR of two arrays. - \param[out] out result of \p lhs || \p rhs; type is b8 - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + Output type is b8. + + \param[out] out 1's where `lhs || rhs`, else 0's + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_or */ @@ -742,9 +799,12 @@ extern "C" { /** C Interface to evaluate the logical NOT of an array. - \param[out] out result of logical NOT; type is b8 - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + Output type is b8. + + \param[out] out !, logical NOT + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_not */ @@ -754,9 +814,10 @@ extern "C" { /** C Interface to evaluate the bitwise NOT of an array. - \param[out] out result of bitwise NOT - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out ~, bitwise NOT + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_bitnot */ @@ -766,11 +827,12 @@ extern "C" { /** C Interface to evaluate the bitwise AND of two arrays. - \param[out] out result of \p lhs & \p rhs - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out &, bitwise AND + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_bitand */ @@ -779,11 +841,12 @@ extern "C" { /** C Interface to evaluate the bitwise OR of two arrays. - \param[out] out result of \p lhs | \p rhs - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out |, bitwise OR + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_bitor */ @@ -792,11 +855,12 @@ extern "C" { /** C Interface to evaluate the bitwise XOR of two arrays. - \param[out] out result of \p lhs ^ \p rhs - \param[in] lhs first input - \param[in] rhs second input - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out ^, bitwise XOR + \param[in] lhs first input + \param[in] rhs second input + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_bitxor */ @@ -805,11 +869,12 @@ extern "C" { /** C Interface to shift the bits of integer arrays left. - \param[out] out result of the left shift - \param[in] lhs values to shift - \param[in] rhs n bits to shift - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out left shift + \param[in] lhs values to shift + \param[in] rhs n bits to shift + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_shiftl */ @@ -818,11 +883,12 @@ extern "C" { /** C Interface to shift the bits of integer arrays right. - \param[out] out result of the right shift - \param[in] lhs values to shift - \param[in] rhs n bits to shift - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out right shift + \param[in] lhs values to shift + \param[in] rhs n bits to shift + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_shiftr */ @@ -832,38 +898,42 @@ extern "C" { C Interface to cast an array from one type to another. This function casts an af_array object from one type to another. If the - type of the original array is the same as \p type then the same array is + type of the original array is the same as `type` then the same array is returned. - \note Consecitive casting operations may be may be optimized out if the + Consecutive casting operations may be may be optimized out if the original type of the af_array is the same as the final type. For example - if the original type is f64 which is then cast to f32 and then back to - f64, then the cast to f32 will be skipped and that operation will *NOT* + if the original type is f64, which is cast to f32 and then back to + f64, then the cast to f32 is skipped and that operation will *NOT* be performed by ArrayFire. The following table shows which casts will be optimized out. outer -> inner -> outer - | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | u8 | b8 | s64 | u64 | s16 | u16 | f16 | - |---------|-----|-----|-----|-----|-----|-----|----|----|-----|-----|-----|-----|-----| - | f32 | x | x | x | x | | | | | | | | | x | - | f64 | x | x | x | x | | | | | | | | | x | - | c32 | x | x | x | x | | | | | | | | | x | - | c64 | x | x | x | x | | | | | | | | | x | - | s32 | x | x | x | x | x | x | | | x | x | | | x | - | u32 | x | x | x | x | x | x | | | x | x | | | x | - | u8 | x | x | x | x | x | x | x | x | x | x | x | x | x | - | b8 | x | x | x | x | x | x | x | x | x | x | x | x | x | - | s64 | x | x | x | x | | | | | x | x | | | x | - | u64 | x | x | x | x | | | | | x | x | | | x | - | s16 | x | x | x | x | x | x | | | x | x | x | x | x | - | u16 | x | x | x | x | x | x | | | x | x | x | x | x | - | f16 | x | x | x | x | | | | | | | | | x | - If you want to avoid this behavior use af_eval after the first cast + + | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | s8 | u8 | b8 | s64 | u64 | s16 | u16 | f16 | + |---------|-----|-----|-----|-----|-----|-----|----|----|----|-----|-----|-----|-----|-----| + | f32 | x | x | x | x | | | | | | | | | | x | + | f64 | x | x | x | x | | | | | | | | | | x | + | c32 | x | x | x | x | | | | | | | | | | x | + | c64 | x | x | x | x | | | | | | | | | | x | + | s32 | x | x | x | x | x | x | | | | x | x | | | x | + | u32 | x | x | x | x | x | x | | | | x | x | | | x | + | s8 | x | x | x | x | x | x | x | x | x | x | x | x | x | x | + | u8 | x | x | x | x | x | x | x | x | x | x | x | x | x | x | + | b8 | x | x | x | x | x | x | x | x | x | x | x | x | x | x | + | s64 | x | x | x | x | | | | | | x | x | | | x | + | u64 | x | x | x | x | | | | | | x | x | | | x | + | s16 | x | x | x | x | x | x | | | | x | x | x | x | x | + | u16 | x | x | x | x | x | x | | | | x | x | x | x | x | + | f16 | x | x | x | x | | | | | | | | | | x | + + If you want to avoid this behavior use, af_eval after the first cast operation. This will ensure that the cast operation is performed on the af_array. - \param[out] out values in the specified type - \param[in] in input - \param[in] type target data type \ref af_dtype - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out values in the specified type + \param[in] in input + \param[in] type target data type \ref af_dtype + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_cast */ @@ -872,24 +942,27 @@ extern "C" { /** C Interface to find the elementwise minimum between two arrays. - \param[out] out minimum of \p lhs and \p rhs - \param[in] lhs input array - \param[in] rhs input array - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out minimum + \param[in] lhs input array + \param[in] rhs input array + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_min */ AFAPI af_err af_minof (af_array *out, const af_array lhs, const af_array rhs, const bool batch); /** - C Interface to find the elementwise minimum between an array and a scalar value. + C Interface to find the elementwise minimum between an array and a + scalar value. - \param[out] out maximum of \p lhs and \p rhs - \param[in] lhs input array - \param[in] rhs input array - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out maximum + \param[in] lhs input array + \param[in] rhs input array + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_max */ @@ -899,12 +972,13 @@ extern "C" { /** C Interface to clamp an array between an upper and a lower limit. - \param[out] out array containing values from \p in clamped between \p lo and \p hi - \param[in] in input array - \param[in] lo lower limit array - \param[in] hi upper limit array - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out clamped array + \param[in] in input array + \param[in] lo lower limit array + \param[in] hi upper limit array + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_clamp */ @@ -915,11 +989,15 @@ extern "C" { /** C Interface to calculate the remainder. - \param[out] out remainder of \p lhs divided by \p rhs - \param[in] lhs numerator - \param[in] rhs denominator - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + For integers, it returns the same output as modulus (% operator) + For floating point numbers, it returns the same as `remainder` from + + \param[out] out remainder + \param[in] lhs numerator + \param[in] rhs denominator + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_rem */ @@ -928,11 +1006,15 @@ extern "C" { /** C Interface to calculate the modulus. - \param[out] out \p lhs modulo \p rhs - \param[in] lhs dividend - \param[in] rhs divisor - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + For integers, it returns the same output as modulus (% operator) + For floating point numbers, it returns the same as `fmod` from + + \param[out] out modulus + \param[in] lhs dividend + \param[in] rhs divisor + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_mod */ @@ -942,19 +1024,22 @@ extern "C" { C Interface to calculate the absolute value. \param[out] out absolute value - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_abs */ AFAPI af_err af_abs (af_array *out, const af_array in); /** - C Interface to calculate the phase angle (in radians) of a complex array. + C Interface to calculate the phase angle (in radians) of a complex + array. \param[out] out phase angle (in radians) - \param[in] in input array, typically complex - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array, typically complex + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_arg */ @@ -964,8 +1049,9 @@ extern "C" { C Interface to calculate the sign of elements in an array. \param[out] out array containing 1's for negative values; 0's otherwise - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_sign */ @@ -974,9 +1060,10 @@ extern "C" { /** C Interface to round numbers. - \param[out] out values rounded to nearest integer - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out nearest integer + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_round */ @@ -985,9 +1072,10 @@ extern "C" { /** C Interface to truncate numbers. - \param[out] out nearest integer not greater in magnitude than \p in - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out nearest integer not greater in magnitude than `in` + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_trunc */ @@ -996,9 +1084,10 @@ extern "C" { /** C Interface to floor numbers. - \param[out] out values rounded to nearest integer less than or equal to \p in - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out nearest integer less than or equal to `in` + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_floor */ @@ -1007,9 +1096,10 @@ extern "C" { /** C Interface to ceil numbers. - \param[out] out values rounded to nearest integer greater than or equal to \p in - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out nearest integer greater than or equal to `in` + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_ceil */ @@ -1018,11 +1108,12 @@ extern "C" { /** C Interface to calculate the length of the hypotenuse of two inputs. - \param[out] out length of the hypotenuse - \param[in] lhs length of first side - \param[in] rhs length of second side - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out length of the hypotenuse + \param[in] lhs length of first side + \param[in] rhs length of second side + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_floor */ @@ -1032,8 +1123,9 @@ extern "C" { C Interface to evaluate the sine function. \param[out] out sine - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_sin */ @@ -1043,8 +1135,9 @@ extern "C" { C Interface to evaluate the cosine function. \param[out] out cosine - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_cos */ @@ -1054,8 +1147,9 @@ extern "C" { C Interface to evaluate the tangent function. \param[out] out tangent - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_tan */ @@ -1065,8 +1159,9 @@ extern "C" { C Interface to evaluate the inverse sine function. \param[out] out inverse sine - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_asin */ @@ -1076,8 +1171,9 @@ extern "C" { C Interface to evaluate the inverse cosine function. \param[out] out inverse cos - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_acos */ @@ -1087,8 +1183,9 @@ extern "C" { C Interface to evaluate the inverse tangent function. \param[out] out inverse tangent - \param[in] in input array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_atan */ @@ -1097,11 +1194,12 @@ extern "C" { /** C Interface to evaluate the inverse tangent of two arrays. - \param[out] out inverse tangent of two arrays - \param[in] lhs numerator - \param[in] rhs denominator - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out inverse tangent of two arrays + \param[in] lhs numerator + \param[in] rhs denominator + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_atan */ @@ -1111,8 +1209,9 @@ extern "C" { C Interface to evaluate the hyperbolic sine function. \param[out] out hyperbolic sine - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_sinh */ @@ -1122,8 +1221,9 @@ extern "C" { C Interface to evaluate the hyperbolic cosine function. \param[out] out hyperbolic cosine - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_cosh */ @@ -1133,8 +1233,9 @@ extern "C" { C Interface to evaluate the hyperbolic tangent function. \param[out] out hyperbolic tangent - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_tanh */ @@ -1144,8 +1245,9 @@ extern "C" { C Interface to evaluate the inverse hyperbolic sine function. \param[out] out inverse hyperbolic sine - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_asinh */ @@ -1155,8 +1257,9 @@ extern "C" { C Interface to evaluate the inverse hyperbolic cosine function. \param[out] out inverse hyperbolic cosine - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_acosh */ @@ -1166,8 +1269,9 @@ extern "C" { C Interface to evaluate the inverse hyperbolic tangent function. \param[out] out inverse hyperbolic tangent - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_atanh */ @@ -1177,8 +1281,9 @@ extern "C" { C Interface to create a complex array from a single real array. \param[out] out complex array - \param[in] in real array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in real array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_cplx */ @@ -1187,11 +1292,14 @@ extern "C" { /** C Interface to create a complex array from two real arrays. - \param[out] out complex array - \param[in] real real array to be assigned as the real component of the returned complex array - \param[in] imag real array to be assigned as the imaginary component of the returned complex array - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out complex array + \param[in] real real array to be assigned as the real component of the + returned complex array + \param[in] imag real array to be assigned as the imaginary component + of the returned complex array + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_cplx */ @@ -1201,8 +1309,9 @@ extern "C" { C Interface to return the real part of a complex array. \param[out] out real part - \param[in] in complex array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in complex array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_real */ @@ -1212,8 +1321,9 @@ extern "C" { C Interface to return the imaginary part of a complex array. \param[out] out imaginary part - \param[in] in complex array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in complex array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_imag */ @@ -1223,8 +1333,9 @@ extern "C" { C Interface to evaluate the complex conjugate of an input array. \param[out] out complex conjugate - \param[in] in complex array - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in complex array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_conjg */ @@ -1233,11 +1344,12 @@ extern "C" { /** C Interface to evaluate the nth root. - \param[out] out \p lhs th root of \p rhs - \param[in] lhs nth root - \param[in] rhs value - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out `lhs` th root of `rhs` + \param[in] lhs nth root + \param[in] rhs value + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_root */ @@ -1247,11 +1359,12 @@ extern "C" { /** C Interface to raise a base to a power (or exponent). - \param[out] out \p lhs raised to the power of \p rhs - \param[in] lhs base - \param[in] rhs exponent - \param[in] batch specifies if operations need to be performed in batch mode - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out `lhs` raised to the power of `rhs` + \param[in] lhs base + \param[in] rhs exponent + \param[in] batch batch mode + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_pow */ @@ -1260,9 +1373,10 @@ extern "C" { /** C Interface to raise 2 to a power (or exponent). - \param[out] out 2 raised to the power of \p in - \param[in] in exponent - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out 2 raised to the power of `in` + \param[in] in exponent + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_pow2 */ @@ -1272,11 +1386,12 @@ extern "C" { /** C Interface to evaluate the logistical sigmoid function. - Computes `1/(1+e^-x)`. + Computes \f$\frac{1}{1+e^{-x}}\f$. \param[out] out output of the logistic sigmoid function - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_sigmoid */ @@ -1286,20 +1401,23 @@ extern "C" { /** C Interface to evaluate the exponential. - \param[out] out e raised to the power of \p in - \param[in] in exponent - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out e raised to the power of `in` + \param[in] in exponent + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_exp */ AFAPI af_err af_exp (af_array *out, const af_array in); /** - C Interface to evaluate the exponential of an array minus 1, `exp(in) - 1`. + C Interface to evaluate the exponential of an array minus 1, + `exp(in) - 1`. \param[out] out exponential of `in - 1` - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_expm1 */ @@ -1309,8 +1427,9 @@ extern "C" { C Interface to evaluate the error function. \param[out] out error function value - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_erf */ @@ -1320,8 +1439,9 @@ extern "C" { C Interface to evaluate the complementary error function. \param[out] out complementary error function - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_erfc */ @@ -1331,8 +1451,9 @@ extern "C" { C Interface to evaluate the natural logarithm. \param[out] out natural logarithm - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_log */ @@ -1342,8 +1463,9 @@ extern "C" { C Interface to evaluate the natural logarithm of 1 + input, `ln(1+in)`. \param[out] out logarithm of `in + 1` - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_log1p */ @@ -1353,8 +1475,9 @@ extern "C" { C Interface to evaluate the base 10 logarithm. \param[out] out base 10 logarithm - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_log10 */ @@ -1364,8 +1487,9 @@ extern "C" { C Interface to evaluate the base 2 logarithm. \param[out] out base 2 logarithm - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup explog_func_log2 */ @@ -1375,8 +1499,9 @@ extern "C" { C Interface to evaluate the square root. \param[out] out square root - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_sqrt */ @@ -1387,8 +1512,9 @@ extern "C" { C Interface to evaluate the reciprocal square root. \param[out] out reciprocal square root - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_rsqrt */ @@ -1398,8 +1524,9 @@ extern "C" { C Interface to evaluate the cube root. \param[out] out cube root - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_cbrt */ @@ -1409,8 +1536,9 @@ extern "C" { C Interface to calculate the factorial. \param[out] out factorial - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_factorial */ @@ -1420,54 +1548,61 @@ extern "C" { C Interface to evaluate the gamma function. \param[out] out gamma function - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_tgamma */ AFAPI af_err af_tgamma (af_array *out, const af_array in); /** - C Interface to evaluate the logarithm of the absolute value of the gamma function. + C Interface to evaluate the logarithm of the absolute value of the + gamma function. \param[out] out logarithm of the absolute value of the gamma function - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup arith_func_lgamma */ AFAPI af_err af_lgamma (af_array *out, const af_array in); /** - C Interface to check if values are zero. + C Interface to check if values are zero. - \param[out] out array containing 1's where input is 0; 0's otherwise - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out array containing 1's where input is 0; 0's otherwise + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \ingroup arith_func_iszero + \ingroup arith_func_iszero */ AFAPI af_err af_iszero (af_array *out, const af_array in); /** - C Interface to check if values are infinite. + C Interface to check if values are infinite. - \param[out] out array containing 1's where input is Inf or -Inf; 0's otherwise - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out array containing 1's where input is Inf or -Inf; 0's + otherwise + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \ingroup arith_func_isinf + \ingroup arith_func_isinf */ AFAPI af_err af_isinf (af_array *out, const af_array in); /** - C Interface to check if values are NaN. + C Interface to check if values are NaN. - \param[out] out array containing 1's where input is NaN; 0's otherwise - \param[in] in input - \return \ref AF_SUCCESS if the execution completes properly + \param[out] out array containing 1's where input is NaN; 0's otherwise + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \ingroup arith_func_isnan + \ingroup arith_func_isnan */ AFAPI af_err af_isnan (af_array *out, const af_array in); diff --git a/include/af/array.h b/include/af/array.h index 0edb9558e1..672c2716eb 100644 --- a/include/af/array.h +++ b/include/af/array.h @@ -71,7 +71,7 @@ namespace af operator array() const; operator array(); -#define ASSIGN(OP) \ +#define ASSIGN_(OP) \ array_proxy& operator OP(const array_proxy &a); \ array_proxy& operator OP(const array &a); \ array_proxy& operator OP(const double &a); \ @@ -88,17 +88,25 @@ namespace af array_proxy& operator OP(const long long &a); \ array_proxy& operator OP(const unsigned long long &a); - ASSIGN(=) - ASSIGN(+=) - ASSIGN(-=) - ASSIGN(*=) - ASSIGN(/=) -#undef ASSIGN - #if AF_API_VERSION >= 32 -#define ASSIGN(OP) \ +#define ASSIGN_32(OP) \ array_proxy& operator OP(const short &a); \ array_proxy& operator OP(const unsigned short &a); +#else +#define ASSIGN_32(OP) +#endif + +#if AF_API_VERSION >= 310 +#define ASSIGN_310(OP) \ + array_proxy& operator OP(const signed char &a); +#else +#define ASSIGN_310(OP) +#endif + +#define ASSIGN(OP) \ + ASSIGN_(OP) \ + ASSIGN_32(OP) \ + ASSIGN_310(OP) ASSIGN(=) ASSIGN(+=) @@ -106,7 +114,9 @@ namespace af ASSIGN(*=) ASSIGN(/=) #undef ASSIGN -#endif +#undef ASSIGN_ +#undef ASSIGN_32 +#undef ASSIGN_310 // af::array member functions. same behavior as those below af_array get(); @@ -655,6 +665,7 @@ namespace af /** Perform deep copy from host/device pointer to an existing array + \note Unlike all other assignment operations, this does NOT result in a copy on write. */ template void write(const T *ptr, const size_t bytes, af::source src = afHost); @@ -761,8 +772,8 @@ namespace af bool isfloating() const; /** - \brief Returns true if the array type is \ref u8, \ref b8, \ref s32 - \ref u32, \ref s64, \ref u64, \ref s16, \ref u16 + \brief Returns true if the array type is \ref s8, \ref u8, \ref b8, + \ref s32, \ref u32, \ref s64, \ref u64, \ref s16, \ref u16 */ bool isinteger() const; @@ -946,27 +957,28 @@ namespace af /// \brief Casts the array into another data type /// - /// \note Consecitive casting operations may be may be optimized out if + /// \note Consecutive casting operations may be optimized out if /// the original type of the af::array is the same as the final type. /// For example if the original type is f64 which is then cast to f32 /// and then back to f64, then the cast to f32 will be skipped and that /// operation will *NOT* be performed by ArrayFire. The following table /// shows which casts will be optimized out. outer -> inner -> outer - /// | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | u8 | b8 | s64 | u64 | s16 | u16 | f16 | - /// |---------|-----|-----|-----|-----|-----|-----|----|----|-----|-----|-----|-----|-----| - /// | f32 | x | x | x | x | | | | | | | | | x | - /// | f64 | x | x | x | x | | | | | | | | | x | - /// | c32 | x | x | x | x | | | | | | | | | x | - /// | c64 | x | x | x | x | | | | | | | | | x | - /// | s32 | x | x | x | x | x | x | | | x | x | | | x | - /// | u32 | x | x | x | x | x | x | | | x | x | | | x | - /// | u8 | x | x | x | x | x | x | x | x | x | x | x | x | x | - /// | b8 | x | x | x | x | x | x | x | x | x | x | x | x | x | - /// | s64 | x | x | x | x | | | | | x | x | | | x | - /// | u64 | x | x | x | x | | | | | x | x | | | x | - /// | s16 | x | x | x | x | x | x | | | x | x | x | x | x | - /// | u16 | x | x | x | x | x | x | | | x | x | x | x | x | - /// | f16 | x | x | x | x | | | | | | | | | x | + /// | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | s8 | u8 | b8 | s64 | u64 | s16 | u16 | f16 | + /// |---------|-----|-----|-----|-----|-----|-----|----|----|----|-----|-----|-----|-----|-----| + /// | f32 | x | x | x | x | | | | | | | | | | x | + /// | f64 | x | x | x | x | | | | | | | | | | x | + /// | c32 | x | x | x | x | | | | | | | | | | x | + /// | c64 | x | x | x | x | | | | | | | | | | x | + /// | s32 | x | x | x | x | x | x | | | | x | x | | | x | + /// | u32 | x | x | x | x | x | x | | | | x | x | | | x | + /// | s8 | x | x | x | x | x | x | x | x | x | x | x | x | x | x | + /// | u8 | x | x | x | x | x | x | x | x | x | x | x | x | x | x | + /// | b8 | x | x | x | x | x | x | x | x | x | x | x | x | x | x | + /// | s64 | x | x | x | x | | | | | | x | x | | | x | + /// | u64 | x | x | x | x | | | | | | x | x | | | x | + /// | s16 | x | x | x | x | x | x | | | | x | x | x | x | x | + /// | u16 | x | x | x | x | x | x | | | | x | x | x | x | x | + /// | f16 | x | x | x | x | | | | | | | | | | x | /// If you want to avoid this behavior use af_eval after the first cast /// operation. This will ensure that the cast operation is performed on /// the af::array @@ -1003,17 +1015,25 @@ namespace af array& OP2(const long long &val); /**< \copydoc OP2##(const array &) */ \ array& OP2(const unsigned long long &val); - #if AF_API_VERSION >= 32 -#define ASSIGN(OP) \ - ASSIGN_(OP) \ - array& OP(const short &val); /**< \copydoc OP##(const array &) */ \ - array& OP(const unsigned short &val); +#define ASSIGN_32(OP) \ + array& OP(const short &val); /**< \copydoc OP##(const array &) */ \ + array& OP(const unsigned short &val); +#else +#define ASSIGN_32(OP) +#endif +#if AF_API_VERSION >= 310 +#define ASSIGN_310(OP) \ + array& OP(const signed char &val); /**< \copydoc OP##(const array &) */ #else -#define ASSIGN(OP) ASSIGN_(OP) +#define ASSIGN_310(OP) #endif +#define ASSIGN(OP) \ + ASSIGN_(OP) \ + ASSIGN_32(OP) \ + ASSIGN_310(OP) /// \ingroup array_mem_operator_eq /// @{ @@ -1079,6 +1099,8 @@ namespace af #undef ASSIGN #undef ASSIGN_ +#undef ASSIGN_32 +#undef ASSIGN_310 /// /// \brief Negates the values of the array @@ -1167,17 +1189,29 @@ namespace af AFAPI array OP (const array& lhs, const cdouble& rhs); #if AF_API_VERSION >= 32 -#define BIN_OP(OP) \ - BIN_OP_(OP) \ +#define BIN_OP_32(OP) \ AFAPI array OP (const short& lhs, const array& rhs); /**< \copydoc OP##(const array&, const array&) */ \ AFAPI array OP (const unsigned short& lhs, const array& rhs); /**< \copydoc OP##(const array&, const array&) */ \ AFAPI array OP (const array& lhs, const short& rhs); /**< \copydoc OP##(const array&, const array&) */ \ AFAPI array OP (const array& lhs, const unsigned short& rhs); #else -#define BIN_OP(OP) BIN_OP_(OP) +#define BIN_OP_32(OP) #endif +#if AF_API_VERSION >= 310 +#define BIN_OP_310(OP) \ + AFAPI array OP (const signed char& lhs, const array& rhs); /**< \copydoc OP##(const array&, const array&) */ \ + AFAPI array OP (const array& lhs, const signed char& rhs); /**< \copydoc OP##(const array&, const array&) */ +#else +#define BIN_OP_310(OP) +#endif + +#define BIN_OP(OP) \ + BIN_OP_(OP) \ + BIN_OP_32(OP) \ + BIN_OP_310(OP) + /// \ingroup arith_func_add /// @{ /// \brief Adds two arrays or an array and a value. @@ -1371,6 +1405,8 @@ namespace af #undef BIN_OP #undef BIN_OP_ +#undef BIN_OP_32 +#undef BIN_OP_310 /// \ingroup arith_func_bitand /// @{ @@ -1393,6 +1429,7 @@ namespace af AFAPI array operator&(const array& lhs, const long long& rhs); AFAPI array operator&(const array& lhs, const long& rhs); AFAPI array operator&(const array& lhs, const short& rhs); + AFAPI array operator&(const array& lhs, const signed char& rhs); AFAPI array operator&(const array& lhs, const unsigned char& rhs); AFAPI array operator&(const array& lhs, const unsigned long long& rhs); AFAPI array operator&(const array& lhs, const unsigned long& rhs); @@ -1408,6 +1445,7 @@ namespace af AFAPI array operator&(const long long& lhs, const array& rhs); AFAPI array operator&(const long& lhs, const array& rhs); AFAPI array operator&(const short& lhs, const array& rhs); + AFAPI array operator&(const signed char& lhs, const array& rhs); AFAPI array operator&(const unsigned char& lhs, const array& rhs); AFAPI array operator&(const unsigned long long& lhs, const array& rhs); AFAPI array operator&(const unsigned long& lhs, const array& rhs); @@ -1436,6 +1474,7 @@ namespace af AFAPI array operator&&(const array& lhs, const long long& rhs); AFAPI array operator&&(const array& lhs, const long& rhs); AFAPI array operator&&(const array& lhs, const short& rhs); + AFAPI array operator&&(const array& lhs, const signed char& rhs); AFAPI array operator&&(const array& lhs, const unsigned char& rhs); AFAPI array operator&&(const array& lhs, const unsigned long long& rhs); AFAPI array operator&&(const array& lhs, const unsigned long& rhs); @@ -1451,6 +1490,7 @@ namespace af AFAPI array operator&&(const long long& lhs, const array& rhs); AFAPI array operator&&(const long& lhs, const array& rhs); AFAPI array operator&&(const short& lhs, const array& rhs); + AFAPI array operator&&(const signed char& lhs, const array& rhs); AFAPI array operator&&(const unsigned char& lhs, const array& rhs); AFAPI array operator&&(const unsigned long long& lhs, const array& rhs); AFAPI array operator&&(const unsigned long& lhs, const array& rhs); diff --git a/include/af/blas.h b/include/af/blas.h index d20986b215..05434ee861 100644 --- a/include/af/blas.h +++ b/include/af/blas.h @@ -1,4 +1,4 @@ -/******************************************************* +/******************************************************** * Copyright (c) 2014, ArrayFire * All rights reserved. * @@ -7,15 +7,7 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ -/** \file blas.h - * - * Contains BLAS related functions - * - * Contains functions for basic BLAS functionallity - */ - #pragma once - #include #ifdef __cplusplus @@ -23,93 +15,95 @@ namespace af { class array; /** - \brief Matrix multiply of two arrays + C++ Interface to multiply two matrices. - \copydetails blas_func_matmul + \copydetails blas_func_matmul - \param[in] lhs The array object on the left hand side - \param[in] rhs The array object on the right hand side - \param[in] optLhs Transpose left hand side before the function is performed - \param[in] optRhs Transpose right hand side before the function is performed - \return The result of the matrix multiplication of lhs, rhs + `optLhs` and `optRhs` can only be one of \ref AF_MAT_NONE, + \ref AF_MAT_TRANS, \ref AF_MAT_CTRANS. - \note optLhs and optRhs can only be one of \ref AF_MAT_NONE, \ref - AF_MAT_TRANS, \ref AF_MAT_CTRANS \note This function is not supported - in GFOR + This function is not supported in GFOR. - \note The following applies for Sparse-Dense matrix multiplication. - \note This function can be used with one sparse input. The sparse input - must always be the \p lhs and the dense matrix must be \p rhs. - \note The sparse array can only be of \ref AF_STORAGE_CSR format. - \note The returned array is always dense. - \note \p optLhs an only be one of \ref AF_MAT_NONE, \ref AF_MAT_TRANS, - \ref AF_MAT_CTRANS. - \note \p optRhs can only be \ref AF_MAT_NONE. + \note The following applies for Sparse-Dense matrix multiplication. + \note This function can be used with one sparse input. The sparse input + must always be the \p lhs and the dense matrix must be \p rhs. + \note The sparse array can only be of \ref AF_STORAGE_CSR format. + \note The returned array is always dense. + \note \p optLhs an only be one of \ref AF_MAT_NONE, \ref AF_MAT_TRANS, + \ref AF_MAT_CTRANS. + \note \p optRhs can only be \ref AF_MAT_NONE. - \ingroup blas_func_matmul + \param[in] lhs input array on the left-hand side + \param[in] rhs input array on the right-hand side + \param[in] optLhs transpose the left-hand side prior to multiplication + \param[in] optRhs transpose the right-hand side prior to multiplication + \return `lhs` * `rhs` - */ + \ingroup blas_func_matmul + */ AFAPI array matmul(const array &lhs, const array &rhs, const matProp optLhs = AF_MAT_NONE, const matProp optRhs = AF_MAT_NONE); /** - \brief Matrix multiply of two arrays + C++ Interface to multiply two matrices. + The second matrix will be transposed. \copydetails blas_func_matmul - \param[in] lhs The array object on the left hand side - \param[in] rhs The array object on the right hand side - \return The result of the matrix multiplication of \p lhs, transpose(\p rhs) + This function is not supported in GFOR. - \note This function is not supported in GFOR + \param[in] lhs input array on the left-hand side + \param[in] rhs input array on the right-hand side + \return `lhs` * transpose(`rhs`) \ingroup blas_func_matmul */ AFAPI array matmulNT(const array &lhs, const array &rhs); /** - \brief Matrix multiply of two arrays + C++ Interface to multiply two matrices. + The first matrix will be transposed. \copydetails blas_func_matmul - \param[in] lhs The array object on the left hand side - \param[in] rhs The array object on the right hand side - \return The result of the matrix multiplication of transpose(\p lhs), \p rhs + This function is not supported in GFOR. - \note This function is not supported in GFOR + \param[in] lhs input array on the left-hand side + \param[in] rhs input array on the right-hand side + \return transpose(`lhs`) * `rhs` \ingroup blas_func_matmul */ AFAPI array matmulTN(const array &lhs, const array &rhs); /** - \brief Matrix multiply of two arrays + C++ Interface to multiply two matrices. + Both matrices will be transposed. \copydetails blas_func_matmul - \param[in] lhs The array object on the left hand side - \param[in] rhs The array object on the right hand side - \return The result of the matrix multiplication of transpose(\p lhs), transpose(\p rhs) + This function is not supported in GFOR. - \note This function is not supported in GFOR + \param[in] lhs input array on the left-hand side + \param[in] rhs input array on the right-hand side + \return transpose(`lhs`) * transpose(`rhs`) \ingroup blas_func_matmul */ AFAPI array matmulTT(const array &lhs, const array &rhs); /** - \brief Chain 2 matrix multiplications + C++ Interface to chain multiply three matrices. - The matrix multiplications are done in a way to reduce temporary memory + The matrix multiplications are done in a way to reduce temporary memory. + + This function is not supported in GFOR. \param[in] a The first array \param[in] b The second array \param[in] c The third array - - \returns out = a x b x c - - \note This function is not supported in GFOR + \return a x b x c \ingroup blas_func_matmul */ @@ -117,18 +111,17 @@ namespace af /** - \brief Chain 3 matrix multiplications + C++ Interface to chain multiply three matrices. - The matrix multiplications are done in a way to reduce temporary memory + The matrix multiplications are done in a way to reduce temporary memory. + + This function is not supported in GFOR. \param[in] a The first array \param[in] b The second array \param[in] c The third array \param[in] d The fourth array - - \returns out = a x b x c x d - - \note This function is not supported in GFOR + \returns a x b x c x d \ingroup blas_func_matmul */ @@ -136,36 +129,34 @@ namespace af #if AF_API_VERSION >= 35 /** - \brief Dot Product + C++ Interface to compute the dot product. - Scalar dot product between two vectors. Also referred to as the inner + Scalar dot product between two vectors, also referred to as the inner product. \code // compute scalar dot product - array x = randu(100), - y = randu(100); + array x = randu(100), y = randu(100); af_print(dot(x, y)); // OR printf("%f\n", dot(x, y)); - \endcode - \tparam T The type of the output - \param[in] lhs The array object on the left hand side - \param[in] rhs The array object on the right hand side - \param[in] optLhs Options for lhs. Currently only \ref AF_MAT_NONE and - AF_MAT_CONJ are supported. - \param[in] optRhs Options for rhs. Currently only \ref AF_MAT_NONE and - AF_MAT_CONJ are supported \return The result of the dot product of lhs, - rhs - - \note optLhs and optRhs can only be one of \ref AF_MAT_NONE or \ref - AF_MAT_CONJ - \note optLhs = AF_MAT_CONJ and optRhs = AF_MAT_NONE will run - conjugate dot operation. - \note This function is not supported in GFOR + Parameters `optLhs` and `optRhs` can only be one of \ref AF_MAT_NONE or + \ref AF_MAT_CONJ. The conjugate dot product can be computed by setting + `optLhs = AF_MAT_CONJ` and `optRhs = AF_MAT_NONE`. + + This function is not supported in GFOR. + + \tparam T type of the output + \param[in] lhs input array on the left-hand side + \param[in] rhs input array on the right-hand side + \param[in] optLhs `lhs` options, only \ref AF_MAT_NONE and \ref + AF_MAT_CONJ are supported + \param[in] optRhs `rhs` options, only \ref AF_MAT_NONE and \ref + AF_MAT_CONJ are supported + \return dot product of `lhs` and `rhs` \ingroup blas_func_dot */ @@ -181,20 +172,21 @@ namespace af const matProp optRhs = AF_MAT_NONE); /** - \brief C++ Interface for transposing a matrix + C++ Interface to transpose a matrix. + + \param[in] in input array + \param[in] conjugate if true, conjugate transposition is performed + \return transpose - \param[in] in an input matrix - \param[in] conjugate if true, a conjugate transposition is performed - \return the transposed matrix \ingroup blas_func_transpose */ AFAPI array transpose(const array &in, const bool conjugate = false); /** - \brief C++ Interface for transposing a matrix in-place + C++ Interface to transpose a matrix in-place. - \param[in,out] in the matrix to be transposed in-place - \param[in] conjugate if true, a conjugate transposition is performed + \param[in,out] in input array to be transposed in-place + \param[in] conjugate if true, conjugate transposition is performed \ingroup blas_func_transpose */ @@ -208,11 +200,10 @@ extern "C" { #if AF_API_VERSION >= 37 /** - \brief BLAS general matrix multiply (GEMM) of two \ref af_array objects + C Interface to multiply two matrices. - \details - This provides a general interface to the BLAS level 3 general matrix - multiply (GEMM), which is generally defined as: + This provides an interface to the BLAS level 3 general matrix multiply + (GEMM) of two \ref af_array objects, which is generally defined as: \f[ C = \alpha * opA(A)opB(B) + \beta * C @@ -251,23 +242,23 @@ extern "C" { \snippet test/blas.cpp ex_af_gemm_overwrite - \param[in,out] C Pointer to the output \ref af_array - - \param[in] opA Operation to perform on A before the multiplication - - \param[in] opB Operation to perform on B before the multiplication - - \param[in] alpha The alpha value; must be the same type as \p lhs - and \p rhs - - \param[in] A Left-hand side operand - - \param[in] B Right-hand side operand - - \param[in] beta The beta value; must be the same type as \p lhs - and \p rhs - - \return AF_SUCCESS if the operation is successful. + \note s8 Support + \note Starting with ArrayFire version v3.10.0, the CUDA backend supports + \p A, \p B input arrays of type \ref s8. + \note Scalars \p alpha, \p beta must be of type \ref f32. + \note Output array \p C will be of type \ref f32. + \note
Requires + \note CUDA version >= 10 on devices with compute capability >= 5.0 + + \param[in,out] C `A` * `B` = `C` + \param[in] opA operation to perform on A before the multiplication + \param[in] opB operation to perform on B before the multiplication + \param[in] alpha alpha value; must be the same type as `A` and `B` + \param[in] A input array on the left-hand side + \param[in] B input array on the right-hand side + \param[in] beta beta value; must be the same type as `A` and `B` + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup blas_func_matmul */ @@ -277,17 +268,9 @@ extern "C" { #endif /** - \brief Matrix multiply of two \ref af_array - - \details Performs a matrix multiplication on two arrays (lhs, rhs). + C Interface to multiply two matrices. - \param[out] out Pointer to the output \ref af_array - \param[in] lhs A 2D matrix \ref af_array object - \param[in] rhs A 2D matrix \ref af_array object - \param[in] optLhs Transpose left hand side before the function is performed - \param[in] optRhs Transpose right hand side before the function is performed - - \return AF_SUCCESS if the process is successful. + Performs matrix multiplication on two arrays. \note The following applies for Sparse-Dense matrix multiplication. \note This function can be used with one sparse input. The sparse input @@ -298,30 +281,41 @@ extern "C" { \ref AF_MAT_CTRANS. \note \p optRhs can only be \ref AF_MAT_NONE. + \param[out] out `lhs` * `rhs` = `out` + \param[in] lhs input array on the left-hand side + \param[in] rhs input array on the right-hand side + \param[in] optLhs transpose `lhs` before the function is performed + \param[in] optRhs transpose `rhs` before the function is performed + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + \ingroup blas_func_matmul */ AFAPI af_err af_matmul( af_array *out , const af_array lhs, const af_array rhs, const af_mat_prop optLhs, const af_mat_prop optRhs); - /** - Scalar dot product between two vectors. Also referred to as the inner + C Interface to compute the dot product. + + Scalar dot product between two vectors, also referred to as the inner product. \code - // compute scalar dot product - array x = randu(100), y = randu(100); - print(dot(x,y)); + // compute scalar dot product + array x = randu(100), y = randu(100); + print(dot(x,y)); \endcode - \param[out] out The array object with the result of the dot operation - \param[in] lhs The array object on the left hand side - \param[in] rhs The array object on the right hand side - \param[in] optLhs Options for lhs. Currently only \ref AF_MAT_NONE and - AF_MAT_CONJ are supported. - \param[in] optRhs Options for rhs. Currently only \ref AF_MAT_NONE and AF_MAT_CONJ are supported - \return AF_SUCCESS if the process is successful. + \param[out] out dot product of `lhs` and `rhs` + \param[in] lhs input array on the left-hand side + \param[in] rhs input array on the right-hand side + \param[in] optLhs `lhs` options, only \ref AF_MAT_NONE and \ref + AF_MAT_CONJ are supported + \param[in] optRhs `rhs` options, only \ref AF_MAT_NONE and \ref + AF_MAT_CONJ are supported + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup blas_func_dot */ @@ -331,18 +325,21 @@ extern "C" { #if AF_API_VERSION >= 35 /** + C Interface to compute the dot product, scalar result returned on host. + Scalar dot product between two vectors. Also referred to as the inner product. Returns the result as a host scalar. - \param[out] real is the real component of the result of dot operation - \param[out] imag is the imaginary component of the result of dot operation - \param[in] lhs The array object on the left hand side - \param[in] rhs The array object on the right hand side - \param[in] optLhs Options for lhs. Currently only \ref AF_MAT_NONE and - AF_MAT_CONJ are supported. - \param[in] optRhs Options for rhs. Currently only \ref AF_MAT_NONE and AF_MAT_CONJ are supported - - \return AF_SUCCESS if the process is successful. + \param[out] real real component of the dot product + \param[out] imag imaginary component of the dot product + \param[in] lhs input array on the left-hand side + \param[in] rhs input array on the right-hand side + \param[in] optLhs `lhs` options, only \ref AF_MAT_NONE and \ref + AF_MAT_CONJ are supported + \param[in] optRhs `rhs` options, only \ref AF_MAT_NONE and \ref + AF_MAT_CONJ are supported + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup blas_func_dot */ @@ -352,22 +349,25 @@ extern "C" { #endif /** - \brief C Interface for transposing a matrix + C Interface to transpose a matrix. - \param[out] out the transposed matrix - \param[in] in an input matrix - \param[in] conjugate if true, a conjugate transposition is performed + \param[out] out transpose + \param[in] in input array + \param[in] conjugate if true, conjugate transposition is performed + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \return AF_SUCCESS if the process is successful. \ingroup blas_func_transpose */ AFAPI af_err af_transpose(af_array *out, af_array in, const bool conjugate); /** - \brief C Interface for transposing a matrix in-place + C Interface to transpose a matrix in-place. - \param[in,out] in is the matrix to be transposed in place - \param[in] conjugate if true, a conjugate transposition is performed + \param[in,out] in input array to be transposed in-place + \param[in] conjugate if true, conjugate transposition is performed + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup blas_func_transpose */ diff --git a/include/af/data.h b/include/af/data.h index 1559ea204f..22e1874439 100644 --- a/include/af/data.h +++ b/include/af/data.h @@ -17,509 +17,479 @@ namespace af { class array; - /** - \param[in] val is the value of each element of the array be genrated - \param[in] dims is the dimensions of the array to be generated - \param[in] ty is the type of the array - - \return array of size \p dims - - \ingroup data_func_constant - */ - + /// C++ Interface to generate an array with elements set to a specified + /// value. + /// + /// \param[in] val constant value + /// \param[in] dims dimensions of the array to be generated + /// \param[in] ty type + /// \return constant array + /// + /// \ingroup data_func_constant template array constant(T val, const dim4 &dims, const dtype ty=(af_dtype)dtype_traits::ctype); - /** - \param[in] val is the value of each element of the array to be generated - \param[in] d0 is the size of the array to be generated - \param[in] ty is the type of the array - - \return array of size \p d0 - - \ingroup data_func_constant - */ - + /// C++ Interface to generate a 1-D array with elements set to a specified + /// value. + /// + /// \param[in] val constant value + /// \param[in] d0 size of the first dimension + /// \param[in] ty type + /// \return constant 1-D array + /// + /// \ingroup data_func_constant template array constant(T val, const dim_t d0, const af_dtype ty=(af_dtype)dtype_traits::ctype); - /** - \param[in] val is the value of each element of the array to be generated - \param[in] d0 is the number of rows of the array to be generated - \param[in] d1 is the number of columns of the array to be generated - \param[in] ty is the type of the array - - \return array of size \p d0 x d1 - - \ingroup data_func_constant - */ + /// C++ Interface to generate a 2-D array with elements set to a specified + /// value. + /// + /// \param[in] val constant value + /// \param[in] d0 size of the first dimension + /// \param[in] d1 size of the second dimension + /// \param[in] ty type + /// \return constant 2-D array + /// + /// \ingroup data_func_constant template array constant(T val, const dim_t d0, const dim_t d1, const af_dtype ty=(af_dtype)dtype_traits::ctype); - /** - \param[in] val is the value of each element of the array to be generated - \param[in] d0 is the size of the 1st dimension of the array to be generated - \param[in] d1 is the size of the 2nd dimension of the array to be generated - \param[in] d2 is the size of the 3rd dimension of the array to be generated - \param[in] ty is the type of the array - - \return array of size \p d0 x d1 x d2 - - \ingroup data_func_constant - */ + /// C++ Interface to generate a 3-D array with elements set to a specified + /// value. + /// + /// \param[in] val constant value + /// \param[in] d0 size of the first dimension + /// \param[in] d1 size of the second dimension + /// \param[in] d2 size of the third dimension + /// \param[in] ty type + /// \return constant 3-D array + /// + /// \ingroup data_func_constant template array constant(T val, const dim_t d0, const dim_t d1, const dim_t d2, const af_dtype ty=(af_dtype)dtype_traits::ctype); - /** - \param[in] val is the value of each element of the array to be generated - \param[in] d0 is the size of the 1st dimension of the array to be generated - \param[in] d1 is the size of the 2nd dimension of the array to be generated - \param[in] d2 is the size of the 3rd dimension of the array to be generated - \param[in] d3 is the size of the 4rd dimension of the array to be generated - \param[in] ty is the type of the array - - \return array of size \p d0 x d1 x d2 x d3 - - \ingroup data_func_constant - */ + /// C++ Interface to generate a 4-D array with elements set to a specified + /// value. + /// + /// \param[in] val constant value + /// \param[in] d0 size of the first dimension + /// \param[in] d1 size of the second dimension + /// \param[in] d2 size of the third dimension + /// \param[in] d3 size of the fourth dimension + /// \param[in] ty type + /// \return constant 4-D array + /// + /// \ingroup data_func_constant template array constant(T val, const dim_t d0, const dim_t d1, const dim_t d2, const dim_t d3, const af_dtype ty=(af_dtype)dtype_traits::ctype); - /** - \param[in] dims is dim4 for size of all dimensions - \param[in] ty is the type of array to generate - - \returns an identity array of specified dimension and type - - \ingroup data_func_identity - */ + /// C++ Interface to generate an identity array. + /// + /// \param[in] dims size + /// \param[in] ty type + /// \return identity array + /// + /// \ingroup data_func_identity AFAPI array identity(const dim4 &dims, const dtype ty=f32); - /** - \param[in] d0 is size of first dimension - \param[in] ty is the type of array to generate - - \returns an identity array of specified dimension and type - - \ingroup data_func_identity - */ + /// C++ Interface to generate a 1-D identity array. + /// + /// \param[in] d0 size of the first dimension + /// \param[in] ty type + /// \return identity array + /// + /// \ingroup data_func_identity AFAPI array identity(const dim_t d0, const dtype ty=f32); - /** - \param[in] d0 is size of first dimension - \param[in] d1 is size of second dimension - \param[in] ty is the type of array to generate - - \returns an identity array of specified dimension and type - - \ingroup data_func_identity - */ + /// C++ Interface to generate a 2-D identity array. + /// + /// \param[in] d0 size of the first dimension + /// \param[in] d1 size of the second dimension + /// \param[in] ty type + /// \return identity array + /// + /// \ingroup data_func_identity AFAPI array identity(const dim_t d0, const dim_t d1, const dtype ty=f32); - /** - \param[in] d0 is size of first dimension - \param[in] d1 is size of second dimension - \param[in] d2 is size of third dimension - \param[in] ty is the type of array to generate - - \returns an identity array of specified dimension and type - - \ingroup data_func_identity - */ + /// C++ Interface to generate a 3-D identity array. + /// + /// \param[in] d0 size of the first dimension + /// \param[in] d1 size of the second dimension + /// \param[in] d2 size of the third dimension + /// \param[in] ty type + /// \return identity array + /// + /// \ingroup data_func_identity AFAPI array identity(const dim_t d0, const dim_t d1, const dim_t d2, const dtype ty=f32); - /** - \param[in] d0 is size of first dimension - \param[in] d1 is size of second dimension - \param[in] d2 is size of third dimension - \param[in] d3 is size of fourth dimension - \param[in] ty is the type of array to generate - - \returns an identity array of specified dimension and type - - \ingroup data_func_identity - */ + /// C++ Interface to generate a 4-D identity array. + /// + /// \param[in] d0 size of the first dimension + /// \param[in] d1 size of the second dimension + /// \param[in] d2 size of the third dimension + /// \param[in] d3 size of the fourth dimension + /// \param[in] ty type + /// \return identity array + /// + /// \ingroup data_func_identity AFAPI array identity(const dim_t d0, const dim_t d1, const dim_t d2, const dim_t d3, const dtype ty=f32); - /** - * C++ Interface for creating an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions of shape `dim4`. - * - \param[in] dims the `dim4` object describing the shape of the generated array - \param[in] seq_dim the dimesion along which `[0, dim[seq_dim] - 1]` is created - \param[in] ty the type of the generated array - - \returns the generated array - - \ingroup data_func_range - */ + /// C++ Interface to generate an array with `[0, n-1]` values along the + /// `seq_dim` dimension and tiled across other dimensions of shape `dim4`. + /// + /// \param[in] dims size + /// \param[in] seq_dim dimesion along which the range is created + /// \param[in] ty type + /// \return range array + /// + /// \ingroup data_func_range AFAPI array range(const dim4 &dims, const int seq_dim = -1, const dtype ty=f32); - /** - * C++ Interface for creating an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions described by dimension parameters. - * - \param[in] d0 the size of first dimension - \param[in] d1 the size of second dimension - \param[in] d2 the size of third dimension - \param[in] d3 the size of fourth dimension - \param[in] seq_dim the dimesion along which `[0, dim[seq_dim] - 1]` is created - \param[in] ty the type of the generated array - - \returns the generated array - - \ingroup data_func_range - */ + /// C++ Interface to generate an array with `[0, n-1]` values along the + /// `seq_dim` dimension and tiled across other dimensions described by + /// dimension parameters. + /// + /// \param[in] d0 size of the first dimension + /// \param[in] d1 size of the second dimension + /// \param[in] d2 size of the third dimension + /// \param[in] d3 size of the fourth dimension + /// \param[in] seq_dim dimesion along which the range is created + /// \param[in] ty type + /// \return range array + /// + /// \ingroup data_func_range AFAPI array range(const dim_t d0, const dim_t d1 = 1, const dim_t d2 = 1, const dim_t d3 = 1, const int seq_dim = -1, const dtype ty=f32); - /** - \param[in] dims is dim4 for unit dimensions of the sequence to be generated - \param[in] tile_dims is dim4 for the number of repetitions of the unit dimensions - \param[in] ty is the type of array to generate - - \returns an array of integral range specified dimension and type - - \ingroup data_func_iota - */ + /// C++ Interface to generate an array with `[0, n-1]` values modified to + /// specified dimensions and tiling. + /// + /// \param[in] dims size + /// \param[in] tile_dims number of tiled repetitions in each dimension + /// \param[in] ty type + /// \return iota array + /// + /// \ingroup data_func_iota AFAPI array iota(const dim4 &dims, const dim4 &tile_dims = dim4(1), const dtype ty=f32); - /** - \param[in] in is the input array - \param[in] num is the diagonal index - \param[in] extract when true returns an array containing diagonal of tha matrix - and when false returns a matrix with \p in as diagonal - - \returns an array with either the diagonal or the matrix based on \p extract - - \ingroup data_func_diag - */ + /// C++ Interface to extract the diagonal from an array. + /// + /// \param[in] in input array + /// \param[in] num diagonal index + /// \param[in] extract if true, returns an array containing diagonal of the + /// matrix; if false, returns a diagonal matrix + /// \return diagonal array (or matrix) + /// + /// \ingroup data_func_diag AFAPI array diag(const array &in, const int num = 0, const bool extract = true); - /** - \brief Join 2 arrays along \p dim - - \param[in] dim is the dimension along which join occurs - \param[in] first is the first input array - \param[in] second is the second input array - \return the array that joins input arrays along the given dimension - - \note empty arrays will be ignored - - \ingroup manip_func_join - */ + /// C++ Interface to join 2 arrays along a dimension. + /// + /// Empty arrays are ignored. + /// + /// \param[in] dim dimension along which the join occurs + /// \param[in] first input array + /// \param[in] second input array + /// \return joined array + /// + /// \ingroup manip_func_join AFAPI array join(const int dim, const array &first, const array &second); - /** - \brief Join 3 arrays along \p dim - - \param[in] dim is the dimension along which join occurs - \param[in] first is the first input array - \param[in] second is the second input array - \param[in] third is the third input array - \return the array that joins input arrays along the given dimension - - \note empty arrays will be ignored - - \ingroup manip_func_join - */ + /// C++ Interface to join 3 arrays along a dimension. + /// + /// Empty arrays are ignored. + /// + /// \param[in] dim dimension along which the join occurs + /// \param[in] first input array + /// \param[in] second input array + /// \param[in] third input array + /// \return joined array + /// + /// \ingroup manip_func_join AFAPI array join(const int dim, const array &first, const array &second, const array &third); - /** - \brief Join 4 arrays along \p dim - - \param[in] dim is the dimension along which join occurs - \param[in] first is the first input array - \param[in] second is the second input array - \param[in] third is the third input array - \param[in] fourth is the fourth input array - \return the array that joins input arrays along the given dimension - - \note empty arrays will be ignored - - \ingroup manip_func_join - */ + /// C++ Interface to join 4 arrays along a dimension. + /// + /// Empty arrays are ignored. + /// + /// \param[in] dim dimension along which the join occurs + /// \param[in] first input array + /// \param[in] second input array + /// \param[in] third input array + /// \param[in] fourth input array + /// \return joined array + /// + /// \ingroup manip_func_join AFAPI array join(const int dim, const array &first, const array &second, const array &third, const array &fourth); - /** - \param[in] in is the input array - \param[in] x is the number of times \p in is copied along the first dimension - \param[in] y is the number of times \p in is copied along the the second dimension - \param[in] z is the number of times \p in is copied along the third dimension - \param[in] w is the number of times \p in is copied along the fourth dimension - \return The tiled version of the input array - - \note \p x, \p y, \p z, and \p w includes the original in the count as - well. Thus, if no duplicates are needed in a certain dimension, - leave it as 1 (the default value for just one copy) - - \ingroup manip_func_tile - */ + /// C++ Interface to generate a tiled array. + /// + /// Note, `x`, `y`, `z`, and `w` include the original in the count. + /// + /// \param[in] in input array + /// \param[in] x number tiles along the first dimension + /// \param[in] y number tiles along the second dimension + /// \param[in] z number tiles along the third dimension + /// \param[in] w number tiles along the fourth dimension + /// \return tiled array + /// + /// \ingroup manip_func_tile AFAPI array tile(const array &in, const unsigned x, const unsigned y=1, const unsigned z=1, const unsigned w=1); - /** - \param[in] in is the input array - \param[in] dims specifies the number of times \p in is copied along each dimension - \return The tiled version of the input array - - \note Each component of \p dims includes the original in the count as - well. Thus, if no duplicates are needed in a certain dimension, - leave it as 1 (the default value for just one copy) - - \ingroup manip_func_tile - */ + /// C++ Interface to generate a tiled array. + /// + /// Each component of `dims` includes the original in the count. Thus, if + /// no duplicates are needed in a certain dimension, it is left as 1, the + /// default value for just one copy. + /// + /// \param[in] in input array + /// \param[in] dims number of times `in` is copied along each dimension + /// \return tiled array + /// + /// \ingroup manip_func_tile AFAPI array tile(const array &in, const dim4 &dims); - /** - \param[in] in is the input array - \param[in] x specifies which dimension should be first - \param[in] y specifies which dimension should be second - \param[in] z specifies which dimension should be third - \param[in] w specifies which dimension should be fourth - \return the reordered output - - \ingroup manip_func_reorder - */ + /// C++ Interface to reorder an array. + /// + /// \param[in] in input array + /// \param[in] x specifies which dimension should be first + /// \param[in] y specifies which dimension should be second + /// \param[in] z specifies which dimension should be third + /// \param[in] w specifies which dimension should be fourth + /// \return reordered array + /// + /// \ingroup manip_func_reorder AFAPI array reorder(const array& in, const unsigned x, const unsigned y=1, const unsigned z=2, const unsigned w=3); - /** - \param[in] in is the input array - \param[in] x specifies the shift along first dimension - \param[in] y specifies the shift along second dimension - \param[in] z specifies the shift along third dimension - \param[in] w specifies the shift along fourth dimension - - \return the shifted output - - \ingroup manip_func_shift - */ + /// C++ Interface to shift an array. + /// + /// \param[in] in input array + /// \param[in] x specifies the shift along the first dimension + /// \param[in] y specifies the shift along the second dimension + /// \param[in] z specifies the shift along the third dimension + /// \param[in] w specifies the shift along the fourth dimension + /// \return shifted array + /// + /// \ingroup manip_func_shift AFAPI array shift(const array& in, const int x, const int y=0, const int z=0, const int w=0); - /** - * C++ Interface for modifying the dimensions of an input array to the shape specified by a `dim4` object - * - \param[in] in the input array - \param[in] dims the array of new dimension sizes - \return the modded output - - \ingroup manip_func_moddims - */ + /// C++ Interface to modify the dimensions of an input array to a specified + /// shape. + /// + /// \param[in] in input array + /// \param[in] dims new dimension sizes + /// \return modded output + /// + /// \ingroup manip_func_moddims AFAPI array moddims(const array& in, const dim4& dims); - /** - * C++ Interface for modifying the dimensions of an input array to the shape specified by dimension length parameters - * - \param[in] in the input array - \param[in] d0 the new size of the first dimension - \param[in] d1 the new size of the second dimension (optional) - \param[in] d2 the new size of the third dimension (optional) - \param[in] d3 the new size of the fourth dimension (optional) - \return the modded output - - \ingroup manip_func_moddims - */ + /// C++ Interface to modify the dimensions of an input array to a specified + /// shape. + /// + /// \param[in] in input array + /// \param[in] d0 new size of the first dimension + /// \param[in] d1 new size of the second dimension (optional) + /// \param[in] d2 new size of the third dimension (optional) + /// \param[in] d3 new size of the fourth dimension (optional) + /// \return modded output + /// + /// \ingroup manip_func_moddims AFAPI array moddims(const array& in, const dim_t d0, const dim_t d1=1, const dim_t d2=1, const dim_t d3=1); - /** - * C++ Interface for modifying the dimensions of an input array to the shape specified by an array of `ndims` dimensions - * - \param[in] in the input array - \param[in] ndims the number of dimensions - \param[in] dims the array of new dimension sizes - \return the modded output - - \ingroup manip_func_moddims - */ + /// C++ Interface to modify the dimensions of an input array to a specified + /// shape. + /// + /// \param[in] in input array + /// \param[in] ndims number of dimensions + /// \param[in] dims new dimension sizes + /// \return modded output + /// + /// \ingroup manip_func_moddims AFAPI array moddims(const array& in, const unsigned ndims, const dim_t* const dims); - /** - \param[in] in is the input array - \return the flat array - - \ingroup manip_func_flat - */ + /// C++ Interface to flatten an array. + /// + /// \param[in] in input array + /// \return flat array + /// + /// \ingroup manip_func_flat AFAPI array flat(const array &in); - /** - \param[in] in is the input array - \param[in] dim is the dimensions to flip the array - \return the flipped array - - \ingroup manip_func_flip - */ + /// C++ Interface to flip an array. + /// + /// \param[in] in input array + /// \param[in] dim dimension to flip + /// \return flipped array + /// + /// \ingroup manip_func_flip AFAPI array flip(const array &in, const unsigned dim); - /** - \param[in] in is the input matrix - \param[in] is_unit_diag is a boolean parameter specifying if the diagonal elements should be 1 - \return the lower triangle array - - \ingroup data_func_lower - */ + /// C++ Interface to return the lower triangle array. + /// + /// \param[in] in input array + /// \param[in] is_unit_diag boolean specifying if diagonal elements are 1's + /// \return lower triangle array + /// + /// \ingroup data_func_lower AFAPI array lower(const array &in, bool is_unit_diag=false); - /** - \param[in] in is the input matrix - \param[in] is_unit_diag is a boolean parameter specifying if the diagonal elements should be 1 - \return the upper triangle matrix - - \ingroup data_func_upper - */ + /// C++ Interface to return the upper triangle array. + /// + /// \param[in] in input array + /// \param[in] is_unit_diag boolean specifying if diagonal elements are 1's + /// \return upper triangle matrix + /// + /// \ingroup data_func_upper AFAPI array upper(const array &in, bool is_unit_diag=false); #if AF_API_VERSION >= 31 - /** - \param[in] cond is the conditional array - \param[in] a is the array containing elements from the true part of the condition - \param[in] b is the array containing elements from the false part of the condition - \return the output containing elements of \p a when \p cond is true else elements from \p b - - \ingroup data_func_select - */ + /// C++ Interface to select elements based on a conditional array. + /// + /// \param[in] cond conditional array + /// \param[in] a when true, select array element + /// \param[in] b when false, select array element + /// \return `a` when `cond` is true, else `b` + /// + /// \ingroup data_func_select AFAPI array select(const array &cond, const array &a, const array &b); #endif #if AF_API_VERSION >= 31 - /** - \param[in] cond is the conditional array - \param[in] a is the array containing elements from the true part of the condition - \param[in] b is a scalar assigned to \p out when \p cond is false - \return the output containing elements of \p a when \p cond is true else the value \p b - - \ingroup data_func_select - */ + /// C++ Interface to select elements based on a conditional array. + /// + /// \param[in] cond conditional array + /// \param[in] a when true, select array element + /// \param[in] b when false, select scalar value + /// \return `a` when `cond` is true, else `b` + /// + /// \ingroup data_func_select AFAPI array select(const array &cond, const array &a, const double &b); #endif #if AF_API_VERSION >= 31 - /** - \param[in] cond is the conditional array - \param[in] a is a scalar assigned to \p out when \p cond is true - \param[in] b is the array containing elements from the false part of the condition - \return the output containing the value \p a when \p cond is true else elements from \p b - - \ingroup data_func_select - */ + /// C++ Interface to select elements based on a conditional array. + /// + /// \param[in] cond conditional array + /// \param[in] a when true, select scalar value + /// \param[in] b when false, select array element + /// \return `a` when `cond` is true, else `b` + /// + /// \ingroup data_func_select AFAPI array select(const array &cond, const double &a, const array &b); #endif #if AF_API_VERSION >= 31 - /** - \param[inout] a is the input array - \param[in] cond is the conditional array. - \param[in] b is the replacement array. - - \note Values of \p a are replaced with corresponding values of \p b, when \p cond is false. - - \ingroup data_func_replace - */ + /// C++ Interface to replace elements of an array with elements of another + /// array. + /// + /// Elements of `a` are replaced with corresponding elements of `b` when + /// `cond` is false. + /// + /// \param[inout] a input array + /// \param[in] cond conditional array + /// \param[in] b replacement array + /// + /// \ingroup data_func_replace AFAPI void replace(array &a, const array &cond, const array &b); #endif #if AF_API_VERSION >= 31 - /** - \param[inout] a is the input array - \param[in] cond is the conditional array. - \param[in] b is the replacement value. - - \note Values of \p a are replaced with value \p b, when \p cond is false. - - \ingroup data_func_replace - */ + /// C++ Interface to replace elements of an array with a scalar value. + /// + /// Elements of `a` are replaced with a scalar value when `cond` is false. + /// + /// \param[inout] a input array + /// \param[in] cond conditional array + /// \param[in] b replacement scalar value + /// + /// \ingroup data_func_replace AFAPI void replace(array &a, const array &cond, const double &b); #endif #if AF_API_VERSION >= 37 - /** - \param[in] in is the input array to be padded - \param[in] beginPadding informs the number of elements to be - padded at beginning of each dimension - \param[in] endPadding informs the number of elements to be - padded at end of each dimension - \param[in] padFillType is indicates what values should fill padded region - - \return the padded array - - \ingroup data_func_pad - */ + /// C++ Interface to pad an array. + /// + /// \param[in] in input array + /// \param[in] beginPadding number of elements to be padded at the start of + /// each dimension + /// \param[in] endPadding number of elements to be padded at the end of + /// each dimension + /// \param[in] padFillType values to fill into the padded region + /// \return padded array + /// + /// \ingroup data_func_pad AFAPI array pad(const array &in, const dim4 &beginPadding, const dim4 &endPadding, const borderType padFillType); #endif #if AF_API_VERSION >= 39 - /** - \param[inout] a is the input array - \param[in] cond is the conditional array. - \param[in] b is the replacement scalar value. - - \note Values of \p a are replaced with value \p b, when \p cond is false. - - \ingroup data_func_replace - */ + /// C++ Interface to replace elements of an array with a scalar value. + /// + /// Elements of `a` are replaced with a scalar value when `cond` is false. + /// + /// \param[inout] a input array + /// \param[in] cond conditional array + /// \param[in] b replacement scalar value + /// + /// \ingroup data_func_replace AFAPI void replace(array &a, const array &cond, const long long b); - /** - \param[inout] a is the input array - \param[in] cond is the conditional array. - \param[in] b is the replacement scalar value. - - \note Values of \p a are replaced with value \p b, when \p cond is false. - - \ingroup data_func_replace - */ + /// C++ Interface to replace elements of an array with a scalar value. + /// + /// Elements of `a` are replaced with a scalar value when `cond` is false. + /// + /// \param[inout] a input array + /// \param[in] cond conditional array + /// \param[in] b replacement scalar value + /// + /// \ingroup data_func_replace AFAPI void replace(array &a, const array &cond, const unsigned long long b); - /** - \param[in] cond is the conditional array - \param[in] a is the array containing elements from the true part of the - condition - \param[in] b is a scalar assigned to \p out when \p cond is false - \return the output containing elements of \p a when \p cond is true - else the value \p b - - \ingroup data_func_select - */ + /// C++ Interface to select elements based on a conditional array. + /// + /// \param[in] cond conditional array + /// \param[in] a when true, select array element + /// \param[in] b when false, select scalar value + /// \return `a` when `cond` is true, else `b` + /// + /// \ingroup data_func_select AFAPI array select(const array &cond, const array &a, const long long b); - /** - \param[in] cond is the conditional array - \param[in] a is the array containing elements from the true part of the - condition - \param[in] b is a scalar assigned to \p out when \p cond is false - \return the output containing elements of \p a when \p cond is true - else the value \p b - - \ingroup data_func_select - */ + /// C++ Interface to select elements based on a conditional array. + /// + /// \param[in] cond conditional array + /// \param[in] a when true, select array element + /// \param[in] b when false, select scalar value + /// \return `a` when `cond` is true, else `b` + /// + /// \ingroup data_func_select AFAPI array select(const array &cond, const array &a, const unsigned long long b); - /** - \param[in] cond is the conditional array - \param[in] a is a scalar assigned to \p out when \p cond is true - \param[in] b is the array containing elements from the false part of the - condition - \return the output containing the value \p a when \p cond is true else - elements from \p b - - \ingroup data_func_select - */ + /// C++ Interface to select elements based on a conditional array. + /// + /// \param[in] cond conditional array + /// \param[in] a when true, select scalar value + /// \param[in] b when false, select array element + /// \return `a` when `cond` is true, else `b` + /// + /// \ingroup data_func_select AFAPI array select(const array &cond, const long long a, const array &b); - /** - \param[in] cond is the conditional array - \param[in] a is a scalar assigned to \p out when \p cond is true - \param[in] b is the array containing elements from the false part of the - condition - \return the output containing the value \p a when \p cond is true else - elements from \p b - - \ingroup data_func_select - */ + /// C++ Interface to select elements based on a conditional array. + /// + /// \param[in] cond conditional array + /// \param[in] a when true, select scalar value + /// \param[in] b when false, select array element + /// \return `a` when `cond` is true, else `b` + /// + /// \ingroup data_func_select AFAPI array select(const array &cond, const unsigned long long a, const array &b); #endif @@ -530,46 +500,65 @@ namespace af extern "C" { #endif /** - \param[out] arr is the generated array of given type - \param[in] val is the value of each element in the generated array - \param[in] ndims is size of dimension array \p dims - \param[in] dims is the array containing sizes of the dimension - \param[in] type is the type of array to generate + C Interface to generate an array with elements set to a specified value. + + \param[out] arr constant array + \param[in] val constant value + \param[in] ndims size of the dimension array + \param[in] dims dimensions of the array to be generated + \param[in] type type + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_constant */ AFAPI af_err af_constant(af_array *arr, const double val, const unsigned ndims, const dim_t * const dims, const af_dtype type); /** - \param[out] arr is the generated array of type \ref c32 or \ref c64 - \param[in] real is the real value of each element in the generated array - \param[in] imag is the imaginary value of each element in the generated array - \param[in] ndims is size of dimension array \p dims - \param[in] dims is the array containing sizes of the dimension - \param[in] type is the type of array to generate + C Interface to generate a complex array with elements set to a specified + value. + + \param[out] arr constant complex array + \param[in] real real constant value + \param[in] imag imaginary constant value + \param[in] ndims size of the dimension array + \param[in] dims dimensions of the array to be generated + \param[in] type type, \ref c32 or \ref c64 + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_constant */ - AFAPI af_err af_constant_complex(af_array *arr, const double real, const double imag, const unsigned ndims, const dim_t * const dims, const af_dtype type); /** - \param[out] arr is the generated array of type \ref s64 - \param[in] val is a complex value of each element in the generated array - \param[in] ndims is size of dimension array \p dims - \param[in] dims is the array containing sizes of the dimension + C Interface to generate an array with elements set to a specified value. + + Output type is \ref s64. + + \param[out] arr constant array + \param[in] val constant value + \param[in] ndims size of the dimension array + \param[in] dims dimensions of the array to be generated + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_constant */ - AFAPI af_err af_constant_long (af_array *arr, const long long val, const unsigned ndims, const dim_t * const dims); /** - \param[out] arr is the generated array of type \ref u64 - \param[in] val is a complex value of each element in the generated array - \param[in] ndims is size of dimension array \p dims - \param[in] dims is the array containing sizes of the dimension + C Interface to generate an array with elements set to a specified value. + + Output type is \ref u64. + + \param[out] arr constant array + \param[in] val constant value + \param[in] ndims size of the dimension array + \param[in] dims dimensions of the array to be generated + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_constant */ @@ -577,186 +566,246 @@ extern "C" { AFAPI af_err af_constant_ulong(af_array *arr, const unsigned long long val, const unsigned ndims, const dim_t * const dims); /** - * C Interface for creating an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions specified by an array of `ndims` dimensions. - * - \param[out] out the generated array - \param[in] ndims the size of dimension array `dims` - \param[in] dims the array containing the dimension sizes - \param[in] seq_dim the dimension along which `[0, dim[seq_dim] - 1]` is created - \param[in] type the type of the generated array - - \ingroup data_func_range + C Interface to generate an identity array. + + \param[out] out identity array + \param[in] ndims number of dimensions + \param[in] dims size + \param[in] type type + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + + \ingroup data_func_identity + */ + AFAPI af_err af_identity(af_array* out, const unsigned ndims, const dim_t* const dims, const af_dtype type); + + /** + C Interface to generate an array with `[0, n-1]` values along the + `seq_dim` dimension and tiled across other dimensions of shape `dim4`. + + \param[out] out range array + \param[in] ndims number of dimensions, specified by the size of `dims` + \param[in] dims size + \param[in] seq_dim dimension along which the range is created + \param[in] type type + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + + \ingroup data_func_range */ AFAPI af_err af_range(af_array *out, const unsigned ndims, const dim_t * const dims, const int seq_dim, const af_dtype type); /** - \param[out] out is the generated array - \param[in] ndims is size of dimension array \p dims - \param[in] dims is the array containing sizes of the dimension - \param[in] t_ndims is size of tile array \p tdims - \param[in] tdims is array containing the number of repetitions of the unit dimensions - \param[in] type is the type of array to generate - - \ingroup data_func_iota + C Interface to generate an array with `[0, n-1]` values modified to + specified dimensions and tiling. + + \param[out] out iota array + \param[in] ndims number of dimensions + \param[in] dims size + \param[in] t_ndims number of dimensions of tiled array + \param[in] tdims number of tiled repetitions in each dimension + \param[in] type type + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + + \ingroup data_func_iota */ AFAPI af_err af_iota(af_array *out, const unsigned ndims, const dim_t * const dims, const unsigned t_ndims, const dim_t * const tdims, const af_dtype type); - /** - \param[out] out is the generated array - \param[in] ndims is size of dimension array \p dims - \param[in] dims is the array containing sizes of the dimension - \param[in] type is the type of array to generate + C Interface to create a diagonal matrix from an extracted diagonal + array. - \ingroup data_func_identity - */ - AFAPI af_err af_identity(af_array *out, const unsigned ndims, const dim_t * const dims, const af_dtype type); + See also, \ref af_diag_extract. - /** - \param[out] out is the array created from the input array \p in - \param[in] in is the input array which is the diagonal - \param[in] num is the diagonal index + \param[out] out diagonal matrix + \param[in] in diagonal array + \param[in] num diagonal index + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \ingroup data_func_diag + \ingroup data_func_diag */ AFAPI af_err af_diag_create(af_array *out, const af_array in, const int num); /** - \param[out] out is the \p num -th diagonal of \p in - \param[in] in is the input matrix - \param[in] num is the diagonal index + C Interface to extract the diagonal from an array. - \ingroup data_func_diag + See also, \ref af_diag_create. + + \param[out] out `num`-th diagonal array + \param[in] in input array + \param[in] num diagonal index + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + + \ingroup data_func_diag */ AFAPI af_err af_diag_extract(af_array *out, const af_array in, const int num); /** - \brief Join 2 arrays along \p dim + C Interface to join 2 arrays along a dimension. - \param[out] out is the generated array - \param[in] dim is the dimension along which join occurs - \param[in] first is the first input array - \param[in] second is the second input array + Empty arrays are ignored. - \note empty arrays will be ignored + \param[out] out joined array + \param[in] dim dimension along which the join occurs + \param[in] first input array + \param[in] second input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \ingroup manip_func_join + \ingroup manip_func_join */ AFAPI af_err af_join(af_array *out, const int dim, const af_array first, const af_array second); /** - \brief Join many arrays along \p dim - - Current limit is set to 10 arrays. + C Interface to join many arrays along a dimension. - \param[out] out is the generated array - \param[in] dim is the dimension along which join occurs - \param[in] n_arrays number of arrays to join - \param[in] inputs is an array of af_arrays containing handles to the arrays to be joined + Limited to 10 arrays. Empty arrays are ignored. - \note empty arrays will be ignored + \param[out] out joined array + \param[in] dim dimension along which the join occurs + \param[in] n_arrays number of arrays to join + \param[in] inputs array of af_arrays containing handles to the + arrays to be joined + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \ingroup manip_func_join + \ingroup manip_func_join */ AFAPI af_err af_join_many(af_array *out, const int dim, const unsigned n_arrays, const af_array *inputs); /** - \param[out] out is the tiled version of the input array - \param[in] in is the input matrix - \param[in] x is the number of times \p in is copied along the first dimension - \param[in] y is the number of times \p in is copied along the the second dimension - \param[in] z is the number of times \p in is copied along the third dimension - \param[in] w is the number of times \p in is copied along the fourth dimension - - \note \p x, \p y, \p z, and \p w includes the original in the count as - well. Thus, if no duplicates are needed in a certain dimension, - leave it as 1 (the default value for just one copy) - - \ingroup manip_func_tile + C Interface to generate a tiled array. + + Note, `x`, `y`, `z`, and `w` include the original in the count. + + \param[out] out tiled array + \param[in] in input array + \param[in] x number of tiles along the first dimension + \param[in] y number of tiles along the second dimension + \param[in] z number of tiles along the third dimension + \param[in] w number of tiles along the fourth dimension + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + + \ingroup manip_func_tile */ AFAPI af_err af_tile(af_array *out, const af_array in, const unsigned x, const unsigned y, const unsigned z, const unsigned w); /** - \param[out] out is the reordered array - \param[in] in is the input matrix - \param[in] x specifies which dimension should be first - \param[in] y specifies which dimension should be second - \param[in] z specifies which dimension should be third - \param[in] w specifies which dimension should be fourth - - \ingroup manip_func_reorder + C Interface to reorder an array. + + \param[out] out reordered array + \param[in] in input array + \param[in] x specifies which dimension should be first + \param[in] y specifies which dimension should be second + \param[in] z specifies which dimension should be third + \param[in] w specifies which dimension should be fourth + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + + \ingroup manip_func_reorder */ AFAPI af_err af_reorder(af_array *out, const af_array in, const unsigned x, const unsigned y, const unsigned z, const unsigned w); /** - \param[in] out is the shifted array - \param[in] in is the input array - \param[in] x specifies the shift along first dimension - \param[in] y specifies the shift along second dimension - \param[in] z specifies the shift along third dimension - \param[in] w specifies the shift along fourth dimension - - \ingroup manip_func_shift + C Interface to shift an array. + + \param[out] out shifted array + \param[in] in input array + \param[in] x specifies the shift along first dimension + \param[in] y specifies the shift along second dimension + \param[in] z specifies the shift along third dimension + \param[in] w specifies the shift along fourth dimension + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + + \ingroup manip_func_shift */ AFAPI af_err af_shift(af_array *out, const af_array in, const int x, const int y, const int z, const int w); /** - * C Interface for modifying the dimensions of an input array to the shape specified by an array of `ndims` dimensions - * - \param[out] out the modded output - \param[in] in the input array - \param[in] ndims the number of dimensions - \param[in] dims the array of new dimension sizes - - \ingroup manip_func_moddims + C Interface to modify the dimensions of an input array to a specified + shape. + + \param[out] out modded output + \param[in] in input array + \param[in] ndims number of dimensions + \param[in] dims new dimension sizes + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + + \ingroup manip_func_moddims */ AFAPI af_err af_moddims(af_array *out, const af_array in, const unsigned ndims, const dim_t * const dims); /** - \param[out] out is the flat array - \param[in] in is the input array + C Interface to flatten an array. + + \param[out] out flat array + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \ingroup manip_func_flat + \ingroup manip_func_flat */ AFAPI af_err af_flat(af_array *out, const af_array in); /** - \param[out] out is the flipped array - \param[in] in is the input array - \param[in] dim is the dimensions to flip the array + C Interface to flip an array. - \ingroup manip_func_flip + \param[out] out flipped array + \param[in] in input array + \param[in] dim dimension to flip + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + + \ingroup manip_func_flip */ AFAPI af_err af_flip(af_array *out, const af_array in, const unsigned dim); /** - \param[out] out is the lower traingle matrix - \param[in] in is the input matrix - \param[in] is_unit_diag is a boolean parameter specifying if the diagonal elements should be 1 + C Interface to return the lower triangle array. + + \param[out] out lower traingle array + \param[in] in input array + \param[in] is_unit_diag boolean specifying if diagonal elements are 1's + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \ingroup data_func_lower + \ingroup data_func_lower */ AFAPI af_err af_lower(af_array *out, const af_array in, bool is_unit_diag); /** - \param[out] out is the upper triangle matrix - \param[in] in is the input matrix - \param[in] is_unit_diag is a boolean parameter specifying if the diagonal elements should be 1 + C Interface to return the upper triangle array. + + \param[out] out upper triangle array + \param[in] in input array + \param[in] is_unit_diag boolean specifying if diagonal elements are 1's + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \ingroup data_func_upper + \ingroup data_func_upper */ AFAPI af_err af_upper(af_array *out, const af_array in, bool is_unit_diag); #if AF_API_VERSION >= 31 /** - \param[out] out is the output containing elements of \p a when \p cond is true else elements from \p b - \param[in] cond is the conditional array - \param[in] a is the array containing elements from the true part of the condition - \param[in] b is the array containing elements from the false part of the condition + C Interface to select elements based on a conditional array. + + \param[out] out `a` when `cond` is true, else `b` + \param[in] cond conditional array + \param[in] a when true, select array element + \param[in] b when false, select array element + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_select */ @@ -765,10 +814,14 @@ extern "C" { #if AF_API_VERSION >= 31 /** - \param[out] out is the output containing elements of \p a when \p cond is true else elements from \p b - \param[in] cond is the conditional array - \param[in] a is the array containing elements from the true part of the condition - \param[in] b is a scalar assigned to \p out when \p cond is false + C Interface to select elements based on a conditional array. + + \param[out] out `a` when `cond` is true, else `b` + \param[in] cond conditional array + \param[in] a when true, select array element + \param[in] b when false, select scalar value + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_select */ @@ -777,10 +830,14 @@ extern "C" { #if AF_API_VERSION >= 31 /** - \param[out] out is the output containing elements of \p a when \p cond is true else elements from \p b - \param[in] cond is the conditional array - \param[in] a is a scalar assigned to \p out when \p cond is true - \param[in] b is the array containing elements from the false part of the condition + C Interface to select elements based on a conditional array. + + \param[out] out `a` when `cond` is true, else `b` + \param[in] cond conditional array + \param[in] a when true, select scalar value + \param[in] b when false, select array element + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_select */ @@ -789,11 +846,17 @@ extern "C" { #if AF_API_VERSION >= 31 /** - \param[inout] a is the input array - \param[in] cond is the conditional array. - \param[in] b is the replacement array. + C Interface to replace elements of an array with elements of another + array. - \note Values of \p a are replaced with corresponding values of \p b, when \p cond is false. + Elements of `a` are replaced with corresponding elements of `b` when + `cond` is false. + + \param[inout] a input array + \param[in] cond conditional array + \param[in] b replacement array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_replace */ @@ -802,11 +865,15 @@ extern "C" { #if AF_API_VERSION >= 31 /** - \param[inout] a is the input array - \param[in] cond is the conditional array. - \param[in] b is the replacement array. + C Interface to replace elements of an array with a scalar value. + + Elements of `a` are replaced with a scalar value when `cond` is false. - \note Values of \p a are replaced with corresponding values of \p b, when \p cond is false. + \param[inout] a input array + \param[in] cond conditional array + \param[in] b replacement scalar value + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_replace */ @@ -815,15 +882,19 @@ extern "C" { #if AF_API_VERSION >= 37 /** - \param[out] out is the padded array - \param[in] in is the input array to be padded - \param[in] begin_ndims is size of \p l_dims array - \param[in] begin_dims array contains padding size at beginning of each - dimension - \param[in] end_ndims is size of \p u_dims array - \param[in] end_dims array contains padding sizes at end of each dimension - \param[in] pad_fill_type is indicates what values should fill - padded region + C Interface to pad an array. + + \param[out] out padded array + \param[in] in input array + \param[in] begin_ndims number of dimensions for start padding + \param[in] begin_dims number of elements to be padded at the start + of each dimension + \param[in] end_ndims number of dimensions for end padding + \param[in] end_dims number of elements to be padded at the end of + each dimension + \param[in] pad_fill_type values to fill into the padded region + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_pad */ @@ -836,12 +907,15 @@ extern "C" { #if AF_API_VERSION >= 39 /** - \param[inout] a is the input array - \param[in] cond is the conditional array. - \param[in] b is the replacement array. + C Interface to replace elements of an array with a scalar value. - \note Values of \p a are replaced with corresponding values of \p b, when - \p cond is false. + Elements of `a` are replaced with a scalar value when `cond` is false. + + \param[inout] a input array + \param[in] cond conditional array + \param[in] b replacement scalar value + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_replace */ @@ -849,12 +923,15 @@ extern "C" { const long long b); /** - \param[inout] a is the input array - \param[in] cond is the conditional array. - \param[in] b is the replacement array. + C Interface to replace elements of an array with a scalar value. + + Elements of `a` are replaced with a scalar value when `cond` is false. - \note Values of \p a are replaced with corresponding values of \p b, when - \p cond is false. + \param[inout] a input array + \param[in] cond conditional array + \param[in] b replacement scalar value + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_replace */ @@ -862,13 +939,14 @@ extern "C" { const unsigned long long b); /** - \param[out] out is the output containing elements of \p a when \p cond is - true else elements from \p b - \param[in] cond is the conditional array - \param[in] a is the array containing elements from the true part of the - condition - \param[in] b is a scalar assigned to \p out when \p cond is - false + C Interface to select elements based on a conditional array. + + \param[out] out `a` when `cond` is true, else `b` + \param[in] cond conditional array + \param[in] a when true, select array element + \param[in] b when false, select scalar value + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_select */ @@ -876,13 +954,14 @@ extern "C" { const af_array a, const long long b); /** - \param[out] out is the output containing elements of \p a when \p cond is - true else elements from \p b - \param[in] cond is the conditional array - \param[in] a is the array containing elements from the true part of the - condition - \param[in] b is a scalar assigned to \p out when \p cond is - false + C Interface to select elements based on a conditional array. + + \param[out] out `a` when `cond` is true, else `b` + \param[in] cond conditional array + \param[in] a when true, select array element + \param[in] b when false, select scalar value + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_select */ @@ -891,12 +970,14 @@ extern "C" { const unsigned long long b); /** - \param[out] out is the output containing elements of \p a when \p cond is - true else elements from \p b - \param[in] cond is the conditional array - \param[in] a is a scalar assigned to \p out when \p cond is true - \param[in] b is the array containing elements from the false part of the - condition + C Interface to select elements based on a conditional array. + + \param[out] out `a` when `cond` is true, else `b` + \param[in] cond conditional array + \param[in] a when true, select scalar value + \param[in] b when false, select array element + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_select */ @@ -904,12 +985,14 @@ extern "C" { const long long a, const af_array b); /** - \param[out] out is the output containing elements of \p a when \p cond is - true else elements from \p b - \param[in] cond is the conditional array - \param[in] a is a scalar assigned to \p out when \p cond is true - \param[in] b is the array containing elements from the false part of the - condition + C Interface to select elements based on a conditional array. + + \param[out] out `a` when `cond` is true, else `b` + \param[in] cond conditional array + \param[in] a when true, select scalar value + \param[in] b when false, select array element + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup data_func_select */ diff --git a/include/af/defines.h b/include/af/defines.h index da6c5591de..42f71024fa 100644 --- a/include/af/defines.h +++ b/include/af/defines.h @@ -227,6 +227,9 @@ typedef enum { #if AF_API_VERSION >= 37 , f16 ///< 16-bit floating point value #endif +#if AF_API_VERSION >= 310 + , s8 ///< 8-bit signed integral values +#endif } af_dtype; typedef enum { diff --git a/include/af/index.h b/include/af/index.h index 3bceb96cbf..8eaaeaa0a5 100644 --- a/include/af/index.h +++ b/include/af/index.h @@ -274,7 +274,7 @@ extern "C" { /// the sequences /// \param[in] lhs is the input array /// \param[in] ndims is the number of \ref af_index_t provided - /// \param[in] indices is an af_array of \ref af_index_t objects + /// \param[in] indices is a C array of \ref af_index_t objects /// \param[in] rhs is the array whose values will be assigned to \p lhs /// /// \ingroup index_func_assign diff --git a/include/af/lapack.h b/include/af/lapack.h index 271d99cf4c..be30cd5900 100644 --- a/include/af/lapack.h +++ b/include/af/lapack.h @@ -16,12 +16,13 @@ namespace af { #if AF_API_VERSION >= 31 /** - C++ Interface for SVD decomposition + C++ Interface to perform singular value decomposition. - \param[out] u is the output array containing U - \param[out] s is the output array containing the diagonal values of sigma, (singular values of the input matrix)) - \param[out] vt is the output array containing V^H - \param[in] in is the input matrix + \param[out] u U + \param[out] s diagonal values of sigma (singular values of the input + matrix) + \param[out] vt V^H + \param[in] in input array \ingroup lapack_factor_func_svd */ @@ -30,18 +31,16 @@ namespace af #if AF_API_VERSION >= 31 /** - C++ Interface for SVD decomposition (in-place) + C++ Interface to perform in-place singular value decomposition. - \param[out] u is the output array containing U - \param[out] s is the output array containing the diagonal values of sigma, - (singular values of the input matrix)) - \param[out] vt is the output array containing V^H - \param[in,out] in is the input matrix and will contain random data after - this operation + This function minimizes memory usage if `in` is dispensable. Input array + `in` is limited to arrays where `dim0` \f$\geq\f$ `dim1`. - \note Currently, \p in is limited to arrays where `dim0` \f$\geq\f$ `dim1` - \note This is best used when minimizing memory usage and \p in is - dispensable + \param[out] u U + \param[out] s diagonal values of sigma (singular values of the input + matrix) + \param[out] vt V^H + \param[inout] in input array; contains random data after the operation this operation \ingroup lapack_factor_func_svd */ @@ -49,158 +48,176 @@ namespace af #endif /** - C++ Interface for LU decomposition in packed format + C++ Interface to perform LU decomposition in packed format. - \param[out] out is the output array containing the packed LU decomposition - \param[out] pivot will contain the permutation indices to map the input to the decomposition - \param[in] in is the input matrix - \param[in] is_lapack_piv specifies if the pivot is returned in original LAPACK compliant format + This function is not supported in GFOR. - \note This function is not supported in GFOR + \param[out] out packed LU decomposition + \param[out] pivot permutation indices mapping the input to the + decomposition + \param[in] in input array + \param[in] is_lapack_piv specifies if the pivot is returned in original + LAPACK compliant format \ingroup lapack_factor_func_lu */ AFAPI void lu(array &out, array &pivot, const array &in, const bool is_lapack_piv=true); /** - C++ Interface for LU decomposition + C++ Interface to perform LU decomposition. - \param[out] lower will contain the lower triangular matrix of the LU decomposition - \param[out] upper will contain the upper triangular matrix of the LU decomposition - \param[out] pivot will contain the permutation indices to map the input to the decomposition - \param[in] in is the input matrix + This function is not supported in GFOR. - \note This function is not supported in GFOR + \param[out] lower lower triangular matrix of the LU decomposition + \param[out] upper upper triangular matrix of the LU decomposition + \param[out] pivot permutation indices mapping the input to the + decomposition + \param[in] in input array \ingroup lapack_factor_func_lu */ AFAPI void lu(array &lower, array &upper, array &pivot, const array &in); /** - C++ Interface for in place LU decomposition + C++ Interface to perform in-place LU decomposition. - \param[out] pivot will contain the permutation indices to map the input to the decomposition - \param[inout] in contains the input on entry, the packed LU decomposition on exit - \param[in] is_lapack_piv specifies if the pivot is returned in original LAPACK compliant format + This function is not supported in GFOR. - \note This function is not supported in GFOR + \param[out] pivot permutation indices mapping the input to the + decomposition + \param[inout] in input array on entry; packed LU + decomposition on exit + \param[in] is_lapack_piv specifies if the pivot is returned in + original LAPACK-compliant format - \ingroup lapack_factor_func_lu + \ingroup lapack_factor_func_lu */ AFAPI void luInPlace(array &pivot, array &in, const bool is_lapack_piv=true); /** - C++ Interface for QR decomposition in packed format + C++ Interface to perform QR decomposition in packed format. - \param[out] out is the output array containing the packed QR decomposition - \param[out] tau will contain additional information needed for unpacking the data - \param[in] in is the input matrix + This function is not supported in GFOR. - \note This function is not supported in GFOR + \param[out] out packed QR decomposition + \param[out] tau additional information needed for unpacking the data + \param[in] in input array \ingroup lapack_factor_func_qr */ AFAPI void qr(array &out, array &tau, const array &in); /** - C++ Interface for QR decomposition + C++ Interface to perform QR decomposition. - \param[out] q is the orthogonal matrix from QR decomposition - \param[out] r is the upper triangular matrix from QR decomposition - \param[out] tau will contain additional information needed for solving a least squares problem using \p q and \p r - \param[in] in is the input matrix + This function is not supported in GFOR. - \note This function is not supported in GFOR + \param[out] q orthogonal matrix from QR decomposition + \param[out] r upper triangular matrix from QR decomposition + \param[out] tau additional information needed for solving a + least-squares problem using `q` and `r` + \param[in] in input array \ingroup lapack_factor_func_qr */ AFAPI void qr(array &q, array &r, array &tau, const array &in); /** - C++ Interface for QR decomposition + C++ Interface to perform QR decomposition. - \param[out] tau will contain additional information needed for unpacking the data - \param[inout] in is the input matrix on entry. It contains packed QR decomposition on exit + This function is not supported in GFOR. - \note This function is not supported in GFOR + \param[out] tau additional information needed for unpacking the data + \param[inout] in input array on entry; packed QR decomposition on exit \ingroup lapack_factor_func_qr */ AFAPI void qrInPlace(array &tau, array &in); /** - C++ Interface for cholesky decomposition - - \param[out] out contains the triangular matrix. Multiply \p out with its conjugate transpose reproduces the input \p in. - \param[in] in is the input matrix - \param[in] is_upper a boolean determining if \p out is upper or lower triangular + C++ Interface to perform Cholesky decomposition. - \returns \p 0 if cholesky decomposition passes, if not it returns the rank at which the decomposition failed. + Multiplying `out` with its conjugate transpose reproduces the input + `in`. + + The input must be positive definite. + + This function is not supported in GFOR. - \note The input matrix \b has to be a positive definite matrix, if it is not zero, the cholesky decomposition functions return a non-zero output. - \note This function is not supported in GFOR + \param[out] out triangular matrix; + \param[in] in input matrix + \param[in] is_upper boolean determining if `out` is upper or lower + triangular + \returns `0` if cholesky decomposition passes; if not, it returns the + rank at which the decomposition fails \ingroup lapack_factor_func_cholesky */ AFAPI int cholesky(array &out, const array &in, const bool is_upper = true); /** - C++ Interface for in place cholesky decomposition + C++ Interface to perform in-place Cholesky decomposition. - \param[inout] in is the input matrix on entry. It contains the triangular matrix on exit. - \param[in] is_upper a boolean determining if \p in is upper or lower triangular + The input must be positive definite. - \returns \p 0 if cholesky decomposition passes, if not it returns the rank at which the decomposition failed. + This function is not supported in GFOR. - \note The input matrix \b has to be a positive definite matrix, if it is not zero, the cholesky decomposition functions return a non-zero output. - \note This function is not supported in GFOR + \param[inout] in input matrix on entry; triangular matrix on exit + \param[in] is_upper boolean determining if `in` is upper or lower + triangular + \returns `0` if cholesky decomposition passes; if not, it returns + the rank at which the decomposition fails \ingroup lapack_factor_func_cholesky */ AFAPI int choleskyInPlace(array &in, const bool is_upper = true); /** - C++ Interface for solving a system of equations + C++ Interface to solve a system of equations. - \param[in] a is the coefficient matrix - \param[in] b is the measured values - \param[in] options determining various properties of matrix \p a - \returns \p x, the matrix of unknown variables + The `options` parameter must be one of \ref AF_MAT_NONE, + \ref AF_MAT_LOWER or \ref AF_MAT_UPPER. - \note \p options needs to be one of \ref AF_MAT_NONE, \ref AF_MAT_LOWER or \ref AF_MAT_UPPER - \note This function is not supported in GFOR + This function is not supported in GFOR. + + \param[in] a coefficient matrix + \param[in] b measured values + \param[in] options determines various properties of matrix `a` + \returns `x`, the matrix of unknown variables \ingroup lapack_solve_func_gen */ AFAPI array solve(const array &a, const array &b, const matProp options = AF_MAT_NONE); - /** - C++ Interface for solving a system of equations + C++ Interface to solve a system of equations. - \param[in] a is the output matrix from packed LU decomposition of the coefficient matrix - \param[in] piv is the pivot array from packed LU decomposition of the coefficient matrix - \param[in] b is the matrix of measured values - \param[in] options determining various properties of matrix \p a - \returns \p x, the matrix of unknown variables + The `options` parameter currently must be \ref AF_MAT_NONE. - \ingroup lapack_solve_lu_func_gen + This function is not supported in GFOR. + + \param[in] a packed LU decomposition of the coefficient matrix + \param[in] piv pivot array from the packed LU decomposition of the + coefficient matrix + \param[in] b measured values + \param[in] options determines various properties of matrix `a` + \returns `x`, the matrix of unknown variables - \note \p options currently needs to be \ref AF_MAT_NONE - \note This function is not supported in GFOR + \ingroup lapack_solve_lu_func_gen */ AFAPI array solveLU(const array &a, const array &piv, const array &b, const matProp options = AF_MAT_NONE); /** - C++ Interface for inverting a matrix + C++ Interface to invert a matrix. + + The `options` parameter currently must be \ref AF_MAT_NONE. - \param[in] in is input matrix - \param[in] options determining various properties of matrix \p in - \returns \p x, the inverse of the input matrix + This function is not supported in GFOR. - \note \p options currently needs to be \ref AF_MAT_NONE - \note This function is not supported in GFOR + \param[in] in input matrix + \param[in] options determines various properties of matrix `in` + \returns inverse matrix \ingroup lapack_ops_func_inv */ @@ -208,19 +225,22 @@ namespace af #if AF_API_VERSION >= 37 /** - C++ Interface for pseudo-inverting (Moore-Penrose) a matrix. + C++ Interface to pseudo-invert (Moore-Penrose) a matrix. + Currently uses the SVD-based approach. - \param[in] in is the input matrix - \param[in] tol defines the lower threshold for singular values from SVD - \param[in] options must be AF_MAT_NONE (more options might be supported - in the future) - \returns the pseudo-inverse of the input matrix + Parameter `tol` is not the actual lower threshold, but it is passed in + as a parameter to the calculation of the actual threshold relative to + the shape and contents of `in`. + + This function is not supported in GFOR. - \note \p tol is not the actual lower threshold, but it is passed in as - a parameter to the calculation of the actual threshold relative to - the shape and contents of \p in. - \note This function is not supported in GFOR + \param[in] in input matrix + \param[in] tol defines the lower threshold for singular values from + SVD + \param[in] options must be AF_MAT_NONE (more options might be supported + in the future) + \returns pseudo-inverse matrix \ingroup lapack_ops_func_pinv */ @@ -229,37 +249,36 @@ namespace af #endif /** - C++ Interface for finding the rank of a matrix - - \param[in] in is input matrix - \param[in] tol is the tolerance value + C++ Interface to find the rank of a matrix. - \returns the rank of the matrix + \param[in] in input matrix + \param[in] tol tolerance value + \returns rank \ingroup lapack_ops_func_rank */ AFAPI unsigned rank(const array &in, const double tol=1E-5); /** - C++ Interface for finding the determinant of a matrix + C++ Interface to find the determinant of a matrix. - \param[in] in is input matrix - - \returns the determinant of the matrix + \param[in] in input matrix + \returns determinant \ingroup lapack_ops_func_det */ template T det(const array &in); /** - C++ Interface for norm of a matrix - - \param[in] in is the input matrix - \param[in] type specifies the \ref af::normType. Default: \ref AF_NORM_VECTOR_1 - \param[in] p specifies the value of P when \p type is one of \ref AF_NORM_VECTOR_P, AF_NORM_MATRIX_L_PQ is used. It is ignored for other values of \p type - \param[in] q specifies the value of Q when \p type is AF_NORM_MATRIX_L_PQ. This parameter is ignored if \p type is anything else + C++ Interface to find the norm of a matrix. - \returns the norm of \p inbased on \p type + \param[in] in input matrix + \param[in] type \ref af::normType. Default: \ref AF_NORM_VECTOR_1 + \param[in] p value of P when `type` is \ref AF_NORM_VECTOR_P or + \ref AF_NORM_MATRIX_L_PQ, else ignored + \param[in] q value of Q when `type` is \ref AF_NORM_MATRIX_L_PQ, else + ignored + \returns norm \ingroup lapack_ops_func_norm */ @@ -268,9 +287,9 @@ namespace af #if AF_API_VERSION >= 33 /** - Returns true is ArrayFire is compiled with LAPACK support + Returns true if ArrayFire is compiled with LAPACK support. - \returns true is LAPACK support is available, false otherwise + \returns true if LAPACK support is available; false otherwise \ingroup lapack_helper_func_available */ @@ -286,12 +305,15 @@ extern "C" { #if AF_API_VERSION >= 31 /** - C Interface for SVD decomposition + C Interface to perform singular value decomposition. - \param[out] u is the output array containing U - \param[out] s is the output array containing the diagonal values of sigma, (singular values of the input matrix)) - \param[out] vt is the output array containing V^H - \param[in] in is the input matrix + \param[out] u U + \param[out] s diagonal values of sigma (singular values of the input + matrix) + \param[out] vt V^H + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_factor_func_svd */ @@ -300,18 +322,18 @@ extern "C" { #if AF_API_VERSION >= 31 /** - C Interface for SVD decomposition (in-place) + C Interface to perform in-place singular value decomposition. - \param[out] u is the output array containing U - \param[out] s is the output array containing the diagonal values of - sigma, (singular values of the input matrix)) - \param[out] vt is the output array containing V^H - \param[in,out] in is the input matrix that will contain random data after - this operation + This function minimizes memory usage if `in` is dispensable. Input array + `in` is limited to arrays where `dim0` \f$\geq\f$ `dim1`. - \note Currently, \p in is limited to arrays where `dim0` \f$\geq\f$ `dim1` - \note This is best used when minimizing memory usage and \p in is - dispensable + \param[out] u U + \param[out] s diagonal values of sigma (singular values of the input + matrix) + \param[out] vt V^H + \param[inout] in input array; contains random data after the operation this operation + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_factor_func_svd */ @@ -319,139 +341,182 @@ extern "C" { #endif /** - C Interface for LU decomposition + C Interface to perform LU decomposition. - \param[out] lower will contain the lower triangular matrix of the LU decomposition - \param[out] upper will contain the upper triangular matrix of the LU decomposition - \param[out] pivot will contain the permutation indices to map the input to the decomposition - \param[in] in is the input matrix + \param[out] lower lower triangular matrix of the LU decomposition + \param[out] upper upper triangular matrix of the LU decomposition + \param[out] pivot permutation indices mapping the input to the + decomposition + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_factor_func_lu */ AFAPI af_err af_lu(af_array *lower, af_array *upper, af_array *pivot, const af_array in); /** - C Interface for in place LU decomposition + C Interface to perform in-place LU decomposition. + + This function is not supported in GFOR. - \param[out] pivot will contain the permutation indices to map the input to the decomposition - \param[inout] in contains the input on entry, the packed LU decomposition on exit - \param[in] is_lapack_piv specifies if the pivot is returned in original LAPACK compliant format + \param[out] pivot permutation indices mapping the input to the + decomposition + \param[inout] in input array on entry; packed LU + decomposition on exit + \param[in] is_lapack_piv specifies if the pivot is returned in + original LAPACK-compliant format + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_factor_func_lu */ AFAPI af_err af_lu_inplace(af_array *pivot, af_array in, const bool is_lapack_piv); /** - C Interface for QR decomposition + C Interface to perform QR decomposition. - \param[out] q is the orthogonal matrix from QR decomposition - \param[out] r is the upper triangular matrix from QR decomposition - \param[out] tau will contain additional information needed for solving a least squares problem using \p q and \p r - \param[in] in is the input matrix + This function is not supported in GFOR. + + \param[out] q orthogonal matrix from QR decomposition + \param[out] r upper triangular matrix from QR decomposition + \param[out] tau additional information needed for solving a + least-squares problem using `q` and `r` + \param[in] in input array + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_factor_func_qr */ AFAPI af_err af_qr(af_array *q, af_array *r, af_array *tau, const af_array in); /** - C Interface for QR decomposition + C Interface to perform QR decomposition. + + This function is not supported in GFOR. - \param[out] tau will contain additional information needed for unpacking the data - \param[inout] in is the input matrix on entry. It contains packed QR decomposition on exit + \param[out] tau additional information needed for unpacking the data + \param[inout] in input array on entry; packed QR decomposition on exit + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_factor_func_qr */ AFAPI af_err af_qr_inplace(af_array *tau, af_array in); /** - C++ Interface for cholesky decomposition + C Interface to perform Cholesky decomposition. - \param[out] out contains the triangular matrix. Multiply \p out with it conjugate transpose reproduces the input \p in. - \param[out] info is \p 0 if cholesky decomposition passes, if not it returns the rank at which the decomposition failed. - \param[in] in is the input matrix - \param[in] is_upper a boolean determining if \p out is upper or lower triangular + Multiplying `out` with its conjugate transpose reproduces the input + `in`. - \note The input matrix \b has to be a positive definite matrix, if it is not zero, the cholesky decomposition functions return a non zero output. + The input must be positive definite. + + \param[out] out triangular matrix; + \param[out] info `0` if cholesky decomposition passes; if not, it + returns the rank at which the decomposition fails + \param[in] in input matrix + \param[in] is_upper boolean determining if `out` is upper or lower + triangular + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_factor_func_cholesky */ AFAPI af_err af_cholesky(af_array *out, int *info, const af_array in, const bool is_upper); /** - C Interface for in place cholesky decomposition + C Interface to perform in-place Cholesky decomposition. - \param[out] info is \p 0 if cholesky decomposition passes, if not it returns the rank at which the decomposition failed. - \param[inout] in is the input matrix on entry. It contains the triangular matrix on exit. - \param[in] is_upper a boolean determining if \p in is upper or lower triangular + The input must be positive definite. - \note The input matrix \b has to be a positive definite matrix, if it is not zero, the cholesky decomposition functions return a non zero output. + \param[out] info `0` if cholesky decomposition passes; if not, it + returns the rank at which the decomposition fails + \param[inout] in input matrix on entry; triangular matrix on exit + \param[in] is_upper boolean determining if `in` is upper or lower + triangular + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_factor_func_cholesky */ AFAPI af_err af_cholesky_inplace(int *info, af_array in, const bool is_upper); /** - C Interface for solving a system of equations + C Interface to solve a system of equations. - \param[out] x is the matrix of unknown variables - \param[in] a is the coefficient matrix - \param[in] b is the measured values - \param[in] options determining various properties of matrix \p a + The `options` parameter must be one of \ref AF_MAT_NONE, + \ref AF_MAT_LOWER or \ref AF_MAT_UPPER. - \ingroup lapack_solve_func_gen + This function is not supported in GFOR. - \note \p options needs to be one of \ref AF_MAT_NONE, \ref AF_MAT_LOWER or \ref AF_MAT_UPPER + \param[out] x matrix of unknown variables + \param[in] a coefficient matrix + \param[in] b measured values + \param[in] options determines various properties of matrix `a` + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given + + \ingroup lapack_solve_func_gen */ AFAPI af_err af_solve(af_array *x, const af_array a, const af_array b, const af_mat_prop options); /** - C Interface for solving a system of equations + C Interface to solve a system of equations. - \param[out] x will contain the matrix of unknown variables - \param[in] a is the output matrix from packed LU decomposition of the coefficient matrix - \param[in] piv is the pivot array from packed LU decomposition of the coefficient matrix - \param[in] b is the matrix of measured values - \param[in] options determining various properties of matrix \p a + The `options` parameter currently must be \ref AF_MAT_NONE. - \ingroup lapack_solve_lu_func_gen + \param[out] x matrix of unknown variables + \param[in] a packed LU decomposition of the coefficient matrix + \param[in] piv pivot array from the packed LU decomposition of the + coefficient matrix + \param[in] b measured values + \param[in] options determines various properties of matrix `a` + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \note \p options currently needs to be \ref AF_MAT_NONE - \note This function is not supported in GFOR + \ingroup lapack_solve_lu_func_gen */ AFAPI af_err af_solve_lu(af_array *x, const af_array a, const af_array piv, const af_array b, const af_mat_prop options); /** - C Interface for inverting a matrix + C Interface to invert a matrix. - \param[out] out will contain the inverse of matrix \p in - \param[in] in is input matrix - \param[in] options determining various properties of matrix \p in + The `options` parameter currently must be \ref AF_MAT_NONE. - \ingroup lapack_ops_func_inv + \param[out] out inverse matrix + \param[in] in input matrix + \param[in] options determines various properties of matrix `in` + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given - \note currently options needs to be \ref AF_MAT_NONE + \ingroup lapack_ops_func_inv */ AFAPI af_err af_inverse(af_array *out, const af_array in, const af_mat_prop options); #if AF_API_VERSION >= 37 /** - C Interface for pseudo-inverting (Moore-Penrose) a matrix. + C Interface to pseudo-invert (Moore-Penrose) a matrix. + Currently uses the SVD-based approach. - \param[out] out will contain the pseudo-inverse of matrix \p in - \param[in] in is the input matrix - \param[in] tol defines the lower threshold for singular values from SVD - \param[in] options must be AF_MAT_NONE (more options might be supported - in the future) + Parameter `tol` is not the actual lower threshold, but it is passed in + as a parameter to the calculation of the actual threshold relative to + the shape and contents of `in`. - \note \p tol is not the actual lower threshold, but it is passed in as a - parameter to the calculation of the actual threshold relative to the - shape and contents of \p in. - \note At first, try setting \p tol to 1e-6 for single precision and 1e-12 - for double. - \note This function is not supported in GFOR + Suggested parameters for `tol`: 1e-6 for single precision and 1e-12 for + double precision. + + \param[out] out pseudo-inverse matrix + \param[in] in input matrix + \param[in] tol defines the lower threshold for singular values from + SVD + \param[in] options must be AF_MAT_NONE (more options might be supported + in the future) + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_ops_func_pinv */ @@ -460,36 +525,43 @@ extern "C" { #endif /** - C Interface for finding the rank of a matrix + C Interface to find the rank of a matrix. - \param[out] rank will contain the rank of \p in - \param[in] in is input matrix - \param[in] tol is the tolerance value + \param[out] rank rank + \param[in] in input matrix + \param[in] tol tolerance value + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_ops_func_rank */ AFAPI af_err af_rank(unsigned *rank, const af_array in, const double tol); /** - C Interface for finding the determinant of a matrix + C Interface to find the determinant of a matrix. - \param[out] det_real will contain the real part of the determinant of \p in - \param[out] det_imag will contain the imaginary part of the determinant of \p in - \param[in] in is input matrix + \param[out] det_real real part of the determinant + \param[out] det_imag imaginary part of the determinant + \param[in] in input matrix + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_ops_func_det */ AFAPI af_err af_det(double *det_real, double *det_imag, const af_array in); /** - C Interface for norm of a matrix - - \param[out] out will contain the norm of \p in - \param[in] in is the input matrix - \param[in] type specifies the \ref af::normType. Default: \ref AF_NORM_VECTOR_1 - \param[in] p specifies the value of P when \p type is one of \ref AF_NORM_VECTOR_P, AF_NORM_MATRIX_L_PQ is used. It is ignored for other values of \p type - \param[in] q specifies the value of Q when \p type is AF_NORM_MATRIX_L_PQ. This parameter is ignored if \p type is anything else + C Interface to find the norm of a matrix. + \param[out] out norm + \param[in] in input matrix + \param[in] type \ref af::normType. Default: \ref AF_NORM_VECTOR_1 + \param[in] p value of P when `type` is \ref AF_NORM_VECTOR_P or + \ref AF_NORM_MATRIX_L_PQ, else ignored + \param[in] q value of Q when `type` is \ref AF_NORM_MATRIX_L_PQ, else + ignored + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup lapack_ops_func_norm */ @@ -497,11 +569,12 @@ extern "C" { #if AF_API_VERSION >= 33 /** - Returns true is ArrayFire is compiled with LAPACK support - - \param[out] out is true if LAPACK support is available, false otherwise + Returns true if ArrayFire is compiled with LAPACK support. - \returns AF_SUCCESS if successful (does not depend on the value of out) + \param[out] out true if LAPACK support is available; false otherwise + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given; does not depend on the value + of `out` \ingroup lapack_helper_func_available */ diff --git a/include/af/random.h b/include/af/random.h index bf81e9218e..53939be226 100644 --- a/include/af/random.h +++ b/include/af/random.h @@ -11,7 +11,7 @@ #include /// -/// \brief Handle for random engine +/// \brief Handle for a random engine object. /// /// This handle is used to reference the internal random engine object. /// @@ -24,7 +24,7 @@ namespace af class array; class dim4; #if AF_API_VERSION >= 34 - /// \brief Random Number Generation Engine Class + /// C++ Interface - Random Number Generation Engine Class /// /// The \ref af::randomEngine class is used to set the type and seed of /// random number generation engine based on \ref af::randomEngineType. @@ -39,79 +39,79 @@ namespace af public: /** - This function creates a \ref af::randomEngine object with a - \ref af::randomEngineType and a seed. + C++ Interface to create a \ref af::randomEngine object with a \ref + af::randomEngineType and a seed. \code - // creates random engine of default type with seed = 1 - randomEngine r(AF_RANDOM_ENGINE_DEFAULT, 1); - \endcode + // create a random engine of default type with seed = 1 + randomEngine r(AF_RANDOM_ENGINE_DEFAULT, 1); + \endcode */ explicit randomEngine(randomEngineType typeIn = AF_RANDOM_ENGINE_DEFAULT, unsigned long long seedIn = 0); /** - Copy constructor for \ref af::randomEngine. + C++ Interface copy constructor for a \ref af::randomEngine. - \param[in] other The input random engine object + \param[in] other input random engine object */ randomEngine(const randomEngine &other); /** - Creates a copy of the random engine object from a \ref - af_random_engine handle. + C++ Interface to create a copy of the random engine object from a + \ref af_random_engine handle. \param[in] engine The input random engine object */ randomEngine(af_random_engine engine); /** - \brief Destructor for \ref af::randomEngine + C++ Interface destructor for a \ref af::randomEngine. */ ~randomEngine(); /** - \brief Assigns the internal state of randome engine + C++ Interface to assign the internal state of randome engine. - \param[in] other The object to be assigned to the random engine + \param[in] other object to be assigned to the random engine - \returns the reference to this + \return the reference to this */ randomEngine &operator=(const randomEngine &other); /** - \brief Sets the random type of the random engine + C++ Interface to set the random type of the random engine. - \param[in] type The type of the random number generator + \param[in] type type of the random number generator */ void setType(const randomEngineType type); /** - \brief Return the random type of the random engine + C++ Interface to get the random type of the random engine. - \returns the \ref af::randomEngineType associated with random engine + \return \ref af::randomEngineType associated with random engine */ randomEngineType getType(void); /** - \brief Sets the seed of the random engine + C++ Interface to set the seed of the random engine. - \param[in] seed The initializing seed of the random number generator + \param[in] seed initializing seed of the random number generator */ void setSeed(const unsigned long long seed); /** - \brief Returns the seed of the random engine + C++ Interface to return the seed of the random engine. - \returns the seed associated with random engine + \return seed associated with random engine */ unsigned long long getSeed(void) const; /** - \brief Returns the af_random_engine handle of this object + C++ Interface to return the af_random_engine handle of this object. - \returns the handle to the af_random_engine associated with this - random engine + \return handle to the af_random_engine associated with this random + engine */ af_random_engine get(void) const; }; @@ -119,11 +119,13 @@ namespace af #if AF_API_VERSION >= 34 /** - \param[in] dims The dimensions of the array to be generated - \param[in] ty The type of the array - \param[in] r The random engine object + C++ Interface to create an array of random numbers uniformly + distributed. - \return array of size \p dims + \param[in] dims dimensions of the array to be generated + \param[in] ty type of the array + \param[in] r random engine object + \return random number array of size `dims` \ingroup random_func_randu */ @@ -132,11 +134,13 @@ namespace af #if AF_API_VERSION >= 34 /** - \param[in] dims The dimensions of the array to be generated - \param[in] ty The type of the array - \param[in] r The random engine object + C++ Interface to create an array of random numbers normally + distributed. - \return array of size \p dims + \param[in] dims dimensions of the array to be generated + \param[in] ty type of the array + \param[in] r random engine object + \return random number array of size `dims` \ingroup random_func_randn */ @@ -144,31 +148,36 @@ namespace af #endif /** - \param[in] dims The dimensions of the array to be generated - \param[in] ty The type of the array + C++ Interface to create an array of random numbers uniformly + distributed. - \return array of size \p dims + \param[in] dims dimensions of the array to be generated + \param[in] ty type of the array \ingroup random_func_randu */ AFAPI array randu(const dim4 &dims, const dtype ty=f32); /** - \param[in] d0 The size of the first dimension - \param[in] ty The type of the array + C++ Interface to create an array of random numbers uniformly + distributed. - \return array of size \p d0 + \param[in] d0 size of the first dimension + \param[in] ty type of the array + \return random number array of size `d0` \ingroup random_func_randu */ AFAPI array randu(const dim_t d0, const dtype ty=f32); /** - \param[in] d0 The size of the first dimension - \param[in] d1 The size of the second dimension - \param[in] ty The type of the array + C++ Interface to create an array of random numbers uniformly + distributed. - \return array of size \p d0 x \p d1 + \param[in] d0 size of the first dimension + \param[in] d1 size of the second dimension + \param[in] ty type of the array + \return random number array of size `d0` x `d1` \ingroup random_func_randu */ @@ -176,12 +185,14 @@ namespace af const dim_t d1, const dtype ty=f32); /** - \param[in] d0 The size of the first dimension - \param[in] d1 The size of the second dimension - \param[in] d2 The size of the third dimension - \param[in] ty The type of the array + C++ Interface to create an array of random numbers uniformly + distributed. - \return array of size \p d0 x \p d1 x \p d2 + \param[in] d0 size of the first dimension + \param[in] d1 size of the second dimension + \param[in] d2 size of the third dimension + \param[in] ty type of the array + \return random number array of size `d0` x `d1` x `d2` \ingroup random_func_randu */ @@ -189,13 +200,15 @@ namespace af const dim_t d1, const dim_t d2, const dtype ty=f32); /** - \param[in] d0 The size of the first dimension - \param[in] d1 The size of the second dimension - \param[in] d2 The size of the third dimension - \param[in] d3 The size of the fourth dimension - \param[in] ty The type of the array + C++ Interface to create an array of random numbers uniformly + distributed. - \return array of size \p d0 x \p d1 x \p d2 x \p d3 + \param[in] d0 size of the first dimension + \param[in] d1 size of the second dimension + \param[in] d2 size of the third dimension + \param[in] d3 size of the fourth dimension + \param[in] ty type of the array + \return random number array of size `d0` x `d1` x `d2` x `d3` \ingroup random_func_randu */ @@ -204,42 +217,50 @@ namespace af const dim_t d3, const dtype ty=f32); /** - \param[in] dims The dimensions of the array to be generated - \param[in] ty The type of the array + C++ Interface to create an array of random numbers normally + distributed. - \return array of size \p dims + \param[in] dims dimensions of the array to be generated + \param[in] ty type of the array + \return random number array of size `dims` \ingroup random_func_randn */ AFAPI array randn(const dim4 &dims, const dtype ty=f32); /** - \param[in] d0 The size of the first dimension - \param[in] ty The type of the array + C++ Interface to create an array of random numbers normally + distributed. - \return array of size \p d0 + \param[in] d0 size of the first dimension + \param[in] ty type of the array + \return random number array of size `d0` \ingroup random_func_randn */ AFAPI array randn(const dim_t d0, const dtype ty=f32); /** - \param[in] d0 The size of the first dimension - \param[in] d1 The size of the second dimension - \param[in] ty The type of the array + C++ Interface to create an array of random numbers normally + distributed. - \return array of size \p d0 x \p d1 + \param[in] d0 size of the first dimension + \param[in] d1 size of the second dimension + \param[in] ty type of the array + \return random number array of size `d0` x `d1` \ingroup random_func_randn */ AFAPI array randn(const dim_t d0, const dim_t d1, const dtype ty=f32); /** - \param[in] d0 The size of the first dimension - \param[in] d1 The size of the second dimension - \param[in] d2 The size of the third dimension - \param[in] ty The type of the array + C++ Interface to create an array of random numbers normally + distributed. - \return array of size \p d0 x \p d1 x \p d2 + \param[in] d0 size of the first dimension + \param[in] d1 size of the second dimension + \param[in] d2 size of the third dimension + \param[in] ty type of the array + \return random number array of size `d0` x `d1` x `d2` \ingroup random_func_randn */ @@ -247,13 +268,15 @@ namespace af const dim_t d1, const dim_t d2, const dtype ty=f32); /** - \param[in] d0 The size of the first dimension - \param[in] d1 The size of the second dimension - \param[in] d2 The size of the third dimension - \param[in] d3 The size of the fourth dimension - \param[in] ty The type of the array + C++ Interface to create an array of random numbers normally + distributed. - \return array of size \p d0 x \p d1 x \p d2 x \p d3 + \param[in] d0 size of the first dimension + \param[in] d1 size of the second dimension + \param[in] d2 size of the third dimension + \param[in] d3 size of the fourth dimension + \param[in] ty type of the array + \return random number array of size `d0` x `d1` x `d2` x `d3` \ingroup random_func_randn */ @@ -263,7 +286,9 @@ namespace af #if AF_API_VERSION >= 34 /** - \param[in] rtype The type of the random number generator + C++ Interface to set the default random engine type. + + \param[in] rtype type of the random number generator \ingroup random_func_set_default_engine */ @@ -272,7 +297,9 @@ namespace af #if AF_API_VERSION >= 34 /** - \returns the \ref af::randomEngine object for the default random engine + C++ Interface to get the default random engine type. + + \return \ref af::randomEngine object for the default random engine \ingroup random_func_get_default_engine */ @@ -280,17 +307,19 @@ namespace af #endif /** - \brief Sets the seed of the default random number generator + C++ Interface to set the seed of the default random number generator. + + \param[in] seed 64-bit unsigned integer - \param[in] seed A 64 bit unsigned integer \ingroup random_func_set_seed */ AFAPI void setSeed(const unsigned long long seed); /** - \brief Gets the seed of the default random number generator + C++ Interface to get the seed of the default random number generator. + + \return seed 64-bit unsigned integer - \returns seed A 64 bit unsigned integer \ingroup random_func_get_seed */ AFAPI unsigned long long getSeed(); @@ -304,13 +333,13 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for creating random engine + C Interface to create a random engine. - \param[out] engine The pointer to the returned random engine object - \param[in] rtype The type of the random number generator - \param[in] seed The initializing seed of the random number generator - - \returns \ref AF_SUCCESS if the execution completes properly + \param[out] engine pointer to the returned random engine object + \param[in] rtype type of the random number generator + \param[in] seed initializing seed of the random number generator + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_random_engine */ @@ -321,12 +350,12 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for retaining random engine - - \param[out] out The pointer to the returned random engine object - \param[in] engine The random engine object + C Interface to retain a random engine. - \returns \ref AF_SUCCESS if the execution completes properly + \param[out] out pointer to the returned random engine object + \param[in] engine random engine object + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_random_engine */ @@ -336,12 +365,12 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for changing random engine type - - \param[in] engine The random engine object - \param[in] rtype The type of the random number generator + C Interface to change random engine type. - \returns \ref AF_SUCCESS if the execution completes properly + \param[in] engine random engine object + \param[in] rtype type of the random number generator + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_random_engine */ @@ -351,12 +380,12 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for getting random engine type + C Interface to get random engine type. - \param[out] rtype The type of the random number generator - \param[in] engine The random engine object - - \returns \ref AF_SUCCESS if the execution completes properly + \param[out] rtype type of the random number generator + \param[in] engine random engine object + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_random_engine */ @@ -366,18 +395,16 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for creating an array of uniform numbers using a random - engine - - \param[out] out The pointer to the returned object. - \param[in] ndims The number of dimensions read from the \p dims - parameter - \param[in] dims A C pointer with \p ndims elements. Each value - represents the size of that dimension - \param[in] type The type of the \ref af_array object - \param[in] engine The random engine object + C Interface to create an array of uniform numbers using a random engine. - \returns \ref AF_SUCCESS if the execution completes properly + \param[out] out pointer to the returned object + \param[in] ndims number of dimensions + \param[in] dims C pointer with `ndims` elements; each value + represents the size of that dimension + \param[in] type type of the \ref af_array object + \param[in] engine random engine object + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_randu */ @@ -388,17 +415,16 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for creating an array of normal numbers using a random engine + C Interface to create an array of normal numbers using a random engine. - \param[out] out The pointer to the returned object. - \param[in] ndims The number of dimensions read from the \p dims - parameter - \param[in] dims A C pointer with \p ndims elements. Each value - represents the size of that dimension - \param[in] type The type of the \ref af_array object - \param[in] engine The random engine object - - \returns \ref AF_SUCCESS if the execution completes properly + \param[out] out pointer to the returned object + \param[in] ndims number of dimensions + \param[in] dims C pointer with `ndims` elements; each value + represents the size of that dimension + \param[in] type type of the \ref af_array object + \param[in] engine random engine object + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_randn */ @@ -409,12 +435,12 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for setting the seed of a random engine - - \param[out] engine The pointer to the returned random engine object - \param[in] seed The initializing seed of the random number generator + C Interface to set the seed of a random engine. - \returns \ref AF_SUCCESS if the execution completes properly + \param[out] engine pointer to the returned random engine object + \param[in] seed initializing seed of the random number generator + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_random_engine */ @@ -424,11 +450,11 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for getting the default random engine + C Interface to get the default random engine. - \param[out] engine The pointer to returned default random engine object - - \returns \ref AF_SUCCESS if the execution completes properly + \param[out] engine pointer to the returned default random engine object + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_get_default_engine */ @@ -437,11 +463,11 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for setting the type of the default random engine - - \param[in] rtype The type of the random number generator + C Interface to set the type of the default random engine. - \returns \ref AF_SUCCESS if the execution completes properly + \param[in] rtype type of the random number generator + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_set_default_engine */ @@ -450,12 +476,12 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for getting the seed of a random engine - - \param[out] seed The pointer to the returned seed. - \param[in] engine The random engine object + C Interface to get the seed of a random engine. - \returns \ref AF_SUCCESS if the execution completes properly + \param[out] seed pointer to the returned seed + \param[in] engine random engine object + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_random_engine */ @@ -465,10 +491,11 @@ extern "C" { #if AF_API_VERSION >= 34 /** - C Interface for releasing random engine + C Interface to release a random engine. - \param[in] engine The random engine object - \returns \ref AF_SUCCESS if the execution completes properly + \param[in] engine random engine object + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_random_engine */ @@ -476,10 +503,12 @@ extern "C" { #endif /** - \param[out] out The generated array - \param[in] ndims Size of dimension array \p dims - \param[in] dims The array containing sizes of the dimension - \param[in] type The type of array to generate + \param[out] out generated array + \param[in] ndims number of dimensions + \param[in] dims array containing sizes of the dimension + \param[in] type type of array to generate + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_randu */ @@ -487,10 +516,12 @@ extern "C" { const dim_t * const dims, const af_dtype type); /** - \param[out] out The generated array - \param[in] ndims Size of dimension array \p dims - \param[in] dims The array containing sizes of the dimension - \param[in] type The type of array to generate + \param[out] out generated array + \param[in] ndims number of dimensions + \param[in] dims array containing sizes of the dimension + \param[in] type type of array to generate + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_randn */ @@ -498,14 +529,18 @@ extern "C" { const dim_t * const dims, const af_dtype type); /** - \param[in] seed A 64 bit unsigned integer + \param[in] seed a 64-bit unsigned integer + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_set_seed */ AFAPI af_err af_set_seed(const unsigned long long seed); /** - \param[out] seed A 64 bit unsigned integer + \param[out] seed a 64-bit unsigned integer + \return \ref AF_SUCCESS, if function returns successfully, else + an \ref af_err code is given \ingroup random_func_get_seed */ diff --git a/include/af/traits.hpp b/include/af/traits.hpp index 6c7d1bf5fa..4216c3f046 100644 --- a/include/af/traits.hpp +++ b/include/af/traits.hpp @@ -175,6 +175,18 @@ struct dtype_traits { static const char* getName() { return "half"; } }; #endif + +#if AF_API_VERSION >= 310 +template<> +struct dtype_traits { + enum { + af_type = s8 , + ctype = f32 + }; + typedef signed char base_type; + static const char* getName() { return "schar"; } +}; +#endif } #endif diff --git a/src/api/c/CMakeLists.txt b/src/api/c/CMakeLists.txt index 870d687382..d374b9a669 100644 --- a/src/api/c/CMakeLists.txt +++ b/src/api/c/CMakeLists.txt @@ -186,10 +186,6 @@ if(FreeImage_FOUND AND AF_WITH_IMAGEIO) endif() if(BUILD_WITH_MKL) - target_compile_definitions(c_api_interface - INTERFACE - AF_MKL_INTERFACE_SIZE=${MKL_INTERFACE_INTEGER_SIZE} - ) # Create mkl thread layer compile option based on cmake cache variable if(MKL_THREAD_LAYER STREQUAL "Sequential") target_compile_definitions(c_api_interface INTERFACE AF_MKL_THREAD_LAYER=0) diff --git a/src/api/c/anisotropic_diffusion.cpp b/src/api/c/anisotropic_diffusion.cpp index 3c77f8644c..6268accb3b 100644 --- a/src/api/c/anisotropic_diffusion.cpp +++ b/src/api/c/anisotropic_diffusion.cpp @@ -90,6 +90,7 @@ af_err af_anisotropic_diffusion(af_array* out, const af_array in, case u32: case s16: case u16: + case s8: case u8: output = diffusion(input, dt, K, iterations, F, eq); break; diff --git a/src/api/c/array.cpp b/src/api/c/array.cpp index 173c52171c..d164faabdb 100644 --- a/src/api/c/array.cpp +++ b/src/api/c/array.cpp @@ -20,6 +20,7 @@ using af::dim4; using arrayfire::copyData; using arrayfire::copySparseArray; using arrayfire::getSparseArrayBase; +using arrayfire::getUseCount; using arrayfire::releaseHandle; using arrayfire::releaseSparseHandle; using arrayfire::retainSparseHandle; @@ -29,6 +30,7 @@ using detail::cdouble; using detail::cfloat; using detail::createDeviceDataArray; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -46,6 +48,7 @@ af_err af_get_data_ptr(void *data, const af_array arr) { case b8: copyData(static_cast(data), arr); break; case s32: copyData(static_cast(data), arr); break; case u32: copyData(static_cast(data), arr); break; + case s8: copyData(static_cast(data), arr); break; case u8: copyData(static_cast(data), arr); break; case s64: copyData(static_cast(data), arr); break; case u64: copyData(static_cast(data), arr); break; @@ -95,6 +98,9 @@ af_err af_create_array(af_array *result, const void *const data, case u32: out = createHandleFromData(d, static_cast(data)); break; + case s8: + out = createHandleFromData(d, static_cast(data)); + break; case u8: out = createHandleFromData(d, static_cast(data)); break; @@ -174,6 +180,7 @@ af_err af_copy_array(af_array *out, const af_array in) { case b8: res = copyArray(in); break; case s32: res = copyArray(in); break; case u32: res = copyArray(in); break; + case s8: res = copyArray(in); break; case u8: res = copyArray(in); break; case s64: res = copyArray(in); break; case u64: res = copyArray(in); break; @@ -192,24 +199,25 @@ af_err af_copy_array(af_array *out, const af_array in) { // Strong Exception Guarantee af_err af_get_data_ref_count(int *use_count, const af_array in) { try { - const ArrayInfo &info = getInfo(in, false, false); + const ArrayInfo &info = getInfo(in, false); const af_dtype type = info.getType(); int res; switch (type) { - case f32: res = getArray(in).useCount(); break; - case c32: res = getArray(in).useCount(); break; - case f64: res = getArray(in).useCount(); break; - case c64: res = getArray(in).useCount(); break; - case b8: res = getArray(in).useCount(); break; - case s32: res = getArray(in).useCount(); break; - case u32: res = getArray(in).useCount(); break; - case u8: res = getArray(in).useCount(); break; - case s64: res = getArray(in).useCount(); break; - case u64: res = getArray(in).useCount(); break; - case s16: res = getArray(in).useCount(); break; - case u16: res = getArray(in).useCount(); break; - case f16: res = getArray(in).useCount(); break; + case f32: res = getUseCount(in); break; + case c32: res = getUseCount(in); break; + case f64: res = getUseCount(in); break; + case c64: res = getUseCount(in); break; + case b8: res = getUseCount(in); break; + case s32: res = getUseCount(in); break; + case u32: res = getUseCount(in); break; + case s8: res = getUseCount(in); break; + case u8: res = getUseCount(in); break; + case s64: res = getUseCount(in); break; + case u64: res = getUseCount(in); break; + case s16: res = getUseCount(in); break; + case u16: res = getUseCount(in); break; + case f16: res = getUseCount(in); break; default: TYPE_ERROR(1, type); } std::swap(*use_count, res); @@ -221,7 +229,7 @@ af_err af_get_data_ref_count(int *use_count, const af_array in) { af_err af_release_array(af_array arr) { try { if (arr == 0) { return AF_SUCCESS; } - const ArrayInfo &info = getInfo(arr, false, false); + const ArrayInfo &info = getInfo(arr, false); af_dtype type = info.getType(); if (info.isSparse()) { @@ -241,6 +249,7 @@ af_err af_release_array(af_array arr) { case b8: releaseHandle(arr); break; case s32: releaseHandle(arr); break; case u32: releaseHandle(arr); break; + case s8: releaseHandle(arr); break; case u8: releaseHandle(arr); break; case s64: releaseHandle(arr); break; case u64: releaseHandle(arr); break; @@ -307,6 +316,9 @@ af_err af_write_array(af_array arr, const void *data, const size_t bytes, case u32: write_array(arr, static_cast(data), bytes, src); break; + case s8: + write_array(arr, static_cast(data), bytes, src); + break; case u8: write_array(arr, static_cast(data), bytes, src); break; @@ -335,7 +347,7 @@ af_err af_write_array(af_array arr, const void *data, const size_t bytes, af_err af_get_elements(dim_t *elems, const af_array arr) { try { // Do not check for device mismatch - *elems = getInfo(arr, false, false).elements(); + *elems = getInfo(arr, false).elements(); } CATCHALL return AF_SUCCESS; @@ -344,7 +356,7 @@ af_err af_get_elements(dim_t *elems, const af_array arr) { af_err af_get_type(af_dtype *type, const af_array arr) { try { // Do not check for device mismatch - *type = getInfo(arr, false, false).getType(); + *type = getInfo(arr, false).getType(); } CATCHALL return AF_SUCCESS; @@ -354,7 +366,7 @@ af_err af_get_dims(dim_t *d0, dim_t *d1, dim_t *d2, dim_t *d3, const af_array in) { try { // Do not check for device mismatch - const ArrayInfo &info = getInfo(in, false, false); + const ArrayInfo &info = getInfo(in, false); *d0 = info.dims()[0]; *d1 = info.dims()[1]; *d2 = info.dims()[2]; @@ -367,7 +379,7 @@ af_err af_get_dims(dim_t *d0, dim_t *d1, dim_t *d2, dim_t *d3, af_err af_get_numdims(unsigned *nd, const af_array in) { try { // Do not check for device mismatch - const ArrayInfo &info = getInfo(in, false, false); + const ArrayInfo &info = getInfo(in, false); *nd = info.ndims(); } CATCHALL @@ -375,14 +387,14 @@ af_err af_get_numdims(unsigned *nd, const af_array in) { } #undef INSTANTIATE -#define INSTANTIATE(fn1, fn2) \ - af_err fn1(bool *result, const af_array in) { \ - try { \ - const ArrayInfo &info = getInfo(in, false, false); \ - *result = info.fn2(); \ - } \ - CATCHALL \ - return AF_SUCCESS; \ +#define INSTANTIATE(fn1, fn2) \ + af_err fn1(bool *result, const af_array in) { \ + try { \ + const ArrayInfo &info = getInfo(in, false); \ + *result = info.fn2(); \ + } \ + CATCHALL \ + return AF_SUCCESS; \ } INSTANTIATE(af_is_empty, isEmpty) @@ -432,6 +444,9 @@ af_err af_get_scalar(void *output_value, const af_array arr) { case u32: getScalar(reinterpret_cast(output_value), arr); break; + case s8: + getScalar(reinterpret_cast(output_value), arr); + break; case u8: getScalar(reinterpret_cast(output_value), arr); break; diff --git a/src/api/c/assign.cpp b/src/api/c/assign.cpp index e53b43a6c5..bdf505048d 100644 --- a/src/api/c/assign.cpp +++ b/src/api/c/assign.cpp @@ -42,6 +42,7 @@ using detail::cdouble; using detail::cfloat; using detail::createSubArray; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -122,6 +123,7 @@ static if_real assign(Array& out, const vector iv, case u64: assign(out, iv, getArray(in)); break; case s16: assign(out, iv, getArray(in)); break; case u16: assign(out, iv, getArray(in)); break; + case s8: assign(out, iv, getArray(in)); break; case u8: assign(out, iv, getArray(in)); break; case b8: assign(out, iv, getArray(in)); break; case f16: assign(out, iv, getArray(in)); break; @@ -201,6 +203,7 @@ af_err af_assign_seq(af_array* out, const af_array lhs, const unsigned ndims, case u64: assign(getArray(res), inSeqs, rhs); break; case s16: assign(getArray(res), inSeqs, rhs); break; case u16: assign(getArray(res), inSeqs, rhs); break; + case s8: assign(getArray(res), inSeqs, rhs); break; case u8: assign(getArray(res), inSeqs, rhs); break; case b8: assign(getArray(res), inSeqs, rhs); break; case f16: assign(getArray(res), inSeqs, rhs); break; @@ -260,8 +263,6 @@ af_err af_assign_gen(af_array* out, const af_array lhs, const dim_t ndims, return af_create_handle(out, 0, nullptr, lhsType); } - ARG_ASSERT(2, (ndims == 1) || (ndims == (dim_t)lInfo.ndims())); - if (ndims == 1 && ndims != static_cast(lInfo.ndims())) { af_array tmp_in = 0; af_array tmp_out = 0; @@ -279,7 +280,6 @@ af_err af_assign_gen(af_array* out, const af_array lhs, const dim_t ndims, ARG_ASSERT(1, (lhsType == rhsType)); ARG_ASSERT(1, (lhsDims.ndims() >= rhsDims.ndims())); - ARG_ASSERT(2, (lhsDims.ndims() >= ndims)); af_array output = 0; if (*out != lhs) { @@ -385,6 +385,7 @@ af_err af_assign_gen(af_array* out, const af_array lhs, const dim_t ndims, case s32: genAssign(output, ptr, rhs); break; case s16: genAssign(output, ptr, rhs); break; case u16: genAssign(output, ptr, rhs); break; + case s8: genAssign(output, ptr, rhs); break; case u8: genAssign(output, ptr, rhs); break; case b8: genAssign(output, ptr, rhs); break; case f16: genAssign(output, ptr, rhs); break; diff --git a/src/api/c/bilateral.cpp b/src/api/c/bilateral.cpp index 44e15c725c..aeec279ea5 100644 --- a/src/api/c/bilateral.cpp +++ b/src/api/c/bilateral.cpp @@ -19,6 +19,7 @@ using af::dim4; using detail::bilateral; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -50,6 +51,7 @@ af_err af_bilateral(af_array *out, const af_array in, const float ssigma, case b8: output = bilateral(in, ssigma, csigma); break; case s32: output = bilateral(in, ssigma, csigma); break; case u32: output = bilateral(in, ssigma, csigma); break; + case s8: output = bilateral(in, ssigma, csigma); break; case u8: output = bilateral(in, ssigma, csigma); break; case s16: output = bilateral(in, ssigma, csigma); break; case u16: output = bilateral(in, ssigma, csigma); break; diff --git a/src/api/c/binary.cpp b/src/api/c/binary.cpp index 566a4b22b5..eebe62bdbb 100644 --- a/src/api/c/binary.cpp +++ b/src/api/c/binary.cpp @@ -43,6 +43,7 @@ using detail::Array; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -139,6 +140,7 @@ static af_err af_arith(af_array *out, const af_array lhs, const af_array rhs, case c64: res = arithOp(lhs, rhs, odims); break; case s32: res = arithOp(lhs, rhs, odims); break; case u32: res = arithOp(lhs, rhs, odims); break; + case s8: res = arithOp(lhs, rhs, odims); break; case u8: res = arithOp(lhs, rhs, odims); break; case b8: res = arithOp(lhs, rhs, odims); break; case s64: res = arithOp(lhs, rhs, odims); break; @@ -159,6 +161,7 @@ static af_err af_arith(af_array *out, const af_array lhs, const af_array rhs, case c64: res = arithOpBroadcast(lhs, rhs); break; case s32: res = arithOpBroadcast(lhs, rhs); break; case u32: res = arithOpBroadcast(lhs, rhs); break; + case s8: res = arithOpBroadcast(lhs, rhs); break; case u8: res = arithOpBroadcast(lhs, rhs); break; case b8: res = arithOpBroadcast(lhs, rhs); break; case s64: res = arithOpBroadcast(lhs, rhs); break; @@ -195,6 +198,7 @@ static af_err af_arith_real(af_array *out, const af_array lhs, case f64: res = arithOp(lhs, rhs, odims); break; case s32: res = arithOp(lhs, rhs, odims); break; case u32: res = arithOp(lhs, rhs, odims); break; + case s8: res = arithOp(lhs, rhs, odims); break; case u8: res = arithOp(lhs, rhs, odims); break; case b8: res = arithOp(lhs, rhs, odims); break; case s64: res = arithOp(lhs, rhs, odims); break; @@ -277,87 +281,101 @@ static af_err af_arith_sparse_dense(af_array *out, const af_array lhs, af_err af_add(af_array *out, const af_array lhs, const af_array rhs, const bool batchMode) { - // Check if inputs are sparse - const ArrayInfo &linfo = getInfo(lhs, false, true); - const ArrayInfo &rinfo = getInfo(rhs, false, true); + try { + // Check if inputs are sparse + const ArrayInfo &linfo = getInfo(lhs, false); + const ArrayInfo &rinfo = getInfo(rhs, false); - if (linfo.isSparse() && rinfo.isSparse()) { - return af_arith_sparse(out, lhs, rhs); - } - if (linfo.isSparse() && !rinfo.isSparse()) { - return af_arith_sparse_dense(out, lhs, rhs); - } - if (!linfo.isSparse() && rinfo.isSparse()) { - // second operand(Array) of af_arith call should be dense - return af_arith_sparse_dense(out, rhs, lhs, true); + if (linfo.isSparse() && rinfo.isSparse()) { + return af_arith_sparse(out, lhs, rhs); + } + if (linfo.isSparse() && !rinfo.isSparse()) { + return af_arith_sparse_dense(out, lhs, rhs); + } + if (!linfo.isSparse() && rinfo.isSparse()) { + // second operand(Array) of af_arith call should be dense + return af_arith_sparse_dense(out, rhs, lhs, true); + } + return af_arith(out, lhs, rhs, batchMode); } - return af_arith(out, lhs, rhs, batchMode); + CATCHALL; } af_err af_mul(af_array *out, const af_array lhs, const af_array rhs, const bool batchMode) { - // Check if inputs are sparse - const ArrayInfo &linfo = getInfo(lhs, false, true); - const ArrayInfo &rinfo = getInfo(rhs, false, true); - - if (linfo.isSparse() && rinfo.isSparse()) { - // return af_arith_sparse(out, lhs, rhs); - // MKL doesn't have mul or div support yet, hence - // this is commented out although alternative cpu code exists - return AF_ERR_NOT_SUPPORTED; - } - if (linfo.isSparse() && !rinfo.isSparse()) { - return af_arith_sparse_dense(out, lhs, rhs); - } - if (!linfo.isSparse() && rinfo.isSparse()) { - return af_arith_sparse_dense(out, rhs, lhs, - true); // dense should be rhs + try { + // Check if inputs are sparse + const ArrayInfo &linfo = getInfo(lhs, false); + const ArrayInfo &rinfo = getInfo(rhs, false); + + if (linfo.isSparse() && rinfo.isSparse()) { + // return af_arith_sparse(out, lhs, rhs); + // MKL doesn't have mul or div support yet, hence + // this is commented out although alternative cpu code exists + return AF_ERR_NOT_SUPPORTED; + } + if (linfo.isSparse() && !rinfo.isSparse()) { + return af_arith_sparse_dense(out, lhs, rhs); + } + if (!linfo.isSparse() && rinfo.isSparse()) { + return af_arith_sparse_dense( + out, rhs, lhs, + true); // dense should be rhs + } + return af_arith(out, lhs, rhs, batchMode); } - return af_arith(out, lhs, rhs, batchMode); + CATCHALL; } af_err af_sub(af_array *out, const af_array lhs, const af_array rhs, const bool batchMode) { - // Check if inputs are sparse - const ArrayInfo &linfo = getInfo(lhs, false, true); - const ArrayInfo &rinfo = getInfo(rhs, false, true); + try { + // Check if inputs are sparse + const ArrayInfo &linfo = getInfo(lhs, false); + const ArrayInfo &rinfo = getInfo(rhs, false); - if (linfo.isSparse() && rinfo.isSparse()) { - return af_arith_sparse(out, lhs, rhs); - } - if (linfo.isSparse() && !rinfo.isSparse()) { - return af_arith_sparse_dense(out, lhs, rhs); - } - if (!linfo.isSparse() && rinfo.isSparse()) { - return af_arith_sparse_dense(out, rhs, lhs, - true); // dense should be rhs + if (linfo.isSparse() && rinfo.isSparse()) { + return af_arith_sparse(out, lhs, rhs); + } + if (linfo.isSparse() && !rinfo.isSparse()) { + return af_arith_sparse_dense(out, lhs, rhs); + } + if (!linfo.isSparse() && rinfo.isSparse()) { + return af_arith_sparse_dense( + out, rhs, lhs, + true); // dense should be rhs + } + return af_arith(out, lhs, rhs, batchMode); } - return af_arith(out, lhs, rhs, batchMode); + CATCHALL; } af_err af_div(af_array *out, const af_array lhs, const af_array rhs, const bool batchMode) { - // Check if inputs are sparse - const ArrayInfo &linfo = getInfo(lhs, false, true); - const ArrayInfo &rinfo = getInfo(rhs, false, true); - - if (linfo.isSparse() && rinfo.isSparse()) { - // return af_arith_sparse(out, lhs, rhs); - // MKL doesn't have mul or div support yet, hence - // this is commented out although alternative cpu code exists - return AF_ERR_NOT_SUPPORTED; - } - if (linfo.isSparse() && !rinfo.isSparse()) { - return af_arith_sparse_dense(out, lhs, rhs); - } - if (!linfo.isSparse() && rinfo.isSparse()) { - // Division by sparse is currently not allowed - for convinence of - // dealing with division by 0 - // return af_arith_sparse_dense(out, rhs, lhs, true); // dense - // should be rhs - return AF_ERR_NOT_SUPPORTED; + try { + // Check if inputs are sparse + const ArrayInfo &linfo = getInfo(lhs, false); + const ArrayInfo &rinfo = getInfo(rhs, false); + + if (linfo.isSparse() && rinfo.isSparse()) { + // return af_arith_sparse(out, lhs, rhs); + // MKL doesn't have mul or div support yet, hence + // this is commented out although alternative cpu code exists + return AF_ERR_NOT_SUPPORTED; + } + if (linfo.isSparse() && !rinfo.isSparse()) { + return af_arith_sparse_dense(out, lhs, rhs); + } + if (!linfo.isSparse() && rinfo.isSparse()) { + // Division by sparse is currently not allowed - for convinence of + // dealing with division by 0 + // return af_arith_sparse_dense(out, rhs, lhs, true); // + // dense should be rhs + return AF_ERR_NOT_SUPPORTED; + } + return af_arith(out, lhs, rhs, batchMode); } - return af_arith(out, lhs, rhs, batchMode); + CATCHALL; } af_err af_maxof(af_array *out, const af_array lhs, const af_array rhs, @@ -462,7 +480,7 @@ af_err af_atan2(af_array *out, const af_array lhs, const af_array rhs, try { const af_dtype type = implicit(lhs, rhs); - if (type != f32 && type != f64) { + if (type != f16 && type != f32 && type != f64) { AF_ERROR("Only floating point arrays are supported for atan2 ", AF_ERR_NOT_SUPPORTED); } @@ -477,6 +495,7 @@ af_err af_atan2(af_array *out, const af_array lhs, const af_array rhs, af_array res; switch (type) { + case f16: res = arithOp(lhs, rhs, odims); break; case f32: res = arithOp(lhs, rhs, odims); break; case f64: res = arithOp(lhs, rhs, odims); break; default: TYPE_ERROR(0, type); @@ -493,7 +512,7 @@ af_err af_hypot(af_array *out, const af_array lhs, const af_array rhs, try { const af_dtype type = implicit(lhs, rhs); - if (type != f32 && type != f64) { + if (type != f16 && type != f32 && type != f64) { AF_ERROR("Only floating point arrays are supported for hypot ", AF_ERR_NOT_SUPPORTED); } @@ -509,6 +528,7 @@ af_err af_hypot(af_array *out, const af_array lhs, const af_array rhs, af_array res; switch (type) { + case f16: res = arithOp(lhs, rhs, odims); break; case f32: res = arithOp(lhs, rhs, odims); break; case f64: res = arithOp(lhs, rhs, odims); break; default: TYPE_ERROR(0, type); @@ -551,6 +571,7 @@ static af_err af_logic(af_array *out, const af_array lhs, const af_array rhs, case c64: res = logicOp(lhs, rhs, odims); break; case s32: res = logicOp(lhs, rhs, odims); break; case u32: res = logicOp(lhs, rhs, odims); break; + case s8: res = logicOp(lhs, rhs, odims); break; case u8: res = logicOp(lhs, rhs, odims); break; case b8: res = logicOp(lhs, rhs, odims); break; case s64: res = logicOp(lhs, rhs, odims); break; @@ -634,6 +655,7 @@ static af_err af_bitwise(af_array *out, const af_array lhs, const af_array rhs, switch (type) { case s32: res = bitOp(lhs, rhs, odims); break; case u32: res = bitOp(lhs, rhs, odims); break; + case s8: res = bitOp(lhs, rhs, odims); break; case u8: res = bitOp(lhs, rhs, odims); break; case b8: res = bitOp(lhs, rhs, odims); break; case s64: res = bitOp(lhs, rhs, odims); break; diff --git a/src/api/c/blas.cpp b/src/api/c/blas.cpp index 0946d42083..f42bc7d57c 100644 --- a/src/api/c/blas.cpp +++ b/src/api/c/blas.cpp @@ -33,6 +33,7 @@ using detail::cdouble; using detail::cfloat; using detail::gemm; using detail::matmul; +using detail::schar; namespace { template @@ -42,12 +43,12 @@ static inline af_array sparseMatmul(const af_array lhs, const af_array rhs, matmul(getSparseArray(lhs), getArray(rhs), optLhs, optRhs)); } -template +template static inline void gemm(af_array *out, af_mat_prop optLhs, af_mat_prop optRhs, - const T *alpha, const af_array lhs, const af_array rhs, - const T *betas) { - gemm(getArray(*out), optLhs, optRhs, alpha, getArray(lhs), - getArray(rhs), betas); + const To *alpha, const af_array lhs, const af_array rhs, + const To *betas) { + gemm(getArray(*out), optLhs, optRhs, alpha, getArray(lhs), + getArray(rhs), betas); } template @@ -134,8 +135,8 @@ af_err af_gemm(af_array *out, const af_mat_prop optLhs, const af_mat_prop optRhs, const void *alpha, const af_array lhs, const af_array rhs, const void *beta) { try { - const ArrayInfo &lhsInfo = getInfo(lhs, false, true); - const ArrayInfo &rhsInfo = getInfo(rhs, true, true); + const ArrayInfo &lhsInfo = getInfo(lhs, false); + const ArrayInfo &rhsInfo = getInfo(rhs, true); af_dtype lhs_type = lhsInfo.getType(); af_dtype rhs_type = rhsInfo.getType(); @@ -178,6 +179,8 @@ af_err af_gemm(af_array *out, const af_mat_prop optLhs, if (*out) { output = *out; } else { + af_dtype out_type = (lhs_type != s8) ? lhs_type : f32; + const int aRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1; const int bColDim = (optRhs == AF_MAT_NONE) ? 1 : 0; const int M = lDims[aRowDim]; @@ -186,7 +189,7 @@ af_err af_gemm(af_array *out, const af_mat_prop optLhs, const dim_t d3 = std::max(lDims[3], rDims[3]); const af::dim4 oDims = af::dim4(M, N, d2, d3); AF_CHECK(af_create_handle(&output, lhsInfo.ndims(), oDims.get(), - lhs_type)); + out_type)); } switch (lhs_type) { @@ -215,6 +218,11 @@ af_err af_gemm(af_array *out, const af_mat_prop optLhs, static_cast(alpha), lhs, rhs, static_cast(beta)); break; + case s8: + gemm(&output, optLhs, optRhs, + static_cast(alpha), lhs, rhs, + static_cast(beta)); + break; default: TYPE_ERROR(3, lhs_type); } @@ -227,8 +235,8 @@ af_err af_gemm(af_array *out, const af_mat_prop optLhs, af_err af_matmul(af_array *out, const af_array lhs, const af_array rhs, const af_mat_prop optLhs, const af_mat_prop optRhs) { try { - const ArrayInfo &lhsInfo = getInfo(lhs, false, true); - const ArrayInfo &rhsInfo = getInfo(rhs, true, true); + const ArrayInfo &lhsInfo = getInfo(lhs, false); + const ArrayInfo &rhsInfo = getInfo(rhs, true); if (lhsInfo.isSparse()) { return af_sparse_matmul(out, lhs, rhs, optLhs, optRhs); @@ -246,11 +254,13 @@ af_err af_matmul(af_array *out, const af_array lhs, const af_array rhs, const dim_t d3 = std::max(lDims[3], rDims[3]); const af::dim4 oDims = af::dim4(M, N, d2, d3); - af_array gemm_out = 0; + af_dtype lhs_type = lhsInfo.getType(); + + af_array gemm_out = 0; + af_dtype gemm_out_type = (lhs_type != s8) ? lhs_type : f32; AF_CHECK(af_create_handle(&gemm_out, oDims.ndims(), oDims.get(), - lhsInfo.getType())); + gemm_out_type)); - af_dtype lhs_type = lhsInfo.getType(); switch (lhs_type) { case f16: { static const half alpha(1.0f); @@ -288,6 +298,13 @@ af_err af_matmul(af_array *out, const af_array lhs, const af_array rhs, &beta)); break; } + case s8: { + float alpha = 1.0; + float beta = 0.0; + AF_CHECK(af_gemm(&gemm_out, optLhs, optRhs, &alpha, lhs, rhs, + &beta)); + break; + } default: TYPE_ERROR(1, lhs_type); } diff --git a/src/api/c/canny.cpp b/src/api/c/canny.cpp index ae1fa8add9..b68b8d4ed0 100644 --- a/src/api/c/canny.cpp +++ b/src/api/c/canny.cpp @@ -53,6 +53,7 @@ using detail::logicOp; using detail::reduce; using detail::reduce_all; using detail::scan; +using detail::schar; using detail::sobelDerivatives; using detail::uchar; using detail::uint; @@ -93,7 +94,6 @@ Array otsuThreshold(const Array& in, const unsigned NUM_BINS, seqBegin[0] = af_make_seq(0, static_cast(hDims[0] - 1), 1); seqRest[0] = af_make_seq(0, static_cast(hDims[0] - 1), 1); - Array TWOS = createValueArray(oDims, 2.0f); Array UnitP = createValueArray(oDims, 1.0f); Array histf = cast(hist); Array totals = createValueArray(hDims, inDims[0] * inDims[1]); @@ -126,7 +126,7 @@ Array otsuThreshold(const Array& in, const unsigned NUM_BINS, auto muL = arithOp(_muL, qL, oDims); auto muH = arithOp(_muH, qH, oDims); auto diff = arithOp(muL, muH, oDims); - auto sqrd = arithOp(diff, TWOS, oDims); + auto sqrd = arithOp(diff, diff, oDims); auto op2 = createSubArray(qLqH, sliceIndex, false); auto sigma = arithOp(sqrd, op2, oDims); @@ -266,6 +266,10 @@ af_err af_canny(af_array* out, const af_array in, const af_canny_threshold ct, output = cannyHelper(getArray(in), t1, ct, t2, sw, isf); break; + case s8: + output = cannyHelper(getArray(in), t1, ct, t2, sw, + isf); + break; case u8: output = cannyHelper(getArray(in), t1, ct, t2, sw, isf); diff --git a/src/api/c/cast.cpp b/src/api/c/cast.cpp index 20e47a1a2d..7b421d28bb 100644 --- a/src/api/c/cast.cpp +++ b/src/api/c/cast.cpp @@ -28,13 +28,14 @@ using arrayfire::common::half; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; using detail::ushort; static af_array cast(const af_array in, const af_dtype type) { - const ArrayInfo& info = getInfo(in, false, true); + const ArrayInfo& info = getInfo(in, false); if (info.getType() == type) { return retain(in); } @@ -54,6 +55,7 @@ static af_array cast(const af_array in, const af_dtype type) { case c64: return getHandle(castArray(in)); case s32: return getHandle(castArray(in)); case u32: return getHandle(castArray(in)); + case s8: return getHandle(castArray(in)); case u8: return getHandle(castArray(in)); case b8: return getHandle(castArray(in)); case s64: return getHandle(castArray(in)); @@ -68,7 +70,7 @@ static af_array cast(const af_array in, const af_dtype type) { af_err af_cast(af_array* out, const af_array in, const af_dtype type) { try { - const ArrayInfo& info = getInfo(in, false, true); + const ArrayInfo& info = getInfo(in, false); af_dtype inType = info.getType(); if ((inType == c32 || inType == c64) && diff --git a/src/api/c/clamp.cpp b/src/api/c/clamp.cpp index fb821d3bf3..8c31469e55 100644 --- a/src/api/c/clamp.cpp +++ b/src/api/c/clamp.cpp @@ -28,6 +28,7 @@ using detail::Array; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -64,6 +65,7 @@ af_err af_clamp(af_array* out, const af_array in, const af_array lo, case c64: res = clampOp(in, lo, hi, odims); break; case s32: res = clampOp(in, lo, hi, odims); break; case u32: res = clampOp(in, lo, hi, odims); break; + case s8: res = clampOp(in, lo, hi, odims); break; case u8: res = clampOp(in, lo, hi, odims); break; case b8: res = clampOp(in, lo, hi, odims); break; case s64: res = clampOp(in, lo, hi, odims); break; diff --git a/src/api/c/confidence_connected.cpp b/src/api/c/confidence_connected.cpp index ceb8ca7b75..903c06f87b 100644 --- a/src/api/c/confidence_connected.cpp +++ b/src/api/c/confidence_connected.cpp @@ -45,8 +45,15 @@ using std::swap; template Array pointList(const Array& in, const Array& x, const Array& y) { - af_array xcoords = getHandle(x); - af_array ycoords = getHandle(y); + + // TODO: Temporary Fix, must fix handling subarrays upstream + // Array has to be a basic array, to be accepted as af_index + Array x_ = (x.getOffset() == 0 && x.isLinear()) ? x : copyArray(x); + Array y_ = (y.getOffset() == 0 && y.isLinear()) ? y : copyArray(y); + + af_array xcoords = getHandle(x_); + af_array ycoords = getHandle(y_); + std::array idxrs = {{{{xcoords}, false, false}, {{ycoords}, false, false}, createSpanIndex(), diff --git a/src/api/c/convolve.cpp b/src/api/c/convolve.cpp index abbcd2f71b..8d37c5d285 100644 --- a/src/api/c/convolve.cpp +++ b/src/api/c/convolve.cpp @@ -33,6 +33,7 @@ using detail::cdouble; using detail::cfloat; using detail::convolve; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -196,6 +197,10 @@ af_err convolve(af_array *out, const af_array signal, const af_array filter, output = convolve(signal, filter, convBT, rank, expand); break; + case s8: + output = convolve(signal, filter, convBT, rank, + expand); + break; case b8: output = convolve(signal, filter, convBT, rank, expand); @@ -311,6 +316,10 @@ af_err af_convolve2_sep(af_array *out, const af_array col_filter, output = convolve2(signal, col_filter, row_filter, expand); break; + case s8: + output = convolve2(signal, col_filter, row_filter, + expand); + break; case b8: output = convolve2(signal, col_filter, row_filter, expand); @@ -437,7 +446,7 @@ af_err af_convolve2_gradient_nn( size_t padding_ndims = padding.ndims(); size_t dilation_ndims = dilation.ndims(); ARG_ASSERT(3, stride_ndims > 0 && stride_ndims <= 2); - ARG_ASSERT(5, padding_ndims > 0 && padding_ndims <= 2); + ARG_ASSERT(5, padding_ndims >= 0 && padding_ndims <= 2); ARG_ASSERT(7, dilation_ndims > 0 && dilation_ndims <= 2); af_dtype type = oinfo.getType(); diff --git a/src/api/c/corrcoef.cpp b/src/api/c/corrcoef.cpp index fd767fb0ba..fde3788dac 100644 --- a/src/api/c/corrcoef.cpp +++ b/src/api/c/corrcoef.cpp @@ -30,6 +30,7 @@ using detail::Array; using detail::getScalar; using detail::intl; using detail::reduce_all; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -89,6 +90,7 @@ af_err af_corrcoef(double* realVal, double* imagVal, const af_array X, case u64: *realVal = corrcoef(X, Y); break; case s16: *realVal = corrcoef(X, Y); break; case u16: *realVal = corrcoef(X, Y); break; + case s8: *realVal = corrcoef(X, Y); break; case u8: *realVal = corrcoef(X, Y); break; case b8: *realVal = corrcoef(X, Y); break; default: TYPE_ERROR(1, xType); diff --git a/src/api/c/covariance.cpp b/src/api/c/covariance.cpp index f364558b11..a4241a8f0a 100644 --- a/src/api/c/covariance.cpp +++ b/src/api/c/covariance.cpp @@ -31,6 +31,7 @@ using detail::intl; using detail::mean; using detail::reduce; using detail::scalar; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -97,6 +98,7 @@ af_err af_cov_v2(af_array* out, const af_array X, const af_array Y, case u64: output = cov(X, Y, bias); break; case s16: output = cov(X, Y, bias); break; case u16: output = cov(X, Y, bias); break; + case s8: output = cov(X, Y, bias); break; case u8: output = cov(X, Y, bias); break; default: TYPE_ERROR(1, xType); } diff --git a/src/api/c/data.cpp b/src/api/c/data.cpp index 60ede3d4f6..324936e76e 100644 --- a/src/api/c/data.cpp +++ b/src/api/c/data.cpp @@ -35,6 +35,7 @@ using detail::iota; using detail::padArrayBorders; using detail::range; using detail::scalar; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -58,6 +59,7 @@ af_err af_constant(af_array *result, const double value, const unsigned ndims, case b8: out = createHandleFromValue(d, value); break; case s32: out = createHandleFromValue(d, value); break; case u32: out = createHandleFromValue(d, value); break; + case s8: out = createHandleFromValue(d, value); break; case u8: out = createHandleFromValue(d, value); break; case s64: out = createHandleFromValue(d, value); break; case u64: out = createHandleFromValue(d, value); break; @@ -159,6 +161,7 @@ af_err af_identity(af_array *out, const unsigned ndims, const dim_t *const dims, case c64: result = identity_(d); break; case s32: result = identity_(d); break; case u32: result = identity_(d); break; + case s8: result = identity_(d); break; case u8: result = identity_(d); break; case u64: result = identity_(d); break; case s64: result = identity_(d); break; @@ -202,6 +205,7 @@ af_err af_range(af_array *result, const unsigned ndims, const dim_t *const dims, case u64: out = range_(d, seq_dim); break; case s16: out = range_(d, seq_dim); break; case u16: out = range_(d, seq_dim); break; + case s8: out = range_(d, seq_dim); break; case u8: out = range_(d, seq_dim); break; case f16: out = range_(d, seq_dim); break; default: TYPE_ERROR(4, type); @@ -242,6 +246,7 @@ af_err af_iota(af_array *result, const unsigned ndims, const dim_t *const dims, case u64: out = iota_(d, t); break; case s16: out = iota_(d, t); break; case u16: out = iota_(d, t); break; + case s8: out = iota_(d, t); break; case u8: out = iota_(d, t); break; case f16: out = iota_(d, t); break; default: TYPE_ERROR(4, type); @@ -285,6 +290,7 @@ af_err af_diag_create(af_array *out, const af_array in, const int num) { case u64: result = diagCreate(in, num); break; case s16: result = diagCreate(in, num); break; case u16: result = diagCreate(in, num); break; + case s8: result = diagCreate(in, num); break; case u8: result = diagCreate(in, num); break; @@ -324,6 +330,7 @@ af_err af_diag_extract(af_array *out, const af_array in, const int num) { case u64: result = diagExtract(in, num); break; case s16: result = diagExtract(in, num); break; case u16: result = diagExtract(in, num); break; + case s8: result = diagExtract(in, num); break; case u8: result = diagExtract(in, num); break; @@ -366,6 +373,7 @@ af_err af_lower(af_array *out, const af_array in, bool is_unit_diag) { case u64: res = triangle(in, false, is_unit_diag); break; case s16: res = triangle(in, false, is_unit_diag); break; case u16: res = triangle(in, false, is_unit_diag); break; + case s8: res = triangle(in, false, is_unit_diag); break; case u8: res = triangle(in, false, is_unit_diag); break; case b8: res = triangle(in, false, is_unit_diag); break; case f16: res = triangle(in, false, is_unit_diag); break; @@ -395,6 +403,7 @@ af_err af_upper(af_array *out, const af_array in, bool is_unit_diag) { case u64: res = triangle(in, true, is_unit_diag); break; case s16: res = triangle(in, true, is_unit_diag); break; case u16: res = triangle(in, true, is_unit_diag); break; + case s8: res = triangle(in, true, is_unit_diag); break; case u8: res = triangle(in, true, is_unit_diag); break; case b8: res = triangle(in, true, is_unit_diag); break; case f16: res = triangle(in, true, is_unit_diag); break; @@ -449,6 +458,7 @@ af_err af_pad(af_array *out, const af_array in, const unsigned begin_ndims, case u64: res = pad(in, lPad, uPad, pad_type); break; case s16: res = pad(in, lPad, uPad, pad_type); break; case u16: res = pad(in, lPad, uPad, pad_type); break; + case s8: res = pad(in, lPad, uPad, pad_type); break; case u8: res = pad(in, lPad, uPad, pad_type); break; case b8: res = pad(in, lPad, uPad, pad_type); break; case f16: res = pad(in, lPad, uPad, pad_type); break; diff --git a/src/api/c/deconvolution.cpp b/src/api/c/deconvolution.cpp index d5327d1efe..19ad89e5db 100644 --- a/src/api/c/deconvolution.cpp +++ b/src/api/c/deconvolution.cpp @@ -43,6 +43,7 @@ using detail::createValueArray; using detail::logicOp; using detail::padArrayBorders; using detail::scalar; +using detail::schar; using detail::select_scalar; using detail::shift; using detail::uchar; @@ -68,9 +69,8 @@ const dim_t GREATEST_PRIME_FACTOR = 7; template Array complexNorm(const Array& input) { - auto mag = detail::abs(input); - auto TWOS = createValueArray(input.dims(), scalar(2)); - return arithOp(mag, TWOS, input.dims()); + auto mag = detail::abs(input); + return arithOp(mag, mag, input.dims()); } std::vector calcPadInfo(dim4& inLPad, dim4& psfLPad, dim4& inUPad, @@ -227,6 +227,7 @@ af_err af_iterative_deconv(af_array* out, const af_array in, const af_array ker, case u16: res = iterDeconv(in, ker, iters, rfac, algo); break; + case s8: res = iterDeconv(in, ker, iters, rfac, algo); break; case u8: res = iterDeconv(in, ker, iters, rfac, algo); break; default: TYPE_ERROR(1, inputType); } @@ -324,6 +325,7 @@ af_err af_inverse_deconv(af_array* out, const af_array in, const af_array psf, case f32: res = invDeconv(in, psf, gamma, algo); break; case s16: res = invDeconv(in, psf, gamma, algo); break; case u16: res = invDeconv(in, psf, gamma, algo); break; + case s8: res = invDeconv(in, psf, gamma, algo); break; case u8: res = invDeconv(in, psf, gamma, algo); break; default: TYPE_ERROR(1, inputType); } diff --git a/src/api/c/device.cpp b/src/api/c/device.cpp index 1b6ef9fb93..7427a1a4e5 100644 --- a/src/api/c/device.cpp +++ b/src/api/c/device.cpp @@ -47,6 +47,7 @@ using detail::init; using detail::intl; using detail::isDoubleSupported; using detail::isHalfSupported; +using detail::schar; using detail::setDevice; using detail::uchar; using detail::uint; @@ -80,7 +81,7 @@ af_err af_get_available_backends(int* result) { af_err af_get_backend_id(af_backend* result, const af_array in) { try { if (in) { - const ArrayInfo& info = getInfo(in, false, false); + const ArrayInfo& info = getInfo(in, false); *result = info.getBackendId(); } else { return AF_ERR_ARG; @@ -93,7 +94,7 @@ af_err af_get_backend_id(af_backend* result, const af_array in) { af_err af_get_device_id(int* device, const af_array in) { try { if (in) { - const ArrayInfo& info = getInfo(in, false, false); + const ArrayInfo& info = getInfo(in, false); *device = static_cast(info.getDevId()); } else { return AF_ERR_ARG; @@ -290,6 +291,7 @@ af_err af_eval(af_array arr) { case c64: eval(arr); break; case s32: eval(arr); break; case u32: eval(arr); break; + case s8: eval(arr); break; case u8: eval(arr); break; case b8: eval(arr); break; case s64: eval(arr); break; @@ -344,6 +346,7 @@ af_err af_eval_multiple(int num, af_array* arrays) { case c64: evalMultiple(num, arrays); break; case s32: evalMultiple(num, arrays); break; case u32: evalMultiple(num, arrays); break; + case s8: evalMultiple(num, arrays); break; case u8: evalMultiple(num, arrays); break; case b8: evalMultiple(num, arrays); break; case s64: evalMultiple(num, arrays); break; diff --git a/src/api/c/diff.cpp b/src/api/c/diff.cpp index c579f0b53e..f75d5c1ab1 100644 --- a/src/api/c/diff.cpp +++ b/src/api/c/diff.cpp @@ -21,6 +21,7 @@ using arrayfire::getHandle; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -64,6 +65,7 @@ af_err af_diff1(af_array* out, const af_array in, const int dim) { case u64: output = diff1(in, dim); break; case s16: output = diff1(in, dim); break; case u16: output = diff1(in, dim); break; + case s8: output = diff1(in, dim); break; case u8: output = diff1(in, dim); break; default: TYPE_ERROR(1, type); } @@ -101,6 +103,7 @@ af_err af_diff2(af_array* out, const af_array in, const int dim) { case u64: output = diff2(in, dim); break; case s16: output = diff2(in, dim); break; case u16: output = diff2(in, dim); break; + case s8: output = diff2(in, dim); break; case u8: output = diff2(in, dim); break; default: TYPE_ERROR(1, type); } diff --git a/src/api/c/dog.cpp b/src/api/c/dog.cpp index fbbe94d211..848262daab 100644 --- a/src/api/c/dog.cpp +++ b/src/api/c/dog.cpp @@ -22,6 +22,7 @@ using af::dim4; using detail::arithOp; using detail::Array; using detail::convolve; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -70,6 +71,7 @@ af_err af_dog(af_array* out, const af_array in, const int radius1, case u32: output = dog(in, radius1, radius2); break; case s16: output = dog(in, radius1, radius2); break; case u16: output = dog(in, radius1, radius2); break; + case s8: output = dog(in, radius1, radius2); break; case u8: output = dog(in, radius1, radius2); break; default: TYPE_ERROR(1, type); } diff --git a/src/api/c/exampleFunction.cpp b/src/api/c/exampleFunction.cpp index 4a7a52f6bd..a58336f90c 100644 --- a/src/api/c/exampleFunction.cpp +++ b/src/api/c/exampleFunction.cpp @@ -76,6 +76,7 @@ af_err af_example_function(af_array* out, const af_array a, case f32: output = example(a, a, param); break; case s32: output = example(a, a, param); break; case u32: output = example(a, a, param); break; + case s8: output = example(a, a, param); break; case u8: output = example(a, a, param); break; case b8: output = example(a, a, param); break; case c32: output = example(a, a, param); break; diff --git a/src/api/c/fast.cpp b/src/api/c/fast.cpp index ed8822c402..08834ce4f4 100644 --- a/src/api/c/fast.cpp +++ b/src/api/c/fast.cpp @@ -22,6 +22,7 @@ using af::dim4; using detail::Array; using detail::createEmptyArray; using detail::createValueArray; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -96,6 +97,10 @@ af_err af_fast(af_features *out, const af_array in, const float thr, *out = fast(in, thr, arc_length, non_max, feature_ratio, edge); break; + case s8: + *out = fast(in, thr, arc_length, non_max, feature_ratio, + edge); + break; case u8: *out = fast(in, thr, arc_length, non_max, feature_ratio, edge); diff --git a/src/api/c/fftconvolve.cpp b/src/api/c/fftconvolve.cpp index f92a3fc655..ead2247c51 100644 --- a/src/api/c/fftconvolve.cpp +++ b/src/api/c/fftconvolve.cpp @@ -35,6 +35,7 @@ using detail::createSubArray; using detail::fftconvolve; using detail::intl; using detail::real; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -211,6 +212,10 @@ af_err fft_convolve(af_array *out, const af_array signal, const af_array filter, output = fftconvolve(signal, filter, expand, convBT, baseDim); break; + case s8: + output = + fftconvolve(signal, filter, expand, convBT, baseDim); + break; case b8: output = fftconvolve(signal, filter, expand, convBT, baseDim); @@ -239,18 +244,24 @@ af_err af_fft_convolve1(af_array *out, const af_array signal, af_err af_fft_convolve2(af_array *out, const af_array signal, const af_array filter, const af_conv_mode mode) { - if (getInfo(signal).dims().ndims() < 2 && - getInfo(filter).dims().ndims() < 2) { - return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 1); + try { + if (getInfo(signal).dims().ndims() < 2 && + getInfo(filter).dims().ndims() < 2) { + return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 1); + } + return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 2); } - return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 2); + CATCHALL; } af_err af_fft_convolve3(af_array *out, const af_array signal, const af_array filter, const af_conv_mode mode) { - if (getInfo(signal).dims().ndims() < 3 && - getInfo(filter).dims().ndims() < 3) { - return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 2); + try { + if (getInfo(signal).dims().ndims() < 3 && + getInfo(filter).dims().ndims() < 3) { + return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 2); + } + return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 3); } - return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 3); + CATCHALL; } diff --git a/src/api/c/filters.cpp b/src/api/c/filters.cpp index dc0067f257..4c154c16fb 100644 --- a/src/api/c/filters.cpp +++ b/src/api/c/filters.cpp @@ -18,6 +18,7 @@ #include using af::dim4; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -64,6 +65,7 @@ af_err af_medfilt1(af_array *out, const af_array in, const dim_t wind_width, case u16: output = medfilt1(in, wind_width, edge_pad); break; + case s8: output = medfilt1(in, wind_width, edge_pad); break; case u8: output = medfilt1(in, wind_width, edge_pad); break; default: TYPE_ERROR(1, type); } @@ -129,6 +131,9 @@ af_err af_medfilt2(af_array *out, const af_array in, const dim_t wind_length, output = medfilt2(in, wind_length, wind_width, edge_pad); break; + case s8: + output = medfilt2(in, wind_length, wind_width, edge_pad); + break; case u8: output = medfilt2(in, wind_length, wind_width, edge_pad); break; diff --git a/src/api/c/flip.cpp b/src/api/c/flip.cpp index 080af47aac..4aea98ec73 100644 --- a/src/api/c/flip.cpp +++ b/src/api/c/flip.cpp @@ -25,6 +25,7 @@ using detail::Array; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uintl; using detail::ushort; @@ -61,6 +62,7 @@ af_err af_flip(af_array *result, const af_array in, const unsigned dim) { case u64: out = flip(in, dim); break; case s16: out = flip(in, dim); break; case u16: out = flip(in, dim); break; + case s8: out = flip(in, dim); break; case u8: out = flip(in, dim); break; default: TYPE_ERROR(1, in_type); } diff --git a/src/api/c/handle.cpp b/src/api/c/handle.cpp index 7a93847826..d67f4ae9a1 100644 --- a/src/api/c/handle.cpp +++ b/src/api/c/handle.cpp @@ -21,6 +21,7 @@ using detail::cdouble; using detail::cfloat; using detail::createDeviceDataArray; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -29,7 +30,7 @@ using detail::ushort; namespace arrayfire { af_array retain(const af_array in) { - const ArrayInfo &info = getInfo(in, false, false); + const ArrayInfo &info = getInfo(in, false); af_dtype ty = info.getType(); if (info.isSparse()) { @@ -46,6 +47,7 @@ af_array retain(const af_array in) { case f64: return retainHandle(in); case s32: return retainHandle(in); case u32: return retainHandle(in); + case s8: return retainHandle(in); case u8: return retainHandle(in); case c32: return retainHandle(in); case c64: return retainHandle(in); @@ -70,6 +72,7 @@ af_array createHandle(const dim4 &d, af_dtype dtype) { case b8: return createHandle(d); case s32: return createHandle(d); case u32: return createHandle(d); + case s8: return createHandle(d); case u8: return createHandle(d); case s64: return createHandle(d); case u64: return createHandle(d); @@ -91,6 +94,7 @@ af_array createHandleFromValue(const dim4 &d, double val, af_dtype dtype) { case b8: return createHandleFromValue(d, val); case s32: return createHandleFromValue(d, val); case u32: return createHandleFromValue(d, val); + case s8: return createHandleFromValue(d, val); case u8: return createHandleFromValue(d, val); case s64: return createHandleFromValue(d, val); case u64: return createHandleFromValue(d, val); @@ -113,6 +117,7 @@ af_array createHandleFromDeviceData(const af::dim4 &d, af_dtype dtype, case b8: return getHandle(createDeviceDataArray(d, data, false)); case s32: return getHandle(createDeviceDataArray(d, data, false)); case u32: return getHandle(createDeviceDataArray(d, data, false)); + case s8: return getHandle(createDeviceDataArray(d, data, false)); case u8: return getHandle(createDeviceDataArray(d, data, false)); case s64: return getHandle(createDeviceDataArray(d, data, false)); case u64: return getHandle(createDeviceDataArray(d, data, false)); @@ -139,9 +144,9 @@ dim4 verifyDims(const unsigned ndims, const dim_t *const dims) { template void releaseHandle(const af_array arr) { - auto &Arr = getArray(arr); + auto &info = getInfo(arr); int old_device = detail::getActiveDeviceId(); - int array_id = Arr.getDevId(); + int array_id = info.getDevId(); if (array_id != old_device) { detail::setDevice(array_id); detail::destroyArray(static_cast *>(arr)); @@ -182,5 +187,6 @@ INSTANTIATE(char); INSTANTIATE(short); INSTANTIATE(ushort); INSTANTIATE(half); +INSTANTIATE(schar); } // namespace arrayfire diff --git a/src/api/c/handle.hpp b/src/api/c/handle.hpp index 97243ac353..b2e3df97cc 100644 --- a/src/api/c/handle.hpp +++ b/src/api/c/handle.hpp @@ -40,8 +40,7 @@ af_array createHandleFromDeviceData(const af::dim4 &d, af_dtype dtype, void *data); namespace common { -const ArrayInfo &getInfo(const af_array arr, bool sparse_check = true, - bool device_check = true); +const ArrayInfo &getInfo(const af_array arr, bool sparse_check = true); template detail::Array castArray(const af_array &in); @@ -53,6 +52,7 @@ const detail::Array &getArray(const af_array &arr) { const detail::Array *A = static_cast *>(arr); if ((af_dtype)af::dtype_traits::af_type != A->getType()) AF_ERROR("Invalid type for input array.", AF_ERR_INTERNAL); + checkAndMigrate(*const_cast *>(A)); return *A; } @@ -61,9 +61,21 @@ detail::Array &getArray(af_array &arr) { detail::Array *A = static_cast *>(arr); if ((af_dtype)af::dtype_traits::af_type != A->getType()) AF_ERROR("Invalid type for input array.", AF_ERR_INTERNAL); + checkAndMigrate(*A); return *A; } +/// Returns the use count +/// +/// \note This function is called separately because we cannot call getArray in +/// case the data was built on a different context. so we are avoiding the check +/// and migrate function +template +int getUseCount(const af_array &arr) { + detail::Array *A = static_cast *>(arr); + return A->useCount(); +} + template af_array getHandle(const detail::Array &A) { detail::Array *ret = new detail::Array(A); diff --git a/src/api/c/hist.cpp b/src/api/c/hist.cpp index f37ba5cea1..0d8f9bfe6b 100644 --- a/src/api/c/hist.cpp +++ b/src/api/c/hist.cpp @@ -29,6 +29,7 @@ using detail::Array; using detail::copy_histogram; using detail::forgeManager; using detail::getScalar; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -68,19 +69,21 @@ fg_chart setup_histogram(fg_window const window, const af_array in, T freqMax = getScalar(detail::reduce_all(histogramInput)); + // For histogram, xMin and xMax should always be the first + // and last bin respectively and should not be rounded if (xMin == 0 && xMax == 0 && yMin == 0 && yMax == 0) { // No previous limits. Set without checking - xMin = static_cast(step_round(minval, false)); - xMax = static_cast(step_round(maxval, true)); + xMin = static_cast(minval); + xMax = static_cast(maxval); yMax = static_cast(step_round(freqMax, true)); // For histogram, always set yMin to 0. yMin = 0; } else { if (xMin > minval) { - xMin = static_cast(step_round(minval, false)); + xMin = static_cast(minval); } if (xMax < maxval) { - xMax = static_cast(step_round(maxval, true)); + xMax = static_cast(maxval); } if (yMax < freqMax) { yMax = static_cast(step_round(freqMax, true)); @@ -131,6 +134,10 @@ af_err af_draw_hist(const af_window window, const af_array X, chart = setup_histogram(window, X, minval, maxval, props); break; + case s8: + chart = + setup_histogram(window, X, minval, maxval, props); + break; case u8: chart = setup_histogram(window, X, minval, maxval, props); diff --git a/src/api/c/histeq.cpp b/src/api/c/histeq.cpp index da2a7579d8..faed6a238c 100644 --- a/src/api/c/histeq.cpp +++ b/src/api/c/histeq.cpp @@ -33,6 +33,7 @@ using detail::intl; using detail::lookup; using detail::reduce_all; using detail::scan; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -95,6 +96,7 @@ af_err af_hist_equal(af_array* out, const af_array in, const af_array hist) { case u16: output = hist_equal(in, hist); break; case s64: output = hist_equal(in, hist); break; case u64: output = hist_equal(in, hist); break; + case s8: output = hist_equal(in, hist); break; case u8: output = hist_equal(in, hist); break; default: TYPE_ERROR(1, dataType); } diff --git a/src/api/c/histogram.cpp b/src/api/c/histogram.cpp index aa2744bb6c..69c6d71de5 100644 --- a/src/api/c/histogram.cpp +++ b/src/api/c/histogram.cpp @@ -15,6 +15,7 @@ #include using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -74,6 +75,10 @@ af_err af_histogram(af_array *out, const af_array in, const unsigned nbins, output = histogram(in, nbins, minval, maxval, info.isLinear()); break; + case s8: + output = histogram(in, nbins, minval, maxval, + info.isLinear()); + break; case u8: output = histogram(in, nbins, minval, maxval, info.isLinear()); diff --git a/src/api/c/image.cpp b/src/api/c/image.cpp index 425530806c..4650c0ec3d 100644 --- a/src/api/c/image.cpp +++ b/src/api/c/image.cpp @@ -39,6 +39,7 @@ using detail::Array; using detail::copy_image; using detail::createValueArray; using detail::forgeManager; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -102,6 +103,7 @@ af_err af_draw_image(const af_window window, const af_array in, case u32: image = convert_and_copy_image(in); break; case s16: image = convert_and_copy_image(in); break; case u16: image = convert_and_copy_image(in); break; + case s8: image = convert_and_copy_image(in); break; case u8: image = convert_and_copy_image(in); break; default: TYPE_ERROR(1, type); } diff --git a/src/api/c/implicit.cpp b/src/api/c/implicit.cpp index f30afda7eb..d045769cbd 100644 --- a/src/api/c/implicit.cpp +++ b/src/api/c/implicit.cpp @@ -14,7 +14,7 @@ Implicit type mimics C/C++ behavior. Order of precedence: - complex > real -- double > float > uintl > intl > uint > int > uchar > char +- double > float > uintl > intl > uint > int > uchar > schar > char */ af_dtype implicit(const af_dtype lty, const af_dtype rty) { @@ -38,6 +38,7 @@ af_dtype implicit(const af_dtype lty, const af_dtype rty) { if ((lty == u16) || (rty == u16)) { return u16; } if ((lty == s16) || (rty == s16)) { return s16; } if ((lty == u8) || (rty == u8)) { return u8; } + if ((lty == s8) || (rty == s8)) { return s8; } if ((lty == b8) && (rty == b8)) { return b8; } return f32; diff --git a/src/api/c/index.cpp b/src/api/c/index.cpp index 1c7484f2bf..792a5a5af7 100644 --- a/src/api/c/index.cpp +++ b/src/api/c/index.cpp @@ -40,6 +40,7 @@ using detail::cdouble; using detail::cfloat; using detail::index; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -115,6 +116,7 @@ af_err af_index(af_array* result, const af_array in, const unsigned ndims, case u16: out = indexBySeqs(in, indices_); break; case s64: out = indexBySeqs(in, indices_); break; case u64: out = indexBySeqs(in, indices_); break; + case s8: out = indexBySeqs(in, indices_); break; case u8: out = indexBySeqs(in, indices_); break; case f16: out = indexBySeqs(in, indices_); break; default: TYPE_ERROR(1, type); @@ -148,6 +150,7 @@ static af_array lookup(const af_array& in, const af_array& idx, case u64: return lookup(in, idx, dim); case s16: return lookup(in, idx, dim); case u16: return lookup(in, idx, dim); + case s8: return lookup(in, idx, dim); case u8: return lookup(in, idx, dim); case b8: return lookup(in, idx, dim); case f16: return lookup(in, idx, dim); @@ -175,21 +178,34 @@ af_err af_lookup(af_array* out, const af_array in, const af_array indices, ARG_ASSERT(2, (idxType != b8)); af_array output = 0; + af_array idx = 0; + + if (!idxInfo.isColumn()) { + // Force a deep copy to flatten the array and handle subarrays of not column vector arrays correctly + AF_CHECK(af_copy_array(&idx, indices)); + } else { + idx = indices; + } switch (idxType) { - case f32: output = lookup(in, indices, dim); break; - case f64: output = lookup(in, indices, dim); break; + case f32: output = lookup(in, idx, dim); break; + case f64: output = lookup(in, idx, dim); break; case s32: output = lookup(in, indices, dim); break; - case u32: output = lookup(in, indices, dim); break; - case s16: output = lookup(in, indices, dim); break; - case u16: output = lookup(in, indices, dim); break; - case s64: output = lookup(in, indices, dim); break; - case u64: output = lookup(in, indices, dim); break; - case u8: output = lookup(in, indices, dim); break; - case f16: output = lookup(in, indices, dim); break; + case u32: output = lookup(in, idx, dim); break; + case s16: output = lookup(in, idx, dim); break; + case u16: output = lookup(in, idx, dim); break; + case s64: output = lookup(in, idx, dim); break; + case u64: output = lookup(in, idx, dim); break; + case s8: output = lookup(in, idx, dim); break; + case u8: output = lookup(in, idx, dim); break; + case f16: output = lookup(in, idx, dim); break; default: TYPE_ERROR(1, idxType); } std::swap(*out, output); + + if (idx != indices) { + AF_CHECK(af_release_array(idx)); // Release indices array if a copy has been made + } } CATCHALL; return AF_SUCCESS; @@ -289,6 +305,7 @@ af_err af_index_gen(af_array* out, const af_array in, const dim_t ndims, case s32: output = genIndex(in, ptr); break; case u16: output = genIndex(in, ptr); break; case s16: output = genIndex(in, ptr); break; + case s8: output = genIndex(in, ptr); break; case u8: output = genIndex(in, ptr); break; case b8: output = genIndex(in, ptr); break; case f16: output = genIndex(in, ptr); break; diff --git a/src/api/c/internal.cpp b/src/api/c/internal.cpp index 38c0c96dfe..c0314981cb 100644 --- a/src/api/c/internal.cpp +++ b/src/api/c/internal.cpp @@ -25,6 +25,7 @@ using detail::cdouble; using detail::cfloat; using detail::createStridedArray; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -120,6 +121,11 @@ af_err af_create_strided_array(af_array *arr, const void *data, dims, strides, offset, static_cast(in_data), isdev)); break; + case s8: + res = getHandle(createStridedArray( + dims, strides, offset, static_cast(in_data), + isdev)); + break; case f16: res = getHandle(createStridedArray( dims, strides, offset, static_cast(in_data), @@ -175,6 +181,7 @@ af_err af_get_raw_ptr(void **ptr, const af_array arr) { case s16: res = getRawPtr(getArray(arr)); break; case b8: res = getRawPtr(getArray(arr)); break; case u8: res = getRawPtr(getArray(arr)); break; + case s8: res = getRawPtr(getArray(arr)); break; case f16: res = getRawPtr(getArray(arr)); break; default: TYPE_ERROR(6, ty); } @@ -212,6 +219,7 @@ af_err af_is_owner(bool *result, const af_array arr) { case s16: res = getArray(arr).isOwner(); break; case b8: res = getArray(arr).isOwner(); break; case u8: res = getArray(arr).isOwner(); break; + case s8: res = getArray(arr).isOwner(); break; case f16: res = getArray(arr).isOwner(); break; default: TYPE_ERROR(6, ty); } @@ -241,6 +249,7 @@ af_err af_get_allocated_bytes(size_t *bytes, const af_array arr) { case s16: res = getArray(arr).getAllocatedBytes(); break; case b8: res = getArray(arr).getAllocatedBytes(); break; case u8: res = getArray(arr).getAllocatedBytes(); break; + case s8: res = getArray(arr).getAllocatedBytes(); break; case f16: res = getArray(arr).getAllocatedBytes(); break; default: TYPE_ERROR(6, ty); } diff --git a/src/api/c/join.cpp b/src/api/c/join.cpp index 4c47fbe495..d3e9cda6b5 100644 --- a/src/api/c/join.cpp +++ b/src/api/c/join.cpp @@ -26,6 +26,7 @@ using detail::cdouble; using detail::cfloat; using detail::createEmptyArray; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -98,6 +99,7 @@ af_err af_join(af_array *out, const int dim, const af_array first, case u64: output = join(dim, first, second); break; case s16: output = join(dim, first, second); break; case u16: output = join(dim, first, second); break; + case s8: output = join(dim, first, second); break; case u8: output = join(dim, first, second); break; case f16: output = join(dim, first, second); break; default: TYPE_ERROR(1, finfo.getType()); @@ -169,6 +171,7 @@ af_err af_join_many(af_array *out, const int dim, const unsigned n_arrays, case u64: output = join_many(dim, n_arrays, inputs); break; case s16: output = join_many(dim, n_arrays, inputs); break; case u16: output = join_many(dim, n_arrays, inputs); break; + case s8: output = join_many(dim, n_arrays, inputs); break; case u8: output = join_many(dim, n_arrays, inputs); break; case f16: output = join_many(dim, n_arrays, inputs); break; default: TYPE_ERROR(1, assertType); diff --git a/src/api/c/match_template.cpp b/src/api/c/match_template.cpp index 6882711a7f..91d81c383c 100644 --- a/src/api/c/match_template.cpp +++ b/src/api/c/match_template.cpp @@ -19,6 +19,7 @@ using af::dim4; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -82,6 +83,10 @@ af_err af_match_template(af_array* out, const af_array search_img, case b8: output = match_template(search_img, template_img, m_type); break; + case s8: + output = + match_template(search_img, template_img, m_type); + break; case u8: output = match_template(search_img, template_img, m_type); diff --git a/src/api/c/mean.cpp b/src/api/c/mean.cpp index af9021983e..65fe057155 100644 --- a/src/api/c/mean.cpp +++ b/src/api/c/mean.cpp @@ -31,6 +31,7 @@ using detail::imag; using detail::intl; using detail::mean; using detail::real; +using detail::schar; using detail::uchar; using detail::uintl; using detail::ushort; @@ -77,6 +78,7 @@ af_err af_mean(af_array *out, const af_array in, const dim_t dim) { case u64: output = mean(in, dim); break; case s16: output = mean(in, dim); break; case u16: output = mean(in, dim); break; + case s8: output = mean(in, dim); break; case u8: output = mean(in, dim); break; case b8: output = mean(in, dim); break; case c32: output = mean(in, dim); break; @@ -127,6 +129,7 @@ af_err af_mean_weighted(af_array *out, const af_array in, case u32: case s16: case u16: + case s8: case u8: case b8: output = mean(in, w, dim); break; case f64: @@ -158,6 +161,7 @@ af_err af_mean_all(double *realVal, double *imagVal, const af_array in) { case u64: *realVal = mean(in); break; case s16: *realVal = mean(in); break; case u16: *realVal = mean(in); break; + case s8: *realVal = mean(in); break; case u8: *realVal = mean(in); break; case b8: *realVal = mean(in); break; case f16: @@ -200,6 +204,7 @@ af_err af_mean_all_weighted(double *realVal, double *imagVal, const af_array in, case u32: case s16: case u16: + case s8: case u8: case b8: case f16: *realVal = mean(in, weights); break; diff --git a/src/api/c/meanshift.cpp b/src/api/c/meanshift.cpp index 0c8322cafe..bf09bc4d2a 100644 --- a/src/api/c/meanshift.cpp +++ b/src/api/c/meanshift.cpp @@ -18,6 +18,7 @@ using af::dim4; using detail::intl; using detail::meanshift; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -84,6 +85,10 @@ af_err af_mean_shift(af_array *out, const af_array in, output = mean_shift(in, spatial_sigma, chromatic_sigma, num_iterations, is_color); break; + case s8: + output = mean_shift(in, spatial_sigma, chromatic_sigma, + num_iterations, is_color); + break; case u8: output = mean_shift(in, spatial_sigma, chromatic_sigma, num_iterations, is_color); diff --git a/src/api/c/median.cpp b/src/api/c/median.cpp index 5e22c1c36a..2fd0de18d8 100644 --- a/src/api/c/median.cpp +++ b/src/api/c/median.cpp @@ -23,6 +23,7 @@ using af::dim4; using detail::Array; using detail::division; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -169,6 +170,7 @@ af_err af_median_all(double* realVal, double* imagVal, // NOLINT case u32: *realVal = median(in); break; case s16: *realVal = median(in); break; case u16: *realVal = median(in); break; + case s8: *realVal = median(in); break; case u8: *realVal = median(in); break; default: TYPE_ERROR(1, type); } @@ -193,6 +195,7 @@ af_err af_median(af_array* out, const af_array in, const dim_t dim) { case u32: output = median(in, dim); break; case s16: output = median(in, dim); break; case u16: output = median(in, dim); break; + case s8: output = median(in, dim); break; case u8: output = median(in, dim); break; default: TYPE_ERROR(1, type); } diff --git a/src/api/c/memory.cpp b/src/api/c/memory.cpp index fbff61720e..665a51ac9c 100644 --- a/src/api/c/memory.cpp +++ b/src/api/c/memory.cpp @@ -42,6 +42,7 @@ using detail::memUnlock; using detail::pinnedAlloc; using detail::pinnedFree; using detail::printMemInfo; +using detail::schar; using detail::signalMemoryCleanup; using detail::uchar; using detail::uint; @@ -95,6 +96,9 @@ af_err af_device_array(af_array *arr, void *data, const unsigned ndims, case u16: res = getHandle(createDeviceDataArray(d, data)); break; + case s8: + res = getHandle(createDeviceDataArray(d, data)); + break; case u8: res = getHandle(createDeviceDataArray(d, data)); break; @@ -130,6 +134,7 @@ af_err af_get_device_ptr(void **data, const af_array arr) { case u64: *data = getDevicePtr(getArray(arr)); break; case s16: *data = getDevicePtr(getArray(arr)); break; case u16: *data = getDevicePtr(getArray(arr)); break; + case s8: *data = getDevicePtr(getArray(arr)); break; case u8: *data = getDevicePtr(getArray(arr)); break; case b8: *data = getDevicePtr(getArray(arr)); break; case f16: *data = getDevicePtr(getArray(arr)); break; @@ -164,6 +169,7 @@ af_err af_lock_array(const af_array arr) { case u64: lockArray(arr); break; case s16: lockArray(arr); break; case u16: lockArray(arr); break; + case s8: lockArray(arr); break; case u8: lockArray(arr); break; case b8: lockArray(arr); break; case f16: lockArray(arr); break; @@ -196,6 +202,7 @@ af_err af_is_locked_array(bool *res, const af_array arr) { case u64: *res = checkUserLock(arr); break; case s16: *res = checkUserLock(arr); break; case u16: *res = checkUserLock(arr); break; + case s8: *res = checkUserLock(arr); break; case u8: *res = checkUserLock(arr); break; case b8: *res = checkUserLock(arr); break; case f16: *res = checkUserLock(arr); break; @@ -229,6 +236,7 @@ af_err af_unlock_array(const af_array arr) { case u64: unlockArray(arr); break; case s16: unlockArray(arr); break; case u16: unlockArray(arr); break; + case s8: unlockArray(arr); break; case u8: unlockArray(arr); break; case b8: unlockArray(arr); break; case f16: unlockArray(arr); break; diff --git a/src/api/c/moddims.cpp b/src/api/c/moddims.cpp index 4f6f0f310d..f419a2fb04 100644 --- a/src/api/c/moddims.cpp +++ b/src/api/c/moddims.cpp @@ -22,6 +22,7 @@ using arrayfire::common::half; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -66,6 +67,7 @@ af_err af_moddims(af_array* out, const af_array in, const unsigned ndims, case b8: output = modDims(in, newDims); break; case s32: output = modDims(in, newDims); break; case u32: output = modDims(in, newDims); break; + case s8: output = modDims(in, newDims); break; case u8: output = modDims(in, newDims); break; case s64: output = modDims(in, newDims); break; case u64: output = modDims(in, newDims); break; @@ -99,6 +101,7 @@ af_err af_flat(af_array* out, const af_array in) { case b8: output = flat(in); break; case s32: output = flat(in); break; case u32: output = flat(in); break; + case s8: output = flat(in); break; case u8: output = flat(in); break; case s64: output = flat(in); break; case u64: output = flat(in); break; diff --git a/src/api/c/morph.cpp b/src/api/c/morph.cpp index efaf6cc53a..418b84e8a9 100644 --- a/src/api/c/morph.cpp +++ b/src/api/c/morph.cpp @@ -34,6 +34,7 @@ using detail::createEmptyArray; using detail::createValueArray; using detail::logicOp; using detail::scalar; +using detail::schar; using detail::uchar; using detail::uint; using detail::unaryOp; @@ -137,6 +138,7 @@ af_err morph(af_array *out, const af_array &in, const af_array &mask, case u32: output = morph(in, mask, isDilation); break; case s16: output = morph(in, mask, isDilation); break; case u16: output = morph(in, mask, isDilation); break; + case s8: output = morph(in, mask, isDilation); break; case u8: output = morph(in, mask, isDilation); break; default: TYPE_ERROR(1, type); } @@ -170,6 +172,7 @@ af_err morph3d(af_array *out, const af_array &in, const af_array &mask, case u32: output = morph3d(in, mask, isDilation); break; case s16: output = morph3d(in, mask, isDilation); break; case u16: output = morph3d(in, mask, isDilation); break; + case s8: output = morph3d(in, mask, isDilation); break; case u8: output = morph3d(in, mask, isDilation); break; default: TYPE_ERROR(1, type); } diff --git a/src/api/c/nearest_neighbour.cpp b/src/api/c/nearest_neighbour.cpp index abc2a7b65b..10543649d9 100644 --- a/src/api/c/nearest_neighbour.cpp +++ b/src/api/c/nearest_neighbour.cpp @@ -21,6 +21,7 @@ using detail::cdouble; using detail::cfloat; using detail::createEmptyArray; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -128,6 +129,10 @@ af_err af_nearest_neighbour(af_array* idx, af_array* dist, const af_array query, dist_dim, n_dist, dist_type); break; + case s8: + nearest_neighbour(&oIdx, &oDist, query, train, + dist_dim, n_dist, dist_type); + break; case u8: nearest_neighbour(&oIdx, &oDist, query, train, dist_dim, n_dist, dist_type); diff --git a/src/api/c/norm.cpp b/src/api/c/norm.cpp index 84444eed58..7eef41afcc 100644 --- a/src/api/c/norm.cpp +++ b/src/api/c/norm.cpp @@ -1,5 +1,5 @@ /******************************************************* - * Copyright (c) 2014, ArrayFire + * Copyright (c) 2025, ArrayFire * All rights reserved. * * This file is distributed under 3-clause BSD license. @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include using af::dim4; +using arrayfire::common::cast; using detail::arithOp; using detail::Array; using detail::cdouble; @@ -35,15 +37,21 @@ using detail::reduce; using detail::reduce_all; using detail::scalar; +template +using normReductionResult = + typename std::conditional::value, float, + T>::type; + template double matrixNorm(const Array &A, double p) { + using RT = normReductionResult; if (p == 1) { - Array colSum = reduce(A, 0); - return getScalar(reduce_all(colSum)); + Array colSum = reduce>(A, 0); + return getScalar(reduce_all(colSum)); } if (p == af::Inf) { - Array rowSum = reduce(A, 1); - return getScalar(reduce_all(rowSum)); + Array rowSum = reduce(A, 1); + return getScalar(reduce_all(rowSum)); } AF_ERROR("This type of norm is not supported in ArrayFire\n", @@ -52,41 +60,45 @@ double matrixNorm(const Array &A, double p) { template double vectorNorm(const Array &A, double p) { - if (p == 1) { return getScalar(reduce_all(A)); } + using RT = normReductionResult; + if (p == 1) { return getScalar(reduce_all(A)); } if (p == af::Inf) { - return getScalar(reduce_all(A)); + return getScalar(reduce_all(cast(A))); } else if (p == 2) { Array A_sq = arithOp(A, A, A.dims()); - return std::sqrt(getScalar(reduce_all(A_sq))); + return std::sqrt(getScalar(reduce_all(A_sq))); } Array P = createValueArray(A.dims(), scalar(p)); Array A_p = arithOp(A, P, A.dims()); - return std::pow(getScalar(reduce_all(A_p)), T(1.0 / p)); + return std::pow(getScalar(reduce_all(A_p)), (1.0 / p)); } template double LPQNorm(const Array &A, double p, double q) { - Array A_p_norm = createEmptyArray(dim4()); + using RT = normReductionResult; + Array A_p_norm = createEmptyArray(dim4()); if (p == 1) { - A_p_norm = reduce(A, 0); + A_p_norm = reduce(A, 0); } else { - Array P = createValueArray(A.dims(), scalar(p)); - Array invP = createValueArray(A.dims(), scalar(1.0 / p)); + Array P = createValueArray(A.dims(), scalar(p)); + Array invP = createValueArray(A.dims(), scalar(1.0 / p)); - Array A_p = arithOp(A, P, A.dims()); - Array A_p_sum = reduce(A_p, 0); - A_p_norm = arithOp(A_p_sum, invP, invP.dims()); + Array A_p = arithOp(A, P, A.dims()); + Array A_p_sum = reduce(A_p, 0); + A_p_norm = arithOp(A_p_sum, invP, invP.dims()); } - if (q == 1) { return getScalar(reduce_all(A_p_norm)); } + if (q == 1) { + return getScalar(reduce_all(A_p_norm)); + } - Array Q = createValueArray(A_p_norm.dims(), scalar(q)); - Array A_p_norm_q = arithOp(A_p_norm, Q, Q.dims()); + Array Q = createValueArray(A_p_norm.dims(), scalar(q)); + Array A_p_norm_q = arithOp(A_p_norm, Q, Q.dims()); - return std::pow(getScalar(reduce_all(A_p_norm_q)), - T(1.0 / q)); + return std::pow(getScalar(reduce_all(A_p_norm_q)), + (1.0 / q)); } template @@ -98,21 +110,13 @@ double norm(const af_array a, const af_norm_type type, const double p, switch (type) { case AF_NORM_EUCLID: return vectorNorm(A, 2); - case AF_NORM_VECTOR_1: return vectorNorm(A, 1); - case AF_NORM_VECTOR_INF: return vectorNorm(A, af::Inf); - case AF_NORM_VECTOR_P: return vectorNorm(A, p); - case AF_NORM_MATRIX_1: return matrixNorm(A, 1); - case AF_NORM_MATRIX_INF: return matrixNorm(A, af::Inf); - case AF_NORM_MATRIX_2: return matrixNorm(A, 2); - case AF_NORM_MATRIX_L_PQ: return LPQNorm(A, p, q); - default: AF_ERROR("This type of norm is not supported in ArrayFire\n", AF_ERR_NOT_SUPPORTED); @@ -123,17 +127,13 @@ af_err af_norm(double *out, const af_array in, const af_norm_type type, const double p, const double q) { try { const ArrayInfo &i_info = getInfo(in); - if (i_info.ndims() > 2) { AF_ERROR("solve can not be used in batch mode", AF_ERR_BATCH); } af_dtype i_type = i_info.getType(); - ARG_ASSERT(1, i_info.isFloating()); // Only floating and complex types - *out = 0; - if (i_info.ndims() == 0) { return AF_SUCCESS; } switch (i_type) { @@ -141,6 +141,7 @@ af_err af_norm(double *out, const af_array in, const af_norm_type type, case f64: *out = norm(in, type, p, q); break; case c32: *out = norm(in, type, p, q); break; case c64: *out = norm(in, type, p, q); break; + case f16: *out = norm(in, type, p, q); break; default: TYPE_ERROR(1, i_type); } } diff --git a/src/api/c/plot.cpp b/src/api/c/plot.cpp index 3cf03d05cf..be5aab06b1 100644 --- a/src/api/c/plot.cpp +++ b/src/api/c/plot.cpp @@ -35,6 +35,7 @@ using detail::Array; using detail::copy_plot; using detail::forgeManager; using detail::reduce; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -166,6 +167,10 @@ af_err plotWrapper(const af_window window, const af_array in, chart = setup_plot(window, in, dims[order_dim], props, ptype, marker); break; + case s8: + chart = setup_plot(window, in, dims[order_dim], props, + ptype, marker); + break; case u8: chart = setup_plot(window, in, dims[order_dim], props, ptype, marker); @@ -240,6 +245,9 @@ af_err plotWrapper(const af_window window, const af_array X, const af_array Y, case u16: chart = setup_plot(window, in, 3, props, ptype, marker); break; + case s8: + chart = setup_plot(window, in, 3, props, ptype, marker); + break; case u8: chart = setup_plot(window, in, 3, props, ptype, marker); break; @@ -307,6 +315,9 @@ af_err plotWrapper(const af_window window, const af_array X, const af_array Y, case u16: chart = setup_plot(window, in, 2, props, ptype, marker); break; + case s8: + chart = setup_plot(window, in, 2, props, ptype, marker); + break; case u8: chart = setup_plot(window, in, 2, props, ptype, marker); break; @@ -385,40 +396,52 @@ af_err af_draw_plot3(const af_window wind, const af_array P, af_err af_draw_scatter_nd(const af_window wind, const af_array in, const af_marker_type af_marker, const af_cell* const props) { - fg_marker_type fg_marker = getFGMarker(af_marker); - return plotWrapper(wind, in, 1, props, FG_PLOT_SCATTER, fg_marker); + try { + fg_marker_type fg_marker = getFGMarker(af_marker); + return plotWrapper(wind, in, 1, props, FG_PLOT_SCATTER, fg_marker); + } + CATCHALL; } af_err af_draw_scatter_2d(const af_window wind, const af_array X, const af_array Y, const af_marker_type af_marker, const af_cell* const props) { - fg_marker_type fg_marker = getFGMarker(af_marker); - return plotWrapper(wind, X, Y, props, FG_PLOT_SCATTER, fg_marker); + try { + fg_marker_type fg_marker = getFGMarker(af_marker); + return plotWrapper(wind, X, Y, props, FG_PLOT_SCATTER, fg_marker); + } + CATCHALL; } af_err af_draw_scatter_3d(const af_window wind, const af_array X, const af_array Y, const af_array Z, const af_marker_type af_marker, const af_cell* const props) { - fg_marker_type fg_marker = getFGMarker(af_marker); - return plotWrapper(wind, X, Y, Z, props, FG_PLOT_SCATTER, fg_marker); + try { + fg_marker_type fg_marker = getFGMarker(af_marker); + return plotWrapper(wind, X, Y, Z, props, FG_PLOT_SCATTER, fg_marker); + } + CATCHALL; } // Deprecated Scatter API af_err af_draw_scatter(const af_window wind, const af_array X, const af_array Y, const af_marker_type af_marker, const af_cell* const props) { - fg_marker_type fg_marker = getFGMarker(af_marker); - return plotWrapper(wind, X, Y, props, FG_PLOT_SCATTER, fg_marker); + try { + fg_marker_type fg_marker = getFGMarker(af_marker); + return plotWrapper(wind, X, Y, props, FG_PLOT_SCATTER, fg_marker); + } + CATCHALL; } af_err af_draw_scatter3(const af_window wind, const af_array P, const af_marker_type af_marker, const af_cell* const props) { - fg_marker_type fg_marker = getFGMarker(af_marker); try { - const ArrayInfo& info = getInfo(P); - af::dim4 dims = info.dims(); + fg_marker_type fg_marker = getFGMarker(af_marker); + const ArrayInfo& info = getInfo(P); + af::dim4 dims = info.dims(); if (dims.ndims() == 2 && dims[1] == 3) { return plotWrapper(wind, P, 1, props, FG_PLOT_SCATTER, fg_marker); diff --git a/src/api/c/print.cpp b/src/api/c/print.cpp index 48fea73b48..2f1ae15c8d 100644 --- a/src/api/c/print.cpp +++ b/src/api/c/print.cpp @@ -36,6 +36,7 @@ using arrayfire::common::SparseArray; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -162,6 +163,7 @@ af_err af_print_array(af_array arr) { case b8: print(NULL, arr, 4); break; case s32: print(NULL, arr, 4); break; case u32: print(NULL, arr, 4); break; + case s8: print(NULL, arr, 4); break; case u8: print(NULL, arr, 4); break; case s64: print(NULL, arr, 4); break; case u64: print(NULL, arr, 4); break; @@ -201,6 +203,7 @@ af_err af_print_array_gen(const char *exp, const af_array arr, case b8: print(exp, arr, precision); break; case s32: print(exp, arr, precision); break; case u32: print(exp, arr, precision); break; + case s8: print(exp, arr, precision); break; case u8: print(exp, arr, precision); break; case s64: print(exp, arr, precision); break; case u64: print(exp, arr, precision); break; @@ -259,6 +262,9 @@ af_err af_array_to_string(char **output, const char *exp, const af_array arr, case u32: print(exp, arr, precision, ss, transpose); break; + case s8: + print(exp, arr, precision, ss, transpose); + break; case u8: print(exp, arr, precision, ss, transpose); break; diff --git a/src/api/c/random.cpp b/src/api/c/random.cpp index 915e733974..6508786f53 100644 --- a/src/api/c/random.cpp +++ b/src/api/c/random.cpp @@ -42,6 +42,7 @@ using detail::createEmptyArray; using detail::createHostDataArray; using detail::intl; using detail::normalDistribution; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -296,6 +297,7 @@ af_err af_random_uniform(af_array *out, const unsigned ndims, case u64: result = uniformDistribution_(d, e); break; case s16: result = uniformDistribution_(d, e); break; case u16: result = uniformDistribution_(d, e); break; + case s8: result = uniformDistribution_(d, e); break; case u8: result = uniformDistribution_(d, e); break; case b8: result = uniformDistribution_(d, e); break; case f16: result = uniformDistribution_(d, e); break; @@ -362,6 +364,7 @@ af_err af_randu(af_array *out, const unsigned ndims, const dim_t *const dims, case u64: result = uniformDistribution_(d, e); break; case s16: result = uniformDistribution_(d, e); break; case u16: result = uniformDistribution_(d, e); break; + case s8: result = uniformDistribution_(d, e); break; case u8: result = uniformDistribution_(d, e); break; case b8: result = uniformDistribution_(d, e); break; case f16: result = uniformDistribution_(d, e); break; diff --git a/src/api/c/reduce.cpp b/src/api/c/reduce.cpp index 8e1e670506..65d3f85209 100644 --- a/src/api/c/reduce.cpp +++ b/src/api/c/reduce.cpp @@ -30,6 +30,7 @@ using detail::getScalar; using detail::imag; using detail::intl; using detail::real; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -107,6 +108,7 @@ static af_err reduce_type(af_array *out, const af_array in, const int dim) { case s16: res = reduce(in, dim); break; case b8: res = reduce(in, dim); break; case u8: res = reduce(in, dim); break; + case s8: res = reduce(in, dim); break; case f16: res = reduce(in, dim); break; default: TYPE_ERROR(1, type); } @@ -171,6 +173,9 @@ static af_err reduce_by_key_type(af_array *keys_out, af_array *vals_out, case u8: reduce_key(keys_out, vals_out, keys, vals, dim); break; + case s8: + reduce_key(keys_out, vals_out, keys, vals, dim); + break; case f16: reduce_key(keys_out, vals_out, keys, vals, dim); break; @@ -210,6 +215,7 @@ static af_err reduce_common(af_array *out, const af_array in, const int dim) { case s16: res = reduce(in, dim); break; case b8: res = reduce(in, dim); break; case u8: res = reduce(in, dim); break; + case s8: res = reduce(in, dim); break; case f16: res = reduce(in, dim); break; default: TYPE_ERROR(1, type); } @@ -280,6 +286,11 @@ static af_err reduce_by_key_common(af_array *keys_out, af_array *vals_out, case u8: reduce_key(keys_out, vals_out, keys, vals, dim); + break; + case s8: + reduce_key(keys_out, vals_out, keys, vals, + dim); + break; case f16: reduce_key(keys_out, vals_out, keys, vals, dim); break; @@ -342,6 +353,9 @@ static af_err reduce_promote(af_array *out, const af_array in, const int dim, case u8: res = reduce(in, dim, change_nan, nanval); break; + case s8: + res = reduce(in, dim, change_nan, nanval); + break; case b8: { if (op == af_mul_t) { res = reduce(in, dim, change_nan, @@ -424,6 +438,10 @@ static af_err reduce_promote_by_key(af_array *keys_out, af_array *vals_out, reduce_key(keys_out, vals_out, keys, vals, dim, change_nan, nanval); break; + case s8: + reduce_key(keys_out, vals_out, keys, vals, dim, + change_nan, nanval); + break; case b8: reduce_key( keys_out, vals_out, keys, vals, dim, change_nan, nanval); @@ -574,6 +592,7 @@ static af_err reduce_all_type(double *real, double *imag, const af_array in) { case s16: *real = reduce_all(in); break; case b8: *real = reduce_all(in); break; case u8: *real = reduce_all(in); break; + case s8: *real = reduce_all(in); break; case f16: *real = reduce_all(in); break; // clang-format on default: TYPE_ERROR(1, type); @@ -605,6 +624,7 @@ static af_err reduce_all_type_array(af_array *out, const af_array in) { case s16: res = reduce_all_array(in); break; case b8: res = reduce_all_array(in); break; case u8: res = reduce_all_array(in); break; + case s8: res = reduce_all_array(in); break; case f16: res = reduce_all_array(in); break; // clang-format on default: TYPE_ERROR(1, type); @@ -643,6 +663,7 @@ static af_err reduce_all_common(double *real_val, double *imag_val, case s16: *real_val = reduce_all(in); break; case b8: *real_val = reduce_all(in); break; case u8: *real_val = reduce_all(in); break; + case s8: *real_val = reduce_all(in); break; case f16: *real_val = reduce_all(in); break; // clang-format on case c32: @@ -688,6 +709,7 @@ static af_err reduce_all_common_array(af_array *out, const af_array in) { case s16: res = reduce_all_array(in); break; case b8: res = reduce_all_array(in); break; case u8: res = reduce_all_array(in); break; + case s8: res = reduce_all_array(in); break; case f16: res = reduce_all_array(in); break; // clang-format on case c32: res = reduce_all_array(in); break; @@ -727,6 +749,7 @@ static af_err reduce_all_promote(double *real_val, double *imag_val, case u16: *real_val = reduce_all(in, change_nan, nanval); break; case s16: *real_val = reduce_all(in, change_nan, nanval); break; case u8: *real_val = reduce_all(in, change_nan, nanval); break; + case s8: *real_val = reduce_all(in, change_nan, nanval); break; // clang-format on case b8: { if (op == af_mul_t) { @@ -812,6 +835,9 @@ static af_err reduce_all_promote_array(af_array *out, const af_array in, case u8: res = reduce_all_array(in, change_nan, nanval); break; + case s8: + res = reduce_all_array(in, change_nan, nanval); + break; case b8: { if (op == af_mul_t) { res = reduce_all_array(in, change_nan, @@ -952,6 +978,7 @@ static af_err ireduce_common(af_array *val, af_array *idx, const af_array in, case s16: ireduce(&res, &loc, in, dim); break; case b8: ireduce(&res, &loc, in, dim); break; case u8: ireduce(&res, &loc, in, dim); break; + case s8: ireduce(&res, &loc, in, dim); break; case f16: ireduce(&res, &loc, in, dim); break; default: TYPE_ERROR(1, type); } @@ -1027,6 +1054,7 @@ static af_err rreduce_common(af_array *val, af_array *idx, const af_array in, break; case b8: rreduce(&res, &loc, in, dim, ragged_len); break; case u8: rreduce(&res, &loc, in, dim, ragged_len); break; + case s8: rreduce(&res, &loc, in, dim, ragged_len); break; case f16: rreduce(&res, &loc, in, dim, ragged_len); break; default: TYPE_ERROR(2, type); } @@ -1085,6 +1113,7 @@ static af_err ireduce_all_common(double *real_val, double *imag_val, break; case b8: *real_val = ireduce_all(loc, in); break; case u8: *real_val = ireduce_all(loc, in); break; + case s8: *real_val = ireduce_all(loc, in); break; case c32: cfval = ireduce_all(loc, in); diff --git a/src/api/c/reorder.cpp b/src/api/c/reorder.cpp index b283c800bf..e29fb621c0 100644 --- a/src/api/c/reorder.cpp +++ b/src/api/c/reorder.cpp @@ -25,6 +25,7 @@ using detail::Array; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -33,12 +34,14 @@ using std::swap; template static inline af_array reorder(const af_array in, const af::dim4 &rdims0) { - Array In = getArray(in); + Array In = detail::createEmptyArray(af::dim4(0)); dim4 rdims = rdims0; if (rdims[0] == 1 && rdims[1] == 0) { - In = transpose(In, false); + In = transpose(getArray(in), false); std::swap(rdims[0], rdims[1]); + } else { + In = getArray(in); } const dim4 idims = In.dims(); const dim4 istrides = In.strides(); @@ -48,8 +51,7 @@ static inline af_array reorder(const af_array in, const af::dim4 &rdims0) { af_array out; if (rdims[0] == 0 && rdims[1] == 1 && rdims[2] == 2 && rdims[3] == 3) { - const Array &Out = In; - out = getHandle(Out); + out = getHandle(In); } else if (rdims[0] == 0) { dim4 odims = dim4(1, 1, 1, 1); dim4 ostrides = dim4(1, 1, 1, 1); @@ -107,6 +109,7 @@ af_err af_reorder(af_array *out, const af_array in, const af::dim4 &rdims) { case b8: output = reorder(in, rdims); break; case s32: output = reorder(in, rdims); break; case u32: output = reorder(in, rdims); break; + case s8: output = reorder(in, rdims); break; case u8: output = reorder(in, rdims); break; case s64: output = reorder(in, rdims); break; case u64: output = reorder(in, rdims); break; diff --git a/src/api/c/replace.cpp b/src/api/c/replace.cpp index b8fdd75e02..7bf66cc439 100644 --- a/src/api/c/replace.cpp +++ b/src/api/c/replace.cpp @@ -27,6 +27,7 @@ using arrayfire::common::half; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::select_scalar; using detail::uchar; using detail::uint; @@ -74,6 +75,7 @@ af_err af_replace(af_array a, const af_array cond, const af_array b) { case u64: replace(a, cond, b); break; case s16: replace(a, cond, b); break; case u16: replace(a, cond, b); break; + case s8: replace(a, cond, b); break; case u8: replace(a, cond, b); break; case b8: replace(a, cond, b); break; default: TYPE_ERROR(2, ainfo.getType()); @@ -116,6 +118,7 @@ af_err replaceScalar(af_array a, const af_array cond, const ScalarType b) { case u64: replace_scalar(a, cond, b); break; case s16: replace_scalar(a, cond, b); break; case u16: replace_scalar(a, cond, b); break; + case s8: replace_scalar(a, cond, b); break; case u8: replace_scalar(a, cond, b); break; case b8: replace_scalar(a, cond, b); break; default: TYPE_ERROR(2, ainfo.getType()); diff --git a/src/api/c/resize.cpp b/src/api/c/resize.cpp index 8b6df743da..814d4df0c8 100644 --- a/src/api/c/resize.cpp +++ b/src/api/c/resize.cpp @@ -19,6 +19,7 @@ using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -68,6 +69,7 @@ af_err af_resize(af_array* out, const af_array in, const dim_t odim0, case u64: output = resize(in, odim0, odim1, method); break; case s16: output = resize(in, odim0, odim1, method); break; case u16: output = resize(in, odim0, odim1, method); break; + case s8: output = resize(in, odim0, odim1, method); break; case u8: output = resize(in, odim0, odim1, method); break; case b8: output = resize(in, odim0, odim1, method); break; default: TYPE_ERROR(1, type); diff --git a/src/api/c/rgb_gray.cpp b/src/api/c/rgb_gray.cpp index 3bea06e855..c7abe042bc 100644 --- a/src/api/c/rgb_gray.cpp +++ b/src/api/c/rgb_gray.cpp @@ -30,6 +30,7 @@ using detail::createEmptyArray; using detail::createValueArray; using detail::join; using detail::scalar; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -157,6 +158,9 @@ af_err convert(af_array* out, const af_array in, const float r, const float g, case u8: output = convert(in, r, g, b); break; + case s8: + output = convert(in, r, g, b); + break; default: TYPE_ERROR(1, iType); break; } std::swap(*out, output); diff --git a/src/api/c/rotate.cpp b/src/api/c/rotate.cpp index 762f77d7f4..50397a310a 100644 --- a/src/api/c/rotate.cpp +++ b/src/api/c/rotate.cpp @@ -19,6 +19,7 @@ using af::dim4; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -76,6 +77,7 @@ af_err af_rotate(af_array *out, const af_array in, const float theta, case u64: output = rotate(in, theta, odims, method); break; case s16: output = rotate(in, theta, odims, method); break; case u16: output = rotate(in, theta, odims, method); break; + case s8: output = rotate(in, theta, odims, method); break; case u8: case b8: output = rotate(in, theta, odims, method); break; default: TYPE_ERROR(1, itype); diff --git a/src/api/c/sat.cpp b/src/api/c/sat.cpp index 3ff72abacc..8715f4865c 100644 --- a/src/api/c/sat.cpp +++ b/src/api/c/sat.cpp @@ -18,6 +18,7 @@ using arrayfire::common::integralImage; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -44,6 +45,7 @@ af_err af_sat(af_array* out, const af_array in) { case s32: output = sat(in); break; case u32: output = sat(in); break; case b8: output = sat(in); break; + case s8: output = sat(in); break; case u8: output = sat(in); break; case s64: output = sat(in); break; case u64: output = sat(in); break; diff --git a/src/api/c/scan.cpp b/src/api/c/scan.cpp index d8a3a7a95d..cac89d6c01 100644 --- a/src/api/c/scan.cpp +++ b/src/api/c/scan.cpp @@ -21,6 +21,7 @@ using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -141,6 +142,7 @@ af_err af_accum(af_array* out, const af_array in, const int dim) { case u16: res = scan(in, dim); break; case s16: res = scan(in, dim); break; case u8: res = scan(in, dim); break; + case s8: res = scan(in, dim); break; // Make sure you are adding only "1" for every non zero value, even // if op == af_add_t case b8: res = scan(in, dim); break; @@ -204,6 +206,9 @@ af_err af_scan(af_array* out, const af_array in, const int dim, af_binary_op op, case u8: res = scan_op(in, dim, op, inclusive_scan); break; + case s8: + res = scan_op(in, dim, op, inclusive_scan); + break; case b8: res = scan_op(in, dim, op, inclusive_scan); break; @@ -252,6 +257,7 @@ af_err af_scan_by_key(af_array* out, const af_array key, const af_array in, break; case s16: case s32: + case s8: res = scan_op(key, in, dim, op, inclusive_scan); break; case u64: diff --git a/src/api/c/select.cpp b/src/api/c/select.cpp index dec47166e7..c161aa5e9b 100644 --- a/src/api/c/select.cpp +++ b/src/api/c/select.cpp @@ -26,6 +26,7 @@ using detail::cdouble; using detail::cfloat; using detail::createSelectNode; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -76,6 +77,7 @@ af_err af_select(af_array* out, const af_array cond, const af_array a, case u64: res = select(cond, a, b, odims); break; case s16: res = select(cond, a, b, odims); break; case u16: res = select(cond, a, b, odims); break; + case s8: res = select(cond, a, b, odims); break; case u8: res = select(cond, a, b, odims); break; case b8: res = select(cond, a, b, odims); break; case f16: res = select(cond, a, b, odims); break; @@ -163,6 +165,10 @@ af_err selectScalar(af_array* out, const af_array cond, const af_array e, res = select_scalar( cond, e, c, odims); break; + case s8: + res = select_scalar( + cond, e, c, odims); + break; case u8: res = select_scalar( cond, e, c, odims); diff --git a/src/api/c/set.cpp b/src/api/c/set.cpp index bf8b66e3c8..3353d7c5ee 100644 --- a/src/api/c/set.cpp +++ b/src/api/c/set.cpp @@ -18,6 +18,7 @@ using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -51,6 +52,7 @@ af_err af_set_unique(af_array* out, const af_array in, const bool is_sorted) { case s64: res = setUnique(in, is_sorted); break; case u64: res = setUnique(in, is_sorted); break; case b8: res = setUnique(in, is_sorted); break; + case s8: res = setUnique(in, is_sorted); break; case u8: res = setUnique(in, is_sorted); break; default: TYPE_ERROR(1, type); } @@ -98,6 +100,7 @@ af_err af_set_union(af_array* out, const af_array first, const af_array second, case s64: res = setUnion(first, second, is_unique); break; case u64: res = setUnion(first, second, is_unique); break; case b8: res = setUnion(first, second, is_unique); break; + case s8: res = setUnion(first, second, is_unique); break; case u8: res = setUnion(first, second, is_unique); break; default: TYPE_ERROR(1, first_type); } @@ -156,6 +159,7 @@ af_err af_set_intersect(af_array* out, const af_array first, res = setIntersect(first, second, is_unique); break; case b8: res = setIntersect(first, second, is_unique); break; + case s8: res = setIntersect(first, second, is_unique); break; case u8: res = setIntersect(first, second, is_unique); break; default: TYPE_ERROR(1, first_type); } diff --git a/src/api/c/shift.cpp b/src/api/c/shift.cpp index 42052fbfbc..cf195d2026 100644 --- a/src/api/c/shift.cpp +++ b/src/api/c/shift.cpp @@ -17,6 +17,7 @@ using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -49,6 +50,7 @@ af_err af_shift(af_array *out, const af_array in, const int sdims[4]) { case u64: output = shift(in, sdims); break; case s16: output = shift(in, sdims); break; case u16: output = shift(in, sdims); break; + case s8: output = shift(in, sdims); break; case u8: output = shift(in, sdims); break; default: TYPE_ERROR(1, type); } diff --git a/src/api/c/sobel.cpp b/src/api/c/sobel.cpp index 6184d5502a..d466db1617 100644 --- a/src/api/c/sobel.cpp +++ b/src/api/c/sobel.cpp @@ -21,6 +21,7 @@ using detail::Array; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -66,6 +67,9 @@ af_err af_sobel_operator(af_array *dx, af_array *dy, const af_array img, output = sobelDerivatives(img, ker_size); break; case b8: output = sobelDerivatives(img, ker_size); break; + case s8: + output = sobelDerivatives(img, ker_size); + break; case u8: output = sobelDerivatives(img, ker_size); break; diff --git a/src/api/c/solve.cpp b/src/api/c/solve.cpp index ec17aafaba..31c1489484 100644 --- a/src/api/c/solve.cpp +++ b/src/api/c/solve.cpp @@ -95,8 +95,9 @@ static inline af_array solve_lu(const af_array a, const af_array pivot, af_err af_solve_lu(af_array* out, const af_array a, const af_array piv, const af_array b, const af_mat_prop options) { try { - const ArrayInfo& a_info = getInfo(a); - const ArrayInfo& b_info = getInfo(b); + const ArrayInfo& a_info = getInfo(a); + const ArrayInfo& b_info = getInfo(b); + const ArrayInfo& piv_info = getInfo(piv); if (a_info.ndims() > 2 || b_info.ndims() > 2) { AF_ERROR("solveLU can not be used in batch mode", AF_ERR_BATCH); @@ -116,6 +117,9 @@ af_err af_solve_lu(af_array* out, const af_array a, const af_array piv, TYPE_ASSERT(a_type == b_type); + af_dtype piv_type = piv_info.getType(); + TYPE_ASSERT(piv_type == s32); // TODO: add support for 64 bit types + DIM_ASSERT(1, adims[0] == adims[1]); DIM_ASSERT(1, bdims[0] == adims[0]); DIM_ASSERT(1, bdims[2] == adims[2]); diff --git a/src/api/c/sort.cpp b/src/api/c/sort.cpp index 4ec1c0a466..b917b8b3c5 100644 --- a/src/api/c/sort.cpp +++ b/src/api/c/sort.cpp @@ -27,6 +27,7 @@ using detail::cdouble; using detail::cfloat; using detail::createEmptyArray; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -59,6 +60,7 @@ af_err af_sort(af_array *out, const af_array in, const unsigned dim, case u16: val = sort(in, dim, isAscending); break; case s64: val = sort(in, dim, isAscending); break; case u64: val = sort(in, dim, isAscending); break; + case s8: val = sort(in, dim, isAscending); break; case u8: val = sort(in, dim, isAscending); break; case b8: val = sort(in, dim, isAscending); break; default: TYPE_ERROR(1, type); @@ -118,6 +120,7 @@ af_err af_sort_index(af_array *out, af_array *indices, const af_array in, case u64: sort_index(&val, &idx, in, dim, isAscending); break; + case s8: sort_index(&val, &idx, in, dim, isAscending); break; case u8: sort_index(&val, &idx, in, dim, isAscending); break; case b8: sort_index(&val, &idx, in, dim, isAscending); break; default: TYPE_ERROR(1, type); @@ -185,6 +188,9 @@ void sort_by_key_tmplt(af_array *okey, af_array *oval, const af_array ikey, case u64: sort_by_key(okey, oval, ikey, ival, dim, isAscending); break; + case s8: + sort_by_key(okey, oval, ikey, ival, dim, isAscending); + break; case u8: sort_by_key(okey, oval, ikey, ival, dim, isAscending); break; @@ -249,6 +255,10 @@ af_err af_sort_by_key(af_array *out_keys, af_array *out_values, sort_by_key_tmplt(&oKey, &oVal, keys, values, dim, isAscending); break; + case s8: + sort_by_key_tmplt(&oKey, &oVal, keys, values, dim, + isAscending); + break; case u8: sort_by_key_tmplt(&oKey, &oVal, keys, values, dim, isAscending); diff --git a/src/api/c/sparse.cpp b/src/api/c/sparse.cpp index 917864dcaf..db57b0077b 100644 --- a/src/api/c/sparse.cpp +++ b/src/api/c/sparse.cpp @@ -347,7 +347,7 @@ af_err af_sparse_convert_to(af_array *out, const af_array in, const af_storage destStorage) { try { // Handle dense case - const ArrayInfo &info = getInfo(in, false, true); + const ArrayInfo &info = getInfo(in, false); if (!info.isSparse()) { // If input is dense return af_create_sparse_array_from_dense(out, in, destStorage); } diff --git a/src/api/c/sparse_handle.hpp b/src/api/c/sparse_handle.hpp index e99bbb36e5..62c5289ebc 100644 --- a/src/api/c/sparse_handle.hpp +++ b/src/api/c/sparse_handle.hpp @@ -30,6 +30,7 @@ const common::SparseArray &getSparseArray(const af_array &arr) { const common::SparseArray *A = static_cast *>(arr); ARG_ASSERT(0, A->isSparse() == true); + checkAndMigrate(*A); return *A; } @@ -37,6 +38,7 @@ template common::SparseArray &getSparseArray(af_array &arr) { common::SparseArray *A = static_cast *>(arr); ARG_ASSERT(0, A->isSparse() == true); + checkAndMigrate(*A); return *A; } @@ -62,7 +64,7 @@ af_array retainSparseHandle(const af_array in) { // based on castArray in handle.hpp template common::SparseArray castSparse(const af_array &in) { - const ArrayInfo &info = getInfo(in, false, true); + const ArrayInfo &info = getInfo(in, false); using namespace common; #define CAST_SPARSE(Ti) \ diff --git a/src/api/c/stdev.cpp b/src/api/c/stdev.cpp index 7f64bf3355..d5589f4d39 100644 --- a/src/api/c/stdev.cpp +++ b/src/api/c/stdev.cpp @@ -38,6 +38,7 @@ using detail::mean; using detail::reduce; using detail::reduce_all; using detail::scalar; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -111,6 +112,7 @@ af_err af_stdev_all_v2(double* realVal, double* imagVal, const af_array in, case u16: *realVal = stdev(in, bias); break; case s64: *realVal = stdev(in, bias); break; case u64: *realVal = stdev(in, bias); break; + case s8: *realVal = stdev(in, bias); break; case u8: *realVal = stdev(in, bias); break; case b8: *realVal = stdev(in, bias); break; // TODO(umar): FIXME: sqrt(complex) is not present in cuda/opencl @@ -152,6 +154,7 @@ af_err af_stdev_v2(af_array* out, const af_array in, const af_var_bias bias, case u16: output = stdev(in, dim, bias); break; case s64: output = stdev(in, dim, bias); break; case u64: output = stdev(in, dim, bias); break; + case s8: output = stdev(in, dim, bias); break; case u8: output = stdev(in, dim, bias); break; case b8: output = stdev(in, dim, bias); break; // TODO(umar): FIXME: sqrt(complex) is not present in cuda/opencl diff --git a/src/api/c/stream.cpp b/src/api/c/stream.cpp index 1be207c66d..45265e69b5 100644 --- a/src/api/c/stream.cpp +++ b/src/api/c/stream.cpp @@ -28,6 +28,7 @@ using detail::cdouble; using detail::cfloat; using detail::createHostDataArray; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -141,6 +142,7 @@ af_err af_save_array(int *index, const char *key, const af_array arr, case b8: id = save(key, arr, filename, append); break; case s32: id = save(key, arr, filename, append); break; case u32: id = save(key, arr, filename, append); break; + case s8: id = save(key, arr, filename, append); break; case u8: id = save(key, arr, filename, append); break; case s64: id = save(key, arr, filename, append); break; case u64: id = save(key, arr, filename, append); break; @@ -240,6 +242,7 @@ static af_array readArrayV1(const char *filename, const unsigned index) { case b8: out = readDataToArray(fs); break; case s32: out = readDataToArray(fs); break; case u32: out = readDataToArray(fs); break; + case s8: out = readDataToArray(fs); break; case u8: out = readDataToArray(fs); break; case s64: out = readDataToArray(fs); break; case u64: out = readDataToArray(fs); break; diff --git a/src/api/c/surface.cpp b/src/api/c/surface.cpp index b2a6404a33..d748677269 100644 --- a/src/api/c/surface.cpp +++ b/src/api/c/surface.cpp @@ -38,6 +38,7 @@ using detail::createEmptyArray; using detail::forgeManager; using detail::getScalar; using detail::reduce_all; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -190,6 +191,9 @@ af_err af_draw_surface(const af_window window, const af_array xVals, case u16: chart = setup_surface(window, xVals, yVals, S, props); break; + case s8: + chart = setup_surface(window, xVals, yVals, S, props); + break; case u8: chart = setup_surface(window, xVals, yVals, S, props); break; diff --git a/src/api/c/susan.cpp b/src/api/c/susan.cpp index 0621f7eb16..8ea7dc8945 100644 --- a/src/api/c/susan.cpp +++ b/src/api/c/susan.cpp @@ -24,6 +24,7 @@ using detail::cfloat; using detail::createEmptyArray; using detail::createValueArray; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::ushort; @@ -98,6 +99,10 @@ af_err af_susan(af_features* out, const af_array in, const unsigned radius, *out = susan(in, radius, diff_thr, geom_thr, feature_ratio, edge); break; + case s8: + *out = susan(in, radius, diff_thr, geom_thr, + feature_ratio, edge); + break; case u8: *out = susan(in, radius, diff_thr, geom_thr, feature_ratio, edge); diff --git a/src/api/c/tile.cpp b/src/api/c/tile.cpp index ce512e9958..2a50f12c43 100644 --- a/src/api/c/tile.cpp +++ b/src/api/c/tile.cpp @@ -26,6 +26,7 @@ using detail::Array; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -60,6 +61,7 @@ af_err af_tile(af_array *out, const af_array in, const af::dim4 &tileDims) { case u64: output = tile(in, tileDims); break; case s16: output = tile(in, tileDims); break; case u16: output = tile(in, tileDims); break; + case s8: output = tile(in, tileDims); break; case u8: output = tile(in, tileDims); break; case f16: output = tile(in, tileDims); break; default: TYPE_ERROR(1, type); diff --git a/src/api/c/transform.cpp b/src/api/c/transform.cpp index 9bdaceb149..259d13840e 100644 --- a/src/api/c/transform.cpp +++ b/src/api/c/transform.cpp @@ -19,6 +19,7 @@ using af::dim4; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -158,6 +159,7 @@ void af_transform_common(af_array *out, const af_array in, const af_array tf, case u64: transform(out, in, tf, method, inverse, perspective); break; case s16: transform(out, in, tf, method, inverse, perspective); break; case u16: transform(out, in, tf, method, inverse, perspective); break; + case s8: transform(out, in, tf, method, inverse, perspective); break; case u8: transform(out, in, tf, method, inverse, perspective); break; case b8: transform(out, in, tf, method, inverse, perspective); break; default: TYPE_ERROR(1, itype); diff --git a/src/api/c/transpose.cpp b/src/api/c/transpose.cpp index 82ae18fef2..9d2fd48cbd 100644 --- a/src/api/c/transpose.cpp +++ b/src/api/c/transpose.cpp @@ -24,6 +24,7 @@ using detail::Array; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -67,6 +68,7 @@ af_err af_transpose(af_array* out, af_array in, const bool conjugate) { case b8: output = trs(in, conjugate); break; case s32: output = trs(in, conjugate); break; case u32: output = trs(in, conjugate); break; + case s8: output = trs(in, conjugate); break; case u8: output = trs(in, conjugate); break; case s64: output = trs(in, conjugate); break; case u64: output = trs(in, conjugate); break; @@ -107,6 +109,7 @@ af_err af_transpose_inplace(af_array in, const bool conjugate) { case b8: transpose_inplace(in, conjugate); break; case s32: transpose_inplace(in, conjugate); break; case u32: transpose_inplace(in, conjugate); break; + case s8: transpose_inplace(in, conjugate); break; case u8: transpose_inplace(in, conjugate); break; case s64: transpose_inplace(in, conjugate); break; case u64: transpose_inplace(in, conjugate); break; diff --git a/src/api/c/type_util.cpp b/src/api/c/type_util.cpp index 4b70df3295..d409c0d868 100644 --- a/src/api/c/type_util.cpp +++ b/src/api/c/type_util.cpp @@ -20,6 +20,7 @@ size_t size_of(af_dtype type) { case f64: return sizeof(double); case s32: return sizeof(int); case u32: return sizeof(unsigned); + case s8: return sizeof(signed char); case u8: return sizeof(unsigned char); case b8: return sizeof(unsigned char); case c32: return sizeof(float) * 2; @@ -38,6 +39,9 @@ size_t size_of(af_dtype type) { } af_err af_get_size_of(size_t *size, af_dtype type) { - *size = size_of(type); - return AF_SUCCESS; + try { + *size = size_of(type); + return AF_SUCCESS; + } + CATCHALL; } diff --git a/src/api/c/type_util.hpp b/src/api/c/type_util.hpp index 4214882492..8e6a7ff9cf 100644 --- a/src/api/c/type_util.hpp +++ b/src/api/c/type_util.hpp @@ -16,6 +16,11 @@ struct ToNum { inline T operator()(T val) { return val; } }; +template<> +struct ToNum { + inline int operator()(signed char val) { return static_cast(val); } +}; + template<> struct ToNum { inline int operator()(unsigned char val) { return static_cast(val); } diff --git a/src/api/c/unary.cpp b/src/api/c/unary.cpp index 6d8b584ace..505c831e74 100644 --- a/src/api/c/unary.cpp +++ b/src/api/c/unary.cpp @@ -43,6 +43,7 @@ using detail::intl; using detail::logicOp; using detail::real; using detail::scalar; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -598,6 +599,7 @@ af_err af_bitnot(af_array *out, const af_array in) { switch (type) { case s32: res = bitOpNot(in); break; case u32: res = bitOpNot(in); break; + case s8: res = bitOpNot(in); break; case u8: res = bitOpNot(in); break; case b8: res = bitOpNot(in); break; case s64: res = bitOpNot(in); break; diff --git a/src/api/c/unwrap.cpp b/src/api/c/unwrap.cpp index ee0ac2a16e..6f09a6b7eb 100644 --- a/src/api/c/unwrap.cpp +++ b/src/api/c/unwrap.cpp @@ -20,6 +20,7 @@ using detail::Array; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -81,6 +82,9 @@ af_err af_unwrap(af_array* out, const af_array in, const dim_t wx, case u16: output = unwrap(in, wx, wy, sx, sy, px, py, is_column); break; + case s8: + output = unwrap(in, wx, wy, sx, sy, px, py, is_column); + break; case u8: output = unwrap(in, wx, wy, sx, sy, px, py, is_column); break; diff --git a/src/api/c/var.cpp b/src/api/c/var.cpp index c82c1ca0cd..64a5d8f693 100644 --- a/src/api/c/var.cpp +++ b/src/api/c/var.cpp @@ -43,6 +43,7 @@ using detail::real; using detail::reduce; using detail::reduce_all; using detail::scalar; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -225,6 +226,9 @@ af_err af_var_v2(af_array* out, const af_array in, const af_var_bias bias, case u64: output = var_(in, no_weights, bias, dim); break; + case s8: + output = var_(in, no_weights, bias, dim); + break; case u8: output = var_(in, no_weights, bias, dim); break; @@ -298,6 +302,10 @@ af_err af_var_weighted(af_array* out, const af_array in, const af_array weights, output = var_(in, weights, AF_VARIANCE_POPULATION, dim); break; + case s8: + output = var_(in, weights, AF_VARIANCE_POPULATION, + dim); + break; case u8: output = var_(in, weights, AF_VARIANCE_POPULATION, dim); @@ -347,6 +355,7 @@ af_err af_var_all_v2(double* realVal, double* imagVal, const af_array in, case u16: *realVal = varAll(in, bias); break; case s64: *realVal = varAll(in, bias); break; case u64: *realVal = varAll(in, bias); break; + case s8: *realVal = varAll(in, bias); break; case u8: *realVal = varAll(in, bias); break; case b8: *realVal = varAll(in, bias); break; case f16: *realVal = varAll(in, bias); break; @@ -390,6 +399,7 @@ af_err af_var_all_weighted(double* realVal, double* imagVal, const af_array in, case u16: *realVal = varAll(in, weights); break; case s64: *realVal = varAll(in, weights); break; case u64: *realVal = varAll(in, weights); break; + case s8: *realVal = varAll(in, weights); break; case u8: *realVal = varAll(in, weights); break; case b8: *realVal = varAll(in, weights); break; case f16: *realVal = varAll(in, weights); break; @@ -453,6 +463,10 @@ af_err af_meanvar(af_array* mean, af_array* var, const af_array in, tie(*mean, *var) = meanvar(in, weights, bias, dim); break; + case s8: + tie(*mean, *var) = + meanvar(in, weights, bias, dim); + break; case u8: tie(*mean, *var) = meanvar(in, weights, bias, dim); diff --git a/src/api/c/vector_field.cpp b/src/api/c/vector_field.cpp index a46d1eed47..9eba21811c 100644 --- a/src/api/c/vector_field.cpp +++ b/src/api/c/vector_field.cpp @@ -35,6 +35,7 @@ using detail::copy_vector_field; using detail::createEmptyArray; using detail::forgeManager; using detail::reduce; +using detail::schar; using detail::transpose; using detail::uchar; using detail::uint; @@ -50,20 +51,21 @@ fg_chart setup_vector_field(fg_window window, const vector& points, vector> pnts; vector> dirs; - for (unsigned i = 0; i < points.size(); ++i) { - pnts.push_back(getArray(points[i])); - dirs.push_back(getArray(directions[i])); - } - - // Join for set up vector - dim4 odims(3, points.size()); - Array out_pnts = createEmptyArray(odims); - Array out_dirs = createEmptyArray(odims); - detail::join(out_pnts, 1, pnts); - detail::join(out_dirs, 1, dirs); - Array pIn = out_pnts; - Array dIn = out_dirs; + Array pIn = getArray(points[0]); + Array dIn = getArray(directions[0]); + if (points.size() > 1) { + for (unsigned i = 0; i < points.size(); ++i) { + pnts.push_back(getArray(points[i])); + dirs.push_back(getArray(directions[i])); + } + // Join for set up vector + const dim4 odims(pIn.dims()[0], points.size()); + pIn = createEmptyArray(odims); + dIn = createEmptyArray(odims); + detail::join(pIn, 1, pnts); + detail::join(dIn, 1, dirs); + } // do transpose if required if (transpose_) { pIn = transpose(pIn, false); @@ -182,6 +184,9 @@ af_err vectorFieldWrapper(const af_window window, const af_array points, case u16: chart = setup_vector_field(window, pnts, dirs, props); break; + case s8: + chart = setup_vector_field(window, pnts, dirs, props); + break; case u8: chart = setup_vector_field(window, pnts, dirs, props); break; @@ -288,6 +293,10 @@ af_err vectorFieldWrapper(const af_window window, const af_array xPoints, chart = setup_vector_field(window, points, directions, props); break; + case s8: + chart = setup_vector_field(window, points, directions, + props); + break; case u8: chart = setup_vector_field(window, points, directions, props); @@ -382,6 +391,10 @@ af_err vectorFieldWrapper(const af_window window, const af_array xPoints, chart = setup_vector_field(window, points, directions, props); break; + case s8: + chart = setup_vector_field(window, points, directions, + props); + break; case u8: chart = setup_vector_field(window, points, directions, props); diff --git a/src/api/c/where.cpp b/src/api/c/where.cpp index 4aeb7b60ba..6f83aed17d 100644 --- a/src/api/c/where.cpp +++ b/src/api/c/where.cpp @@ -18,6 +18,7 @@ using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -51,6 +52,7 @@ af_err af_where(af_array* idx, const af_array in) { case u64: res = where(in); break; case s16: res = where(in); break; case u16: res = where(in); break; + case s8: res = where(in); break; case u8: res = where(in); break; case b8: res = where(in); break; default: TYPE_ERROR(1, type); diff --git a/src/api/c/wrap.cpp b/src/api/c/wrap.cpp index f436f37350..e3c06a4642 100644 --- a/src/api/c/wrap.cpp +++ b/src/api/c/wrap.cpp @@ -19,6 +19,7 @@ using af::dim4; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -75,6 +76,7 @@ void af_wrap_common(af_array* out, const af_array in, const dim_t ox, case u64: wrap(out, in, wx, wy, sx, sy, px, py, is_column); break; case s16: wrap(out, in, wx, wy, sx, sy, px, py, is_column); break; case u16: wrap(out, in, wx, wy, sx, sy, px, py, is_column); break; + case s8: wrap(out, in, wx, wy, sx, sy, px, py, is_column); break; case u8: wrap(out, in, wx, wy, sx, sy, px, py, is_column); break; case b8: wrap(out, in, wx, wy, sx, sy, px, py, is_column); break; default: TYPE_ERROR(1, in_type); diff --git a/src/api/cpp/array.cpp b/src/api/cpp/array.cpp index 1d61c63c2d..418d94c52b 100644 --- a/src/api/cpp/array.cpp +++ b/src/api/cpp/array.cpp @@ -236,6 +236,7 @@ INSTANTIATE(double) INSTANTIATE(float) INSTANTIATE(unsigned) INSTANTIATE(int) +INSTANTIATE(signed char) INSTANTIATE(unsigned char) INSTANTIATE(char) INSTANTIATE(long long) @@ -701,6 +702,7 @@ MEM_FUNC(af_array, get) ASSIGN_TYPE(long long, OP) \ ASSIGN_TYPE(unsigned long long, OP) \ ASSIGN_TYPE(char, OP) \ + ASSIGN_TYPE(signed char, OP) \ ASSIGN_TYPE(unsigned char, OP) \ ASSIGN_TYPE(bool, OP) \ ASSIGN_TYPE(short, OP) \ @@ -828,6 +830,7 @@ array &array::operator=(const array &other) { ASSIGN_TYPE(long long, OP) \ ASSIGN_TYPE(unsigned long long, OP) \ ASSIGN_TYPE(char, OP) \ + ASSIGN_TYPE(signed char, OP) \ ASSIGN_TYPE(unsigned char, OP) \ ASSIGN_TYPE(bool, OP) \ ASSIGN_TYPE(short, OP) \ @@ -863,6 +866,7 @@ ASSIGN_OP(/=, af_div) ASSIGN_TYPE(long long, OP) \ ASSIGN_TYPE(unsigned long long, OP) \ ASSIGN_TYPE(char, OP) \ + ASSIGN_TYPE(signed char, OP) \ ASSIGN_TYPE(unsigned char, OP) \ ASSIGN_TYPE(bool, OP) \ ASSIGN_TYPE(short, OP) \ @@ -939,6 +943,7 @@ af::dtype implicit_dtype(af::dtype scalar_type, af::dtype array_type) { BINARY_TYPE(long long, OP, release_func, s64) \ BINARY_TYPE(unsigned long long, OP, release_func, u64) \ BINARY_TYPE(char, OP, release_func, b8) \ + BINARY_TYPE(signed char, OP, release_func, s8) \ BINARY_TYPE(unsigned char, OP, release_func, u8) \ BINARY_TYPE(bool, OP, release_func, b8) \ BINARY_TYPE(short, OP, release_func, s16) \ @@ -1038,6 +1043,7 @@ INSTANTIATE(double) INSTANTIATE(float) INSTANTIATE(unsigned) INSTANTIATE(int) +INSTANTIATE(signed char) INSTANTIATE(unsigned char) INSTANTIATE(char) INSTANTIATE(long long) @@ -1080,6 +1086,7 @@ INSTANTIATE(double) INSTANTIATE(float) INSTANTIATE(unsigned) INSTANTIATE(int) +INSTANTIATE(signed char) INSTANTIATE(unsigned char) INSTANTIATE(char) INSTANTIATE(long long) diff --git a/src/api/cpp/corrcoef.cpp b/src/api/cpp/corrcoef.cpp index f90be68b5f..dbedad5aee 100644 --- a/src/api/cpp/corrcoef.cpp +++ b/src/api/cpp/corrcoef.cpp @@ -26,6 +26,7 @@ INSTANTIATE_CORRCOEF(double); INSTANTIATE_CORRCOEF(int); INSTANTIATE_CORRCOEF(unsigned int); INSTANTIATE_CORRCOEF(char); +INSTANTIATE_CORRCOEF(signed char); INSTANTIATE_CORRCOEF(unsigned char); INSTANTIATE_CORRCOEF(long long); INSTANTIATE_CORRCOEF(unsigned long long); diff --git a/src/api/cpp/data.cpp b/src/api/cpp/data.cpp index 3f86520bd0..f5eb8c2544 100644 --- a/src/api/cpp/data.cpp +++ b/src/api/cpp/data.cpp @@ -130,6 +130,7 @@ CONSTANT(float); CONSTANT(int); CONSTANT(unsigned); CONSTANT(char); +CONSTANT(signed char); CONSTANT(unsigned char); CONSTANT(cfloat); CONSTANT(cdouble); diff --git a/src/api/cpp/device.cpp b/src/api/cpp/device.cpp index 89aab84754..b62589097e 100644 --- a/src/api/cpp/device.cpp +++ b/src/api/cpp/device.cpp @@ -192,6 +192,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(unsigned) +INSTANTIATE(signed char) INSTANTIATE(unsigned char) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/api/cpp/mean.cpp b/src/api/cpp/mean.cpp index c03a83fa51..61693ca40d 100644 --- a/src/api/cpp/mean.cpp +++ b/src/api/cpp/mean.cpp @@ -81,6 +81,7 @@ INSTANTIATE_MEAN(double); INSTANTIATE_MEAN(int); INSTANTIATE_MEAN(unsigned int); INSTANTIATE_MEAN(char); +INSTANTIATE_MEAN(signed char); INSTANTIATE_MEAN(unsigned char); INSTANTIATE_MEAN(long long); INSTANTIATE_MEAN(unsigned long long); diff --git a/src/api/cpp/median.cpp b/src/api/cpp/median.cpp index 5f4b88fb2a..b288df74a9 100644 --- a/src/api/cpp/median.cpp +++ b/src/api/cpp/median.cpp @@ -27,6 +27,7 @@ INSTANTIATE_MEDIAN(double); INSTANTIATE_MEDIAN(int); INSTANTIATE_MEDIAN(unsigned int); INSTANTIATE_MEDIAN(char); +INSTANTIATE_MEDIAN(signed char); INSTANTIATE_MEDIAN(unsigned char); INSTANTIATE_MEDIAN(long long); INSTANTIATE_MEDIAN(unsigned long long); diff --git a/src/api/cpp/reduce.cpp b/src/api/cpp/reduce.cpp index cfdadf85ae..8dc47fcab9 100644 --- a/src/api/cpp/reduce.cpp +++ b/src/api/cpp/reduce.cpp @@ -191,6 +191,7 @@ void max(array &val, array &idx, const array &in, const int dim) { INSTANTIATE_REAL(fnC, fnCPP, short) \ INSTANTIATE_REAL(fnC, fnCPP, unsigned short) \ INSTANTIATE_REAL(fnC, fnCPP, char) \ + INSTANTIATE_REAL(fnC, fnCPP, signed char) \ INSTANTIATE_REAL(fnC, fnCPP, unsigned char) \ INSTANTIATE_CPLX(fnC, fnCPP, af_cfloat, float) \ INSTANTIATE_CPLX(fnC, fnCPP, af_cdouble, double) @@ -294,6 +295,7 @@ INSTANTIATE(product_nan, product) INSTANTIATE_COMPAT(fnCPP, fnCompat, long long) \ INSTANTIATE_COMPAT(fnCPP, fnCompat, unsigned long long) \ INSTANTIATE_COMPAT(fnCPP, fnCompat, char) \ + INSTANTIATE_COMPAT(fnCPP, fnCompat, signed char) \ INSTANTIATE_COMPAT(fnCPP, fnCompat, unsigned char) \ INSTANTIATE_COMPAT(fnCPP, fnCompat, af_cfloat) \ INSTANTIATE_COMPAT(fnCPP, fnCompat, af_cdouble) \ @@ -332,6 +334,7 @@ INSTANTIATE_COMPAT(anyTrue, anytrue, bool) INSTANTIATE_REAL(fn, int) \ INSTANTIATE_REAL(fn, unsigned) \ INSTANTIATE_REAL(fn, char) \ + INSTANTIATE_REAL(fn, signed char) \ INSTANTIATE_REAL(fn, unsigned char) \ INSTANTIATE_REAL(fn, short) \ INSTANTIATE_REAL(fn, unsigned short) \ diff --git a/src/api/cpp/stdev.cpp b/src/api/cpp/stdev.cpp index a9e22d58f6..66edaf816a 100644 --- a/src/api/cpp/stdev.cpp +++ b/src/api/cpp/stdev.cpp @@ -60,6 +60,7 @@ INSTANTIATE_STDEV(unsigned long long); INSTANTIATE_STDEV(short); INSTANTIATE_STDEV(unsigned short); INSTANTIATE_STDEV(char); +INSTANTIATE_STDEV(signed char); INSTANTIATE_STDEV(unsigned char); #undef INSTANTIATE_STDEV diff --git a/src/api/cpp/var.cpp b/src/api/cpp/var.cpp index 80cd6a63c5..66f2d76252 100644 --- a/src/api/cpp/var.cpp +++ b/src/api/cpp/var.cpp @@ -112,6 +112,7 @@ INSTANTIATE_VAR(unsigned long long); INSTANTIATE_VAR(short); INSTANTIATE_VAR(unsigned short); INSTANTIATE_VAR(char); +INSTANTIATE_VAR(signed char); INSTANTIATE_VAR(unsigned char); INSTANTIATE_VAR(af_half); INSTANTIATE_VAR(half_float::half); diff --git a/src/api/unified/CMakeLists.txt b/src/api/unified/CMakeLists.txt index ca6805c7a4..bd373acab8 100644 --- a/src/api/unified/CMakeLists.txt +++ b/src/api/unified/CMakeLists.txt @@ -82,8 +82,8 @@ target_include_directories(af target_include_directories(af SYSTEM PRIVATE $ - $<$: $> - $<$: ${CUDA_INCLUDE_DIRS}> + $<$:$> + $<$:${CUDA_INCLUDE_DIRS}> ) target_link_libraries(af diff --git a/src/api/unified/error.cpp b/src/api/unified/error.cpp index 9fd89c0166..24a2dbfac9 100644 --- a/src/api/unified/error.cpp +++ b/src/api/unified/error.cpp @@ -42,7 +42,7 @@ void af_get_last_error(char **str, dim_t *len) { typedef void (*af_func)(char **, dim_t *); void *vfn = LOAD_SYMBOL(); af_func func = nullptr; - memcpy(&func, vfn, sizeof(void *)); + memcpy(&func, &vfn, sizeof(void *)); func(str, len); } } diff --git a/src/api/unified/symbol_manager.cpp b/src/api/unified/symbol_manager.cpp index d3aed5f498..93ca06938f 100644 --- a/src/api/unified/symbol_manager.cpp +++ b/src/api/unified/symbol_manager.cpp @@ -193,16 +193,15 @@ AFSymbolManager::AFSymbolManager() // In order of priority. static const af_backend order[] = {AF_BACKEND_CUDA, AF_BACKEND_ONEAPI, AF_BACKEND_OPENCL, AF_BACKEND_CPU}; - - LibHandle handle = nullptr; - af::Backend backend = AF_BACKEND_DEFAULT; + LibHandle handle = nullptr; + af::Backend backend = AF_BACKEND_DEFAULT; // Decremeting loop. The last successful backend loaded will be the most // prefered one. for (int i = NUM_BACKENDS - 1; i >= 0; i--) { - int backend_index = order[i] >> 1U; // 2 4 1 -> 1 2 0 - bkndHandles[backend_index] = openDynLibrary(order[i]); - if (bkndHandles[backend_index]) { - handle = bkndHandles[backend_index]; + int bknd_idx = backend_index(order[i]); + bkndHandles[bknd_idx] = openDynLibrary(order[i]); + if (bkndHandles[bknd_idx]) { + handle = bkndHandles[bknd_idx]; backend = order[i]; numBackends++; backendsAvailable += order[i]; @@ -242,7 +241,7 @@ af_err setBackend(af::Backend bknd) { UNIFIED_ERROR_LOAD_LIB(); } } - int idx = bknd >> 1U; // Convert 1, 2, 4 -> 0, 1, 2 + int idx = backend_index(bknd); if (instance.getHandle(idx)) { getActiveHandle() = instance.getHandle(idx); getActiveBackend() = bknd; diff --git a/src/backend/common/ArrayInfo.cpp b/src/backend/common/ArrayInfo.cpp index d919c942f8..60c55c3e52 100644 --- a/src/backend/common/ArrayInfo.cpp +++ b/src/backend/common/ArrayInfo.cpp @@ -221,8 +221,7 @@ dim4 toStride(const vector &seqs, const af::dim4 &parentDims) { namespace arrayfire { namespace common { -const ArrayInfo &getInfo(const af_array arr, bool sparse_check, - bool device_check) { +const ArrayInfo &getInfo(const af_array arr, bool sparse_check) { const ArrayInfo *info = nullptr; memcpy(&info, &arr, sizeof(af_array)); @@ -230,11 +229,6 @@ const ArrayInfo &getInfo(const af_array arr, bool sparse_check, // are accepted Otherwise only regular Array is accepted if (sparse_check) { ARG_ASSERT(0, info->isSparse() == false); } - if (device_check && info->getDevId() != static_cast( - detail::getActiveDeviceId())) { - AF_ERROR("Input Array not created on current device", AF_ERR_DEVICE); - } - return *info; } diff --git a/src/backend/common/Binary.hpp b/src/backend/common/Binary.hpp index 6ad8654f83..128cf18988 100644 --- a/src/backend/common/Binary.hpp +++ b/src/backend/common/Binary.hpp @@ -40,6 +40,13 @@ struct Binary { __DH__ T operator()(T lhs, T rhs) { return lhs + rhs; } }; +template +struct Binary { + static __DH__ T init() { return scalar(0); } + + __DH__ T operator()(T lhs, T rhs) { return lhs - rhs; } +}; + template struct Binary { static __DH__ T init() { return scalar(1); } @@ -47,6 +54,13 @@ struct Binary { __DH__ T operator()(T lhs, T rhs) { return lhs * rhs; } }; +template +struct Binary { + static __DH__ T init() { return scalar(1); } + + __DH__ T operator()(T lhs, T rhs) { return lhs / rhs; } +}; + template struct Binary { static __DH__ T init() { return scalar(0); } diff --git a/src/backend/common/DefaultMemoryManager.cpp b/src/backend/common/DefaultMemoryManager.cpp index d4aae2138e..0e0694631d 100644 --- a/src/backend/common/DefaultMemoryManager.cpp +++ b/src/backend/common/DefaultMemoryManager.cpp @@ -140,10 +140,19 @@ float DefaultMemoryManager::getMemoryPressure() { } } -bool DefaultMemoryManager::jitTreeExceedsMemoryPressure(size_t bytes) { +bool DefaultMemoryManager::jitTreeExceedsMemoryPressure( + size_t jit_tree_buffer_bytes) { lock_guard_t lock(this->memory_mutex); memory_info ¤t = this->getCurrentMemoryInfo(); - return 2 * bytes > current.lock_bytes; + if (current.lock_bytes > 0.25f * current.max_bytes) { + /// Evaluate JIT if half of all locked buffers are locked by this JIT + /// tree + return jit_tree_buffer_bytes > current.lock_bytes * 0.5f; + } else { + /// Evaluate if this JIT Tree accounts for 10% of total memory on the + /// device + return jit_tree_buffer_bytes > 0.10f * current.max_bytes; + } } void *DefaultMemoryManager::alloc(bool user_lock, const unsigned ndims, diff --git a/src/backend/common/Logger.hpp b/src/backend/common/Logger.hpp index a004e773fb..a9a8feaa0b 100644 --- a/src/backend/common/Logger.hpp +++ b/src/backend/common/Logger.hpp @@ -22,6 +22,7 @@ /* Intel ICC/ICPC */ // Fix the warning code here, if any #elif defined(__GNUC__) || defined(__GNUG__) +#pragma GCC diagnostic push /* GNU GCC/G++ */ #elif defined(_MSC_VER) /* Microsoft Visual Studio */ diff --git a/src/backend/common/SparseArray.cpp b/src/backend/common/SparseArray.cpp index ac91a29f31..052dc97e86 100644 --- a/src/backend/common/SparseArray.cpp +++ b/src/backend/common/SparseArray.cpp @@ -171,6 +171,13 @@ void destroySparseArray(SparseArray *sparse) { delete sparse; } +template +void checkAndMigrate(const SparseArray &arr) { + checkAndMigrate(const_cast &>(arr.getColIdx())); + checkAndMigrate(const_cast &>(arr.getRowIdx())); + checkAndMigrate(const_cast &>(arr.getValues())); +} + //////////////////////////////////////////////////////////////////////////// // Sparse Array Class Implementations //////////////////////////////////////////////////////////////////////////// @@ -250,7 +257,8 @@ SparseArray::SparseArray(const SparseArray &other, bool copy) template SparseArray::SparseArray( \ const af::dim4 &_dims, const Array &_values, \ const Array &_rowIdx, const Array &_colIdx, \ - const af::storage _storage, bool _copy) + const af::storage _storage, bool _copy); \ + template void checkAndMigrate(const SparseArray &arr) // Instantiate only floating types INSTANTIATE(float); diff --git a/src/backend/common/SparseArray.hpp b/src/backend/common/SparseArray.hpp index 860f7814ac..046a92fbe7 100644 --- a/src/backend/common/SparseArray.hpp +++ b/src/backend/common/SparseArray.hpp @@ -248,5 +248,12 @@ class SparseArray { friend void destroySparseArray(SparseArray *sparse); }; +/// Checks if the Array object can be migrated to the current device and if not, +/// an error is thrown +/// +/// \param[in] arr The Array that will be checked. +template +void checkAndMigrate(const SparseArray &arr); + } // namespace common } // namespace arrayfire diff --git a/src/backend/common/TemplateTypename.hpp b/src/backend/common/TemplateTypename.hpp index 47286af899..96dfb3c6fe 100644 --- a/src/backend/common/TemplateTypename.hpp +++ b/src/backend/common/TemplateTypename.hpp @@ -33,6 +33,7 @@ struct TemplateTypename { operator std::string() const noexcept { return #NAME; } \ } +SPECIALIZE(signed char, detail::schar); SPECIALIZE(unsigned char, detail::uchar); SPECIALIZE(unsigned int, detail::uint); SPECIALIZE(unsigned short, detail::ushort); diff --git a/src/backend/common/cast.cpp b/src/backend/common/cast.cpp index cc98f0504f..bcb2dfb519 100644 --- a/src/backend/common/cast.cpp +++ b/src/backend/common/cast.cpp @@ -14,6 +14,7 @@ using arrayfire::common::half; using detail::cdouble; using detail::cfloat; using detail::intl; +using detail::schar; using detail::uchar; using detail::uint; using detail::uintl; @@ -38,6 +39,7 @@ detail::Array castArray(const af_array &in) { case c64: return common::cast(getArray(in)); case s32: return common::cast(getArray(in)); case u32: return common::cast(getArray(in)); + case s8: return common::cast(getArray(in)); case u8: return common::cast(getArray(in)); case b8: return common::cast(getArray(in)); case s64: return common::cast(getArray(in)); @@ -56,6 +58,7 @@ template detail::Array castArray(const af_array &in); template detail::Array castArray(const af_array &in); template detail::Array castArray(const af_array &in); template detail::Array castArray(const af_array &in); +template detail::Array castArray(const af_array &in); template detail::Array castArray(const af_array &in); template detail::Array castArray(const af_array &in); template detail::Array castArray(const af_array &in); diff --git a/src/backend/common/cast.hpp b/src/backend/common/cast.hpp index 4186a03914..c60614a8a9 100644 --- a/src/backend/common/cast.hpp +++ b/src/backend/common/cast.hpp @@ -31,20 +31,21 @@ namespace common { /// outer -> inner -> outer /// /// inner cast -/// f32 f64 c32 c64 s32 u32 u8 b8 s64 u64 s16 u16 f16 -/// f32 x x x x x -/// f64 x x x x x -/// o c32 x x x x x -/// u c64 x x x x x -/// t s32 x x x x x x x x x -/// e u32 x x x x x x x x x -/// r u8 x x x x x x x x x x x x x -/// b8 x x x x x x x x x x x x x -/// c s64 x x x x x x x -/// a u64 x x x x x x x -/// s s16 x x x x x x x x x x x -/// t u16 x x x x x x x x x x x -/// f16 x x x x x +/// f32 f64 c32 c64 s32 u32 s8 u8 b8 s64 u64 s16 u16 f16 +/// f32 x x x x x +/// f64 x x x x x +/// o c32 x x x x x +/// u c64 x x x x x +/// t s32 x x x x x x x x x +/// e u32 x x x x x x x x x +/// r s8 x x x x x x x x x x x x x x +/// u8 x x x x x x x x x x x x x x +/// c b8 x x x x x x x x x x x x x x +/// a s64 x x x x x x x +/// s u64 x x x x x x x +/// t s16 x x x x x x x x x x x +/// u16 x x x x x x x x x x x +/// f16 x x x x x /// /// \param[in] outer The type of the second cast and the child of the /// previous cast diff --git a/src/backend/common/complex.hpp b/src/backend/common/complex.hpp index b7663580dc..e6c5bb79ce 100644 --- a/src/backend/common/complex.hpp +++ b/src/backend/common/complex.hpp @@ -6,8 +6,8 @@ * The complete license agreement can be obtained at: * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +#pragma once -#include #include #include diff --git a/src/backend/common/err_common.cpp b/src/backend/common/err_common.cpp index 9e2b2e8a2f..672afe6da0 100644 --- a/src/backend/common/err_common.cpp +++ b/src/backend/common/err_common.cpp @@ -25,6 +25,7 @@ #include #include #elif defined(AF_ONEAPI) +#include #include #endif @@ -91,8 +92,8 @@ int ArgumentError::getArgIndex() const noexcept { return argIndex; } SupportError::SupportError(const char *const func, const char *const file, const int line, const char *const back, - stacktrace st) - : AfError(func, file, line, "Unsupported Error", AF_ERR_NOT_SUPPORTED, + const char *const message, stacktrace st) + : AfError(func, file, line, message, AF_ERR_NOT_SUPPORTED, std::move(st)) , backend(back) {} @@ -169,6 +170,16 @@ af_err processException() { snprintf(oneapi_err_msg, sizeof(oneapi_err_msg), "oneAPI Error (%d): %s", ex.code().value(), ex.what()); + if (ex.code() == sycl::errc::memory_allocation) { + err = set_global_error_string(oneapi_err_msg, AF_ERR_NO_MEM); + } else { + err = set_global_error_string(oneapi_err_msg, AF_ERR_INTERNAL); + } + } catch (const oneapi::mkl::exception &ex) { + char oneapi_err_msg[1024]; + snprintf(oneapi_err_msg, sizeof(oneapi_err_msg), "MKL Error: %s", + ex.what()); + err = set_global_error_string(oneapi_err_msg, AF_ERR_INTERNAL); #endif #ifdef AF_OPENCL @@ -184,6 +195,8 @@ af_err processException() { err = set_global_error_string(opencl_err_msg, AF_ERR_INTERNAL); } #endif + } catch (const std::exception &ex) { + err = set_global_error_string(ex.what(), AF_ERR_UNKNOWN); } catch (...) { err = set_global_error_string(ss.str(), AF_ERR_UNKNOWN); } return err; diff --git a/src/backend/common/err_common.hpp b/src/backend/common/err_common.hpp index 3936cee77c..846f4b516f 100644 --- a/src/backend/common/err_common.hpp +++ b/src/backend/common/err_common.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -112,7 +113,7 @@ class SupportError : public AfError { public: SupportError(const char* const func, const char* const file, const int line, - const char* const back, + const char* const back, const char* const message, const boost::stacktrace::stacktrace st); SupportError(SupportError&& other) noexcept = default; diff --git a/src/backend/common/graphics_common.cpp b/src/backend/common/graphics_common.cpp index 217722eb36..01f94078d4 100644 --- a/src/backend/common/graphics_common.cpp +++ b/src/backend/common/graphics_common.cpp @@ -139,6 +139,7 @@ INSTANTIATE_GET_FG_TYPE(float, FG_FLOAT32); INSTANTIATE_GET_FG_TYPE(int, FG_INT32); INSTANTIATE_GET_FG_TYPE(unsigned, FG_UINT32); INSTANTIATE_GET_FG_TYPE(char, FG_INT8); +INSTANTIATE_GET_FG_TYPE(signed char, FG_INT8); INSTANTIATE_GET_FG_TYPE(unsigned char, FG_UINT8); INSTANTIATE_GET_FG_TYPE(unsigned short, FG_UINT16); INSTANTIATE_GET_FG_TYPE(short, FG_INT16); diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp index 67bd47829f..42d18be47b 100644 --- a/src/backend/common/half.hpp +++ b/src/backend/common/half.hpp @@ -87,7 +87,9 @@ using uint16_t = unsigned short; #define AF_CONSTEXPR constexpr #else #include +#include #include +#include #include #include #include @@ -162,6 +164,10 @@ AF_CONSTEXPR __DH__ native_half_t int2half_impl(char value) noexcept { return __ull2half_rn(value); } template<> +AF_CONSTEXPR __DH__ native_half_t int2half_impl(signed char value) noexcept { + return __ull2half_rn(value); +} +template<> AF_CONSTEXPR __DH__ native_half_t int2half_impl(unsigned char value) noexcept { return __ull2half_rn(value); } @@ -244,9 +250,9 @@ AF_CONSTEXPR __DH__ native_half_t int2half_impl(T value) noexcept { /// \return binary representation of half-precision value template __DH__ native_half_t float2half_impl(float value) noexcept { - uint32_t bits = 0; // = *reinterpret_cast(&value); - // //violating strict aliasing! - std::memcpy(&bits, &value, sizeof(float)); + alignas(std::max(alignof(uint32_t), alignof(float))) float _value = value; + uint32_t bits = *reinterpret_cast(&_value); + constexpr uint16_t base_table[512] = { 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, @@ -336,9 +342,10 @@ __DH__ native_half_t float2half_impl(float value) noexcept { 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13}; - uint16_t hbits = - base_table[bits >> 23] + - static_cast((bits & 0x7FFFFF) >> shift_table[bits >> 23]); + alignas(std::max(alignof(uint16_t), alignof(native_half_t))) + uint16_t hbits = + base_table[bits >> 23] + + static_cast((bits & 0x7FFFFF) >> shift_table[bits >> 23]); AF_IF_CONSTEXPR(R == std::round_to_nearest) hbits += (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) | @@ -366,7 +373,8 @@ __DH__ native_half_t float2half_impl(float value) noexcept { (((bits >> 23) <= 358) & ((bits >> 23) != 256))) & (hbits < 0xFC00) & (hbits >> 15)) - ((hbits == 0x7C00) & ((bits >> 23) != 255)); - return hbits; + + return *reinterpret_cast(&hbits); } /// Convert IEEE double-precision to half-precision. @@ -378,11 +386,11 @@ __DH__ native_half_t float2half_impl(float value) noexcept { /// \return binary representation of half-precision value template __DH__ native_half_t float2half_impl(double value) { - uint64_t bits{0}; // = *reinterpret_cast(&value); //violating - // strict aliasing! - std::memcpy(&bits, &value, sizeof(double)); + alignas(std::max(alignof(uint64_t), alignof(double))) double _value = value; + uint64_t bits = *reinterpret_cast(&_value); uint32_t hi = bits >> 32, lo = bits & 0xFFFFFFFF; - uint16_t hbits = (hi >> 16) & 0x8000; + alignas(std::max(alignof(uint16_t), alignof(native_half_t))) + uint16_t hbits = (hi >> 16) & 0x8000; hi &= 0x7FFFFFFF; int exp = hi >> 20; if (exp == 2047) @@ -419,7 +427,8 @@ __DH__ native_half_t float2half_impl(double value) { ~(hbits >> 15) & (s | g); else AF_IF_CONSTEXPR(R == std::round_toward_neg_infinity) hbits += (hbits >> 15) & (g | s); - return hbits; + + return *reinterpret_cast(&hbits); } __DH__ inline float half2float_impl(native_half_t value) noexcept { @@ -789,14 +798,14 @@ __DH__ inline float half2float_impl(native_half_t value) noexcept { 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024}; - uint16_t value_bits = 0; - std::memcpy(&value_bits, &value, sizeof(uint16_t)); - uint32_t bits = + alignas(std::max(alignof(uint16_t), alignof(native_half_t))) + native_half_t _value = value; + uint16_t value_bits = *reinterpret_cast(&_value); + + alignas(std::max(alignof(uint32_t), alignof(float))) uint32_t bits = mantissa_table[offset_table[value_bits >> 10] + (value_bits & 0x3FF)] + exponent_table[value_bits >> 10]; - float out = 0.0f; - std::memcpy(&out, &bits, sizeof(float)); - return out; + return *reinterpret_cast(&bits); } #endif // __CUDACC_RTC__ @@ -854,23 +863,25 @@ AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept { template AF_CONSTEXPR T half2int(native_half_t value) { #ifdef __CUDA_ARCH__ - AF_IF_CONSTEXPR(std::is_same_v || std::is_same_v || - std::is_same_v) { + AF_IF_CONSTEXPR(std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value) { return __half2short_rn(value); } - else AF_IF_CONSTEXPR(std::is_same_v) { + else AF_IF_CONSTEXPR(std::is_same::value) { return __half2ushort_rn(value); } - else AF_IF_CONSTEXPR(std::is_same_v) { + else AF_IF_CONSTEXPR(std::is_same::value) { return __half2ll_rn(value); } - else AF_IF_CONSTEXPR(std::is_same_v) { + else AF_IF_CONSTEXPR(std::is_same::value) { return __half2ull_rn(value); } - else AF_IF_CONSTEXPR(std::is_same_v) { + else AF_IF_CONSTEXPR(std::is_same::value) { return __half2int_rn(value); } - else AF_IF_CONSTEXPR(std::is_same_v) { + else { return __half2uint_rn(value); } #elif defined(AF_ONEAPI) @@ -1038,6 +1049,10 @@ class alignas(2) half { return half2int(data_); } + AF_CONSTEXPR __DH__ explicit operator signed char() const noexcept { + return half2int(data_); + } + AF_CONSTEXPR __DH__ explicit operator unsigned char() const noexcept { return half2int(data_); } diff --git a/src/backend/common/jit/BinaryNode.cpp b/src/backend/common/jit/BinaryNode.cpp index 84c5597e31..b017394876 100644 --- a/src/backend/common/jit/BinaryNode.cpp +++ b/src/backend/common/jit/BinaryNode.cpp @@ -69,6 +69,7 @@ INSTANTIATE(cdouble, double, af_cplx2_t); INSTANTIATE(unsigned short, unsigned short, op); \ INSTANTIATE(unsigned long long, unsigned long long, op); \ INSTANTIATE(long long, long long, op); \ + INSTANTIATE(signed char, signed char, op); \ INSTANTIATE(unsigned char, unsigned char, op); \ INSTANTIATE(char, char, op); \ INSTANTIATE(common::half, common::half, op); \ @@ -91,6 +92,7 @@ INSTANTIATE_ARITH(af_max_t); INSTANTIATE(unsigned short, unsigned short, op); \ INSTANTIATE(unsigned long long, unsigned long long, op); \ INSTANTIATE(long long, long long, op); \ + INSTANTIATE(signed char, signed char, op); \ INSTANTIATE(unsigned char, unsigned char, op); \ INSTANTIATE(char, char, op); \ INSTANTIATE(common::half, common::half, op); \ @@ -114,6 +116,7 @@ INSTANTIATE_FLOATOPS(af_atan2_t); INSTANTIATE(unsigned short, unsigned short, op); \ INSTANTIATE(unsigned long long, unsigned long long, op); \ INSTANTIATE(long long, long long, op); \ + INSTANTIATE(signed char, signed char, op); \ INSTANTIATE(unsigned char, unsigned char, op); \ INSTANTIATE(char, char, op); \ INSTANTIATE(int, int, op) @@ -136,6 +139,7 @@ INSTANTIATE_BITOP(af_bitxor_t); INSTANTIATE(char, unsigned short, op); \ INSTANTIATE(char, unsigned long long, op); \ INSTANTIATE(char, long long, op); \ + INSTANTIATE(char, signed char, op); \ INSTANTIATE(char, unsigned char, op); \ INSTANTIATE(char, char, op); \ INSTANTIATE(char, int, op) diff --git a/src/backend/common/jit/BufferNodeBase.hpp b/src/backend/common/jit/BufferNodeBase.hpp index 061aa37a8c..85576304ad 100644 --- a/src/backend/common/jit/BufferNodeBase.hpp +++ b/src/backend/common/jit/BufferNodeBase.hpp @@ -30,9 +30,9 @@ class BufferNodeBase : public common::Node { public: ParamType m_param; BufferNodeBase(af::dtype type) - : Node(type, 0, {}), m_bytes(0), m_linear_buffer(true) {} - - bool isBuffer() const final { return true; } + : Node(type, 0, {}, kNodeType::Buffer) + , m_bytes(0) + , m_linear_buffer(true) {} std::unique_ptr clone() final { return std::make_unique(*this); @@ -71,10 +71,11 @@ class BufferNodeBase : public common::Node { } int setArgs(int start_id, bool is_linear, - std::function + std::function setArg) const override { - return detail::setKernelArguments(start_id, is_linear, setArg, m_data, - m_param); + return detail::setBufferKernelArguments(start_id, is_linear, setArg, + m_data, m_param); } void genOffsets(std::stringstream &kerStream, int id, @@ -118,6 +119,18 @@ class BufferNodeBase : public common::Node { } return false; } + + virtual void modDims(const af::dim4 &newDim) override { + af::dim4 strides(1, 1, 1, 1); + for(dim_t i = 1; i < 4; ++i) { + strides[i] = strides[i - 1] * newDim[i - 1]; + } + + for(dim_t i = 0; i < 4; ++i) { + m_param.dims[i] = newDim[i]; + m_param.strides[i] = strides[i]; + } + } }; } // namespace common diff --git a/src/backend/common/jit/NaryNode.hpp b/src/backend/common/jit/NaryNode.hpp index 0d78b9e86c..5f1e91a570 100644 --- a/src/backend/common/jit/NaryNode.hpp +++ b/src/backend/common/jit/NaryNode.hpp @@ -40,7 +40,8 @@ class NaryNode : public Node { type, height, std::forward< const std::array>( - children)) + children), + kNodeType::Nary) , m_num_children(num_children) , m_op_str(op_str) , m_op(op) { diff --git a/src/backend/common/jit/Node.cpp b/src/backend/common/jit/Node.cpp index 0e67228f91..09c001a724 100644 --- a/src/backend/common/jit/Node.cpp +++ b/src/backend/common/jit/Node.cpp @@ -42,6 +42,7 @@ int Node::getNodesMap(Node_map_t &node_map, vector &full_nodes, } std::string getFuncName(const vector &output_nodes, + const vector &output_ids, const vector &full_nodes, const vector &full_ids, const bool is_linear, const bool loop0, const bool loop1, const bool loop2, @@ -59,6 +60,11 @@ std::string getFuncName(const vector &output_nodes, funcName += node->getNameStr(); } + for (const int id : output_ids) { + funcName += '-'; + funcName += std::to_string(id); + } + for (int i = 0; i < static_cast(full_nodes.size()); i++) { full_nodes[i]->genKerName(funcName, full_ids[i]); } @@ -76,6 +82,12 @@ auto isScalar(const Node &ptr) -> bool { return ptr.isScalar(); } bool Node::isLinear(const dim_t dims[4]) const { return true; } +/// This function returns true if the \p node is a Shift node or a Buffer node +auto isBufferOrShift(const Node_ptr &node) -> bool { + return node->getNodeType() == kNodeType::Buffer || + node->getNodeType() == kNodeType::Shift; +} + } // namespace common } // namespace arrayfire diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp index 8a262e0734..794c10c14c 100644 --- a/src/backend/common/jit/Node.hpp +++ b/src/backend/common/jit/Node.hpp @@ -13,7 +13,9 @@ #include #include #include +#include +#include #include #include #include @@ -32,6 +34,15 @@ enum class kJITHeuristics { namespace arrayfire { namespace common { + +enum class kNodeType { + Generic = 0, + Scalar = 1, + Buffer = 2, + Nary = 3, + Shift = 4, +}; + class Node; } // namespace common } // namespace arrayfire @@ -88,6 +99,7 @@ static const char *getFullName(af::dtype type) { case u16: return detail::getFullName(); case s16: return detail::getFullName(); case b8: return detail::getFullName(); + case s8: return detail::getFullName(); case u8: return detail::getFullName(); case f16: return "half"; } @@ -107,6 +119,7 @@ static const char *getShortName(af::dtype type) { case u16: return detail::shortname(); case s16: return detail::shortname(); case b8: return detail::shortname(); + case s8: return detail::shortname(); case u8: return detail::shortname(); case f16: return "h"; } @@ -122,13 +135,17 @@ class Node { std::array m_children; af::dtype m_type; int m_height; + kNodeType m_node_type = kNodeType::Generic; template friend class NodeIterator; Node() = default; Node(const af::dtype type, const int height, - const std::array children) - : m_children(children), m_type(type), m_height(height) { + const std::array children, kNodeType node_type) + : m_children(children) + , m_type(type) + , m_height(height) + , m_node_type(node_type) { static_assert(std::is_nothrow_move_assignable::value, "Node is not move assignable"); } @@ -225,10 +242,10 @@ class Node { /// /// \returns the next index that will need to be set in the kernl. This /// is usually start_id + the number of times setArg is called - virtual int setArgs( - int start_id, bool is_linear, - std::function setArg) - const { + virtual int setArgs(int start_id, bool is_linear, + std::function + setArg) const { UNUSED(is_linear); UNUSED(setArg); return start_id; @@ -249,14 +266,17 @@ class Node { virtual size_t getBytes() const { return 0; } // Returns true if this node is a Buffer - virtual bool isBuffer() const { return false; } + bool isBuffer() const { return m_node_type == kNodeType::Buffer; } // Returns true if this node is a Scalar - virtual bool isScalar() const { return false; } + bool isScalar() const { return m_node_type == kNodeType::Scalar; } /// Returns true if the buffer is linear virtual bool isLinear(const dim_t dims[4]) const; + /// Returns the node type + kNodeType getNodeType() const { return m_node_type; } + /// Returns the type af::dtype getType() const { return m_type; } @@ -292,6 +312,10 @@ class Node { } virtual std::unique_ptr clone() = 0; + virtual void modDims(const af::dim4 &newDim) { + UNUSED(newDim); + } + #ifdef AF_CPU template friend void arrayfire::cpu::kernel::evalMultiple( @@ -309,14 +333,85 @@ struct Node_ids { }; std::string getFuncName(const std::vector &output_nodes, + const std::vector &output_ids, const std::vector &full_nodes, const std::vector &full_ids, const bool is_linear, const bool loop0, const bool loop1, const bool loop2, const bool loop3); +/// Returns true if the \p ptr is a Buffer Node auto isBuffer(const Node &ptr) -> bool; +/// Returns true if the \p ptr is a Scalar Node auto isScalar(const Node &ptr) -> bool; +/// Returns true if \p node is a Buffer or a Shift node +auto isBufferOrShift(const Node_ptr &node) -> bool; + +template +inline void applyShifts(std::array &shifts, nonstd::span dims) { + std::array out; + for (size_t i = 0; i < shifts.size(); i++) { out[i] = dims[shifts[i]]; } + std::copy(begin(out), std::end(out), std::begin(dims)); +} + +template +inline std::array compressArray(ArrayT dims) { + std::array shifts{0, 1, 2, 3}; + bool changed; + do { + changed = false; + for (int i = 0; i < AF_MAX_DIMS - 1; i++) { + if (dims[i] == 1 && dims[i + 1] != 1) { + std::swap(dims[i], dims[i + 1]); + std::swap(shifts[i], shifts[i + 1]); + changed = true; + } + } + } while (changed); + return shifts; +} + +/// Removes empty columns from output and the other node pointers in \p nodes +template +void removeEmptyDimensions(nonstd::span outputs, + nonstd::span nodes) { + dim_t *outDims{outputs[0].dims_ptr()}; + dim_t *outStrides{outputs[0].strides_ptr()}; + auto shifts = compressArray(outDims); + applyShifts(shifts, {outStrides, AF_MAX_DIMS}); + for (auto nodeIt{begin(nodes)}, endIt{end(nodes)}; + (nodeIt = find_if(nodeIt, endIt, isBufferOrShift)) != endIt; + ++nodeIt) { + switch ((*nodeIt)->getNodeType()) { + case kNodeType::Buffer: { + BufferNodeT *buf{static_cast(nodeIt->get())}; + applyShifts(shifts, + {buf->m_param.dims_ptr(), AF_MAX_DIMS}); + applyShifts(shifts, + {buf->m_param.strides_ptr(), AF_MAX_DIMS}); + } break; + case kNodeType::Shift: { + ShiftNodeT &shiftNode{ + *static_cast(nodeIt->get())}; + BufferNodeT &buf{shiftNode.getBufferNode()}; + applyShifts(shifts, + {buf.m_param.dims_ptr(), AF_MAX_DIMS}); + applyShifts(shifts, + {buf.m_param.strides_ptr(), AF_MAX_DIMS}); + + auto &node_shifts = shiftNode.getShifts(); + applyShifts(shifts, node_shifts); + } break; + default: break; + } + } + std::for_each( + std::begin(outputs) + 1, std::end(outputs), [&shifts](ParamT &output) { + applyShifts(shifts, {output.dims_ptr(), AF_MAX_DIMS}); + applyShifts(shifts, {output.strides_ptr(), AF_MAX_DIMS}); + }); +} + } // namespace common } // namespace arrayfire diff --git a/src/backend/common/jit/ScalarNode.hpp b/src/backend/common/jit/ScalarNode.hpp index 3a530a6911..4236ec4725 100644 --- a/src/backend/common/jit/ScalarNode.hpp +++ b/src/backend/common/jit/ScalarNode.hpp @@ -26,7 +26,8 @@ class ScalarNode : public common::Node { public: ScalarNode(T val) - : Node(static_cast(af::dtype_traits::af_type), 0, {}) + : Node(static_cast(af::dtype_traits::af_type), 0, {}, + kNodeType::Scalar) , m_val(val) { static_assert(std::is_nothrow_move_assignable::value, "ScalarNode is not move assignable"); @@ -72,10 +73,11 @@ class ScalarNode : public common::Node { } int setArgs(int start_id, bool is_linear, - std::function + std::function setArg) const final { UNUSED(is_linear); - setArg(start_id, static_cast(&m_val), sizeof(T)); + setArg(start_id, static_cast(&m_val), sizeof(T), false); return start_id + 1; } @@ -85,9 +87,6 @@ class ScalarNode : public common::Node { << ";\n"; } - // Returns true if this node is a Buffer - virtual bool isScalar() const { return false; } - std::string getNameStr() const final { return detail::shortname(false); } // Return the info for the params and the size of the buffers diff --git a/src/backend/common/jit/ShiftNodeBase.hpp b/src/backend/common/jit/ShiftNodeBase.hpp index bbc0f5863f..553f4a16a1 100644 --- a/src/backend/common/jit/ShiftNodeBase.hpp +++ b/src/backend/common/jit/ShiftNodeBase.hpp @@ -32,7 +32,9 @@ class ShiftNodeBase : public Node { public: ShiftNodeBase(const af::dtype type, std::shared_ptr buffer_node, const std::array shifts) - : Node(type, 0, {}), m_buffer_node(buffer_node), m_shifts(shifts) { + : Node(type, 0, {}, kNodeType::Shift) + , m_buffer_node(buffer_node) + , m_shifts(shifts) { static_assert(std::is_nothrow_move_assignable::value, "ShiftNode is not move assignable"); static_assert(std::is_nothrow_move_constructible::value, @@ -51,6 +53,8 @@ class ShiftNodeBase : public Node { return *this; } + std::array &getShifts() { return m_shifts; } + std::unique_ptr clone() final { return std::make_unique(*this); } @@ -63,6 +67,9 @@ class ShiftNodeBase : public Node { swap(m_shifts, other.m_shifts); } + BufferNode &getBufferNode() { return *m_buffer_node; } + const BufferNode &getBufferNode() const { return *m_buffer_node; } + bool isLinear(const dim_t dims[4]) const final { UNUSED(dims); return false; @@ -85,12 +92,14 @@ class ShiftNodeBase : public Node { } int setArgs(int start_id, bool is_linear, - std::function + std::function setArg) const { int curr_id = m_buffer_node->setArgs(start_id, is_linear, setArg); for (int i = 0; i < 4; i++) { const int &d = m_shifts[i]; - setArg(curr_id + i, static_cast(&d), sizeof(int)); + setArg(curr_id + i, static_cast(&d), sizeof(int), + false); } return curr_id + 4; } diff --git a/src/backend/common/moddims.cpp b/src/backend/common/moddims.cpp index 6fbd99650e..25edfa5b0a 100644 --- a/src/backend/common/moddims.cpp +++ b/src/backend/common/moddims.cpp @@ -20,21 +20,27 @@ using detail::createNodeArray; using std::make_shared; using std::shared_ptr; +using std::array; +using arrayfire::common::Node; +using arrayfire::common::Node_ptr; using std::vector; namespace arrayfire { namespace common { + +Node_ptr copyModdims(const Node_ptr &in, const af::dim4 &newDim) { + + Node_ptr out = in->clone(); + for(int i = 0; i < in->kMaxChildren && in->m_children[i] != nullptr; ++i) { + out->m_children[i] = copyModdims(in->m_children[i], newDim); + } + if(out->isBuffer()) out->modDims(newDim); + + return out; +} + template Array moddimOp(const Array &in, af::dim4 outDim) { - using arrayfire::common::Node; - using arrayfire::common::Node_ptr; - using std::array; - - auto createModdim = [outDim](array &operands) { - return make_shared( - outDim, static_cast(af::dtype_traits::af_type), - operands[0]); - }; const auto &node = in.getNode(); @@ -49,8 +55,9 @@ Array moddimOp(const Array &in, af::dim4 outDim) { } if (all_linear == false) in.eval(); - Node_ptr out = createNaryNode(outDim, createModdim, {&in}); - return createNodeArray(outDim, out); + Array out = createNodeArray(outDim, copyModdims(in.getNode(), outDim)); + + return out; } template @@ -94,6 +101,7 @@ INSTANTIATE(double); INSTANTIATE(detail::cfloat); INSTANTIATE(detail::cdouble); INSTANTIATE(arrayfire::common::half); +INSTANTIATE(signed char); INSTANTIATE(unsigned char); INSTANTIATE(char); INSTANTIATE(unsigned short); diff --git a/src/backend/common/traits.hpp b/src/backend/common/traits.hpp index 2b9090727c..51a4b53899 100644 --- a/src/backend/common/traits.hpp +++ b/src/backend/common/traits.hpp @@ -24,6 +24,7 @@ namespace { inline size_t dtypeSize(af::dtype type) { switch (type) { + case s8: case u8: case b8: return 1; case s16: @@ -59,7 +60,7 @@ constexpr bool isRealFloating(af::dtype type) { constexpr bool isInteger(af::dtype type) { return (type == s32 || type == u32 || type == s64 || type == u64 || - type == s16 || type == u16 || type == u8); + type == s16 || type == u16 || type == s8 || type == u8); } constexpr bool isBool(af::dtype type) { return (type == b8); } @@ -68,6 +69,12 @@ constexpr bool isFloating(af::dtype type) { return (!isInteger(type) && !isBool(type)); } +template +constexpr bool is_any_of() { + AF_IF_CONSTEXPR(!sizeof...(Args)) { return std::is_same::value; } + else { return std::is_same::value || is_any_of(); } +} + } // namespace } // namespace common } // namespace arrayfire diff --git a/src/backend/common/util.cpp b/src/backend/common/util.cpp index 2d4a8e5ea0..87be74fa83 100644 --- a/src/backend/common/util.cpp +++ b/src/backend/common/util.cpp @@ -103,6 +103,7 @@ const char* getName(af_dtype type) { case u64: return "unsigned long long"; case s64: return "long long"; case u8: return "unsigned char"; + case s8: return "signed char"; case b8: return "bool"; default: return "unknown type"; } @@ -125,7 +126,13 @@ void saveKernel(const string& funcName, const string& jit_ker, // Path to a folder const string ffp = string(jitKernelsOutput) + AF_PATH_SEPARATOR + funcName + ext; + +#if defined(OS_WIN) + FILE* f = fopen(ffp.c_str(), "w"); +#else FILE* f = fopen(ffp.c_str(), "we"); +#endif + if (!f) { fprintf(stderr, "Cannot open file %s\n", ffp.c_str()); return; @@ -269,6 +276,7 @@ template string toString(int); template string toString(unsigned short); template string toString(short); template string toString(unsigned char); +template string toString(signed char); template string toString(char); template string toString(long); template string toString(long long); diff --git a/src/backend/cpu/Array.cpp b/src/backend/cpu/Array.cpp index 88f4bcabee..276ea952b4 100644 --- a/src/backend/cpu/Array.cpp +++ b/src/backend/cpu/Array.cpp @@ -134,6 +134,11 @@ Array::Array(const dim4 &dims, const dim4 &strides, dim_t offset_, } } +template +void checkAndMigrate(const Array &arr) { + return; +} + template void Array::eval() { evalMultiple({this}); @@ -353,7 +358,8 @@ void Array::setDataDims(const dim4 &new_dims) { Array & arr, const void *const data, const size_t bytes); \ template void evalMultiple(vector *> arrays); \ template kJITHeuristics passesJitHeuristics(span n); \ - template void Array::setDataDims(const dim4 &new_dims); + template void Array::setDataDims(const dim4 &new_dims); \ + template void checkAndMigrate(const Array &arr); INSTANTIATE(float) INSTANTIATE(double) @@ -361,6 +367,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) diff --git a/src/backend/cpu/Array.hpp b/src/backend/cpu/Array.hpp index 3c7b54c5ec..7afed3501e 100644 --- a/src/backend/cpu/Array.hpp +++ b/src/backend/cpu/Array.hpp @@ -127,6 +127,13 @@ void *getRawPtr(const Array &arr) { return (void *)(arr.get(false)); } +/// Checks if the Array object can be migrated to the current device and if not, +/// an error is thrown +/// +/// \param[in] arr The Array that will be checked. +template +void checkAndMigrate(const Array &arr); + // Array Array Implementation template class Array { diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt index b8025d53a2..8a83a55894 100644 --- a/src/backend/cpu/CMakeLists.txt +++ b/src/backend/cpu/CMakeLists.txt @@ -15,6 +15,10 @@ generate_product_version(af_cpu_ver_res_file add_library(afcpu "") add_library(ArrayFire::afcpu ALIAS afcpu) +# CPU back end needs to use MKL LP64 interface +set(MKL_INTERFACE_INTEGER_SIZE 4) +set(MKL_INTERFACE "lp64") + # CPU backend source files target_sources(afcpu PRIVATE @@ -313,6 +317,7 @@ target_link_libraries(afcpu ) if(BUILD_WITH_MKL) target_compile_definitions(afcpu PRIVATE USE_MKL) + target_compile_definitions(afcpu PRIVATE AF_MKL_INTERFACE_SIZE=${MKL_INTERFACE_INTEGER_SIZE}) if(MKL_BATCH) target_compile_definitions(afcpu PRIVATE AF_USE_MKL_BATCH) diff --git a/src/backend/cpu/assign.cpp b/src/backend/cpu/assign.cpp index cfeb5e168e..32af00e487 100644 --- a/src/backend/cpu/assign.cpp +++ b/src/backend/cpu/assign.cpp @@ -66,6 +66,7 @@ INSTANTIATE(uintl) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(int) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/cpu/bilateral.cpp b/src/backend/cpu/bilateral.cpp index 027afb2c3b..19af80f3cb 100644 --- a/src/backend/cpu/bilateral.cpp +++ b/src/backend/cpu/bilateral.cpp @@ -38,6 +38,7 @@ INSTANTIATE(float, float) INSTANTIATE(char, float) INSTANTIATE(int, float) INSTANTIATE(uint, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(short, float) INSTANTIATE(ushort, float) diff --git a/src/backend/cpu/binary.hpp b/src/backend/cpu/binary.hpp index 3d130ba520..8d28501053 100644 --- a/src/backend/cpu/binary.hpp +++ b/src/backend/cpu/binary.hpp @@ -1,5 +1,5 @@ /******************************************************* - * Copyright (c) 2021, ArrayFire + * Copyright (c) 2025, ArrayFire * All rights reserved. * * This file is distributed under 3-clause BSD license. @@ -89,8 +89,7 @@ LOGIC_CPLX_FN(double, af_or_t, ||) template static T __mod(T lhs, T rhs) { - T res = lhs % rhs; - return (res < 0) ? abs(rhs - res) : res; + return lhs % rhs; // Same as other backends } template diff --git a/src/backend/cpu/blas.cpp b/src/backend/cpu/blas.cpp index b7d158eb21..60cd9be655 100644 --- a/src/backend/cpu/blas.cpp +++ b/src/backend/cpu/blas.cpp @@ -219,9 +219,10 @@ toCblasTranspose(af_mat_prop opt) { return out; } -template -void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, - const Array &lhs, const Array &rhs, const T *beta) { +template +void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, + const To *alpha, const Array &lhs, const Array &rhs, + const To *beta) { const CBLAS_TRANSPOSE lOpts = toCblasTranspose(optLhs); const CBLAS_TRANSPOSE rOpts = toCblasTranspose(optRhs); @@ -236,17 +237,17 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, const int K = lDims[aColDim]; const dim4 oDims = out.dims(); - using BT = typename blas_base::type; - using CBT = const typename blas_base::type; + using BT = typename blas_base::type; + using CBT = const typename blas_base::type; - auto alpha_ = scale_type(alpha); - auto beta_ = scale_type(beta); + auto alpha_ = scale_type(alpha); + auto beta_ = scale_type(beta); #ifdef USE_MKL - auto alpha_batched = scale_type(alpha); - auto beta_batched = scale_type(beta); + auto alpha_batched = scale_type(alpha); + auto beta_batched = scale_type(beta); #endif - auto func = [=](Param output, CParam left, CParam right) { + auto func = [=](Param output, CParam left, CParam right) { dim4 lStrides = left.strides(); dim4 rStrides = right.strides(); dim4 oStrides = output.strides(); @@ -255,14 +256,14 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, if (right.dims()[bColDim] == 1) { dim_t incr = (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1]; - gemv_func()( + gemv_func()( CblasColMajor, lOpts, lDims[0], lDims[1], alpha_.getScale(), reinterpret_cast(left.get()), lStrides[1], reinterpret_cast(right.get()), incr, beta_.getScale(), reinterpret_cast(output.get()), oStrides[0]); } else { - gemm_func()( + gemm_func()( CblasColMajor, lOpts, rOpts, M, N, K, alpha_.getScale(), reinterpret_cast(left.get()), lStrides[1], reinterpret_cast(right.get()), rStrides[1], @@ -303,24 +304,24 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, const MKL_INT ldb = rStrides[1]; const MKL_INT ldc = oStrides[1]; - gemm_batch_func()(CblasColMajor, &lOpts, &rOpts, &M, &N, &K, - alpha_batched.getScale(), lptrs.data(), &lda, - rptrs.data(), &ldb, beta_batched.getScale(), - optrs.data(), &ldc, 1, &batchSize); + gemm_batch_func()(CblasColMajor, &lOpts, &rOpts, &M, &N, &K, + alpha_batched.getScale(), lptrs.data(), &lda, + rptrs.data(), &ldb, beta_batched.getScale(), + optrs.data(), &ldc, 1, &batchSize); #else for (int n = 0; n < batchSize; n++) { if (rDims[bColDim] == 1) { dim_t incr = (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1]; - gemv_func()(CblasColMajor, lOpts, lDims[0], lDims[1], - alpha_.getScale(), lptrs[n], lStrides[1], - rptrs[n], incr, beta_.getScale(), optrs[n], - oStrides[0]); + gemv_func()(CblasColMajor, lOpts, lDims[0], lDims[1], + alpha_.getScale(), lptrs[n], lStrides[1], + rptrs[n], incr, beta_.getScale(), optrs[n], + oStrides[0]); } else { - gemm_func()(CblasColMajor, lOpts, rOpts, M, N, K, - alpha_.getScale(), lptrs[n], lStrides[1], - rptrs[n], rStrides[1], beta_.getScale(), - optrs[n], oStrides[1]); + gemm_func()(CblasColMajor, lOpts, rOpts, M, N, K, + alpha_.getScale(), lptrs[n], lStrides[1], + rptrs[n], rStrides[1], beta_.getScale(), + optrs[n], oStrides[1]); } } #endif @@ -341,6 +342,14 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, copyArray(out, outArr); } +template<> +void gemm(Array &out, af_mat_prop optLhs, + af_mat_prop optRhs, const float *alpha, + const Array &lhs, const Array &rhs, + const float *beta) { + TYPE_ERROR(3, af_dtype::s8); +} + template Array dot(const Array &lhs, const Array &rhs, af_mat_prop optLhs, af_mat_prop optRhs) { diff --git a/src/backend/cpu/blas.hpp b/src/backend/cpu/blas.hpp index 1043a567e9..c16916dafb 100644 --- a/src/backend/cpu/blas.hpp +++ b/src/backend/cpu/blas.hpp @@ -13,9 +13,10 @@ namespace arrayfire { namespace cpu { -template -void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, - const Array &lhs, const Array &rhs, const T *beta); +template +void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, + const To *alpha, const Array &lhs, const Array &rhs, + const To *beta); template Array matmul(const Array &lhs, const Array &rhs, af_mat_prop optLhs, diff --git a/src/backend/cpu/cast.hpp b/src/backend/cpu/cast.hpp index dd756eb2b3..d51b7838b8 100644 --- a/src/backend/cpu/cast.hpp +++ b/src/backend/cpu/cast.hpp @@ -150,6 +150,7 @@ struct UnOp, std::complex, af_cast_t> { CAST_B8(float) CAST_B8(double) CAST_B8(int) +CAST_B8(schar) CAST_B8(uchar) CAST_B8(char) diff --git a/src/backend/cpu/convolve.cpp b/src/backend/cpu/convolve.cpp index 20138fd9e5..2fd0e3bce3 100644 --- a/src/backend/cpu/convolve.cpp +++ b/src/backend/cpu/convolve.cpp @@ -111,6 +111,7 @@ INSTANTIATE(double, double) INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(char, float) INSTANTIATE(ushort, float) diff --git a/src/backend/cpu/copy.cpp b/src/backend/cpu/copy.cpp index b1d0985680..ea98c0f613 100644 --- a/src/backend/cpu/copy.cpp +++ b/src/backend/cpu/copy.cpp @@ -72,6 +72,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) @@ -101,6 +102,8 @@ INSTANTIATE(half) Array const &src); \ template void copyArray(Array & dst, \ Array const &src); \ + template void copyArray(Array & dst, \ + Array const &src); \ template void copyArray(Array & dst, \ Array const &src); \ template void copyArray(Array & dst, \ @@ -114,6 +117,7 @@ INSTANTIATE_COPY_ARRAY(int) INSTANTIATE_COPY_ARRAY(uint) INSTANTIATE_COPY_ARRAY(intl) INSTANTIATE_COPY_ARRAY(uintl) +INSTANTIATE_COPY_ARRAY(schar) INSTANTIATE_COPY_ARRAY(uchar) INSTANTIATE_COPY_ARRAY(char) INSTANTIATE_COPY_ARRAY(ushort) @@ -144,6 +148,7 @@ INSTANTIATE_GETSCALAR(cfloat) INSTANTIATE_GETSCALAR(cdouble) INSTANTIATE_GETSCALAR(int) INSTANTIATE_GETSCALAR(uint) +INSTANTIATE_GETSCALAR(schar) INSTANTIATE_GETSCALAR(uchar) INSTANTIATE_GETSCALAR(char) INSTANTIATE_GETSCALAR(intl) diff --git a/src/backend/cpu/diagonal.cpp b/src/backend/cpu/diagonal.cpp index eddd8c0a49..1767096ed0 100644 --- a/src/backend/cpu/diagonal.cpp +++ b/src/backend/cpu/diagonal.cpp @@ -62,6 +62,7 @@ INSTANTIATE_DIAGONAL(uint) INSTANTIATE_DIAGONAL(intl) INSTANTIATE_DIAGONAL(uintl) INSTANTIATE_DIAGONAL(char) +INSTANTIATE_DIAGONAL(schar) INSTANTIATE_DIAGONAL(uchar) INSTANTIATE_DIAGONAL(short) INSTANTIATE_DIAGONAL(ushort) diff --git a/src/backend/cpu/diff.cpp b/src/backend/cpu/diff.cpp index 8e9c67cae1..f9ced50f52 100644 --- a/src/backend/cpu/diff.cpp +++ b/src/backend/cpu/diff.cpp @@ -56,6 +56,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/cpu/err_cpu.hpp b/src/backend/cpu/err_cpu.hpp index d618cecb1e..58c7b59aab 100644 --- a/src/backend/cpu/err_cpu.hpp +++ b/src/backend/cpu/err_cpu.hpp @@ -11,6 +11,6 @@ #define CPU_NOT_SUPPORTED(message) \ do { \ - throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \ - boost::stacktrace::stacktrace()); \ + throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, "CPU", \ + message, boost::stacktrace::stacktrace()); \ } while (0) diff --git a/src/backend/cpu/exampleFunction.cpp b/src/backend/cpu/exampleFunction.cpp index ee7b847524..3f677bc24b 100644 --- a/src/backend/cpu/exampleFunction.cpp +++ b/src/backend/cpu/exampleFunction.cpp @@ -56,6 +56,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(cfloat) diff --git a/src/backend/cpu/fast.cpp b/src/backend/cpu/fast.cpp index b8ac38eeaf..ac93345797 100644 --- a/src/backend/cpu/fast.cpp +++ b/src/backend/cpu/fast.cpp @@ -120,6 +120,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/fftconvolve.cpp b/src/backend/cpu/fftconvolve.cpp index 728238c1ef..ff2e5b68c4 100644 --- a/src/backend/cpu/fftconvolve.cpp +++ b/src/backend/cpu/fftconvolve.cpp @@ -207,6 +207,7 @@ INSTANTIATE(double) INSTANTIATE(float) INSTANTIATE(uint) INSTANTIATE(int) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(uintl) diff --git a/src/backend/cpu/hist_graphics.cpp b/src/backend/cpu/hist_graphics.cpp index 7635004c91..a77e9fe77e 100644 --- a/src/backend/cpu/hist_graphics.cpp +++ b/src/backend/cpu/hist_graphics.cpp @@ -43,6 +43,7 @@ void copy_histogram(const Array &data, fg_histogram hist) { INSTANTIATE(float) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/histogram.cpp b/src/backend/cpu/histogram.cpp index e2f8e15433..9d9c6ba8fa 100644 --- a/src/backend/cpu/histogram.cpp +++ b/src/backend/cpu/histogram.cpp @@ -48,6 +48,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/identity.cpp b/src/backend/cpu/identity.cpp index 05695d7629..ce7f35bdb0 100644 --- a/src/backend/cpu/identity.cpp +++ b/src/backend/cpu/identity.cpp @@ -42,6 +42,7 @@ INSTANTIATE_IDENTITY(uint) INSTANTIATE_IDENTITY(intl) INSTANTIATE_IDENTITY(uintl) INSTANTIATE_IDENTITY(char) +INSTANTIATE_IDENTITY(schar) INSTANTIATE_IDENTITY(uchar) INSTANTIATE_IDENTITY(short) INSTANTIATE_IDENTITY(ushort) diff --git a/src/backend/cpu/image.cpp b/src/backend/cpu/image.cpp index f11a2db4ca..2e24dec9be 100644 --- a/src/backend/cpu/image.cpp +++ b/src/backend/cpu/image.cpp @@ -49,6 +49,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/cpu/index.cpp b/src/backend/cpu/index.cpp index 315406b46d..84cff747bd 100644 --- a/src/backend/cpu/index.cpp +++ b/src/backend/cpu/index.cpp @@ -35,7 +35,16 @@ Array index(const Array& in, const af_index_t idxrs[]) { // create seq vector to retrieve output // dimensions, offsets & offsets for (unsigned x = 0; x < isSeq.size(); ++x) { - if (idxrs[x].isSeq) { seqs[x] = idxrs[x].idx.seq; } + if (idxrs[x].isSeq) { + af_seq seq = idxrs[x].idx.seq; + // Handle af_span as a sequence that covers the complete axis + if (seq.begin == af_span.begin && seq.end == af_span.end && + seq.step == af_span.step) { + seqs[x] = af_seq{0, (double)(in.dims()[x] - 1), 1}; + } else { + seqs[x] = seq; + } + } isSeq[x] = idxrs[x].isSeq; } @@ -72,6 +81,7 @@ INSTANTIATE(uintl) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(int) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/cpu/iota.cpp b/src/backend/cpu/iota.cpp index 1e7155bcd9..fe50919783 100644 --- a/src/backend/cpu/iota.cpp +++ b/src/backend/cpu/iota.cpp @@ -41,6 +41,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/ireduce.cpp b/src/backend/cpu/ireduce.cpp index 435d6ea44d..b87c12bc87 100644 --- a/src/backend/cpu/ireduce.cpp +++ b/src/backend/cpu/ireduce.cpp @@ -58,11 +58,13 @@ void rreduce(Array &out, Array &loc, const Array &in, const int dim, template T ireduce_all(unsigned *loc, const Array &in) { + in.eval(); getQueue().sync(); af::dim4 dims = in.dims(); af::dim4 strides = in.strides(); const T *inPtr = in.get(); + dim_t idx = 0; kernel::MinMaxOp Op(inPtr[0], 0); @@ -76,8 +78,8 @@ T ireduce_all(unsigned *loc, const Array &in) { dim_t off1 = j * strides[1]; for (dim_t i = 0; i < dims[0]; i++) { - dim_t idx = i + off1 + off2 + off3; - Op(inPtr[idx], idx); + dim_t d_idx = i + off1 + off2 + off3; + Op(inPtr[d_idx], idx++); } } } @@ -105,6 +107,7 @@ INSTANTIATE(af_min_t, uint) INSTANTIATE(af_min_t, intl) INSTANTIATE(af_min_t, uintl) INSTANTIATE(af_min_t, char) +INSTANTIATE(af_min_t, schar) INSTANTIATE(af_min_t, uchar) INSTANTIATE(af_min_t, short) INSTANTIATE(af_min_t, ushort) @@ -120,6 +123,7 @@ INSTANTIATE(af_max_t, uint) INSTANTIATE(af_max_t, intl) INSTANTIATE(af_max_t, uintl) INSTANTIATE(af_max_t, char) +INSTANTIATE(af_max_t, schar) INSTANTIATE(af_max_t, uchar) INSTANTIATE(af_max_t, short) INSTANTIATE(af_max_t, ushort) diff --git a/src/backend/cpu/jit/BinaryNode.hpp b/src/backend/cpu/jit/BinaryNode.hpp index 8c1cc39d68..424e37a63f 100644 --- a/src/backend/cpu/jit/BinaryNode.hpp +++ b/src/backend/cpu/jit/BinaryNode.hpp @@ -32,7 +32,7 @@ class BinaryNode : public TNode> { BinaryNode(common::Node_ptr lhs, common::Node_ptr rhs) : TNode>(compute_t(0), std::max(lhs->getHeight(), rhs->getHeight()) + 1, - {{lhs, rhs}}) {} + {{lhs, rhs}}, common::kNodeType::Nary) {} std::unique_ptr clone() final { return std::make_unique(*this); @@ -71,7 +71,8 @@ class BinaryNode : public TNode> { } int setArgs(int start_id, bool is_linear, - std::function + std::function setArg) const override { UNUSED(is_linear); UNUSED(setArg); diff --git a/src/backend/cpu/jit/BufferNode.hpp b/src/backend/cpu/jit/BufferNode.hpp index e6be492b7f..ca3cfe7bb5 100644 --- a/src/backend/cpu/jit/BufferNode.hpp +++ b/src/backend/cpu/jit/BufferNode.hpp @@ -35,7 +35,7 @@ class BufferNode : public TNode { public: BufferNode() - : TNode(T(0), 0, {}) + : TNode(T(0), 0, {}, common::kNodeType::Buffer) , m_bytes(0) , m_strides{0, 0, 0, 0} , m_dims{0, 0, 0, 0} @@ -119,7 +119,8 @@ class BufferNode : public TNode { } int setArgs(int start_id, bool is_linear, - std::function + std::function setArg) const override { UNUSED(is_linear); UNUSED(setArg); @@ -145,8 +146,6 @@ class BufferNode : public TNode { dims[3] == m_dims[3]; } - bool isBuffer() const final { return true; } - size_t getHash() const noexcept final { std::hash ptr_hash; std::hash aftype_hash; @@ -176,6 +175,19 @@ class BufferNode : public TNode { } return false; } + + virtual void modDims(const af::dim4 &newDim) override { + af::dim4 strides(1, 1, 1, 1); + for(dim_t i = 1; i < 4; ++i) { + strides[i] = strides[i - 1] * newDim[i - 1]; + } + + for(dim_t i = 0; i < 4; ++i) { + m_dims[i] = newDim[i]; + m_strides[i] = strides[i]; + } + } + }; } // namespace jit diff --git a/src/backend/cpu/jit/Node.hpp b/src/backend/cpu/jit/Node.hpp index b3914cbc70..c40b0adf92 100644 --- a/src/backend/cpu/jit/Node.hpp +++ b/src/backend/cpu/jit/Node.hpp @@ -43,9 +43,10 @@ class TNode : public common::Node { public: TNode(T val, const int height, - const std::array &&children) + const std::array &&children, + common::kNodeType node_type) : Node(static_cast(af::dtype_traits::af_type), height, - move(children)) { + move(children), node_type) { using namespace common; m_val.fill(static_cast>(val)); } diff --git a/src/backend/cpu/jit/ScalarNode.hpp b/src/backend/cpu/jit/ScalarNode.hpp index a6d7eff5df..0b119deb82 100644 --- a/src/backend/cpu/jit/ScalarNode.hpp +++ b/src/backend/cpu/jit/ScalarNode.hpp @@ -20,7 +20,7 @@ namespace jit { template class ScalarNode : public TNode { public: - ScalarNode(T val) : TNode(val, 0, {}) {} + ScalarNode(T val) : TNode(val, 0, {}, common::kNodeType::Scalar) {} std::unique_ptr clone() final { return std::make_unique(*this); @@ -40,7 +40,8 @@ class ScalarNode : public TNode { } int setArgs(int start_id, bool is_linear, - std::function + std::function setArg) const override { UNUSED(is_linear); UNUSED(setArg); @@ -59,8 +60,6 @@ class ScalarNode : public TNode { UNUSED(kerStream); UNUSED(ids); } - - bool isScalar() const final { return true; } }; } // namespace jit } // namespace cpu diff --git a/src/backend/cpu/jit/UnaryNode.hpp b/src/backend/cpu/jit/UnaryNode.hpp index 9ae8e0aa94..5ca37ca8f4 100644 --- a/src/backend/cpu/jit/UnaryNode.hpp +++ b/src/backend/cpu/jit/UnaryNode.hpp @@ -34,7 +34,8 @@ class UnaryNode : public TNode { public: UnaryNode(common::Node_ptr child) - : TNode(To(0), child->getHeight() + 1, {{child}}) {} + : TNode(To(0), child->getHeight() + 1, {{child}}, + common::kNodeType::Nary) {} std::unique_ptr clone() final { return std::make_unique(*this); diff --git a/src/backend/cpu/join.cpp b/src/backend/cpu/join.cpp index e9fed65df1..602f2db7f9 100644 --- a/src/backend/cpu/join.cpp +++ b/src/backend/cpu/join.cpp @@ -70,6 +70,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) @@ -90,6 +91,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/cpu/kernel/index.hpp b/src/backend/cpu/kernel/index.hpp index 2a6a6d9bc4..962b0713dc 100644 --- a/src/backend/cpu/kernel/index.hpp +++ b/src/backend/cpu/kernel/index.hpp @@ -34,25 +34,27 @@ void index(Param out, CParam in, const af::dim4 dDims, for (dim_t l = 0; l < oDims[3]; ++l) { dim_t lOff = l * oStrides[3]; - dim_t inIdx3 = trimIndex(isSeq[3] ? l + iOffs[3] : ptr3[l], iDims[3]); + dim_t inIdx3 = trimIndex( + isSeq[3] ? l * seqs[3].step + iOffs[3] : ptr3[l], iDims[3]); dim_t inOff3 = inIdx3 * iStrds[3]; for (dim_t k = 0; k < oDims[2]; ++k) { - dim_t kOff = k * oStrides[2]; - dim_t inIdx2 = - trimIndex(isSeq[2] ? k + iOffs[2] : ptr2[k], iDims[2]); + dim_t kOff = k * oStrides[2]; + dim_t inIdx2 = trimIndex( + isSeq[2] ? k * seqs[2].step + iOffs[2] : ptr2[k], iDims[2]); dim_t inOff2 = inIdx2 * iStrds[2]; for (dim_t j = 0; j < oDims[1]; ++j) { - dim_t jOff = j * oStrides[1]; - dim_t inIdx1 = - trimIndex(isSeq[1] ? j + iOffs[1] : ptr1[j], iDims[1]); + dim_t jOff = j * oStrides[1]; + dim_t inIdx1 = trimIndex( + isSeq[1] ? j * seqs[1].step + iOffs[1] : ptr1[j], iDims[1]); dim_t inOff1 = inIdx1 * iStrds[1]; for (dim_t i = 0; i < oDims[0]; ++i) { - dim_t iOff = i * oStrides[0]; - dim_t inIdx0 = - trimIndex(isSeq[0] ? i + iOffs[0] : ptr0[i], iDims[0]); + dim_t iOff = i * oStrides[0]; + dim_t inIdx0 = trimIndex( + isSeq[0] ? i * seqs[0].step + iOffs[0] : ptr0[i], + iDims[0]); dim_t inOff0 = inIdx0 * iStrds[0]; dst[lOff + kOff + jOff + iOff] = diff --git a/src/backend/cpu/kernel/ireduce.hpp b/src/backend/cpu/kernel/ireduce.hpp index 9c371498c7..9d2598af4b 100644 --- a/src/backend/cpu/kernel/ireduce.hpp +++ b/src/backend/cpu/kernel/ireduce.hpp @@ -10,7 +10,9 @@ #pragma once #include #include +#include #include +#include namespace arrayfire { namespace cpu { @@ -23,16 +25,13 @@ double cabs(const T in) { static double cabs(const char in) { return (double)(in > 0); } static double cabs(const cfloat &in) { return (double)abs(in); } static double cabs(const cdouble &in) { return (double)abs(in); } -template -static bool is_nan(T in) { - return in != in; -} template struct MinMaxOp { T m_val; uint m_idx; MinMaxOp(T val, uint idx) : m_val(val), m_idx(idx) { + using arrayfire::cpu::is_nan; if (is_nan(val)) { m_val = common::Binary::init(); } } @@ -50,6 +49,7 @@ struct MinMaxOp { T m_val; uint m_idx; MinMaxOp(T val, uint idx) : m_val(val), m_idx(idx) { + using arrayfire::cpu::is_nan; if (is_nan(val)) { m_val = common::Binary::init(); } } diff --git a/src/backend/cpu/kernel/random_engine.hpp b/src/backend/cpu/kernel/random_engine.hpp index 09c2bff20c..0ab49f8a80 100644 --- a/src/backend/cpu/kernel/random_engine.hpp +++ b/src/backend/cpu/kernel/random_engine.hpp @@ -115,6 +115,11 @@ uchar transform(uint *val, uint index) { return v; } +template<> +schar transform(uint *val, uint index) { + return transform(val, index); +} + template<> ushort transform(uint *val, uint index) { ushort v = val[index >> 1U] >> (16U * (index & 1U)) & 0x0000ffff; diff --git a/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp b/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp index 6ac6875f3e..5873e93117 100644 --- a/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp +++ b/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp @@ -9,7 +9,7 @@ #include -// SBK_TYPES:float double int uint intl uintl short ushort char uchar +// SBK_TYPES:float double int uint intl uintl short ushort char schar uchar namespace arrayfire { namespace cpu { diff --git a/src/backend/cpu/kernel/sort_by_key_impl.hpp b/src/backend/cpu/kernel/sort_by_key_impl.hpp index acd7524a9b..e77e868d78 100644 --- a/src/backend/cpu/kernel/sort_by_key_impl.hpp +++ b/src/backend/cpu/kernel/sort_by_key_impl.hpp @@ -169,6 +169,7 @@ void sort0ByKey(Param okey, Param oval, bool isAscending) { INSTANTIATE(Tk, short) \ INSTANTIATE(Tk, ushort) \ INSTANTIATE(Tk, char) \ + INSTANTIATE(Tk, schar) \ INSTANTIATE(Tk, uchar) \ INSTANTIATE(Tk, intl) \ INSTANTIATE(Tk, uintl) diff --git a/src/backend/cpu/lookup.cpp b/src/backend/cpu/lookup.cpp index 8a5c40d55c..b8c56e297c 100644 --- a/src/backend/cpu/lookup.cpp +++ b/src/backend/cpu/lookup.cpp @@ -51,6 +51,8 @@ Array lookup(const Array &input, const Array &indices, const unsigned); \ template Array lookup(const Array &, const Array &, \ const unsigned); \ + template Array lookup(const Array &, const Array &, \ + const unsigned); \ template Array lookup(const Array &, const Array &, \ const unsigned); \ template Array lookup(const Array &, const Array &, \ @@ -64,6 +66,7 @@ INSTANTIATE(int); INSTANTIATE(unsigned); INSTANTIATE(intl); INSTANTIATE(uintl); +INSTANTIATE(schar); INSTANTIATE(uchar); INSTANTIATE(char); INSTANTIATE(ushort); diff --git a/src/backend/cpu/match_template.cpp b/src/backend/cpu/match_template.cpp index d3cfb26b4a..6b4d0f1b91 100644 --- a/src/backend/cpu/match_template.cpp +++ b/src/backend/cpu/match_template.cpp @@ -51,6 +51,7 @@ INSTANTIATE(float, float) INSTANTIATE(char, float) INSTANTIATE(int, float) INSTANTIATE(uint, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(short, float) INSTANTIATE(ushort, float) diff --git a/src/backend/cpu/math.hpp b/src/backend/cpu/math.hpp index d2735acd2a..06c1027edf 100644 --- a/src/backend/cpu/math.hpp +++ b/src/backend/cpu/math.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -42,6 +43,36 @@ static inline T max(T lhs, T rhs) { cfloat max(cfloat lhs, cfloat rhs); cdouble max(cdouble lhs, cdouble rhs); +template +static inline auto is_nan(const T &val) -> bool { + return false; +} + +template<> +inline auto is_nan(const float &val) -> bool { + return std::isnan(val); +} + +template<> +inline auto is_nan(const double &val) -> bool { + return std::isnan(val); +} + +template<> +inline auto is_nan(const common::half &val) -> bool { + return isnan(val); +} + +template<> +inline auto is_nan(const cfloat &in) -> bool { + return std::isnan(real(in)) || std::isnan(imag(in)); +} + +template<> +inline auto is_nan(const cdouble &in) -> bool { + return std::isnan(real(in)) || std::isnan(imag(in)); +} + template static inline T division(T lhs, double rhs) { return lhs / rhs; diff --git a/src/backend/cpu/mean.cpp b/src/backend/cpu/mean.cpp index 6a256113f7..2323442110 100644 --- a/src/backend/cpu/mean.cpp +++ b/src/backend/cpu/mean.cpp @@ -141,6 +141,7 @@ INSTANTIATE(intl, double, double); INSTANTIATE(uintl, double, double); INSTANTIATE(short, float, float); INSTANTIATE(ushort, float, float); +INSTANTIATE(schar, float, float); INSTANTIATE(uchar, float, float); INSTANTIATE(char, float, float); INSTANTIATE(cfloat, float, cfloat); diff --git a/src/backend/cpu/meanshift.cpp b/src/backend/cpu/meanshift.cpp index d52b56a99e..878aa4cacb 100644 --- a/src/backend/cpu/meanshift.cpp +++ b/src/backend/cpu/meanshift.cpp @@ -50,6 +50,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/medfilt.cpp b/src/backend/cpu/medfilt.cpp index 53497be8c9..4c952fc762 100644 --- a/src/backend/cpu/medfilt.cpp +++ b/src/backend/cpu/medfilt.cpp @@ -63,6 +63,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(ushort) INSTANTIATE(short) diff --git a/src/backend/cpu/memory.cpp b/src/backend/cpu/memory.cpp index 9bbb41d458..0a32186f2e 100644 --- a/src/backend/cpu/memory.cpp +++ b/src/backend/cpu/memory.cpp @@ -106,6 +106,7 @@ INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/cpu/moments.cpp b/src/backend/cpu/moments.cpp index bd5c520eac..09db606bd4 100644 --- a/src/backend/cpu/moments.cpp +++ b/src/backend/cpu/moments.cpp @@ -49,6 +49,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/cpu/morph.cpp b/src/backend/cpu/morph.cpp index add13de416..e526e7c066 100644 --- a/src/backend/cpu/morph.cpp +++ b/src/backend/cpu/morph.cpp @@ -67,6 +67,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(ushort) INSTANTIATE(short) diff --git a/src/backend/cpu/nearest_neighbour.cpp b/src/backend/cpu/nearest_neighbour.cpp index 2979090dd9..0581e97ab6 100644 --- a/src/backend/cpu/nearest_neighbour.cpp +++ b/src/backend/cpu/nearest_neighbour.cpp @@ -67,6 +67,7 @@ INSTANTIATE(int, int) INSTANTIATE(uint, uint) INSTANTIATE(intl, intl) INSTANTIATE(uintl, uintl) +INSTANTIATE(schar, int) INSTANTIATE(uchar, uint) INSTANTIATE(ushort, uint) INSTANTIATE(short, int) diff --git a/src/backend/cpu/plot.cpp b/src/backend/cpu/plot.cpp index abf1a7b397..1ca6ae7882 100644 --- a/src/backend/cpu/plot.cpp +++ b/src/backend/cpu/plot.cpp @@ -46,6 +46,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/queue.hpp b/src/backend/cpu/queue.hpp index 594396a78e..cdcfb8092f 100644 --- a/src/backend/cpu/queue.hpp +++ b/src/backend/cpu/queue.hpp @@ -38,6 +38,42 @@ class queue_impl { } }; +class event_impl { + public: + event_impl() noexcept = default; + ~event_impl() noexcept = default; + explicit event_impl(const event_impl &other) = default; + event_impl(event_impl &&other) noexcept = default; + event_impl &operator=(event_impl &&other) noexcept = default; + event_impl &operator=(event_impl &other) noexcept = default; + + explicit event_impl(const int val) {} + + event_impl &operator=(int val) noexcept { return *this; } + + int create() { + AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); + return 0; + } + + int mark(queue_impl &queue) { + AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); + return 0; + } + + int wait(queue_impl &queue) const { + AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); + return 0; + } + + int sync() const noexcept { + AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); + return 0; + } + + operator bool() const noexcept { return false; } +}; + #else #include diff --git a/src/backend/cpu/random_engine.cpp b/src/backend/cpu/random_engine.cpp index 3e1c8745c8..d42a7bdae1 100644 --- a/src/backend/cpu/random_engine.cpp +++ b/src/backend/cpu/random_engine.cpp @@ -149,6 +149,7 @@ INSTANTIATE_UNIFORM(uint) INSTANTIATE_UNIFORM(intl) INSTANTIATE_UNIFORM(uintl) INSTANTIATE_UNIFORM(char) +INSTANTIATE_UNIFORM(schar) INSTANTIATE_UNIFORM(uchar) INSTANTIATE_UNIFORM(short) INSTANTIATE_UNIFORM(ushort) diff --git a/src/backend/cpu/range.cpp b/src/backend/cpu/range.cpp index 3b782837e0..ad100da4d4 100644 --- a/src/backend/cpu/range.cpp +++ b/src/backend/cpu/range.cpp @@ -54,6 +54,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(ushort) INSTANTIATE(short) diff --git a/src/backend/cpu/reduce.cpp b/src/backend/cpu/reduce.cpp index 6ce141b316..5b13d6f96f 100644 --- a/src/backend/cpu/reduce.cpp +++ b/src/backend/cpu/reduce.cpp @@ -145,6 +145,7 @@ INSTANTIATE(af_min_t, uint, uint) INSTANTIATE(af_min_t, intl, intl) INSTANTIATE(af_min_t, uintl, uintl) INSTANTIATE(af_min_t, char, char) +INSTANTIATE(af_min_t, schar, schar) INSTANTIATE(af_min_t, uchar, uchar) INSTANTIATE(af_min_t, short, short) INSTANTIATE(af_min_t, ushort, ushort) @@ -160,6 +161,7 @@ INSTANTIATE(af_max_t, uint, uint) INSTANTIATE(af_max_t, intl, intl) INSTANTIATE(af_max_t, uintl, uintl) INSTANTIATE(af_max_t, char, char) +INSTANTIATE(af_max_t, schar, schar) INSTANTIATE(af_max_t, uchar, uchar) INSTANTIATE(af_max_t, short, short) INSTANTIATE(af_max_t, ushort, ushort) @@ -180,6 +182,8 @@ INSTANTIATE(af_add_t, uintl, uintl) INSTANTIATE(af_add_t, uintl, double) INSTANTIATE(af_add_t, char, int) INSTANTIATE(af_add_t, char, float) +INSTANTIATE(af_add_t, schar, int) +INSTANTIATE(af_add_t, schar, float) INSTANTIATE(af_add_t, uchar, uint) INSTANTIATE(af_add_t, uchar, float) INSTANTIATE(af_add_t, short, int) @@ -199,6 +203,7 @@ INSTANTIATE(af_mul_t, uint, uint) INSTANTIATE(af_mul_t, intl, intl) INSTANTIATE(af_mul_t, uintl, uintl) INSTANTIATE(af_mul_t, char, int) +INSTANTIATE(af_mul_t, schar, int) INSTANTIATE(af_mul_t, uchar, uint) INSTANTIATE(af_mul_t, short, int) INSTANTIATE(af_mul_t, ushort, uint) @@ -214,6 +219,7 @@ INSTANTIATE(af_notzero_t, uint, uint) INSTANTIATE(af_notzero_t, intl, uint) INSTANTIATE(af_notzero_t, uintl, uint) INSTANTIATE(af_notzero_t, char, uint) +INSTANTIATE(af_notzero_t, schar, uint) INSTANTIATE(af_notzero_t, uchar, uint) INSTANTIATE(af_notzero_t, short, uint) INSTANTIATE(af_notzero_t, ushort, uint) @@ -229,6 +235,7 @@ INSTANTIATE(af_or_t, uint, char) INSTANTIATE(af_or_t, intl, char) INSTANTIATE(af_or_t, uintl, char) INSTANTIATE(af_or_t, char, char) +INSTANTIATE(af_or_t, schar, char) INSTANTIATE(af_or_t, uchar, char) INSTANTIATE(af_or_t, short, char) INSTANTIATE(af_or_t, ushort, char) @@ -244,6 +251,7 @@ INSTANTIATE(af_and_t, uint, char) INSTANTIATE(af_and_t, intl, char) INSTANTIATE(af_and_t, uintl, char) INSTANTIATE(af_and_t, char, char) +INSTANTIATE(af_and_t, schar, char) INSTANTIATE(af_and_t, uchar, char) INSTANTIATE(af_and_t, short, char) INSTANTIATE(af_and_t, ushort, char) diff --git a/src/backend/cpu/reorder.cpp b/src/backend/cpu/reorder.cpp index 67233542bd..dd0a43ccac 100644 --- a/src/backend/cpu/reorder.cpp +++ b/src/backend/cpu/reorder.cpp @@ -39,6 +39,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) diff --git a/src/backend/cpu/reshape.cpp b/src/backend/cpu/reshape.cpp index b2d46eb066..31a0053684 100644 --- a/src/backend/cpu/reshape.cpp +++ b/src/backend/cpu/reshape.cpp @@ -40,6 +40,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) @@ -68,6 +69,8 @@ INSTANTIATE(ushort) const dim4 &, short, double); \ template Array reshape( \ const Array &, const dim4 &, ushort, double); \ + template Array reshape(const Array &, \ + const dim4 &, schar, double); \ template Array reshape(const Array &, \ const dim4 &, uchar, double); \ template Array reshape(const Array &, \ @@ -79,6 +82,7 @@ INSTANTIATE_PAD_ARRAY(int) INSTANTIATE_PAD_ARRAY(uint) INSTANTIATE_PAD_ARRAY(intl) INSTANTIATE_PAD_ARRAY(uintl) +INSTANTIATE_PAD_ARRAY(schar) INSTANTIATE_PAD_ARRAY(uchar) INSTANTIATE_PAD_ARRAY(char) INSTANTIATE_PAD_ARRAY(ushort) diff --git a/src/backend/cpu/resize.cpp b/src/backend/cpu/resize.cpp index 4f899d89d8..ffc473fd4e 100644 --- a/src/backend/cpu/resize.cpp +++ b/src/backend/cpu/resize.cpp @@ -53,6 +53,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cpu/rotate.cpp b/src/backend/cpu/rotate.cpp index 0e9806a2af..bed34b7bf3 100644 --- a/src/backend/cpu/rotate.cpp +++ b/src/backend/cpu/rotate.cpp @@ -53,6 +53,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cpu/scan.cpp b/src/backend/cpu/scan.cpp index af5c4d9efe..7f6843f99a 100644 --- a/src/backend/cpu/scan.cpp +++ b/src/backend/cpu/scan.cpp @@ -84,6 +84,7 @@ Array scan(const Array& in, const int dim, bool inclusive_scan) { INSTANTIATE_SCAN(ROp, uintl, uintl) \ INSTANTIATE_SCAN(ROp, char, int) \ INSTANTIATE_SCAN(ROp, char, uint) \ + INSTANTIATE_SCAN(ROp, schar, int) \ INSTANTIATE_SCAN(ROp, uchar, uint) \ INSTANTIATE_SCAN(ROp, short, int) \ INSTANTIATE_SCAN(ROp, ushort, uint) diff --git a/src/backend/cpu/select.cpp b/src/backend/cpu/select.cpp index 96849cecd1..8258cae47a 100644 --- a/src/backend/cpu/select.cpp +++ b/src/backend/cpu/select.cpp @@ -51,6 +51,7 @@ INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/set.cpp b/src/backend/cpu/set.cpp index 838ad7675e..6db13c8760 100644 --- a/src/backend/cpu/set.cpp +++ b/src/backend/cpu/set.cpp @@ -120,6 +120,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/shift.cpp b/src/backend/cpu/shift.cpp index f8942f641f..d812cbde89 100644 --- a/src/backend/cpu/shift.cpp +++ b/src/backend/cpu/shift.cpp @@ -37,6 +37,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cpu/sobel.cpp b/src/backend/cpu/sobel.cpp index 68bddee784..5708348295 100644 --- a/src/backend/cpu/sobel.cpp +++ b/src/backend/cpu/sobel.cpp @@ -44,6 +44,7 @@ INSTANTIATE(double, double) INSTANTIATE(int, int) INSTANTIATE(uint, int) INSTANTIATE(char, int) +INSTANTIATE(schar, int) INSTANTIATE(uchar, int) INSTANTIATE(short, int) INSTANTIATE(ushort, int) diff --git a/src/backend/cpu/sort.cpp b/src/backend/cpu/sort.cpp index e5067a8dba..41c6b75147 100644 --- a/src/backend/cpu/sort.cpp +++ b/src/backend/cpu/sort.cpp @@ -98,6 +98,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/sort_by_key.cpp b/src/backend/cpu/sort_by_key.cpp index 169b598558..efe8eba2f1 100644 --- a/src/backend/cpu/sort_by_key.cpp +++ b/src/backend/cpu/sort_by_key.cpp @@ -71,6 +71,7 @@ void sort_by_key(Array &okey, Array &oval, const Array &ikey, INSTANTIATE(Tk, int) \ INSTANTIATE(Tk, uint) \ INSTANTIATE(Tk, char) \ + INSTANTIATE(Tk, schar) \ INSTANTIATE(Tk, uchar) \ INSTANTIATE(Tk, short) \ INSTANTIATE(Tk, ushort) \ @@ -82,6 +83,7 @@ INSTANTIATE1(double) INSTANTIATE1(int) INSTANTIATE1(uint) INSTANTIATE1(char) +INSTANTIATE1(schar) INSTANTIATE1(uchar) INSTANTIATE1(short) INSTANTIATE1(ushort) diff --git a/src/backend/cpu/sort_index.cpp b/src/backend/cpu/sort_index.cpp index cec724c85d..8b1f4a1319 100644 --- a/src/backend/cpu/sort_index.cpp +++ b/src/backend/cpu/sort_index.cpp @@ -75,6 +75,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/surface.cpp b/src/backend/cpu/surface.cpp index e861dbeac7..d86bd6f469 100644 --- a/src/backend/cpu/surface.cpp +++ b/src/backend/cpu/surface.cpp @@ -47,6 +47,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/susan.cpp b/src/backend/cpu/susan.cpp index 6ab2bfba78..c5321deb16 100644 --- a/src/backend/cpu/susan.cpp +++ b/src/backend/cpu/susan.cpp @@ -73,6 +73,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/tile.cpp b/src/backend/cpu/tile.cpp index d2a8d3ab7c..884bfed40d 100644 --- a/src/backend/cpu/tile.cpp +++ b/src/backend/cpu/tile.cpp @@ -47,6 +47,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cpu/transform.cpp b/src/backend/cpu/transform.cpp index 9a57424250..0fbe10ea5c 100644 --- a/src/backend/cpu/transform.cpp +++ b/src/backend/cpu/transform.cpp @@ -8,6 +8,7 @@ ********************************************************/ #include +#include #include #include #include @@ -22,23 +23,27 @@ void transform(Array &out, const Array &in, const Array &tf, const bool perspective) { out.eval(); in.eval(); + + // TODO: Temporary Fix, must fix handling subarrays upstream + // tf has to be linear, although offset is allowed + const Array tf_Lin = tf.isLinear() ? tf : copyArray(tf); tf.eval(); switch (method) { case AF_INTERP_NEAREST: case AF_INTERP_LOWER: - getQueue().enqueue(kernel::transform, out, in, tf, inverse, - perspective, method); + getQueue().enqueue(kernel::transform, out, in, tf_Lin, + inverse, perspective, method); break; case AF_INTERP_BILINEAR: case AF_INTERP_BILINEAR_COSINE: - getQueue().enqueue(kernel::transform, out, in, tf, inverse, - perspective, method); + getQueue().enqueue(kernel::transform, out, in, tf_Lin, + inverse, perspective, method); break; case AF_INTERP_BICUBIC: case AF_INTERP_BICUBIC_SPLINE: - getQueue().enqueue(kernel::transform, out, in, tf, inverse, - perspective, method); + getQueue().enqueue(kernel::transform, out, in, tf_Lin, + inverse, perspective, method); break; default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG); break; } @@ -58,6 +63,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cpu/transpose.cpp b/src/backend/cpu/transpose.cpp index 7cd713afd6..a9f6f9d3d5 100644 --- a/src/backend/cpu/transpose.cpp +++ b/src/backend/cpu/transpose.cpp @@ -51,6 +51,7 @@ INSTANTIATE(cdouble) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/cpu/triangle.cpp b/src/backend/cpu/triangle.cpp index 8e3b0569b2..6c276ca4bd 100644 --- a/src/backend/cpu/triangle.cpp +++ b/src/backend/cpu/triangle.cpp @@ -58,6 +58,7 @@ INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/types.hpp b/src/backend/cpu/types.hpp index 27a678af82..f1f58e7006 100644 --- a/src/backend/cpu/types.hpp +++ b/src/backend/cpu/types.hpp @@ -31,6 +31,7 @@ using cdouble = std::complex; using cfloat = std::complex; using intl = long long; using uint = unsigned int; +using schar = signed char; using uchar = unsigned char; using uintl = unsigned long long; using ushort = unsigned short; diff --git a/src/backend/cpu/unwrap.cpp b/src/backend/cpu/unwrap.cpp index 49086fad49..dca2433ff8 100644 --- a/src/backend/cpu/unwrap.cpp +++ b/src/backend/cpu/unwrap.cpp @@ -55,6 +55,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cpu/vector_field.cpp b/src/backend/cpu/vector_field.cpp index 2a7549de81..efe207be09 100644 --- a/src/backend/cpu/vector_field.cpp +++ b/src/backend/cpu/vector_field.cpp @@ -58,6 +58,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/where.cpp b/src/backend/cpu/where.cpp index 3eb65015f0..30f70efcb0 100644 --- a/src/backend/cpu/where.cpp +++ b/src/backend/cpu/where.cpp @@ -73,6 +73,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cpu/wrap.cpp b/src/backend/cpu/wrap.cpp index d502bc85ad..0c0d397e3f 100644 --- a/src/backend/cpu/wrap.cpp +++ b/src/backend/cpu/wrap.cpp @@ -49,6 +49,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp index db03d1b3e5..e0d5f73f5a 100644 --- a/src/backend/cuda/Array.cpp +++ b/src/backend/cuda/Array.cpp @@ -8,6 +8,7 @@ ********************************************************/ #include +#include #include #include #include @@ -56,6 +57,22 @@ std::shared_ptr> bufferNodePtr() { static_cast(dtype_traits::af_type)); } +template +void checkAndMigrate(Array &arr) { + int arr_id = arr.getDevId(); + int cur_id = detail::getActiveDeviceId(); + if (!isDeviceBufferAccessible(arr_id, cur_id)) { + static auto getLogger = [&] { return spdlog::get("platform"); }; + AF_TRACE("Migrating array from {} to {}.", arr_id, cur_id); + auto migrated_data = memAlloc(arr.elements()); + CUDA_CHECK( + cudaMemcpyPeerAsync(migrated_data.get(), getDeviceNativeId(cur_id), + arr.get(), getDeviceNativeId(arr_id), + arr.elements() * sizeof(T), getActiveStream())); + arr.data.reset(migrated_data.release(), memFree); + } +} + template Array::Array(const af::dim4 &dims) : info(getActiveDeviceId(), dims, 0, calcStrides(dims), @@ -253,8 +270,13 @@ Node_ptr Array::getNode() const { template kJITHeuristics passesJitHeuristics(span root_nodes) { if (!evalFlag()) { return kJITHeuristics::Pass; } + static auto getLogger = [&] { return spdlog::get("jit"); }; for (Node *n : root_nodes) { if (n->getHeight() > static_cast(getMaxJitSize())) { + AF_TRACE( + "JIT tree evaluated because of tree height exceeds limit: {} > " + "{}", + n->getHeight(), getMaxJitSize()); return kJITHeuristics::TreeHeight; } } @@ -313,9 +335,14 @@ kJITHeuristics passesJitHeuristics(span root_nodes) { // should be checking the amount of memory available to guard // this eval if (param_size >= max_param_size) { + AF_TRACE( + "JIT tree evaluated because of kernel parameter size: {} >= {}", + param_size, max_param_size); return kJITHeuristics::KernelParameterSize; } if (jitTreeExceedsMemoryPressure(info.total_buffer_size)) { + AF_TRACE("JIT tree evaluated because of memory pressure: {}", + info.total_buffer_size); return kJITHeuristics::MemoryPressure; } } @@ -457,7 +484,8 @@ void Array::setDataDims(const dim4 &new_dims) { Array & arr, const void *const data, const size_t bytes); \ template void evalMultiple(std::vector *> arrays); \ template kJITHeuristics passesJitHeuristics(span n); \ - template void Array::setDataDims(const dim4 &new_dims); + template void Array::setDataDims(const dim4 &new_dims); \ + template void checkAndMigrate(Array & arr); INSTANTIATE(float) INSTANTIATE(double) @@ -465,6 +493,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) diff --git a/src/backend/cuda/Array.hpp b/src/backend/cuda/Array.hpp index 7e1324d016..82e8bb9583 100644 --- a/src/backend/cuda/Array.hpp +++ b/src/backend/cuda/Array.hpp @@ -34,6 +34,13 @@ using af::dim4; template class Array; +/// Checks if the Array object can be migrated to the current device and if not, +/// an error is thrown +/// +/// \param[in] arr The Array that will be checked. +template +void checkAndMigrate(Array &arr); + template void evalNodes(Param out, common::Node *node); @@ -298,6 +305,7 @@ class Array { friend void destroyArray(Array *arr); friend void *getDevicePtr(const Array &arr); friend void *getRawPtr(const Array &arr); + friend void checkAndMigrate(Array &arr); }; } // namespace cuda diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt index 0dc208fd8b..5085c57717 100644 --- a/src/backend/cuda/CMakeLists.txt +++ b/src/backend/cuda/CMakeLists.txt @@ -51,6 +51,9 @@ set(CUDA_architecture_build_targets "Auto" CACHE find_cuda_helper_libs(nvrtc) find_cuda_helper_libs(nvrtc-builtins) list(APPEND nvrtc_libs ${CUDA_nvrtc_LIBRARY}) +if(UNIX) + list(APPEND nvrtc_libs ${CUDA_nvrtc-builtins_LIBRARY}) +endif() if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS) # The libraries that may be staticly linked or may be loaded at runtime @@ -99,7 +102,12 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS) # contains GPU accelerated stedc and bdsqr. The user has to link # libcusolver_static.a with liblapack_static.a in order to build # successfully. - af_find_static_cuda_libs(lapack_static) + # Cuda Versions >= 12.0 changed lib name to libcusolver_lapack_static.a + if (CUDA_VERSION VERSION_GREATER_EQUAL 12.0) + af_find_static_cuda_libs(cusolver_lapack_static) + else() + af_find_static_cuda_libs(lapack_static) + endif() set(af_cuda_static_flags "${af_cuda_static_flags};-lcusolver_static") else() @@ -142,6 +150,8 @@ set(nvrtc_src ${CUDA_INCLUDE_DIRS}/cuda_fp16.hpp ${CUDA_TOOLKIT_ROOT_DIR}/include/cuComplex.h ${CUDA_TOOLKIT_ROOT_DIR}/include/math_constants.h + ${CUDA_TOOLKIT_ROOT_DIR}/include/vector_types.h + ${CUDA_TOOLKIT_ROOT_DIR}/include/vector_functions.h ${PROJECT_SOURCE_DIR}/src/api/c/optypes.hpp ${PROJECT_SOURCE_DIR}/include/af/defines.h @@ -551,6 +561,7 @@ add_library(afcuda wrap.hpp jit/BufferNode.hpp + jit/ShiftNode.hpp jit/kernel_generators.hpp ${scan_by_key_sources} @@ -781,7 +792,9 @@ function(afcu_collect_libs libname) NAMES "${PX}${libname}64_${lib_major}${SX}" "${PX}${libname}64_${lib_major}${lib_minor}${SX}" + "${PX}${libname}64_${lib_major}0_0${SX}" "${PX}${libname}64_${lib_major}${lib_minor}_0${SX}" + "${PX}${libname}_${lib_major}0_0${SX}" PATHS ${dlib_path_prefix} ) mark_as_advanced(CUDA_${libname}_LIBRARY_DLL) @@ -830,18 +843,24 @@ endfunction() if(AF_INSTALL_STANDALONE) if(AF_WITH_CUDNN) afcu_collect_cudnn_libs("") - if(cuDNN_VERSION_MAJOR VERSION_GREATER 8 OR cuDNN_VERSION_MAJOR VERSION_EQUAL 8) + if(cuDNN_VERSION_MAJOR VERSION_EQUAL 8) # cudnn changed how dlls are shipped starting major version 8 # except the main dll a lot of the other DLLs are loaded upon demand afcu_collect_cudnn_libs(cnn_infer) afcu_collect_cudnn_libs(cnn_train) afcu_collect_cudnn_libs(ops_infer) afcu_collect_cudnn_libs(ops_train) + elseif(cuDNN_VERSION_MAJOR VERSION_GREATER_EQUAL 9) + # infer and train libraries are now combined in version 9 + afcu_collect_cudnn_libs(cnn) + afcu_collect_cudnn_libs(ops) endif() endif() if(WIN32 OR NOT AF_WITH_STATIC_CUDA_NUMERIC_LIBS) - if(CUDA_VERSION_MAJOR VERSION_EQUAL 11) + if(CUDA_VERSION_MAJOR VERSION_EQUAL 12) + afcu_collect_libs(cufft LIB_MAJOR 11 LIB_MINOR 3) + elseif(CUDA_VERSION_MAJOR VERSION_EQUAL 11) afcu_collect_libs(cufft LIB_MAJOR 10 LIB_MINOR 4) else() afcu_collect_libs(cufft) @@ -850,14 +869,25 @@ if(AF_INSTALL_STANDALONE) if(CUDA_VERSION VERSION_GREATER 10.0) afcu_collect_libs(cublasLt) endif() - afcu_collect_libs(cusolver) + if(CUDA_VERSION_MAJOR VERSION_EQUAL 12) + afcu_collect_libs(cusolver LIB_MAJOR 11 LIB_MINOR 7) + else() + afcu_collect_libs(cusolver) + endif() afcu_collect_libs(cusparse) + if(CUDA_VERSION VERSION_GREATER 12.0) + afcu_collect_libs(nvJitLink) + endif() elseif(NOT ${use_static_cuda_lapack}) - afcu_collect_libs(cusolver) + if(CUDA_VERSION_MAJOR VERSION_EQUAL 12) + afcu_collect_libs(cusolver LIB_MAJOR 11 LIB_MINOR 7) + else() + afcu_collect_libs(cusolver) + endif() endif() if(WIN32 OR CUDA_VERSION VERSION_LESS 11.5 OR NOT AF_WITH_STATIC_CUDA_NUMERIC_LIBS) - afcu_collect_libs(nvrtc FULL_VERSION) + afcu_collect_libs(nvrtc) if(CUDA_VERSION VERSION_GREATER 10.0) afcu_collect_libs(nvrtc-builtins FULL_VERSION) else() diff --git a/src/backend/cuda/Param.hpp b/src/backend/cuda/Param.hpp index 817d601eaa..496d4eea68 100644 --- a/src/backend/cuda/Param.hpp +++ b/src/backend/cuda/Param.hpp @@ -35,6 +35,9 @@ class Param { return dims[0] * dims[1] * dims[2] * dims[3]; } + dim_t *dims_ptr() { return dims; } + dim_t *strides_ptr() { return strides; } + Param(const Param &other) noexcept = default; Param(Param &&other) noexcept = default; Param &operator=(const Param &other) noexcept = default; diff --git a/src/backend/cuda/ThrustArrayFirePolicy.hpp b/src/backend/cuda/ThrustArrayFirePolicy.hpp index 189ee558b3..339d3ea088 100644 --- a/src/backend/cuda/ThrustArrayFirePolicy.hpp +++ b/src/backend/cuda/ThrustArrayFirePolicy.hpp @@ -37,7 +37,11 @@ inline void return_temporary_buffer(ThrustArrayFirePolicy, Pointer p) { } // namespace cuda } // namespace arrayfire +#if defined(_WIN32) +THRUST_NAMESPACE_BEGIN +#else namespace thrust { +#endif namespace cuda_cub { template<> __DH__ inline cudaStream_t get_stream( @@ -60,4 +64,8 @@ inline cudaError_t synchronize_stream( } } // namespace cuda_cub +#if defined(_WIN32) +THRUST_NAMESPACE_END +#else } // namespace thrust +#endif diff --git a/src/backend/cuda/all.cu b/src/backend/cuda/all.cu index 3ff42ad599..fa0681dbaf 100644 --- a/src/backend/cuda/all.cu +++ b/src/backend/cuda/all.cu @@ -24,6 +24,7 @@ INSTANTIATE(af_and_t, uint, char) INSTANTIATE(af_and_t, intl, char) INSTANTIATE(af_and_t, uintl, char) INSTANTIATE(af_and_t, char, char) +INSTANTIATE(af_and_t, schar, char) INSTANTIATE(af_and_t, uchar, char) INSTANTIATE(af_and_t, short, char) INSTANTIATE(af_and_t, ushort, char) diff --git a/src/backend/cuda/any.cu b/src/backend/cuda/any.cu index 34092c94d3..801dcb6c10 100644 --- a/src/backend/cuda/any.cu +++ b/src/backend/cuda/any.cu @@ -24,6 +24,7 @@ INSTANTIATE(af_or_t, uint, char) INSTANTIATE(af_or_t, intl, char) INSTANTIATE(af_or_t, uintl, char) INSTANTIATE(af_or_t, char, char) +INSTANTIATE(af_or_t, schar, char) INSTANTIATE(af_or_t, uchar, char) INSTANTIATE(af_or_t, short, char) INSTANTIATE(af_or_t, ushort, char) diff --git a/src/backend/cuda/assign.cpp b/src/backend/cuda/assign.cpp index 67bcbd1291..b65265dc8b 100644 --- a/src/backend/cuda/assign.cpp +++ b/src/backend/cuda/assign.cpp @@ -73,6 +73,7 @@ INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/assign_kernel_param.hpp b/src/backend/cuda/assign_kernel_param.hpp index 0591ca80ad..350893f911 100644 --- a/src/backend/cuda/assign_kernel_param.hpp +++ b/src/backend/cuda/assign_kernel_param.hpp @@ -15,6 +15,7 @@ namespace cuda { typedef struct { int offs[4]; int strds[4]; + int steps[4]; bool isSeq[4]; unsigned int* ptr[4]; } AssignKernelParam; diff --git a/src/backend/cuda/bilateral.cpp b/src/backend/cuda/bilateral.cpp index f9f828018d..6d56640fa8 100644 --- a/src/backend/cuda/bilateral.cpp +++ b/src/backend/cuda/bilateral.cpp @@ -34,6 +34,7 @@ INSTANTIATE(float, float) INSTANTIATE(char, float) INSTANTIATE(int, float) INSTANTIATE(uint, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(short, float) INSTANTIATE(ushort, float) diff --git a/src/backend/cuda/binary.hpp b/src/backend/cuda/binary.hpp index 20f2bea9a6..ca707f30be 100644 --- a/src/backend/cuda/binary.hpp +++ b/src/backend/cuda/binary.hpp @@ -1,5 +1,5 @@ /******************************************************* - * Copyright (c) 2014, ArrayFire + * Copyright (c) 2025, ArrayFire * All rights reserved. * * This file is distributed under 3-clause BSD license. @@ -60,7 +60,7 @@ BINARY_TYPE_1(bitshiftr) }; \ template \ struct BinOp { \ - const char *name() { return "f" #fn; } \ + const char *name() { return "f" #fn "f"; } \ }; \ template \ struct BinOp { \ @@ -80,6 +80,11 @@ BINARY_TYPE_2(max) BINARY_TYPE_2(rem) BINARY_TYPE_2(mod) +template<> +struct BinOp { + const char *name() { return "hmod"; } +}; + template struct BinOp { const char *name() { return "__pow"; } diff --git a/src/backend/cuda/blas.cu b/src/backend/cuda/blas.cu index 6c88ea002a..08df398a8d 100644 --- a/src/backend/cuda/blas.cu +++ b/src/backend/cuda/blas.cu @@ -91,6 +91,17 @@ BLAS_FUNC(gemmBatched, double, D) BLAS_FUNC(gemmBatched, cdouble, Z) BLAS_FUNC(gemmBatched, __half, H) +template<> +gemm_func_def gemm_func() { + TYPE_ERROR(3, af_dtype::s8); + return gemm_func_def(); +} +template<> +gemmBatched_func_def gemmBatched_func() { + TYPE_ERROR(3, af_dtype::s8); + return gemmBatched_func_def(); +} + BLAS_FUNC_DEF(trsm) BLAS_FUNC(trsm, float, S) BLAS_FUNC(trsm, cfloat, C) @@ -161,20 +172,20 @@ cublasGemmAlgo_t selectGEMMAlgorithm<__half>() { return selectGEMMAlgorithm(); } -template +template cublasStatus_t gemmDispatch(BlasHandle handle, cublasOperation_t lOpts, cublasOperation_t rOpts, int M, int N, int K, - const T *alpha, const Array &lhs, dim_t lStride, - const Array &rhs, dim_t rStride, const T *beta, - Array &out, dim_t oleading) { + const To *alpha, const Array &lhs, dim_t lStride, + const Array &rhs, dim_t rStride, const To *beta, + Array &out, dim_t oleading) { auto prop = getDeviceProp(getActiveDeviceId()); #if __CUDACC_VER_MAJOR__ >= 10 if (prop.major > 3 && __CUDACC_VER_MAJOR__ >= 10) { return cublasGemmEx( - blasHandle(), lOpts, rOpts, M, N, K, alpha, lhs.get(), getType(), - lStride, rhs.get(), getType(), rStride, beta, out.get(), - getType(), out.strides()[1], - getComputeType(), // Compute type + blasHandle(), lOpts, rOpts, M, N, K, alpha, lhs.get(), getType(), + lStride, rhs.get(), getType(), rStride, beta, out.get(), + getType(), out.strides()[1], + getComputeType(), // Compute type // NOTE: When using the CUBLAS_GEMM_DEFAULT_TENSOR_OP algorithm // for the cublasGemm*Ex functions, the performance of the @@ -184,10 +195,10 @@ cublasStatus_t gemmDispatch(BlasHandle handle, cublasOperation_t lOpts, // this change. Does this imply that the TENSOR_OP function // performs the computation in fp16 bit even when the compute // type is CUDA_R_32F? - selectGEMMAlgorithm()); + selectGEMMAlgorithm()); } else { #endif - using Nt = typename common::kernel_type::native; + using Nt = typename common::kernel_type::native; return gemm_func()(blasHandle(), lOpts, rOpts, M, N, K, (Nt *)alpha, (Nt *)lhs.get(), lStride, (Nt *)rhs.get(), rStride, (Nt *)beta, (Nt *)out.get(), oleading); @@ -197,21 +208,21 @@ cublasStatus_t gemmDispatch(BlasHandle handle, cublasOperation_t lOpts, #endif } -template +template cublasStatus_t gemmBatchedDispatch(BlasHandle handle, cublasOperation_t lOpts, cublasOperation_t rOpts, int M, int N, int K, - const T *alpha, const T **lptrs, - int lStrides, const T **rptrs, int rStrides, - const T *beta, T **optrs, int oStrides, + const To *alpha, const Ti **lptrs, + int lStrides, const Ti **rptrs, int rStrides, + const To *beta, To **optrs, int oStrides, int batchSize) { auto prop = getDeviceProp(getActiveDeviceId()); #if __CUDACC_VER_MAJOR__ >= 10 if (prop.major > 3) { return cublasGemmBatchedEx( blasHandle(), lOpts, rOpts, M, N, K, alpha, (const void **)lptrs, - getType(), lStrides, (const void **)rptrs, getType(), - rStrides, beta, (void **)optrs, getType(), oStrides, batchSize, - getComputeType(), // compute type + getType(), lStrides, (const void **)rptrs, getType(), + rStrides, beta, (void **)optrs, getType(), oStrides, batchSize, + getComputeType(), // compute type // NOTE: When using the CUBLAS_GEMM_DEFAULT_TENSOR_OP algorithm // for the cublasGemm*Ex functions, the performance of the // fp32 numbers seem to increase dramatically. Their numerical @@ -220,10 +231,10 @@ cublasStatus_t gemmBatchedDispatch(BlasHandle handle, cublasOperation_t lOpts, // this change. Does this imply that the TENSOR_OP function // performs the computation in fp16 bit even when the compute // type is CUDA_R_32F? - selectGEMMAlgorithm()); + selectGEMMAlgorithm()); } else { #endif - using Nt = typename common::kernel_type::native; + using Nt = typename common::kernel_type::native; return gemmBatched_func()( blasHandle(), lOpts, rOpts, M, N, K, (const Nt *)alpha, (const Nt **)lptrs, lStrides, (const Nt **)rptrs, rStrides, @@ -233,9 +244,9 @@ cublasStatus_t gemmBatchedDispatch(BlasHandle handle, cublasOperation_t lOpts, #endif } -template -void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, - const Array &lhs, const Array &rhs, const T *beta) { +template +void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const To *alpha, + const Array &lhs, const Array &rhs, const To *beta) { const cublasOperation_t lOpts = toCblasTranspose(optLhs); const cublasOperation_t rOpts = toCblasTranspose(optRhs); @@ -255,14 +266,14 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, dim4 oStrides = out.strides(); if (oDims.ndims() <= 2) { - CUBLAS_CHECK(gemmDispatch(blasHandle(), lOpts, rOpts, M, N, K, alpha, - lhs, lStrides[1], rhs, rStrides[1], beta, - out, oStrides[1])); + CUBLAS_CHECK((gemmDispatch(blasHandle(), lOpts, rOpts, M, N, K, alpha, + lhs, lStrides[1], rhs, rStrides[1], beta, + out, oStrides[1]))); } else { int batchSize = oDims[2] * oDims[3]; - vector lptrs(batchSize); - vector rptrs(batchSize); - vector optrs(batchSize); + vector lptrs(batchSize); + vector rptrs(batchSize); + vector optrs(batchSize); bool is_l_d2_batched = oDims[2] == lDims[2]; bool is_l_d3_batched = oDims[3] == lDims[3]; @@ -270,9 +281,9 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, bool is_r_d2_batched = oDims[2] == rDims[2]; bool is_r_d3_batched = oDims[3] == rDims[3]; - const T *lptr = lhs.get(); - const T *rptr = rhs.get(); - T *optr = out.get(); + const Ti *lptr = lhs.get(); + const Ti *rptr = rhs.get(); + To *optr = out.get(); for (int n = 0; n < batchSize; n++) { int w = n / oDims[2]; @@ -286,7 +297,7 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, optrs[n] = optr + z * oStrides[2] + w * oStrides[3]; } - size_t bytes = batchSize * sizeof(T **); + size_t bytes = batchSize * sizeof(Ti **); auto d_lptrs = memAlloc(bytes); auto d_rptrs = memAlloc(bytes); auto d_optrs = memAlloc(bytes); @@ -302,11 +313,11 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, // afterwards CUDA_CHECK(cudaStreamSynchronize(getActiveStream())); - using Nt = typename common::kernel_type::native; + using Nt = typename common::kernel_type::native; CUBLAS_CHECK(gemmBatchedDispatch( blasHandle(), lOpts, rOpts, M, N, K, alpha, - (const T **)d_lptrs.get(), lStrides[1], (const T **)d_rptrs.get(), - rStrides[1], beta, (T **)d_optrs.get(), oStrides[1], batchSize)); + (const Ti **)d_lptrs.get(), lStrides[1], (const Ti **)d_rptrs.get(), + rStrides[1], beta, (To **)d_optrs.get(), oStrides[1], batchSize)); } } @@ -340,17 +351,18 @@ void trsm(const Array &lhs, Array &rhs, af_mat_prop trans, bool is_upper, lhs.get(), lStrides[1], rhs.get(), rStrides[1])); } -#define INSTANTIATE_GEMM(TYPE) \ - template void gemm(Array & out, af_mat_prop optLhs, \ - af_mat_prop optRhs, const TYPE *alpha, \ +#define INSTANTIATE_GEMM(TYPE, OUTTYPE) \ + template void gemm(Array & out, af_mat_prop optLhs, \ + af_mat_prop optRhs, const OUTTYPE *alpha, \ const Array &lhs, const Array &rhs, \ - const TYPE *beta); - -INSTANTIATE_GEMM(float) -INSTANTIATE_GEMM(cfloat) -INSTANTIATE_GEMM(double) -INSTANTIATE_GEMM(cdouble) -INSTANTIATE_GEMM(half) + const OUTTYPE *beta); + +INSTANTIATE_GEMM(float, float) +INSTANTIATE_GEMM(cfloat, cfloat) +INSTANTIATE_GEMM(double, double) +INSTANTIATE_GEMM(cdouble, cdouble) +INSTANTIATE_GEMM(half, half) +INSTANTIATE_GEMM(schar, float) #define INSTANTIATE_DOT(TYPE) \ template Array dot(const Array &lhs, \ diff --git a/src/backend/cuda/blas.hpp b/src/backend/cuda/blas.hpp index dc4382d013..37432911e2 100644 --- a/src/backend/cuda/blas.hpp +++ b/src/backend/cuda/blas.hpp @@ -11,9 +11,10 @@ namespace arrayfire { namespace cuda { -template -void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, - const Array &lhs, const Array &rhs, const T *beta); +template +void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, + const To *alpha, const Array &lhs, const Array &rhs, + const To *beta); template Array matmul(const Array &lhs, const Array &rhs, af_mat_prop optLhs, diff --git a/src/backend/cuda/cast.hpp b/src/backend/cuda/cast.hpp index 9328dd5052..214d24845a 100644 --- a/src/backend/cuda/cast.hpp +++ b/src/backend/cuda/cast.hpp @@ -34,6 +34,7 @@ struct CastOp { CAST_FN(int) CAST_FN(unsigned int) CAST_FN(unsigned char) +CAST_FN(signed char) CAST_FN(unsigned short) CAST_FN(short) CAST_FN(float) diff --git a/src/backend/cuda/compile_module.cpp b/src/backend/cuda/compile_module.cpp index 36014049a8..d7ee8182bc 100644 --- a/src/backend/cuda/compile_module.cpp +++ b/src/backend/cuda/compile_module.cpp @@ -39,6 +39,8 @@ #include #include #include +#include +#include #include #include #include @@ -155,15 +157,12 @@ Module compileModule(const string &moduleKey, span sources, using namespace arrayfire::cuda; if (sourceIsJIT) { constexpr const char *header_names[] = { - "utility", - "cuda_fp16.hpp", - "cuda_fp16.h", + "utility", "cuda_fp16.hpp", "cuda_fp16.h", + "vector_types.h", "vector_functions.h", }; constexpr size_t numHeaders = extent::value; array headers = { - "", - cuda_fp16_hpp, - cuda_fp16_h, + "", cuda_fp16_hpp, cuda_fp16_h, vector_types_h, vector_functions_h, }; static_assert(headers.size() == numHeaders, "headers array contains fewer sources than header_names"); @@ -176,7 +175,7 @@ Module compileModule(const string &moduleKey, span sources, "stdbool.h", // DUMMY ENTRY TO SATISFY af/defines.h inclusion "stdlib.h", // DUMMY ENTRY TO SATISFY af/defines.h inclusion "vector_types.h", // DUMMY ENTRY TO SATISFY cuComplex_h inclusion - "utility", // DUMMY ENTRY TO SATISFY cuda_fp16.hpp inclusion + "utility", // DUMMY ENTRY TO SATISFY utility inclusion "backend.hpp", "cuComplex.h", "jit.cuh", @@ -201,6 +200,7 @@ Module compileModule(const string &moduleKey, span sources, "dims_param.hpp", "common/internal_enums.hpp", "minmax_op.hpp", + "vector_functions.h", }; constexpr size_t numHeaders = extent::value; @@ -234,6 +234,7 @@ Module compileModule(const string &moduleKey, span sources, string(dims_param_hpp, dims_param_hpp_len), string(internal_enums_hpp, internal_enums_hpp_len), string(minmax_op_hpp, minmax_op_hpp_len), + string(vector_functions_h, vector_functions_h_len), }}; static const char *headers[] = { @@ -251,7 +252,7 @@ Module compileModule(const string &moduleKey, span sources, sourceStrings[22].c_str(), sourceStrings[23].c_str(), sourceStrings[24].c_str(), sourceStrings[25].c_str(), sourceStrings[26].c_str(), sourceStrings[27].c_str(), - sourceStrings[28].c_str()}; + sourceStrings[28].c_str(), sourceStrings[29].c_str()}; static_assert(extent::value == numHeaders, "headers array contains fewer sources than includeNames"); NVRTC_CHECK(nvrtcCreateProgram(&prog, sources[0].c_str(), diff --git a/src/backend/cuda/complex.hpp b/src/backend/cuda/complex.hpp index d9d143ddbf..81f39dd785 100644 --- a/src/backend/cuda/complex.hpp +++ b/src/backend/cuda/complex.hpp @@ -7,6 +7,8 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +#pragma once + #include #include #include diff --git a/src/backend/cuda/convolve.cpp b/src/backend/cuda/convolve.cpp index 3a33c6f64f..043bfdcc9e 100644 --- a/src/backend/cuda/convolve.cpp +++ b/src/backend/cuda/convolve.cpp @@ -95,6 +95,7 @@ INSTANTIATE(double, double) INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(char, float) INSTANTIATE(ushort, float) diff --git a/src/backend/cuda/copy.cpp b/src/backend/cuda/copy.cpp index f8472a7dfb..5d1701d965 100644 --- a/src/backend/cuda/copy.cpp +++ b/src/backend/cuda/copy.cpp @@ -113,6 +113,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) @@ -142,6 +143,8 @@ INSTANTIATE(half) Array const &src); \ template void copyArray(Array & dst, \ Array const &src); \ + template void copyArray(Array & dst, \ + Array const &src); \ template void copyArray(Array & dst, \ Array const &src); \ template void copyArray(Array & dst, \ @@ -157,6 +160,7 @@ INSTANTIATE_COPY_ARRAY(intl) INSTANTIATE_COPY_ARRAY(uintl) INSTANTIATE_COPY_ARRAY(short) INSTANTIATE_COPY_ARRAY(ushort) +INSTANTIATE_COPY_ARRAY(schar) INSTANTIATE_COPY_ARRAY(uchar) INSTANTIATE_COPY_ARRAY(char) INSTANTIATE_COPY_ARRAY(half) @@ -187,6 +191,7 @@ INSTANTIATE_GETSCALAR(cfloat) INSTANTIATE_GETSCALAR(cdouble) INSTANTIATE_GETSCALAR(int) INSTANTIATE_GETSCALAR(uint) +INSTANTIATE_GETSCALAR(schar) INSTANTIATE_GETSCALAR(uchar) INSTANTIATE_GETSCALAR(char) INSTANTIATE_GETSCALAR(intl) diff --git a/src/backend/cuda/count.cu b/src/backend/cuda/count.cu index 373def999c..3cb5806a88 100644 --- a/src/backend/cuda/count.cu +++ b/src/backend/cuda/count.cu @@ -26,6 +26,7 @@ INSTANTIATE(af_notzero_t, uintl, uint) INSTANTIATE(af_notzero_t, short, uint) INSTANTIATE(af_notzero_t, ushort, uint) INSTANTIATE(af_notzero_t, char, uint) +INSTANTIATE(af_notzero_t, schar, uint) INSTANTIATE(af_notzero_t, uchar, uint) INSTANTIATE(af_notzero_t, half, uint) } // namespace cuda diff --git a/src/backend/cuda/cudaDataType.hpp b/src/backend/cuda/cudaDataType.hpp index 1da3429e60..3746d0b4b9 100644 --- a/src/backend/cuda/cudaDataType.hpp +++ b/src/backend/cuda/cudaDataType.hpp @@ -44,6 +44,22 @@ inline cudaDataType_t getType() { return CUDA_R_16F; } +template<> +inline cudaDataType_t getType() { + return CUDA_R_8I; +} + +template<> +inline cudaDataType_t getType() { + return CUDA_R_8I; +} + +/* only supports LStride/RStride % 4 == 0 */ +template<> +inline cudaDataType_t getType() { + return CUDA_R_32I; +} + template inline cudaDataType_t getComputeType() { return getType(); diff --git a/src/backend/cuda/cudnn.cpp b/src/backend/cuda/cudnn.cpp index 39ee3305e6..5b8a500d00 100644 --- a/src/backend/cuda/cudnn.cpp +++ b/src/backend/cuda/cudnn.cpp @@ -64,6 +64,12 @@ cudnnDataType_t getCudnnDataType() { } #if CUDNN_VERSION >= 7100 +/// TODONT COMMIT +template<> +cudnnDataType_t getCudnnDataType() { + return CUDNN_DATA_INT8; +} + template<> cudnnDataType_t getCudnnDataType() { return CUDNN_DATA_UINT8; diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp index 8000f2f635..ee7ce76980 100644 --- a/src/backend/cuda/device_manager.cpp +++ b/src/backend/cuda/device_manager.cpp @@ -101,6 +101,14 @@ static const int jetsonComputeCapabilities[] = { // clang-format off static const cuNVRTCcompute Toolkit2MaxCompute[] = { + {12090, 9, 0, 0}, + {12080, 9, 0, 0}, + {12070, 9, 0, 0}, + {12060, 9, 0, 0}, + {12050, 9, 0, 0}, + {12040, 9, 0, 0}, + {12030, 9, 0, 0}, + {12020, 9, 0, 0}, {12010, 9, 0, 0}, {12000, 9, 0, 0}, {11080, 9, 0, 0}, @@ -139,8 +147,16 @@ struct ComputeCapabilityToStreamingProcessors { // clang-format off static const ToolkitDriverVersions CudaToDriverVersion[] = { - {12010, 525.60f, 527.41f}, - {12000, 525.60f, 527.41f}, + {12090, 525.60f, 528.33f}, + {12080, 525.60f, 528.33f}, + {12070, 525.60f, 528.33f}, + {12060, 525.60f, 528.33f}, + {12050, 525.60f, 528.33f}, + {12040, 525.60f, 528.33f}, + {12030, 525.60f, 528.33f}, + {12020, 525.60f, 528.33f}, + {12010, 525.60f, 528.33f}, + {12000, 525.60f, 528.33f}, {11080, 450.80f, 452.39f}, {11070, 450.80f, 452.39f}, {11060, 450.80f, 452.39f}, @@ -497,7 +513,10 @@ void DeviceManager::checkCudaVsDriverVersion() { debugRuntimeCheck(getLogger(), runtime, driver); - if (runtime > driver) { + int runtime_major = runtime / 1000; + int driver_major = driver / 1000; + + if (runtime_major > driver_major) { string msg = "ArrayFire was built with CUDA {} which requires GPU driver " "version {} or later. Please download and install the latest " @@ -611,6 +630,29 @@ DeviceManager::DeviceManager() sortDevices(); + // Set all default peer access to false + for (auto &dev_map : device_peer_access_map) + for (auto &dev_access : dev_map) { dev_access = false; } + + // Enable peer 2 peer access to device memory if available + for (int i = 0; i < nDevices; i++) { + for (int j = 0; j < nDevices; j++) { + if (i != j) { + int can_access_peer; + CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, i, j)); + if (can_access_peer) { + CUDA_CHECK(cudaSetDevice(i)); + AF_TRACE("Peer access enabled for {}({}) and {}({})", i, + cuDevices[i].prop.name, j, cuDevices[j].prop.name); + CUDA_CHECK(cudaDeviceEnablePeerAccess(j, 0)); + device_peer_access_map[i][j] = true; + } + } else { + device_peer_access_map[i][j] = true; + } + } + } + // Initialize all streams to 0. // Streams will be created in setActiveDevice() for (int i = 0; i < MAX_DEVICES; i++) { diff --git a/src/backend/cuda/device_manager.hpp b/src/backend/cuda/device_manager.hpp index 9275386011..ca43efaf1f 100644 --- a/src/backend/cuda/device_manager.hpp +++ b/src/backend/cuda/device_manager.hpp @@ -11,6 +11,7 @@ #include +#include #include #include #include @@ -95,6 +96,8 @@ class DeviceManager { friend std::pair getComputeCapability(const int device); + friend bool isDeviceBufferAccessible(int buf_device_id, int execution_id); + private: DeviceManager(); @@ -117,6 +120,12 @@ class DeviceManager { std::shared_ptr logger; + /// A matrix of booleans where true indicates that the corresponding + /// corrdinate devices can access each other buffers. False indicates + /// buffers need to be copied over to the other device + std::array, MAX_DEVICES> + device_peer_access_map; + std::vector cuDevices; std::vector> devJitComputes; diff --git a/src/backend/cuda/diagonal.cpp b/src/backend/cuda/diagonal.cpp index cbf3180a70..b5dd2b5c0b 100644 --- a/src/backend/cuda/diagonal.cpp +++ b/src/backend/cuda/diagonal.cpp @@ -54,6 +54,7 @@ INSTANTIATE_DIAGONAL(uint) INSTANTIATE_DIAGONAL(intl) INSTANTIATE_DIAGONAL(uintl) INSTANTIATE_DIAGONAL(char) +INSTANTIATE_DIAGONAL(schar) INSTANTIATE_DIAGONAL(uchar) INSTANTIATE_DIAGONAL(short) INSTANTIATE_DIAGONAL(ushort) diff --git a/src/backend/cuda/diff.cpp b/src/backend/cuda/diff.cpp index 55bb68ece0..b21ab36b72 100644 --- a/src/backend/cuda/diff.cpp +++ b/src/backend/cuda/diff.cpp @@ -55,6 +55,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cuda/err_cuda.hpp b/src/backend/cuda/err_cuda.hpp index 77926cdd79..f6db7e6822 100644 --- a/src/backend/cuda/err_cuda.hpp +++ b/src/backend/cuda/err_cuda.hpp @@ -14,8 +14,8 @@ #define CUDA_NOT_SUPPORTED(message) \ do { \ - throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \ - boost::stacktrace::stacktrace()); \ + throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, "CUDA", \ + message, boost::stacktrace::stacktrace()); \ } while (0) #define CU_CHECK(fn) \ diff --git a/src/backend/cuda/exampleFunction.cpp b/src/backend/cuda/exampleFunction.cpp index b94f9f8e54..12bf635785 100644 --- a/src/backend/cuda/exampleFunction.cpp +++ b/src/backend/cuda/exampleFunction.cpp @@ -60,6 +60,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(cfloat) diff --git a/src/backend/cuda/fast.cu b/src/backend/cuda/fast.cu index 7744d4b6d6..63e9a57cb4 100644 --- a/src/backend/cuda/fast.cu +++ b/src/backend/cuda/fast.cu @@ -62,6 +62,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/fast_pyramid.cpp b/src/backend/cuda/fast_pyramid.cpp index 97228af248..ba0b6dfbf4 100644 --- a/src/backend/cuda/fast_pyramid.cpp +++ b/src/backend/cuda/fast_pyramid.cpp @@ -120,6 +120,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/fftconvolve.cpp b/src/backend/cuda/fftconvolve.cpp index ed22d0ea85..cb8359423e 100644 --- a/src/backend/cuda/fftconvolve.cpp +++ b/src/backend/cuda/fftconvolve.cpp @@ -112,6 +112,7 @@ INSTANTIATE(float) INSTANTIATE(uint) INSTANTIATE(int) INSTANTIATE(uchar) +INSTANTIATE(schar) INSTANTIATE(char) INSTANTIATE(uintl) INSTANTIATE(intl) diff --git a/src/backend/cuda/hist_graphics.cpp b/src/backend/cuda/hist_graphics.cpp index 6678281db6..cabadeb1ad 100644 --- a/src/backend/cuda/hist_graphics.cpp +++ b/src/backend/cuda/hist_graphics.cpp @@ -69,6 +69,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace cuda diff --git a/src/backend/cuda/histogram.cpp b/src/backend/cuda/histogram.cpp index ca7e6ced86..f012d6e64b 100644 --- a/src/backend/cuda/histogram.cpp +++ b/src/backend/cuda/histogram.cpp @@ -41,6 +41,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/identity.cpp b/src/backend/cuda/identity.cpp index 995b09a9d9..ee62dcf549 100644 --- a/src/backend/cuda/identity.cpp +++ b/src/backend/cuda/identity.cpp @@ -37,6 +37,7 @@ INSTANTIATE_IDENTITY(uint) INSTANTIATE_IDENTITY(intl) INSTANTIATE_IDENTITY(uintl) INSTANTIATE_IDENTITY(char) +INSTANTIATE_IDENTITY(schar) INSTANTIATE_IDENTITY(uchar) INSTANTIATE_IDENTITY(short) INSTANTIATE_IDENTITY(ushort) diff --git a/src/backend/cuda/image.cpp b/src/backend/cuda/image.cpp index 810d36d968..23bccf616e 100644 --- a/src/backend/cuda/image.cpp +++ b/src/backend/cuda/image.cpp @@ -70,6 +70,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/cuda/index.cpp b/src/backend/cuda/index.cpp index 88a95da73b..dbb7d1ad60 100644 --- a/src/backend/cuda/index.cpp +++ b/src/backend/cuda/index.cpp @@ -44,6 +44,16 @@ Array index(const Array& in, const af_index_t idxrs[]) { p.isSeq[i] = idxrs[i].isSeq; p.offs[i] = iOffs[i]; p.strds[i] = iStrds[i]; + p.steps[i] = 0; + if (idxrs[i].isSeq) { + af_seq seq = idxrs[i].idx.seq; + // The step for af_span used in the kernel must be 1 + if (seq.begin == af_span.begin && seq.end == af_span.end && + seq.step == af_span.step) + p.steps[i] = 1; + else + p.steps[i] = seq.step; + } } std::vector> idxArrs(4, createEmptyArray(dim4())); @@ -80,6 +90,7 @@ INSTANTIATE(int) INSTANTIATE(uintl) INSTANTIATE(intl) INSTANTIATE(uchar) +INSTANTIATE(schar) INSTANTIATE(char) INSTANTIATE(ushort) INSTANTIATE(short) diff --git a/src/backend/cuda/iota.cpp b/src/backend/cuda/iota.cpp index d9afef41c5..0ac6dbee74 100644 --- a/src/backend/cuda/iota.cpp +++ b/src/backend/cuda/iota.cpp @@ -38,6 +38,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/ireduce.cpp b/src/backend/cuda/ireduce.cpp index 94cd340a66..a2236230d4 100644 --- a/src/backend/cuda/ireduce.cpp +++ b/src/backend/cuda/ireduce.cpp @@ -62,6 +62,7 @@ INSTANTIATE(af_min_t, uintl) INSTANTIATE(af_min_t, short) INSTANTIATE(af_min_t, ushort) INSTANTIATE(af_min_t, char) +INSTANTIATE(af_min_t, schar) INSTANTIATE(af_min_t, uchar) INSTANTIATE(af_min_t, half) @@ -77,6 +78,7 @@ INSTANTIATE(af_max_t, uintl) INSTANTIATE(af_max_t, short) INSTANTIATE(af_max_t, ushort) INSTANTIATE(af_max_t, char) +INSTANTIATE(af_max_t, schar) INSTANTIATE(af_max_t, uchar) INSTANTIATE(af_max_t, half) } // namespace cuda diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp index 33a80adb50..171ec66f61 100644 --- a/src/backend/cuda/jit.cpp +++ b/src/backend/cuda/jit.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,8 @@ using arrayfire::common::findModule; using arrayfire::common::getEnvVar; using arrayfire::common::getFuncName; using arrayfire::common::half; +using arrayfire::common::isBufferOrShift; +using arrayfire::common::kNodeType; using arrayfire::common::ModdimNode; using arrayfire::common::Node; using arrayfire::common::Node_ids; @@ -45,6 +48,8 @@ using arrayfire::common::Node_map_t; using arrayfire::common::Node_ptr; using arrayfire::common::NodeIterator; using arrayfire::common::saveKernel; +using arrayfire::cuda::jit::BufferNode; +using arrayfire::cuda::jit::ShiftNode; using std::array; using std::equal; @@ -58,7 +63,6 @@ using std::vector; namespace arrayfire { namespace cuda { -using jit::BufferNode; static string getKernelString(const string& funcName, const vector& full_nodes, @@ -240,16 +244,18 @@ struct Param { node->genOffsets(inOffsetsStream, ids_curr.id, is_linear); // Generate the core function body, needs children ids as well node->genFuncs(opsStream, ids_curr); - for (auto outIt{begin(output_ids)}, endIt{end(output_ids)}; - (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) { - // Generate also output parameters - outParamStream << (oid == 0 ? "" : ",\n") << "Param<" - << full_nodes[ids_curr.id]->getTypeStr() - << "> out" << oid; - // Generate code to write the output (offset already in ptr) - opsStream << "out" << oid << ".ptr[idx] = val" << ids_curr.id - << ";\n"; - ++oid; + for (size_t output_idx{0}; output_idx < output_ids.size(); + ++output_idx) { + if (output_ids[output_idx] == ids_curr.id) { + // Generate also output parameters + outParamStream << (oid == 0 ? "" : ",\n") << "Param<" + << full_nodes[ids_curr.id]->getTypeStr() + << "> out" << oid; + // Generate code to write the output (offset already in ptr) + opsStream << "out" << output_idx << ".ptr[idx] = val" + << ids_curr.id << ";\n"; + ++oid; + } } } @@ -318,8 +324,9 @@ static CUfunction getKernel(const vector& output_nodes, const bool is_linear, const bool loop0, const bool loop1, const bool loop2, const bool loop3) { - const string funcName{getFuncName(output_nodes, full_nodes, full_ids, - is_linear, loop0, loop1, loop2, loop3)}; + const string funcName{getFuncName(output_nodes, output_ids, full_nodes, + full_ids, is_linear, loop0, loop1, loop2, + loop3)}; // A forward lookup in module cache helps avoid recompiling // the JIT source generated from identical JIT-trees. const auto entry{ @@ -474,22 +481,9 @@ void evalNodes(vector>& outputs, const vector& output_nodes) { } } if (emptyColumnsFound) { - const auto isBuffer{ - [](const Node_ptr& node) { return node->isBuffer(); }}; - for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)}; - (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt; - ++nodeIt) { - BufferNode* buf{ - static_cast*>(nodeIt->get())}; - removeEmptyColumns(outDims, ndims, buf->m_param.dims, - buf->m_param.strides); - } - for_each(++begin(outputs), end(outputs), - [outDims, ndims](Param& output) { - removeEmptyColumns(outDims, ndims, output.dims, - output.strides); - }); - ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides); + common::removeEmptyDimensions, BufferNode, + ShiftNode>(outputs, + node_clones); } full_nodes.clear(); @@ -508,7 +502,8 @@ void evalNodes(vector>& outputs, const vector& output_nodes) { vector args; for (const Node* node : full_nodes) { node->setArgs(0, is_linear, - [&](int /*id*/, const void* ptr, size_t /*size*/) { + [&](int /*id*/, const void* ptr, size_t /*size*/, + bool /*is_buffer*/) { args.push_back(const_cast(ptr)); }); } @@ -557,6 +552,7 @@ template void evalNodes(Param out, Node* node); template void evalNodes(Param out, Node* node); template void evalNodes(Param out, Node* node); template void evalNodes(Param out, Node* node); +template void evalNodes(Param out, Node* node); template void evalNodes(Param out, Node* node); template void evalNodes(Param out, Node* node); template void evalNodes(Param out, Node* node); @@ -578,6 +574,8 @@ template void evalNodes(vector>& out, const vector& node); template void evalNodes(vector>& out, const vector& node); +template void evalNodes(vector>& out, + const vector& node); template void evalNodes(vector>& out, const vector& node); template void evalNodes(vector>& out, diff --git a/src/backend/cuda/jit/BufferNode.hpp b/src/backend/cuda/jit/BufferNode.hpp index 195353fdd8..8692b72515 100644 --- a/src/backend/cuda/jit/BufferNode.hpp +++ b/src/backend/cuda/jit/BufferNode.hpp @@ -27,7 +27,16 @@ bool BufferNodeBase::operator==( // clang-format off return m_data.get() == other.m_data.get() && m_bytes == other.m_bytes && - m_param.ptr == other.m_param.ptr; + m_param.ptr == other.m_param.ptr && + m_linear_buffer == other.m_linear_buffer && + m_param.dims[0] == other.m_param.dims[0] && + m_param.dims[1] == other.m_param.dims[1] && + m_param.dims[2] == other.m_param.dims[2] && + m_param.dims[3] == other.m_param.dims[3] && + m_param.strides[0] == other.m_param.strides[0] && + m_param.strides[1] == other.m_param.strides[1] && + m_param.strides[2] == other.m_param.strides[2] && + m_param.strides[3] == other.m_param.strides[3]; // clang-format on } diff --git a/src/backend/cuda/jit/ShiftNode.hpp b/src/backend/cuda/jit/ShiftNode.hpp new file mode 100644 index 0000000000..16bdf5d0f9 --- /dev/null +++ b/src/backend/cuda/jit/ShiftNode.hpp @@ -0,0 +1,22 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#include +#include + +namespace arrayfire { +namespace cuda { +namespace jit { + +template +using ShiftNode = common::ShiftNodeBase>; + +} // namespace jit +} // namespace cuda +} // namespace arrayfire diff --git a/src/backend/cuda/jit/kernel_generators.hpp b/src/backend/cuda/jit/kernel_generators.hpp index f675faf4b4..02f58f432d 100644 --- a/src/backend/cuda/jit/kernel_generators.hpp +++ b/src/backend/cuda/jit/kernel_generators.hpp @@ -33,15 +33,17 @@ void generateParamDeclaration(std::stringstream& kerStream, int id, /// Calls the setArg function to set the arguments for a kernel call template -int setKernelArguments( +int setBufferKernelArguments( int start_id, bool is_linear, - std::function& setArg, + std::function& setArg, const std::shared_ptr& ptr, const Param& info) { UNUSED(ptr); if (is_linear) { - setArg(start_id, static_cast(&info.ptr), sizeof(T*)); + setArg(start_id, static_cast(&info.ptr), sizeof(T*), true); } else { - setArg(start_id, static_cast(&info), sizeof(Param)); + setArg(start_id, static_cast(&info), sizeof(Param), + true); } return start_id + 1; } diff --git a/src/backend/cuda/join.cpp b/src/backend/cuda/join.cpp index 3eed6f7fb5..5065412342 100644 --- a/src/backend/cuda/join.cpp +++ b/src/backend/cuda/join.cpp @@ -209,6 +209,7 @@ INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(half) @@ -229,6 +230,7 @@ INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(half) diff --git a/src/backend/cuda/kernel/convolve_separable.cpp b/src/backend/cuda/kernel/convolve_separable.cpp index 3c18a02240..14a62d1f1e 100644 --- a/src/backend/cuda/kernel/convolve_separable.cpp +++ b/src/backend/cuda/kernel/convolve_separable.cpp @@ -22,6 +22,7 @@ INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) INSTANTIATE(uchar, float) +INSTANTIATE(schar, float) INSTANTIATE(char, float) INSTANTIATE(ushort, float) INSTANTIATE(short, float) diff --git a/src/backend/cuda/kernel/copy.cuh b/src/backend/cuda/kernel/copy.cuh index 9e771e8c52..20f6bfa021 100644 --- a/src/backend/cuda/kernel/copy.cuh +++ b/src/backend/cuda/kernel/copy.cuh @@ -49,6 +49,18 @@ convertType>(char value) { return compute_t(value); } +template<> +__inline__ __device__ schar +convertType, schar>(compute_t value) { + return (schar)((short)value); +} + +template<> +__inline__ __device__ compute_t +convertType>(schar value) { + return compute_t(value); +} + template<> __inline__ __device__ uchar convertType, uchar>(compute_t value) { @@ -90,6 +102,7 @@ OTHER_SPECIALIZATIONS(intl) OTHER_SPECIALIZATIONS(uintl) OTHER_SPECIALIZATIONS(short) OTHER_SPECIALIZATIONS(ushort) +OTHER_SPECIALIZATIONS(schar) OTHER_SPECIALIZATIONS(uchar) OTHER_SPECIALIZATIONS(char) OTHER_SPECIALIZATIONS(common::half) diff --git a/src/backend/cuda/kernel/index.cuh b/src/backend/cuda/kernel/index.cuh index 37b6b63d46..968e9ae0c6 100644 --- a/src/backend/cuda/kernel/index.cuh +++ b/src/backend/cuda/kernel/index.cuh @@ -43,13 +43,17 @@ __global__ void index(Param out, CParam in, const IndexKernelParam p, gw < out.dims[3]) { // calculate pointer offsets for input int i = - p.strds[0] * trimIndex(s0 ? gx + p.offs[0] : ptr0[gx], in.dims[0]); + p.strds[0] * + trimIndex(s0 ? gx * p.steps[0] + p.offs[0] : ptr0[gx], in.dims[0]); int j = - p.strds[1] * trimIndex(s1 ? gy + p.offs[1] : ptr1[gy], in.dims[1]); + p.strds[1] * + trimIndex(s1 ? gy * p.steps[1] + p.offs[1] : ptr1[gy], in.dims[1]); int k = - p.strds[2] * trimIndex(s2 ? gz + p.offs[2] : ptr2[gz], in.dims[2]); + p.strds[2] * + trimIndex(s2 ? gz * p.steps[2] + p.offs[2] : ptr2[gz], in.dims[2]); int l = - p.strds[3] * trimIndex(s3 ? gw + p.offs[3] : ptr3[gw], in.dims[3]); + p.strds[3] * + trimIndex(s3 ? gw * p.steps[3] + p.offs[3] : ptr3[gw], in.dims[3]); // offset input and output pointers const T* src = (const T*)in.ptr + (i + j + k + l); T* dst = (T*)out.ptr + (gx * out.strides[0] + gy * out.strides[1] + diff --git a/src/backend/cuda/kernel/ireduce.hpp b/src/backend/cuda/kernel/ireduce.hpp index c394c01f83..992d0871c4 100644 --- a/src/backend/cuda/kernel/ireduce.hpp +++ b/src/backend/cuda/kernel/ireduce.hpp @@ -165,14 +165,14 @@ T ireduce_all(uint *idx, CParam in) { using std::unique_ptr; int in_elements = in.dims[0] * in.dims[1] * in.dims[2] * in.dims[3]; - // FIXME: Use better heuristics to get to the optimum number - if (in_elements > 4096) { - bool is_linear = (in.strides[0] == 1); - for (int k = 1; k < 4; k++) { - is_linear &= - (in.strides[k] == (in.strides[k - 1] * in.dims[k - 1])); - } + bool is_linear = (in.strides[0] == 1); + for (int k = 1; k < 4; k++) { + is_linear &= + (in.strides[k] == (in.strides[k - 1] * in.dims[k - 1])); + } + // FIXME: Use better heuristics to get to the optimum number + if (!is_linear || in_elements > 4096) { if (is_linear) { in.dims[0] = in_elements; for (int k = 1; k < 4; k++) { diff --git a/src/backend/cuda/kernel/jit.cuh b/src/backend/cuda/kernel/jit.cuh index cfb5837719..879d46f3c2 100644 --- a/src/backend/cuda/kernel/jit.cuh +++ b/src/backend/cuda/kernel/jit.cuh @@ -1,5 +1,5 @@ /******************************************************* - * Copyright (c) 2014, ArrayFire + * Copyright (c) 2025, ArrayFire * All rights reserved. * * This file is distributed under 3-clause BSD license. @@ -59,14 +59,9 @@ typedef cuDoubleComplex cdouble; #define __rem(lhs, rhs) ((lhs) % (rhs)) #define __mod(lhs, rhs) ((lhs) % (rhs)) -#ifdef AF_WITH_FAST_MATH #define __pow(lhs, rhs) \ static_cast( \ pow(static_cast(lhs), static_cast(rhs))); -#else -#define __pow(lhs, rhs) \ - __float2int_rn(powf(__int2float_rn((int)lhs), __int2float_rn((int)rhs))) -#endif #define __powll(lhs, rhs) \ __double2ll_rn(pow(__ll2double_rn(lhs), __ll2double_rn(rhs))) #define __powul(lhs, rhs) \ @@ -78,6 +73,7 @@ typedef cuDoubleComplex cdouble; #define __convert_char(val) (char)((val) != 0) #define frem(lhs, rhs) remainder((lhs), (rhs)) +#define fremf(lhs, rhs) remainderf((lhs), (rhs)) // ---------------------------------------------- // COMPLEX FLOAT OPERATIONS @@ -219,6 +215,15 @@ __device__ __inline__ int __isinf<__half>(const __half in) { #endif } +__device__ __inline__ +__half hmod(const __half lhs, const __half rhs) { +#if __CUDA_ARCH__ >= 530 + return __hsub(lhs, __hmul(htrunc(__hdiv(lhs, rhs)), rhs)); +#else + return __float2half(fmodf(__half2float(lhs), __half2float(rhs))); +#endif +} + template static __device__ __inline__ int __isnan(const T in) { return isnan(in); diff --git a/src/backend/cuda/kernel/random_engine.hpp b/src/backend/cuda/kernel/random_engine.hpp index 07ba4163a2..a5e2305885 100644 --- a/src/backend/cuda/kernel/random_engine.hpp +++ b/src/backend/cuda/kernel/random_engine.hpp @@ -312,6 +312,12 @@ __device__ static void writeOut128Bytes(uchar *out, const uint &index, out[index + 15 * blockDim.x] = r4 >> 24; } +__device__ static void writeOut128Bytes(schar *out, const uint &index, + const uint &r1, const uint &r2, + const uint &r3, const uint &r4) { + writeOut128Bytes((uchar *)(out), index, r1, r2, r3, r4); +} + __device__ static void writeOut128Bytes(char *out, const uint &index, const uint &r1, const uint &r2, const uint &r3, const uint &r4) { @@ -535,6 +541,13 @@ __device__ static void partialWriteOut128Bytes(uchar *out, const uint &index, } } +__device__ static void partialWriteOut128Bytes(schar *out, const uint &index, + const uint &r1, const uint &r2, + const uint &r3, const uint &r4, + const uint &elements) { + partialWriteOut128Bytes((uchar *)(out), index, r1, r2, r3, r4, elements); +} + __device__ static void partialWriteOut128Bytes(char *out, const uint &index, const uint &r1, const uint &r2, const uint &r3, const uint &r4, diff --git a/src/backend/cuda/kernel/reduce_by_key.hpp b/src/backend/cuda/kernel/reduce_by_key.hpp index ea015aaff2..1e04a123ec 100644 --- a/src/backend/cuda/kernel/reduce_by_key.hpp +++ b/src/backend/cuda/kernel/reduce_by_key.hpp @@ -25,8 +25,6 @@ using std::unique_ptr; -const static unsigned int FULL_MASK = 0xFFFFFFFF; - namespace arrayfire { namespace cuda { namespace kernel { @@ -68,9 +66,9 @@ __global__ void test_needs_reduction(int *needs_another_reduction, if (tid < n) { k = keys_in.ptr[tid]; } - int update_key = (k == shfl_down_sync(FULL_MASK, k, 1)) && + int update_key = (k == shfl_down_sync(k, 1)) && (tid < (n - 1)) && ((threadIdx.x % 32) < 31); - int remaining_updates = any_sync(FULL_MASK, update_key); + int remaining_updates = any_sync(update_key); __syncthreads(); @@ -83,7 +81,7 @@ __global__ void test_needs_reduction(int *needs_another_reduction, && (threadIdx.x < (blockDim.x - 1)) // not last thread in block // next value valid and equal && ((tid + 1) < n) && (k == keys_in.ptr[tid + 1])); - remaining_updates = any_sync(FULL_MASK, update_key); + remaining_updates = any_sync(update_key); // TODO: single per warp? change to assignment rather than atomicOr if (remaining_updates) atomicOr(needs_another_reduction, remaining_updates); @@ -243,7 +241,7 @@ __global__ static void reduce_blocks_by_key(int *reduced_block_sizes, v = common::Binary, op>::init(); } - compute_t eq_check = (k != shfl_up_sync(FULL_MASK, k, 1)); + compute_t eq_check = (k != shfl_up_sync(k, 1)); // mark threads containing unique keys char unique_flag = (eq_check || (laneid == 0)) && (tidx < n); @@ -251,42 +249,33 @@ __global__ static void reduce_blocks_by_key(int *reduced_block_sizes, char unique_id = unique_flag; #pragma unroll for (int offset = 1; offset < 32; offset <<= 1) { - char y = shfl_up_sync(FULL_MASK, unique_id, offset); + char y = shfl_up_sync(unique_id, offset); if (laneid >= offset) unique_id += y; } // // Reduce each warp by key - char all_eq = (k == shfl_down_sync(FULL_MASK, k, 1)); - if (all_sync(FULL_MASK, - all_eq)) { // check special case of single key per warp - v = reduce(v, shfl_down_sync(FULL_MASK, v, 1)); - v = reduce(v, shfl_down_sync(FULL_MASK, v, 2)); - v = reduce(v, shfl_down_sync(FULL_MASK, v, 4)); - v = reduce(v, shfl_down_sync(FULL_MASK, v, 8)); - v = reduce(v, shfl_down_sync(FULL_MASK, v, 16)); + char all_eq = (k == shfl_down_sync(k, 1)); + if (all_sync(all_eq)) { // check special case of single key per warp + v = reduce(v, shfl_down_sync(v, 1)); + v = reduce(v, shfl_down_sync(v, 2)); + v = reduce(v, shfl_down_sync(v, 4)); + v = reduce(v, shfl_down_sync(v, 8)); + v = reduce(v, shfl_down_sync(v, 16)); } else { compute_t init = common::Binary, op>::init(); int eq_check, update_key; - unsigned shflmask; #pragma unroll for (int delta = 1; delta < 32; delta <<= 1) { eq_check = - (unique_id == shfl_down_sync(FULL_MASK, unique_id, delta)); + (unique_id == shfl_down_sync(unique_id, delta)); // checks if this thread should perform a reduction update_key = eq_check && (laneid < (32 - delta)) && ((tidx + delta) < n); - // obtains mask of all threads that should be reduced - shflmask = ballot_sync(FULL_MASK, update_key); - - // shifts mask to include source threads that should participate in - // _shfl - shflmask |= (shflmask << delta); - // shfls data from neighboring threads - compute_t uval = shfl_down_sync(shflmask, v, delta); + compute_t uval = shfl_down_sync(v, delta); // update if thread requires it v = reduce(v, (update_key ? uval : init)); @@ -479,7 +468,7 @@ __global__ static void reduce_blocks_dim_by_key( v = init; } - Tk eq_check = (k != shfl_up_sync(FULL_MASK, k, 1)); + Tk eq_check = (k != shfl_up_sync(k, 1)); // mark threads containing unique keys char unique_flag = (eq_check || (laneid == 0)) && (tidx < n); @@ -487,42 +476,33 @@ __global__ static void reduce_blocks_dim_by_key( char unique_id = unique_flag; #pragma unroll for (int offset = 1; offset < 32; offset <<= 1) { - char y = shfl_up_sync(FULL_MASK, unique_id, offset); + char y = shfl_up_sync(unique_id, offset); if (laneid >= offset) unique_id += y; } // // Reduce each warp by key - char all_eq = (k == shfl_down_sync(FULL_MASK, k, 1)); - if (all_sync(FULL_MASK, - all_eq)) { // check special case of single key per warp - v = reduce(v, shfl_down_sync(FULL_MASK, v, 1)); - v = reduce(v, shfl_down_sync(FULL_MASK, v, 2)); - v = reduce(v, shfl_down_sync(FULL_MASK, v, 4)); - v = reduce(v, shfl_down_sync(FULL_MASK, v, 8)); - v = reduce(v, shfl_down_sync(FULL_MASK, v, 16)); + char all_eq = (k == shfl_down_sync(k, 1)); + if (all_sync(all_eq)) { // check special case of single key per warp + v = reduce(v, shfl_down_sync(v, 1)); + v = reduce(v, shfl_down_sync(v, 2)); + v = reduce(v, shfl_down_sync(v, 4)); + v = reduce(v, shfl_down_sync(v, 8)); + v = reduce(v, shfl_down_sync(v, 16)); } else { compute_t init = common::Binary, op>::init(); int eq_check, update_key; - unsigned shflmask; #pragma unroll for (int delta = 1; delta < 32; delta <<= 1) { eq_check = - (unique_id == shfl_down_sync(FULL_MASK, unique_id, delta)); + (unique_id == shfl_down_sync(unique_id, delta)); // checks if this thread should perform a reduction update_key = eq_check && (laneid < (32 - delta)) && ((tidx + delta) < n); - // obtains mask of all threads that should be reduced - shflmask = ballot_sync(FULL_MASK, update_key); - - // shifts mask to include source threads that should participate in - // _shfl - shflmask |= (shflmask << delta); - // shfls data from neighboring threads - compute_t uval = shfl_down_sync(shflmask, v, delta); + compute_t uval = shfl_down_sync(v, delta); // update if thread requires it v = reduce(v, (update_key ? uval : init)); diff --git a/src/backend/cuda/kernel/shared.hpp b/src/backend/cuda/kernel/shared.hpp index 55d9f70a64..d1f15653c3 100644 --- a/src/backend/cuda/kernel/shared.hpp +++ b/src/backend/cuda/kernel/shared.hpp @@ -53,6 +53,7 @@ SPECIALIZE(int) SPECIALIZE(uint) SPECIALIZE(short) SPECIALIZE(ushort) +SPECIALIZE(schar) SPECIALIZE(uchar) SPECIALIZE(intl) SPECIALIZE(uintl) diff --git a/src/backend/cuda/kernel/shfl_intrinsics.hpp b/src/backend/cuda/kernel/shfl_intrinsics.hpp index 687abf5144..a91dc74148 100644 --- a/src/backend/cuda/kernel/shfl_intrinsics.hpp +++ b/src/backend/cuda/kernel/shfl_intrinsics.hpp @@ -11,11 +11,13 @@ namespace arrayfire { namespace cuda { namespace kernel { +constexpr unsigned int FULL_MASK = 0xffffffff; + //__all_sync wrapper template -__device__ T all_sync(unsigned mask, T var) { +__device__ T all_sync(T var) { #if (CUDA_VERSION >= 9000) - return __all_sync(mask, var); + return __all_sync(FULL_MASK, var); #else return __all(var); #endif @@ -23,9 +25,9 @@ __device__ T all_sync(unsigned mask, T var) { //__all_sync wrapper template -__device__ T any_sync(unsigned mask, T var) { +__device__ T any_sync(T var) { #if (CUDA_VERSION >= 9000) - return __any_sync(mask, var); + return __any_sync(FULL_MASK, var); #else return __any(var); #endif @@ -33,9 +35,9 @@ __device__ T any_sync(unsigned mask, T var) { //__shfl_down_sync wrapper template -__device__ T ballot_sync(unsigned mask, T var) { +__device__ T ballot_sync(T var) { #if (CUDA_VERSION >= 9000) - return __ballot_sync(mask, var); + return __ballot_sync(FULL_MASK, var); #else return __ballot(var); #endif @@ -43,19 +45,19 @@ __device__ T ballot_sync(unsigned mask, T var) { //__shfl_down_sync wrapper template -__device__ T shfl_down_sync(unsigned mask, T var, int delta) { +__device__ T shfl_down_sync(T var, int delta) { #if (CUDA_VERSION >= 9000) - return __shfl_down_sync(mask, var, delta); + return __shfl_down_sync(FULL_MASK, var, delta); #else return __shfl_down(var, delta); #endif } // specialization for cfloat template<> -inline __device__ cfloat shfl_down_sync(unsigned mask, cfloat var, int delta) { +inline __device__ cfloat shfl_down_sync(cfloat var, int delta) { #if (CUDA_VERSION >= 9000) - cfloat res = {__shfl_down_sync(mask, var.x, delta), - __shfl_down_sync(mask, var.y, delta)}; + cfloat res = {__shfl_down_sync(FULL_MASK, var.x, delta), + __shfl_down_sync(FULL_MASK, var.y, delta)}; #else cfloat res = {__shfl_down(var.x, delta), __shfl_down(var.y, delta)}; #endif @@ -63,11 +65,11 @@ inline __device__ cfloat shfl_down_sync(unsigned mask, cfloat var, int delta) { } // specialization for cdouble template<> -inline __device__ cdouble shfl_down_sync(unsigned mask, cdouble var, +inline __device__ cdouble shfl_down_sync(cdouble var, int delta) { #if (CUDA_VERSION >= 9000) - cdouble res = {__shfl_down_sync(mask, var.x, delta), - __shfl_down_sync(mask, var.y, delta)}; + cdouble res = {__shfl_down_sync(FULL_MASK, var.x, delta), + __shfl_down_sync(FULL_MASK, var.y, delta)}; #else cdouble res = {__shfl_down(var.x, delta), __shfl_down(var.y, delta)}; #endif @@ -76,19 +78,19 @@ inline __device__ cdouble shfl_down_sync(unsigned mask, cdouble var, //__shfl_up_sync wrapper template -__device__ T shfl_up_sync(unsigned mask, T var, int delta) { +__device__ T shfl_up_sync(T var, int delta) { #if (CUDA_VERSION >= 9000) - return __shfl_up_sync(mask, var, delta); + return __shfl_up_sync(FULL_MASK, var, delta); #else return __shfl_up(var, delta); #endif } // specialization for cfloat template<> -inline __device__ cfloat shfl_up_sync(unsigned mask, cfloat var, int delta) { +inline __device__ cfloat shfl_up_sync(cfloat var, int delta) { #if (CUDA_VERSION >= 9000) - cfloat res = {__shfl_up_sync(mask, var.x, delta), - __shfl_up_sync(mask, var.y, delta)}; + cfloat res = {__shfl_up_sync(FULL_MASK, var.x, delta), + __shfl_up_sync(FULL_MASK, var.y, delta)}; #else cfloat res = {__shfl_up(var.x, delta), __shfl_up(var.y, delta)}; #endif @@ -96,10 +98,10 @@ inline __device__ cfloat shfl_up_sync(unsigned mask, cfloat var, int delta) { } // specialization for cdouble template<> -inline __device__ cdouble shfl_up_sync(unsigned mask, cdouble var, int delta) { +inline __device__ cdouble shfl_up_sync(cdouble var, int delta) { #if (CUDA_VERSION >= 9000) - cdouble res = {__shfl_up_sync(mask, var.x, delta), - __shfl_up_sync(mask, var.y, delta)}; + cdouble res = {__shfl_up_sync(FULL_MASK, var.x, delta), + __shfl_up_sync(FULL_MASK, var.y, delta)}; #else cdouble res = {__shfl_up(var.x, delta), __shfl_up(var.y, delta)}; #endif diff --git a/src/backend/cuda/kernel/sparse.cuh b/src/backend/cuda/kernel/sparse.cuh index bdf0e20884..84825bdd24 100644 --- a/src/backend/cuda/kernel/sparse.cuh +++ b/src/backend/cuda/kernel/sparse.cuh @@ -17,15 +17,13 @@ namespace cuda { template __global__ void coo2Dense(Param output, CParam values, CParam rowIdx, CParam colIdx) { - int id = blockIdx.x * blockDim.x * reps + threadIdx.x; - if (id >= values.dims[0]) return; + for (int i = threadIdx.x; i < reps * blockDim.x; i += blockDim.x) { + int id = i + blockIdx.x * blockDim.x * reps; + if (id >= values.dims[0]) return; - for (int i = threadIdx.x; i <= reps * blockDim.x; i += blockDim.x) { - if (i >= values.dims[0]) return; - - T v = values.ptr[i]; - int r = rowIdx.ptr[i]; - int c = colIdx.ptr[i]; + T v = values.ptr[id]; + int r = rowIdx.ptr[id]; + int c = colIdx.ptr[id]; int offset = r + c * output.strides[1]; diff --git a/src/backend/cuda/kernel/sparse.hpp b/src/backend/cuda/kernel/sparse.hpp index 6629d0fec6..60068d3e20 100644 --- a/src/backend/cuda/kernel/sparse.hpp +++ b/src/backend/cuda/kernel/sparse.hpp @@ -30,7 +30,7 @@ void coo2dense(Param output, CParam values, CParam rowIdx, dim3 threads(256, 1, 1); - dim3 blocks(divup(output.dims[0], threads.x * reps), 1, 1); + dim3 blocks(divup(values.dims[0], threads.x * reps), 1, 1); EnqueueArgs qArgs(blocks, threads, getActiveStream()); diff --git a/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu b/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu index 19b291356c..7a7e3616c9 100644 --- a/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu +++ b/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu @@ -11,7 +11,7 @@ // This file instantiates sort_by_key as separate object files from CMake // The 3 lines below are read by CMake to determenine the instantiations -// SBK_TYPES:float double int uint intl uintl short ushort char uchar +// SBK_TYPES:float double int uint intl uintl short ushort char schar uchar // SBK_INSTS:0 1 namespace arrayfire { diff --git a/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp b/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp index e4695ac48e..e909a786de 100644 --- a/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp +++ b/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp @@ -39,6 +39,7 @@ void thrustSortByKey(Tk *keyPtr, Tv *valPtr, int elements, bool isAscending) { INSTANTIATE(Tk, cfloat) \ INSTANTIATE(Tk, cdouble) \ INSTANTIATE(Tk, char) \ + INSTANTIATE(Tk, schar) \ INSTANTIATE(Tk, uchar) #define INSTANTIATE1(Tk) \ diff --git a/src/backend/cuda/lookup.cpp b/src/backend/cuda/lookup.cpp index 133db5ba26..ca5b8f79ed 100644 --- a/src/backend/cuda/lookup.cpp +++ b/src/backend/cuda/lookup.cpp @@ -54,6 +54,8 @@ Array lookup(const Array &input, const Array &indices, const unsigned); \ template Array lookup(const Array &, const Array &, \ const unsigned); \ + template Array lookup(const Array &, const Array &, \ + const unsigned); \ template Array lookup(const Array &, const Array &, \ const unsigned); \ template Array lookup(const Array &, const Array &, \ @@ -67,6 +69,7 @@ INSTANTIATE(int); INSTANTIATE(unsigned); INSTANTIATE(intl); INSTANTIATE(uintl); +INSTANTIATE(schar); INSTANTIATE(uchar); INSTANTIATE(char); INSTANTIATE(short); diff --git a/src/backend/cuda/match_template.cpp b/src/backend/cuda/match_template.cpp index d82137bb5c..63b50435b7 100644 --- a/src/backend/cuda/match_template.cpp +++ b/src/backend/cuda/match_template.cpp @@ -38,6 +38,7 @@ INSTANTIATE(float, float) INSTANTIATE(char, float) INSTANTIATE(int, float) INSTANTIATE(uint, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(short, float) INSTANTIATE(ushort, float) diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp index 3562565a86..28574ac7e2 100644 --- a/src/backend/cuda/math.hpp +++ b/src/backend/cuda/math.hpp @@ -18,6 +18,7 @@ #endif //__CUDACC__ #include +#include #include #endif //__CUDACC_RTC__ @@ -191,6 +192,14 @@ inline __device__ uintl maxval() { return 1ULL << (8 * sizeof(uintl) - 1); } template<> +inline __device__ schar maxval() { + return 0x7f; +} +template<> +inline __device__ schar minval() { + return 0x80; +} +template<> inline __device__ char maxval() { return 0x7f; } @@ -260,6 +269,42 @@ __SDH__ double real(cdouble c) { return cuCreal(c); } __SDH__ float imag(cfloat c) { return cuCimagf(c); } __SDH__ double imag(cdouble c) { return cuCimag(c); } +template +static inline __DH__ auto is_nan(const T &val) -> bool { + return false; +} + +template<> +inline __DH__ auto is_nan(const float &val) -> bool { + return ::isnan(val); +} + +template<> +inline __DH__ auto is_nan(const double &val) -> bool { + return ::isnan(val); +} + +#ifdef __CUDA_ARCH__ +template<> +inline __device__ auto is_nan<__half>(const __half &val) -> bool { +#if __CUDA_ARCH__ >= 530 + return __hisnan(val); +#else + return ::isnan(__half2float(val)); +#endif +} +#endif + +template<> +inline auto is_nan(const cfloat &in) -> bool { + return ::isnan(real(in)) || ::isnan(imag(in)); +} + +template<> +inline auto is_nan(const cdouble &in) -> bool { + return ::isnan(real(in)) || ::isnan(imag(in)); +} + template T __SDH__ conj(T x) { return x; diff --git a/src/backend/cuda/max.cu b/src/backend/cuda/max.cu index 03f712b303..9fe7b92409 100644 --- a/src/backend/cuda/max.cu +++ b/src/backend/cuda/max.cu @@ -24,6 +24,7 @@ INSTANTIATE(af_max_t, uint, uint) INSTANTIATE(af_max_t, intl, intl) INSTANTIATE(af_max_t, uintl, uintl) INSTANTIATE(af_max_t, char, char) +INSTANTIATE(af_max_t, schar, schar) INSTANTIATE(af_max_t, uchar, uchar) INSTANTIATE(af_max_t, short, short) INSTANTIATE(af_max_t, ushort, ushort) diff --git a/src/backend/cuda/mean.cu b/src/backend/cuda/mean.cu index 9b1eea74e9..b4dab3b866 100644 --- a/src/backend/cuda/mean.cu +++ b/src/backend/cuda/mean.cu @@ -63,6 +63,7 @@ INSTANTIATE(uintl, double, double); INSTANTIATE(short, float, float); INSTANTIATE(ushort, float, float); INSTANTIATE(uchar, float, float); +INSTANTIATE(schar, float, float); INSTANTIATE(char, float, float); INSTANTIATE(cfloat, float, cfloat); INSTANTIATE(cdouble, double, cdouble); diff --git a/src/backend/cuda/meanshift.cpp b/src/backend/cuda/meanshift.cpp index d72d1aa041..83d12cb3ef 100644 --- a/src/backend/cuda/meanshift.cpp +++ b/src/backend/cuda/meanshift.cpp @@ -38,6 +38,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/medfilt.cpp b/src/backend/cuda/medfilt.cpp index c80c95c21f..cca97dd644 100644 --- a/src/backend/cuda/medfilt.cpp +++ b/src/backend/cuda/medfilt.cpp @@ -58,6 +58,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/memory.cpp b/src/backend/cuda/memory.cpp index dafbef1ce8..616547d6af 100644 --- a/src/backend/cuda/memory.cpp +++ b/src/backend/cuda/memory.cpp @@ -117,6 +117,7 @@ INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/cuda/min.cu b/src/backend/cuda/min.cu index 72a3f1beef..b0fad5733c 100644 --- a/src/backend/cuda/min.cu +++ b/src/backend/cuda/min.cu @@ -24,6 +24,7 @@ INSTANTIATE(af_min_t, uint, uint) INSTANTIATE(af_min_t, intl, intl) INSTANTIATE(af_min_t, uintl, uintl) INSTANTIATE(af_min_t, char, char) +INSTANTIATE(af_min_t, schar, schar) INSTANTIATE(af_min_t, uchar, uchar) INSTANTIATE(af_min_t, short, short) INSTANTIATE(af_min_t, ushort, ushort) diff --git a/src/backend/cuda/minmax_op.hpp b/src/backend/cuda/minmax_op.hpp index 4fcc995c0b..a2b7149a07 100644 --- a/src/backend/cuda/minmax_op.hpp +++ b/src/backend/cuda/minmax_op.hpp @@ -34,26 +34,12 @@ double cabs(const cdouble &in) { return (double)abs(in); } -template -static bool is_nan(const T &in) { - return in != in; -} - -template<> -bool is_nan(const cfloat &in) { - return in.x != in.x || in.y != in.y; -} - -template<> -bool is_nan(const cdouble &in) { - return in.x != in.x || in.y != in.y; -} - template struct MinMaxOp { T m_val; uint m_idx; MinMaxOp(T val, uint idx) : m_val(val), m_idx(idx) { + using arrayfire::cuda::is_nan; if (is_nan(val)) { m_val = common::Binary, op>::init(); } } @@ -71,6 +57,7 @@ struct MinMaxOp { T m_val; uint m_idx; MinMaxOp(T val, uint idx) : m_val(val), m_idx(idx) { + using arrayfire::cuda::is_nan; if (is_nan(val)) { m_val = common::Binary::init(); } } diff --git a/src/backend/cuda/moments.cpp b/src/backend/cuda/moments.cpp index 34c8cf753f..fa37b033e1 100644 --- a/src/backend/cuda/moments.cpp +++ b/src/backend/cuda/moments.cpp @@ -51,6 +51,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/cuda/morph.cpp b/src/backend/cuda/morph.cpp index a49fd5a40e..f09f20bded 100644 --- a/src/backend/cuda/morph.cpp +++ b/src/backend/cuda/morph.cpp @@ -53,6 +53,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/nearest_neighbour.cu b/src/backend/cuda/nearest_neighbour.cu index ca6a11a1c6..dc10695f8a 100644 --- a/src/backend/cuda/nearest_neighbour.cu +++ b/src/backend/cuda/nearest_neighbour.cu @@ -67,6 +67,7 @@ INSTANTIATE(int, int) INSTANTIATE(uint, uint) INSTANTIATE(intl, intl) INSTANTIATE(uintl, uintl) +INSTANTIATE(schar, int) INSTANTIATE(uchar, uint) INSTANTIATE(short, int) INSTANTIATE(ushort, uint) diff --git a/src/backend/cuda/pad_array_borders.cpp b/src/backend/cuda/pad_array_borders.cpp index bf41b5f2e7..af563733d2 100644 --- a/src/backend/cuda/pad_array_borders.cpp +++ b/src/backend/cuda/pad_array_borders.cpp @@ -48,6 +48,7 @@ INSTANTIATE_PAD_ARRAY_BORDERS(int) INSTANTIATE_PAD_ARRAY_BORDERS(uint) INSTANTIATE_PAD_ARRAY_BORDERS(intl) INSTANTIATE_PAD_ARRAY_BORDERS(uintl) +INSTANTIATE_PAD_ARRAY_BORDERS(schar) INSTANTIATE_PAD_ARRAY_BORDERS(uchar) INSTANTIATE_PAD_ARRAY_BORDERS(char) INSTANTIATE_PAD_ARRAY_BORDERS(ushort) diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp index 3fab99bb7f..0de2451c4d 100644 --- a/src/backend/cuda/platform.cpp +++ b/src/backend/cuda/platform.cpp @@ -49,6 +49,7 @@ #include #include #include +#include using std::call_once; using std::make_unique; @@ -123,8 +124,10 @@ unique_handle *nnManager(const int deviceId) { if (!(*handle)) { getLogger()->error("Error initalizing cuDNN"); } }); if (error) { - string error_msg = fmt::format("Error initializing cuDNN({}): {}.", - error, errorString(error)); + string error_msg = fmt::format( + "Error initializing cuDNN({}): {}.", + static_cast::type>(error), + errorString(error)); AF_ERROR(error_msg, AF_ERR_RUNTIME); } CUDNN_CHECK(getCudnnPlugin().cudnnSetStream(cudnnHandles[deviceId], @@ -208,6 +211,12 @@ DeviceManager::~DeviceManager() { } } +bool isDeviceBufferAccessible(int buf_device_id, int execution_id) { + DeviceManager &mngr = DeviceManager::getInstance(); + return buf_device_id == execution_id || + mngr.device_peer_access_map[buf_device_id][execution_id]; +} + int getBackend() { return AF_BACKEND_CUDA; } string getDeviceInfo(int device) noexcept { diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp index cac1281b59..be9f0b9996 100644 --- a/src/backend/cuda/platform.hpp +++ b/src/backend/cuda/platform.hpp @@ -88,6 +88,13 @@ cudaStream_t getStream(int device); cudaStream_t getActiveStream(); +/// Returns true if the buffer on device buf_device_id can be accessed by +/// kernels on device execution_id +/// +/// \param[in] buf_device_id The device id of the buffer +/// \param[in] execution_id The device where the buffer will be accessed. +bool isDeviceBufferAccessible(int buf_device_id, int execution_id); + /// Return a handle to the stream for the device. /// /// \param[in] device The device of the returned stream diff --git a/src/backend/cuda/plot.cpp b/src/backend/cuda/plot.cpp index e012377305..e69b149790 100644 --- a/src/backend/cuda/plot.cpp +++ b/src/backend/cuda/plot.cpp @@ -70,6 +70,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace cuda diff --git a/src/backend/cuda/product.cu b/src/backend/cuda/product.cu index c4fff43b93..fb26c95562 100644 --- a/src/backend/cuda/product.cu +++ b/src/backend/cuda/product.cu @@ -24,6 +24,7 @@ INSTANTIATE(af_mul_t, uint, uint) INSTANTIATE(af_mul_t, intl, intl) INSTANTIATE(af_mul_t, uintl, uintl) INSTANTIATE(af_mul_t, char, int) +INSTANTIATE(af_mul_t, schar, int) INSTANTIATE(af_mul_t, uchar, uint) INSTANTIATE(af_mul_t, short, int) INSTANTIATE(af_mul_t, ushort, uint) diff --git a/src/backend/cuda/qr.cpp b/src/backend/cuda/qr.cpp index c28a41523f..f388944127 100644 --- a/src/backend/cuda/qr.cpp +++ b/src/backend/cuda/qr.cpp @@ -67,6 +67,16 @@ struct mqr_func_def_t { int, T *, int, int *); }; +template +struct mqr_buf_func_def_t { + using mqr_buf_func_def = cusolverStatus_t (*)(cusolverDnHandle_t, + cublasSideMode_t, + cublasOperation_t, int, int, int, + const T *, int, const T *, T *, + int, int *); +}; + + #define QR_FUNC_DEF(FUNC) \ template \ typename FUNC##_func_def_t::FUNC##_func_def FUNC##_func(); \ @@ -94,15 +104,25 @@ QR_FUNC(geqrf, double, D) QR_FUNC(geqrf, cfloat, C) QR_FUNC(geqrf, cdouble, Z) -#define MQR_FUNC_DEF(FUNC) \ - template \ - typename FUNC##_func_def_t::FUNC##_func_def FUNC##_func(); +#define MQR_FUNC_DEF(FUNC) \ + template \ + typename FUNC##_func_def_t::FUNC##_func_def FUNC##_func(); \ + \ + template \ + typename FUNC##_buf_func_def_t::FUNC##_buf_func_def FUNC##_buf_func(); #define MQR_FUNC(FUNC, TYPE, PREFIX) \ template<> \ typename FUNC##_func_def_t::FUNC##_func_def FUNC##_func() { \ return (FUNC##_func_def_t::FUNC##_func_def) & \ cusolverDn##PREFIX; \ + } \ + \ + template<> \ + typename FUNC##_buf_func_def_t::FUNC##_buf_func_def \ + FUNC##_buf_func() { \ + return (FUNC##_buf_func_def_t::FUNC##_buf_func_def) & \ + cusolverDn##PREFIX##_bufferSize; \ } MQR_FUNC_DEF(mqr) @@ -143,6 +163,13 @@ void qr(Array &q, Array &r, Array &t, const Array &in) { dim4 qdims(M, mn); q = identity(qdims); + CUSOLVER_CHECK(mqr_buf_func()( + solverDnHandle(), CUBLAS_SIDE_LEFT, CUBLAS_OP_N, q.dims()[0], + q.dims()[1], min(M, N), in_copy.get(), in_copy.strides()[1], t.get(), + q.get(), q.strides()[1], &lwork)); + + workspace = memAlloc(lwork); + CUSOLVER_CHECK(mqr_func()( solverDnHandle(), CUBLAS_SIDE_LEFT, CUBLAS_OP_N, q.dims()[0], q.dims()[1], min(M, N), in_copy.get(), in_copy.strides()[1], t.get(), diff --git a/src/backend/cuda/random_engine.cu b/src/backend/cuda/random_engine.cu index a63ead0bf8..26cdbdc23b 100644 --- a/src/backend/cuda/random_engine.cu +++ b/src/backend/cuda/random_engine.cu @@ -143,6 +143,7 @@ INSTANTIATE_UNIFORM(uint) INSTANTIATE_UNIFORM(intl) INSTANTIATE_UNIFORM(uintl) INSTANTIATE_UNIFORM(char) +INSTANTIATE_UNIFORM(schar) INSTANTIATE_UNIFORM(uchar) INSTANTIATE_UNIFORM(short) INSTANTIATE_UNIFORM(ushort) diff --git a/src/backend/cuda/range.cpp b/src/backend/cuda/range.cpp index 55a2553649..f821f283f7 100644 --- a/src/backend/cuda/range.cpp +++ b/src/backend/cuda/range.cpp @@ -48,6 +48,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/reorder.cpp b/src/backend/cuda/reorder.cpp index c81fd02f6a..286dcde6ad 100644 --- a/src/backend/cuda/reorder.cpp +++ b/src/backend/cuda/reorder.cpp @@ -43,6 +43,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) diff --git a/src/backend/cuda/reshape.cpp b/src/backend/cuda/reshape.cpp index 9d6e57549f..329b7883cb 100644 --- a/src/backend/cuda/reshape.cpp +++ b/src/backend/cuda/reshape.cpp @@ -49,6 +49,8 @@ Array reshape(const Array &in, const dim4 &outDims, dim4 const &, short, double); \ template Array reshape( \ Array const &, dim4 const &, ushort, double); \ + template Array reshape(Array const &, \ + dim4 const &, schar, double); \ template Array reshape(Array const &, \ dim4 const &, uchar, double); \ template Array reshape(Array const &, \ @@ -64,6 +66,7 @@ INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(half) diff --git a/src/backend/cuda/resize.cpp b/src/backend/cuda/resize.cpp index 97dc8a7da8..dec6f09d26 100644 --- a/src/backend/cuda/resize.cpp +++ b/src/backend/cuda/resize.cpp @@ -41,6 +41,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cuda/rotate.cpp b/src/backend/cuda/rotate.cpp index 2f46894aef..7edb0de7a6 100644 --- a/src/backend/cuda/rotate.cpp +++ b/src/backend/cuda/rotate.cpp @@ -36,6 +36,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cuda/scan.cpp b/src/backend/cuda/scan.cpp index 10002cbbad..cf3f2a0b70 100644 --- a/src/backend/cuda/scan.cpp +++ b/src/backend/cuda/scan.cpp @@ -47,6 +47,7 @@ Array scan(const Array& in, const int dim, bool inclusive_scan) { INSTANTIATE_SCAN(ROp, uintl, uintl) \ INSTANTIATE_SCAN(ROp, char, int) \ INSTANTIATE_SCAN(ROp, char, uint) \ + INSTANTIATE_SCAN(ROp, schar, int) \ INSTANTIATE_SCAN(ROp, uchar, uint) \ INSTANTIATE_SCAN(ROp, short, int) \ INSTANTIATE_SCAN(ROp, ushort, uint) diff --git a/src/backend/cuda/select.cpp b/src/backend/cuda/select.cpp index b13df55bfe..0b78263efd 100644 --- a/src/backend/cuda/select.cpp +++ b/src/backend/cuda/select.cpp @@ -127,6 +127,7 @@ INSTANTIATE(uint); INSTANTIATE(intl); INSTANTIATE(uintl); INSTANTIATE(char); +INSTANTIATE(schar); INSTANTIATE(uchar); INSTANTIATE(short); INSTANTIATE(ushort); diff --git a/src/backend/cuda/set.cu b/src/backend/cuda/set.cu index fbbbc28c0a..d558d6e938 100644 --- a/src/backend/cuda/set.cu +++ b/src/backend/cuda/set.cu @@ -122,6 +122,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/shift.cpp b/src/backend/cuda/shift.cpp index 82aab5e1fe..f073d3c844 100644 --- a/src/backend/cuda/shift.cpp +++ b/src/backend/cuda/shift.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -18,9 +19,8 @@ using af::dim4; using arrayfire::common::Node_ptr; -using arrayfire::common::ShiftNodeBase; - using arrayfire::cuda::jit::BufferNode; +using arrayfire::cuda::jit::ShiftNode; using std::array; using std::make_shared; @@ -29,8 +29,6 @@ using std::string; namespace arrayfire { namespace cuda { -template -using ShiftNode = ShiftNodeBase>; template Array shift(const Array &in, const int sdims[4]) { @@ -70,6 +68,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cuda/sobel.cpp b/src/backend/cuda/sobel.cpp index 5200f69a45..1861d0c76c 100644 --- a/src/backend/cuda/sobel.cpp +++ b/src/backend/cuda/sobel.cpp @@ -38,6 +38,7 @@ INSTANTIATE(double, double) INSTANTIATE(int, int) INSTANTIATE(uint, int) INSTANTIATE(char, int) +INSTANTIATE(schar, int) INSTANTIATE(uchar, int) INSTANTIATE(short, int) INSTANTIATE(ushort, int) diff --git a/src/backend/cuda/solve.cu b/src/backend/cuda/solve.cu index 884d7735b1..568e44b136 100644 --- a/src/backend/cuda/solve.cu +++ b/src/backend/cuda/solve.cu @@ -164,6 +164,13 @@ struct mqr_solve_func_def_t { const T *, int, const T *, T *, int, T *, int, int *); }; +template +struct mqr_solve_buf_func_def_t { + typedef cusolverStatus_t (*mqr_solve_buf_func_def)( + cusolverDnHandle_t, cublasSideMode_t, cublasOperation_t, int, int, int, + const T *, int, const T *, T *, int, int *); +}; + #define QR_FUNC_DEF(FUNC) \ template \ static typename FUNC##_solve_func_def_t::FUNC##_solve_func_def \ @@ -195,17 +202,28 @@ QR_FUNC(geqrf, double, D) QR_FUNC(geqrf, cfloat, C) QR_FUNC(geqrf, cdouble, Z) -#define MQR_FUNC_DEF(FUNC) \ - template \ - static typename FUNC##_solve_func_def_t::FUNC##_solve_func_def \ - FUNC##_solve_func(); - -#define MQR_FUNC(FUNC, TYPE, PREFIX) \ - template<> \ - typename FUNC##_solve_func_def_t::FUNC##_solve_func_def \ - FUNC##_solve_func() { \ - return (FUNC##_solve_func_def_t::FUNC##_solve_func_def) & \ - cusolverDn##PREFIX; \ +#define MQR_FUNC_DEF(FUNC) \ + template \ + static typename FUNC##_solve_func_def_t::FUNC##_solve_func_def \ + FUNC##_solve_func(); \ + \ + template \ + static typename FUNC##_solve_buf_func_def_t::FUNC##_solve_buf_func_def \ + FUNC##_solve_buf_func(); + +#define MQR_FUNC(FUNC, TYPE, PREFIX) \ + template<> \ + typename FUNC##_solve_func_def_t::FUNC##_solve_func_def \ + FUNC##_solve_func() { \ + return (FUNC##_solve_func_def_t::FUNC##_solve_func_def) & \ + cusolverDn##PREFIX; \ + } \ + \ + template<> \ + typename FUNC##_solve_buf_func_def_t::FUNC##_solve_buf_func_def \ + FUNC##_solve_buf_func() { \ + return (FUNC##_solve_buf_func_def_t::FUNC##_solve_buf_func_def) & \ + cusolverDn##PREFIX##_bufferSize; \ } MQR_FUNC_DEF(mqr) @@ -393,6 +411,13 @@ Array leastSquares(const Array &a, const Array &b) { B.resetDims(dim4(N, K)); // matmul(Q, Bpad) + CUSOLVER_CHECK(mqr_solve_buf_func()( + solverDnHandle(), CUBLAS_SIDE_LEFT, CUBLAS_OP_N, B.dims()[0], + B.dims()[1], A.dims()[0], A.get(), A.strides()[1], t.get(), B.get(), + B.strides()[1], &lwork)); + + workspace = memAlloc(lwork); + CUSOLVER_CHECK(mqr_solve_func()( solverDnHandle(), CUBLAS_SIDE_LEFT, CUBLAS_OP_N, B.dims()[0], B.dims()[1], A.dims()[0], A.get(), A.strides()[1], t.get(), B.get(), @@ -427,10 +452,17 @@ Array leastSquares(const Array &a, const Array &b) { t.get(), workspace.get(), lwork, info.get())); // matmul(Q1, B) + CUSOLVER_CHECK(mqr_solve_buf_func()( + solverDnHandle(), CUBLAS_SIDE_LEFT, trans(), M, K, N, A.get(), + A.strides()[1], t.get(), B.get(), B.strides()[1], &lwork)); + + workspace = memAlloc(lwork); + CUSOLVER_CHECK(mqr_solve_func()( solverDnHandle(), CUBLAS_SIDE_LEFT, trans(), M, K, N, A.get(), A.strides()[1], t.get(), B.get(), B.strides()[1], workspace.get(), lwork, info.get())); + // tri_solve(R1, Bt) A.resetDims(dim4(N, N)); B.resetDims(dim4(N, K)); diff --git a/src/backend/cuda/sort.cu b/src/backend/cuda/sort.cu index 9970ddd8b2..d56899a87d 100644 --- a/src/backend/cuda/sort.cu +++ b/src/backend/cuda/sort.cu @@ -54,6 +54,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/sort_by_key.cu b/src/backend/cuda/sort_by_key.cu index bd19d16240..21d9efc5b2 100644 --- a/src/backend/cuda/sort_by_key.cu +++ b/src/backend/cuda/sort_by_key.cu @@ -67,6 +67,7 @@ void sort_by_key(Array &okey, Array &oval, const Array &ikey, INSTANTIATE(Tk, short) \ INSTANTIATE(Tk, ushort) \ INSTANTIATE(Tk, char) \ + INSTANTIATE(Tk, schar) \ INSTANTIATE(Tk, uchar) \ INSTANTIATE(Tk, intl) \ INSTANTIATE(Tk, uintl) @@ -78,6 +79,7 @@ INSTANTIATE1(uint) INSTANTIATE1(short) INSTANTIATE1(ushort) INSTANTIATE1(char) +INSTANTIATE1(schar) INSTANTIATE1(uchar) INSTANTIATE1(intl) INSTANTIATE1(uintl) diff --git a/src/backend/cuda/sort_index.cu b/src/backend/cuda/sort_index.cu index 039e77a147..d923f7c6e9 100644 --- a/src/backend/cuda/sort_index.cu +++ b/src/backend/cuda/sort_index.cu @@ -63,6 +63,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/sum.cu b/src/backend/cuda/sum.cu index 44cfec9449..6a52c2c369 100644 --- a/src/backend/cuda/sum.cu +++ b/src/backend/cuda/sum.cu @@ -29,6 +29,8 @@ INSTANTIATE(af_add_t, uintl, uintl) INSTANTIATE(af_add_t, uintl, double) INSTANTIATE(af_add_t, char, int) INSTANTIATE(af_add_t, char, float) +INSTANTIATE(af_add_t, schar, int) +INSTANTIATE(af_add_t, schar, float) INSTANTIATE(af_add_t, uchar, uint) INSTANTIATE(af_add_t, uchar, float) INSTANTIATE(af_add_t, short, int) diff --git a/src/backend/cuda/surface.cpp b/src/backend/cuda/surface.cpp index bef751239b..61f3457036 100644 --- a/src/backend/cuda/surface.cpp +++ b/src/backend/cuda/surface.cpp @@ -71,6 +71,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace cuda diff --git a/src/backend/cuda/susan.cpp b/src/backend/cuda/susan.cpp index 4d0fcc078c..5f1d07d913 100644 --- a/src/backend/cuda/susan.cpp +++ b/src/backend/cuda/susan.cpp @@ -74,6 +74,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/tile.cpp b/src/backend/cuda/tile.cpp index f93982eb43..edd2a7b686 100644 --- a/src/backend/cuda/tile.cpp +++ b/src/backend/cuda/tile.cpp @@ -48,6 +48,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cuda/transform.cpp b/src/backend/cuda/transform.cpp index baba9b1a04..af8b561191 100644 --- a/src/backend/cuda/transform.cpp +++ b/src/backend/cuda/transform.cpp @@ -9,6 +9,7 @@ #include +#include #include #include @@ -19,7 +20,11 @@ template void transform(Array &out, const Array &in, const Array &tf, const af::interpType method, const bool inverse, const bool perspective) { - kernel::transform(out, in, tf, inverse, perspective, method, + // TODO: Temporary Fix, must fix handling subarrays upstream + // tf has to be linear, although offset is allowed. + const Array tf_Lin = tf.isLinear() ? tf : copyArray(tf); + + kernel::transform(out, in, tf_Lin, inverse, perspective, method, interpOrder(method)); } @@ -37,6 +42,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cuda/transpose.cpp b/src/backend/cuda/transpose.cpp index faa4659b68..03d6f3b91d 100644 --- a/src/backend/cuda/transpose.cpp +++ b/src/backend/cuda/transpose.cpp @@ -45,6 +45,7 @@ INSTANTIATE(cdouble) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/cuda/transpose_inplace.cpp b/src/backend/cuda/transpose_inplace.cpp index ff89730d47..dcc8c5664b 100644 --- a/src/backend/cuda/transpose_inplace.cpp +++ b/src/backend/cuda/transpose_inplace.cpp @@ -37,6 +37,7 @@ INSTANTIATE(cdouble) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/cuda/triangle.cpp b/src/backend/cuda/triangle.cpp index 4ec0a04e6f..c32e984626 100644 --- a/src/backend/cuda/triangle.cpp +++ b/src/backend/cuda/triangle.cpp @@ -48,6 +48,7 @@ INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/types.hpp b/src/backend/cuda/types.hpp index 34815cba66..2230948f3a 100644 --- a/src/backend/cuda/types.hpp +++ b/src/backend/cuda/types.hpp @@ -35,6 +35,7 @@ namespace cuda { using cdouble = cuDoubleComplex; using cfloat = cuFloatComplex; using intl = long long; +using schar = signed char; using uchar = unsigned char; using uint = unsigned int; using uintl = unsigned long long; @@ -82,6 +83,10 @@ inline const char *shortname(bool caps) { return caps ? "J" : "j"; } template<> +inline const char *shortname(bool caps) { + return caps ? "A" : "a"; // TODO +} +template<> inline const char *shortname(bool caps) { return caps ? "V" : "v"; } @@ -120,6 +125,7 @@ SPECIALIZE(double) SPECIALIZE(cfloat) SPECIALIZE(cdouble) SPECIALIZE(char) +SPECIALIZE(signed char) SPECIALIZE(unsigned char) SPECIALIZE(short) SPECIALIZE(unsigned short) diff --git a/src/backend/cuda/unwrap.cpp b/src/backend/cuda/unwrap.cpp index 6eae7d428b..9d96aec1d9 100644 --- a/src/backend/cuda/unwrap.cpp +++ b/src/backend/cuda/unwrap.cpp @@ -55,6 +55,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/cuda/vector_field.cpp b/src/backend/cuda/vector_field.cpp index 2868979772..a0528cddb1 100644 --- a/src/backend/cuda/vector_field.cpp +++ b/src/backend/cuda/vector_field.cpp @@ -105,6 +105,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace cuda diff --git a/src/backend/cuda/where.cpp b/src/backend/cuda/where.cpp index efd488d26e..862b25fa24 100644 --- a/src/backend/cuda/where.cpp +++ b/src/backend/cuda/where.cpp @@ -36,6 +36,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/cuda/wrap.cpp b/src/backend/cuda/wrap.cpp index d8963cacd9..dd7901cc0e 100644 --- a/src/backend/cuda/wrap.cpp +++ b/src/backend/cuda/wrap.cpp @@ -44,6 +44,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp index 3a9fbff3be..57c8f111ee 100644 --- a/src/backend/oneapi/Array.cpp +++ b/src/backend/oneapi/Array.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -87,6 +88,13 @@ void verifyTypeSupport() { } } // namespace +template +void checkAndMigrate(const Array &arr) { + if (arr.getDevId() != detail::getActiveDeviceId()) { + AF_ERROR("Input Array not created on current device", AF_ERR_DEVICE); + } +} + template Array::Array(const dim4 &dims) : info(getActiveDeviceId(), dims, 0, calcStrides(dims), @@ -125,10 +133,10 @@ Array::Array(const dim4 &dims, const T *const in_data) static_assert( offsetof(Array, info) == 0, "Array::info must be the first member variable of Array"); - // getQueue().enqueueWriteBuffer(*data.get(), CL_TRUE, 0, - // sizeof(T) * info.elements(), in_data); getQueue() - .submit([&](sycl::handler &h) { h.copy(in_data, data->get_access(h)); }) + .submit([&](sycl::handler &h) { + h.copy(in_data, data->get_access(h, sycl::range(info.elements()))); + }) .wait(); } @@ -145,7 +153,8 @@ Array::Array(const af::dim4 &dims, buffer *const mem, size_t offset, if (copy) { getQueue() .submit([&](sycl::handler &h) { - h.copy(mem->get_access(h), data->get_access(h)); + h.copy(mem->get_access(h, sycl::range(info.elements())), + data->get_access(h)); }) .wait(); } @@ -193,8 +202,9 @@ Array::Array(const dim4 &dims, const dim4 &strides, dim_t offset_, } else { data = memAlloc(info.elements()); getQueue() - .submit( - [&](sycl::handler &h) { h.copy(in_data, data->get_access(h)); }) + .submit([&](sycl::handler &h) { + h.copy(in_data, data->get_access(h, sycl::range(info.total()))); + }) .wait(); } } @@ -309,8 +319,13 @@ Node_ptr Array::getNode() const { template kJITHeuristics passesJitHeuristics(span root_nodes) { if (!evalFlag()) { return kJITHeuristics::Pass; } + static auto getLogger = [&] { return common::loggerFactory("jit"); }; for (const Node *n : root_nodes) { if (n->getHeight() > static_cast(getMaxJitSize())) { + AF_TRACE( + "JIT tree evaluated because of tree height exceeds limit: {} > " + "{}", + n->getHeight(), getMaxJitSize()); return kJITHeuristics::TreeHeight; } } @@ -383,17 +398,21 @@ kJITHeuristics passesJitHeuristics(span root_nodes) { bool isParamLimit = param_size >= max_param_size; - if (isParamLimit) { return kJITHeuristics::KernelParameterSize; } - // TODO(umar): check buffer limit for JIT kernel generation - // if (isBufferLimit) { return kJITHeuristics::MemoryPressure; } + if (isParamLimit) { + AF_TRACE( + "JIT tree evaluated because of kernel parameter size: {} >= {}", + param_size, max_param_size); + return kJITHeuristics::KernelParameterSize; + } + if (isBufferLimit) { + AF_TRACE("JIT tree evaluated because of memory pressure: {}", + info.total_buffer_size); + return kJITHeuristics::MemoryPressure; + } } return kJITHeuristics::Pass; } -// Doesn't make sense with sycl::buffer -// TODO: accessors? or return sycl::buffer? -// TODO: return accessor.get_pointer() for access::target::global_buffer or -// (host_buffer?) template void *getDevicePtr(const Array &arr) { const buffer *buf = arr.device(); @@ -481,17 +500,15 @@ template void writeHostDataArray(Array &arr, const T *const data, const size_t bytes) { if (!arr.isOwner()) { arr = copyArray(arr); } + auto arr_get = arr.get(); getQueue() .submit([&](sycl::handler &h) { - buffer &buf = *arr.get(); - // auto offset_acc = buf.get_access(h, sycl::range, sycl::id<>) - // TODO: offset accessor - auto offset_acc = buf.get_access(h); - h.copy(data, offset_acc); + auto host_acc = + arr_get->template get_access( + h, sycl::range(bytes / sizeof(T)), arr.getOffset()); + h.copy(data, host_acc); }) .wait(); - // getQueue().enqueueWriteBuffer(*arr.get(), CL_TRUE, arr.getOffset(), - // bytes, data); } template @@ -499,14 +516,16 @@ void writeDeviceDataArray(Array &arr, const void *const data, const size_t bytes) { if (!arr.isOwner()) { arr = copyArray(arr); } - // clRetainMemObject( - // reinterpret_cast *>(const_cast(data))); - // buffer data_buf = - // buffer(reinterpret_cast*>(const_cast(data))); - - ONEAPI_NOT_SUPPORTED("writeDeviceDataArray not supported"); - // getQueue().enqueueCopyBuffer(data_buf, buf, 0, - // static_cast(arr.getOffset()), bytes); + sycl::buffer *dataptr = + static_cast *>(const_cast(data)); + auto arr_get = arr.get(); + getQueue().submit([&](sycl::handler &h) { + auto src_acc = dataptr->template get_access( + h, sycl::range(bytes / sizeof(T))); + auto dst_acc = arr_get->template get_access( + h, sycl::range(bytes / sizeof(T)), arr.getOffset()); + h.copy(src_acc, dst_acc); + }); } template @@ -554,7 +573,8 @@ size_t Array::getAllocatedBytes() const { template kJITHeuristics passesJitHeuristics(span node); \ template void *getDevicePtr(const Array &arr); \ template void Array::setDataDims(const dim4 &new_dims); \ - template size_t Array::getAllocatedBytes() const; + template size_t Array::getAllocatedBytes() const; \ + template void checkAndMigrate(const Array &arr); INSTANTIATE(float) INSTANTIATE(double) @@ -562,6 +582,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp index a6ca6c402c..5e7ec490f1 100644 --- a/src/backend/oneapi/Array.hpp +++ b/src/backend/oneapi/Array.hpp @@ -23,6 +23,7 @@ #include #include #include +#include #include enum class kJITHeuristics; @@ -50,6 +51,13 @@ using af::dim4; template class Array; +/// Checks if the Array object can be migrated to the current device and if not, +/// an error is thrown +/// +/// \param[in] arr The Array that will be checked. +template +void checkAndMigrate(const Array &arr); + template void evalMultiple(std::vector *> arrays); @@ -258,6 +266,22 @@ class Array { return data.get(); } + template + sycl::buffer getBufferWithOffset(dim_t offset = -1) const { + offset = (offset == -1) ? getOffset() : offset; + dim_t sz_remaining = data_dims.elements() - offset; + if constexpr (std::is_same_v) { + if (offset == 0) { return *get(); } + return sycl::buffer(*get(), sycl::id<1>(offset), + sycl::range<1>(sz_remaining)); + } else { + if (offset == 0) { return get()->template reinterpret(); } + return sycl::buffer(*get(), sycl::id<1>(offset), + sycl::range<1>(sz_remaining)) + .template reinterpret(); + } + } + int useCount() const { return data.use_count(); } dim_t getOffset() const { return info.getOffset(); } diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt index 46e20c88d4..a41d3fa3b7 100644 --- a/src/backend/oneapi/CMakeLists.txt +++ b/src/backend/oneapi/CMakeLists.txt @@ -1,9 +1,13 @@ -# Copyright (c) 2022, ArrayFire -# All rights reserved. +#Copyright(c) 2022, ArrayFire +#All rights reserved. # -# This file is distributed under 3-clause BSD license. -# The complete license agreement can be obtained at: -# http://arrayfire.com/licenses/BSD-3-Clause +#This file is distributed under 3 - clause BSD license. +#The complete license agreement can be obtained at: +#http: // arrayfire.com/licenses/BSD-3-Clause + +if(AF_BUILD_ONEAPI) + enable_language(SYCL) +endif() include(InternalUtils) include(build_cl2hpp) @@ -95,6 +99,7 @@ add_library(afoneapi ireduce.hpp jit.cpp jit/BufferNode.hpp + jit/ShiftNode.hpp jit/kernel_generators.hpp join.cpp join.hpp @@ -218,6 +223,11 @@ target_sources(afoneapi kernel/convolve_separable.cpp kernel/diagonal.hpp kernel/diff.hpp + kernel/fftconvolve_common.hpp + kernel/fftconvolve_multiply.hpp + kernel/fftconvolve_pack.hpp + kernel/fftconvolve_pad.hpp + kernel/fftconvolve_reorder.hpp kernel/histogram.hpp kernel/iir.hpp kernel/identity.hpp @@ -236,6 +246,7 @@ target_sources(afoneapi kernel/range.hpp kernel/reduce.hpp kernel/reduce_all.hpp + kernel/reduce_by_key.hpp kernel/reduce_first.hpp kernel/reduce_dim.hpp kernel/reorder.hpp @@ -243,6 +254,8 @@ target_sources(afoneapi kernel/scan_dim.hpp kernel/sort.hpp kernel/sort_by_key.hpp + kernel/sparse.hpp + kernel/sparse_arith.hpp kernel/transpose.hpp kernel/transpose_inplace.hpp kernel/triangle.hpp @@ -252,6 +265,31 @@ target_sources(afoneapi kernel/wrap_dilated.hpp ) +function(set_sycl_language) + foreach(target ${ARGV}) + set_target_properties(${target} + PROPERTIES + LINKER_LANGUAGE SYCL) + + get_target_property(target_type ${target} TYPE) + if(NOT (${target_type} STREQUAL "INTERFACE_LIBRARY")) + target_compile_options(${target} PRIVATE ${MSVC_RUNTIME}) + endif() + + get_target_property(TGT_SOURCES ${target} SOURCES) + if(NOT TGT_SOURCES) + get_target_property(TGT_SOURCES ${target} INTERFACE_SOURCES) + endif() + + foreach(FILE ${TGT_SOURCES}) + get_filename_component(FILE_EXTENSION ${FILE} EXT) + if(FILE_EXTENSION STREQUAL ".cpp") + set_source_files_properties(${FILE} PROPERTIES LANGUAGE SYCL) + endif() + endforeach() + endforeach() +endfunction() + set(kernel_src ${CMAKE_CURRENT_SOURCE_DIR}/../opencl/kernel/KParam.hpp ${CMAKE_CURRENT_SOURCE_DIR}/../opencl/kernel/jit.cl @@ -293,10 +331,11 @@ target_include_directories(afoneapi target_compile_options(afoneapi PRIVATE - -fsycl + $<$: -fno-sycl-id-queries-fit-in-int -sycl-std=2020 - -fno-sycl-rdc + $<$: -fno-sycl-rdc> + > ) target_compile_definitions(afoneapi @@ -307,15 +346,17 @@ target_compile_definitions(afoneapi CL_HPP_TARGET_OPENCL_VERSION=300 CL_HPP_MINIMUM_OPENCL_VERSION=110 CL_HPP_ENABLE_EXCEPTIONS + AF_MKL_INTERFACE_SIZE=${MKL_INTERFACE_INTEGER_SIZE} ) +if(MKL_INTERFACE_INTEGER_SIZE EQUAL 8) + target_compile_definitions(afoneapi PRIVATE MKL_ILP64) +endif() cmake_host_system_information(RESULT NumberOfThreads QUERY NUMBER_OF_LOGICAL_CORES) target_link_libraries(afoneapi PRIVATE - -fsycl - -fvisibility-inlines-hidden c_api_interface cpp_api_interface oneapi_sort_by_key @@ -323,14 +364,21 @@ target_link_libraries(afoneapi OpenCL::OpenCL OpenCL::cl2hpp -fno-sycl-id-queries-fit-in-int - -fno-sycl-rdc - -fsycl-device-code-split=per_kernel - -fsycl-link-huge-device-code + $<$:-flink-huge-device-code> + $<$:-fvisibility-inlines-hidden> + $<$:-fno-sycl-rdc> + $<$:-Wl,--build-id> -fsycl-max-parallel-link-jobs=${NumberOfThreads} - MKL::MKL_DPCPP + MKL::MKL_SYCL ) + set_sycl_language(afcommon_interface + oneapi_sort_by_key + c_api_interface + cpp_api_interface + afoneapi) + -af_split_debug_info(afoneapi ${AF_INSTALL_LIB_DIR}) +#af_split_debug_info(afoneapi ${AF_INSTALL_LIB_DIR}) install(TARGETS afoneapi EXPORT ArrayFireoneAPITargets diff --git a/src/backend/oneapi/Event.cpp b/src/backend/oneapi/Event.cpp index 056c6cf950..60bc8bcb77 100644 --- a/src/backend/oneapi/Event.cpp +++ b/src/backend/oneapi/Event.cpp @@ -24,56 +24,50 @@ namespace arrayfire { namespace oneapi { /// \brief Creates a new event and marks it in the queue Event makeEvent(sycl::queue& queue) { - ONEAPI_NOT_SUPPORTED("makeEvent"); - return Event(); + Event e; + if (e.create() == 0) { e.mark(queue); } + return e; } af_event createEvent() { - ONEAPI_NOT_SUPPORTED(""); - return 0; - // auto e = make_unique(); - // // Ensure the default CL command queue is initialized - // getQueue(); - // if (e->create() != CL_SUCCESS) { - // AF_ERROR("Could not create event", AF_ERR_RUNTIME); - // } - // Event& ref = *e.release(); - // return getHandle(ref); + auto e = make_unique(); + // Ensure the default CL command queue is initialized + getQueue(); + if (e->create() != 0) { + AF_ERROR("Could not create event", AF_ERR_RUNTIME); + } + Event& ref = *e.release(); + return getHandle(ref); } void markEventOnActiveQueue(af_event eventHandle) { - ONEAPI_NOT_SUPPORTED(""); - // Event& event = getEvent(eventHandle); - //// Use the currently-active stream - // if (event.mark(getQueue()()) != CL_SUCCESS) { - // AF_ERROR("Could not mark event on active queue", AF_ERR_RUNTIME); - //} + Event& event = getEvent(eventHandle); + // Use the currently-active stream + if (event.mark(getQueue()) != 0) { + AF_ERROR("Could not mark event on active queue", AF_ERR_RUNTIME); + } } void enqueueWaitOnActiveQueue(af_event eventHandle) { - ONEAPI_NOT_SUPPORTED(""); - // Event& event = getEvent(eventHandle); - //// Use the currently-active stream - // if (event.enqueueWait(getQueue()()) != CL_SUCCESS) { - // AF_ERROR("Could not enqueue wait on active queue for event", - // AF_ERR_RUNTIME); - //} + Event& event = getEvent(eventHandle); + // Use the currently-active stream + if (event.enqueueWait(getQueue()) != 0) { + AF_ERROR("Could not enqueue wait on active queue for event", + AF_ERR_RUNTIME); + } } void block(af_event eventHandle) { - ONEAPI_NOT_SUPPORTED(""); - // Event& event = getEvent(eventHandle); - // if (event.block() != CL_SUCCESS) { - // AF_ERROR("Could not block on active queue for event", AF_ERR_RUNTIME); - //} + Event& event = getEvent(eventHandle); + if (event.block() != 0) { + AF_ERROR("Could not block on active queue for event", AF_ERR_RUNTIME); + } } af_event createAndMarkEvent() { - ONEAPI_NOT_SUPPORTED(""); - return 0; - // af_event handle = createEvent(); - // markEventOnActiveQueue(handle); - // return handle; + af_event handle = createEvent(); + markEventOnActiveQueue(handle); + return handle; } } // namespace oneapi diff --git a/src/backend/oneapi/Event.hpp b/src/backend/oneapi/Event.hpp index ae7fdd8c29..44af139cda 100644 --- a/src/backend/oneapi/Event.hpp +++ b/src/backend/oneapi/Event.hpp @@ -17,33 +17,32 @@ namespace arrayfire { namespace oneapi { class OneAPIEventPolicy { public: - using EventType = sycl::event; + using EventType = sycl::event *; using QueueType = sycl::queue; - // using ErrorType = sycl::exception; //does this make sense using ErrorType = int; static ErrorType createAndMarkEvent(EventType *e) noexcept { - // Events are created when you mark them + *e = new sycl::event; return 0; } static ErrorType markEvent(EventType *e, QueueType stream) noexcept { - // return clEnqueueMarkerWithWaitList(stream, 0, nullptr, e); + **e = stream.ext_oneapi_submit_barrier(); return 0; } static ErrorType waitForEvent(EventType *e, QueueType stream) noexcept { - // return clEnqueueMarkerWithWaitList(stream, 1, e, nullptr); + stream.ext_oneapi_submit_barrier({**e}); return 0; } static ErrorType syncForEvent(EventType *e) noexcept { - // return clWaitForEvents(1, e); + (*e)->wait(); return 0; } static ErrorType destroyEvent(EventType *e) noexcept { - // return clReleaseEvent(*e); + delete *e; return 0; } }; diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp index 7df0a73f85..4a935c5e2c 100644 --- a/src/backend/oneapi/Param.hpp +++ b/src/backend/oneapi/Param.hpp @@ -27,6 +27,9 @@ struct Param { Param(const Param& other) = default; Param(Param&& other) = default; + dim_t* dims_ptr() { return info.dims; } + dim_t* strides_ptr() { return info.strides; } + // AF_DEPRECATED("Use Array") Param() : data(nullptr), info{{0, 0, 0, 0}, {0, 0, 0, 0}, 0} {} @@ -54,6 +57,9 @@ struct AParam { AParam(const AParam& other) = default; AParam(AParam&& other) = default; + dim_t* dims_ptr() { return dims.get(); } + dim_t* strides_ptr() { return strides.get(); } + // AF_DEPRECATED("Use Array") AParam() : data(), dims{0, 0, 0, 0}, strides{0, 0, 0, 0}, offset(0) {} @@ -72,7 +78,7 @@ struct AParam { return *data; } - void require(sycl::handler& h) { h.require(data); } + void require(sycl::handler& h) const { h.require(data); } operator KParam() const { return KParam{{dims[0], dims[1], dims[2], dims[3]}, diff --git a/src/backend/oneapi/all.cpp b/src/backend/oneapi/all.cpp index ad09e4aff1..e4e86232d2 100644 --- a/src/backend/oneapi/all.cpp +++ b/src/backend/oneapi/all.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_and_t, uint, char) INSTANTIATE(af_and_t, intl, char) INSTANTIATE(af_and_t, uintl, char) INSTANTIATE(af_and_t, char, char) +INSTANTIATE(af_and_t, schar, char) INSTANTIATE(af_and_t, uchar, char) INSTANTIATE(af_and_t, short, char) INSTANTIATE(af_and_t, ushort, char) diff --git a/src/backend/oneapi/any.cpp b/src/backend/oneapi/any.cpp index bdf600e9a9..82e242a989 100644 --- a/src/backend/oneapi/any.cpp +++ b/src/backend/oneapi/any.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_or_t, uint, char) INSTANTIATE(af_or_t, intl, char) INSTANTIATE(af_or_t, uintl, char) INSTANTIATE(af_or_t, char, char) +INSTANTIATE(af_or_t, schar, char) INSTANTIATE(af_or_t, uchar, char) INSTANTIATE(af_or_t, short, char) INSTANTIATE(af_or_t, ushort, char) diff --git a/src/backend/oneapi/assign.cpp b/src/backend/oneapi/assign.cpp index def9378d2d..de436495db 100644 --- a/src/backend/oneapi/assign.cpp +++ b/src/backend/oneapi/assign.cpp @@ -80,6 +80,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/oneapi/bilateral.cpp b/src/backend/oneapi/bilateral.cpp index d7d5dd33b9..6520cf9ffa 100644 --- a/src/backend/oneapi/bilateral.cpp +++ b/src/backend/oneapi/bilateral.cpp @@ -35,6 +35,7 @@ INSTANTIATE(float, float) INSTANTIATE(char, float) INSTANTIATE(int, float) INSTANTIATE(uint, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(short, float) INSTANTIATE(ushort, float) diff --git a/src/backend/oneapi/binary.hpp b/src/backend/oneapi/binary.hpp index a9bc4900e8..8bd36aff7e 100644 --- a/src/backend/oneapi/binary.hpp +++ b/src/backend/oneapi/binary.hpp @@ -9,6 +9,9 @@ #pragma once #include +#include + +using arrayfire::common::half; namespace arrayfire { namespace oneapi { @@ -93,6 +96,7 @@ struct BinOp { POW_BINARY_OP(double, "pow") POW_BINARY_OP(float, "pow") +POW_BINARY_OP(half, "pow") POW_BINARY_OP(intl, "__powll") POW_BINARY_OP(uintl, "__powul") POW_BINARY_OP(uint, "__powui") diff --git a/src/backend/oneapi/blas.cpp b/src/backend/oneapi/blas.cpp index 73dbadfcfd..93ae6559a4 100644 --- a/src/backend/oneapi/blas.cpp +++ b/src/backend/oneapi/blas.cpp @@ -43,19 +43,25 @@ static oneapi::mkl::transpose toBlasTranspose(af_mat_prop opt) { } template -static void gemvDispatch(sycl::queue queue, oneapi::mkl::transpose lOpts, int M, - int N, const T *alpha, - const arrayfire::oneapi::Array &lhs, dim_t lStride, - const arrayfire::oneapi::Array &x, dim_t incx, - const T *beta, arrayfire::oneapi::Array &out, - dim_t oInc) { +static void gemvDispatch(sycl::queue queue, oneapi::mkl::transpose lOpts, + oneapi::mkl::transpose rOpts, int M, int N, + const T *alpha, const arrayfire::oneapi::Array &lhs, + dim_t lStride, const arrayfire::oneapi::Array &x, + dim_t incx, const T *beta, + arrayfire::oneapi::Array &out, dim_t oInc) { using Dt = arrayfire::oneapi::data_t; - sycl::buffer lhsBuf = lhs.get()->template reinterpret(); - sycl::buffer xBuf = x.get()->template reinterpret(); - sycl::buffer outBuf = out.get()->template reinterpret(); - ::oneapi::mkl::blas::gemv(queue, lOpts, (int64_t)M, (int64_t)N, (T)*alpha, - lhsBuf, (int64_t)lStride, xBuf, (int64_t)incx, - (T)*beta, outBuf, (int64_t)oInc); + const af::dim4 lStrides = lhs.strides(); + const af::dim4 xStrides = x.strides(); + const af::dim4 oStrides = out.strides(); + sycl::buffer lhsBuf = lhs.template getBufferWithOffset
(); + sycl::buffer xBuf = x.template getBufferWithOffset
(); + sycl::buffer outBuf = out.template getBufferWithOffset
(); + if constexpr (!std::is_same_v) { + ::oneapi::mkl::blas::gemv(queue, lOpts, (int64_t)M, (int64_t)N, + (T)*alpha, lhsBuf, (int64_t)lStride, xBuf, + (int64_t)incx, (T)*beta, outBuf, + (int64_t)oInc); + } } template @@ -65,10 +71,14 @@ static void gemmDispatch(sycl::queue queue, oneapi::mkl::transpose lOpts, dim_t lStride, const arrayfire::oneapi::Array &rhs, dim_t rStride, const T *beta, arrayfire::oneapi::Array &out, dim_t oleading) { - using Dt = arrayfire::oneapi::data_t; - sycl::buffer lhsBuf = lhs.get()->template reinterpret(); - sycl::buffer rhsBuf = rhs.get()->template reinterpret(); - sycl::buffer outBuf = out.get()->template reinterpret(); + using Dt = arrayfire::oneapi::data_t; + const af::dim4 lStrides = lhs.strides(); + + const af::dim4 rStrides = rhs.strides(); + const af::dim4 oStrides = out.strides(); + sycl::buffer lhsBuf = lhs.template getBufferWithOffset
(); + sycl::buffer rhsBuf = rhs.template getBufferWithOffset
(); + sycl::buffer outBuf = out.template getBufferWithOffset
(); ::oneapi::mkl::blas::gemm(queue, lOpts, rOpts, M, N, K, *alpha, lhsBuf, lStride, rhsBuf, rStride, *beta, outBuf, oleading); @@ -83,9 +93,14 @@ void initBlas() { /*gpu_blas_init();*/ void deInitBlas() { /*gpu_blas_deinit();*/ } -template -void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, - const Array &lhs, const Array &rhs, const T *beta) { +bool isStrideMonotonic(const af::dim4 &dim) { + return (dim[0] <= dim[1]) && (dim[1] <= dim[2]) && (dim[2] <= dim[3]); +} + +template +void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, + const To *alpha, const Array &lhs, const Array &rhs, + const To *beta) { const auto lOpts = toBlasTranspose(optLhs); const auto rOpts = toBlasTranspose(optRhs); @@ -106,42 +121,100 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, if (oDims.ndims() <= 2) { // if non-batched if (rhs.dims()[bColDim] == 1) { - dim_t incr = (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1]; - gemvDispatch(getQueue(), lOpts, lDims[0], lDims[1], alpha, lhs, - lStrides[1], rhs, incr, beta, out, oStrides[0]); + if constexpr (std::is_same_v) { + // currently no half support for gemv, use gemm instead + gemmDispatch(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs, + lStrides[1], rhs, rStrides[1], beta, out, + oStrides[1]); + } else { + dim_t incr = + (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1]; + gemvDispatch(getQueue(), lOpts, rOpts, lDims[0], lDims[1], + alpha, lhs, lStrides[1], rhs, incr, beta, out, + oStrides[0]); + } } else { - gemmDispatch(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs, - lStrides[1], rhs, rStrides[1], beta, out, - oStrides[1]); + gemmDispatch(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs, + lStrides[1], rhs, rStrides[1], beta, out, + oStrides[1]); } } else { // if batched - using Dt = arrayfire::oneapi::data_t; - - sycl::buffer lhsBuf = lhs.get()->template reinterpret(); - sycl::buffer rhsBuf = rhs.get()->template reinterpret(); - sycl::buffer outBuf = out.get()->template reinterpret(); - - const int64_t lda = lStrides[1]; - const int64_t ldb = rStrides[1]; - const int64_t ldc = oStrides[1]; + using Dt = arrayfire::oneapi::data_t; int64_t batchSize = static_cast(oDims[2] * oDims[3]); - const bool not_l_batched = - (oDims[2] != lDims[2] && oDims[3] != lDims[3]); - const bool not_r_batched = - (oDims[2] != rDims[2] && oDims[3] != rDims[3]); - - ::oneapi::mkl::blas::gemm_batch( - getQueue(), lOpts, rOpts, M, N, K, *alpha, lhsBuf, lda, - not_l_batched ? 0 : lStrides[2], rhsBuf, ldb, - not_r_batched ? 0 : rStrides[2], *beta, outBuf, ldc, oStrides[2], - batchSize); + bool is_l_d2_batched = (oDims[2] == lDims[2]) && lDims[2] != 1; + bool is_l_d3_batched = (oDims[3] == lDims[3]) && lDims[3] != 1; + bool is_r_d2_batched = (oDims[2] == rDims[2]) && rDims[2] != 1; + bool is_r_d3_batched = (oDims[3] == rDims[3]) && rDims[3] != 1; + + // MKL requires stridec >= ldc * n, which may not be true with reordered + // outputs if the stride is monotonic, then MKL requirements for + // batching can be met + bool canBatchMKL = isStrideMonotonic(oStrides); + if (canBatchMKL) { + sycl::buffer lhsBuf = lhs.template getBufferWithOffset
(); + sycl::buffer rhsBuf = rhs.template getBufferWithOffset
(); + sycl::buffer outBuf = out.template getBufferWithOffset
(); + + const int64_t lda = lStrides[1]; + const int64_t ldb = rStrides[1]; + const int64_t ldc = oStrides[1]; + + dim_t lstride = (is_l_d2_batched) ? lStrides[2] + : is_l_d3_batched ? lStrides[3] + : 0; + dim_t rstride = (is_r_d2_batched) ? rStrides[2] + : is_r_d3_batched ? rStrides[3] + : 0; + + ::oneapi::mkl::blas::gemm_batch(getQueue(), lOpts, rOpts, M, N, K, + *alpha, lhsBuf, lda, lstride, + rhsBuf, ldb, rstride, *beta, outBuf, + ldc, oStrides[2], batchSize); + } else { + std::vector> lptrs; + std::vector> rptrs; + std::vector> optrs; + + lptrs.reserve(batchSize); + rptrs.reserve(batchSize); + optrs.reserve(batchSize); + + for (int n = 0; n < batchSize; n++) { + ptrdiff_t w = n / oDims[2]; + ptrdiff_t z = n - w * oDims[2]; + + ptrdiff_t loff = z * (is_l_d2_batched * lStrides[2]) + + w * (is_l_d3_batched * lStrides[3]); + ptrdiff_t roff = z * (is_r_d2_batched * rStrides[2]) + + w * (is_r_d3_batched * rStrides[3]); + ptrdiff_t zoff = z * oStrides[2] + w * oStrides[3]; + + lptrs.emplace_back(lhs.template getBufferWithOffset
(loff)); + rptrs.emplace_back(rhs.template getBufferWithOffset
(roff)); + optrs.emplace_back(out.template getBufferWithOffset
(zoff)); + } + + for (int n = 0; n < batchSize; n++) { + ::oneapi::mkl::blas::gemm(getQueue(), lOpts, rOpts, M, N, K, + *alpha, lptrs[n], lStrides[1], + rptrs[n], rStrides[1], *beta, + optrs[n], oStrides[1]); + } + } } - ONEAPI_DEBUG_FINISH(getQueue()); } +template<> +void gemm(Array &out, af_mat_prop optLhs, + af_mat_prop optRhs, const float *alpha, + const Array &lhs, const Array &rhs, + const float *beta) { + TYPE_ERROR(3, af_dtype::s8); +} + template Array dot(const Array &lhs, const Array &rhs, af_mat_prop optLhs, af_mat_prop optRhs) { @@ -161,13 +234,7 @@ INSTANTIATE_GEMM(float) INSTANTIATE_GEMM(cfloat) INSTANTIATE_GEMM(double) INSTANTIATE_GEMM(cdouble) -// INSTANTIATE_GEMM(half) -template<> -void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, - const half *alpha, const Array &lhs, const Array &rhs, - const half *beta) { - ONEAPI_NOT_SUPPORTED(""); -} +INSTANTIATE_GEMM(half) #define INSTANTIATE_DOT(TYPE) \ template Array dot(const Array &lhs, \ diff --git a/src/backend/oneapi/blas.hpp b/src/backend/oneapi/blas.hpp index 194fc4e6fb..af65f56d12 100644 --- a/src/backend/oneapi/blas.hpp +++ b/src/backend/oneapi/blas.hpp @@ -9,6 +9,7 @@ #pragma once #include +#include // This file contains the common interface for OneAPI BLAS // functions @@ -19,9 +20,10 @@ namespace oneapi { void initBlas(); void deInitBlas(); -template -void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, - const Array &lhs, const Array &rhs, const T *beta); +template +void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, + const To *alpha, const Array &lhs, const Array &rhs, + const To *beta); template Array matmul(const Array &lhs, const Array &rhs, af_mat_prop optLhs, @@ -30,8 +32,8 @@ Array matmul(const Array &lhs, const Array &rhs, af_mat_prop optLhs, int Ndim = optRhs == AF_MAT_NONE ? 1 : 0; Array res = createEmptyArray( dim4(lhs.dims()[Mdim], rhs.dims()[Ndim], lhs.dims()[2], lhs.dims()[3])); - static constexpr T alpha = 1.0; - static constexpr T beta = 0.0; + static const T alpha = scalar(1.0); + static const T beta = scalar(0.0); gemm(res, optLhs, optRhs, &alpha, lhs, rhs, &beta); return res; } diff --git a/src/backend/oneapi/cast.hpp b/src/backend/oneapi/cast.hpp index c9b015c4f2..11b64c9631 100644 --- a/src/backend/oneapi/cast.hpp +++ b/src/backend/oneapi/cast.hpp @@ -38,6 +38,11 @@ CAST_FN(uchar) CAST_FN(float) CAST_FN(double) +template +struct CastOp { + const char *name() { return "convert_char"; } +}; + #define CAST_CFN(TYPE) \ template \ struct CastOp { \ diff --git a/src/backend/oneapi/cholesky.cpp b/src/backend/oneapi/cholesky.cpp index 4fb0e08c58..d399034383 100644 --- a/src/backend/oneapi/cholesky.cpp +++ b/src/backend/oneapi/cholesky.cpp @@ -14,8 +14,10 @@ #include #if defined(WITH_LINEAR_ALGEBRA) +#include +#include #include -#include "oneapi/mkl/lapack.hpp" +#include namespace arrayfire { namespace oneapi { @@ -32,15 +34,16 @@ int cholesky_inplace(Array &in, const bool is_upper) { ::oneapi::mkl::uplo uplo = ::oneapi::mkl::uplo::lower; if (is_upper) { uplo = ::oneapi::mkl::uplo::upper; } - lwork = ::oneapi::mkl::lapack::potrf_scratchpad_size(getQueue(), uplo, N, - LDA); + lwork = ::oneapi::mkl::lapack::potrf_scratchpad_size>( + getQueue(), uplo, N, LDA); - Array workspace = createEmptyArray(af::dim4(lwork)); - Array d_info = createEmptyArray(af::dim4(1)); + auto workspace = memAlloc>(std::max(lwork, 1)); + sycl::buffer> in_buffer = + in.template getBufferWithOffset>(); try { - ::oneapi::mkl::lapack::potrf(getQueue(), uplo, N, *in.get(), LDA, - *workspace.get(), lwork); + ::oneapi::mkl::lapack::potrf(getQueue(), uplo, N, in_buffer, LDA, + *workspace, workspace->size()); } catch (::oneapi::mkl::lapack::exception const &e) { AF_ERROR( "Unexpected exception caught during synchronous\ diff --git a/src/backend/oneapi/convolve.cpp b/src/backend/oneapi/convolve.cpp index 69c120569b..0e443d7b77 100644 --- a/src/backend/oneapi/convolve.cpp +++ b/src/backend/oneapi/convolve.cpp @@ -98,6 +98,7 @@ INSTANTIATE(double, double) INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(char, float) INSTANTIATE(ushort, float) @@ -149,15 +150,9 @@ Array convolve2_unwrap(const Array &signal, const Array &filter, template Array convolve2(Array const &signal, Array const &filter, const dim4 stride, const dim4 padding, const dim4 dilation) { - if constexpr (!std::is_same::value) { - Array out = - convolve2_unwrap(signal, filter, stride, padding, dilation); - return out; - } else { - ONEAPI_NOT_SUPPORTED(""); - Array out = createEmptyArray(dim4(1)); - return out; - } + Array out = + convolve2_unwrap(signal, filter, stride, padding, dilation); + return out; } #define INSTANTIATE(T) \ @@ -177,39 +172,33 @@ Array conv2DataGradient(const Array &incoming_gradient, const Array & /*convolved_output*/, af::dim4 stride, af::dim4 padding, af::dim4 dilation) { - if constexpr (!std::is_same::value) { - const dim4 &cDims = incoming_gradient.dims(); - const dim4 &sDims = original_signal.dims(); - const dim4 &fDims = original_filter.dims(); - - Array collapsed_filter = original_filter; - - collapsed_filter = flip(collapsed_filter, {1, 1, 0, 0}); - collapsed_filter = modDims( - collapsed_filter, dim4(fDims[0] * fDims[1] * fDims[2], fDims[3])); - - Array collapsed_gradient = incoming_gradient; - collapsed_gradient = reorder(collapsed_gradient, dim4(0, 1, 3, 2)); - collapsed_gradient = modDims( - collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2])); - - Array res = matmul(collapsed_gradient, collapsed_filter, AF_MAT_NONE, - AF_MAT_TRANS); - res = modDims(res, dim4(res.dims()[0] / sDims[3], sDims[3], - fDims[0] * fDims[1], sDims[2])); - res = reorder(res, dim4(0, 2, 3, 1)); - - const bool retCols = false; - res = wrap_dilated(res, sDims[0], sDims[1], fDims[0], fDims[1], - stride[0], stride[1], padding[0], padding[1], - dilation[0], dilation[1], retCols); - - return res; - } else { - ONEAPI_NOT_SUPPORTED(""); - Array out = createEmptyArray(dim4(1)); - return out; - } + const dim4 &cDims = incoming_gradient.dims(); + const dim4 &sDims = original_signal.dims(); + const dim4 &fDims = original_filter.dims(); + + Array collapsed_filter = original_filter; + + collapsed_filter = flip(collapsed_filter, {1, 1, 0, 0}); + collapsed_filter = modDims(collapsed_filter, + dim4(fDims[0] * fDims[1] * fDims[2], fDims[3])); + + Array collapsed_gradient = incoming_gradient; + collapsed_gradient = reorder(collapsed_gradient, dim4(0, 1, 3, 2)); + collapsed_gradient = modDims( + collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2])); + + Array res = + matmul(collapsed_gradient, collapsed_filter, AF_MAT_NONE, AF_MAT_TRANS); + res = modDims(res, dim4(res.dims()[0] / sDims[3], sDims[3], + fDims[0] * fDims[1], sDims[2])); + res = reorder(res, dim4(0, 2, 3, 1)); + + const bool retCols = false; + res = wrap_dilated(res, sDims[0], sDims[1], fDims[0], fDims[1], stride[0], + stride[1], padding[0], padding[1], dilation[0], + dilation[1], retCols); + + return res; } template @@ -219,36 +208,30 @@ Array conv2FilterGradient(const Array &incoming_gradient, const Array & /*convolved_output*/, af::dim4 stride, af::dim4 padding, af::dim4 dilation) { - if constexpr (!std::is_same::value) { - const dim4 &cDims = incoming_gradient.dims(); - const dim4 &fDims = original_filter.dims(); - - const bool retCols = false; - Array unwrapped = - unwrap(original_signal, fDims[0], fDims[1], stride[0], stride[1], - padding[0], padding[1], dilation[0], dilation[1], retCols); - - unwrapped = reorder(unwrapped, dim4(1, 2, 0, 3)); - dim4 uDims = unwrapped.dims(); - unwrapped = - modDims(unwrapped, dim4(uDims[0] * uDims[1], uDims[2] * uDims[3])); - - Array collapsed_gradient = incoming_gradient; - collapsed_gradient = reorder(collapsed_gradient, dim4(0, 1, 3, 2)); - collapsed_gradient = modDims( - collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2])); - - Array res = - matmul(unwrapped, collapsed_gradient, AF_MAT_NONE, AF_MAT_NONE); - res = modDims(res, dim4(fDims[0], fDims[1], fDims[2], fDims[3])); - - auto out = flip(res, {1, 1, 0, 0}); - return out; - } else { - ONEAPI_NOT_SUPPORTED(""); - Array out = createEmptyArray(dim4(1)); - return out; - } + const dim4 &cDims = incoming_gradient.dims(); + const dim4 &fDims = original_filter.dims(); + + const bool retCols = false; + Array unwrapped = + unwrap(original_signal, fDims[0], fDims[1], stride[0], stride[1], + padding[0], padding[1], dilation[0], dilation[1], retCols); + + unwrapped = reorder(unwrapped, dim4(1, 2, 0, 3)); + dim4 uDims = unwrapped.dims(); + unwrapped = + modDims(unwrapped, dim4(uDims[0] * uDims[1], uDims[2] * uDims[3])); + + Array collapsed_gradient = incoming_gradient; + collapsed_gradient = reorder(collapsed_gradient, dim4(0, 1, 3, 2)); + collapsed_gradient = modDims( + collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2])); + + Array res = + matmul(unwrapped, collapsed_gradient, AF_MAT_NONE, AF_MAT_NONE); + res = modDims(res, dim4(fDims[0], fDims[1], fDims[2], fDims[3])); + + auto out = flip(res, {1, 1, 0, 0}); + return out; } #define INSTANTIATE(T) \ diff --git a/src/backend/oneapi/convolve_separable.cpp b/src/backend/oneapi/convolve_separable.cpp index fdf9fc952f..ddf5c27a7e 100644 --- a/src/backend/oneapi/convolve_separable.cpp +++ b/src/backend/oneapi/convolve_separable.cpp @@ -65,6 +65,7 @@ INSTANTIATE(double, double) INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(char, float) INSTANTIATE(short, float) diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp index f99f79854e..a89023261e 100644 --- a/src/backend/oneapi/copy.cpp +++ b/src/backend/oneapi/copy.cpp @@ -155,6 +155,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) @@ -184,6 +185,8 @@ INSTANTIATE(half) Array const &src); \ template void copyArray(Array & dst, \ Array const &src); \ + template void copyArray(Array & dst, \ + Array const &src); \ template void copyArray(Array & dst, \ Array const &src); \ template void copyArray(Array & dst, \ @@ -197,6 +200,7 @@ INSTANTIATE_COPY_ARRAY(int) INSTANTIATE_COPY_ARRAY(uint) INSTANTIATE_COPY_ARRAY(intl) INSTANTIATE_COPY_ARRAY(uintl) +INSTANTIATE_COPY_ARRAY(schar) INSTANTIATE_COPY_ARRAY(uchar) INSTANTIATE_COPY_ARRAY(char) INSTANTIATE_COPY_ARRAY(short) @@ -216,10 +220,11 @@ template T getScalar(const Array &in) { T retVal{}; + auto in_get = in.get(); getQueue() .submit([&](sycl::handler &h) { auto acc_in = - in.get()->template get_access( + in_get->template get_access( h, sycl::range{1}, sycl::id{static_cast(in.getOffset())}); h.copy(acc_in, &retVal); @@ -237,6 +242,7 @@ INSTANTIATE_GETSCALAR(cfloat) INSTANTIATE_GETSCALAR(cdouble) INSTANTIATE_GETSCALAR(int) INSTANTIATE_GETSCALAR(uint) +INSTANTIATE_GETSCALAR(schar) INSTANTIATE_GETSCALAR(uchar) INSTANTIATE_GETSCALAR(char) INSTANTIATE_GETSCALAR(intl) diff --git a/src/backend/oneapi/count.cpp b/src/backend/oneapi/count.cpp index f8ef354169..4ed59eb3b9 100644 --- a/src/backend/oneapi/count.cpp +++ b/src/backend/oneapi/count.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_notzero_t, uint, uint) INSTANTIATE(af_notzero_t, intl, uint) INSTANTIATE(af_notzero_t, uintl, uint) INSTANTIATE(af_notzero_t, char, uint) +INSTANTIATE(af_notzero_t, schar, uint) INSTANTIATE(af_notzero_t, uchar, uint) INSTANTIATE(af_notzero_t, short, uint) INSTANTIATE(af_notzero_t, ushort, uint) diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp index ac06d5768c..56125382a0 100644 --- a/src/backend/oneapi/device_manager.cpp +++ b/src/backend/oneapi/device_manager.cpp @@ -104,13 +104,7 @@ DeviceManager::DeviceManager() // Iterate through platforms, get all available devices and store them for (auto& platform : platforms) { vector current_devices; - try { - current_devices = platform.get_devices(); - } catch (sycl::exception& err) { - printf("DeviceManager::DeviceManager() exception: %s\n", - err.what()); - throw; - } + current_devices = platform.get_devices(); AF_TRACE("Found {} devices on platform {}", current_devices.size(), platform.get_info()); diff --git a/src/backend/oneapi/diagonal.cpp b/src/backend/oneapi/diagonal.cpp index a18d024585..900f53ba3c 100644 --- a/src/backend/oneapi/diagonal.cpp +++ b/src/backend/oneapi/diagonal.cpp @@ -54,6 +54,7 @@ INSTANTIATE_DIAGONAL(uint) INSTANTIATE_DIAGONAL(intl) INSTANTIATE_DIAGONAL(uintl) INSTANTIATE_DIAGONAL(char) +INSTANTIATE_DIAGONAL(schar) INSTANTIATE_DIAGONAL(uchar) INSTANTIATE_DIAGONAL(short) INSTANTIATE_DIAGONAL(ushort) diff --git a/src/backend/oneapi/diff.cpp b/src/backend/oneapi/diff.cpp index a3c37f6a4a..01cd18e37e 100644 --- a/src/backend/oneapi/diff.cpp +++ b/src/backend/oneapi/diff.cpp @@ -50,6 +50,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/oneapi/err_oneapi.hpp b/src/backend/oneapi/err_oneapi.hpp index fad7d449c0..4f187b6273 100644 --- a/src/backend/oneapi/err_oneapi.hpp +++ b/src/backend/oneapi/err_oneapi.hpp @@ -13,8 +13,8 @@ #define ONEAPI_NOT_SUPPORTED(message) \ do { \ - throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \ - boost::stacktrace::stacktrace()); \ + throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, "oneAPI",\ + message, boost::stacktrace::stacktrace()); \ } while (0) #define CL_CHECK(call) \ diff --git a/src/backend/oneapi/exampleFunction.cpp b/src/backend/oneapi/exampleFunction.cpp index 6159d9d1d4..9a006febff 100644 --- a/src/backend/oneapi/exampleFunction.cpp +++ b/src/backend/oneapi/exampleFunction.cpp @@ -59,6 +59,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(cfloat) diff --git a/src/backend/oneapi/fast.cpp b/src/backend/oneapi/fast.cpp index cb9ae28d4c..a5b0934f97 100644 --- a/src/backend/oneapi/fast.cpp +++ b/src/backend/oneapi/fast.cpp @@ -38,6 +38,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/fft.cpp b/src/backend/oneapi/fft.cpp index eff8770bfc..03ae19efc6 100644 --- a/src/backend/oneapi/fft.cpp +++ b/src/backend/oneapi/fft.cpp @@ -7,32 +7,164 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ -#include - #include +#include #include #include #include #include +#include +#include #include -#include -using std::array; +#include +#include -using af::dim4; +#include +#include -#include +using std::make_shared; + +using af::dim4; namespace arrayfire { namespace oneapi { void setFFTPlanCacheSize(size_t numPlans) {} -inline array computeDims(const int rank, const dim4 &idims) { - array retVal = {}; - for (int i = 0; i < rank; i++) { retVal[i] = idims[(rank - 1) - i]; } - return retVal; +std::string genPlanHashStr(int rank, ::oneapi::mkl::dft::precision precision, + ::oneapi::mkl::dft::domain domain, + const bool isInPlace, const dim_t *n, + std::int64_t *istrides, int ibatch, + std::int64_t *ostrides, int obatch, int nbatch) { + // create the key string + char key_str_temp[64]; + sprintf(key_str_temp, "%d:", rank); + + std::string key_string(key_str_temp); + + if (precision == ::oneapi::mkl::dft::precision::SINGLE) { + key_string.append("S:"); + } else if (precision == ::oneapi::mkl::dft::precision::DOUBLE) { + key_string.append("D:"); + } + if (domain == ::oneapi::mkl::dft::domain::REAL) { + key_string.append("R:"); + } else if (domain == ::oneapi::mkl::dft::domain::COMPLEX) { + key_string.append("C:"); + } + if (isInPlace) { + key_string.append("IIP:"); + } else { + key_string.append("OOP:"); + } + + for (int r = 0; r < rank; ++r) { + sprintf(key_str_temp, "%lld:", n[r]); + key_string.append(std::string(key_str_temp)); + } + + if (istrides != nullptr) { + for (int r = 0; r < rank + 1; ++r) { + sprintf(key_str_temp, "%ld:", istrides[r]); + key_string.append(std::string(key_str_temp)); + } + sprintf(key_str_temp, "%d:", ibatch); + key_string.append(std::string(key_str_temp)); + } + + if (ostrides != nullptr) { + for (int r = 0; r < rank + 1; ++r) { + sprintf(key_str_temp, "%ld:", ostrides[r]); + key_string.append(std::string(key_str_temp)); + } + sprintf(key_str_temp, "%d:", obatch); + key_string.append(std::string(key_str_temp)); + } + + sprintf(key_str_temp, "%d", nbatch); + key_string.append(std::string(key_str_temp)); + + return key_string; +} + +std::vector computeStrides(const int rank, const dim4 istrides, + const dim_t offset) { + if (rank == 2) return {offset, istrides[1], istrides[0]}; + if (rank == 3) return {offset, istrides[2], istrides[1], istrides[0]}; + if (rank == 4) + return {offset, istrides[3], istrides[2], istrides[1], istrides[0]}; + return {offset, istrides[0]}; +} + +template<::oneapi::mkl::dft::precision precision, + ::oneapi::mkl::dft::domain domain> +PlanType findPlan(int rank, const bool isInPlace, const dim_t *idims, + std::int64_t *istrides, int ibatch, std::int64_t *ostrides, + int obatch, int nbatch) { + using desc_ty = ::oneapi::mkl::dft::descriptor; + + std::string key_string = + genPlanHashStr(rank, precision, domain, isInPlace, idims, istrides, + ibatch, ostrides, obatch, nbatch); + + PlanCache &planner = arrayfire::oneapi::fftManager(); + std::shared_ptr retVal = (planner.find(key_string)); + if (retVal) { return *retVal; } + + desc_ty *desc = [rank, &idims]() { + if (rank == 1) return new desc_ty(static_cast(idims[0])); + if (rank == 2) return new desc_ty({idims[1], idims[0]}); + if (rank == 3) return new desc_ty({idims[2], idims[1], idims[0]}); + return new desc_ty({idims[3], idims[2], idims[1], idims[0]}); + }(); + + if (rank > 1) { + desc->set_value(::oneapi::mkl::dft::config_param::INPUT_STRIDES, + istrides); + desc->set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES, + ostrides); + } + + if (isInPlace) { + desc->set_value(::oneapi::mkl::dft::config_param::PLACEMENT, + DFTI_INPLACE); + } else { + desc->set_value(::oneapi::mkl::dft::config_param::PLACEMENT, + DFTI_NOT_INPLACE); + } + + desc->set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS, + (int64_t)nbatch); + + desc->set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE, ibatch); + desc->set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE, obatch); + + if constexpr (domain == ::oneapi::mkl::dft::domain::COMPLEX) { + desc->set_value(::oneapi::mkl::dft::config_param::COMPLEX_STORAGE, + DFTI_COMPLEX_COMPLEX); + } else { + desc->set_value( + ::oneapi::mkl::dft::config_param::CONJUGATE_EVEN_STORAGE, + DFTI_COMPLEX_COMPLEX); + desc->set_value(::oneapi::mkl::dft::config_param::PACKED_FORMAT, + DFTI_CCE_FORMAT); + } + + try { + desc->commit(getQueue()); + } catch (::oneapi::mkl::device_bad_alloc &e) { + // If plan creation fails, clean up the memory we hold on to and try + // again + arrayfire::oneapi::signalMemoryCleanup(); + desc->commit(getQueue()); + } + + // push the plan into plan cache + std::shared_ptr ptr(desc); + planner.push(key_string, make_shared(ptr)); + return ptr; } template @@ -48,30 +180,28 @@ void fft_inplace(Array &in, const int rank, const bool direction) { ::oneapi::mkl::dft::descriptor; - auto desc = [rank, &idims]() { - if (rank == 1) return desc_ty(idims[0]); - if (rank == 2) return desc_ty({idims[0], idims[1]}); - if (rank == 3) return desc_ty({idims[0], idims[1], idims[2]}); - return desc_ty({idims[0], idims[1], idims[2], idims[3]}); - }(); - - desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT, DFTI_INPLACE); - + // TODO[STF]: WTF + // getOffset() for s0 throwing Invalid Descriptor when targeting gpu + // on CPU, results are wrong but does not throw + // strides not working? TODO: test standalone oneMKL + // perhaps in.getDataDims() needed instead of in.dims()? + std::vector fft_input_strides = + computeStrides(rank, istrides, 0); + // computeStrides(rank, istrides, in.getOffset()); //TODO[STF]: WTF, int batch = 1; for (int i = rank; i < 4; i++) { batch *= idims[i]; } - desc.set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS, - (int64_t)batch); - desc.set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE, - istrides[rank]); - desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE, - istrides[rank]); + const bool isInPlace = true; + PlanType descP = findPlan( + rank, isInPlace, idims.get(), fft_input_strides.data(), istrides[rank], + fft_input_strides.data(), istrides[rank], batch); + + desc_ty *desc = (desc_ty *)descP.get(); - desc.commit(getQueue()); if (direction) - ::oneapi::mkl::dft::compute_forward(desc, *in.get()); + ::oneapi::mkl::dft::compute_forward(*desc, *in.get()); else - ::oneapi::mkl::dft::compute_backward(desc, *in.get()); + ::oneapi::mkl::dft::compute_backward(*desc, *in.get()); } template @@ -90,34 +220,22 @@ Array fft_r2c(const Array &in, const int rank) { ::oneapi::mkl::dft::descriptor; - auto desc = [rank, &idims]() { - if (rank == 1) return desc_ty(idims[0]); - if (rank == 2) return desc_ty({idims[0], idims[1]}); - if (rank == 3) return desc_ty({idims[0], idims[1], idims[2]}); - return desc_ty({idims[0], idims[1], idims[2], idims[3]}); - }(); - - desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT, - DFTI_NOT_INPLACE); + std::vector fft_input_strides = + computeStrides(rank, istrides, in.getOffset()); + std::vector fft_output_strides = + computeStrides(rank, ostrides, out.getOffset()); int batch = 1; for (int i = rank; i < 4; i++) { batch *= idims[i]; } - desc.set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS, - (int64_t)batch); - desc.set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE, - ostrides[rank]); - desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE, - istrides[rank]); + const bool isInPlace = false; + PlanType descP = findPlan( + rank, isInPlace, idims.get(), fft_input_strides.data(), istrides[rank], + fft_output_strides.data(), ostrides[rank], batch); - const std::int64_t fft_output_strides[5] = { - 0, ostrides[(rank == 2) ? 1 : 0], ostrides[(rank == 2) ? 0 : 1], - ostrides[2], ostrides[3]}; - desc.set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES, - fft_output_strides, rank); + desc_ty *desc = (desc_ty *)descP.get(); - desc.commit(getQueue()); - ::oneapi::mkl::dft::compute_forward(desc, *in.get(), *out.get()); + ::oneapi::mkl::dft::compute_forward(*desc, *in.get(), *out.get()); return out; } @@ -137,34 +255,22 @@ Array fft_c2r(const Array &in, const dim4 &odims, const int rank) { ::oneapi::mkl::dft::descriptor; - auto desc = [rank, &odims]() { - if (rank == 1) return desc_ty(odims[0]); - if (rank == 2) return desc_ty({odims[0], odims[1]}); - if (rank == 3) return desc_ty({odims[0], odims[1], odims[2]}); - return desc_ty({odims[0], odims[1], odims[2], odims[3]}); - }(); - - desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT, - DFTI_NOT_INPLACE); + std::vector fft_input_strides = + computeStrides(rank, istrides, in.getOffset()); + std::vector fft_output_strides = + computeStrides(rank, ostrides, out.getOffset()); int batch = 1; - for (int i = rank; i < 4; i++) { batch *= idims[i]; } - desc.set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS, - (int64_t)batch); - - desc.set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE, - istrides[rank]); - desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE, - ostrides[rank]); - - const std::int64_t fft_output_strides[5] = { - 0, ostrides[(rank == 2) ? 1 : 0], ostrides[(rank == 2) ? 0 : 1], - ostrides[2], ostrides[3]}; - desc.set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES, - fft_output_strides, rank); - - desc.commit(getQueue()); - ::oneapi::mkl::dft::compute_backward(desc, *in.get(), *out.get()); + for (int i = rank; i < 4; i++) { batch *= odims[i]; } + + const bool isInPlace = false; + PlanType descP = findPlan( + rank, isInPlace, odims.get(), fft_input_strides.data(), ostrides[rank], + fft_output_strides.data(), istrides[rank], batch); + + desc_ty *desc = (desc_ty *)descP.get(); + + ::oneapi::mkl::dft::compute_backward(*desc, *in.get(), *out.get()); return out; } diff --git a/src/backend/oneapi/fft.hpp b/src/backend/oneapi/fft.hpp index 0138970ba9..ca82f06118 100644 --- a/src/backend/oneapi/fft.hpp +++ b/src/backend/oneapi/fft.hpp @@ -6,6 +6,7 @@ * The complete license agreement can be obtained at: * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +#pragma once #include diff --git a/src/backend/oneapi/fftconvolve.cpp b/src/backend/oneapi/fftconvolve.cpp index c4aea5689c..85718f4f4f 100644 --- a/src/backend/oneapi/fftconvolve.cpp +++ b/src/backend/oneapi/fftconvolve.cpp @@ -15,6 +15,12 @@ #include #include +#include +#include +#include +#include +#include + #include #include #include @@ -59,9 +65,78 @@ dim4 calcPackedSize(Array const& i1, Array const& i2, const dim_t rank) { template Array fftconvolve(Array const& signal, Array const& filter, const bool expand, AF_BATCH_KIND kind, const int rank) { - ONEAPI_NOT_SUPPORTED(""); + using convT = typename conditional::value || + is_same::value || + is_same::value, + float, double>::type; + using cT = typename conditional::value, cfloat, + cdouble>::type; + + const dim4& sDims = signal.dims(); + const dim4& fDims = filter.dims(); + dim4 oDims(1); + if (expand) { + for (int d = 0; d < AF_MAX_DIMS; ++d) { + if (kind == AF_BATCH_NONE || kind == AF_BATCH_RHS) { + oDims[d] = sDims[d] + fDims[d] - 1; + } else { + oDims[d] = (d < rank ? sDims[d] + fDims[d] - 1 : sDims[d]); + } + } + } else { + oDims = sDims; + if (kind == AF_BATCH_RHS) { + for (int i = rank; i < AF_MAX_DIMS; ++i) { oDims[i] = fDims[i]; } + } + } + + const dim4 pDims = calcPackedSize(signal, filter, rank); + Array packed = createEmptyArray(pDims); + + kernel::packDataHelper(packed, signal, filter, rank, kind); + kernel::padDataHelper(packed, signal, filter, rank, kind); + + fft_inplace(packed, rank, true); + + kernel::complexMultiplyHelper(packed, signal, filter, rank, kind); + + // Compute inverse FFT only on complex-multiplied data + if (kind == AF_BATCH_RHS) { + vector seqs; + for (int k = 0; k < AF_MAX_DIMS; k++) { + if (k < rank) { + seqs.push_back({0., static_cast(pDims[k] - 1), 1.}); + } else if (k == rank) { + seqs.push_back({1., static_cast(pDims[k] - 1), 1.}); + } else { + seqs.push_back({0., 0., 1.}); + } + } + + Array subPacked = createSubArray(packed, seqs); + fft_inplace(subPacked, rank, false); + } else { + vector seqs; + for (int k = 0; k < AF_MAX_DIMS; k++) { + if (k < rank) { + seqs.push_back({0., static_cast(pDims[k]) - 1, 1.}); + } else if (k == rank) { + seqs.push_back({0., static_cast(pDims[k] - 2), 1.}); + } else { + seqs.push_back({0., 0., 1.}); + } + } + + Array subPacked = createSubArray(packed, seqs); + fft_inplace(subPacked, rank, false); + } + Array out = createEmptyArray(oDims); + + kernel::reorderOutputHelper(out, packed, signal, filter, rank, kind, + expand); + return out; } @@ -73,6 +148,7 @@ INSTANTIATE(double) INSTANTIATE(float) INSTANTIATE(uint) INSTANTIATE(int) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(uintl) diff --git a/src/backend/oneapi/hist_graphics.cpp b/src/backend/oneapi/hist_graphics.cpp index 3b280592b1..e016337a54 100644 --- a/src/backend/oneapi/hist_graphics.cpp +++ b/src/backend/oneapi/hist_graphics.cpp @@ -28,6 +28,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace oneapi diff --git a/src/backend/oneapi/histogram.cpp b/src/backend/oneapi/histogram.cpp index 4dfece0640..872431f14c 100644 --- a/src/backend/oneapi/histogram.cpp +++ b/src/backend/oneapi/histogram.cpp @@ -41,6 +41,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/identity.cpp b/src/backend/oneapi/identity.cpp index 5a838a4cf0..68a592ab88 100644 --- a/src/backend/oneapi/identity.cpp +++ b/src/backend/oneapi/identity.cpp @@ -37,6 +37,7 @@ INSTANTIATE_IDENTITY(uint) INSTANTIATE_IDENTITY(intl) INSTANTIATE_IDENTITY(uintl) INSTANTIATE_IDENTITY(char) +INSTANTIATE_IDENTITY(schar) INSTANTIATE_IDENTITY(uchar) INSTANTIATE_IDENTITY(short) INSTANTIATE_IDENTITY(ushort) diff --git a/src/backend/oneapi/iir.cpp b/src/backend/oneapi/iir.cpp index f60db52e8e..4a7654bd38 100644 --- a/src/backend/oneapi/iir.cpp +++ b/src/backend/oneapi/iir.cpp @@ -37,6 +37,19 @@ Array iir(const Array &b, const Array &a, const Array &x) { if (num_a == 1) { return c; } + size_t local_bytes_req = (num_a * 2 + 1) * sizeof(T); + if (local_bytes_req > + getDevice().get_info()) { + char errMessage[256]; + snprintf(errMessage, sizeof(errMessage), + "\ncurrent OneAPI device does not have sufficient local " + "memory,\n" + "for iir kernel, %zu(required) > %zu(available)\n", + local_bytes_req, + getDevice().get_info()); + AF_ERROR(errMessage, AF_ERR_RUNTIME); + } + dim4 ydims = c.dims(); Array y = createEmptyArray(ydims); diff --git a/src/backend/oneapi/image.cpp b/src/backend/oneapi/image.cpp index 723c29fb8b..7aa8b4b667 100644 --- a/src/backend/oneapi/image.cpp +++ b/src/backend/oneapi/image.cpp @@ -29,6 +29,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/index.cpp b/src/backend/oneapi/index.cpp index bec65902d8..af204b0820 100644 --- a/src/backend/oneapi/index.cpp +++ b/src/backend/oneapi/index.cpp @@ -44,6 +44,16 @@ Array index(const Array& in, const af_index_t idxrs[]) { p.isSeq[i] = idxrs[i].isSeq; p.offs[i] = iOffs[i]; p.strds[i] = iStrds[i]; + p.steps[i] = 0; + if (idxrs[i].isSeq) { + af_seq seq = idxrs[i].idx.seq; + // The step for af_span used in the kernel must be 1 + if (seq.begin == af_span.begin && seq.end == af_span.end && + seq.step == af_span.step) + p.steps[i] = 1; + else + p.steps[i] = seq.step; + } } std::vector> idxArrs(4, createEmptyArray(dim4(1))); @@ -73,6 +83,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/oneapi/inverse.cpp b/src/backend/oneapi/inverse.cpp index 97d91f4db4..2779393906 100644 --- a/src/backend/oneapi/inverse.cpp +++ b/src/backend/oneapi/inverse.cpp @@ -19,9 +19,8 @@ namespace oneapi { template Array inverse(const Array &in) { - ONEAPI_NOT_SUPPORTED(""); Array I = identity(in.dims()); - return I; + return solve(in, I); } #define INSTANTIATE(T) template Array inverse(const Array &in); diff --git a/src/backend/oneapi/iota.cpp b/src/backend/oneapi/iota.cpp index 6d511df23f..e775f0dde6 100644 --- a/src/backend/oneapi/iota.cpp +++ b/src/backend/oneapi/iota.cpp @@ -38,6 +38,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/ireduce.cpp b/src/backend/oneapi/ireduce.cpp index c7b4d263ab..c4bfc7604f 100644 --- a/src/backend/oneapi/ireduce.cpp +++ b/src/backend/oneapi/ireduce.cpp @@ -58,6 +58,7 @@ INSTANTIATE(af_min_t, uint) INSTANTIATE(af_min_t, intl) INSTANTIATE(af_min_t, uintl) INSTANTIATE(af_min_t, char) +INSTANTIATE(af_min_t, schar) INSTANTIATE(af_min_t, uchar) INSTANTIATE(af_min_t, short) INSTANTIATE(af_min_t, ushort) @@ -73,6 +74,7 @@ INSTANTIATE(af_max_t, uint) INSTANTIATE(af_max_t, intl) INSTANTIATE(af_max_t, uintl) INSTANTIATE(af_max_t, char) +INSTANTIATE(af_max_t, schar) INSTANTIATE(af_max_t, uchar) INSTANTIATE(af_max_t, short) INSTANTIATE(af_max_t, ushort) diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp index 794bb7796f..bda9e43ccf 100644 --- a/src/backend/oneapi/jit.cpp +++ b/src/backend/oneapi/jit.cpp @@ -8,17 +8,20 @@ ********************************************************/ #include +#include #include #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -34,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -42,14 +46,17 @@ using arrayfire::common::getFuncName; using arrayfire::common::half; +using arrayfire::common::kNodeType; using arrayfire::common::ModdimNode; using arrayfire::common::Node; using arrayfire::common::Node_ids; using arrayfire::common::Node_map_t; using arrayfire::common::Node_ptr; using arrayfire::common::NodeIterator; +using arrayfire::common::ShiftNodeBase; using arrayfire::oneapi::getActiveDeviceBaseBuildFlags; using arrayfire::oneapi::jit::BufferNode; +using arrayfire::oneapi::jit::ShiftNode; using std::array; using std::begin; @@ -211,61 +218,75 @@ __kernel void )JIT"; thread_local stringstream outOffsetStream; thread_local stringstream inOffsetsStream; thread_local stringstream opsStream; + thread_local stringstream kerStream; - int oid{0}; - for (size_t i{0}; i < full_nodes.size(); i++) { - const auto& node{full_nodes[i]}; - const auto& ids_curr{full_ids[i]}; - // Generate input parameters, only needs current id - node->genParams(inParamStream, ids_curr.id, is_linear); - // Generate input offsets, only needs current id - node->genOffsets(inOffsetsStream, ids_curr.id, is_linear); - // Generate the core function body, needs children ids as well - node->genFuncs(opsStream, ids_curr); - for (auto outIt{begin(output_ids)}, endIt{end(output_ids)}; - (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) { - // Generate also output parameters - outParamStream << "__global " - << full_nodes[ids_curr.id]->getTypeStr() << " *out" - << oid << ", int offset" << oid << ",\n"; - // Apply output offset - outOffsetStream << "\nout" << oid << " += offset" << oid << ';'; - // Generate code to write the output - opsStream << "out" << oid << "[idx] = val" << ids_curr.id << ";\n"; - ++oid; + string ret; + try { + int oid{0}; + for (size_t i{0}; i < full_nodes.size(); i++) { + const auto& node{full_nodes[i]}; + const auto& ids_curr{full_ids[i]}; + // Generate input parameters, only needs current id + node->genParams(inParamStream, ids_curr.id, is_linear); + // Generate input offsets, only needs current id + node->genOffsets(inOffsetsStream, ids_curr.id, is_linear); + // Generate the core function body, needs children ids as well + node->genFuncs(opsStream, ids_curr); + for (size_t output_idx{0}; output_idx < output_ids.size(); + ++output_idx) { + if (output_ids[output_idx] == ids_curr.id) { + outParamStream + << "__global " << full_nodes[ids_curr.id]->getTypeStr() + << " *out" << oid << ", int offset" << oid << ",\n"; + // Apply output offset + outOffsetStream << "\nout" << oid << " += offset" << oid + << ';'; + // Generate code to write the output + opsStream << "out" << output_idx << "[idx] = val" + << ids_curr.id << ";\n"; + ++oid; + } + } } - } - thread_local stringstream kerStream; - kerStream << DEFAULT_MACROS_STR << kernelVoid << funcName << "(\n" - << inParamStream.str() << outParamStream.str() << dimParams << ")" - << blockStart; - if (is_linear) { - kerStream << linearInit << inOffsetsStream.str() - << outOffsetStream.str() << '\n'; - if (loop0) kerStream << linearLoop0Start; - kerStream << "\n\n" << opsStream.str(); - if (loop0) kerStream << linearLoop0End; - kerStream << linearEnd; - } else { - if (loop0) { - kerStream << stridedLoop0Init << outOffsetStream.str() << '\n' - << stridedLoop0Start; + kerStream << DEFAULT_MACROS_STR << kernelVoid << funcName << "(\n" + << inParamStream.str() << outParamStream.str() << dimParams + << ")" << blockStart; + if (is_linear) { + kerStream << linearInit << inOffsetsStream.str() + << outOffsetStream.str() << '\n'; + if (loop0) kerStream << linearLoop0Start; + kerStream << "\n\n" << opsStream.str(); + if (loop0) kerStream << linearLoop0End; + kerStream << linearEnd; } else { - kerStream << stridedLoopNInit << outOffsetStream.str() << '\n'; - if (loop3) kerStream << stridedLoop3Init; - if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start; - if (loop3) kerStream << stridedLoop3Start; + if (loop0) { + kerStream << stridedLoop0Init << outOffsetStream.str() << '\n' + << stridedLoop0Start; + } else { + kerStream << stridedLoopNInit << outOffsetStream.str() << '\n'; + if (loop3) kerStream << stridedLoop3Init; + if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start; + if (loop3) kerStream << stridedLoop3Start; + } + kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str(); + if (loop3) kerStream << stridedLoop3End; + if (loop1) kerStream << stridedLoop1End; + if (loop0) kerStream << stridedLoop0End; + kerStream << stridedEnd; } - kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str(); - if (loop3) kerStream << stridedLoop3End; - if (loop1) kerStream << stridedLoop1End; - if (loop0) kerStream << stridedLoop0End; - kerStream << stridedEnd; + kerStream << blockEnd; + ret = kerStream.str(); + } catch (...) { + // Prepare for next round, limit memory + inParamStream.str(""); + outParamStream.str(""); + inOffsetsStream.str(""); + outOffsetStream.str(""); + opsStream.str(""); + kerStream.str(""); + throw; } - kerStream << blockEnd; - const string ret{kerStream.str()}; - // Prepare for next round, limit memory inParamStream.str(""); outParamStream.str(""); @@ -286,6 +307,11 @@ __kernel void )JIT"; // return common::getKernel("", "", true).get(); // } +static unordered_map device_name_map; +static std::mutex device_name_map_mutex; +static unordered_map kernel_map; +static std::mutex kernel_map_mutex; + template cl_kernel getKernel( std::string funcName, cl_context ctx, cl_device_id dev, cl_command_queue q, @@ -293,10 +319,36 @@ cl_kernel getKernel( nonstd::span full_ids, nonstd::span output_ids, nonstd::span const> ap, bool is_linear) { - static unordered_map kernel_map; + std::string devName; + { + std::lock_guard lock(device_name_map_mutex); + + auto devNameIt = device_name_map.find(dev); + if (devNameIt == device_name_map.end()) { + size_t devNameSz; + CL_CHECK( + clGetDeviceInfo(dev, CL_DEVICE_NAME, 0, nullptr, &devNameSz)); + string newDevName(devNameSz, '\0'); + CL_CHECK(clGetDeviceInfo(dev, CL_DEVICE_NAME, devNameSz, + newDevName.data(), nullptr)); + device_name_map[dev] = newDevName; + devName = newDevName; + } else { + devName = devNameIt->second; + } + } vector kernels(10); - if (kernel_map.find(funcName) == end(kernel_map)) { + bool kernel_found; + string kernelHash = funcName + devName; + { + std::lock_guard lock(kernel_map_mutex); + kernel_found = !(kernel_map.find(kernelHash) == end(kernel_map)); + } + if (kernel_found) { + std::lock_guard lock(kernel_map_mutex); + kernels[0] = kernel_map[kernelHash]; + } else { string jitstr = arrayfire::opencl::getKernelString( funcName, full_nodes, full_ids, output_ids, is_linear, false, false, ap[0].dims[2] > 1); @@ -320,10 +372,10 @@ cl_kernel getKernel( cl_uint ret_kernels = 0; CL_CHECK( clCreateKernelsInProgram(prog, 1, kernels.data(), &ret_kernels)); - kernel_map[funcName] = kernels[0]; + + std::lock_guard lock(kernel_map_mutex); + kernel_map[kernelHash] = kernels[0]; CL_CHECK(clReleaseProgram(prog)); - } else { - kernels[0] = kernel_map[funcName]; } return kernels[0]; } @@ -343,9 +395,11 @@ void evalNodes(vector>& outputs, const vector& output_nodes) { bool is_linear{true}; dim_t numOutElems{1}; + assert(outputs.size() == output_nodes.size()); KParam& out_info{outputs[0].info}; dim_t* outDims{out_info.dims}; dim_t* outStrides{out_info.strides}; + // unsigned nrInputs{0}; dim_t ndims{outDims[3] > 1 ? 4 : outDims[2] > 1 ? 3 @@ -371,6 +425,7 @@ void evalNodes(vector>& outputs, const vector& output_nodes) { for (const Node* node : full_nodes) { is_linear &= node->isLinear(outDims); moddimsFound |= (node->getOp() == af_moddims_t); + // if (node->isBuffer()) { ++nrInputs; } } bool emptyColumnsFound{false}; @@ -391,12 +446,10 @@ void evalNodes(vector>& outputs, const vector& output_nodes) { : 4); } - // for (auto* node : full_nodes) SHOW(*node); // Keep in global scope, so that the nodes remain active for later // referral in case moddims operations or column elimination have to - // take place - // Avoid all cloning/copying when no moddims node is present (high - // chance) + // take place Avoid all cloning/copying when no moddims node is present + // (high chance) if (moddimsFound || emptyColumnsFound) { for (const Node_ids& ids : full_ids) { auto& children{node_clones[ids.id]->m_children}; @@ -434,36 +487,31 @@ void evalNodes(vector>& outputs, const vector& output_nodes) { } } if (emptyColumnsFound) { - const auto isBuffer{ - [](const Node_ptr& ptr) { return ptr->isBuffer(); }}; - for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)}; - (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt; - ++nodeIt) { - BufferNode* buf{static_cast*>(nodeIt->get())}; - removeEmptyColumns(outDims, ndims, buf->m_param.dims.get(), - buf->m_param.strides.get()); - } - for_each(++begin(outputs), end(outputs), - [outDims, ndims](Param& output) { - removeEmptyColumns(outDims, ndims, output.info.dims, - output.info.strides); - }); - ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides); + common::removeEmptyDimensions, BufferNode, + ShiftNode>(outputs, node_clones); } } full_nodes.clear(); for (Node_ptr& node : node_clones) { full_nodes.push_back(node.get()); } - const string funcName{getFuncName(output_nodes, full_nodes, full_ids, - is_linear, false, false, false, + const string funcName{getFuncName(output_nodes, output_ids, full_nodes, + full_ids, is_linear, false, false, false, outputs[0].info.dims[2] > 1)}; getQueue().submit([&](sycl::handler& h) { for (Node* node : full_nodes) { - if (node->isBuffer()) { - BufferNode* n = static_cast*>(node); - n->m_param.require(h); + switch (node->getNodeType()) { + case kNodeType::Buffer: { + BufferNode* n = static_cast*>(node); + n->m_param.require(h); + } break; + case kNodeType::Shift: { + ShiftNodeBase>* sn = + static_cast>*>(node); + sn->getBufferNode().m_param.require(h); + } break; + default: break; } } vector> ap; @@ -489,15 +537,15 @@ void evalNodes(vector>& outputs, const vector& output_nodes) { ap, is_linear); int nargs{0}; for (Node* node : full_nodes) { - if (node->isBuffer()) { - nargs = node->setArgs( - nargs, is_linear, - [&kernel, &hh, &is_linear]( - int id, const void* ptr, size_t arg_size) { - AParam* info = - static_cast*>( - const_cast(ptr)); + nargs = node->setArgs( + nargs, is_linear, + [&kernel, &hh, &is_linear](int id, const void* ptr, + size_t arg_size, + bool is_buffer) { + if (is_buffer) { + auto* info = static_cast< + AParam*>( + const_cast(ptr)); vector mem = hh.get_native_mem( info->data); @@ -517,16 +565,12 @@ void evalNodes(vector>& outputs, const vector& output_nodes) { sizeof(KParam), &ooo)); } - }); - } else { - nargs = node->setArgs( - nargs, is_linear, - [&kernel](int id, const void* ptr, - size_t arg_size) { + + } else { CL_CHECK(clSetKernelArg(kernel, id, arg_size, ptr)); - }); - } + } + }); } // Set output parameters @@ -554,7 +598,19 @@ void evalNodes(vector>& outputs, const vector& output_nodes) { (size_t)ap[0].dims[2]}; ndims = 3; } - // SHOW(global); + + { + using namespace oneapi::kernel_logger; + AF_TRACE( + "Launching {}: Dims: [{},{},{},{}] Global: " + "[{},{},{}] threads: {}", + funcName, ap[0].dims[0], ap[0].dims[1], + ap[0].dims[2], ap[0].dims[3], global[0], global[1], + global[2], + global[0] * std::max(1, global[1]) * + std::max(1, global[2])); + } + cl_event kernel_event; CL_CHECK(clEnqueueNDRangeKernel( q, kernel, ndims, offset.data(), global.data(), nullptr, @@ -588,6 +644,7 @@ template void evalNodes(Param& out, Node* node); template void evalNodes(Param& out, Node* node); template void evalNodes(Param& out, Node* node); template void evalNodes(Param& out, Node* node); +template void evalNodes(Param& out, Node* node); template void evalNodes(Param& out, Node* node); template void evalNodes(Param& out, Node* node); template void evalNodes(Param& out, Node* node); @@ -609,6 +666,8 @@ template void evalNodes(vector>& out, const vector& node); template void evalNodes(vector>& out, const vector& node); +template void evalNodes(vector>& out, + const vector& node); template void evalNodes(vector>& out, const vector& node); template void evalNodes(vector>& out, diff --git a/src/backend/oneapi/jit/BufferNode.hpp b/src/backend/oneapi/jit/BufferNode.hpp index 94655f23e7..d10ca24cc3 100644 --- a/src/backend/oneapi/jit/BufferNode.hpp +++ b/src/backend/oneapi/jit/BufferNode.hpp @@ -31,7 +31,16 @@ bool BufferNodeBase::operator==( // clang-format off return m_data.get() == other.m_data.get() && m_bytes == other.m_bytes && - m_param.offset == other.m_param.offset; + m_param.offset == other.m_param.offset && + m_linear_buffer == other.m_linear_buffer && + m_param.dims[0] == other.m_param.dims[0] && + m_param.dims[1] == other.m_param.dims[1] && + m_param.dims[2] == other.m_param.dims[2] && + m_param.dims[3] == other.m_param.dims[3] && + m_param.strides[0] == other.m_param.strides[0] && + m_param.strides[1] == other.m_param.strides[1] && + m_param.strides[2] == other.m_param.strides[2] && + m_param.strides[3] == other.m_param.strides[3]; // clang-format on } diff --git a/src/backend/oneapi/jit/ShiftNode.hpp b/src/backend/oneapi/jit/ShiftNode.hpp new file mode 100644 index 0000000000..6a87b28729 --- /dev/null +++ b/src/backend/oneapi/jit/ShiftNode.hpp @@ -0,0 +1,22 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#include +#include + +namespace arrayfire { +namespace oneapi { +namespace jit { + +template +using ShiftNode = common::ShiftNodeBase>; + +} // namespace jit +} // namespace oneapi +} // namespace arrayfire diff --git a/src/backend/oneapi/jit/kernel_generators.hpp b/src/backend/oneapi/jit/kernel_generators.hpp index bc12929fe6..9ca9cd984e 100644 --- a/src/backend/oneapi/jit/kernel_generators.hpp +++ b/src/backend/oneapi/jit/kernel_generators.hpp @@ -38,12 +38,14 @@ inline void generateParamDeclaration(std::stringstream& kerStream, int id, /// Calls the setArg function to set the arguments for a kernel call template -inline int setKernelArguments( +inline int setBufferKernelArguments( int start_id, bool is_linear, - std::function& setArg, + std::function& setArg, const std::shared_ptr>& ptr, const AParam& info) { - setArg(start_id + 0, static_cast(&info), sizeof(Param)); + setArg(start_id + 0, static_cast(&info), + sizeof(AParam), true); return start_id + 2; } @@ -61,8 +63,9 @@ inline void generateBufferOffsets(std::stringstream& kerStream, int id, << info_str << ".strides[3] * id3 + (id2 < " << info_str << ".dims[2]) * " << info_str << ".strides[2] * id2 + (id1 < " << info_str << ".dims[1]) * " << info_str - << ".strides[1] * id1 + (id0 < " << info_str - << ".dims[0]) * id0 + " << info_str << ".offset;\n"; + << ".strides[1] * id1 + (id0 < " << info_str << ".dims[0]) * " + << info_str << ".strides[0] * id0 + " << info_str + << ".offset;\n"; } } diff --git a/src/backend/oneapi/join.cpp b/src/backend/oneapi/join.cpp index 37c7c14fc9..a64e6edb9d 100644 --- a/src/backend/oneapi/join.cpp +++ b/src/backend/oneapi/join.cpp @@ -94,15 +94,17 @@ Array join(const int jdim, const Array &first, const Array &second) { if (first.isReady()) { if (1LL + jdim >= first.ndims() && first.isLinear()) { // first & out are linear + auto first_array = first.get(); + auto out_array = out.get(); getQueue().submit([&](sycl::handler &h) { sycl::range sz(first.elements()); sycl::id src_offset(first.getOffset()); sycl::accessor offset_acc_src = - first.get()->template get_access( + first_array->template get_access( h, sz, src_offset); sycl::id dst_offset(0); sycl::accessor offset_acc_dst = - out.get()->template get_access( + out_array->template get_access( h, sz, dst_offset); h.copy(offset_acc_src, offset_acc_dst); }); @@ -125,16 +127,18 @@ Array join(const int jdim, const Array &first, const Array &second) { if (second.isReady()) { if (1LL + jdim >= second.ndims() && second.isLinear()) { // second & out are linear + auto second_array = second.get(); + auto out_array = out.get(); getQueue().submit([&](sycl::handler &h) { sycl::range sz(second.elements()); sycl::id src_offset(second.getOffset()); sycl::accessor offset_acc_src = - second.get()->template get_access( + second_array->template get_access( h, sz, src_offset); sycl::id dst_offset(fdims.dims[jdim] * out.strides().dims[jdim]); sycl::accessor offset_acc_dst = - out.get()->template get_access( + out_array->template get_access( h, sz, dst_offset); h.copy(offset_acc_src, offset_acc_dst); }); @@ -216,11 +220,12 @@ void join(Array &out, const int jdim, const vector> &inputs) { for (const Array *in : s.ins) { if (in->isReady()) { if (1LL + jdim >= in->ndims() && in->isLinear()) { + auto in_array = in->get(); getQueue().submit([&](sycl::handler &h) { sycl::range sz(in->elements()); sycl::id src_offset(in->getOffset()); sycl::accessor offset_acc_src = - in->get() + in_array ->template get_access< sycl::access_mode::read>(h, sz, src_offset); @@ -267,6 +272,7 @@ INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(half) @@ -287,6 +293,7 @@ INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(half) diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp index 5e3ef6c666..1b69827d18 100644 --- a/src/backend/oneapi/kernel/assign.hpp +++ b/src/backend/oneapi/kernel/assign.hpp @@ -88,7 +88,7 @@ class assignKernel { p_.strds[3] * trimIndex(s3 ? gw + p_.offs[3] : p_.ptr[3][gw], oInfo_.dims[3]); - T* iptr = in_.get_pointer(); + const T* iptr = in_.get_pointer(); // offset input and output pointers const T* src = iptr + (gx * iInfo_.strides[0] + gy * iInfo_.strides[1] + diff --git a/src/backend/oneapi/kernel/assign_kernel_param.hpp b/src/backend/oneapi/kernel/assign_kernel_param.hpp index e2539ed2b3..e2eec56d18 100644 --- a/src/backend/oneapi/kernel/assign_kernel_param.hpp +++ b/src/backend/oneapi/kernel/assign_kernel_param.hpp @@ -19,6 +19,7 @@ namespace oneapi { typedef struct { int offs[4]; int strds[4]; + int steps[4]; bool isSeq[4]; std::array, diff --git a/src/backend/oneapi/kernel/convolve1.hpp b/src/backend/oneapi/kernel/convolve1.hpp index e156308b34..41c6facae6 100644 --- a/src/backend/oneapi/kernel/convolve1.hpp +++ b/src/backend/oneapi/kernel/convolve1.hpp @@ -174,6 +174,7 @@ INSTANTIATE_CONV1(double, double) INSTANTIATE_CONV1(float, float) INSTANTIATE_CONV1(uint, float) INSTANTIATE_CONV1(int, float) +INSTANTIATE_CONV1(schar, float) INSTANTIATE_CONV1(uchar, float) INSTANTIATE_CONV1(char, float) INSTANTIATE_CONV1(ushort, float) diff --git a/src/backend/oneapi/kernel/convolve2.hpp b/src/backend/oneapi/kernel/convolve2.hpp index b216e50917..45bfa6c108 100644 --- a/src/backend/oneapi/kernel/convolve2.hpp +++ b/src/backend/oneapi/kernel/convolve2.hpp @@ -195,4 +195,5 @@ INSTANTIATE_CONV2(intl, float) INSTANTIATE_CONV2(ushort, float) INSTANTIATE_CONV2(uint, float) INSTANTIATE_CONV2(uintl, float) +INSTANTIATE_CONV2(schar, float) INSTANTIATE_CONV2(uchar, float) diff --git a/src/backend/oneapi/kernel/convolve3.hpp b/src/backend/oneapi/kernel/convolve3.hpp index 3ac4a50aa2..bdfcc4eb24 100644 --- a/src/backend/oneapi/kernel/convolve3.hpp +++ b/src/backend/oneapi/kernel/convolve3.hpp @@ -193,6 +193,7 @@ INSTANTIATE_CONV3(double, double) INSTANTIATE_CONV3(float, float) INSTANTIATE_CONV3(uint, float) INSTANTIATE_CONV3(int, float) +INSTANTIATE_CONV3(schar, float) INSTANTIATE_CONV3(uchar, float) INSTANTIATE_CONV3(char, float) INSTANTIATE_CONV3(ushort, float) diff --git a/src/backend/oneapi/kernel/convolve_separable.cpp b/src/backend/oneapi/kernel/convolve_separable.cpp index 712570a558..0f3dfacb30 100644 --- a/src/backend/oneapi/kernel/convolve_separable.cpp +++ b/src/backend/oneapi/kernel/convolve_separable.cpp @@ -76,7 +76,6 @@ class convolveSeparableCreateKernel { if (CONV_DIM_ == 0) { gx += (EXPAND_ ? 0 : FLEN_ >> 1); int endX = ((FLEN_ - 1) << 1) + g.get_local_range(0); -#pragma unroll for (int lx = it.get_local_id(0), glb_x = gx; lx < endX; lx += g.get_local_range(0), glb_x += g.get_local_range(0)) { int i = glb_x - radius; @@ -90,7 +89,6 @@ class convolveSeparableCreateKernel { } else if (CONV_DIM_ == 1) { gy += (EXPAND_ ? 0 : FLEN_ >> 1); int endY = ((FLEN_ - 1) << 1) + g.get_local_range(1); -#pragma unroll for (int ly = it.get_local_id(1), glb_y = gy; ly < endY; ly += g.get_local_range(1), glb_y += g.get_local_range(1)) { int i = gx; @@ -108,7 +106,6 @@ class convolveSeparableCreateKernel { // kernel compilation int i = (CONV_DIM_ == 0 ? lx : ly) + radius; accType accum = (accType)(0); -#pragma unroll for (int f = 0; f < FLEN_; ++f) { accType f_val = impulse_[f]; // below conditional statement is based on MACRO value passed @@ -163,8 +160,6 @@ void convSep(Param out, const Param signal, const Param filter, } constexpr int THREADS_X = 16; constexpr int THREADS_Y = 16; - constexpr bool IsComplex = - std::is_same::value || std::is_same::value; const int fLen = filter.info.dims[0] * filter.info.dims[1]; const size_t C0_SIZE = (THREADS_X + 2 * (fLen - 1)) * THREADS_Y; @@ -205,6 +200,7 @@ INSTANTIATE(double, double) INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(char, float) INSTANTIATE(ushort, float) diff --git a/src/backend/oneapi/kernel/fftconvolve_common.hpp b/src/backend/oneapi/kernel/fftconvolve_common.hpp new file mode 100644 index 0000000000..6caf9923d2 --- /dev/null +++ b/src/backend/oneapi/kernel/fftconvolve_common.hpp @@ -0,0 +1,74 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace arrayfire { +namespace oneapi { +namespace kernel { + +constexpr int THREADS = 256; + +template +void calcParamSizes(Param& sig_tmp, Param& filter_tmp, + Param& packed, Param& sig, Param& filter, + const int rank, AF_BATCH_KIND kind) { + sig_tmp.info.dims[0] = filter_tmp.info.dims[0] = packed.info.dims[0]; + sig_tmp.info.strides[0] = filter_tmp.info.strides[0] = 1; + + for (int k = 1; k < 4; k++) { + if (k < rank) { + sig_tmp.info.dims[k] = packed.info.dims[k]; + filter_tmp.info.dims[k] = packed.info.dims[k]; + } else { + sig_tmp.info.dims[k] = sig.info.dims[k]; + filter_tmp.info.dims[k] = filter.info.dims[k]; + } + + sig_tmp.info.strides[k] = + sig_tmp.info.strides[k - 1] * sig_tmp.info.dims[k - 1]; + filter_tmp.info.strides[k] = + filter_tmp.info.strides[k - 1] * filter_tmp.info.dims[k - 1]; + } + + // NOTE: The OpenCL implementation on which this oneAPI port is + // based treated the incoming `packed` buffer as a string of real + // scalars instead of complex numbers. OpenCL accomplished this + // with the hack depicted in the trailing two lines. This note + // remains here in an explanation of SYCL buffer reinterpret's in + // fftconvolve kernel invocations. + + // sig_tmp.data = packed.data; + // filter_tmp.data = packed.data; + + // Calculate memory offsets for packed signal and filter + if (kind == AF_BATCH_RHS) { + filter_tmp.info.offset = 0; + sig_tmp.info.offset = + filter_tmp.info.strides[3] * filter_tmp.info.dims[3] * 2; + } else { + sig_tmp.info.offset = 0; + filter_tmp.info.offset = + sig_tmp.info.strides[3] * sig_tmp.info.dims[3] * 2; + } +} + +} // namespace kernel +} // namespace oneapi +} // namespace arrayfire diff --git a/src/backend/oneapi/kernel/fftconvolve_multiply.hpp b/src/backend/oneapi/kernel/fftconvolve_multiply.hpp new file mode 100644 index 0000000000..32516f4056 --- /dev/null +++ b/src/backend/oneapi/kernel/fftconvolve_multiply.hpp @@ -0,0 +1,153 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +namespace arrayfire { +namespace oneapi { +namespace kernel { + +template +class fftconvolve_multiplyCreateKernel { + public: + fftconvolve_multiplyCreateKernel(write_accessor d_out, KParam oInfo, + read_accessor d_in1, KParam i1Info, + read_accessor d_in2, KParam i2Info, + const int nelem, const int kind) + : d_out_(d_out) + , oInfo_(oInfo) + , d_in1_(d_in1) + , i1Info_(i1Info) + , d_in2_(d_in2) + , i2Info_(i2Info) + , nelem_(nelem) + , kind_(kind) {} + void operator()(sycl::nd_item<1> it) const { + const int t = it.get_global_id(0); + + if (t >= nelem_) return; + + if (kind_ == AF_BATCH_NONE || kind_ == AF_BATCH_SAME) { + // Complex multiply each signal to equivalent filter + const int ridx = t * 2; + const int iidx = t * 2 + 1; + + T a = d_in1_[i1Info_.offset + ridx]; + T b = d_in1_[i1Info_.offset + iidx]; + T c = d_in2_[i2Info_.offset + ridx]; + T d = d_in2_[i2Info_.offset + iidx]; + + d_out_[oInfo_.offset + ridx] = a * c - b * d; + d_out_[oInfo_.offset + iidx] = a * d + b * c; + } else if (kind_ == AF_BATCH_LHS) { + // Complex multiply all signals to filter + const int ridx1 = t * 2; + const int iidx1 = t * 2 + 1; + + // Treating complex output array as real-only array, + // thus, multiply strides by 2 + const int ridx2 = + ridx1 % (i2Info_.strides[3] * i2Info_.dims[3] * 2); + const int iidx2 = + iidx1 % (i2Info_.strides[3] * i2Info_.dims[3] * 2); + + T a = d_in1_[i1Info_.offset + ridx1]; + T b = d_in1_[i1Info_.offset + iidx1]; + T c = d_in2_[i2Info_.offset + ridx2]; + T d = d_in2_[i2Info_.offset + iidx2]; + + d_out_[oInfo_.offset + ridx1] = a * c - b * d; + d_out_[oInfo_.offset + iidx1] = a * d + b * c; + } else if (kind_ == AF_BATCH_RHS) { + // Complex multiply signal to all filters + const int ridx2 = t * 2; + const int iidx2 = t * 2 + 1; + + // Treating complex output array as real-only array, + // thus, multiply strides by 2 + const int ridx1 = + ridx2 % (i1Info_.strides[3] * i1Info_.dims[3] * 2); + const int iidx1 = + iidx2 % (i1Info_.strides[3] * i1Info_.dims[3] * 2); + + T a = d_in1_[i1Info_.offset + ridx1]; + T b = d_in1_[i1Info_.offset + iidx1]; + T c = d_in2_[i2Info_.offset + ridx2]; + T d = d_in2_[i2Info_.offset + iidx2]; + + d_out_[oInfo_.offset + ridx2] = a * c - b * d; + d_out_[oInfo_.offset + iidx2] = a * d + b * c; + } + } + + private: + write_accessor d_out_; + KParam oInfo_; + read_accessor d_in1_; + KParam i1Info_; + read_accessor d_in2_; + KParam i2Info_; + const int nelem_; + const int kind_; +}; + +template +void complexMultiplyHelper(Param packed, Param sig, Param filter, + const int rank, AF_BATCH_KIND kind) { + Param sig_tmp, filter_tmp; + calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind); + + int sig_packed_elem = sig_tmp.info.strides[3] * sig_tmp.info.dims[3]; + int filter_packed_elem = + filter_tmp.info.strides[3] * filter_tmp.info.dims[3]; + int mul_elem = (sig_packed_elem < filter_packed_elem) ? filter_packed_elem + : sig_packed_elem; + int blocks = divup(mul_elem, THREADS); + + auto local = sycl::range(THREADS); + auto global = sycl::range(blocks * THREADS); + + // Treat complex output as an array of scalars + using convScalarT = typename convT::value_type; + auto packed_num_elem = (*packed.data).get_range().size(); + auto packed_tmp_buffer = (*packed.data) + .template reinterpret( + sycl::range<1>{packed_num_elem * 2}); + auto sig_tmp_buffer = (*packed.data) + .template reinterpret( + sycl::range<1>{packed_num_elem * 2}); + auto filter_tmp_buffer = (*packed.data) + .template reinterpret( + sycl::range<1>{packed_num_elem * 2}); + + getQueue().submit([&](auto &h) { + write_accessor d_packed = {packed_tmp_buffer, h}; + read_accessor d_sig_tmp = {sig_tmp_buffer, h}; + read_accessor d_filter_tmp = {filter_tmp_buffer, h}; + h.parallel_for( + sycl::nd_range{global, local}, + fftconvolve_multiplyCreateKernel( + d_packed, packed.info, d_sig_tmp, sig_tmp.info, d_filter_tmp, + filter_tmp.info, mul_elem, (int)kind)); + }); + + ONEAPI_DEBUG_FINISH(getQueue()); +} +} // namespace kernel +} // namespace oneapi +} // namespace arrayfire diff --git a/src/backend/oneapi/kernel/fftconvolve_pack.hpp b/src/backend/oneapi/kernel/fftconvolve_pack.hpp new file mode 100644 index 0000000000..5f8afc2b7a --- /dev/null +++ b/src/backend/oneapi/kernel/fftconvolve_pack.hpp @@ -0,0 +1,142 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace arrayfire { +namespace oneapi { +namespace kernel { + +template +class fftconvolve_packCreateKernel { + public: + fftconvolve_packCreateKernel(write_accessor d_out, KParam oInfo, + read_accessor d_in, KParam iInfo, + const int di0_half, const int odd_di0) + : d_out_(d_out) + , oInfo_(oInfo) + , d_in_(d_in) + , iInfo_(iInfo) + , di0_half_(di0_half) + , odd_di0_(odd_di0) {} + void operator()(sycl::nd_item<1> it) const { + const int t = it.get_global_id(0); + + const int tMax = oInfo_.strides[3] * oInfo_.dims[3]; + + if (t >= tMax) return; + + // const int do0 = oInfo_.dims[0]; + const int do1 = oInfo_.dims[1]; + const int do2 = oInfo_.dims[2]; + + const int so1 = oInfo_.strides[1]; + const int so2 = oInfo_.strides[2]; + const int so3 = oInfo_.strides[3]; + + const int to0 = t % so1; + const int to1 = (t / so1) % do1; + const int to2 = (t / so2) % do2; + const int to3 = t / so3; + + // const int di0 = iInfo_.dims[0]; + const int di1 = iInfo_.dims[1]; + const int di2 = iInfo_.dims[2]; + + const int si1 = iInfo_.strides[1]; + const int si2 = iInfo_.strides[2]; + const int si3 = iInfo_.strides[3]; + + const int ti0 = to0; + const int ti1 = to1 * si1; + const int ti2 = to2 * si2; + const int ti3 = to3 * si3; + + const int iidx1 = iInfo_.offset + ti3 + ti2 + ti1 + ti0; + const int iidx2 = iidx1 + di0_half_; + + // Treating complex output array as real-only array, + // thus, multiply strides by 2 + const int oidx1 = oInfo_.offset + to3 * so3 * 2 + to2 * so2 * 2 + + to1 * so1 * 2 + to0 * 2; + const int oidx2 = oidx1 + 1; + + if (to0 < di0_half_ && to1 < di1 && to2 < di2) { + d_out_[oidx1] = (outputType)d_in_[iidx1]; + if (ti0 == di0_half_ - 1 && odd_di0_ == 1) + d_out_[oidx2] = (outputType)0; + else + d_out_[oidx2] = (outputType)d_in_[iidx2]; + } else { + // Pad remaining elements with 0s + d_out_[oidx1] = (outputType)0; + d_out_[oidx2] = (outputType)0; + } + } + + private: + write_accessor d_out_; + KParam oInfo_; + read_accessor d_in_; + KParam iInfo_; + const int di0_half_; + const int odd_di0_; +}; + +template +void packDataHelper(Param packed, Param sig, Param filter, + const int rank, AF_BATCH_KIND kind) { + Param sig_tmp, filter_tmp; + calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind); + + int sig_packed_elem = sig_tmp.info.strides[3] * sig_tmp.info.dims[3]; + + // Number of packed complex elements in dimension 0 + int sig_half_d0 = divup(sig.info.dims[0], 2); + int sig_half_d0_odd = sig.info.dims[0] % 2; + + int blocks = divup(sig_packed_elem, THREADS); + + // Locate features kernel sizes + auto local = sycl::range(THREADS); + auto global = sycl::range(blocks * THREADS); + + // Treat complex output as an array of scalars + using convScalarT = typename convT::value_type; + auto packed_num_elem = (*packed.data).get_range().size(); + auto sig_tmp_buffer = (*packed.data) + .template reinterpret( + sycl::range<1>{packed_num_elem * 2}); + + getQueue().submit([&](auto &h) { + read_accessor d_sig = {*sig.data, h}; + write_accessor d_sig_tmp = {sig_tmp_buffer, h}; + h.parallel_for(sycl::nd_range{global, local}, + fftconvolve_packCreateKernel( + d_sig_tmp, sig_tmp.info, d_sig, sig.info, + sig_half_d0, sig_half_d0_odd)); + }); + + ONEAPI_DEBUG_FINISH(getQueue()); +} + +} // namespace kernel +} // namespace oneapi +} // namespace arrayfire diff --git a/src/backend/oneapi/kernel/fftconvolve_pad.hpp b/src/backend/oneapi/kernel/fftconvolve_pad.hpp new file mode 100644 index 0000000000..6d60506236 --- /dev/null +++ b/src/backend/oneapi/kernel/fftconvolve_pad.hpp @@ -0,0 +1,122 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +namespace arrayfire { +namespace oneapi { +namespace kernel { + +template +class fftconvolve_padCreateKernel { + public: + fftconvolve_padCreateKernel(write_accessor d_out, KParam oInfo, + read_accessor d_in, KParam iInfo) + : d_out_(d_out), oInfo_(oInfo), d_in_(d_in), iInfo_(iInfo) {} + void operator()(sycl::nd_item<1> it) const { + const int t = it.get_global_id(0); + + const int tMax = oInfo_.strides[3] * oInfo_.dims[3]; + + if (t >= tMax) return; + + // const int do0 = oInfo_.dims[0]; + const int do1 = oInfo_.dims[1]; + const int do2 = oInfo_.dims[2]; + + const int so1 = oInfo_.strides[1]; + const int so2 = oInfo_.strides[2]; + const int so3 = oInfo_.strides[3]; + + const int to0 = t % so1; + const int to1 = (t / so1) % do1; + const int to2 = (t / so2) % do2; + const int to3 = (t / so3); + + const int di0 = iInfo_.dims[0]; + const int di1 = iInfo_.dims[1]; + const int di2 = iInfo_.dims[2]; + const int di3 = iInfo_.dims[3]; + + const int si1 = iInfo_.strides[1]; + const int si2 = iInfo_.strides[2]; + const int si3 = iInfo_.strides[3]; + + const int ti0 = to0; + const int ti1 = to1 * si1; + const int ti2 = to2 * si2; + const int ti3 = to3 * si3; + + const int iidx = iInfo_.offset + ti3 + ti2 + ti1 + ti0; + + const int oidx = oInfo_.offset + t * 2; + + if (to0 < di0 && to1 < di1 && to2 < di2 && to3 < di3) { + // Copy input elements to real elements, set imaginary elements to 0 + d_out_[oidx] = (outputType)d_in_[iidx]; + d_out_[oidx + 1] = (outputType)0; + } else { + // Pad remaining of the matrix to 0s + d_out_[oidx] = (outputType)0; + d_out_[oidx + 1] = (outputType)0; + } + } + + private: + write_accessor d_out_; + KParam oInfo_; + read_accessor d_in_; + KParam iInfo_; +}; + +template +void padDataHelper(Param packed, Param sig, Param filter, + const int rank, AF_BATCH_KIND kind) { + Param sig_tmp, filter_tmp; + calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind); + + int filter_packed_elem = + filter_tmp.info.strides[3] * filter_tmp.info.dims[3]; + + int blocks = divup(filter_packed_elem, THREADS); + + // Locate features kernel sizes + auto local = sycl::range(THREADS); + auto global = sycl::range(blocks * THREADS); + + // Treat complex output as an array of scalars + using convScalarT = typename convT::value_type; + auto packed_num_elem = (*packed.data).get_range().size(); + auto filter_tmp_buffer = (*packed.data) + .template reinterpret( + sycl::range<1>{packed_num_elem * 2}); + + getQueue().submit([&](auto &h) { + read_accessor d_filter = {*filter.data, h, sycl::read_only}; + write_accessor d_filter_tmp = {filter_tmp_buffer, h}; + h.parallel_for( + sycl::nd_range{global, local}, + fftconvolve_padCreateKernel( + d_filter_tmp, filter_tmp.info, d_filter, filter.info)); + }); + + ONEAPI_DEBUG_FINISH(getQueue()); +} +} // namespace kernel +} // namespace oneapi +} // namespace arrayfire diff --git a/src/backend/oneapi/kernel/fftconvolve_reorder.hpp b/src/backend/oneapi/kernel/fftconvolve_reorder.hpp new file mode 100644 index 0000000000..589242007a --- /dev/null +++ b/src/backend/oneapi/kernel/fftconvolve_reorder.hpp @@ -0,0 +1,187 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +namespace arrayfire { +namespace oneapi { +namespace kernel { + +template +class fftconvolve_reorderCreateKernel { + public: + fftconvolve_reorderCreateKernel(write_accessor d_out, KParam oInfo, + read_accessor d_in, + KParam iInfo, KParam fInfo, + const int half_di0, const int baseDim, + const int fftScale, const bool EXPAND, + const bool ROUND_OUT) + : d_out_(d_out) + , oInfo_(oInfo) + , d_in_(d_in) + , iInfo_(iInfo) + , fInfo_(fInfo) + , half_di0_(half_di0) + , baseDim_(baseDim) + , fftScale_(fftScale) + , EXPAND_(EXPAND) + , ROUND_OUT_(ROUND_OUT) {} + void operator()(sycl::nd_item<1> it) const { + const int t = it.get_global_id(0); + + const int tMax = oInfo_.strides[3] * oInfo_.dims[3]; + + if (t >= tMax) return; + + // const int do0 = oInfo_.dims[0]; + const int do1 = oInfo_.dims[1]; + const int do2 = oInfo_.dims[2]; + + const int so1 = oInfo_.strides[1]; + const int so2 = oInfo_.strides[2]; + const int so3 = oInfo_.strides[3]; + + // Treating complex input array as real-only array, + // thus, multiply dimension 0 and strides by 2 + const int si1 = iInfo_.strides[1] * 2; + const int si2 = iInfo_.strides[2] * 2; + const int si3 = iInfo_.strides[3] * 2; + + const int to0 = t % so1; + const int to1 = (t / so1) % do1; + const int to2 = (t / so2) % do2; + const int to3 = (t / so3); + + int oidx = to3 * so3 + to2 * so2 + to1 * so1 + to0; + + int ti0, ti1, ti2, ti3; + if (EXPAND_) { + ti0 = to0; + ti1 = to1 * si1; + ti2 = to2 * si2; + ti3 = to3 * si3; + } else { + ti0 = to0 + fInfo_.dims[0] / 2; + ti1 = (to1 + (baseDim_ > 1) * (fInfo_.dims[1] / 2)) * si1; + ti2 = (to2 + (baseDim_ > 2) * (fInfo_.dims[2] / 2)) * si2; + ti3 = to3 * si3; + } + + // Divide output elements to cuFFT resulting scale, round result if + // output type is single or double precision floating-point + if (ti0 < half_di0_) { + // Copy top elements + int iidx = iInfo_.offset + ti3 + ti2 + ti1 + ti0 * 2; + if (ROUND_OUT_) + d_out_[oidx] = (T)round(d_in_[iidx] / fftScale_); + else + d_out_[oidx] = (T)(d_in_[iidx] / fftScale_); + } else if (ti0 < half_di0_ + fInfo_.dims[0] - 1) { + // Add central elements + int iidx1 = iInfo_.offset + ti3 + ti2 + ti1 + ti0 * 2; + int iidx2 = + iInfo_.offset + ti3 + ti2 + ti1 + (ti0 - half_di0_) * 2 + 1; + if (ROUND_OUT_) + d_out_[oidx] = + (T)round((d_in_[iidx1] + d_in_[iidx2]) / fftScale_); + else + d_out_[oidx] = (T)((d_in_[iidx1] + d_in_[iidx2]) / fftScale_); + } else { + // Copy bottom elements + const int iidx = + iInfo_.offset + ti3 + ti2 + ti1 + (ti0 - half_di0_) * 2 + 1; + if (ROUND_OUT_) + d_out_[oidx] = (T)round(d_in_[iidx] / fftScale_); + else + d_out_[oidx] = (T)(d_in_[iidx] / fftScale_); + } + } + + private: + write_accessor d_out_; + KParam oInfo_; + read_accessor d_in_; + KParam iInfo_; + KParam fInfo_; + const int half_di0_; + const int baseDim_; + const int fftScale_; + const bool EXPAND_; + const bool ROUND_OUT_; +}; + +template +void reorderOutputHelper(Param out, Param packed, Param sig, + Param filter, const int rank, AF_BATCH_KIND kind, + bool expand) { + int fftScale = 1; + + // Calculate the scale by which to divide clFFT results + for (int k = 0; k < rank; k++) fftScale *= packed.info.dims[k]; + + Param sig_tmp, filter_tmp; + calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind); + + // Number of packed complex elements in dimension 0 + int sig_half_d0 = divup(sig.info.dims[0], 2); + + int blocks = divup(out.info.strides[3] * out.info.dims[3], THREADS); + + constexpr bool round_out = std::is_integral::value; + + auto local = sycl::range(THREADS); + auto global = sycl::range(blocks * THREADS); + + using convScalarT = typename convT::value_type; + + if (kind == AF_BATCH_RHS) { + auto packed_num_elem = (*packed.data).get_range().size(); + auto filter_tmp_buffer = (*packed.data) + .template reinterpret( + sycl::range<1>{packed_num_elem * 2}); + getQueue().submit([&](auto &h) { + read_accessor d_filter_tmp = {filter_tmp_buffer, h}; + write_accessor d_out = {*out.data, h, sycl::write_only}; + h.parallel_for( + sycl::nd_range{global, local}, + fftconvolve_reorderCreateKernel( + d_out, out.info, d_filter_tmp, filter_tmp.info, filter.info, + sig_half_d0, rank, fftScale, expand, round_out)); + }); + } else { + auto packed_num_elem = (*packed.data).get_range().size(); + auto sig_tmp_buffer = (*packed.data) + .template reinterpret( + sycl::range<1>{packed_num_elem * 2}); + getQueue().submit([&](auto &h) { + read_accessor d_sig_tmp = {sig_tmp_buffer, h, + sycl::read_only}; + write_accessor d_out = {*out.data, h}; + h.parallel_for( + sycl::nd_range{global, local}, + fftconvolve_reorderCreateKernel( + d_out, out.info, d_sig_tmp, sig_tmp.info, filter.info, + sig_half_d0, rank, fftScale, expand, round_out)); + }); + } + + ONEAPI_DEBUG_FINISH(getQueue()); +} +} // namespace kernel +} // namespace oneapi +} // namespace arrayfire diff --git a/src/backend/oneapi/kernel/iir.hpp b/src/backend/oneapi/kernel/iir.hpp index 38769ad46a..938202f32f 100644 --- a/src/backend/oneapi/kernel/iir.hpp +++ b/src/backend/oneapi/kernel/iir.hpp @@ -21,8 +21,6 @@ namespace arrayfire { namespace oneapi { namespace kernel { -constexpr int MAX_A_SIZE = 1024; - template class iirKernel { public: @@ -67,10 +65,9 @@ class iirKernel { const int repeat = (num_a + g.get_local_range(0) - 1) / g.get_local_range(0); - for (int ii = 0; ii < MAX_A_SIZE / g.get_local_range(0); ii++) { - int id = ii * g.get_local_range(0) + tx; - s_z_[id] = scalar(0); - s_a_[id] = (id < num_a) ? d_a[id] : scalar(0); + for (int ii = tx; ii < num_a; ii += g.get_local_range(0)) { + s_z_[ii] = scalar(0); + s_a_[ii] = (ii < num_a) ? d_a[ii] : scalar(0); } group_barrier(g); @@ -81,14 +78,19 @@ class iirKernel { } group_barrier(g); -#pragma unroll for (int ii = 0; ii < repeat; ii++) { int id = ii * g.get_local_range(0) + tx + 1; - T z = s_z_[id] - s_a_[id] * s_y_[0]; + T z; + + if (id < num_a) { + z = s_z_[id] - s_a_[id] * s_y_[0]; + } else { + z = scalar(0); + } group_barrier(g); - s_z_[id - 1] = z; + if ((id - 1) < num_a) { s_z_[id - 1] = z; } group_barrier(g); } } @@ -124,8 +126,10 @@ void iir(Param y, Param c, Param a) { read_accessor cAcc{*c.data, h}; read_accessor aAcc{*a.data, h}; - auto s_z = sycl::local_accessor(MAX_A_SIZE, h); - auto s_a = sycl::local_accessor(MAX_A_SIZE, h); + unsigned num_a = a.info.dims[0]; + + auto s_z = sycl::local_accessor(num_a, h); + auto s_a = sycl::local_accessor(num_a, h); auto s_y = sycl::local_accessor(1, h); if (batch_a) { diff --git a/src/backend/oneapi/kernel/index.hpp b/src/backend/oneapi/kernel/index.hpp index 857b299aef..e86c0bd808 100644 --- a/src/backend/oneapi/kernel/index.hpp +++ b/src/backend/oneapi/kernel/index.hpp @@ -88,13 +88,17 @@ class indexKernel { if (gx < odims0 && gy < odims1 && gz < odims2 && gw < odims3) { // calculate pointer offsets for input int i = p.strds[0] * - trimIndex(s0 ? gx + p.offs[0] : ptr0[gx], inp.dims[0]); + trimIndex(s0 ? gx * p.steps[0] + p.offs[0] : ptr0[gx], + inp.dims[0]); int j = p.strds[1] * - trimIndex(s1 ? gy + p.offs[1] : ptr1[gy], inp.dims[1]); + trimIndex(s1 ? gy * p.steps[1] + p.offs[1] : ptr1[gy], + inp.dims[1]); int k = p.strds[2] * - trimIndex(s2 ? gz + p.offs[2] : ptr2[gz], inp.dims[2]); + trimIndex(s2 ? gz * p.steps[2] + p.offs[2] : ptr2[gz], + inp.dims[2]); int l = p.strds[3] * - trimIndex(s3 ? gw + p.offs[3] : ptr3[gw], inp.dims[3]); + trimIndex(s3 ? gw * p.steps[3] + p.offs[3] : ptr3[gw], + inp.dims[3]); // offset input and output pointers const T* src = (const T*)in.get_pointer() + (i + j + k + l); T* dst = (T*)out.get_pointer() + @@ -133,11 +137,14 @@ void index(Param out, Param in, IndexKernelParam& p, blocks[0] *= threads[0]; sycl::nd_range<3> marange(blocks, threads); + sycl::buffer *idxArrs_get[4]; + for (dim_t x = 0; x < 4; ++x) + idxArrs_get[x] = idxArrs[x].get(); getQueue().submit([&](sycl::handler& h) { auto pp = p; for (dim_t x = 0; x < 4; ++x) { pp.ptr[x] = - idxArrs[x].get()->get_access(h); + idxArrs_get[x]->get_access(h); } h.parallel_for( diff --git a/src/backend/oneapi/kernel/ireduce.hpp b/src/backend/oneapi/kernel/ireduce.hpp index 0c6ae70383..9ba79ed61b 100644 --- a/src/backend/oneapi/kernel/ireduce.hpp +++ b/src/backend/oneapi/kernel/ireduce.hpp @@ -42,7 +42,8 @@ class ireduceDimKernelSMEM { read_accessor in, KParam iInfo, read_accessor iloc, KParam ilocInfo, uint groups_x, uint groups_y, uint groups_dim, - read_accessor rlen, KParam rlenInfo, + bool rlenValid, read_accessor rlen, + KParam rlenInfo, sycl::local_accessor, 1> s_val, sycl::local_accessor s_idx) : out_(out) @@ -56,6 +57,7 @@ class ireduceDimKernelSMEM { , groups_x_(groups_x) , groups_y_(groups_y) , groups_dim_(groups_dim) + , rlenValid_(rlenValid) , rlen_(rlen) , rlenInfo_(rlenInfo) , s_val_(s_val) @@ -90,9 +92,8 @@ class ireduceDimKernelSMEM { const bool rlen_valid = (ids[0] < rlenInfo_.dims[0]) && (ids[1] < rlenInfo_.dims[1]) && (ids[2] < rlenInfo_.dims[2]) && (ids[3] < rlenInfo_.dims[3]); - const bool rlen_nonnull = (rlenInfo_.dims[0] * rlenInfo_.dims[1] * - rlenInfo_.dims[2] * rlenInfo_.dims[3]) > 0; - uint *const rlenptr = + const bool rlen_nonnull = rlenValid_; + const uint *rlenptr = (rlen_nonnull && rlen_valid) ? rlen_.get_pointer() + ids[3] * rlenInfo_.strides[3] + ids[2] * rlenInfo_.strides[2] + @@ -104,10 +105,10 @@ class ireduceDimKernelSMEM { // add thread offset for reduced dim for inputs ids[dim] = ids[dim] * g.get_local_range(1) + lidy; - T *iptr = in_.get_pointer() + ids[3] * iInfo_.strides[3] + - ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] + - ids[0] + iInfo_.offset; - uint *ilptr; + const T *iptr = in_.get_pointer() + ids[3] * iInfo_.strides[3] + + ids[2] * iInfo_.strides[2] + + ids[1] * iInfo_.strides[1] + ids[0] + iInfo_.offset; + const uint *ilptr; if (!is_first) { ilptr = iloc_.get_pointer() + ids[3] * iInfo_.strides[3] + ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] + @@ -204,6 +205,7 @@ class ireduceDimKernelSMEM { read_accessor iloc_; KParam ilocInfo_; uint groups_x_, groups_y_, groups_dim_; + bool rlenValid_; read_accessor rlen_; KParam rlenInfo_; sycl::local_accessor, 1> s_val_; @@ -218,25 +220,25 @@ void ireduce_dim_launcher(Param out, Param oloc, Param in, sycl::range<2> global(groups_dim[0] * groups_dim[2] * local[0], groups_dim[1] * groups_dim[3] * local[1]); - sycl::buffer empty{sycl::range<1>(1)}; + auto iempty = memAlloc(1); + auto rempty = memAlloc(1); getQueue().submit([&](sycl::handler &h) { write_accessor out_acc{*out.data, h}; write_accessor oloc_acc{*oloc.data, h}; read_accessor in_acc{*in.data, h}; - read_accessor iloc_acc{empty, h}; + read_accessor iloc_acc{*iempty, h}; if (iloc.info.dims[0] * iloc.info.dims[1] * iloc.info.dims[2] * iloc.info.dims[3] > 0) { iloc_acc = read_accessor{*iloc.data, h}; } - read_accessor rlen_acc{empty, h}; - if (rlen.info.dims[0] * rlen.info.dims[1] * rlen.info.dims[2] * - rlen.info.dims[3] > - 0) { - rlen_acc = read_accessor{*rlen.data, h}; - } + read_accessor rlen_acc{*rempty, h}; + bool rlenValid = (rlen.info.dims[0] * rlen.info.dims[1] * + rlen.info.dims[2] * rlen.info.dims[3] > + 0); + if (rlenValid) { rlen_acc = read_accessor{*rlen.data, h}; } auto shrdVal = sycl::local_accessor, 1>( creduce::THREADS_PER_BLOCK, h); @@ -250,35 +252,35 @@ void ireduce_dim_launcher(Param out, Param oloc, Param in, ireduceDimKernelSMEM( out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info, iloc_acc, iloc.info, groups_dim[0], groups_dim[1], - groups_dim[dim], rlen_acc, rlen.info, shrdVal, - shrdLoc)); + groups_dim[dim], rlenValid, rlen_acc, rlen.info, + shrdVal, shrdLoc)); break; case 4: h.parallel_for( sycl::nd_range<2>(global, local), - ireduceDimKernelSMEM( + ireduceDimKernelSMEM( out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info, iloc_acc, iloc.info, groups_dim[0], groups_dim[1], - groups_dim[dim], rlen_acc, rlen.info, shrdVal, - shrdLoc)); + groups_dim[dim], rlenValid, rlen_acc, rlen.info, + shrdVal, shrdLoc)); break; case 2: h.parallel_for( sycl::nd_range<2>(global, local), - ireduceDimKernelSMEM( + ireduceDimKernelSMEM( out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info, iloc_acc, iloc.info, groups_dim[0], groups_dim[1], - groups_dim[dim], rlen_acc, rlen.info, shrdVal, - shrdLoc)); + groups_dim[dim], rlenValid, rlen_acc, rlen.info, + shrdVal, shrdLoc)); break; case 1: h.parallel_for( sycl::nd_range<2>(global, local), - ireduceDimKernelSMEM( + ireduceDimKernelSMEM( out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info, iloc_acc, iloc.info, groups_dim[0], groups_dim[1], - groups_dim[dim], rlen_acc, rlen.info, shrdVal, - shrdLoc)); + groups_dim[dim], rlenValid, rlen_acc, rlen.info, + shrdVal, shrdLoc)); break; } }); @@ -335,7 +337,8 @@ class ireduceFirstKernelSMEM { read_accessor in, KParam iInfo, read_accessor iloc, KParam ilocInfo, uint groups_x, uint groups_y, uint repeat, - read_accessor rlen, KParam rlenInfo, + bool rlenValid, read_accessor rlen, + KParam rlenInfo, sycl::local_accessor, 1> s_val, sycl::local_accessor s_idx) : out_(out) @@ -349,6 +352,7 @@ class ireduceFirstKernelSMEM { , groups_x_(groups_x) , groups_y_(groups_y) , repeat_(repeat) + , rlenValid_(rlenValid) , rlen_(rlen) , rlenInfo_(rlenInfo) , s_val_(s_val) @@ -367,28 +371,29 @@ class ireduceFirstKernelSMEM { const uint xid = groupId_x * g.get_local_range(0) * repeat_ + lidx; const uint yid = groupId_y * g.get_local_range(1) + lidy; - T *const iptr = in_.get_pointer() + wid * iInfo_.strides[3] + + const T *iptr = in_.get_pointer() + wid * iInfo_.strides[3] + zid * iInfo_.strides[2] + yid * iInfo_.strides[1] + iInfo_.offset; T *optr = out_.get_pointer() + wid * oInfo_.strides[3] + - zid * oInfo_.strides[2] + yid * oInfo_.strides[1]; - - const bool rlenvalid = (rlenInfo_.dims[0] * rlenInfo_.dims[1] * - rlenInfo_.dims[2] * rlenInfo_.dims[3]) > 0; - uint *const rlenptr = - (rlenvalid) - ? rlen_.get_pointer() + wid * rlenInfo_.strides[3] + - zid * rlenInfo_.strides[2] + yid * rlenInfo_.strides[1] - : nullptr; + zid * oInfo_.strides[2] + yid * oInfo_.strides[1] + + oInfo_.offset; - uint *ilptr; + const uint *rlenptr = + (rlenValid_) ? rlen_.get_pointer() + wid * rlenInfo_.strides[3] + + zid * rlenInfo_.strides[2] + + yid * rlenInfo_.strides[1] + rlenInfo_.offset + : nullptr; + + const uint *ilptr; if (!is_first) { ilptr = iloc_.get_pointer() + wid * iInfo_.strides[3] + - zid * iInfo_.strides[2] + yid * iInfo_.strides[1]; + zid * iInfo_.strides[2] + yid * iInfo_.strides[1] + + iInfo_.offset; } uint *olptr = oloc_.get_pointer() + wid * oInfo_.strides[3] + - zid * oInfo_.strides[2] + yid * oInfo_.strides[1]; + zid * oInfo_.strides[2] + yid * oInfo_.strides[1] + + oInfo_.offset; size_t ylim = iInfo_.dims[1]; size_t zlim = iInfo_.dims[2]; @@ -404,7 +409,7 @@ class ireduceFirstKernelSMEM { compute_t out_val = common::Binary, op>::init(); uint idx = xid; - if (xid < lim) { + if (xid < lim && is_valid) { out_val = static_cast>(iptr[xid]); if (!is_first) idx = ilptr[xid]; } @@ -501,6 +506,7 @@ class ireduceFirstKernelSMEM { read_accessor iloc_; KParam ilocInfo_; uint groups_x_, groups_y_, repeat_; + bool rlenValid_; read_accessor rlen_; KParam rlenInfo_; sycl::local_accessor, 1> s_val_; @@ -518,25 +524,25 @@ void ireduce_first_launcher(Param out, Param oloc, Param in, uint repeat = divup(in.info.dims[0], (groups_x * threads_x)); - sycl::buffer empty{sycl::range<1>(1)}; + auto iempty = memAlloc(1); + auto rempty = memAlloc(1); getQueue().submit([&](sycl::handler &h) { write_accessor out_acc{*out.data, h}; write_accessor oloc_acc{*oloc.data, h}; read_accessor in_acc{*in.data, h}; - read_accessor iloc_acc{empty, h}; + read_accessor iloc_acc{*iempty, h}; if (iloc.info.dims[0] * iloc.info.dims[1] * iloc.info.dims[2] * iloc.info.dims[3] > 0) { iloc_acc = read_accessor{*iloc.data, h}; } - read_accessor rlen_acc{empty, h}; - if (rlen.info.dims[0] * rlen.info.dims[1] * rlen.info.dims[2] * - rlen.info.dims[3] > - 0) { - rlen_acc = read_accessor{*rlen.data, h}; - } + read_accessor rlen_acc{*rempty, h}; + bool rlenValid = (rlen.info.dims[0] * rlen.info.dims[1] * + rlen.info.dims[2] * rlen.info.dims[3] > + 0); + if (rlenValid) { rlen_acc = read_accessor{*rlen.data, h}; } auto shrdVal = sycl::local_accessor, 1>( creduce::THREADS_PER_BLOCK, h); @@ -550,7 +556,7 @@ void ireduce_first_launcher(Param out, Param oloc, Param in, ireduceFirstKernelSMEM( out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info, iloc_acc, iloc.info, groups_x, groups_y, repeat, - rlen_acc, rlen.info, shrdVal, shrdLoc)); + rlenValid, rlen_acc, rlen.info, shrdVal, shrdLoc)); break; case 64: h.parallel_for( @@ -558,7 +564,7 @@ void ireduce_first_launcher(Param out, Param oloc, Param in, ireduceFirstKernelSMEM( out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info, iloc_acc, iloc.info, groups_x, groups_y, repeat, - rlen_acc, rlen.info, shrdVal, shrdLoc)); + rlenValid, rlen_acc, rlen.info, shrdVal, shrdLoc)); break; case 128: h.parallel_for( @@ -566,7 +572,7 @@ void ireduce_first_launcher(Param out, Param oloc, Param in, ireduceFirstKernelSMEM( out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info, iloc_acc, iloc.info, groups_x, groups_y, repeat, - rlen_acc, rlen.info, shrdVal, shrdLoc)); + rlenValid, rlen_acc, rlen.info, shrdVal, shrdLoc)); break; case 256: h.parallel_for( @@ -574,7 +580,7 @@ void ireduce_first_launcher(Param out, Param oloc, Param in, ireduceFirstKernelSMEM( out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info, iloc_acc, iloc.info, groups_x, groups_y, repeat, - rlen_acc, rlen.info, shrdVal, shrdLoc)); + rlenValid, rlen_acc, rlen.info, shrdVal, shrdLoc)); break; } }); @@ -669,6 +675,16 @@ T ireduce_all(uint *idx, Param in) { sycl::host_accessor h_ptr_raw{*tmp.get()}; sycl::host_accessor h_lptr_raw{*tlptr.get()}; + if (!is_linear) { + // Converting n-d index into a linear index + // in is of size [ dims0, dims1, dims2, dims3] + // tidx is of size [blocks_x, dims1, dims2, dims3] + // i / blocks_x gives you the batch number "N" + // "N * dims0 + i" gives the linear index + for (int i = 0; i < tmp_elements; i++) { + h_lptr_raw[i] += (i / groups_x) * in.info.dims[0]; + } + } MinMaxOp Op(h_ptr_raw[0], h_lptr_raw[0]); diff --git a/src/backend/oneapi/kernel/lookup.hpp b/src/backend/oneapi/kernel/lookup.hpp index f3e2fcdcde..6bceca3e97 100644 --- a/src/backend/oneapi/kernel/lookup.hpp +++ b/src/backend/oneapi/kernel/lookup.hpp @@ -64,7 +64,7 @@ class lookupNDCreateKernel { int gx = g.get_local_range(0) * (g.get_group_id(0) - gz * nBBS0_) + lx; int gy = g.get_local_range(1) * (g.get_group_id(1) - gw * nBBS1_) + ly; - const idx_t *idxPtr = indices_.get_pointer(); + const idx_t *idxPtr = indices_.get_pointer() + idxInfo_.offset; int i = iInfo_.strides[0] * (DIM_ == 0 ? trimIndex((int)idxPtr[gx], iInfo_.dims[0]) : gx); diff --git a/src/backend/oneapi/kernel/lu_split.hpp b/src/backend/oneapi/kernel/lu_split.hpp index fb69001ebc..6d52fb3835 100644 --- a/src/backend/oneapi/kernel/lu_split.hpp +++ b/src/backend/oneapi/kernel/lu_split.hpp @@ -51,9 +51,9 @@ class luSplitKernel { const int incy = groupsPerMatY_ * g.get_local_range(1); const int incx = groupsPerMatX_ * g.get_local_range(0); - T *d_l = lower_.get_pointer(); - T *d_u = upper_.get_pointer(); - T *d_i = in_.get_pointer(); + T *d_l = lower_.get_pointer(); + T *d_u = upper_.get_pointer(); + const T *d_i = in_.get_pointer(); if (oz < iInfo_.dims[2] && ow < iInfo_.dims[3]) { d_i = d_i + oz * iInfo_.strides[2] + ow * iInfo_.strides[3]; @@ -61,9 +61,9 @@ class luSplitKernel { d_u = d_u + oz * uInfo_.strides[2] + ow * uInfo_.strides[3]; for (int oy = yy; oy < iInfo_.dims[1]; oy += incy) { - T *Yd_i = d_i + oy * iInfo_.strides[1]; - T *Yd_l = d_l + oy * lInfo_.strides[1]; - T *Yd_u = d_u + oy * uInfo_.strides[1]; + const T *Yd_i = d_i + oy * iInfo_.strides[1]; + T *Yd_l = d_l + oy * lInfo_.strides[1]; + T *Yd_u = d_u + oy * uInfo_.strides[1]; for (int ox = xx; ox < iInfo_.dims[0]; ox += incx) { if (ox > oy) { if (same_dims || oy < lInfo_.dims[1]) diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp index ef98cb0954..4c8533b1ec 100644 --- a/src/backend/oneapi/kernel/mean.hpp +++ b/src/backend/oneapi/kernel/mean.hpp @@ -1,3 +1,4 @@ + /******************************************************* * Copyright (c) 2022, ArrayFire * All rights reserved. @@ -29,17 +30,6 @@ namespace arrayfire { namespace oneapi { -/* -TODO: port half -__device__ auto operator*(float lhs, __half rhs) -> __half { - return __float2half(lhs * __half2float(rhs)); -} - -__device__ auto operator/(__half lhs, float rhs) -> __half { - return __float2half(__half2float(lhs) / rhs); -} -*/ - namespace kernel { template @@ -101,7 +91,7 @@ class meanDimKernelSMEM { To *optr = out_.get_pointer(); uint ooffset = ids[3] * oInfo_.strides[3] + ids[2] * oInfo_.strides[2] + - ids[1] * oInfo_.strides[1] + ids[0]; + ids[1] * oInfo_.strides[1] + ids[0] + oInfo_.offset; // There is only one element per block for out // There are blockDim.y elements per block for in // Hence increment ids[dim] just after offseting out and before @@ -112,11 +102,11 @@ class meanDimKernelSMEM { ids[dim] = ids[dim] * g.get_local_range(1) + lidy; uint ioffset = ids[3] * iInfo_.strides[3] + ids[2] * iInfo_.strides[2] + - ids[1] * iInfo_.strides[1] + ids[0]; + ids[1] * iInfo_.strides[1] + ids[0] + iInfo_.offset; iptr += ioffset; - const Tw *iwptr; - Tw *owptr; + const Tw *iwptr = nullptr; + Tw *owptr = nullptr; if (output_weight_) owptr = owt_.get_pointer() + ooffset; if (input_weight_) iwptr = iwt_.get_pointer() + ioffset; @@ -135,7 +125,7 @@ class meanDimKernelSMEM { if (is_valid && id_dim_in < iInfo_.dims[dim]) { val = transform(*iptr); - if (iwptr != NULL) { + if (iwptr) { weight = *iwptr; } else { weight = (Tw)1; @@ -143,14 +133,14 @@ class meanDimKernelSMEM { } const uint id_dim_in_start = - id_dim_in + offset_dim_ * g.get_local_range(0); + id_dim_in + offset_dim_ * g.get_local_range(1); for (int id = id_dim_in_start; is_valid && (id < iInfo_.dims[dim]); - id += offset_dim_ * g.get_local_range(0)) { - iptr = iptr + offset_dim_ * g.get_local_range(0) * istride_dim; + id += offset_dim_ * g.get_local_range(1)) { + iptr = iptr + offset_dim_ * g.get_local_range(1) * istride_dim; if (input_weight_) { iwptr = - iwptr + offset_dim_ * g.get_local_range(0) * istride_dim; + iwptr + offset_dim_ * g.get_local_range(1) * istride_dim; stable_mean(&val, &weight, transform(*iptr), compute_t(*iwptr)); } else { @@ -358,19 +348,21 @@ class meanFirstKernelSMEM { To *optr = out_.get_pointer(); iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] + - yid * iInfo_.strides[1]; + yid * iInfo_.strides[1] + iInfo_.offset; optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] + - yid * oInfo_.strides[1]; + yid * oInfo_.strides[1] + oInfo_.offset; - const Tw *iwptr; - Tw *owptr; + const Tw *iwptr = nullptr; + Tw *owptr = nullptr; if (input_weight_) iwptr = iwt_.get_pointer() + wid * iwInfo_.strides[3] + - zid * iwInfo_.strides[2] + yid * iwInfo_.strides[1]; + zid * iwInfo_.strides[2] + yid * iwInfo_.strides[1] + + iwInfo_.offset; if (output_weight_) - owptr = owt_.get_pointer() + wid * oInfo_.strides[3] + - zid * oInfo_.strides[2] + yid * oInfo_.strides[1]; + owptr = owt_.get_pointer() + wid * owInfo_.strides[3] + + zid * owInfo_.strides[2] + yid * owInfo_.strides[1] + + owInfo_.offset; bool cond = (yid < iInfo_.dims[1] && zid < iInfo_.dims[2] && wid < iInfo_.dims[3]); @@ -485,9 +477,9 @@ class meanFirstKernelSMEM { }; template -sycl::event mean_first_launcher(Param out, Param owt, Param in, - Param iwt, const uint groups_x, - const uint groups_y, const uint threads_x) { +void mean_first_launcher(Param out, Param owt, Param in, + Param iwt, const uint groups_x, + const uint groups_y, const uint threads_x) { sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x); sycl::range<2> global(groups_x * in.info.dims[2] * local[0], groups_y * in.info.dims[3] * local[1]); @@ -496,7 +488,7 @@ sycl::event mean_first_launcher(Param out, Param owt, Param in, auto empty = memAlloc(1); auto oempty = memAlloc(1); - return getQueue().submit([&](sycl::handler &h) { + getQueue().submit([&](sycl::handler &h) { write_accessor out_acc{*out.data, h}; read_accessor in_acc{*in.data, h}; @@ -521,6 +513,7 @@ sycl::event mean_first_launcher(Param out, Param owt, Param in, iwt.info, threads_x, groups_x, groups_y, repeat, s_val, s_idx, input_weight, output_weight)); }); + ONEAPI_DEBUG_FINISH(getQueue()); } template @@ -616,16 +609,14 @@ T mean_all_weighted(Param in, Param iwt) { blocks_y, threads_x); compute_t val; + auto tmpOut_get = tmpOut.get(); + auto tmpWt_get = tmpWt.get(); getQueue() .submit([&](sycl::handler &h) { auto acc_in = - tmpOut.get() - ->template get_access(h); + tmpOut_get->get_host_access(h, sycl::read_only); auto acc_wt = - tmpWt.get() - ->template get_access(h); + tmpWt_get->get_host_access(h, sycl::read_only); h.host_task([acc_in, acc_wt, tmp_elements, &val] { val = static_cast>(acc_in[0]); @@ -644,13 +635,10 @@ T mean_all_weighted(Param in, Param iwt) { compute_t val; getQueue() .submit([&](sycl::handler &h) { - auto acc_in = - in.data->template get_access( - h, sycl::range{in_elements}); - auto acc_wt = - iwt.data->template get_access( - h, sycl::range{in_elements}); + auto acc_in = in.data->get_host_access( + h, sycl::range{in_elements}, sycl::read_only); + auto acc_wt = iwt.data->get_host_access( + h, sycl::range{in_elements}, sycl::read_only); h.host_task([acc_in, acc_wt, in_elements, &val]() { val = acc_in[0]; @@ -707,16 +695,14 @@ To mean_all(Param in) { uintl tmp_elements = tmpOut.elements(); compute_t val; + auto tmpOut_get = tmpOut.get(); + auto tmpCt_get = tmpCt.get(); getQueue() .submit([&](sycl::handler &h) { auto out = - tmpOut.get() - ->template get_access(h); + tmpOut_get->get_host_access(h, sycl::read_only); auto ct = - tmpCt.get() - ->template get_access(h); + tmpCt_get->get_host_access(h, sycl::read_only); h.host_task([out, ct, tmp_elements, &val] { val = static_cast>(out[0]); @@ -735,8 +721,7 @@ To mean_all(Param in) { getQueue() .submit([&](sycl::handler &h) { auto acc_in = - in.data->template get_access(h); + in.data->get_host_access(h, sycl::read_only); h.host_task([acc_in, in_elements, &val]() { common::Transform, af_add_t> transform; compute_t count = static_cast>(1); diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp index c6b8dbb04c..64bd26ba1e 100644 --- a/src/backend/oneapi/kernel/memcopy.hpp +++ b/src/backend/oneapi/kernel/memcopy.hpp @@ -28,6 +28,27 @@ namespace arrayfire { namespace oneapi { namespace kernel { +template +using factortypes = typename std::conditional || + std::is_same_v, + double, float>::type; + +template> +inline T scale(T value, FACTORTYPE factor) { + return (T)(FACTORTYPE(value) * factor); +} + +template<> +inline cfloat scale(cfloat value, float factor) { + return cfloat{static_cast(value.real() * factor), + static_cast(value.imag() * factor)}; +} + +template<> +inline cdouble scale(cdouble value, double factor) { + return cdouble{value.real() * factor, value.imag() * factor}; +} + typedef struct { dim_t dim[4]; } dims_t; @@ -60,7 +81,7 @@ class memCopy { const int id0 = group_id_0 * gg.get_local_range(0) + lid0; const int id1 = group_id_1 * gg.get_local_range(1) + lid1; - T *iptr = in_.get_pointer(); + const T *iptr = in_.get_pointer(); // FIXME: Do more work per work group T *optr = out_.get_pointer(); @@ -126,22 +147,6 @@ void memcopy(sycl::buffer *out, const dim_t *ostrides, ONEAPI_DEBUG_FINISH(getQueue()); } -template -inline T scale(T value, double factor) { - return (T)(double(value) * factor); -} - -template<> -inline cfloat scale(cfloat value, double factor) { - return cfloat{static_cast(value.real() * factor), - static_cast(value.imag() * factor)}; -} - -template<> -inline cdouble scale(cdouble value, double factor) { - return cdouble{value.real() * factor, value.imag() * factor}; -} - template inline outType convertType(inType value) { return static_cast(value); @@ -159,6 +164,19 @@ convertType>(char value) { return compute_t(value); } +template<> +signed char inline convertType, signed char>( + compute_t value) { + return (signed char)((short)value); +} + +template<> +inline compute_t +convertType>( + signed char value) { + return compute_t(value); +} + template<> unsigned char inline convertType, unsigned char>( @@ -192,6 +210,7 @@ OTHER_SPECIALIZATIONS(intl) OTHER_SPECIALIZATIONS(uintl) OTHER_SPECIALIZATIONS(short) OTHER_SPECIALIZATIONS(ushort) +OTHER_SPECIALIZATIONS(schar) OTHER_SPECIALIZATIONS(uchar) OTHER_SPECIALIZATIONS(char) OTHER_SPECIALIZATIONS(arrayfire::common::half) @@ -201,7 +220,7 @@ class reshapeCopy { public: reshapeCopy(write_accessor dst, KParam oInfo, read_accessor src, KParam iInfo, outType default_value, - float factor, dims_t trgt, int blk_x, int blk_y) + factortypes factor, dims_t trgt, int blk_x, int blk_y) : dst_(dst) , src_(src) , oInfo_(oInfo) @@ -266,7 +285,7 @@ class reshapeCopy { read_accessor src_; KParam oInfo_, iInfo_; outType default_value_; - float factor_; + factortypes factor_; dims_t trgt_; int blk_x_, blk_y_; }; @@ -309,15 +328,15 @@ void copy(Param dst, const Param src, const int ndims, *const_cast *>(src.data), h}; if (same_dims) { - h.parallel_for(ndrange, reshapeCopy( - dst_acc, dst.info, src_acc, src.info, - default_value, (float)factor, trgt_dims, - blk_x, blk_y)); + h.parallel_for(ndrange, + reshapeCopy( + dst_acc, dst.info, src_acc, src.info, + default_value, factor, trgt_dims, blk_x, blk_y)); } else { - h.parallel_for(ndrange, reshapeCopy( - dst_acc, dst.info, src_acc, src.info, - default_value, (float)factor, trgt_dims, - blk_x, blk_y)); + h.parallel_for(ndrange, + reshapeCopy( + dst_acc, dst.info, src_acc, src.info, + default_value, factor, trgt_dims, blk_x, blk_y)); } }); ONEAPI_DEBUG_FINISH(getQueue()); diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp index dcd20dec13..a96d7d07fe 100644 --- a/src/backend/oneapi/kernel/random_engine_write.hpp +++ b/src/backend/oneapi/kernel/random_engine_write.hpp @@ -8,71 +8,12 @@ ********************************************************/ #pragma once #include +#include namespace arrayfire { namespace oneapi { namespace kernel { -// TODO: !!!! half functions still need to be ported !!!! - -//// Conversion to half adapted from Random123 -//// #define HALF_FACTOR (1.0f) / (std::numeric_limits::max() + (1.0f)) -//// #define HALF_HALF_FACTOR ((0.5f) * HALF_FACTOR) -//// -//// NOTE: The following constants for half were calculated using the formulas -//// above. This is done so that we can avoid unnecessary computations because -/// the / __half datatype is not a constexprable type. This prevents the -/// compiler from / peforming these operations at compile time. -// #define HALF_FACTOR __ushort_as_half(0x100u) -// #define HALF_HALF_FACTOR __ushort_as_half(0x80) -// -//// Conversion to half adapted from Random123 -////#define SIGNED_HALF_FACTOR \ -// //((1.0f) / (std::numeric_limits::max() + (1.0f))) -////#define SIGNED_HALF_HALF_FACTOR ((0.5f) * SIGNED_HALF_FACTOR) -//// -//// NOTE: The following constants for half were calculated using the formulas -//// above. This is done so that we can avoid unnecessary computations because -/// the / __half datatype is not a constexprable type. This prevents the -/// compiler from / peforming these operations at compile time -// #define SIGNED_HALF_FACTOR __ushort_as_half(0x200u) -// #define SIGNED_HALF_HALF_FACTOR __ushort_as_half(0x100u) -// -///// This is the largest integer representable by fp16. We need to -///// make sure that the value converted from ushort is smaller than this -///// value to avoid generating infinity -// constexpr ushort max_int_before_infinity = 65504; -// -//// Generates rationals in (0, 1] -//__device__ static __half oneMinusGetHalf01(uint num) { -// // convert to ushort before the min operation -// ushort v = min(max_int_before_infinity, ushort(num)); -// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 530 -// return (1.0f - __half2float(__hfma(__ushort2half_rn(v), HALF_FACTOR, -// HALF_HALF_FACTOR))); -// #else -// __half out = __ushort_as_half(0x3c00u) /*1.0h*/ - -// __hfma(__ushort2half_rn(v), HALF_FACTOR, HALF_HALF_FACTOR); -// if (__hisinf(out)) printf("val: %d ushort: %d\n", num, v); -// return out; -// #endif -//} -// -//// Generates rationals in (0, 1] -//__device__ static __half getHalf01(uint num) { -// // convert to ushort before the min operation -// ushort v = min(max_int_before_infinity, ushort(num)); -// return __hfma(__ushort2half_rn(v), HALF_FACTOR, HALF_HALF_FACTOR); -//} -// -//// Generates rationals in (-1, 1] -//__device__ static __half getHalfNegative11(uint num) { -// // convert to ushort before the min operation -// ushort v = min(max_int_before_infinity, ushort(num)); -// return __hfma(__ushort2half_rn(v), SIGNED_HALF_FACTOR, -// SIGNED_HALF_HALF_FACTOR); -//} -// // Generates rationals in (0, 1] static float getFloat01(uint num) { // Conversion to floats adapted from Random123 @@ -126,94 +67,43 @@ static double getDoubleNegative11(uint num1, uint num2) { return sycl::fma(static_cast(num), signed_factor, half_factor); } +/// This is the largest integer representable by fp16. We need to +/// make sure that the value converted from ushort is smaller than this +/// value to avoid generating infinity +#define MAX_INT_BEFORE_INFINITY (ushort)65504u + +// Generates rationals in (0, 1] +sycl::half getHalf01(uint num, uint index) { + sycl::half v = static_cast(min(MAX_INT_BEFORE_INFINITY, + static_cast(num >> (16U * (index & 1U)) & 0x0000ffff))); + + const sycl::half half_factor{1.526e-5}; // (1 / (USHRT_MAX + 1)) + const sycl::half half_half_factor{7.6e-6}; // (0.5 * half_factor) + return sycl::fma(v, half_factor, half_half_factor); +} + +sycl::half oneMinusGetHalf01(uint num, uint index) { + return static_cast(1.) - getHalf01(num, index); +} + +// Generates rationals in (-1, 1] +sycl::half getHalfNegative11(uint num, uint index) { + sycl::half v = static_cast(min(MAX_INT_BEFORE_INFINITY, + static_cast(num >> (16U * (index & 1U)) & 0x0000ffff))); + + const sycl::half signed_half_factor{3.05e-5}; // (1 / (SHRT_MAX + 1)) + const sycl::half signed_half_half_factor{1.526e-5}; // (0.5 * signed_half_factor) + return sycl::fma(v, signed_half_factor, signed_half_half_factor); +} + namespace { -// -// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 -// #define HALF_MATH_FUNC(OP, HALF_OP) \ -// template<> \ -// __device__ __half OP(__half val) { \ -// return ::HALF_OP(val); \ -// } -// #else -// #define HALF_MATH_FUNC(OP, HALF_OP) \ -// template<> \ -// __device__ __half OP(__half val) { \ -// float fval = __half2float(val); \ -// return __float2half(OP(fval)); \ -// } -// #endif -// -// #define MATH_FUNC(OP, DOUBLE_OP, FLOAT_OP, HALF_OP) \ -// template \ -// __device__ T OP(T val); \ -// template<> \ -// __device__ double OP(double val) { \ -// return ::DOUBLE_OP(val); \ -// } \ -// template<> \ -// __device__ float OP(float val) { \ -// return ::FLOAT_OP(val); \ -// } \ -// HALF_MATH_FUNC(OP, HALF_OP) -// -// MATH_FUNC(log, log, logf, hlog) -// MATH_FUNC(sqrt, sqrt, sqrtf, hsqrt) -// MATH_FUNC(sin, sin, sinf, hsin) -// MATH_FUNC(cos, cos, cosf, hcos) -// -// template -//__device__ void sincos(T val, T *sptr, T *cptr); -// -// template<> -//__device__ void sincos(double val, double *sptr, double *cptr) { -// ::sincos(val, sptr, cptr); -//} -// -// template<> -//__device__ void sincos(float val, float *sptr, float *cptr) { -// sincosf(val, sptr, cptr); -//} -// -// template<> -//__device__ void sincos(__half val, __half *sptr, __half *cptr) { -// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 -// *sptr = sin(val); -// *cptr = cos(val); -// #else -// float s, c; -// float fval = __half2float(val); -// sincos(fval, &s, &c); -// *sptr = __float2half(s); -// *cptr = __float2half(c); -// #endif -//} -// template void sincospi(T val, T *sptr, T *cptr) { *sptr = sycl::sinpi(val); *cptr = sycl::cospi(val); } - -// template<> -//__device__ void sincospi(__half val, __half *sptr, __half *cptr) { -// // CUDA cannot make __half into a constexpr as of CUDA 11 so we are -// // converting this offline -// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530 -// const __half pi_val = __ushort_as_half(0x4248); // 0x4248 == 3.14062h -// val *= pi_val; -// *sptr = sin(val); -// *cptr = cos(val); -// #else -// float fval = __half2float(val); -// float s, c; -// sincospi(fval, &s, &c); -// *sptr = __float2half(s); -// *cptr = __float2half(c); -// #endif -//} -// } // namespace -// + template constexpr T neg_two() { return -2.0; @@ -273,13 +163,6 @@ static void boxMullerTransform(Td *const out1, Td *const out2, const Tc &r1, *out1 = static_cast(r * s); *out2 = static_cast(r * c); } -// template<> -//__device__ void boxMullerTransform( -// arrayfire::common::half *const out1, arrayfire::common::half *const out2, -// const __half &r1, const __half &r2) { float o1, o2; float fr1 = -// __half2float(r1); float fr2 = __half2float(r2); boxMullerTransform(&o1, -// &o2, fr1, fr2); *out1 = o1; *out2 = o2; -//} // Writes without boundary checking static void writeOut128Bytes(uchar *out, const uint &index, const uint groupSz, @@ -303,6 +186,12 @@ static void writeOut128Bytes(uchar *out, const uint &index, const uint groupSz, out[index + 15 * groupSz] = r4 >> 24; } +static void writeOut128Bytes(schar *out, const uint &index, const uint groupSz, + const uint &r1, const uint &r2, const uint &r3, + const uint &r4) { + writeOut128Bytes((uchar *)(out), index, groupSz, r1, r2, r3, r4); +} + static void writeOut128Bytes(char *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2, const uint &r3, const uint &r4) { @@ -407,14 +296,14 @@ static void writeOut128Bytes(cdouble *out, const uint &index, static void writeOut128Bytes(arrayfire::common::half *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2, const uint &r3, const uint &r4) { - // out[index] = oneMinusGetHalf01(r1); - // out[index + groupSz] = oneMinusGetHalf01(r1 >> 16); - // out[index + 2 * groupSz] = oneMinusGetHalf01(r2); - // out[index + 3 * groupSz] = oneMinusGetHalf01(r2 >> 16); - // out[index + 4 * groupSz] = oneMinusGetHalf01(r3); - // out[index + 5 * groupSz] = oneMinusGetHalf01(r3 >> 16); - // out[index + 6 * groupSz] = oneMinusGetHalf01(r4); - // out[index + 7 * groupSz] = oneMinusGetHalf01(r4 >> 16); + out[index] = oneMinusGetHalf01(r1, 0); + out[index + groupSz] = oneMinusGetHalf01(r1, 1); + out[index + 2 * groupSz] = oneMinusGetHalf01(r2, 0); + out[index + 3 * groupSz] = oneMinusGetHalf01(r2, 1); + out[index + 4 * groupSz] = oneMinusGetHalf01(r3, 0); + out[index + 5 * groupSz] = oneMinusGetHalf01(r3, 1); + out[index + 6 * groupSz] = oneMinusGetHalf01(r4, 0); + out[index + 7 * groupSz] = oneMinusGetHalf01(r4, 1); } // Normalized writes without boundary checking @@ -458,17 +347,14 @@ static void boxMullerWriteOut128Bytes(arrayfire::common::half *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2, const uint &r3, const uint &r4) { - // boxMullerTransform(&out[index], &out[index + groupSz], - // getHalfNegative11(r1), getHalf01(r1 >> 16)); - // boxMullerTransform(&out[index + 2 * groupSz], - // &out[index + 3 * groupSz], getHalfNegative11(r2), - // getHalf01(r2 >> 16)); - // boxMullerTransform(&out[index + 4 * groupSz], - // &out[index + 5 * groupSz], getHalfNegative11(r3), - // getHalf01(r3 >> 16)); - // boxMullerTransform(&out[index + 6 * groupSz], - // &out[index + 7 * groupSz], getHalfNegative11(r4), - // getHalf01(r4 >> 16)); + boxMullerTransform(&out[index], &out[index + groupSz], + getHalfNegative11(r1, 0), getHalf01(r1, 1)); + boxMullerTransform(&out[index + 2 * groupSz], &out[index + 3 * groupSz], + getHalfNegative11(r2, 0), getHalf01(r2, 1)); + boxMullerTransform(&out[index + 4 * groupSz], &out[index + 5 * groupSz], + getHalfNegative11(r3, 0), getHalf01(r3, 1)); + boxMullerTransform(&out[index + 6 * groupSz], &out[index + 7 * groupSz], + getHalfNegative11(r4, 0), getHalf01(r4, 1)); } // Writes with boundary checking @@ -505,6 +391,14 @@ static void partialWriteOut128Bytes(uchar *out, const uint &index, } } +static void partialWriteOut128Bytes(schar *out, const uint &index, + const uint groupSz, const uint &r1, + const uint &r2, const uint &r3, + const uint &r4, const uint &elements) { + partialWriteOut128Bytes((uchar *)(out), index, groupSz, r1, r2, r3, r4, + elements); +} + static void partialWriteOut128Bytes(char *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2, const uint &r3, @@ -713,28 +607,28 @@ static void partialWriteOut128Bytes(arrayfire::common::half *out, const uint &r1, const uint &r2, const uint &r3, const uint &r4, const uint &elements) { - // if (index < elements) { out[index] = oneMinusGetHalf01(r1); } - // if (index + groupSz < elements) { - // out[index + groupSz] = oneMinusGetHalf01(r1 >> 16); - // } - // if (index + 2 * groupSz < elements) { - // out[index + 2 * groupSz] = oneMinusGetHalf01(r2); - // } - // if (index + 3 * groupSz < elements) { - // out[index + 3 * groupSz] = oneMinusGetHalf01(r2 >> 16); - // } - // if (index + 4 * groupSz < elements) { - // out[index + 4 * groupSz] = oneMinusGetHalf01(r3); - // } - // if (index + 5 * groupSz < elements) { - // out[index + 5 * groupSz] = oneMinusGetHalf01(r3 >> 16); - // } - // if (index + 6 * groupSz < elements) { - // out[index + 6 * groupSz] = oneMinusGetHalf01(r4); - // } - // if (index + 7 * groupSz < elements) { - // out[index + 7 * groupSz] = oneMinusGetHalf01(r4 >> 16); - // } + if (index < elements) { out[index] = oneMinusGetHalf01(r1, 0); } + if (index + groupSz < elements) { + out[index + groupSz] = oneMinusGetHalf01(r1, 1); + } + if (index + 2 * groupSz < elements) { + out[index + 2 * groupSz] = oneMinusGetHalf01(r2, 0); + } + if (index + 3 * groupSz < elements) { + out[index + 3 * groupSz] = oneMinusGetHalf01(r2, 1); + } + if (index + 4 * groupSz < elements) { + out[index + 4 * groupSz] = oneMinusGetHalf01(r3, 0); + } + if (index + 5 * groupSz < elements) { + out[index + 5 * groupSz] = oneMinusGetHalf01(r3, 1); + } + if (index + 6 * groupSz < elements) { + out[index + 6 * groupSz] = oneMinusGetHalf01(r4, 0); + } + if (index + 7 * groupSz < elements) { + out[index + 7 * groupSz] = oneMinusGetHalf01(r4, 1); + } } // Normalized writes with boundary checking @@ -744,35 +638,22 @@ static void partialBoxMullerWriteOut128Bytes(arrayfire::common::half *out, const uint &r2, const uint &r3, const uint &r4, const uint &elements) { - // arrayfire::common::half n[8]; - // boxMullerTransform(n + 0, n + 1, getHalfNegative11(r1), - // getHalf01(r1 >> 16)); - // boxMullerTransform(n + 2, n + 3, getHalfNegative11(r2), - // getHalf01(r2 >> 16)); - // boxMullerTransform(n + 4, n + 5, getHalfNegative11(r3), - // getHalf01(r3 >> 16)); - // boxMullerTransform(n + 6, n + 7, getHalfNegative11(r4), - // getHalf01(r4 >> 16)); - // if (index < elements) { out[index] = n[0]; } - // if (index + groupSz < elements) { out[index + groupSz] = n[1]; } - // if (index + 2 * groupSz < elements) { - // out[index + 2 * groupSz] = n[2]; - // } - // if (index + 3 * groupSz < elements) { - // out[index + 3 * groupSz] = n[3]; - // } - // if (index + 4 * groupSz < elements) { - // out[index + 4 * groupSz] = n[4]; - // } - // if (index + 5 * groupSz < elements) { - // out[index + 5 * groupSz] = n[5]; - // } - // if (index + 6 * groupSz < elements) { - // out[index + 6 * groupSz] = n[6]; - // } - // if (index + 7 * groupSz < elements) { - // out[index + 7 * groupSz] = n[7]; - // } + sycl::half n1, n2; + boxMullerTransform(&n1, &n2, getHalfNegative11(r1, 0), getHalf01(r1, 1)); + if (index < elements) { out[index] = n1; } + if (index + groupSz < elements) { out[index + groupSz] = n2; } + + boxMullerTransform(&n1, &n2, getHalfNegative11(r2, 0), getHalf01(r2, 1)); + if (index + 2 * groupSz < elements) { out[index + 2 * groupSz] = n1; } + if (index + 3 * groupSz < elements) { out[index + 3 * groupSz] = n2; } + + boxMullerTransform(&n1, &n2, getHalfNegative11(r3, 0), getHalf01(r3, 1)); + if (index + 4 * groupSz < elements) { out[index + 4 * groupSz] = n1; } + if (index + 5 * groupSz < elements) { out[index + 5 * groupSz] = n2; } + + boxMullerTransform(&n1, &n2, getHalfNegative11(r4, 0), getHalf01(r4, 1)); + if (index + 6 * groupSz < elements) { out[index + 6 * groupSz] = n1; } + if (index + 7 * groupSz < elements) { out[index + 7 * groupSz] = n2; } } } // namespace kernel diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp index 4bc3d5254d..7a1e842425 100644 --- a/src/backend/oneapi/kernel/reduce_all.hpp +++ b/src/backend/oneapi/kernel/reduce_all.hpp @@ -249,13 +249,17 @@ void reduce_all_launcher_default(Param out, Param in, "Too many blocks requested (typeof(retirementCount) == unsigned)", AF_ERR_RUNTIME); } - Array tmp = createEmptyArray(tmp_elements); + Array tmp = createEmptyArray(tmp_elements); + auto tmp_get = tmp.get(); + Array retirementCount = createValueArray(1, 0); + auto ret_get = retirementCount.get(); + getQueue().submit([&](sycl::handler &h) { write_accessor out_acc{*out.data, h}; - auto retCount_acc = retirementCount.get()->get_access(h); - auto tmp_acc = tmp.get()->get_access(h); + auto retCount_acc = ret_get->get_access(h); + auto tmp_acc = tmp_get->get_access(h); read_accessor in_acc{*in.data, h}; auto shrdMem = sycl::local_accessor, 1>( diff --git a/src/backend/oneapi/kernel/reduce_by_key.hpp b/src/backend/oneapi/kernel/reduce_by_key.hpp new file mode 100644 index 0000000000..329fd33109 --- /dev/null +++ b/src/backend/oneapi/kernel/reduce_by_key.hpp @@ -0,0 +1,694 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using std::unique_ptr; + +namespace arrayfire { +namespace oneapi { +namespace kernel { + +// Reduces keys across block boundaries +template +class finalBoundaryReduceKernel { + public: + finalBoundaryReduceKernel(write_accessor reduced_block_sizes, + read_accessor iKeys, KParam iKInfo, + sycl::accessor oVals, KParam oVInfo, + const int n) + : reduced_block_sizes_(reduced_block_sizes) + , iKeys_(iKeys) + , iKInfo_(iKInfo) + , oVals_(oVals) + , oVInfo_(oVInfo) + , n_(n) {} + + void operator()(sycl::nd_item<1> it) const { + sycl::group g = it.get_group(); + const uint lid = it.get_local_id(0); + const uint gid = it.get_global_id(0); + const uint bid = g.get_group_id(0); + + common::Binary, op> binOp; + if (gid == ((bid + 1) * it.get_local_range(0)) - 1 && + bid < g.get_group_range(0) - 1) { + Tk k0 = iKeys_[gid + iKInfo_.offset]; + Tk k1 = iKeys_[gid + 1 + iKInfo_.offset]; + + if (k0 == k1) { + compute_t v0 = compute_t(oVals_[gid]); + compute_t v1 = compute_t(oVals_[gid + 1]); + oVals_[gid + 1] = binOp(v0, v1); + reduced_block_sizes_[bid] = it.get_local_range(0) - 1; + } else { + reduced_block_sizes_[bid] = it.get_local_range(0); + } + } + + // if last block, set block size to difference between n and block + // boundary + if (lid == 0 && bid == g.get_group_range(0) - 1) { + reduced_block_sizes_[bid] = n_ - (bid * it.get_local_range(0)); + } + } + + protected: + write_accessor reduced_block_sizes_; + read_accessor iKeys_; + KParam iKInfo_; + sycl::accessor oVals_; + KParam oVInfo_; + int n_; +}; + +template +class finalBoundaryReduceDimKernel { + public: + finalBoundaryReduceDimKernel(write_accessor reduced_block_sizes, + read_accessor iKeys, KParam iKInfo, + sycl::accessor oVals, KParam oVInfo, + const int n, const int nGroupsZ) + : reduced_block_sizes_(reduced_block_sizes) + , iKeys_(iKeys) + , iKInfo_(iKInfo) + , oVals_(oVals) + , oVInfo_(oVInfo) + , n_(n) + , nGroupsZ_(nGroupsZ) {} + + void operator()(sycl::nd_item<3> it) const { + sycl::group g = it.get_group(); + const uint lid = it.get_local_id(0); + const uint gid = it.get_global_id(0); + const uint bid = g.get_group_id(0); + + common::Binary, op> binOp; + if (gid == ((bid + 1) * it.get_local_range(0)) - 1 && + bid < g.get_group_range(0) - 1) { + Tk k0 = iKeys_[gid + iKInfo_.offset]; + Tk k1 = iKeys_[gid + 1 + iKInfo_.offset]; + + if (k0 == k1) { + compute_t v0 = compute_t(oVals_[gid]); + compute_t v1 = compute_t(oVals_[gid + 1]); + oVals_[gid + 1] = binOp(v0, v1); + reduced_block_sizes_[bid] = it.get_local_range(0) - 1; + } else { + reduced_block_sizes_[bid] = it.get_local_range(0); + } + } + + // if last block, set block size to difference between n and block + // boundary + if (lid == 0 && bid == g.get_group_range(0) - 1) { + reduced_block_sizes_[bid] = n_ - (bid * it.get_local_range(0)); + } + } + + protected: + write_accessor reduced_block_sizes_; + read_accessor iKeys_; + KParam iKInfo_; + sycl::accessor oVals_; + KParam oVInfo_; + int n_; + int nGroupsZ_; +}; + +template +using global_atomic_ref = + sycl::atomic_ref; + +// Tests if data needs further reduction, including across block boundaries +template +class testNeedsReductionKernel { + public: + testNeedsReductionKernel(sycl::accessor needs_another_reduction, + sycl::accessor needs_block_boundary_reduced, + read_accessor iKeys, KParam iKInfo, + const int n, const int DIMX, + sycl::local_accessor l_keys) + : needs_another_reduction_(needs_another_reduction) + , needs_block_boundary_reduced_(needs_block_boundary_reduced) + , iKeys_(iKeys) + , iKInfo_(iKInfo) + , n_(n) + , DIMX_(DIMX) + , l_keys_(l_keys) {} + + void operator()(sycl::nd_item<1> it) const { + sycl::group g = it.get_group(); + const uint lid = it.get_local_id(0); + const uint gid = it.get_global_id(0); + const uint bid = g.get_group_id(0); + + Tk k = scalar(0); + if (gid < n_) { k = iKeys_[gid + iKInfo_.offset]; } + + l_keys_[lid] = k; + it.barrier(); + + int update_key = + (lid < DIMX_ - 2) && (k == l_keys_[lid + 1]) && (gid < (n_ - 1)); + + if (update_key) { + global_atomic_ref(needs_another_reduction_[0]) |= update_key; + } + + it.barrier(); + + // last thread in each block checks if any inter-block keys need further + // reduction + if (gid == ((bid + 1) * DIMX_) - 1 && + bid < (g.get_group_range(0) - 1)) { + int k0 = iKeys_[gid + iKInfo_.offset]; + int k1 = iKeys_[gid + 1 + iKInfo_.offset]; + if (k0 == k1) { + global_atomic_ref(needs_block_boundary_reduced_[0]) |= 1; + } + } + } + + protected: + sycl::accessor needs_another_reduction_; + sycl::accessor needs_block_boundary_reduced_; + read_accessor iKeys_; + KParam iKInfo_; + int n_; + int DIMX_; + sycl::local_accessor l_keys_; +}; + +// Compacts "incomplete" block-sized chunks of data in global memory +template +class compactKernel { + public: + compactKernel(read_accessor reduced_block_sizes, + write_accessor oKeys, KParam oKInfo, + write_accessor oVals, KParam oVInfo, + read_accessor iKeys, KParam iKInfo, + read_accessor iVals, KParam iVInfo, int nGroupsZ) + : reduced_block_sizes_(reduced_block_sizes) + , oKeys_(oKeys) + , oKInfo_(oKInfo) + , oVals_(oVals) + , oVInfo_(oVInfo) + , iKeys_(iKeys) + , iKInfo_(iKInfo) + , iVals_(iVals) + , iVInfo_(iVInfo) + , nGroupsZ_(nGroupsZ) {} + + void operator()(sycl::nd_item<3> it) const { + sycl::group g = it.get_group(); + const uint lid = it.get_local_id(0); + const uint bid = g.get_group_id(0); + const uint gid = it.get_global_id(0); + + const int bidy = g.get_group_id(1); + const int bidz = g.get_group_id(2) % nGroupsZ_; + const int bidw = g.get_group_id(2) / nGroupsZ_; + + const int bOffset = bidw * oVInfo_.strides[3] + + bidz * oVInfo_.strides[2] + + bidy * oVInfo_.strides[1]; + + // reduced_block_sizes should have inclusive sum of block sizes + int nwrite = + (bid == 0) + ? reduced_block_sizes_[0] + : (reduced_block_sizes_[bid] - reduced_block_sizes_[bid - 1]); + int writeloc = (bid == 0) ? 0 : reduced_block_sizes_[bid - 1]; + + Tk k = iKeys_[gid + iKInfo_.offset]; + To v = iVals_[bOffset + gid + iVInfo_.offset]; + + if (lid < nwrite) { + oKeys_[writeloc + lid] = k; + oVals_[bOffset + writeloc + lid] = v; + } + } + + protected: + read_accessor reduced_block_sizes_; + write_accessor oKeys_; + KParam oKInfo_; + write_accessor oVals_; + KParam oVInfo_; + read_accessor iKeys_; + KParam iKInfo_; + read_accessor iVals_; + KParam iVInfo_; + int nGroupsZ_; +}; + +// Compacts "incomplete" block-sized chunks of data in global memory +template +class compactDimKernel { + public: + compactDimKernel(read_accessor reduced_block_sizes, + write_accessor oKeys, KParam oKInfo, + write_accessor oVals, KParam oVInfo, + read_accessor iKeys, KParam iKInfo, + read_accessor iVals, KParam iVInfo, int nGroupsZ, + int DIM) + : reduced_block_sizes_(reduced_block_sizes) + , oKeys_(oKeys) + , oKInfo_(oKInfo) + , oVals_(oVals) + , oVInfo_(oVInfo) + , iKeys_(iKeys) + , iKInfo_(iKInfo) + , iVals_(iVals) + , iVInfo_(iVInfo) + , nGroupsZ_(nGroupsZ) + , DIM_(DIM) {} + + void operator()(sycl::nd_item<3> it) const { + sycl::group g = it.get_group(); + + const uint lid = it.get_local_id(0); + const uint gidx = it.get_global_id(0); + const uint bid = g.get_group_id(0); + + const int bidy = g.get_group_id(1); + const int bidz = g.get_group_id(2) % nGroupsZ_; + const int bidw = g.get_group_id(2) / nGroupsZ_; + + int dims_ordering[4]; + dims_ordering[0] = DIM_; + int d = 1; + for (int i = 0; i < 4; ++i) { + if (i != DIM_) dims_ordering[d++] = i; + } + + Tk k; + To v; + + // reduced_block_sizes should have inclusive sum of block sizes + int nwrite = + (bid == 0) + ? reduced_block_sizes_[0] + : (reduced_block_sizes_[bid] - reduced_block_sizes_[bid - 1]); + int writeloc = (bid == 0) ? 0 : reduced_block_sizes_[bid - 1]; + + const int tid = bidw * iVInfo_.strides[dims_ordering[3]] + + bidz * iVInfo_.strides[dims_ordering[2]] + + bidy * iVInfo_.strides[dims_ordering[1]] + + gidx * iVInfo_.strides[DIM_]; + k = iKeys_[gidx + iKInfo_.offset]; + v = iVals_[tid + iVInfo_.offset]; + + if (lid < nwrite) { + oKeys_[writeloc + lid] = k; + const int bOffset = bidw * oVInfo_.strides[dims_ordering[3]] + + bidz * oVInfo_.strides[dims_ordering[2]] + + bidy * oVInfo_.strides[dims_ordering[1]]; + oVals_[bOffset + (writeloc + lid) * oVInfo_.strides[DIM_]] = v; + } + } + + protected: + read_accessor reduced_block_sizes_; + write_accessor oKeys_; + KParam oKInfo_; + write_accessor oVals_; + KParam oVInfo_; + read_accessor iKeys_; + KParam iKInfo_; + read_accessor iVals_; + KParam iVInfo_; + int nGroupsZ_; + int DIM_; +}; + +// Reduces each block by key +template +class reduceBlocksByKeyKernel { + public: + reduceBlocksByKeyKernel(sycl::accessor reduced_block_sizes, + write_accessor oKeys, KParam oKInfo, + write_accessor oVals, KParam oVInfo, + read_accessor iKeys, KParam iKInfo, + read_accessor iVals, KParam iVInfo, + int change_nan, To nanval, int n, int nGroupsZ, + int DIMX, sycl::local_accessor l_keys, + sycl::local_accessor> l_vals, + sycl::local_accessor l_reduced_keys, + sycl::local_accessor> l_reduced_vals, + sycl::local_accessor l_unique_ids, + sycl::local_accessor l_wg_temp, + sycl::local_accessor l_unique_flags, + sycl::local_accessor l_reduced_block_size) + : reduced_block_sizes_(reduced_block_sizes) + , oKeys_(oKeys) + , oKInfo_(oKInfo) + , oVals_(oVals) + , oVInfo_(oVInfo) + , iKeys_(iKeys) + , iKInfo_(iKInfo) + , iVals_(iVals) + , iVInfo_(iVInfo) + , change_nan_(change_nan) + , nanval_(nanval) + , n_(n) + , nGroupsZ_(nGroupsZ) + , DIMX_(DIMX) + , l_keys_(l_keys) + , l_vals_(l_vals) + , l_reduced_keys_(l_reduced_keys) + , l_reduced_vals_(l_reduced_vals) + , l_unique_ids_(l_unique_ids) + , l_wg_temp_(l_wg_temp) + , l_unique_flags_(l_unique_flags) + , l_reduced_block_size_(l_reduced_block_size) {} + + void operator()(sycl::nd_item<3> it) const { + sycl::group g = it.get_group(); + const uint lid = it.get_local_id(0); + const uint gid = it.get_global_id(0); + + const int bidy = g.get_group_id(1); + const int bidz = g.get_group_id(2) % nGroupsZ_; + const int bidw = g.get_group_id(2) / nGroupsZ_; + + const compute_t init_val = + common::Binary, op>::init(); + common::Binary, op> binOp; + common::Transform, op> transform; + + if (lid == 0) { l_reduced_block_size_[0] = 0; } + + // load keys and values to threads + Tk k = scalar(0); + compute_t v = init_val; + if (gid < n_) { + k = iKeys_[gid + iKInfo_.offset]; + const int bOffset = bidw * iVInfo_.strides[3] + + bidz * iVInfo_.strides[2] + + bidy * iVInfo_.strides[1]; + v = transform(iVals_[bOffset + gid + iVInfo_.offset]); + if (change_nan_) v = IS_NAN(v) ? nanval_ : v; + } + + l_keys_[lid] = k; + l_vals_[lid] = v; + + l_reduced_keys_[lid] = k; + it.barrier(); + + // mark threads containing unique keys + int eq_check = (lid > 0) ? (k != l_reduced_keys_[lid - 1]) : 0; + int unique_flag = (eq_check || (lid == 0)) && (gid < n_); + + l_unique_flags_[lid] = unique_flag; + int unique_id = + work_group_scan_inclusive_add(it, l_wg_temp_, l_unique_flags_); + + l_unique_ids_[lid] = unique_id; + + if (lid == DIMX_ - 1) l_reduced_block_size_[0] = unique_id; + + for (int off = 1; off < DIMX_; off *= 2) { + it.barrier(); + int test_unique_id = + (lid + off < DIMX_) ? l_unique_ids_[lid + off] : ~unique_id; + eq_check = (unique_id == test_unique_id); + int update_key = + eq_check && (lid < (DIMX_ - off)) && + ((gid + off) < + n_); // checks if this thread should perform a reduction + compute_t uval = (update_key) ? l_vals_[lid + off] : init_val; + it.barrier(); + l_vals_[lid] = + binOp(l_vals_[lid], uval); // update if thread requires it + } + + if (unique_flag) { + l_reduced_keys_[unique_id - 1] = k; + l_reduced_vals_[unique_id - 1] = l_vals_[lid]; + } + it.barrier(); + + const int bid = g.get_group_id(0); + if (lid < l_reduced_block_size_[0]) { + const int bOffset = bidw * oVInfo_.strides[3] + + bidz * oVInfo_.strides[2] + + bidy * oVInfo_.strides[1]; + oKeys_[bid * DIMX_ + lid] = l_reduced_keys_[lid]; + oVals_[bOffset + ((bid * DIMX_) + lid)] = l_reduced_vals_[lid]; + } + + reduced_block_sizes_[bid] = l_reduced_block_size_[0]; + } + + int work_group_scan_inclusive_add(sycl::nd_item<3> it, + sycl::local_accessor wg_temp, + sycl::local_accessor arr) const { + const uint lid = it.get_local_id(0); + int *active_buf; + + int val = arr[lid]; + active_buf = arr.get_pointer(); + + bool swap_buffer = false; + for (int off = 1; off <= DIMX_; off *= 2) { + it.barrier(); + if (lid >= off) { val = val + active_buf[lid - off]; } + swap_buffer = !swap_buffer; + active_buf = + swap_buffer ? wg_temp.get_pointer() : arr.get_pointer(); + active_buf[lid] = val; + } + + int res = active_buf[lid]; + return res; + } + + protected: + sycl::accessor reduced_block_sizes_; + write_accessor oKeys_; + KParam oKInfo_; + write_accessor oVals_; + KParam oVInfo_; + read_accessor iKeys_; + KParam iKInfo_; + read_accessor iVals_; + KParam iVInfo_; + int change_nan_; + To nanval_; + int n_; + int nGroupsZ_; + int DIMX_; + sycl::local_accessor l_keys_; + sycl::local_accessor> l_vals_; + sycl::local_accessor l_reduced_keys_; + sycl::local_accessor> l_reduced_vals_; + sycl::local_accessor l_unique_ids_; + sycl::local_accessor l_wg_temp_; + sycl::local_accessor l_unique_flags_; + sycl::local_accessor l_reduced_block_size_; +}; + +// Reduces each block by key +template +class reduceBlocksByKeyDimKernel { + public: + reduceBlocksByKeyDimKernel( + sycl::accessor reduced_block_sizes, write_accessor oKeys, + KParam oKInfo, write_accessor oVals, KParam oVInfo, + read_accessor iKeys, KParam iKInfo, read_accessor iVals, + KParam iVInfo, int change_nan, To nanval, int n, int nGroupsZ, int DIMX, + int DIM, sycl::local_accessor l_keys, + sycl::local_accessor> l_vals, + sycl::local_accessor l_reduced_keys, + sycl::local_accessor> l_reduced_vals, + sycl::local_accessor l_unique_ids, + sycl::local_accessor l_wg_temp, + sycl::local_accessor l_unique_flags, + sycl::local_accessor l_reduced_block_size) + : reduced_block_sizes_(reduced_block_sizes) + , oKeys_(oKeys) + , oKInfo_(oKInfo) + , oVals_(oVals) + , oVInfo_(oVInfo) + , iKeys_(iKeys) + , iKInfo_(iKInfo) + , iVals_(iVals) + , iVInfo_(iVInfo) + , change_nan_(change_nan) + , nanval_(nanval) + , n_(n) + , nGroupsZ_(nGroupsZ) + , DIMX_(DIMX) + , DIM_(DIM) + , l_keys_(l_keys) + , l_vals_(l_vals) + , l_reduced_keys_(l_reduced_keys) + , l_reduced_vals_(l_reduced_vals) + , l_unique_ids_(l_unique_ids) + , l_wg_temp_(l_wg_temp) + , l_unique_flags_(l_unique_flags) + , l_reduced_block_size_(l_reduced_block_size) {} + + void operator()(sycl::nd_item<3> it) const { + sycl::group g = it.get_group(); + const uint lid = it.get_local_id(0); + const uint gid = it.get_global_id(0); + + const int bidy = g.get_group_id(1); + const int bidz = g.get_group_id(2) % nGroupsZ_; + const int bidw = g.get_group_id(2) / nGroupsZ_; + + const compute_t init_val = + common::Binary, op>::init(); + common::Binary, op> binOp; + common::Transform, op> transform; + + if (lid == 0) { l_reduced_block_size_[0] = 0; } + + int dims_ordering[4]; + dims_ordering[0] = DIM_; + int d = 1; + for (int i = 0; i < 4; ++i) { + if (i != DIM_) dims_ordering[d++] = i; + } + it.barrier(); + + // load keys and values to threads + Tk k = scalar(0); + compute_t v = init_val; + if (gid < n_) { + k = iKeys_[gid + iKInfo_.offset]; + const int bOffset = bidw * iVInfo_.strides[dims_ordering[3]] + + bidz * iVInfo_.strides[dims_ordering[2]] + + bidy * iVInfo_.strides[dims_ordering[1]]; + v = transform( + iVals_[bOffset + gid * iVInfo_.strides[DIM_] + iVInfo_.offset]); + if (change_nan_) v = IS_NAN(v) ? nanval_ : v; + } + + l_keys_[lid] = k; + l_vals_[lid] = v; + + l_reduced_keys_[lid] = k; + it.barrier(); + + // mark threads containing unique keys + int eq_check = (lid > 0) ? (k != l_reduced_keys_[lid - 1]) : 0; + int unique_flag = (eq_check || (lid == 0)) && (gid < n_); + + l_unique_flags_[lid] = unique_flag; + int unique_id = + work_group_scan_inclusive_add(it, l_wg_temp_, l_unique_flags_); + + l_unique_ids_[lid] = unique_id; + + if (lid == DIMX_ - 1) l_reduced_block_size_[0] = unique_id; + + for (int off = 1; off < DIMX_; off *= 2) { + it.barrier(); + int test_unique_id = + (lid + off < DIMX_) ? l_unique_ids_[lid + off] : ~unique_id; + eq_check = (unique_id == test_unique_id); + int update_key = + eq_check && (lid < (DIMX_ - off)) && + ((gid + off) < + n_); // checks if this thread should perform a reduction + compute_t uval = (update_key) ? l_vals_[lid + off] : init_val; + it.barrier(); + l_vals_[lid] = + binOp(l_vals_[lid], uval); // update if thread requires it + } + + if (unique_flag) { + l_reduced_keys_[unique_id - 1] = k; + l_reduced_vals_[unique_id - 1] = l_vals_[lid]; + } + it.barrier(); + + const int bid = g.get_group_id(0); + if (lid < l_reduced_block_size_[0]) { + const int bOffset = bidw * oVInfo_.strides[dims_ordering[3]] + + bidz * oVInfo_.strides[dims_ordering[2]] + + bidy * oVInfo_.strides[dims_ordering[1]]; + oKeys_[gid] = l_reduced_keys_[lid]; + oVals_[bOffset + (gid)*oVInfo_.strides[DIM_]] = + l_reduced_vals_[lid]; + } + + reduced_block_sizes_[bid] = l_reduced_block_size_[0]; + } + + int work_group_scan_inclusive_add(sycl::nd_item<3> it, + sycl::local_accessor wg_temp, + sycl::local_accessor arr) const { + const uint lid = it.get_local_id(0); + int *active_buf; + + int val = arr[lid]; + active_buf = arr.get_pointer(); + + bool swap_buffer = false; + for (int off = 1; off <= DIMX_; off *= 2) { + it.barrier(); + if (lid >= off) { val = val + active_buf[lid - off]; } + swap_buffer = !swap_buffer; + active_buf = + swap_buffer ? wg_temp.get_pointer() : arr.get_pointer(); + active_buf[lid] = val; + } + + int res = active_buf[lid]; + return res; + } + + protected: + sycl::accessor reduced_block_sizes_; + write_accessor oKeys_; + KParam oKInfo_; + write_accessor oVals_; + KParam oVInfo_; + read_accessor iKeys_; + KParam iKInfo_; + read_accessor iVals_; + KParam iVInfo_; + int change_nan_; + To nanval_; + int n_; + int nGroupsZ_; + int DIMX_; + int DIM_; + sycl::local_accessor l_keys_; + sycl::local_accessor> l_vals_; + sycl::local_accessor l_reduced_keys_; + sycl::local_accessor> l_reduced_vals_; + sycl::local_accessor l_unique_ids_; + sycl::local_accessor l_wg_temp_; + sycl::local_accessor l_unique_flags_; + sycl::local_accessor l_reduced_block_size_; +}; + +} // namespace kernel +} // namespace oneapi +} // namespace arrayfire diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp index 6b51801fa7..0cc7055f14 100644 --- a/src/backend/oneapi/kernel/reduce_dim.hpp +++ b/src/backend/oneapi/kernel/reduce_dim.hpp @@ -65,16 +65,17 @@ class reduceDimKernelSMEM { uint ids[4] = {xid, yid, zid, wid}; using sycl::global_ptr; - global_ptr> optr = - out_.get_pointer() + ids[3] * oInfo_.strides[3] + - ids[2] * oInfo_.strides[2] + ids[1] * oInfo_.strides[1] + ids[0]; + data_t *optr = out_.get_pointer() + ids[3] * oInfo_.strides[3] + + ids[2] * oInfo_.strides[2] + + ids[1] * oInfo_.strides[1] + ids[0]; const uint groupIdx_dim = ids[dim]; ids[dim] = ids[dim] * g.get_local_range(1) + lidy; - global_ptr> iptr = + const data_t *iptr = in_.get_pointer() + ids[3] * iInfo_.strides[3] + - ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] + ids[0]; + ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] + ids[0] + + iInfo_.offset; const uint id_dim_in = ids[dim]; const uint istride_dim = iInfo_.strides[dim]; diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp index f105d63671..152120648b 100644 --- a/src/backend/oneapi/kernel/reduce_first.hpp +++ b/src/backend/oneapi/kernel/reduce_first.hpp @@ -68,7 +68,7 @@ class reduceFirstKernelSMEM { common::Binary, op> reduce; common::Transform, op> transform; - Ti *const iptr = in_.get_pointer() + wid * iInfo_.strides[3] + + const Ti *iptr = in_.get_pointer() + wid * iInfo_.strides[3] + zid * iInfo_.strides[2] + yid * iInfo_.strides[1] + iInfo_.offset; diff --git a/src/backend/oneapi/kernel/rotate.hpp b/src/backend/oneapi/kernel/rotate.hpp index a6d255d369..2bb945f9a2 100644 --- a/src/backend/oneapi/kernel/rotate.hpp +++ b/src/backend/oneapi/kernel/rotate.hpp @@ -53,8 +53,7 @@ class rotateCreateKernel { , batches_(batches) , blocksXPerImage_(blocksXPerImage) , blocksYPerImage_(blocksYPerImage) - , method_(method) - , INTERP_ORDER_(INTERP_ORDER) {} + , method_(method) {} void operator()(sycl::nd_item<2> it) const { sycl::group g = it.get_group(); @@ -72,7 +71,8 @@ class rotateCreateKernel { const int limages = std::min((int)out_.dims[2] - setId * nimages_, nimages_); - if (xido >= out_.dims[0] || yido >= out_.dims[1]) return; + if (xido >= (unsigned)out_.dims[0] || yido >= (unsigned)out_.dims[1]) + return; InterpPosTy xidi = xido * t_.tmat[0] + yido * t_.tmat[1] + t_.tmat[2]; InterpPosTy yidi = xido * t_.tmat[3] + yido * t_.tmat[4] + t_.tmat[5]; @@ -85,7 +85,7 @@ class rotateCreateKernel { const int loco = outoff + (yido * out_.strides[1] + xido); InterpInTy zero = (InterpInTy)0; - if (INTERP_ORDER_ > 1) { + if constexpr (INTERP_ORDER > 1) { // Special conditions to deal with boundaries for bilinear and // bicubic // FIXME: Ideally this condition should be removed or be present for @@ -102,8 +102,8 @@ class rotateCreateKernel { // FIXME: Nearest and lower do not do clamping, but other methods do // Make it consistent - const bool doclamp = INTERP_ORDER_ != 1; - Interp2 interp2; // INTERP_ORDER> interp2; + constexpr bool doclamp = INTERP_ORDER != 1; + Interp2 interp2; interp2(d_out_, out_, loco, d_in_, in_, inoff, xidi, yidi, 0, 1, method_, limages, doclamp, 2); } @@ -119,7 +119,6 @@ class rotateCreateKernel { const int blocksXPerImage_; const int blocksYPerImage_; af::interpType method_; - const int INTERP_ORDER_; }; template diff --git a/src/backend/oneapi/kernel/scan_dim.hpp b/src/backend/oneapi/kernel/scan_dim.hpp index eea34ffff7..52450f5c98 100644 --- a/src/backend/oneapi/kernel/scan_dim.hpp +++ b/src/backend/oneapi/kernel/scan_dim.hpp @@ -82,7 +82,7 @@ class scanDimKernel { optr += ids[3] * oInfo_.strides[3] + ids[2] * oInfo_.strides[2] + ids[1] * oInfo_.strides[1] + ids[0]; iptr += ids[3] * iInfo_.strides[3] + ids[2] * iInfo_.strides[2] + - ids[1] * iInfo_.strides[1] + ids[0]; + ids[1] * iInfo_.strides[1] + ids[0] + iInfo_.offset; int id_dim = ids[dim]; const int out_dim = oInfo_.dims[dim]; diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp index 649e031b03..4aa7fc502e 100644 --- a/src/backend/oneapi/kernel/scan_first.hpp +++ b/src/backend/oneapi/kernel/scan_first.hpp @@ -71,7 +71,7 @@ class scanFirstKernel { To *tptr = tmp_acc_.get_pointer(); iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] + - yid * iInfo_.strides[1]; + yid * iInfo_.strides[1] + iInfo_.offset; optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] + yid * oInfo_.strides[1]; tptr += wid * tInfo_.strides[3] + zid * tInfo_.strides[2] + @@ -181,7 +181,7 @@ class scanFirstBcastKernel { // Shift broadcast one step to the right for exclusive scan (#2366) int offset = !inclusive_scan_; for (int k = 0, id = xid + offset; k < lim_ && id < oInfo_.dims[0]; - k++, id += g.get_group_range(0)) { + k++, id += g.get_local_range(0)) { optr[id] = binop(accum, optr[id]); } } diff --git a/src/backend/oneapi/kernel/select.hpp b/src/backend/oneapi/kernel/select.hpp index b5a6ae5954..06db45ad79 100644 --- a/src/backend/oneapi/kernel/select.hpp +++ b/src/backend/oneapi/kernel/select.hpp @@ -59,9 +59,9 @@ class selectKernelCreateKernel { void operator()(sycl::nd_item<2> it) const { sycl::group g = it.get_group(); - char *cptr = cptr__.get_pointer() + cinfo_.offset; - T *aptr = aptr__.get_pointer() + ainfo_.offset; - T *bptr = bptr__.get_pointer() + binfo_.offset; + const char *cptr = cptr__.get_pointer() + cinfo_.offset; + const T *aptr = aptr__.get_pointer() + ainfo_.offset; + const T *bptr = bptr__.get_pointer() + binfo_.offset; const int idz = g.get_group_id(0) / groups_0_; const int idw = g.get_group_id(1) / groups_1_; @@ -169,8 +169,8 @@ class selectScalarCreateKernel { void operator()(sycl::nd_item<2> it) const { sycl::group g = it.get_group(); - char *cptr = cptr__.get_pointer() + cinfo_.offset; - T *aptr = aptr__.get_pointer() + ainfo_.offset; + const char *cptr = cptr__.get_pointer() + cinfo_.offset; + const T *aptr = aptr__.get_pointer() + ainfo_.offset; const int idz = g.get_group_id(0) / groups_0_; const int idw = g.get_group_id(1) / groups_1_; @@ -185,7 +185,8 @@ class selectScalarCreateKernel { idy * oinfo_.strides[1]; int ids[] = {idx0, idy, idz, idw}; - optr_.get_pointer() += off; + T *optr = optr_.get_pointer(); + optr += off; aptr += getOffset(ainfo_.dims, ainfo_.strides, oinfo_.dims, ids); cptr += getOffset(cinfo_.dims, cinfo_.strides, oinfo_.dims, ids); @@ -196,7 +197,7 @@ class selectScalarCreateKernel { for (int idx = idx0; idx < oinfo_.dims[0]; idx += g.get_local_range(0) * groups_0_) { - optr_.get_pointer()[idx] = (cptr[idx] ^ flip_) ? aptr[idx] : b_; + optr[idx] = (cptr[idx] ^ flip_) ? aptr[idx] : b_; } } diff --git a/src/backend/oneapi/kernel/sort.hpp b/src/backend/oneapi/kernel/sort.hpp index 1789887b82..71bedd1f50 100644 --- a/src/backend/oneapi/kernel/sort.hpp +++ b/src/backend/oneapi/kernel/sort.hpp @@ -80,9 +80,9 @@ void sortBatched(Param pVal, int dim, bool isAscending) { auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue()); auto key_begin = ::oneapi::dpl::begin(*pKey.get()); - auto key_end = ::oneapi::dpl::end(*pKey.get()); + auto key_end = key_begin + pKey.dims()[0]; auto val_begin = ::oneapi::dpl::begin(*pVal.data); - auto val_end = ::oneapi::dpl::end(*pVal.data); + auto val_end = val_begin + pVal.info.dims[0]; auto zipped_begin = dpl::make_zip_iterator(key_begin, val_begin); auto zipped_end = dpl::make_zip_iterator(key_end, val_end); diff --git a/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt b/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt index ce184639eb..08b1d35f73 100644 --- a/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt +++ b/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt @@ -20,6 +20,10 @@ foreach(SBK_TYPE ${SBK_TYPES}) "${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key/sort_by_key_impl.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key_impl.hpp" ) + + set_source_files_properties("${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key/sort_by_key_impl.cpp" + PROPERTIES + LANGUAGE SYCL) set_target_properties(oneapi_sort_by_key_${SBK_TYPE} PROPERTIES COMPILE_DEFINITIONS "TYPE=${SBK_TYPE};AFDLL;$" @@ -41,12 +45,18 @@ foreach(SBK_TYPE ${SBK_TYPES}) .. ) + target_compile_options(oneapi_sort_by_key_${SBK_TYPE} + PRIVATE + $<$: -fno-sycl-id-queries-fit-in-int + -sycl-std=2020 + ${MSVC_RUNTIME} + $<$: -fno-sycl-rdc>>) + target_include_directories(oneapi_sort_by_key_${SBK_TYPE} SYSTEM PRIVATE ${span-lite_SOURCE_DIR}/include $) - target_compile_options(oneapi_sort_by_key_${SBK_TYPE} PUBLIC -fsycl) set_target_properties(oneapi_sort_by_key_${SBK_TYPE} PROPERTIES POSITION_INDEPENDENT_CODE ON) target_sources(oneapi_sort_by_key INTERFACE $) diff --git a/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp b/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp index 9b04402904..0b0a8fb13f 100644 --- a/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp +++ b/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp @@ -9,7 +9,7 @@ #include -// SBK_TYPES:float double int uint intl uintl short ushort char uchar half +// SBK_TYPES:float double int uint intl uintl short ushort char schar uchar half namespace arrayfire { namespace oneapi { diff --git a/src/backend/oneapi/kernel/sort_by_key_impl.hpp b/src/backend/oneapi/kernel/sort_by_key_impl.hpp index c0c57d8eff..2e462db4b6 100644 --- a/src/backend/oneapi/kernel/sort_by_key_impl.hpp +++ b/src/backend/oneapi/kernel/sort_by_key_impl.hpp @@ -8,6 +8,13 @@ ********************************************************/ #pragma once +#if defined(__clang__) +#pragma clang diagnostic push +// temporary ignores for DPL internals +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + // oneDPL headers should be included before standard headers #define ONEDPL_USE_PREDEFINED_POLICIES 0 #include @@ -98,23 +105,25 @@ void sortByKeyBatched(Param pKey, Param pVal, const int dim, // set up iterators for seq, key, val, and new cKey auto seq_begin = ::oneapi::dpl::begin(*Seq.get()); - auto seq_end = ::oneapi::dpl::end(*Seq.get()); + auto seq_end = seq_begin + elements; auto key_begin = ::oneapi::dpl::begin(pKey.data->template reinterpret>()); - auto key_end = - ::oneapi::dpl::end(pKey.data->template reinterpret>()); + auto key_end = key_begin + elements; + auto val_begin = ::oneapi::dpl::begin(*pVal.data); - auto val_end = ::oneapi::dpl::end(*pVal.data); + auto val_end = val_begin + elements; auto cKey = memAlloc(elements); + auto cKey_get = cKey.get(); getQueue().submit([&](sycl::handler &h) { - h.copy(pKey.data->template reinterpret>().get_access(), - cKey.get()->template reinterpret>().get_access()); + h.copy(pKey.data->template reinterpret>().get_access( + h, elements), + cKey_get->template reinterpret>().get_access( + h, elements)); }); auto ckey_begin = ::oneapi::dpl::begin(cKey.get()->template reinterpret>()); - auto ckey_end = - ::oneapi::dpl::end(cKey.get()->template reinterpret>()); + auto ckey_end = ckey_begin + elements; { auto zipped_begin_KV = dpl::make_zip_iterator(key_begin, val_begin); @@ -142,12 +151,15 @@ void sortByKeyBatched(Param pKey, Param pVal, const int dim, } } + auto Seq_get = Seq.get(); auto cSeq = memAlloc(elements); + auto cSeq_get = cSeq.get(); getQueue().submit([&](sycl::handler &h) { - h.copy(Seq.get()->get_access(), cSeq.get()->get_access()); + h.copy(Seq_get->get_access(h, elements), + cSeq_get->get_access(h, elements)); }); auto cseq_begin = ::oneapi::dpl::begin(*cSeq.get()); - auto cseq_end = ::oneapi::dpl::end(*cSeq.get()); + auto cseq_end = cseq_begin + elements; { auto zipped_begin_SV = dpl::make_zip_iterator(seq_begin, val_begin); @@ -197,6 +209,7 @@ void sort0ByKey(Param pKey, Param pVal, bool isAscending) { INSTANTIATE(Tk, short) \ INSTANTIATE(Tk, ushort) \ INSTANTIATE(Tk, char) \ + INSTANTIATE(Tk, schar) \ INSTANTIATE(Tk, uchar) \ INSTANTIATE(Tk, intl) \ INSTANTIATE(Tk, uintl) @@ -204,3 +217,8 @@ void sort0ByKey(Param pKey, Param pVal, bool isAscending) { } // namespace kernel } // namespace oneapi } // namespace arrayfire + +#if defined(__clang__) +/* Clang/LLVM */ +#pragma clang diagnostic pop +#endif diff --git a/src/backend/oneapi/kernel/sparse.hpp b/src/backend/oneapi/kernel/sparse.hpp new file mode 100644 index 0000000000..24458ed77d --- /dev/null +++ b/src/backend/oneapi/kernel/sparse.hpp @@ -0,0 +1,472 @@ +/******************************************************* + * Copyright (c) 2014, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace arrayfire { +namespace oneapi { +namespace kernel { + +template +class coo2DenseCreateKernel { + public: + coo2DenseCreateKernel(write_accessor oPtr, const KParam output, + write_accessor vPtr, const KParam values, + read_accessor rPtr, const KParam rowIdx, + read_accessor cPtr, const KParam colIdx) + : oPtr_(oPtr) + , output_(output) + , vPtr_(vPtr) + , values_(values) + , rPtr_(rPtr) + , rowIdx_(rowIdx) + , cPtr_(cPtr) + , colIdx_(colIdx) {} + + void operator()(sycl::nd_item<2> it) const { + sycl::group g = it.get_group(); + + const int dimSize = g.get_local_range(0); + + for (int i = it.get_local_id(0); i < REPEAT * dimSize; i += dimSize) { + const int id = + g.get_group_id(0) * g.get_local_range(0) * REPEAT + i; + if (id >= values_.dims[0]) return; + + T v = vPtr_[id + values_.offset]; + int r = rPtr_[id + rowIdx_.offset]; + int c = cPtr_[id + colIdx_.offset]; + + int offset = r + c * output_.strides[1]; + + oPtr_[offset] = v; + } + } + + private: + write_accessor oPtr_; + const KParam output_; + write_accessor vPtr_; + const KParam values_; + read_accessor rPtr_; + const KParam rowIdx_; + read_accessor cPtr_; + const KParam colIdx_; +}; + +template +void coo2dense(Param out, const Param values, const Param rowIdx, + const Param colIdx) { + auto local = sycl::range(THREADS_PER_BLOCK, 1); + auto global = sycl::range( + divup(values.info.dims[0], local[0] * REPEAT) * THREADS_PER_BLOCK, 1); + + getQueue().submit([&](auto &h) { + sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only}; + sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only}; + sycl::accessor d_out{*out.data, h, sycl::write_only, sycl::no_init}; + sycl::accessor d_values{*values.data, h, sycl::write_only, + sycl::no_init}; + h.parallel_for(sycl::nd_range{global, local}, + coo2DenseCreateKernel( + d_out, out.info, d_values, values.info, d_rowIdx, + rowIdx.info, d_colIdx, colIdx.info)); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +class csr2DenseCreateKernel { + public: + csr2DenseCreateKernel(write_accessor output, read_accessor values, + read_accessor rowidx, read_accessor colidx, + const int M, const int v_off, const int r_off, const int c_off) + : output_(output) + , values_(values) + , rowidx_(rowidx) + , colidx_(colidx) + , M_(M) + , v_off_(v_off) + , r_off_(r_off) + , c_off_(c_off) {} + + void operator()(sycl::nd_item<2> it) const { + sycl::group g = it.get_group(); + + int lid = it.get_local_id(0); + for (int rowId = g.get_group_id(0); rowId < M_; + rowId += it.get_group_range(0)) { + int colStart = rowidx_[rowId + r_off_]; + int colEnd = rowidx_[rowId + r_off_ + 1]; + for (int colId = colStart + lid; colId < colEnd; colId += THREADS) { + output_[rowId + colidx_[colId + c_off_] * M_] = values_[colId + v_off_]; + } + } + } + + private: + write_accessor output_; + read_accessor values_; + read_accessor rowidx_; + read_accessor colidx_; + const int M_; + const int v_off_; + const int r_off_; + const int c_off_; +}; + +template +void csr2dense(Param output, const Param values, const Param rowIdx, + const Param colIdx) { + constexpr int MAX_GROUPS = 4096; + // FIXME: This needs to be based non nonzeros per row + constexpr int threads = 64; + + const int M = rowIdx.info.dims[0] - 1; + + auto local = sycl::range(threads, 1); + int groups_x = std::min((int)(divup(M, local[0])), MAX_GROUPS); + auto global = sycl::range(local[0] * groups_x, 1); + + getQueue().submit([&](auto &h) { + sycl::accessor d_values{*values.data, h, sycl::read_only}; + sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only}; + sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only}; + sycl::accessor d_output{*output.data, h, sycl::write_only, + sycl::no_init}; + h.parallel_for(sycl::nd_range{global, local}, + csr2DenseCreateKernel( + d_output, d_values, d_rowIdx, d_colIdx, M, + static_cast(values.info.offset), + static_cast(rowIdx.info.offset), + static_cast(colIdx.info.offset))); + }); + + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +class dense2csrCreateKernel { + public: + dense2csrCreateKernel(write_accessor svalptr, + write_accessor scolptr, read_accessor dvalptr, + const KParam valinfo, read_accessor dcolptr, + const KParam colinfo, read_accessor rowptr) + : svalptr_(svalptr) + , scolptr_(scolptr) + , dvalptr_(dvalptr) + , valinfo_(valinfo) + , dcolptr_(dcolptr) + , colinfo_(colinfo) + , rowptr_(rowptr) {} + + void operator()(sycl::nd_item<2> it) const { + // sycl::group g = it.get_group(); + + int gidx = it.get_global_id(0); + int gidy = it.get_global_id(1); + + if (gidx >= (unsigned)valinfo_.dims[0]) return; + if (gidy >= (unsigned)valinfo_.dims[1]) return; + + int rowoff = rowptr_[gidx]; + auto svalptr_ptr = svalptr_.get_pointer(); + auto scolptr_ptr = scolptr_.get_pointer(); + + auto dvalptr_ptr = dvalptr_.get_pointer(); + auto dcolptr_ptr = dcolptr_.get_pointer(); + + T val = dvalptr_ptr[gidx + gidy * (unsigned)valinfo_.strides[1] + valinfo_.offset]; + + if constexpr (std::is_same_v> || + std::is_same_v>) { + if (val.real() == 0 && val.imag() == 0) return; + } else { + if (val == 0) return; + } + + int oloc = dcolptr_ptr[gidx + gidy * colinfo_.strides[1] + colinfo_.offset]; + svalptr_ptr[oloc + rowoff - 1] = val; + scolptr_ptr[oloc + rowoff - 1] = gidy; + } + + private: + write_accessor svalptr_; + write_accessor scolptr_; + read_accessor dvalptr_; + const KParam valinfo_; + read_accessor dcolptr_; + const KParam colinfo_; + read_accessor rowptr_; +}; + +template +void dense2csr(Param values, Param rowIdx, Param colIdx, + const Param dense) { + int num_rows = dense.info.dims[0]; + int num_cols = dense.info.dims[1]; + + // sd1 contains output of scan along dim 1 of dense + Array sd1 = createEmptyArray(dim4(num_rows, num_cols)); + // rd1 contains output of nonzero count along dim 1 along dense + Array rd1 = createEmptyArray(num_rows); + + scan_dim(sd1, dense, true); + reduce_dim_default(rd1, dense, 0, 0); + scan_first(rowIdx, rd1, false); + + const int nnz = values.info.dims[0]; + + const sycl::id<1> fillOffset(rowIdx.info.offset + + (rowIdx.info.dims[0] - 1)); + const sycl::range<1> fillRange(rowIdx.info.dims[0] - fillOffset[0]); + getQueue().submit([&](auto &h) { + sycl::accessor d_rowIdx{*rowIdx.data, h, fillRange, fillOffset}; + h.fill(d_rowIdx, nnz); + }); + + auto local = sycl::range(THREADS_X, THREADS_Y); + int groups_x = divup(dense.info.dims[0], local[0]); + int groups_y = divup(dense.info.dims[1], local[1]); + auto global = sycl::range(groups_x * local[0], groups_y * local[1]); + + const Param sdParam = sd1; + + getQueue().submit([&](auto &h) { + sycl::accessor d_dense{*dense.data, h, sycl::read_only}; + sycl::accessor d_sdParam{*sdParam.data, h, sycl::read_only}; + sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only}; + sycl::accessor d_values{*values.data, h, sycl::write_only, + sycl::no_init}; + sycl::accessor d_colIdx{*colIdx.data, h, sycl::write_only, + sycl::no_init}; + h.parallel_for( + sycl::nd_range{global, local}, + dense2csrCreateKernel(d_values, d_colIdx, d_dense, dense.info, + d_sdParam, sdParam.info, d_rowIdx)); + }); + + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +class swapIndexCreateKernel { + public: + swapIndexCreateKernel(write_accessor ovalues, write_accessor oindex, + read_accessor ivalues, read_accessor iindex, + read_accessor swapIdx, const int nNZ) + : ovalues_(ovalues) + , oindex_(oindex) + , ivalues_(ivalues) + , iindex_(iindex) + , swapIdx_(swapIdx) + , nNZ_(nNZ) {} + + void operator()(sycl::item<1> it) const { + int id = it.get_id(0); + if (id < nNZ_) { + int idx = swapIdx_[id]; + + ovalues_[id] = ivalues_[idx]; + oindex_[id] = iindex_[idx]; + } + } + + private: + write_accessor ovalues_; + write_accessor oindex_; + read_accessor ivalues_; + read_accessor iindex_; + read_accessor swapIdx_; + const int nNZ_; +}; + +template +void swapIndex(Param ovalues, Param oindex, const Param ivalues, + sycl::buffer iindex, const Param swapIdx) { + auto global = sycl::range(ovalues.info.dims[0]); + + getQueue().submit([&](auto &h) { + sycl::accessor d_ivalues{*ivalues.data, h, sycl::read_only}; + sycl::accessor d_iindex{iindex, h, sycl::read_only}; + sycl::accessor d_swapIdx{*swapIdx.data, h, sycl::read_only}; + sycl::accessor d_ovalues{*ovalues.data, h, sycl::write_only, + sycl::no_init}; + sycl::accessor d_oindex{*oindex.data, h, sycl::write_only, + sycl::no_init}; + + h.parallel_for(global, + swapIndexCreateKernel( + d_ovalues, d_oindex, d_ivalues, d_iindex, d_swapIdx, + static_cast(ovalues.info.dims[0]))); + }); + + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +class csr2CooCreateKernel { + public: + csr2CooCreateKernel(write_accessor orowidx, + write_accessor ocolidx, read_accessor irowidx, + read_accessor icolidx, const int M) + : orowidx_(orowidx) + , ocolidx_(ocolidx) + , irowidx_(irowidx) + , icolidx_(icolidx) + , M_(M) {} + + void operator()(sycl::nd_item<2> it) const { + sycl::group g = it.get_group(); + + int lid = it.get_local_id(0); + for (int rowId = g.get_group_id(0); rowId < M_; + rowId += it.get_group_range(0)) { + int colStart = irowidx_[rowId]; + int colEnd = irowidx_[rowId + 1]; + for (int colId = colStart + lid; colId < colEnd; + colId += g.get_local_range(0)) { + orowidx_[colId] = rowId; + ocolidx_[colId] = icolidx_[colId]; + } + } + } + + private: + write_accessor orowidx_; + write_accessor ocolidx_; + read_accessor irowidx_; + read_accessor icolidx_; + const int M_; +}; + +template +void csr2coo(Param ovalues, Param orowIdx, Param ocolIdx, + const Param ivalues, const Param irowIdx, + const Param icolIdx, Param index) { + const int MAX_GROUPS = 4096; + int M = irowIdx.info.dims[0] - 1; + // FIXME: This needs to be based non nonzeros per row + int threads = 64; + + auto scratch = memAlloc(orowIdx.info.dims[0]); + + auto local = sycl::range(threads, 1); + int groups_x = std::min((int)(divup(M, local[0])), MAX_GROUPS); + auto global = sycl::range(local[0] * groups_x, 1); + + getQueue().submit([&](auto &h) { + sycl::accessor d_irowIdx{*irowIdx.data, h, sycl::read_only}; + sycl::accessor d_icolIdx{*icolIdx.data, h, sycl::read_only}; + sycl::accessor d_scratch{*scratch, h, sycl::write_only, sycl::no_init}; + sycl::accessor d_ocolIdx{*ocolIdx.data, h, sycl::write_only, + sycl::no_init}; + h.parallel_for(sycl::nd_range{global, local}, + csr2CooCreateKernel(d_scratch, d_ocolIdx, d_irowIdx, + d_icolIdx, M)); + }); + + // Now we need to sort this into column major + kernel::sort0ByKeyIterative(ocolIdx, index, true); + + // Now use index to sort values and rows + kernel::swapIndex(ovalues, orowIdx, ivalues, *scratch, index); + + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +class csrReduceKernel { + public: + csrReduceKernel(write_accessor orowidx, read_accessor irowidx, + const int M, const int nNZ) + : orowidx_(orowidx), irowidx_(irowidx), M_(M), nNZ_(nNZ) {} + + void operator()(sycl::item<1> it) const { + int id = it.get_id(0); + + if (id < nNZ_) { + // Read COO row indices + int iRId = irowidx_[id]; + int iRId1 = 0; + if (id > 0) iRId1 = irowidx_[id - 1]; + + // If id is 0, then mark the edge cases of csrRow[0] and csrRow[M] + if (id == 0) { + orowidx_[id] = 0; + orowidx_[M_] = nNZ_; + } else if (iRId1 != iRId) { + // If iRId1 and iRId are not same, that means the row has + // incremented For example, if iRId is 5 and iRId1 is 4, that + // means row 4 has ended and row 5 has begun at index id. We use + // the for-loop because there can be any number of empty rows + // between iRId1 and iRId, all of which should be marked by id + for (int i = iRId1 + 1; i <= iRId; i++) orowidx_[i] = id; + } + + // The last X rows are corner cases if they dont have any values + if (id < M_) { + if (id > irowidx_[nNZ_ - 1] && orowidx_[id] == 0) { + orowidx_[id] = nNZ_; + } + } + } + } + + private: + write_accessor orowidx_; + read_accessor irowidx_; + const int M_; + const int nNZ_; +}; + +template +void coo2csr(Param ovalues, Param orowIdx, Param ocolIdx, + const Param ivalues, const Param irowIdx, + const Param icolIdx, Param index, Param rowCopy, + const int M) { + // Now we need to sort this into column major + kernel::sort0ByKeyIterative(rowCopy, index, true); + + // Now use index to sort values and rows + kernel::swapIndex(ovalues, ocolIdx, ivalues, *icolIdx.data, index); + + ONEAPI_DEBUG_FINISH(getQueue()); + + auto global = sycl::range(irowIdx.info.dims[0]); + + getQueue().submit([&](auto &h) { + sycl::accessor d_orowIdx{*orowIdx.data, h, sycl::write_only}; + sycl::accessor d_rowCopy{*rowCopy.data, h, sycl::read_only}; + h.parallel_for( + sycl::range{global}, + csrReduceKernel(d_orowIdx, d_rowCopy, M, + static_cast(ovalues.info.dims[0]))); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +} // namespace kernel +} // namespace oneapi +} // namespace arrayfire diff --git a/src/backend/oneapi/kernel/sparse_arith.hpp b/src/backend/oneapi/kernel/sparse_arith.hpp new file mode 100644 index 0000000000..b46baa69df --- /dev/null +++ b/src/backend/oneapi/kernel/sparse_arith.hpp @@ -0,0 +1,570 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace arrayfire { +namespace oneapi { +namespace kernel { + +constexpr unsigned TX = 32; +constexpr unsigned TY = 8; +constexpr unsigned THREADS = TX * TY; + +template +using global_atomic_ref = + sycl::atomic_ref; + +template +class sparseArithCSRKernel { + public: + sparseArithCSRKernel(write_accessor oPtr, const KParam out, + read_accessor values, read_accessor rowIdx, + read_accessor colIdx, const int nNZ, + read_accessor rPtr, const KParam rhs, + const int reverse) + : oPtr_(oPtr) + , out_(out) + , values_(values) + , rowIdx_(rowIdx) + , colIdx_(colIdx) + , nNZ_(nNZ) + , rPtr_(rPtr) + , rhs_(rhs) + , reverse_(reverse) {} + + void operator()(sycl::nd_item<2> it) const { + sycl::group g = it.get_group(); + common::Binary binOP; + + const int row = + g.get_group_id(0) * g.get_local_range(1) + it.get_local_id(1); + + if (row < out_.dims[0]) { + const int rowStartIdx = rowIdx_[row]; + const int rowEndIdx = rowIdx_[row + 1]; + + // Repeat loop until all values in the row are computed + for (int idx = rowStartIdx + it.get_local_id(0); idx < rowEndIdx; + idx += g.get_local_range(0)) { + const int col = colIdx_[idx]; + + if (row >= out_.dims[0] || col >= out_.dims[1]) + continue; // Bad indices + + // Get Values + const T val = values_[idx]; + const T rval = rPtr_[col * rhs_.strides[1] + row]; + + const int offset = col * out_.strides[1] + row; + if (reverse_) + oPtr_[offset] = binOP(rval, val); + else + oPtr_[offset] = binOP(val, rval); + } + } + } + + private: + write_accessor oPtr_; + const KParam out_; + read_accessor values_; + read_accessor rowIdx_; + read_accessor colIdx_; + const int nNZ_; + read_accessor rPtr_; + const KParam rhs_; + const int reverse_; +}; + +template +void sparseArithOpCSR(Param out, const Param values, + const Param rowIdx, const Param colIdx, + const Param rhs, const bool reverse) { + auto local = sycl::range(TX, TY); + auto global = sycl::range(divup(out.info.dims[0], TY) * TX, TY); + + getQueue().submit([&](auto &h) { + sycl::accessor d_out{*out.data, h, sycl::write_only}; + sycl::accessor d_values{*values.data, h, sycl::read_only}; + sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only}; + sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only}; + sycl::accessor d_rhs{*rhs.data, h, sycl::read_only}; + + h.parallel_for(sycl::nd_range{global, local}, + sparseArithCSRKernel( + d_out, out.info, d_values, d_rowIdx, d_colIdx, + static_cast(values.info.dims[0]), d_rhs, + rhs.info, static_cast(reverse))); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +class sparseArithCOOKernel { + public: + sparseArithCOOKernel(write_accessor oPtr, const KParam out, + read_accessor values, read_accessor rowIdx, + read_accessor colIdx, const int nNZ, + read_accessor rPtr, const KParam rhs, + const int reverse) + : oPtr_(oPtr) + , out_(out) + , values_(values) + , rowIdx_(rowIdx) + , colIdx_(colIdx) + , nNZ_(nNZ) + , rPtr_(rPtr) + , rhs_(rhs) + , reverse_(reverse) {} + + void operator()(sycl::nd_item<1> it) const { + common::Binary binOP; + + const int idx = it.get_global_id(0); + + if (idx < nNZ_) { + const int row = rowIdx_[idx]; + const int col = colIdx_[idx]; + + if (row >= out_.dims[0] || col >= out_.dims[1]) + return; // Bad indices + + // Get Values + const T val = values_[idx]; + const T rval = rPtr_[col * rhs_.strides[1] + row]; + + const int offset = col * out_.strides[1] + row; + if (reverse_) + oPtr_[offset] = binOP(rval, val); + else + oPtr_[offset] = binOP(val, rval); + } + } + + private: + write_accessor oPtr_; + const KParam out_; + read_accessor values_; + read_accessor rowIdx_; + read_accessor colIdx_; + const int nNZ_; + read_accessor rPtr_; + const KParam rhs_; + const int reverse_; +}; + +template +void sparseArithOpCOO(Param out, const Param values, + const Param rowIdx, const Param colIdx, + const Param rhs, const bool reverse) { + auto local = sycl::range(THREADS); + auto global = sycl::range(divup(values.info.dims[0], THREADS) * THREADS); + + getQueue().submit([&](auto &h) { + sycl::accessor d_out{*out.data, h, sycl::write_only}; + sycl::accessor d_values{*values.data, h, sycl::read_only}; + sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only}; + sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only}; + sycl::accessor d_rhs{*rhs.data, h, sycl::read_only}; + + h.parallel_for(sycl::nd_range{global, local}, + sparseArithCOOKernel( + d_out, out.info, d_values, d_rowIdx, d_colIdx, + static_cast(values.info.dims[0]), d_rhs, + rhs.info, static_cast(reverse))); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +class sparseArithCSR2Kernel { + public: + sparseArithCSR2Kernel(sycl::accessor values, read_accessor rowIdx, + read_accessor colIdx, const int nNZ, + read_accessor rPtr, const KParam rhs, + const int reverse) + : values_(values) + , rowIdx_(rowIdx) + , colIdx_(colIdx) + , nNZ_(nNZ) + , rPtr_(rPtr) + , rhs_(rhs) + , reverse_(reverse) {} + + void operator()(sycl::nd_item<2> it) const { + sycl::group g = it.get_group(); + common::Binary binOP; + + const int row = + g.get_group_id(0) * g.get_local_range(1) + it.get_local_id(1); + + if (row < rhs_.dims[0]) { + const int rowStartIdx = rowIdx_[row]; + const int rowEndIdx = rowIdx_[row + 1]; + + // Repeat loop until all values in the row are computed + for (int idx = rowStartIdx + it.get_local_id(0); idx < rowEndIdx; + idx += g.get_local_range(0)) { + const int col = colIdx_[idx]; + + if (row >= rhs_.dims[0] || col >= rhs_.dims[1]) + continue; // Bad indices + + // Get Values + const T val = values_[idx]; + const T rval = rPtr_[col * rhs_.strides[1] + row]; + + if (reverse_) + values_[idx] = binOP(rval, val); + else + values_[idx] = binOP(val, rval); + } + } + } + + private: + sycl::accessor values_; + read_accessor rowIdx_; + read_accessor colIdx_; + const int nNZ_; + read_accessor rPtr_; + const KParam rhs_; + const int reverse_; +}; + +template +void sparseArithOpCSR(Param values, Param rowIdx, Param colIdx, + const Param rhs, const bool reverse) { + auto local = sycl::range(TX, TY); + auto global = sycl::range(divup(values.info.dims[0], TY) * TX, TY); + + getQueue().submit([&](auto &h) { + sycl::accessor d_values{*values.data, h, sycl::read_write}; + sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only}; + sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only}; + sycl::accessor d_rhs{*rhs.data, h, sycl::read_only}; + + h.parallel_for(sycl::nd_range{global, local}, + sparseArithCSR2Kernel( + d_values, d_rowIdx, d_colIdx, + static_cast(values.info.dims[0]), d_rhs, + rhs.info, static_cast(reverse))); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +class sparseArithCOO2Kernel { + public: + sparseArithCOO2Kernel(sycl::accessor values, read_accessor rowIdx, + read_accessor colIdx, const int nNZ, + read_accessor rPtr, const KParam rhs, + const int reverse) + : values_(values) + , rowIdx_(rowIdx) + , colIdx_(colIdx) + , nNZ_(nNZ) + , rPtr_(rPtr) + , rhs_(rhs) + , reverse_(reverse) {} + + void operator()(sycl::nd_item<1> it) const { + common::Binary binOP; + + const int idx = it.get_global_id(0); + + if (idx < nNZ_) { + const int row = rowIdx_[idx]; + const int col = colIdx_[idx]; + + if (row >= rhs_.dims[0] || col >= rhs_.dims[1]) + return; // Bad indices + + // Get Values + const T val = values_[idx]; + const T rval = rPtr_[col * rhs_.strides[1] + row]; + + if (reverse_) + values_[idx] = binOP(rval, val); + else + values_[idx] = binOP(val, rval); + } + } + + private: + sycl::accessor values_; + read_accessor rowIdx_; + read_accessor colIdx_; + const int nNZ_; + read_accessor rPtr_; + const KParam rhs_; + const int reverse_; +}; + +template +void sparseArithOpCOO(Param values, Param rowIdx, Param colIdx, + const Param rhs, const bool reverse) { + auto local = sycl::range(THREADS); + auto global = sycl::range(divup(values.info.dims[0], THREADS) * THREADS); + + getQueue().submit([&](auto &h) { + sycl::accessor d_values{*values.data, h, sycl::read_write}; + sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only}; + sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only}; + sycl::accessor d_rhs{*rhs.data, h, sycl::read_only}; + + h.parallel_for(sycl::nd_range{global, local}, + sparseArithCOO2Kernel( + d_values, d_rowIdx, d_colIdx, + static_cast(values.info.dims[0]), d_rhs, + rhs.info, static_cast(reverse))); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +class csrCalcOutNNZKernel { + public: + csrCalcOutNNZKernel(write_accessor nnzc, + write_accessor oRowIdx, unsigned M, + read_accessor lRowIdx, read_accessor lColIdx, + read_accessor rRowIdx, read_accessor rColIdx, + sycl::local_accessor blkNNZ) + : nnzc_(nnzc) + , oRowIdx_(oRowIdx) + , M_(M) + , lRowIdx_(lRowIdx) + , lColIdx_(lColIdx) + , rRowIdx_(rRowIdx) + , rColIdx_(rColIdx) + , blkNNZ_(blkNNZ) {} + + void operator()(sycl::nd_item<1> it) const { + sycl::group g = it.get_group(); + + const uint row = it.get_global_id(0); + const uint tid = it.get_local_id(0); + + const bool valid = row < M_; + + const uint lEnd = (valid ? lRowIdx_[row + 1] : 0); + const uint rEnd = (valid ? rRowIdx_[row + 1] : 0); + + blkNNZ_[tid] = 0; + it.barrier(); + + uint l = (valid ? lRowIdx_[row] : 0); + uint r = (valid ? rRowIdx_[row] : 0); + uint nnz = 0; + while (l < lEnd && r < rEnd) { + uint lci = lColIdx_[l]; + uint rci = rColIdx_[r]; + l += (lci <= rci); + r += (lci >= rci); + nnz++; + } + nnz += (lEnd - l); + nnz += (rEnd - r); + + blkNNZ_[tid] = nnz; + it.barrier(); + + if (valid) oRowIdx_[row + 1] = nnz; + + for (uint s = g.get_local_range(0) / 2; s > 0; s >>= 1) { + if (tid < s) { blkNNZ_[tid] += blkNNZ_[tid + s]; } + it.barrier(); + } + + if (tid == 0) { + nnz = blkNNZ_[0]; + global_atomic_ref(nnzc_[0]) += nnz; + } + } + + private: + write_accessor nnzc_; + write_accessor oRowIdx_; + unsigned M_; + read_accessor lRowIdx_; + read_accessor lColIdx_; + read_accessor rRowIdx_; + read_accessor rColIdx_; + sycl::local_accessor blkNNZ_; +}; + +static void csrCalcOutNNZ(Param outRowIdx, unsigned &nnzC, const uint M, + const uint N, uint nnzA, const Param lrowIdx, + const Param lcolIdx, uint nnzB, + const Param rrowIdx, const Param rcolIdx) { + UNUSED(N); + UNUSED(nnzA); + UNUSED(nnzB); + + auto local = sycl::range(256); + auto global = sycl::range(divup(M, local[0]) * local[0]); + + Array out = createValueArray(1, 0); + auto out_get = out.get(); + + getQueue().submit([&](auto &h) { + sycl::accessor d_out{*out_get, h, sycl::write_only}; + sycl::accessor d_outRowIdx{*outRowIdx.data, h, sycl::write_only}; + sycl::accessor d_lRowIdx{*lrowIdx.data, h, sycl::read_only}; + sycl::accessor d_lColIdx{*lcolIdx.data, h, sycl::read_only}; + sycl::accessor d_rRowIdx{*rrowIdx.data, h, sycl::read_only}; + sycl::accessor d_rColIdx{*rcolIdx.data, h, sycl::read_only}; + + auto blkNNZ = sycl::local_accessor(local[0], h); + h.parallel_for( + sycl::nd_range{global, local}, + csrCalcOutNNZKernel(d_out, d_outRowIdx, M, d_lRowIdx, d_lColIdx, + d_rRowIdx, d_rColIdx, blkNNZ)); + }); + + { + sycl::host_accessor nnz_acc{*out.get(), sycl::read_only}; + nnzC = nnz_acc[0]; + } + + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +class ssarithCSRKernel { + public: + ssarithCSRKernel(write_accessor oVals, write_accessor oColIdx, + read_accessor oRowIdx, unsigned M, unsigned N, + unsigned nnza, read_accessor lVals, + read_accessor lRowIdx, read_accessor lColIdx, + unsigned nnzb, read_accessor rVals, + read_accessor rRowIdx, read_accessor rColIdx) + : oVals_(oVals) + , oColIdx_(oColIdx) + , oRowIdx_(oRowIdx) + , M_(M) + , N_(N) + , nnza_(nnza) + , lVals_(lVals) + , lRowIdx_(lRowIdx) + , lColIdx_(lColIdx) + , nnzb_(nnzb) + , rVals_(rVals) + , rRowIdx_(rRowIdx) + , rColIdx_(rColIdx) {} + + void operator()(sycl::nd_item<1> it) const { + common::Binary binOP; + + const uint row = it.get_global_id(0); + + const bool valid = row < M_; + const uint lEnd = (valid ? lRowIdx_[row + 1] : 0); + const uint rEnd = (valid ? rRowIdx_[row + 1] : 0); + const uint offset = (valid ? oRowIdx_[row] : 0); + + T *ovPtr = oVals_.get_pointer() + offset; + int *ocPtr = oColIdx_.get_pointer() + offset; + + uint l = (valid ? lRowIdx_[row] : 0); + uint r = (valid ? rRowIdx_[row] : 0); + + uint nnz = 0; + while (l < lEnd && r < rEnd) { + uint lci = lColIdx_[l]; + uint rci = rColIdx_[r]; + + T lhs = (lci <= rci ? lVals_[l] : common::Binary::init()); + T rhs = (lci >= rci ? rVals_[r] : common::Binary::init()); + + ovPtr[nnz] = binOP(lhs, rhs); + ocPtr[nnz] = (lci <= rci) ? lci : rci; + + l += (lci <= rci); + r += (lci >= rci); + nnz++; + } + while (l < lEnd) { + ovPtr[nnz] = binOP(lVals_[l], common::Binary::init()); + ocPtr[nnz] = lColIdx_[l]; + l++; + nnz++; + } + while (r < rEnd) { + ovPtr[nnz] = binOP(common::Binary::init(), rVals_[r]); + ocPtr[nnz] = rColIdx_[r]; + r++; + nnz++; + } + } + + private: + write_accessor oVals_; + write_accessor oColIdx_; + read_accessor oRowIdx_; + unsigned M_, N_; + unsigned nnza_; + read_accessor lVals_; + read_accessor lRowIdx_; + read_accessor lColIdx_; + unsigned nnzb_; + read_accessor rVals_; + read_accessor rRowIdx_; + read_accessor rColIdx_; +}; + +template +void ssArithCSR(Param oVals, Param oColIdx, const Param oRowIdx, + const uint M, const uint N, unsigned nnzA, const Param lVals, + const Param lRowIdx, const Param lColIdx, + unsigned nnzB, const Param rVals, const Param rRowIdx, + const Param rColIdx) { + auto local = sycl::range(256); + auto global = sycl::range(divup(M, local[0]) * local[0]); + + getQueue().submit([&](auto &h) { + sycl::accessor d_oVals{*oVals.data, h, sycl::write_only}; + sycl::accessor d_oColIdx{*oColIdx.data, h, sycl::write_only}; + sycl::accessor d_oRowIdx{*oRowIdx.data, h, sycl::read_only}; + + sycl::accessor d_lVals{*lVals.data, h, sycl::read_only}; + sycl::accessor d_lRowIdx{*lRowIdx.data, h, sycl::read_only}; + sycl::accessor d_lColIdx{*lColIdx.data, h, sycl::read_only}; + + sycl::accessor d_rVals{*rVals.data, h, sycl::read_only}; + sycl::accessor d_rRowIdx{*rRowIdx.data, h, sycl::read_only}; + sycl::accessor d_rColIdx{*rColIdx.data, h, sycl::read_only}; + + h.parallel_for( + sycl::nd_range{global, local}, + ssarithCSRKernel(d_oVals, d_oColIdx, d_oRowIdx, M, N, nnzA, + d_lVals, d_lRowIdx, d_lColIdx, nnzB, + d_rVals, d_rRowIdx, d_rColIdx)); + }); +} + +} // namespace kernel +} // namespace oneapi +} // namespace arrayfire diff --git a/src/backend/oneapi/kernel/transform.hpp b/src/backend/oneapi/kernel/transform.hpp index 07f70a3a62..874e9638c7 100644 --- a/src/backend/oneapi/kernel/transform.hpp +++ b/src/backend/oneapi/kernel/transform.hpp @@ -178,7 +178,8 @@ class transformCreateKernel { using TMatTy = typename std::conditional::type; TMatTy tmat; - const float *tmat_ptr = c_tmat_.get_pointer() + t_idx * transf_len; + const float *tmat_ptr = + c_tmat_.get_pointer() + tf_.offset + t_idx * transf_len; // We expect a inverse transform matrix by default // If it is an forward transform, then we need its inverse diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp index bf7c7a874b..2752111534 100644 --- a/src/backend/oneapi/kernel/transpose.hpp +++ b/src/backend/oneapi/kernel/transpose.hpp @@ -95,7 +95,8 @@ class transposeKernel { // offset in_ and out_ based on batch id // also add the subBuffer offsets - T *iDataPtr = iData_.get_pointer(), *oDataPtr = oData_.get_pointer(); + const T *iDataPtr = iData_.get_pointer(); + T *oDataPtr = oData_.get_pointer(); iDataPtr += batchId_x * in_.strides[2] + batchId_y * in_.strides[3] + in_.offset; oDataPtr += batchId_x * out_.strides[2] + batchId_y * out_.strides[3] + diff --git a/src/backend/oneapi/kernel/unwrap.hpp b/src/backend/oneapi/kernel/unwrap.hpp index 0c88bd4348..43301fd744 100644 --- a/src/backend/oneapi/kernel/unwrap.hpp +++ b/src/backend/oneapi/kernel/unwrap.hpp @@ -149,7 +149,7 @@ void unwrap(Param out, const Param in, const dim_t wx, const dim_t wy, reps = divup((wx * wy), TX); } else { TX = THREADS_X; - TY = THREADS_X; + TY = THREADS_Y; BX = divup(out.info.dims[0], TX); reps = divup((wx * wy), TY); } diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp index b65e0d9333..69f2f7719a 100644 --- a/src/backend/oneapi/kernel/where.hpp +++ b/src/backend/oneapi/kernel/where.hpp @@ -73,7 +73,7 @@ class whereKernel { otptr += wid * otInfo_.strides[3] + zid * otInfo_.strides[2] + yid * otInfo_.strides[1]; iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] + - yid * iInfo_.strides[1]; + yid * iInfo_.strides[1] + iInfo_.offset; size_t odims0 = otInfo_.dims[0]; size_t odims1 = otInfo_.dims[1]; diff --git a/src/backend/oneapi/kernel/wrap.hpp b/src/backend/oneapi/kernel/wrap.hpp index ef8d2eba21..e29403b604 100644 --- a/src/backend/oneapi/kernel/wrap.hpp +++ b/src/backend/oneapi/kernel/wrap.hpp @@ -63,8 +63,8 @@ class wrapCreateKernel { T *optr = optrAcc_.get_pointer() + idx2 * out_.strides[2] + idx3 * out_.strides[3] + out_.offset; - T *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] + - idx3 * in_.strides[3] + in_.offset; + const T *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] + + idx3 * in_.strides[3] + in_.offset; if (oidx0 >= out_.dims[0] || oidx1 >= out_.dims[1]) return; @@ -140,7 +140,7 @@ void wrap(Param out, const Param in, const dim_t wx, const dim_t wy, dim_t groups_y = divup(out.info.dims[1], local[1]); auto global = sycl::range{groups_x * local[0] * out.info.dims[2], - groups_y * local[1]}; + groups_y * local[1] * out.info.dims[3]}; auto Q = getQueue(); Q.submit([&](sycl::handler &h) { diff --git a/src/backend/oneapi/kernel/wrap_dilated.hpp b/src/backend/oneapi/kernel/wrap_dilated.hpp index 63bdf342a8..41112fbce4 100644 --- a/src/backend/oneapi/kernel/wrap_dilated.hpp +++ b/src/backend/oneapi/kernel/wrap_dilated.hpp @@ -28,12 +28,13 @@ namespace kernel { template class wrapDilatedCreateKernel { public: - wrapDilatedCreateKernel(write_accessor optrAcc, KParam out, - read_accessor iptrAcc, KParam in, const int wx, - const int wy, const int sx, const int sy, - const int px, const int py, const int dx, - const int dy, const int nx, const int ny, - int groups_x, int groups_y, const bool is_column) + wrapDilatedCreateKernel(write_accessor> optrAcc, KParam out, + read_accessor> iptrAcc, KParam in, + const int wx, const int wy, const int sx, + const int sy, const int px, const int py, + const int dx, const int dy, const int nx, + const int ny, int groups_x, int groups_y, + const bool is_column) : optrAcc_(optrAcc) , out_(out) , iptrAcc_(iptrAcc) @@ -63,10 +64,10 @@ class wrapDilatedCreateKernel { int oidx0 = it.get_local_id(0) + g.get_local_range(0) * groupId_x; int oidx1 = it.get_local_id(1) + g.get_local_range(1) * groupId_y; - T *optr = optrAcc_.get_pointer() + idx2 * out_.strides[2] + - idx3 * out_.strides[3]; - T *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] + - idx3 * in_.strides[3] + in_.offset; + data_t *optr = optrAcc_.get_pointer() + idx2 * out_.strides[2] + + idx3 * out_.strides[3]; + const data_t *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] + + idx3 * in_.strides[3] + in_.offset; if (oidx0 >= out_.dims[0] || oidx1 >= out_.dims[1]) return; @@ -86,7 +87,7 @@ class wrapDilatedCreateKernel { const int x_start = (pidx0 < eff_wx) ? 0 : (pidx0 - eff_wx) / sx_ + 1; const int x_end = sycl::min(pidx0 / sx_ + 1, nx_); - T val = (T)0; + compute_t val(0); int idx = 1; for (int y = y_start; y < y_end; y++) { @@ -111,8 +112,8 @@ class wrapDilatedCreateKernel { idx = dim_end + win_end * in_.strides[1]; } - T ival; - ival = (yvalid && xvalid) ? iptr[idx] : (T)0; + compute_t ival; + ival = (yvalid && xvalid) ? iptr[idx] : compute_t(0); val = val + ival; } } @@ -121,9 +122,9 @@ class wrapDilatedCreateKernel { } private: - write_accessor optrAcc_; + write_accessor> optrAcc_; KParam out_; - read_accessor iptrAcc_; + read_accessor> iptrAcc_; KParam in_; const int wx_; const int wy_; @@ -158,8 +159,10 @@ void wrap_dilated(Param out, const Param in, const dim_t wx, auto Q = getQueue(); Q.submit([&](sycl::handler &h) { - sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init}; - sycl::accessor inAcc{*in.data, h, sycl::read_only}; + write_accessor> outAcc = + out.template get_accessor(h); + read_accessor> inAcc = + in.template get_accessor(h); h.parallel_for(sycl::nd_range{global, local}, wrapDilatedCreateKernel( outAcc, out.info, inAcc, in.info, wx, wy, sx, sy, px, diff --git a/src/backend/oneapi/lookup.cpp b/src/backend/oneapi/lookup.cpp index 9c87003375..da658e12aa 100644 --- a/src/backend/oneapi/lookup.cpp +++ b/src/backend/oneapi/lookup.cpp @@ -25,8 +25,8 @@ Array lookup(const Array &input, const Array &indices, const dim4 &iDims = input.dims(); dim4 oDims(1); - for (int d = 0; d < 4; ++d) { - oDims[d] = (d == int(dim) ? indices.elements() : iDims[d]); + for (dim_t d = 0; d < 4; ++d) { + oDims[d] = (d == dim ? indices.elements() : iDims[d]); } Array out = createEmptyArray(oDims); @@ -53,6 +53,8 @@ Array lookup(const Array &input, const Array &indices, const unsigned); \ template Array lookup(const Array &, const Array &, \ const unsigned); \ + template Array lookup(const Array &, const Array &, \ + const unsigned); \ template Array lookup(const Array &, const Array &, \ const unsigned); \ template Array lookup(const Array &, const Array &, \ @@ -66,6 +68,7 @@ INSTANTIATE(int); INSTANTIATE(unsigned); INSTANTIATE(intl); INSTANTIATE(uintl); +INSTANTIATE(schar); INSTANTIATE(uchar); INSTANTIATE(char); INSTANTIATE(ushort); diff --git a/src/backend/oneapi/lu.cpp b/src/backend/oneapi/lu.cpp index 200b85d23b..27e6bd4bf3 100644 --- a/src/backend/oneapi/lu.cpp +++ b/src/backend/oneapi/lu.cpp @@ -14,23 +14,22 @@ #include #include #include +#include +#include #include -#include "oneapi/mkl/lapack.hpp" namespace arrayfire { namespace oneapi { -Array convertPivot(sycl::buffer &pivot, int out_sz, +Array convertPivot(sycl::buffer &pivot, int in_sz, int out_sz, bool convert_pivot) { - dim_t d0 = pivot.get_range()[0]; - std::vector d_po(out_sz); for (int i = 0; i < out_sz; i++) { d_po[i] = i; } auto d_pi = pivot.get_host_access(); if (convert_pivot) { - for (int j = 0; j < d0; j++) { + for (int j = 0; j < in_sz; j++) { // 1 indexed in pivot std::swap(d_po[j], d_po[d_pi[j] - 1]); } @@ -38,10 +37,10 @@ Array convertPivot(sycl::buffer &pivot, int out_sz, Array res = createHostDataArray(dim4(out_sz), &d_po[0]); return res; } else { - d_po.resize(d0); - for (int j = 0; j < d0; j++) { d_po[j] = static_cast(d_pi[j]); } + d_po.resize(in_sz); + for (int j = 0; j < in_sz; j++) { d_po[j] = static_cast(d_pi[j]); } } - Array res = createHostDataArray(dim4(d0), &d_po[0]); + Array res = createHostDataArray(dim4(in_sz), &d_po[0]); return res; } @@ -76,13 +75,15 @@ Array lu_inplace(Array &in, const bool convert_pivot) { std::int64_t scratchpad_size = ::oneapi::mkl::lapack::getrf_scratchpad_size(getQueue(), M, N, LDA); - sycl::buffer ipiv{sycl::range<1>(MN)}; - Array scratch = createEmptyArray(af::dim4(scratchpad_size)); + auto ipiv = memAlloc(MN); + auto scratchpad = memAlloc>(scratchpad_size); - ::oneapi::mkl::lapack::getrf(getQueue(), M, N, *in.get(), LDA, ipiv, - *scratch.get(), scratchpad_size); + sycl::buffer> in_buffer = + in.template getBufferWithOffset>(); + ::oneapi::mkl::lapack::getrf(getQueue(), M, N, in_buffer, LDA, *ipiv, + *scratchpad, scratchpad->size()); - Array pivot = convertPivot(ipiv, M, convert_pivot); + Array pivot = convertPivot(*ipiv, MN, M, convert_pivot); return pivot; } diff --git a/src/backend/oneapi/match_template.cpp b/src/backend/oneapi/match_template.cpp index 28794ff2eb..10b84757ac 100644 --- a/src/backend/oneapi/match_template.cpp +++ b/src/backend/oneapi/match_template.cpp @@ -32,6 +32,7 @@ INSTANTIATE(float, float) INSTANTIATE(char, float) INSTANTIATE(int, float) INSTANTIATE(uint, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(short, float) INSTANTIATE(ushort, float) diff --git a/src/backend/oneapi/math.cpp b/src/backend/oneapi/math.cpp index a673f9293b..18bafd324b 100644 --- a/src/backend/oneapi/math.cpp +++ b/src/backend/oneapi/math.cpp @@ -12,43 +12,14 @@ namespace arrayfire { namespace oneapi { -cfloat operator+(cfloat lhs, cfloat rhs) { - // cfloat res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}}; - cfloat res; - return res; -} - -cdouble operator+(cdouble lhs, cdouble rhs) { - // cdouble res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}}; - cdouble res; - return res; -} - -cfloat operator*(cfloat lhs, cfloat rhs) { - cfloat out; - // out.s[0] = lhs.s[0] * rhs.s[0] - lhs.s[1] * rhs.s[1]; - // out.s[1] = lhs.s[0] * rhs.s[1] + lhs.s[1] * rhs.s[0]; - return out; -} - -cdouble operator*(cdouble lhs, cdouble rhs) { - cdouble out; - // out.s[0] = lhs.s[0] * rhs.s[0] - lhs.s[1] * rhs.s[1]; - // out.s[1] = lhs.s[0] * rhs.s[1] + lhs.s[1] * rhs.s[0]; - return out; -} cfloat division(cfloat lhs, double rhs) { - cfloat retVal; - // retVal.s[0] = real(lhs) / rhs; - // retVal.s[1] = imag(lhs) / rhs; + cfloat retVal(real(lhs) / rhs, imag(lhs) / rhs); return retVal; } cdouble division(cdouble lhs, double rhs) { - cdouble retVal; - // retVal.s[0] = real(lhs) / rhs; - // retVal.s[1] = imag(lhs) / rhs; + cdouble retVal(real(lhs) / rhs, imag(lhs) / rhs); return retVal; } } // namespace oneapi diff --git a/src/backend/oneapi/math.hpp b/src/backend/oneapi/math.hpp index 063d82f370..7362874442 100644 --- a/src/backend/oneapi/math.hpp +++ b/src/backend/oneapi/math.hpp @@ -18,6 +18,7 @@ #include #include +#include #include #if defined(__GNUC__) || defined(__GNUG__) @@ -71,6 +72,36 @@ inline cdouble min(cdouble lhs, cdouble rhs) { return abs(lhs) < abs(rhs) ? lhs : rhs; } +template +static inline auto is_nan(const T &val) -> bool { + return false; +} + +template<> +inline auto is_nan(const sycl::half &val) -> bool { + return sycl::isnan(val); +} + +template<> +inline auto is_nan(const float &val) -> bool { + return sycl::isnan(val); +} + +template<> +inline auto is_nan(const double &val) -> bool { + return sycl::isnan(val); +} + +template<> +inline auto is_nan(const cfloat &in) -> bool { + return sycl::isnan(real(in)) || sycl::isnan(imag(in)); +} + +template<> +inline auto is_nan(const cdouble &in) -> bool { + return sycl::isnan(real(in)) || sycl::isnan(imag(in)); +} + template static T scalar(double val) { return (T)(val); @@ -79,8 +110,6 @@ static T scalar(double val) { template<> inline cfloat scalar(double val) { cfloat cval(static_cast(val)); - // cval.real() = (float)val; - // cval.imag() = 0; return cval; } @@ -128,8 +157,8 @@ inline double minval() { return -std::numeric_limits::infinity(); } template<> -inline arrayfire::common::half minval() { - return -std::numeric_limits::infinity(); +inline sycl::half minval() { + return -1 * std::numeric_limits::infinity(); } template @@ -142,11 +171,6 @@ static inline T imag(T in) { return std::imag(in); } -inline arrayfire::common::half operator+(arrayfire::common::half lhs, - arrayfire::common::half rhs) noexcept { - return arrayfire::common::half(static_cast(lhs) + - static_cast(rhs)); -} } // namespace oneapi } // namespace arrayfire diff --git a/src/backend/oneapi/max.cpp b/src/backend/oneapi/max.cpp index 8b6ef71a10..fa21d78c1c 100644 --- a/src/backend/oneapi/max.cpp +++ b/src/backend/oneapi/max.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_max_t, uint, uint) INSTANTIATE(af_max_t, intl, intl) INSTANTIATE(af_max_t, uintl, uintl) INSTANTIATE(af_max_t, char, char) +INSTANTIATE(af_max_t, schar, schar) INSTANTIATE(af_max_t, uchar, uchar) INSTANTIATE(af_max_t, short, short) INSTANTIATE(af_max_t, ushort, ushort) diff --git a/src/backend/oneapi/mean.cpp b/src/backend/oneapi/mean.cpp index 09763bb739..2f94101f56 100644 --- a/src/backend/oneapi/mean.cpp +++ b/src/backend/oneapi/mean.cpp @@ -60,6 +60,7 @@ INSTANTIATE(intl, double, double); INSTANTIATE(uintl, double, double); INSTANTIATE(short, float, float); INSTANTIATE(ushort, float, float); +INSTANTIATE(schar, float, float); INSTANTIATE(uchar, float, float); INSTANTIATE(char, float, float); INSTANTIATE(cfloat, float, cfloat); diff --git a/src/backend/oneapi/meanshift.cpp b/src/backend/oneapi/meanshift.cpp index 1017b9074b..825b26eb88 100644 --- a/src/backend/oneapi/meanshift.cpp +++ b/src/backend/oneapi/meanshift.cpp @@ -38,6 +38,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/medfilt.cpp b/src/backend/oneapi/medfilt.cpp index 3b1ff319c5..50c2cc3dd8 100644 --- a/src/backend/oneapi/medfilt.cpp +++ b/src/backend/oneapi/medfilt.cpp @@ -59,6 +59,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp index 2b383b9520..3482742b73 100644 --- a/src/backend/oneapi/memory.cpp +++ b/src/backend/oneapi/memory.cpp @@ -62,7 +62,7 @@ template std::unique_ptr, std::function *)>> memAlloc(const size_t &elements) { if (elements) { - dim4 dims(elements * sizeof(T)); + dim4 dims(elements); // The alloc function returns a pointer to a buffer object. // We need to reinterpret that object into buffer while keeping the @@ -71,7 +71,7 @@ memAlloc(const size_t &elements) { // This would delete the buffer object and replace it with // the buffer object. We do the reverse in the memFree function auto *ptr = static_cast *>( - memoryManager().alloc(false, 1, dims.get(), 1)); + memoryManager().alloc(false, 1, dims.get(), sizeof(T))); sycl::buffer *optr = static_cast *>((void *)ptr); size_t bytes = ptr->byte_size(); @@ -104,14 +104,7 @@ void memFree(sycl::buffer *ptr) { } } -void memFreeUser(void *ptr) { - ONEAPI_NOT_SUPPORTED("memFreeUser Not supported"); - - // cl::Buffer *buf = static_cast(ptr); - // cl_mem mem = (*buf)(); - // delete buf; - memoryManager().unlock(ptr, true); -} +void memFreeUser(void *ptr) { memoryManager().unlock(ptr, true); } template void memLock(const sycl::buffer *ptr) { @@ -159,26 +152,26 @@ INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(short) INSTANTIATE(ushort) INSTANTIATE(arrayfire::common::half) +INSTANTIATE(int64_t) template<> void *pinnedAlloc(const size_t &elements) { - ONEAPI_NOT_SUPPORTED("pinnedAlloc Not supported"); - - // // TODO: make pinnedAlloc aware of array shapes - // dim4 dims(elements); - // void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), sizeof(T)); - return static_cast(nullptr); + // TODO: make pinnedAlloc aware of array shapes + dim4 dims(elements); + void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), 1); + return ptr; } Allocator::Allocator() { logger = common::loggerFactory("mem"); } -void Allocator::shutdown() {} +void Allocator::shutdown() { shutdownMemoryManager(); } int Allocator::getActiveDeviceId() { return oneapi::getActiveDeviceId(); } diff --git a/src/backend/oneapi/min.cpp b/src/backend/oneapi/min.cpp index ea9900543c..fe1a5a3fa4 100644 --- a/src/backend/oneapi/min.cpp +++ b/src/backend/oneapi/min.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_min_t, uint, uint) INSTANTIATE(af_min_t, intl, intl) INSTANTIATE(af_min_t, uintl, uintl) INSTANTIATE(af_min_t, char, char) +INSTANTIATE(af_min_t, schar, schar) INSTANTIATE(af_min_t, uchar, uchar) INSTANTIATE(af_min_t, short, short) INSTANTIATE(af_min_t, ushort, ushort) diff --git a/src/backend/oneapi/minmax_op.hpp b/src/backend/oneapi/minmax_op.hpp index f006ff419c..40159d3ec9 100644 --- a/src/backend/oneapi/minmax_op.hpp +++ b/src/backend/oneapi/minmax_op.hpp @@ -10,6 +10,7 @@ #pragma once #include +#include namespace arrayfire { namespace oneapi { @@ -34,21 +35,6 @@ double cabs(const cdouble &in) { return (double)abs(in); } -template -static bool is_nan(const T &in) { - return in != in; -} - -template<> -bool is_nan(const cfloat &in) { - return in.real() != in.real() || in.imag() != in.imag(); -} - -template<> -bool is_nan(const cdouble &in) { - return in.real() != in.real() || in.imag() != in.imag(); -} - template struct MinMaxOp { T m_val; diff --git a/src/backend/oneapi/moments.cpp b/src/backend/oneapi/moments.cpp index 50efe4ccd5..76e385990b 100644 --- a/src/backend/oneapi/moments.cpp +++ b/src/backend/oneapi/moments.cpp @@ -49,6 +49,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/morph.cpp b/src/backend/oneapi/morph.cpp index 44fe6a6529..11f3d3df7a 100644 --- a/src/backend/oneapi/morph.cpp +++ b/src/backend/oneapi/morph.cpp @@ -62,6 +62,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/nearest_neighbour.cpp b/src/backend/oneapi/nearest_neighbour.cpp index 7a34ba0fba..bec80b5cce 100644 --- a/src/backend/oneapi/nearest_neighbour.cpp +++ b/src/backend/oneapi/nearest_neighbour.cpp @@ -82,6 +82,7 @@ INSTANTIATE(intl, intl) INSTANTIATE(uintl, uintl) INSTANTIATE(short, int) INSTANTIATE(ushort, uint) +INSTANTIATE(schar, int) INSTANTIATE(uchar, uint) INSTANTIATE(uintl, uint) // For Hamming diff --git a/src/backend/oneapi/onefft.hpp b/src/backend/oneapi/onefft.hpp new file mode 100644 index 0000000000..a31a91d1e1 --- /dev/null +++ b/src/backend/oneapi/onefft.hpp @@ -0,0 +1,39 @@ +/******************************************************* + * Copyright (c) 2016, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ +#pragma once + +#include +#include +#include + +#include + +namespace arrayfire { +namespace oneapi { + +using ::oneapi::mkl::dft::domain; +using ::oneapi::mkl::dft::precision; + +using PlanType = std::shared_ptr; +using SharedPlan = std::shared_ptr; + +template +PlanType findPlan(int rank, const bool isInPlace, int *n, + std::int64_t *istrides, int ibatch, std::int64_t *ostrides, + int obatch, int nbatch); + +class PlanCache : public common::FFTPlanCache { + template + friend PlanType findPlan(int rank, const bool isInPlace, int *n, + std::int64_t *istrides, int ibatch, + std::int64_t *ostrides, int obatch, int nbatch); +}; + +} // namespace oneapi +} // namespace arrayfire diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp index d9b6f1d832..3994a907a5 100644 --- a/src/backend/oneapi/platform.cpp +++ b/src/backend/oneapi/platform.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -163,7 +164,9 @@ string getDeviceInfo() noexcept { << ", " << msize / 1048576 << " MB"; info << " ("; if (device->has(aspect::fp64)) { info << "fp64 "; } - if (device->has(aspect::fp16)) { info << "fp16 "; } + if (device->has(aspect::fp16) && + device->get_info() != 0) + { info << "fp16 "; } info << "\b)"; #ifndef NDEBUG info << " -- "; @@ -385,7 +388,8 @@ bool isHalfSupported(unsigned device) { DeviceManager& devMngr = DeviceManager::getInstance(); common::lock_guard_t lock(devMngr.deviceMutex); - return devMngr.mDevices[device]->has(sycl::aspect::fp16); + return devMngr.mDevices[device]->has(sycl::aspect::fp16) && + devMngr.mDevices[device]->get_info() != 0; } void devprop(char* d_name, char* d_platform, char* d_toolkit, char* d_compute) { @@ -605,7 +609,7 @@ void setMemoryManager(unique_ptr mgr) { } void resetMemoryManager() { - return DeviceManager::getInstance().resetMemoryManagerPinned(); + return DeviceManager::getInstance().resetMemoryManager(); } void setMemoryManagerPinned(unique_ptr mgr) { @@ -634,6 +638,16 @@ GraphicsResourceManager& interopManager() { return *(inst.gfxManagers[id].get()); } +unique_ptr& oneFFTManager(const int deviceId) { + thread_local unique_ptr caches[DeviceManager::MAX_DEVICES]; + thread_local once_flag initFlags[DeviceManager::MAX_DEVICES]; + call_once(initFlags[deviceId], + [&] { caches[deviceId] = make_unique(); }); + return caches[deviceId]; +} + +PlanCache& fftManager() { return *oneFFTManager(getActiveDeviceId()); } + } // namespace oneapi } // namespace arrayfire diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp index 86439a685c..bceb1e5db6 100644 --- a/src/backend/oneapi/platform.hpp +++ b/src/backend/oneapi/platform.hpp @@ -131,6 +131,8 @@ arrayfire::common::ForgeManager& forgeManager(); GraphicsResourceManager& interopManager(); +PlanCache& fftManager(); + // afcl::platform getPlatformEnum(cl::Device dev); void setActiveContext(int device); diff --git a/src/backend/oneapi/plot.cpp b/src/backend/oneapi/plot.cpp index d2fa041291..3bd287fbd6 100644 --- a/src/backend/oneapi/plot.cpp +++ b/src/backend/oneapi/plot.cpp @@ -78,6 +78,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace oneapi diff --git a/src/backend/oneapi/product.cpp b/src/backend/oneapi/product.cpp index bc3f9421ae..4aa9cb61dd 100644 --- a/src/backend/oneapi/product.cpp +++ b/src/backend/oneapi/product.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_mul_t, uint, uint) INSTANTIATE(af_mul_t, intl, intl) INSTANTIATE(af_mul_t, uintl, uintl) INSTANTIATE(af_mul_t, char, int) +INSTANTIATE(af_mul_t, schar, int) INSTANTIATE(af_mul_t, uchar, uint) INSTANTIATE(af_mul_t, short, int) INSTANTIATE(af_mul_t, ushort, uint) diff --git a/src/backend/oneapi/qr.cpp b/src/backend/oneapi/qr.cpp index 32bf559f4c..64884e4c24 100644 --- a/src/backend/oneapi/qr.cpp +++ b/src/backend/oneapi/qr.cpp @@ -11,94 +11,110 @@ #include -#if defined(WITH_LINEAR_ALGEBRA) && !defined(AF_ONEAPI) +#if defined(WITH_LINEAR_ALGEBRA) #include #include -#include #include -// #include -#include -#include -#include +#include +#include +#include #include namespace arrayfire { namespace oneapi { -template -void qr(Array &q, Array &r, Array &t, const Array &orig) { - if (OpenCLCPUOffload()) { return cpu::qr(q, r, t, orig); } - - const dim4 NullShape(0, 0, 0, 0); +using sycl::buffer; - dim4 iDims = orig.dims(); +template +void qr(Array &q, Array &r, Array &t, const Array &in) { + dim4 iDims = in.dims(); int M = iDims[0]; int N = iDims[1]; - dim4 endPadding(M - iDims[0], max(M, N) - iDims[1], 0, 0); - Array in = - (endPadding == NullShape - ? copyArray(orig) - : padArrayBorders(orig, NullShape, endPadding, AF_PAD_ZERO)); - in.resetDims(iDims); - - int MN = std::min(M, N); - int NB = magma_get_geqrf_nb(M); - - int NUM = (2 * MN + ((N + 31) / 32) * 32) * NB; - Array tmp = createEmptyArray(dim4(NUM)); - - std::vector h_tau(MN); - - int info = 0; - cl::Buffer *in_buf = in.get(); - cl::Buffer *dT = tmp.get(); - - magma_geqrf3_gpu(M, N, (*in_buf)(), in.getOffset(), in.strides()[1], - &h_tau[0], (*dT)(), tmp.getOffset(), getQueue()(), - &info); - - r = createEmptyArray(in.dims()); - kernel::triangle(r, in, true, false); - - cl::Buffer *r_buf = r.get(); - magmablas_swapdblk(MN - 1, NB, (*r_buf)(), r.getOffset(), r.strides()[1], - 1, (*dT)(), tmp.getOffset() + MN * NB, NB, 0, - getQueue()()); - - q = in; // No need to copy + Array in_copy = copyArray(in); + + // Get workspace needed for QR + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::geqrf_scratchpad_size>( + getQueue(), iDims[0], iDims[1], in_copy.strides()[1]); + + auto scratchpad = memAlloc>(scratchpad_size); + + t = createEmptyArray(af::dim4(min(M, N), 1, 1, 1)); + + buffer> iBuf = + in_copy.template getBufferWithOffset>(); + buffer> tBuf = t.template getBufferWithOffset>(); + ::oneapi::mkl::lapack::geqrf(getQueue(), M, N, iBuf, in_copy.strides()[1], + tBuf, *scratchpad, scratchpad->size()); + // SPLIT into q and r + dim4 rdims(M, N); + r = createEmptyArray(rdims); + + constexpr bool is_upper = true; + constexpr bool is_unit_diag = false; + kernel::triangle(r, in_copy, is_upper, is_unit_diag); + + int mn = max(M, N); + dim4 qdims(M, mn); + q = identity(qdims); + + buffer> qBuf = q.template getBufferWithOffset>(); + if constexpr (std::is_floating_point>()) { + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::ormqr_scratchpad_size>( + getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::nontrans, q.dims()[0], q.dims()[1], + min(M, N), in_copy.strides()[1], q.strides()[1]); + + auto scratchpad_ormqr = memAlloc>(scratchpad_size); + ::oneapi::mkl::lapack::ormqr( + getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::nontrans, q.dims()[0], q.dims()[1], + min(M, N), iBuf, in_copy.strides()[1], tBuf, qBuf, q.strides()[1], + *scratchpad_ormqr, scratchpad_ormqr->size()); + + } else if constexpr (common::isComplex(static_cast( + dtype_traits>::af_type))) { + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::unmqr_scratchpad_size>( + getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::nontrans, q.dims()[0], q.dims()[1], + min(M, N), in_copy.strides()[1], q.strides()[1]); + + auto scratchpad_ormqr = memAlloc>(scratchpad_size); + ::oneapi::mkl::lapack::unmqr( + getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::nontrans, q.dims()[0], q.dims()[1], + min(M, N), iBuf, in_copy.strides()[1], tBuf, qBuf, q.strides()[1], + *scratchpad_ormqr, scratchpad_ormqr->size()); + } q.resetDims(dim4(M, M)); - cl::Buffer *q_buf = q.get(); - - magma_ungqr_gpu(q.dims()[0], q.dims()[1], std::min(M, N), (*q_buf)(), - q.getOffset(), q.strides()[1], &h_tau[0], (*dT)(), - tmp.getOffset(), NB, getQueue()(), &info); - - t = createHostDataArray(dim4(MN), &h_tau[0]); } template Array qr_inplace(Array &in) { - if (OpenCLCPUOffload()) { return cpu::qr_inplace(in); } - - dim4 iDims = in.dims(); - int M = iDims[0]; - int N = iDims[1]; - int MN = std::min(M, N); - - getQueue().finish(); // FIXME: Does this need to be here? - cl::CommandQueue Queue2(getContext(), getDevice()); - cl_command_queue queues[] = {getQueue()(), Queue2()}; - - std::vector h_tau(MN); - cl::Buffer *in_buf = in.get(); - - int info = 0; - magma_geqrf2_gpu(M, N, (*in_buf)(), in.getOffset(), in.strides()[1], - &h_tau[0], queues, &info); - - Array t = createHostDataArray(dim4(MN), &h_tau[0]); + dim4 iDims = in.dims(); + dim4 iStrides = in.strides(); + int M = iDims[0]; + int N = iDims[1]; + + Array t = createEmptyArray(af::dim4(min(M, N), 1, 1, 1)); + + // Get workspace needed for QR + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::geqrf_scratchpad_size>( + getQueue(), iDims[0], iDims[1], iStrides[1]); + + auto scratchpad = memAlloc>(scratchpad_size); + + buffer> iBuf = in.template getBufferWithOffset>(); + buffer> tBuf = t.template getBufferWithOffset>(); + // In place Perform in place QR + ::oneapi::mkl::lapack::geqrf(getQueue(), iDims[0], iDims[1], iBuf, + iStrides[1], tBuf, *scratchpad, + scratchpad->size()); return t; } diff --git a/src/backend/oneapi/random_engine.cpp b/src/backend/oneapi/random_engine.cpp index 7045dcc8cc..e3eac5da0b 100644 --- a/src/backend/oneapi/random_engine.cpp +++ b/src/backend/oneapi/random_engine.cpp @@ -92,6 +92,7 @@ INSTANTIATE_UNIFORM(uint) INSTANTIATE_UNIFORM(intl) INSTANTIATE_UNIFORM(uintl) INSTANTIATE_UNIFORM(char) +INSTANTIATE_UNIFORM(schar) INSTANTIATE_UNIFORM(uchar) INSTANTIATE_UNIFORM(short) INSTANTIATE_UNIFORM(ushort) diff --git a/src/backend/oneapi/range.cpp b/src/backend/oneapi/range.cpp index caa8ed48bc..c08a7bea91 100644 --- a/src/backend/oneapi/range.cpp +++ b/src/backend/oneapi/range.cpp @@ -48,6 +48,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/reduce_impl.hpp b/src/backend/oneapi/reduce_impl.hpp index 14b5a9e269..b2c478c71f 100644 --- a/src/backend/oneapi/reduce_impl.hpp +++ b/src/backend/oneapi/reduce_impl.hpp @@ -6,11 +6,26 @@ * The complete license agreement can be obtained at: * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +#pragma once + +#if defined(__clang__) +#pragma clang diagnostic push +// temporary ignores for DPL internals +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + +// oneDPL headers should be included before standard headers +#define ONEDPL_USE_PREDEFINED_POLICIES 0 +#include +#include +#include #include #include +#include #include -// #include +#include #include #include #include @@ -31,11 +46,579 @@ Array reduce(const Array &in, const int dim, bool change_nan, return out; } +template +void reduceBlocksByKey(sycl::buffer &reduced_block_sizes, + Array keys_out, Array vals_out, + const Array keys, const Array vals, + int change_nan, double nanval, const int n, + const int threads_x) { + int numBlocks = divup(n, threads_x); + + sycl::range<3> local(threads_x, 1, 1); + sycl::range<3> global(local[0] * numBlocks, vals_out.dims()[1], + vals_out.dims()[2] * vals_out.dims()[3]); + + auto keys_out_get = keys_out.get(); + auto vals_out_get = vals_out.get(); + auto keys_get = keys.get(); + auto vals_get = vals.get(); + getQueue().submit([&](sycl::handler &h) { + sycl::accessor reduced_block_sizes_acc{reduced_block_sizes, h}; + write_accessor keys_out_acc{*keys_out_get, h}; + write_accessor vals_out_acc{*vals_out_get, h}; + read_accessor keys_acc{*keys_get, h}; + read_accessor vals_acc{*vals_get, h}; + + auto l_keys = sycl::local_accessor(threads_x, h); + auto l_vals = sycl::local_accessor>(threads_x, h); + auto l_reduced_keys = sycl::local_accessor(threads_x, h); + auto l_reduced_vals = sycl::local_accessor>(threads_x, h); + auto l_unique_ids = sycl::local_accessor(threads_x, h); + auto l_wq_temp = sycl::local_accessor(threads_x, h); + auto l_unique_flags = sycl::local_accessor(threads_x, h); + auto l_reduced_block_size = sycl::local_accessor(1, h); + + h.parallel_for( + sycl::nd_range<3>(global, local), + kernel::reduceBlocksByKeyKernel( + reduced_block_sizes_acc, keys_out_acc, keys_out, vals_out_acc, + vals_out, keys_acc, keys, vals_acc, vals, change_nan, + scalar(nanval), n, static_cast(vals_out.dims()[2]), + threads_x, l_keys, l_vals, l_reduced_keys, l_reduced_vals, + l_unique_ids, l_wq_temp, l_unique_flags, l_reduced_block_size)); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +void reduceBlocksByKeyDim(sycl::buffer &reduced_block_sizes, + Array keys_out, Array vals_out, + const Array keys, const Array vals, + int change_nan, double nanval, const int n, + const int threads_x, const int dim, + std::vector dim_ordering) { + int numBlocks = divup(n, threads_x); + + sycl::range<3> local(threads_x, 1, 1); + sycl::range<3> global( + local[0] * numBlocks, vals_out.dims()[dim_ordering[1]], + vals_out.dims()[dim_ordering[2]] * vals_out.dims()[dim_ordering[3]]); + + auto keys_out_get = keys_out.get(); + auto vals_out_get = vals_out.get(); + auto keys_get = keys.get(); + auto vals_get = vals.get(); + getQueue().submit([&](sycl::handler &h) { + sycl::accessor reduced_block_sizes_acc{reduced_block_sizes, h}; + write_accessor keys_out_acc{*keys_out_get, h}; + write_accessor vals_out_acc{*vals_out_get, h}; + read_accessor keys_acc{*keys_get, h}; + read_accessor vals_acc{*vals_get, h}; + + auto l_keys = sycl::local_accessor(threads_x, h); + auto l_vals = sycl::local_accessor>(threads_x, h); + auto l_reduced_keys = sycl::local_accessor(threads_x, h); + auto l_reduced_vals = sycl::local_accessor>(threads_x, h); + auto l_unique_ids = sycl::local_accessor(threads_x, h); + auto l_wq_temp = sycl::local_accessor(threads_x, h); + auto l_unique_flags = sycl::local_accessor(threads_x, h); + auto l_reduced_block_size = sycl::local_accessor(1, h); + + h.parallel_for( + sycl::nd_range<3>(global, local), + kernel::reduceBlocksByKeyDimKernel( + reduced_block_sizes_acc, keys_out_acc, keys_out, vals_out_acc, + vals_out, keys_acc, keys, vals_acc, vals, change_nan, + scalar(nanval), n, static_cast(vals_out.dims()[2]), + threads_x, dim, l_keys, l_vals, l_reduced_keys, l_reduced_vals, + l_unique_ids, l_wq_temp, l_unique_flags, l_reduced_block_size)); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +void finalBoundaryReduce(sycl::buffer &reduced_block_sizes, Array keys, + Array vals_out, const int n, const int numBlocks, + const int threads_x) { + sycl::range<1> local(threads_x); + sycl::range<1> global(local[0] * numBlocks); + + auto vals_out_get = vals_out.get(); + auto keys_get = keys.get(); + getQueue().submit([&](sycl::handler &h) { + write_accessor reduced_block_sizes_acc{reduced_block_sizes, h}; + read_accessor keys_acc{*keys_get, h}; + sycl::accessor vals_out_acc{*vals_out_get, h}; + + h.parallel_for(sycl::nd_range<1>(global, local), + kernel::finalBoundaryReduceKernel( + reduced_block_sizes_acc, keys_acc, keys, + vals_out_acc, vals_out, n)); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +void finalBoundaryReduceDim(sycl::buffer &reduced_block_sizes, + Array keys, Array vals_out, const int n, + const int numBlocks, const int threads_x, + const int dim, std::vector dim_ordering) { + sycl::range<3> local(threads_x, 1, 1); + sycl::range<3> global( + local[0] * numBlocks, vals_out.dims()[dim_ordering[1]], + vals_out.dims()[dim_ordering[2]] * vals_out.dims()[dim_ordering[3]]); + + auto vals_out_get = vals_out.get(); + auto keys_get = keys.get(); + getQueue().submit([&](sycl::handler &h) { + write_accessor reduced_block_sizes_acc{reduced_block_sizes, h}; + read_accessor keys_acc{*keys_get, h}; + sycl::accessor vals_out_acc{*vals_out_get, h}; + + // TODO: fold 3,4 dimensions + h.parallel_for( + sycl::nd_range<3>(global, local), + kernel::finalBoundaryReduceDimKernel( + reduced_block_sizes_acc, keys_acc, keys, vals_out_acc, vals_out, + n, vals_out.dims()[dim_ordering[2]])); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +void compact(sycl::buffer reduced_block_sizes, Array &keys_out, + Array &vals_out, const Array &keys, const Array &vals, + const int numBlocks, const int threads_x) { + sycl::range<3> local(threads_x, 1, 1); + sycl::range<3> global(local[0] * numBlocks, vals_out.dims()[1], + vals_out.dims()[2] * vals_out.dims()[3]); + + auto keys_out_get = keys_out.get(); + auto vals_out_get = vals_out.get(); + auto keys_get = keys.get(); + auto vals_get = vals.get(); + getQueue().submit([&](sycl::handler &h) { + read_accessor reduced_block_sizes_acc{reduced_block_sizes, h}; + write_accessor keys_out_acc{*keys_out_get, h}; + write_accessor vals_out_acc{*vals_out_get, h}; + read_accessor keys_acc{*keys_get, h}; + read_accessor vals_acc{*vals_get, h}; + + h.parallel_for(sycl::nd_range<3>(global, local), + kernel::compactKernel( + reduced_block_sizes_acc, keys_out_acc, keys_out, + vals_out_acc, vals_out, keys_acc, keys, vals_acc, + vals, static_cast(vals_out.dims()[2]))); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +void compactDim(sycl::buffer &reduced_block_sizes, Array &keys_out, + Array &vals_out, const Array &keys, + const Array &vals, const int numBlocks, const int threads_x, + const int dim, std::vector dim_ordering) { + sycl::range<3> local(threads_x, 1, 1); + sycl::range<3> global( + local[0] * numBlocks, vals_out.dims()[dim_ordering[1]], + vals_out.dims()[dim_ordering[2]] * vals_out.dims()[dim_ordering[3]]); + + auto keys_out_get = keys_out.get(); + auto vals_out_get = vals_out.get(); + auto keys_get = keys.get(); + auto vals_get = vals.get(); + getQueue().submit([&](sycl::handler &h) { + read_accessor reduced_block_sizes_acc{reduced_block_sizes, h}; + write_accessor keys_out_acc{*keys_out_get, h}; + write_accessor vals_out_acc{*vals_out_get, h}; + read_accessor keys_acc{*keys_get, h}; + read_accessor vals_acc{*vals_get, h}; + + h.parallel_for( + sycl::nd_range<3>(global, local), + kernel::compactDimKernel( + reduced_block_sizes_acc, keys_out_acc, keys_out, vals_out_acc, + vals_out, keys_acc, keys, vals_acc, vals, + static_cast(vals_out.dims()[dim_ordering[2]]), dim)); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +void testNeedsReduction(sycl::buffer needs_reduction, + sycl::buffer needs_boundary, const Array &keys, + const int n, const int numBlocks, const int threads_x) { + sycl::range<1> local(threads_x); + sycl::range<1> global(local[0] * numBlocks); + + auto keys_get = keys.get(); + getQueue().submit([&](sycl::handler &h) { + sycl::accessor needs_reduction_acc{needs_reduction, h}; + sycl::accessor needs_boundary_acc{needs_boundary, h}; + read_accessor keys_acc{*keys_get, h}; + auto l_keys = sycl::local_accessor(threads_x, h); + + h.parallel_for(sycl::nd_range<1>(global, local), + kernel::testNeedsReductionKernel( + needs_reduction_acc, needs_boundary_acc, keys_acc, + keys, n, threads_x, l_keys)); + }); + ONEAPI_DEBUG_FINISH(getQueue()); +} + +template +int reduce_by_key_first(Array &keys_out, Array &vals_out, + const Array &keys, const Array &vals, + bool change_nan, double nanval) { + auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue()); + + dim4 kdims = keys.dims(); + dim4 odims = vals.dims(); + + Array reduced_keys = createEmptyArray(kdims); + Array reduced_vals = createEmptyArray(odims); + Array t_reduced_keys = createEmptyArray(kdims); + Array t_reduced_vals = createEmptyArray(odims); + + // flags determining more reduction is necessary + auto needs_another_reduction = memAlloc(1); + auto needs_block_boundary_reduction = memAlloc(1); + + // reset flags + getQueue().submit([&](sycl::handler &h) { + auto wacc = + needs_another_reduction->get_access(h); + h.fill(wacc, 0); + }); + getQueue().submit([&](sycl::handler &h) { + auto wacc = needs_block_boundary_reduction + ->get_access(h); + h.fill(wacc, 0); + }); + + size_t nelems = kdims[0]; + + const unsigned int numThreads = 128; + int numBlocksD0 = divup(nelems, numThreads); + auto reduced_block_sizes = memAlloc(numBlocksD0); + + int n_reduced_host = nelems; + + int needs_another_reduction_host = 0; + int needs_block_boundary_reduction_host = 0; + + bool first_pass = true; + do { + numBlocksD0 = divup(n_reduced_host, numThreads); + + if (first_pass) { + reduceBlocksByKey( + *reduced_block_sizes.get(), reduced_keys, reduced_vals, keys, + vals, change_nan, nanval, n_reduced_host, numThreads); + first_pass = false; + } else { + constexpr af_op_t op2 = (op == af_notzero_t) ? af_add_t : op; + reduceBlocksByKey( + *reduced_block_sizes.get(), reduced_keys, reduced_vals, + t_reduced_keys, t_reduced_vals, change_nan, nanval, + n_reduced_host, numThreads); + } + + auto val_buf_begin = ::oneapi::dpl::begin(*reduced_block_sizes.get()); + auto val_buf_end = val_buf_begin + numBlocksD0; + std::inclusive_scan(dpl_policy, val_buf_begin, val_buf_end, + val_buf_begin); + + compact(*reduced_block_sizes.get(), t_reduced_keys, + t_reduced_vals, reduced_keys, reduced_vals, numBlocksD0, + numThreads); + + sycl::event reduce_host_event = + getQueue().submit([&](sycl::handler &h) { + sycl::range rr(1); + sycl::id offset_id(numBlocksD0 - 1); + auto offset_acc = + reduced_block_sizes + ->template get_access( + h, rr, offset_id); + h.copy(offset_acc, &n_reduced_host); + }); + + // reset flags + getQueue().submit([&](sycl::handler &h) { + auto wacc = + needs_another_reduction->get_access( + h); + h.fill(wacc, 0); + }); + getQueue().submit([&](sycl::handler &h) { + auto wacc = needs_block_boundary_reduction + ->get_access(h); + h.fill(wacc, 0); + }); + + reduce_host_event.wait(); + + numBlocksD0 = divup(n_reduced_host, numThreads); + + testNeedsReduction(*needs_another_reduction.get(), + *needs_block_boundary_reduction.get(), + t_reduced_keys, n_reduced_host, numBlocksD0, + numThreads); + + sycl::event host_flag0_event = getQueue().submit([&](sycl::handler &h) { + sycl::range rr(1); + auto acc = + needs_another_reduction + ->template get_access(h, rr); + h.copy(acc, &needs_another_reduction_host); + }); + sycl::event host_flag1_event = getQueue().submit([&](sycl::handler &h) { + sycl::range rr(1); + auto acc = + needs_block_boundary_reduction + ->template get_access(h, rr); + h.copy(acc, &needs_block_boundary_reduction_host); + }); + + host_flag1_event.wait(); + host_flag0_event.wait(); + + if (needs_block_boundary_reduction_host && + !needs_another_reduction_host) { + finalBoundaryReduce( + *reduced_block_sizes.get(), t_reduced_keys, t_reduced_vals, + n_reduced_host, numBlocksD0, numThreads); + + auto val_buf_begin = + ::oneapi::dpl::begin(*reduced_block_sizes.get()); + auto val_buf_end = val_buf_begin + numBlocksD0; + std::inclusive_scan(dpl_policy, val_buf_begin, val_buf_end, + val_buf_begin); + + sycl::event reduce_host_event = + getQueue().submit([&](sycl::handler &h) { + sycl::range rr(1); + sycl::id offset_id(numBlocksD0 - 1); + auto offset_acc = + reduced_block_sizes + ->template get_access( + h, rr, offset_id); + h.copy(offset_acc, &n_reduced_host); + }); + + compact(*reduced_block_sizes.get(), reduced_keys, + reduced_vals, t_reduced_keys, t_reduced_vals, + numBlocksD0, numThreads); + + std::swap(t_reduced_keys, reduced_keys); + std::swap(t_reduced_vals, reduced_vals); + reduce_host_event.wait(); + } + } while (needs_another_reduction_host || + needs_block_boundary_reduction_host); + + keys_out = t_reduced_keys; + vals_out = t_reduced_vals; + return n_reduced_host; +} + +template +int reduce_by_key_dim(Array &keys_out, Array &vals_out, + const Array &keys, const Array &vals, + bool change_nan, double nanval, const int dim) { + auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue()); + + std::vector dim_ordering = {dim}; + for (int i = 0; i < 4; ++i) { + if (i != dim) { dim_ordering.push_back(i); } + } + + dim4 kdims = keys.dims(); + dim4 odims = vals.dims(); + + Array reduced_keys = createEmptyArray(kdims); + Array reduced_vals = createEmptyArray(odims); + Array t_reduced_keys = createEmptyArray(kdims); + Array t_reduced_vals = createEmptyArray(odims); + + // flags determining more reduction is necessary + auto needs_another_reduction = memAlloc(1); + auto needs_block_boundary_reduction = memAlloc(1); + + // reset flags + getQueue().submit([&](sycl::handler &h) { + auto wacc = + needs_another_reduction->get_access(h); + h.fill(wacc, 0); + }); + getQueue().submit([&](sycl::handler &h) { + auto wacc = needs_block_boundary_reduction + ->get_access(h); + h.fill(wacc, 0); + }); + + int nelems = kdims[0]; + + const unsigned int numThreads = 128; + int numBlocksD0 = divup(nelems, numThreads); + auto reduced_block_sizes = memAlloc(numBlocksD0); + + int n_reduced_host = nelems; + + int needs_another_reduction_host = 0; + int needs_block_boundary_reduction_host = 0; + + bool first_pass = true; + do { + numBlocksD0 = divup(n_reduced_host, numThreads); + + if (first_pass) { + reduceBlocksByKeyDim( + *reduced_block_sizes.get(), reduced_keys, reduced_vals, keys, + vals, change_nan, nanval, n_reduced_host, numThreads, dim, + dim_ordering); + first_pass = false; + } else { + constexpr af_op_t op2 = op == af_notzero_t ? af_add_t : op; + reduceBlocksByKeyDim( + *reduced_block_sizes.get(), reduced_keys, reduced_vals, + t_reduced_keys, t_reduced_vals, change_nan, nanval, + n_reduced_host, numThreads, dim, dim_ordering); + } + + auto val_buf_begin = ::oneapi::dpl::begin(*reduced_block_sizes.get()); + auto val_buf_end = val_buf_begin + numBlocksD0; + std::inclusive_scan(dpl_policy, val_buf_begin, val_buf_end, + val_buf_begin); + + compactDim(*reduced_block_sizes.get(), t_reduced_keys, + t_reduced_vals, reduced_keys, reduced_vals, + numBlocksD0, numThreads, dim, dim_ordering); + + sycl::event reduce_host_event = + getQueue().submit([&](sycl::handler &h) { + sycl::range rr(1); + sycl::id offset_id(numBlocksD0 - 1); + auto offset_acc = + reduced_block_sizes + ->template get_access( + h, rr, offset_id); + h.copy(offset_acc, &n_reduced_host); + }); + + // reset flags + getQueue().submit([&](sycl::handler &h) { + auto wacc = + needs_another_reduction->get_access( + h); + h.fill(wacc, 0); + }); + getQueue().submit([&](sycl::handler &h) { + auto wacc = needs_block_boundary_reduction + ->get_access(h); + h.fill(wacc, 0); + }); + + reduce_host_event.wait(); + + numBlocksD0 = divup(n_reduced_host, numThreads); + + testNeedsReduction(*needs_another_reduction.get(), + *needs_block_boundary_reduction.get(), + t_reduced_keys, n_reduced_host, numBlocksD0, + numThreads); + + sycl::event host_flag0_event = getQueue().submit([&](sycl::handler &h) { + sycl::range rr(1); + auto acc = + needs_another_reduction + ->template get_access(h, rr); + h.copy(acc, &needs_another_reduction_host); + }); + sycl::event host_flag1_event = getQueue().submit([&](sycl::handler &h) { + sycl::range rr(1); + auto acc = + needs_block_boundary_reduction + ->template get_access(h, rr); + h.copy(acc, &needs_block_boundary_reduction_host); + }); + + host_flag1_event.wait(); + host_flag0_event.wait(); + + if (needs_block_boundary_reduction_host && + !needs_another_reduction_host) { + finalBoundaryReduceDim( + *reduced_block_sizes.get(), t_reduced_keys, t_reduced_vals, + n_reduced_host, numBlocksD0, numThreads, dim, dim_ordering); + + auto val_buf_begin = + ::oneapi::dpl::begin(*reduced_block_sizes.get()); + auto val_buf_end = val_buf_begin + numBlocksD0; + std::inclusive_scan(dpl_policy, val_buf_begin, val_buf_end, + val_buf_begin); + + sycl::event reduce_host_event = + getQueue().submit([&](sycl::handler &h) { + sycl::range rr(1); + sycl::id offset_id(numBlocksD0 - 1); + auto offset_acc = + reduced_block_sizes + ->template get_access( + h, rr, offset_id); + h.copy(offset_acc, &n_reduced_host); + }); + + compactDim(*reduced_block_sizes.get(), reduced_keys, + reduced_vals, t_reduced_keys, t_reduced_vals, + numBlocksD0, numThreads, dim, dim_ordering); + + std::swap(t_reduced_keys, reduced_keys); + std::swap(t_reduced_vals, reduced_vals); + reduce_host_event.wait(); + } + } while (needs_another_reduction_host || + needs_block_boundary_reduction_host); + + keys_out = t_reduced_keys; + vals_out = t_reduced_vals; + + return n_reduced_host; +} + template void reduce_by_key(Array &keys_out, Array &vals_out, const Array &keys, const Array &vals, const int dim, bool change_nan, double nanval) { - ONEAPI_NOT_SUPPORTED(""); + dim4 kdims = keys.dims(); + dim4 odims = vals.dims(); + + // prepare output arrays + Array reduced_keys = createEmptyArray(dim4()); + Array reduced_vals = createEmptyArray(dim4()); + + size_t n_reduced = 0; + if (dim == 0) { + n_reduced = reduce_by_key_first( + reduced_keys, reduced_vals, keys, vals, change_nan, nanval); + } else { + n_reduced = reduce_by_key_dim( + reduced_keys, reduced_vals, keys, vals, change_nan, nanval, dim); + } + + kdims[0] = n_reduced; + odims[dim] = n_reduced; + std::vector kindex, vindex; + for (int i = 0; i < odims.ndims(); ++i) { + af_seq sk = {0.0, (double)kdims[i] - 1, 1.0}; + af_seq sv = {0.0, (double)odims[i] - 1, 1.0}; + kindex.push_back(sk); + vindex.push_back(sv); + } + + keys_out = createSubArray(reduced_keys, kindex, true); + vals_out = createSubArray(reduced_vals, vindex, true); } template @@ -59,3 +642,8 @@ Array reduce_all(const Array &in, bool change_nan, double nanval) { const Array &vals, const int dim, bool change_nan, double nanval); \ template Array reduce_all(const Array &in, \ bool change_nan, double nanval); + +#if defined(__clang__) +/* Clang/LLVM */ +#pragma clang diagnostic pop +#endif diff --git a/src/backend/oneapi/reorder.cpp b/src/backend/oneapi/reorder.cpp index d62db984e9..d9e264f70c 100644 --- a/src/backend/oneapi/reorder.cpp +++ b/src/backend/oneapi/reorder.cpp @@ -40,6 +40,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) diff --git a/src/backend/oneapi/reshape.cpp b/src/backend/oneapi/reshape.cpp index 8f1b6f0ecb..2b15f686e9 100644 --- a/src/backend/oneapi/reshape.cpp +++ b/src/backend/oneapi/reshape.cpp @@ -50,6 +50,8 @@ Array reshape(const Array &in, const dim4 &outDims, dim4 const &, short, double); \ template Array reshape( \ Array const &, dim4 const &, ushort, double); \ + template Array reshape(Array const &, \ + dim4 const &, schar, double); \ template Array reshape(Array const &, \ dim4 const &, uchar, double); \ template Array reshape(Array const &, \ @@ -65,6 +67,7 @@ INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(half) diff --git a/src/backend/oneapi/resize.cpp b/src/backend/oneapi/resize.cpp index 005faf6b2b..b73f42eabb 100644 --- a/src/backend/oneapi/resize.cpp +++ b/src/backend/oneapi/resize.cpp @@ -40,6 +40,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/oneapi/rotate.cpp b/src/backend/oneapi/rotate.cpp index 10f1f93480..bcd7b5810a 100644 --- a/src/backend/oneapi/rotate.cpp +++ b/src/backend/oneapi/rotate.cpp @@ -50,6 +50,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/oneapi/scan.cpp b/src/backend/oneapi/scan.cpp index f7151ce076..9aaae59b49 100644 --- a/src/backend/oneapi/scan.cpp +++ b/src/backend/oneapi/scan.cpp @@ -45,6 +45,7 @@ Array scan(const Array& in, const int dim, bool inclusiveScan) { INSTANTIATE_SCAN(ROp, intl, intl) \ INSTANTIATE_SCAN(ROp, uintl, uintl) \ INSTANTIATE_SCAN(ROp, char, uint) \ + INSTANTIATE_SCAN(ROp, schar, int) \ INSTANTIATE_SCAN(ROp, uchar, uint) \ INSTANTIATE_SCAN(ROp, short, int) \ INSTANTIATE_SCAN(ROp, ushort, uint) diff --git a/src/backend/oneapi/select.cpp b/src/backend/oneapi/select.cpp index 8cb80c919d..b24b1fa340 100644 --- a/src/backend/oneapi/select.cpp +++ b/src/backend/oneapi/select.cpp @@ -128,6 +128,7 @@ INSTANTIATE(uint); INSTANTIATE(intl); INSTANTIATE(uintl); INSTANTIATE(char); +INSTANTIATE(schar); INSTANTIATE(uchar); INSTANTIATE(short); INSTANTIATE(ushort); diff --git a/src/backend/oneapi/set.cpp b/src/backend/oneapi/set.cpp index a76363f10b..4c4b68e4b0 100644 --- a/src/backend/oneapi/set.cpp +++ b/src/backend/oneapi/set.cpp @@ -6,6 +6,11 @@ * The complete license agreement can be obtained at: * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +// oneDPL headers should be included before standard headers +#define ONEDPL_USE_PREDEFINED_POLICIES 0 +#include +#include +#include #include #include @@ -30,115 +35,84 @@ using type_t = template Array setUnique(const Array &in, const bool is_sorted) { - ONEAPI_NOT_SUPPORTED("setUnique Not supported"); - return createEmptyArray(dim4(1, 1, 1, 1)); + auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue()); - // try { - // Array out = copyArray(in); + Array out = copyArray(in); - // compute::command_queue queue(getQueue()()); + auto out_begin = ::oneapi::dpl::begin(*out.get()); + auto out_end = out_begin + out.elements(); - // compute::buffer out_data((*out.get())()); + if (!is_sorted) { + std::sort(dpl_policy, out_begin, out_end, + [](auto lhs, auto rhs) { return lhs < rhs; }); + } - // compute::buffer_iterator> begin(out_data, 0); - // compute::buffer_iterator> end(out_data, out.elements()); + out_end = std::unique(dpl_policy, out_begin, out_end); - // if (!is_sorted) { compute::sort(begin, end, queue); } + out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1)); - // end = compute::unique(begin, end, queue); - - // out.resetDims(dim4(std::distance(begin, end), 1, 1, 1)); - - // return out; - // } catch (const std::exception &ex) { AF_ERROR(ex.what(), - // AF_ERR_INTERNAL); } + return out; } template Array setUnion(const Array &first, const Array &second, const bool is_unique) { - ONEAPI_NOT_SUPPORTED("setUnion Not supported"); - return createEmptyArray(dim4(1, 1, 1, 1)); - - // try { - // Array unique_first = first; - // Array unique_second = second; + Array unique_first = first; + Array unique_second = second; - // if (!is_unique) { - // unique_first = setUnique(first, false); - // unique_second = setUnique(second, false); - // } + if (!is_unique) { + unique_first = setUnique(first, false); + unique_second = setUnique(second, false); + } - // size_t out_size = unique_first.elements() + unique_second.elements(); - // Array out = createEmptyArray(dim4(out_size, 1, 1, 1)); + size_t out_size = unique_first.elements() + unique_second.elements(); + Array out = createEmptyArray(dim4(out_size, 1, 1, 1)); - // compute::command_queue queue(getQueue()()); + auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue()); - // compute::buffer first_data((*unique_first.get())()); - // compute::buffer second_data((*unique_second.get())()); - // compute::buffer out_data((*out.get())()); + auto first_begin = ::oneapi::dpl::begin(*unique_first.get()); + auto first_end = first_begin + unique_first.elements(); - // compute::buffer_iterator> first_begin(first_data, 0); - // compute::buffer_iterator> first_end(first_data, - // unique_first.elements()); - // compute::buffer_iterator> second_begin(second_data, 0); - // compute::buffer_iterator> second_end( - // second_data, unique_second.elements()); - // compute::buffer_iterator> out_begin(out_data, 0); + auto second_begin = ::oneapi::dpl::begin(*unique_second.get()); + auto second_end = second_begin + unique_second.elements(); - // compute::buffer_iterator> out_end = compute::set_union( - // first_begin, first_end, second_begin, second_end, out_begin, - // queue); + auto out_begin = ::oneapi::dpl::begin(*out.get()); - // out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1)); - // return out; - - // } catch (const std::exception &ex) { AF_ERROR(ex.what(), - // AF_ERR_INTERNAL); } + auto out_end = std::set_union(dpl_policy, first_begin, first_end, + second_begin, second_end, out_begin); + out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1)); + return out; } template Array setIntersect(const Array &first, const Array &second, const bool is_unique) { - ONEAPI_NOT_SUPPORTED("setIntersect Not supported"); - return createEmptyArray(dim4(1, 1, 1, 1)); - - // try { - // Array unique_first = first; - // Array unique_second = second; - - // if (!is_unique) { - // unique_first = setUnique(first, false); - // unique_second = setUnique(second, false); - // } - - // size_t out_size = - // std::max(unique_first.elements(), unique_second.elements()); - // Array out = createEmptyArray(dim4(out_size, 1, 1, 1)); - - // compute::command_queue queue(getQueue()()); - - // compute::buffer first_data((*unique_first.get())()); - // compute::buffer second_data((*unique_second.get())()); - // compute::buffer out_data((*out.get())()); - - // compute::buffer_iterator> first_begin(first_data, 0); - // compute::buffer_iterator> first_end(first_data, - // unique_first.elements()); - // compute::buffer_iterator> second_begin(second_data, 0); - // compute::buffer_iterator> second_end( - // second_data, unique_second.elements()); - // compute::buffer_iterator> out_begin(out_data, 0); - - // compute::buffer_iterator> out_end = - // compute::set_intersection( - // first_begin, first_end, second_begin, second_end, out_begin, - // queue); - - // out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1)); - // return out; - // } catch (const std::exception &ex) { AF_ERROR(ex.what(), - // AF_ERR_INTERNAL); } + Array unique_first = first; + Array unique_second = second; + + if (!is_unique) { + unique_first = setUnique(first, false); + unique_second = setUnique(second, false); + } + + size_t out_size = + std::max(unique_first.elements(), unique_second.elements()); + Array out = createEmptyArray(dim4(out_size, 1, 1, 1)); + + auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue()); + + auto first_begin = ::oneapi::dpl::begin(*unique_first.get()); + auto first_end = first_begin + unique_first.elements(); + + auto second_begin = ::oneapi::dpl::begin(*unique_second.get()); + auto second_end = second_begin + unique_second.elements(); + + auto out_begin = ::oneapi::dpl::begin(*out.get()); + + auto out_end = std::set_intersection(dpl_policy, first_begin, first_end, + second_begin, second_end, out_begin); + out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1)); + return out; } #define INSTANTIATE(T) \ @@ -153,6 +127,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/set.hpp b/src/backend/oneapi/set.hpp index 85d3386489..beef4a44b4 100644 --- a/src/backend/oneapi/set.hpp +++ b/src/backend/oneapi/set.hpp @@ -6,6 +6,7 @@ * The complete license agreement can be obtained at: * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +#pragma once #include diff --git a/src/backend/oneapi/shift.cpp b/src/backend/oneapi/shift.cpp index d72477c770..7e5e31bf37 100644 --- a/src/backend/oneapi/shift.cpp +++ b/src/backend/oneapi/shift.cpp @@ -23,13 +23,11 @@ using std::string; namespace arrayfire { namespace oneapi { +template +using ShiftNode = ShiftNodeBase>; template Array shift(const Array &in, const int sdims[4]) { - ONEAPI_NOT_SUPPORTED(""); - Array o = createEmptyArray(dim4(1)); - return o; - /* // Shift should only be the first node in the JIT tree. // Force input to be evaluated so that in is always a buffer. in.eval(); @@ -49,11 +47,10 @@ Array shift(const Array &in, const int sdims[4]) { assert(shifts[i] >= 0 && shifts[i] <= oDims[i]); } - auto node = make_shared( + auto node = make_shared>( static_cast(dtype_traits::af_type), - static_pointer_cast(in.getNode()), shifts); + static_pointer_cast>(in.getNode()), shifts); return createNodeArray(oDims, common::Node_ptr(node)); - */ } #define INSTANTIATE(T) \ @@ -67,6 +64,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/oneapi/sobel.cpp b/src/backend/oneapi/sobel.cpp index 54ba117be7..e919a37b77 100644 --- a/src/backend/oneapi/sobel.cpp +++ b/src/backend/oneapi/sobel.cpp @@ -42,6 +42,7 @@ INSTANTIATE(double, double) INSTANTIATE(int, int) INSTANTIATE(uint, int) INSTANTIATE(char, int) +INSTANTIATE(schar, int) INSTANTIATE(uchar, int) INSTANTIATE(short, int) INSTANTIATE(ushort, int) diff --git a/src/backend/oneapi/solve.cpp b/src/backend/oneapi/solve.cpp index a4082c0d1f..4d213d25ae 100644 --- a/src/backend/oneapi/solve.cpp +++ b/src/backend/oneapi/solve.cpp @@ -11,111 +11,129 @@ #include -#if defined(WITH_LINEAR_ALGEBRA) && !defined(AF_ONEAPI) +#if defined(WITH_LINEAR_ALGEBRA) +#include #include +#include #include -#include #include -#include -#include -#include -#include #include +#include +#include +#include #include #include -#include +#include #include +#include #include -using cl::Buffer; +using arrayfire::common::cast; using std::min; using std::vector; +using sycl::buffer; namespace arrayfire { namespace oneapi { +static ::oneapi::mkl::transpose toMKLTranspose(af_mat_prop opt) { + switch (opt) { + case AF_MAT_NONE: return ::oneapi::mkl::transpose::nontrans; + case AF_MAT_TRANS: return ::oneapi::mkl::transpose::trans; + case AF_MAT_CTRANS: return ::oneapi::mkl::transpose::conjtrans; + default: AF_ERROR("INVALID af_mat_prop", AF_ERR_ARG); + } +} + template Array solveLU(const Array &A, const Array &pivot, const Array &b, const af_mat_prop options) { - ONEAPI_NOT_SUPPORTED("solveLU Not supported"); + const int64_t N = A.dims()[0]; + const int64_t NRHS = b.dims()[1]; + const int64_t LDA = A.strides()[1]; + const int64_t LDB = b.strides()[1]; + + ::oneapi::mkl::transpose opts = toMKLTranspose(options); + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::getrs_scratchpad_size>( + getQueue(), opts, N, NRHS, LDA, LDB); + + Array ipiv = cast(pivot); + buffer ipivBuf = ipiv.get()->reinterpret(); + auto scratchpad = memAlloc>(scratchpad_size); + + Array> B = copyArray>(b); + buffer> aBuf = A.template getBufferWithOffset>(); + buffer> bBuf = B.template getBufferWithOffset>(); + + ::oneapi::mkl::lapack::getrs(getQueue(), opts, N, NRHS, aBuf, LDA, ipivBuf, + bBuf, LDB, *scratchpad, scratchpad->size()); + return B; +} - if (OpenCLCPUOffload()) { return cpu::solveLU(A, pivot, b, options); } +template +Array generalSolve(const Array &a, const Array &b) { + int batches = a.dims()[2] * a.dims()[3]; - int N = A.dims()[0]; - int NRHS = b.dims()[1]; + dim4 aDims = a.dims(); + dim4 bDims = b.dims(); + int M = aDims[0]; + int N = aDims[1]; + int K = bDims[1]; + int MN = std::min(M, N); - vector ipiv(N); - copyData(&ipiv[0], pivot); + int lda = a.strides()[1]; + int astride = a.strides()[2]; + auto ipiv = memAlloc(MN * batches); + int ipivstride = MN; - Array B = copyArray(b); + int ldb = b.strides()[1]; + int bstride = b.strides()[2]; - const Buffer *A_buf = A.get(); - Buffer *B_buf = B.get(); + vector info(batches, 0); - int info = 0; - magma_getrs_gpu(MagmaNoTrans, N, NRHS, (*A_buf)(), A.getOffset(), - A.strides()[1], &ipiv[0], (*B_buf)(), B.getOffset(), - B.strides()[1], getQueue()(), &info); - return B; -} + Array A = copyArray(a); // A will be overwritten by L,U + Array B = copyArray(b); // will be overwritten with solution + + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::getrf_batch_scratchpad_size>( + getQueue(), M, N, lda, astride, ipivstride, batches); + + auto scratchpad = memAlloc>(scratchpad_size); + + buffer> aBuf = A.template getBufferWithOffset>(); + buffer> bBuf = B.template getBufferWithOffset>(); + ::oneapi::mkl::lapack::getrf_batch(getQueue(), M, N, aBuf, lda, astride, + *ipiv, ipivstride, batches, *scratchpad, + scratchpad->size()); + + scratchpad_size = + ::oneapi::mkl::lapack::getrs_batch_scratchpad_size>( + getQueue(), ::oneapi::mkl::transpose::nontrans, N, K, lda, astride, + ipivstride, ldb, bstride, batches); + + auto scratchpad_rs = memAlloc>(scratchpad_size); + + ::oneapi::mkl::lapack::getrs_batch( + getQueue(), ::oneapi::mkl::transpose::nontrans, N, K, aBuf, lda, + astride, *ipiv, ipivstride, bBuf, ldb, bstride, batches, *scratchpad_rs, + scratchpad_rs->size()); -template -Array generalSolve(const Array &a, const Array &b) { - ONEAPI_NOT_SUPPORTED("generalSolve Not supported"); - - // dim4 aDims = a.dims(); - // int batchz = aDims[2]; - // int batchw = aDims[3]; - - // Array A = copyArray(a); - Array B = copyArray(b); - - // for (int i = 0; i < batchw; i++) { - // for (int j = 0; j < batchz; j++) { - // int M = aDims[0]; - // int N = aDims[1]; - // int MN = min(M, N); - // vector ipiv(MN); - - // Buffer *A_buf = A.get(); - // int info = 0; - // cl_command_queue q = getQueue()(); - // auto aoffset = - // A.getOffset() + j * A.strides()[2] + i * A.strides()[3]; - // magma_getrf_gpu(M, N, (*A_buf)(), aoffset, A.strides()[1], - // &ipiv[0], q, &info); - - // Buffer *B_buf = B.get(); - // int K = B.dims()[1]; - - // auto boffset = - // B.getOffset() + j * B.strides()[2] + i * B.strides()[3]; - // magma_getrs_gpu(MagmaNoTrans, M, K, (*A_buf)(), aoffset, - // A.strides()[1], &ipiv[0], (*B_buf)(), boffset, - // B.strides()[1], q, &info); - // } - // } return B; } template Array leastSquares(const Array &a, const Array &b) { - ONEAPI_NOT_SUPPORTED("leastSquares Not supported"); - - int M = a.dims()[0]; - int N = a.dims()[1]; - int K = b.dims()[1]; - int MN = min(M, N); + int64_t M = a.dims()[0]; + int64_t N = a.dims()[1]; + int64_t K = b.dims()[1]; + int64_t MN = min(M, N); Array B = createEmptyArray(dim4()); - gpu_blas_trsm_func gpu_blas_trsm; - - cl_event event; - cl_command_queue queue = getQueue()(); if (M < N) { -#define UNMQR 0 // FIXME: UNMQR == 1 should be faster but does not work + const dim4 NullShape(0, 0, 0, 0); // Least squres for this case is solved using the following // solve(A, B) == matmul(Q, Xpad); @@ -127,71 +145,76 @@ Array leastSquares(const Array &a, const Array &b) { // QR is performed on the transpose of A Array A = transpose(a, true); - -#if UNMQR - const dim4 NullShape(0, 0, 0, 0); dim4 endPadding(N - b.dims()[0], K - b.dims()[1], 0, 0); B = (endPadding == NullShape ? copyArray(b) : padArrayBorders(b, NullShape, endPadding, AF_PAD_ZERO)); - B.resetDims(dim4(M, K)); -#else - B = copyArray(b); -#endif - int NB = magma_get_geqrf_nb(A.dims()[1]); - int NUM = (2 * MN + ((M + 31) / 32) * 32) * NB; - Array tmp = createEmptyArray(dim4(NUM)); + // Get workspace needed for QR + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::geqrf_scratchpad_size>( + getQueue(), A.dims()[0], A.dims()[1], A.strides()[1]); - vector h_tau(MN); + auto scratchpad = memAlloc>(scratchpad_size); + auto t = memAlloc>(MN); - int info = 0; - Buffer *dA = A.get(); - Buffer *dT = tmp.get(); - Buffer *dB = B.get(); - - magma_geqrf3_gpu(A.dims()[0], A.dims()[1], (*dA)(), A.getOffset(), - A.strides()[1], &h_tau[0], (*dT)(), tmp.getOffset(), - getQueue()(), &info); + buffer> aBuf = + A.template getBufferWithOffset>(); + // In place Perform in place QR + ::oneapi::mkl::lapack::geqrf(getQueue(), A.dims()[0], A.dims()[1], aBuf, + A.strides()[1], *t, *scratchpad, + scratchpad->size()); + // R1 = R(seq(M), seq(M)); A.resetDims(dim4(M, M)); - magmablas_swapdblk(MN - 1, NB, (*dA)(), A.getOffset(), - A.strides()[1], 1, (*dT)(), - tmp.getOffset() + MN * NB, NB, 0, queue); + // Bt = tri_solve(R1, B); + B.resetDims(dim4(M, K)); - OPENCL_BLAS_CHECK( - gpu_blas_trsm(OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_UPPER, - OPENCL_BLAS_CONJ_TRANS, OPENCL_BLAS_NON_UNIT_DIAGONAL, - B.dims()[0], B.dims()[1], scalar(1), (*dA)(), - A.getOffset(), A.strides()[1], (*dB)(), B.getOffset(), - B.strides()[1], 1, &queue, 0, nullptr, &event)); + buffer> bBuf = + B.template getBufferWithOffset>(); + // TODO: move to helper? trsm(A, B, AF_MAT_CTRANS, true, true, + // false); + compute_t alpha = scalar>(1); + ::oneapi::mkl::blas::trsm( + getQueue(), ::oneapi::mkl::side::left, ::oneapi::mkl::uplo::upper, + ::oneapi::mkl::transpose::conjtrans, ::oneapi::mkl::diag::nonunit, + B.dims()[0], B.dims()[1], alpha, aBuf, A.strides()[1], bBuf, + B.strides()[1]); + + // Bpad = pad(Bt, ..) + B.resetDims(dim4(N, K)); - magmablas_swapdblk(MN - 1, NB, (*dT)(), tmp.getOffset() + MN * NB, - NB, 0, (*dA)(), A.getOffset(), A.strides()[1], 1, - queue); + // matmul(Q, Bpad) + if constexpr (std::is_floating_point>()) { + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::ormqr_scratchpad_size>( + getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::nontrans, B.dims()[0], + B.dims()[1], A.dims()[0], A.strides()[1], B.strides()[1]); + + auto scratchpad_ormqr = memAlloc>(scratchpad_size); + ::oneapi::mkl::lapack::ormqr( + getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::nontrans, B.dims()[0], B.dims()[1], + A.dims()[0], aBuf, A.strides()[1], *t, bBuf, B.strides()[1], + *scratchpad_ormqr, scratchpad_ormqr->size()); + } else if constexpr (common::isComplex(static_cast( + dtype_traits>::af_type))) { + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::unmqr_scratchpad_size>( + getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::nontrans, B.dims()[0], + B.dims()[1], A.dims()[0], A.strides()[1], B.strides()[1]); + + auto scratchpad_unmqr = memAlloc>(scratchpad_size); + ::oneapi::mkl::lapack::unmqr( + getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::nontrans, B.dims()[0], B.dims()[1], + A.dims()[0], aBuf, A.strides()[1], *t, bBuf, B.strides()[1], + *scratchpad_unmqr, scratchpad_unmqr->size()); + } -#if UNMQR - int lwork = (B.dims()[0] - A.dims()[0] + NB) * (B.dims()[1] + 2 * NB); - vector h_work(lwork); - B.resetDims(dim4(N, K)); - magma_unmqr_gpu(MagmaLeft, MagmaNoTrans, B.dims()[0], B.dims()[1], - A.dims()[0], (*dA)(), A.getOffset(), A.strides()[1], - &h_tau[0], (*dB)(), B.getOffset(), B.strides()[1], - &h_work[0], lwork, (*dT)(), tmp.getOffset(), NB, - queue, &info); -#else - A.resetDims(dim4(N, M)); - magma_ungqr_gpu(A.dims()[0], A.dims()[1], min(M, N), (*dA)(), - A.getOffset(), A.strides()[1], &h_tau[0], (*dT)(), - tmp.getOffset(), NB, queue, &info); - - Array B_new = createEmptyArray(dim4(A.dims()[0], B.dims()[1])); - T alpha = scalar(1.0); - T beta = scalar(0.0); - gemm(B_new, AF_MAT_NONE, AF_MAT_NONE, &alpha, A, B, &beta); - B = B_new; -#endif } else if (M > N) { // Least squres for this case is solved using the following // solve(A, B) == tri_solve(R1, Bt); @@ -204,56 +227,61 @@ Array leastSquares(const Array &a, const Array &b) { Array A = copyArray(a); B = copyArray(b); - int MN = min(M, N); - int NB = magma_get_geqrf_nb(M); - - int NUM = (2 * MN + ((N + 31) / 32) * 32) * NB; - Array tmp = createEmptyArray(dim4(NUM)); - - vector h_tau(NUM); - - int info = 0; - Buffer *A_buf = A.get(); - Buffer *B_buf = B.get(); - Buffer *dT = tmp.get(); - - magma_geqrf3_gpu(M, N, (*A_buf)(), A.getOffset(), A.strides()[1], - &h_tau[0], (*dT)(), tmp.getOffset(), getQueue()(), - &info); - - int NRHS = B.dims()[1]; - int lhwork = (M - N + NB) * (NRHS + NB) + NRHS * NB; - - vector h_work(lhwork); - h_work[0] = scalar(lhwork); - - magma_unmqr_gpu(MagmaLeft, MagmaConjTrans, M, NRHS, N, (*A_buf)(), - A.getOffset(), A.strides()[1], &h_tau[0], (*B_buf)(), - B.getOffset(), B.strides()[1], &h_work[0], lhwork, - (*dT)(), tmp.getOffset(), NB, queue, &info); - - magmablas_swapdblk(MN - 1, NB, (*A_buf)(), A.getOffset(), - A.strides()[1], 1, (*dT)(), - tmp.getOffset() + NB * MN, NB, 0, queue); - - if (getActivePlatform() == AFCL_PLATFORM_NVIDIA) { - Array AT = transpose(A, true); - Buffer *AT_buf = AT.get(); - OPENCL_BLAS_CHECK(gpu_blas_trsm( - OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_LOWER, - OPENCL_BLAS_CONJ_TRANS, OPENCL_BLAS_NON_UNIT_DIAGONAL, N, NRHS, - scalar(1), (*AT_buf)(), AT.getOffset(), AT.strides()[1], - (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0, - nullptr, &event)); - } else { - OPENCL_BLAS_CHECK(gpu_blas_trsm( - OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_UPPER, - OPENCL_BLAS_NO_TRANS, OPENCL_BLAS_NON_UNIT_DIAGONAL, N, NRHS, - scalar(1), (*A_buf)(), A.getOffset(), A.strides()[1], - (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0, - nullptr, &event)); + // Get workspace needed for QR + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::geqrf_scratchpad_size>( + getQueue(), M, N, A.strides()[1]); + + auto scratchpad = memAlloc>(scratchpad_size); + auto t = memAlloc>(MN); + + buffer> aBuf = + A.template getBufferWithOffset>(); + // In place Perform in place QR + ::oneapi::mkl::lapack::geqrf(getQueue(), M, N, aBuf, A.strides()[1], *t, + *scratchpad, scratchpad->size()); + + // matmul(Q1, B) + buffer> bBuf = + B.template getBufferWithOffset>(); + if constexpr (std::is_floating_point>()) { + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::ormqr_scratchpad_size>( + getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::trans, M, K, N, A.strides()[1], + b.strides()[1]); + + auto scratchpad_ormqr = memAlloc>(scratchpad_size); + ::oneapi::mkl::lapack::ormqr(getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::trans, M, K, + N, aBuf, A.strides()[1], *t, bBuf, + b.strides()[1], *scratchpad_ormqr, + scratchpad_ormqr->size()); + } else if constexpr (common::isComplex(static_cast( + dtype_traits>::af_type))) { + std::int64_t scratchpad_size = + ::oneapi::mkl::lapack::unmqr_scratchpad_size>( + getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::conjtrans, M, K, N, + A.strides()[1], b.strides()[1]); + + auto scratchpad_unmqr = memAlloc>(scratchpad_size); + ::oneapi::mkl::lapack::unmqr(getQueue(), ::oneapi::mkl::side::left, + ::oneapi::mkl::transpose::conjtrans, M, + K, N, aBuf, A.strides()[1], *t, bBuf, + b.strides()[1], *scratchpad_unmqr, + scratchpad_unmqr->size()); } + + // tri_solve(R1, Bt) + A.resetDims(dim4(N, N)); B.resetDims(dim4(N, K)); + + compute_t alpha = scalar>(1); + ::oneapi::mkl::blas::trsm( + getQueue(), ::oneapi::mkl::side::left, ::oneapi::mkl::uplo::upper, + ::oneapi::mkl::transpose::nontrans, ::oneapi::mkl::diag::nonunit, N, + K, alpha, aBuf, A.strides()[1], bBuf, B.strides()[1]); } return B; @@ -262,53 +290,30 @@ Array leastSquares(const Array &a, const Array &b) { template Array triangleSolve(const Array &A, const Array &b, const af_mat_prop options) { - gpu_blas_trsm_func gpu_blas_trsm; - - Array B = copyArray(b); - - int N = B.dims()[0]; - int NRHS = B.dims()[1]; + Array> B = copyArray(b); - const Buffer *A_buf = A.get(); - Buffer *B_buf = B.get(); + compute_t alpha = scalar>(1); + ::oneapi::mkl::uplo uplo = (options & AF_MAT_UPPER) + ? ::oneapi::mkl::uplo::upper + : ::oneapi::mkl::uplo::lower; - cl_event event = 0; - cl_command_queue queue = getQueue()(); + ::oneapi::mkl::diag unitdiag = (options & AF_MAT_DIAG_UNIT) + ? ::oneapi::mkl::diag::unit + : ::oneapi::mkl::diag::nonunit; - if (getActivePlatform() == AFCL_PLATFORM_NVIDIA && - (options & AF_MAT_UPPER)) { - Array AT = transpose(A, true); - - cl::Buffer *AT_buf = AT.get(); - OPENCL_BLAS_CHECK(gpu_blas_trsm( - OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_LOWER, - OPENCL_BLAS_CONJ_TRANS, - options & AF_MAT_DIAG_UNIT ? OPENCL_BLAS_UNIT_DIAGONAL - : OPENCL_BLAS_NON_UNIT_DIAGONAL, - N, NRHS, scalar(1), (*AT_buf)(), AT.getOffset(), AT.strides()[1], - (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0, nullptr, - &event)); - } else { - OPENCL_BLAS_CHECK(gpu_blas_trsm( - OPENCL_BLAS_SIDE_LEFT, - options & AF_MAT_LOWER ? OPENCL_BLAS_TRIANGLE_LOWER - : OPENCL_BLAS_TRIANGLE_UPPER, - OPENCL_BLAS_NO_TRANS, - options & AF_MAT_DIAG_UNIT ? OPENCL_BLAS_UNIT_DIAGONAL - : OPENCL_BLAS_NON_UNIT_DIAGONAL, - N, NRHS, scalar(1), (*A_buf)(), A.getOffset(), A.strides()[1], - (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0, nullptr, - &event)); - } + buffer> aBuf = A.template getBufferWithOffset>(); + buffer> bBuf = B.template getBufferWithOffset>(); + ::oneapi::mkl::blas::trsm(getQueue(), ::oneapi::mkl::side::left, uplo, + ::oneapi::mkl::transpose::nontrans, unitdiag, + B.dims()[0], B.dims()[1], alpha, aBuf, + A.strides()[1], bBuf, B.strides()[1]); return B; } template Array solve(const Array &a, const Array &b, const af_mat_prop options) { - if (OpenCLCPUOffload()) { return cpu::solve(a, b, options); } - if (options & AF_MAT_UPPER || options & AF_MAT_LOWER) { return triangleSolve(a, b, options); } diff --git a/src/backend/oneapi/solve.hpp b/src/backend/oneapi/solve.hpp index acea9327b4..a0c8924fa9 100644 --- a/src/backend/oneapi/solve.hpp +++ b/src/backend/oneapi/solve.hpp @@ -11,6 +11,7 @@ namespace arrayfire { namespace oneapi { + template Array solve(const Array &a, const Array &b, const af_mat_prop options = AF_MAT_NONE); diff --git a/src/backend/oneapi/sort.cpp b/src/backend/oneapi/sort.cpp index 002385a320..9bfbeb9094 100644 --- a/src/backend/oneapi/sort.cpp +++ b/src/backend/oneapi/sort.cpp @@ -7,6 +7,13 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +#if defined(__clang__) +#pragma clang diagnostic push +// temporary ignores for DPL internals +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + #include #include @@ -22,31 +29,29 @@ namespace oneapi { template Array sort(const Array &in, const unsigned dim, bool isAscending) { - try { - Array out = copyArray(in); - switch (dim) { - case 0: kernel::sort0(out, isAscending); break; - case 1: kernel::sortBatched(out, 1, isAscending); break; - case 2: kernel::sortBatched(out, 2, isAscending); break; - case 3: kernel::sortBatched(out, 3, isAscending); break; - default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED); - } - - if (dim != 0) { - af::dim4 preorderDims = out.dims(); - af::dim4 reorderDims(0, 1, 2, 3); - reorderDims[dim] = 0; - preorderDims[0] = out.dims()[dim]; - for (int i = 1; i <= static_cast(dim); i++) { - reorderDims[i - 1] = i; - preorderDims[i] = out.dims()[i - 1]; - } + Array out = copyArray(in); + switch (dim) { + case 0: kernel::sort0(out, isAscending); break; + case 1: kernel::sortBatched(out, 1, isAscending); break; + case 2: kernel::sortBatched(out, 2, isAscending); break; + case 3: kernel::sortBatched(out, 3, isAscending); break; + default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED); + } - out.setDataDims(preorderDims); - out = reorder(out, reorderDims); + if (dim != 0) { + af::dim4 preorderDims = out.dims(); + af::dim4 reorderDims(0, 1, 2, 3); + reorderDims[dim] = 0; + preorderDims[0] = out.dims()[dim]; + for (int i = 1; i <= static_cast(dim); i++) { + reorderDims[i - 1] = i; + preorderDims[i] = out.dims()[i - 1]; } - return out; - } catch (std::exception &ex) { AF_ERROR(ex.what(), AF_ERR_INTERNAL); } + + out.setDataDims(preorderDims); + out = reorder(out, reorderDims); + } + return out; } #define INSTANTIATE(T) \ @@ -58,6 +63,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) @@ -66,3 +72,8 @@ INSTANTIATE(uintl) } // namespace oneapi } // namespace arrayfire + +#if defined(__clang__) +/* Clang/LLVM */ +#pragma clang diagnostic pop +#endif diff --git a/src/backend/oneapi/sort_by_key.cpp b/src/backend/oneapi/sort_by_key.cpp index 9ec60130cd..ba24249955 100644 --- a/src/backend/oneapi/sort_by_key.cpp +++ b/src/backend/oneapi/sort_by_key.cpp @@ -67,6 +67,7 @@ void sort_by_key(Array &okey, Array &oval, const Array &ikey, INSTANTIATE(Tk, short) \ INSTANTIATE(Tk, ushort) \ INSTANTIATE(Tk, char) \ + INSTANTIATE(Tk, schar) \ INSTANTIATE(Tk, uchar) \ INSTANTIATE(Tk, intl) \ INSTANTIATE(Tk, uintl) @@ -78,6 +79,7 @@ INSTANTIATE1(uint) INSTANTIATE1(short) INSTANTIATE1(ushort) INSTANTIATE1(char) +INSTANTIATE1(schar) INSTANTIATE1(uchar) INSTANTIATE1(intl) INSTANTIATE1(uintl) diff --git a/src/backend/oneapi/sort_index.cpp b/src/backend/oneapi/sort_index.cpp index 17de33fbad..a8c547f8a1 100644 --- a/src/backend/oneapi/sort_index.cpp +++ b/src/backend/oneapi/sort_index.cpp @@ -68,6 +68,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/sparse.cpp b/src/backend/oneapi/sparse.cpp index 37e5826430..2e9a67213f 100644 --- a/src/backend/oneapi/sparse.cpp +++ b/src/backend/oneapi/sparse.cpp @@ -7,7 +7,7 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ -// #include +#include #include #include @@ -26,154 +26,151 @@ #include #include +#include + namespace arrayfire { namespace oneapi { using namespace common; +#define P(exp) af_print_array_gen(#exp, getHandle(exp), 2) + // Partial template specialization of sparseConvertDenseToStorage for COO // However, template specialization is not allowed template SparseArray sparseConvertDenseToCOO(const Array &in) { - ONEAPI_NOT_SUPPORTED("sparseConvertDenseToCOO Not supported"); - // in.eval(); + in.eval(); - // Array nonZeroIdx_ = where(in); - // Array nonZeroIdx = cast(nonZeroIdx_); + Array nonZeroIdx_ = where(in); + Array nonZeroIdx = cast(nonZeroIdx_); + nonZeroIdx.eval(); - // dim_t nNZ = nonZeroIdx.elements(); + dim_t nNZ = nonZeroIdx.elements(); - // Array constDim = createValueArray(dim4(nNZ), in.dims()[0]); - // constDim.eval(); + Array constDim = createValueArray(dim4(nNZ), in.dims()[0]); + constDim.eval(); - // Array rowIdx = - // arithOp(nonZeroIdx, constDim, nonZeroIdx.dims()); - // Array colIdx = - // arithOp(nonZeroIdx, constDim, nonZeroIdx.dims()); + Array rowIdx = + arithOp(nonZeroIdx, constDim, nonZeroIdx.dims()); + Array colIdx = + arithOp(nonZeroIdx, constDim, nonZeroIdx.dims()); - // Array values = copyArray(in); - // values = modDims(values, dim4(values.elements())); - // values = lookup(values, nonZeroIdx, 0); + Array values = copyArray(in); + values = modDims(values, dim4(values.elements())); + values = lookup(values, nonZeroIdx, 0); - // return createArrayDataSparseArray(in.dims(), values, rowIdx, colIdx, - // AF_STORAGE_COO); + return createArrayDataSparseArray(in.dims(), values, rowIdx, colIdx, + AF_STORAGE_COO); } template SparseArray sparseConvertDenseToStorage(const Array &in_) { - ONEAPI_NOT_SUPPORTED("sparseConvertDenseToStorage Not supported"); - // in_.eval(); - // - // uint nNZ = getScalar(reduce_all(in_)); - // - // SparseArray sparse_ = createEmptySparseArray(in_.dims(), nNZ, - // stype); sparse_.eval(); - // - // Array &values = sparse_.getValues(); - // Array &rowIdx = sparse_.getRowIdx(); - // Array &colIdx = sparse_.getColIdx(); - - // kernel::dense2csr(values, rowIdx, colIdx, in_); - - // return sparse_; + in_.eval(); + + uint nNZ = getScalar(reduce_all(in_)); + + SparseArray sparse_ = createEmptySparseArray(in_.dims(), nNZ, stype); + sparse_.eval(); + + Array &values = sparse_.getValues(); + Array &rowIdx = sparse_.getRowIdx(); + Array &colIdx = sparse_.getColIdx(); + + kernel::dense2csr(values, rowIdx, colIdx, in_); + + return sparse_; } // Partial template specialization of sparseConvertStorageToDense for COO // However, template specialization is not allowed template Array sparseConvertCOOToDense(const SparseArray &in) { - ONEAPI_NOT_SUPPORTED("sparseConvertCOOToDense Not supported"); - // in.eval(); - // - // Array dense = createValueArray(in.dims(), scalar(0)); - // dense.eval(); - // - // const Array values = in.getValues(); - // const Array rowIdx = in.getRowIdx(); - // const Array colIdx = in.getColIdx(); - - // kernel::coo2dense(dense, values, rowIdx, colIdx); - - // return dense; + in.eval(); + + Array dense = createValueArray(in.dims(), scalar(0)); + dense.eval(); + + const Array values = in.getValues(); + const Array rowIdx = in.getRowIdx(); + const Array colIdx = in.getColIdx(); + + kernel::coo2dense(dense, values, rowIdx, colIdx); + + return dense; } template Array sparseConvertStorageToDense(const SparseArray &in_) { - ONEAPI_NOT_SUPPORTED("sparseConvertStorageToDense Not supported"); - // - // if (stype != AF_STORAGE_CSR) { - // AF_ERROR("OpenCL Backend only supports CSR or COO to Dense", - // AF_ERR_NOT_SUPPORTED); - // } - // - // in_.eval(); - // - // Array dense_ = createValueArray(in_.dims(), scalar(0)); - // dense_.eval(); - // - // const Array &values = in_.getValues(); - // const Array &rowIdx = in_.getRowIdx(); - // const Array &colIdx = in_.getColIdx(); - // - // if (stype == AF_STORAGE_CSR) { - // // kernel::csr2dense(dense_, values, rowIdx, colIdx); - // } else { - // AF_ERROR("OpenCL Backend only supports CSR or COO to Dense", - // AF_ERR_NOT_SUPPORTED); - // } - // - // return dense_; + if (stype != AF_STORAGE_CSR) { + AF_ERROR("oneAPI Backend only supports CSR or COO to Dense", + AF_ERR_NOT_SUPPORTED); + } + + in_.eval(); + + Array dense_ = createValueArray(in_.dims(), scalar(0)); + dense_.eval(); + + const Array &values = in_.getValues(); + const Array &rowIdx = in_.getRowIdx(); + const Array &colIdx = in_.getColIdx(); + + if (stype == AF_STORAGE_CSR) { + kernel::csr2dense(dense_, values, rowIdx, colIdx); + } else { + AF_ERROR("oneAPI Backend only supports CSR or COO to Dense", + AF_ERR_NOT_SUPPORTED); + } + + return dense_; } template SparseArray sparseConvertStorageToStorage(const SparseArray &in) { - ONEAPI_NOT_SUPPORTED("sparseConvertStorageToStorage Not supported"); - // in.eval(); - - // SparseArray converted = createEmptySparseArray( - // in.dims(), static_cast(in.getNNZ()), dest); - // converted.eval(); - - // if (src == AF_STORAGE_CSR && dest == AF_STORAGE_COO) { - // Array index = range(in.getNNZ(), 0); - // index.eval(); - - // Array &ovalues = converted.getValues(); - // Array &orowIdx = converted.getRowIdx(); - // Array &ocolIdx = converted.getColIdx(); - // const Array &ivalues = in.getValues(); - // const Array &irowIdx = in.getRowIdx(); - // const Array &icolIdx = in.getColIdx(); - - // // kernel::csr2coo(ovalues, orowIdx, ocolIdx, ivalues, irowIdx, - // // icolIdx, - // // index); - - //} else if (src == AF_STORAGE_COO && dest == AF_STORAGE_CSR) { - // Array index = range(in.getNNZ(), 0); - // index.eval(); - - // Array &ovalues = converted.getValues(); - // Array &orowIdx = converted.getRowIdx(); - // Array &ocolIdx = converted.getColIdx(); - // const Array &ivalues = in.getValues(); - // const Array &irowIdx = in.getRowIdx(); - // const Array &icolIdx = in.getColIdx(); - - // Array rowCopy = copyArray(irowIdx); - // rowCopy.eval(); - - // kernel::coo2csr(ovalues, orowIdx, ocolIdx, ivalues, irowIdx, - // icolIdx, - // index, rowCopy, in.dims()[0]); - - //} else { - // // Should never come here - // AF_ERROR("OpenCL Backend invalid conversion combination", - // AF_ERR_NOT_SUPPORTED); - //} - - // return converted; + in.eval(); + + SparseArray converted = createEmptySparseArray( + in.dims(), static_cast(in.getNNZ()), dest); + converted.eval(); + + if (src == AF_STORAGE_CSR && dest == AF_STORAGE_COO) { + Array index = range(in.getNNZ(), 0); + index.eval(); + + Array &ovalues = converted.getValues(); + Array &orowIdx = converted.getRowIdx(); + Array &ocolIdx = converted.getColIdx(); + const Array &ivalues = in.getValues(); + const Array &irowIdx = in.getRowIdx(); + const Array &icolIdx = in.getColIdx(); + + kernel::csr2coo(ovalues, orowIdx, ocolIdx, ivalues, irowIdx, icolIdx, + index); + + } else if (src == AF_STORAGE_COO && dest == AF_STORAGE_CSR) { + Array index = range(in.getNNZ(), 0); + index.eval(); + + Array &ovalues = converted.getValues(); + Array &orowIdx = converted.getRowIdx(); + Array &ocolIdx = converted.getColIdx(); + const Array &ivalues = in.getValues(); + const Array &irowIdx = in.getRowIdx(); + const Array &icolIdx = in.getColIdx(); + + Array rowCopy = copyArray(irowIdx); + rowCopy.eval(); + + kernel::coo2csr(ovalues, orowIdx, ocolIdx, ivalues, irowIdx, icolIdx, + index, rowCopy, in.dims()[0]); + + } else { + // Should never come here + AF_ERROR("oneAPI Backend invalid conversion combination", + AF_ERR_NOT_SUPPORTED); + } + + return converted; } #define INSTANTIATE_TO_STORAGE(T, S) \ diff --git a/src/backend/oneapi/sparse_arith.cpp b/src/backend/oneapi/sparse_arith.cpp index 856d300553..4b3e7301c4 100644 --- a/src/backend/oneapi/sparse_arith.cpp +++ b/src/backend/oneapi/sparse_arith.cpp @@ -7,8 +7,8 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ -// #include #include +#include #include #include @@ -51,104 +51,101 @@ cdouble getInf() { template Array arithOpD(const SparseArray &lhs, const Array &rhs, const bool reverse) { - ONEAPI_NOT_SUPPORTED("arithOpD Not supported"); - // lhs.eval(); - // rhs.eval(); - - // Array out = createEmptyArray(dim4(0)); - // Array zero = createValueArray(rhs.dims(), scalar(0)); - // switch (op) { - // case af_add_t: out = copyArray(rhs); break; - // case af_sub_t: - // out = reverse ? copyArray(rhs) - // : arithOp(zero, rhs, rhs.dims()); - // break; - // default: out = copyArray(rhs); - // } - // out.eval(); - // switch (lhs.getStorage()) { - // case AF_STORAGE_CSR: - // kernel::sparseArithOpCSR(out, lhs.getValues(), - // lhs.getRowIdx(), lhs.getColIdx(), - // rhs, reverse); - // break; - // case AF_STORAGE_COO: - // kernel::sparseArithOpCOO(out, lhs.getValues(), - // lhs.getRowIdx(), lhs.getColIdx(), - // rhs, reverse); - // break; - // default: - // AF_ERROR("Sparse Arithmetic only supported for CSR or COO", - // AF_ERR_NOT_SUPPORTED); - // } - - // return out; + lhs.eval(); + rhs.eval(); + + Array out = createEmptyArray(dim4(0)); + Array zero = createValueArray(rhs.dims(), scalar(0)); + switch (op) { + case af_add_t: out = copyArray(rhs); break; + case af_sub_t: + out = reverse ? copyArray(rhs) + : arithOp(zero, rhs, rhs.dims()); + break; + default: out = copyArray(rhs); + } + out.eval(); + switch (lhs.getStorage()) { + case AF_STORAGE_CSR: + kernel::sparseArithOpCSR(out, lhs.getValues(), + lhs.getRowIdx(), lhs.getColIdx(), + rhs, reverse); + break; + case AF_STORAGE_COO: + kernel::sparseArithOpCOO(out, lhs.getValues(), + lhs.getRowIdx(), lhs.getColIdx(), + rhs, reverse); + break; + default: + AF_ERROR("Sparse Arithmetic only supported for CSR or COO", + AF_ERR_NOT_SUPPORTED); + } + + return out; } template SparseArray arithOp(const SparseArray &lhs, const Array &rhs, const bool reverse) { - ONEAPI_NOT_SUPPORTED("arithOp Not supported"); - // lhs.eval(); - // rhs.eval(); - - // SparseArray out = createArrayDataSparseArray( - // lhs.dims(), lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(), - // lhs.getStorage(), true); - // out.eval(); - // switch (lhs.getStorage()) { - // case AF_STORAGE_CSR: - // kernel::sparseArithOpCSR(out.getValues(), out.getRowIdx(), - // out.getColIdx(), rhs, reverse); - // break; - // case AF_STORAGE_COO: - // kernel::sparseArithOpCOO(out.getValues(), out.getRowIdx(), - // out.getColIdx(), rhs, reverse); - // break; - // default: - // AF_ERROR("Sparse Arithmetic only supported for CSR or COO", - // AF_ERR_NOT_SUPPORTED); - // } - - // return out; + lhs.eval(); + rhs.eval(); + + SparseArray out = createArrayDataSparseArray( + lhs.dims(), lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(), + lhs.getStorage(), true); + out.eval(); + switch (lhs.getStorage()) { + case AF_STORAGE_CSR: + kernel::sparseArithOpCSR(out.getValues(), out.getRowIdx(), + out.getColIdx(), rhs, reverse); + break; + case AF_STORAGE_COO: + kernel::sparseArithOpCOO(out.getValues(), out.getRowIdx(), + out.getColIdx(), rhs, reverse); + break; + default: + AF_ERROR("Sparse Arithmetic only supported for CSR or COO", + AF_ERR_NOT_SUPPORTED); + } + + return out; } template SparseArray arithOp(const SparseArray &lhs, const SparseArray &rhs) { - ONEAPI_NOT_SUPPORTED("arithOp Not supported"); - // lhs.eval(); - // rhs.eval(); - // af::storage sfmt = lhs.getStorage(); + lhs.eval(); + rhs.eval(); + af::storage sfmt = lhs.getStorage(); - // const dim4 &ldims = lhs.dims(); + const dim4 &ldims = lhs.dims(); - // const uint M = ldims[0]; - // const uint N = ldims[1]; + const uint M = ldims[0]; + const uint N = ldims[1]; - // const dim_t nnzA = lhs.getNNZ(); - // const dim_t nnzB = rhs.getNNZ(); + const dim_t nnzA = lhs.getNNZ(); + const dim_t nnzB = rhs.getNNZ(); - // auto temp = createValueArray(dim4(M + 1), scalar(0)); - // temp.eval(); + auto temp = createValueArray(dim4(M + 1), scalar(0)); + temp.eval(); - // unsigned nnzC = 0; - // kernel::csrCalcOutNNZ(temp, nnzC, M, N, nnzA, lhs.getRowIdx(), - // lhs.getColIdx(), nnzB, rhs.getRowIdx(), - // rhs.getColIdx()); + unsigned nnzC = 0; + kernel::csrCalcOutNNZ(temp, nnzC, M, N, nnzA, lhs.getRowIdx(), + lhs.getColIdx(), nnzB, rhs.getRowIdx(), + rhs.getColIdx()); - // auto outRowIdx = scan(temp, 0); + auto outRowIdx = scan(temp, 0); - // auto outColIdx = createEmptyArray(dim4(nnzC)); - // auto outValues = createEmptyArray(dim4(nnzC)); + auto outColIdx = createEmptyArray(dim4(nnzC)); + auto outValues = createEmptyArray(dim4(nnzC)); - // kernel::ssArithCSR(outValues, outColIdx, outRowIdx, M, N, nnzA, - // lhs.getValues(), lhs.getRowIdx(), - // lhs.getColIdx(), nnzB, rhs.getValues(), - // rhs.getRowIdx(), rhs.getColIdx()); + kernel::ssArithCSR(outValues, outColIdx, outRowIdx, M, N, nnzA, + lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(), + nnzB, rhs.getValues(), rhs.getRowIdx(), + rhs.getColIdx()); - // SparseArray retVal = createArrayDataSparseArray( - // ldims, outValues, outRowIdx, outColIdx, sfmt); - // return retVal; + SparseArray retVal = createArrayDataSparseArray( + ldims, outValues, outRowIdx, outColIdx, sfmt); + return retVal; } #define INSTANTIATE(T) \ diff --git a/src/backend/oneapi/sparse_blas.cpp b/src/backend/oneapi/sparse_blas.cpp index 67d7cb8352..0494a5806e 100644 --- a/src/backend/oneapi/sparse_blas.cpp +++ b/src/backend/oneapi/sparse_blas.cpp @@ -9,15 +9,6 @@ #include -// #include -// #include -// #include -// #include - -#include -#include -#include - #include #include #include @@ -26,68 +17,77 @@ #include #include -#if defined(WITH_LINEAR_ALGEBRA) -// #include -#endif // WITH_LINEAR_ALGEBRA +#include + +#include + +#include +#include +#include namespace arrayfire { namespace oneapi { using namespace common; +// Converts an af_mat_prop options to a transpose type for mkl +static ::oneapi::mkl::transpose toBlasTranspose(af_mat_prop opt) { + switch (opt) { + case AF_MAT_NONE: return ::oneapi::mkl::transpose::nontrans; + case AF_MAT_TRANS: return ::oneapi::mkl::transpose::trans; + case AF_MAT_CTRANS: return ::oneapi::mkl::transpose::conjtrans; + default: AF_ERROR("INVALID af_mat_prop", AF_ERR_ARG); + } +} + template Array matmul(const common::SparseArray& lhs, const Array& rhsIn, af_mat_prop optLhs, af_mat_prop optRhs) { - ONEAPI_NOT_SUPPORTED("sparse matmul Not supported"); - // #if defined(WITH_LINEAR_ALGEBRA) - // if (OpenCLCPUOffload( - // false)) { // Do not force offload gemm on OSX Intel devices - // return cpu::matmul(lhs, rhsIn, optLhs, optRhs); - // } - // #endif - // - // int lRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1; - // // int lColDim = (optLhs == AF_MAT_NONE) ? 1 : 0; - // static const int rColDim = - // 1; // Unsupported : (optRhs == AF_MAT_NONE) ? 1 : 0; - // - // dim4 lDims = lhs.dims(); - // dim4 rDims = rhsIn.dims(); - // int M = lDims[lRowDim]; - // int N = rDims[rColDim]; - // // int K = lDims[lColDim]; - // - // const Array rhs = - // (N != 1 && optLhs == AF_MAT_NONE) ? transpose(rhsIn, false) : - // rhsIn; - // Array out = createEmptyArray(af::dim4(M, N, 1, 1)); - // - // static const T alpha = scalar(1.0); - // static const T beta = scalar(0.0); - // - // const Array& values = lhs.getValues(); - // const Array& rowIdx = lhs.getRowIdx(); - // const Array& colIdx = lhs.getColIdx(); - // - // if (optLhs == AF_MAT_NONE) { - // if (N == 1) { - // kernel::csrmv(out, values, rowIdx, colIdx, rhs, alpha, beta); - // } else { - // kernel::csrmm_nt(out, values, rowIdx, colIdx, rhs, alpha, - // beta); - // } - // } else { - // // CSR transpose is a CSC matrix - // if (N == 1) { - // kernel::cscmv(out, values, rowIdx, colIdx, rhs, alpha, beta, - // optLhs == AF_MAT_CTRANS); - // } else { - // kernel::cscmm_nn(out, values, rowIdx, colIdx, rhs, alpha, - // beta, - // optLhs == AF_MAT_CTRANS); - // } - // } - // return out; + int lRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1; + static const int rColDim = + 1; // Unsupported : (optRhs == AF_MAT_NONE) ? 1 : 0; + + dim4 lDims = lhs.dims(); + dim4 rDims = rhsIn.dims(); + dim4 rStrides = rhsIn.strides(); + int M = lDims[lRowDim]; + int N = rDims[rColDim]; + + Array out = createEmptyArray(af::dim4(M, N, 1, 1)); + dim4 oStrides = out.strides(); + + static const T alpha = scalar(1.0); + static const T beta = scalar(0.0); + + const Array& values = lhs.getValues(); + const Array& rowIdx = lhs.getRowIdx(); + const Array& colIdx = lhs.getColIdx(); + sycl::buffer valBuf = values.template getBufferWithOffset(); + sycl::buffer rowBuf = rowIdx.template getBufferWithOffset(); + sycl::buffer colBuf = colIdx.template getBufferWithOffset(); + + const auto lOpts = toBlasTranspose(optLhs); + const auto rOpts = toBlasTranspose(optRhs); + + sycl::buffer rhsBuf = rhsIn.template getBufferWithOffset(); + sycl::buffer outBuf = out.template getBufferWithOffset(); + + ::oneapi::mkl::sparse::matrix_handle_t CSRHandle = nullptr; + ::oneapi::mkl::sparse::init_matrix_handle(&CSRHandle); + ::oneapi::mkl::sparse::set_csr_data( + getQueue(), CSRHandle, lDims[0], lDims[1], + ::oneapi::mkl::index_base::zero, rowBuf, colBuf, valBuf); + + if (N == 1) { + ::oneapi::mkl::sparse::gemv(getQueue(), lOpts, alpha, CSRHandle, rhsBuf, + beta, outBuf); + } else { + ::oneapi::mkl::sparse::gemm( + getQueue(), ::oneapi::mkl::layout::col_major, lOpts, rOpts, alpha, + CSRHandle, rhsBuf, N, rStrides[1], beta, outBuf, oStrides[1]); + } + ::oneapi::mkl::sparse::release_matrix_handle(getQueue(), &CSRHandle); + return out; } #define INSTANTIATE_SPARSE(T) \ diff --git a/src/backend/oneapi/sum.cpp b/src/backend/oneapi/sum.cpp index fb20ce6121..990979ba25 100644 --- a/src/backend/oneapi/sum.cpp +++ b/src/backend/oneapi/sum.cpp @@ -29,6 +29,8 @@ INSTANTIATE(af_add_t, uintl, uintl) INSTANTIATE(af_add_t, uintl, double) INSTANTIATE(af_add_t, char, int) INSTANTIATE(af_add_t, char, float) +INSTANTIATE(af_add_t, schar, int) +INSTANTIATE(af_add_t, schar, float) INSTANTIATE(af_add_t, uchar, uint) INSTANTIATE(af_add_t, uchar, float) INSTANTIATE(af_add_t, short, int) diff --git a/src/backend/oneapi/surface.cpp b/src/backend/oneapi/surface.cpp index 2a8d604772..ac50627938 100644 --- a/src/backend/oneapi/surface.cpp +++ b/src/backend/oneapi/surface.cpp @@ -80,6 +80,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace oneapi diff --git a/src/backend/oneapi/susan.cpp b/src/backend/oneapi/susan.cpp index 437259681c..b51acf13df 100644 --- a/src/backend/oneapi/susan.cpp +++ b/src/backend/oneapi/susan.cpp @@ -70,6 +70,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/svd.cpp b/src/backend/oneapi/svd.cpp index 2c9b751d15..7255226e1b 100644 --- a/src/backend/oneapi/svd.cpp +++ b/src/backend/oneapi/svd.cpp @@ -38,15 +38,27 @@ void svdInPlace(Array &s, Array &u, Array &vt, Array &in) { int64_t LDU = uStrides[1]; int64_t LDVt = vStrides[1]; - int64_t scratch_size = ::oneapi::mkl::lapack::gesvd_scratchpad_size( - getQueue(), ::oneapi::mkl::jobsvd::vectors, - ::oneapi::mkl::jobsvd::vectors, M, N, LDA, LDU, LDVt); - Array scratchpad = createEmptyArray(af::dim4(scratch_size)); - - ::oneapi::mkl::lapack::gesvd( - getQueue(), ::oneapi::mkl::jobsvd::vectors, - ::oneapi::mkl::jobsvd::vectors, M, N, *in.get(), LDA, *s.get(), - *u.get(), LDU, *vt.get(), LDVt, *scratchpad.get(), scratch_size); + int64_t scratch_size = + ::oneapi::mkl::lapack::gesvd_scratchpad_size>( + getQueue(), ::oneapi::mkl::jobsvd::vectors, + ::oneapi::mkl::jobsvd::vectors, M, N, LDA, LDU, LDVt); + + auto scratchpad = memAlloc>(scratch_size); + + sycl::buffer> in_buffer = + in.template getBufferWithOffset>(); + + sycl::buffer> sBuf = + s.template getBufferWithOffset>(); + sycl::buffer> uBuf = + u.template getBufferWithOffset>(); + sycl::buffer> vtBuf = + vt.template getBufferWithOffset>(); + + ::oneapi::mkl::lapack::gesvd(getQueue(), ::oneapi::mkl::jobsvd::vectors, + ::oneapi::mkl::jobsvd::vectors, M, N, + in_buffer, LDA, sBuf, uBuf, LDU, vtBuf, LDVt, + *scratchpad, scratchpad->size()); } template diff --git a/src/backend/oneapi/tile.cpp b/src/backend/oneapi/tile.cpp index aca96e4ec6..928d0e2b19 100644 --- a/src/backend/oneapi/tile.cpp +++ b/src/backend/oneapi/tile.cpp @@ -42,6 +42,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/oneapi/transform.cpp b/src/backend/oneapi/transform.cpp index 54b328f7fd..00edc15817 100644 --- a/src/backend/oneapi/transform.cpp +++ b/src/backend/oneapi/transform.cpp @@ -9,6 +9,7 @@ #include +#include #include #include @@ -19,18 +20,25 @@ template void transform(Array &out, const Array &in, const Array &tf, const af_interp_type method, const bool inverse, const bool perspective) { + // TODO: Temporary Fix, must fix handling subarrays upstream + // tf has to be linear, although offset is allowed. + const Array tf_Lin = tf.isLinear() ? tf : copyArray(tf); + switch (method) { case AF_INTERP_NEAREST: case AF_INTERP_LOWER: - kernel::transform(out, in, tf, inverse, perspective, method, 1); + kernel::transform(out, in, tf_Lin, inverse, perspective, method, + 1); break; case AF_INTERP_BILINEAR: case AF_INTERP_BILINEAR_COSINE: - kernel::transform(out, in, tf, inverse, perspective, method, 2); + kernel::transform(out, in, tf_Lin, inverse, perspective, method, + 2); break; case AF_INTERP_BICUBIC: case AF_INTERP_BICUBIC_SPLINE: - kernel::transform(out, in, tf, inverse, perspective, method, 3); + kernel::transform(out, in, tf_Lin, inverse, perspective, method, + 3); break; default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG); } @@ -50,6 +58,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/oneapi/transpose.cpp b/src/backend/oneapi/transpose.cpp index 580573125f..1f41e96cde 100644 --- a/src/backend/oneapi/transpose.cpp +++ b/src/backend/oneapi/transpose.cpp @@ -43,6 +43,7 @@ INSTANTIATE(cdouble) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/oneapi/transpose_inplace.cpp b/src/backend/oneapi/transpose_inplace.cpp index ddbb14e419..013027f780 100644 --- a/src/backend/oneapi/transpose_inplace.cpp +++ b/src/backend/oneapi/transpose_inplace.cpp @@ -40,6 +40,7 @@ INSTANTIATE(cdouble) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/oneapi/triangle.cpp b/src/backend/oneapi/triangle.cpp index e418c15b93..c8ab5e2b16 100644 --- a/src/backend/oneapi/triangle.cpp +++ b/src/backend/oneapi/triangle.cpp @@ -49,6 +49,7 @@ INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/types.hpp b/src/backend/oneapi/types.hpp index 4537f27987..395687396c 100644 --- a/src/backend/oneapi/types.hpp +++ b/src/backend/oneapi/types.hpp @@ -43,6 +43,7 @@ namespace oneapi { using cdouble = std::complex; using cfloat = std::complex; using intl = long long; +using schar = signed char; using uchar = unsigned char; using uint = unsigned int; using uintl = unsigned long long; @@ -95,6 +96,10 @@ inline const char *shortname(bool caps) { return caps ? "J" : "j"; } template<> +inline const char *shortname(bool caps) { + return caps ? "A" : "a"; // TODO +} +template<> inline const char *shortname(bool caps) { return caps ? "V" : "v"; } @@ -120,6 +125,11 @@ inline const char *getFullName() { return af::dtype_traits::getName(); } +template<> +inline const char *getFullName() { + return "signed char"; +} + template<> inline const char *getFullName() { return "float2"; diff --git a/src/backend/oneapi/unwrap.cpp b/src/backend/oneapi/unwrap.cpp index 15d60afe5d..bfc95e0f18 100644 --- a/src/backend/oneapi/unwrap.cpp +++ b/src/backend/oneapi/unwrap.cpp @@ -53,6 +53,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/oneapi/vector_field.cpp b/src/backend/oneapi/vector_field.cpp index 92f310698a..d67fa73c51 100644 --- a/src/backend/oneapi/vector_field.cpp +++ b/src/backend/oneapi/vector_field.cpp @@ -31,6 +31,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace oneapi diff --git a/src/backend/oneapi/where.cpp b/src/backend/oneapi/where.cpp index bc9e45a515..fd08b975b8 100644 --- a/src/backend/oneapi/where.cpp +++ b/src/backend/oneapi/where.cpp @@ -36,6 +36,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/oneapi/wrap.cpp b/src/backend/oneapi/wrap.cpp index 19e8c0260e..21c47ac007 100644 --- a/src/backend/oneapi/wrap.cpp +++ b/src/backend/oneapi/wrap.cpp @@ -44,6 +44,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp index d479ac5752..38fbfc4d84 100644 --- a/src/backend/opencl/Array.cpp +++ b/src/backend/opencl/Array.cpp @@ -9,6 +9,7 @@ #include +#include #include #include #include @@ -191,6 +192,30 @@ Array::Array(const dim4 &dims, const dim4 &strides, dim_t offset_, } } +template +void checkAndMigrate(Array &arr) { + int arr_id = arr.getDevId(); + int cur_id = detail::getActiveDeviceId(); + if (!isDeviceBufferAccessible(arr_id, cur_id)) { + auto getLogger = [&] { return spdlog::get("platform"); }; + AF_TRACE("Migrating array from {} to {}.", arr_id, cur_id); + auto migrated_data = memAlloc(arr.elements()); + void *mapped_migrated_buffer = getQueue().enqueueMapBuffer( + *migrated_data, CL_TRUE, CL_MAP_WRITE_INVALIDATE_REGION, 0, + sizeof(T) * arr.elements()); + setDevice(arr_id); + Buffer &buf = *arr.get(); + getQueue().enqueueReadBuffer(buf, CL_TRUE, 0, + sizeof(T) * arr.elements(), + mapped_migrated_buffer); + setDevice(cur_id); + getQueue().enqueueUnmapMemObject(*migrated_data, + mapped_migrated_buffer); + arr.data.reset(migrated_data.release(), bufferFree); + arr.setId(cur_id); + } +} + template void Array::eval() { if (isReady()) { return; } @@ -301,8 +326,13 @@ Node_ptr Array::getNode() const { template kJITHeuristics passesJitHeuristics(span root_nodes) { if (!evalFlag()) { return kJITHeuristics::Pass; } + static auto getLogger = [&] { return common::loggerFactory("jit"); }; for (const Node *n : root_nodes) { if (n->getHeight() > static_cast(getMaxJitSize())) { + AF_TRACE( + "JIT tree evaluated because of tree height exceeds limit: {} > " + "{}", + n->getHeight(), getMaxJitSize()); return kJITHeuristics::TreeHeight; } } @@ -377,8 +407,17 @@ kJITHeuristics passesJitHeuristics(span root_nodes) { bool isParamLimit = param_size >= max_param_size; - if (isParamLimit) { return kJITHeuristics::KernelParameterSize; } - if (isBufferLimit) { return kJITHeuristics::MemoryPressure; } + if (isParamLimit) { + AF_TRACE( + "JIT tree evaluated because of kernel parameter size: {} >= {}", + param_size, max_param_size); + return kJITHeuristics::KernelParameterSize; + } + if (isBufferLimit) { + AF_TRACE("JIT tree evaluated because of memory pressure: {}", + info.total_buffer_size); + return kJITHeuristics::MemoryPressure; + } } return kJITHeuristics::Pass; } @@ -537,7 +576,8 @@ size_t Array::getAllocatedBytes() const { template kJITHeuristics passesJitHeuristics(span node); \ template void *getDevicePtr(const Array &arr); \ template void Array::setDataDims(const dim4 &new_dims); \ - template size_t Array::getAllocatedBytes() const; + template size_t Array::getAllocatedBytes() const; \ + template void checkAndMigrate(Array & arr); INSTANTIATE(float) INSTANTIATE(double) @@ -545,6 +585,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) diff --git a/src/backend/opencl/Array.hpp b/src/backend/opencl/Array.hpp index 3a672d00f6..05b0468333 100644 --- a/src/backend/opencl/Array.hpp +++ b/src/backend/opencl/Array.hpp @@ -41,6 +41,13 @@ using af::dim4; template class Array; +/// Checks if the Array object can be migrated to the current device and if not, +/// an error is thrown +/// +/// \param[in] arr The Array that will be checked. +template +void checkAndMigrate(Array &arr); + template void evalMultiple(std::vector *> arrays); @@ -323,6 +330,7 @@ class Array { friend void destroyArray(Array *arr); friend void *getDevicePtr(const Array &arr); friend void *getRawPtr(const Array &arr); + friend void checkAndMigrate(Array &arr); }; } // namespace opencl diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt index 8a0e55d2e4..23bedeedab 100644 --- a/src/backend/opencl/CMakeLists.txt +++ b/src/backend/opencl/CMakeLists.txt @@ -7,6 +7,10 @@ dependency_check(OpenCL_FOUND "OpenCL not found.") +# OpenCL back end needs to use MKL LP64 interface +set(MKL_INTERFACE_INTEGER_SIZE 4) +set(MKL_INTERFACE "lp64") + include(InternalUtils) include(build_cl2hpp) include(build_CLBlast) @@ -454,6 +458,7 @@ target_sources(afopencl kernel/convolve/conv2_f32.cpp kernel/convolve/conv2_f64.cpp kernel/convolve/conv2_impl.hpp + kernel/convolve/conv2_s8.cpp kernel/convolve/conv2_s16.cpp kernel/convolve/conv2_s32.cpp kernel/convolve/conv2_s64.cpp @@ -468,6 +473,7 @@ target_sources(afopencl target_sources(afopencl PRIVATE jit/BufferNode.hpp + jit/ShiftNode.hpp jit/kernel_generators.hpp ) @@ -576,6 +582,7 @@ if(LAPACK_FOUND OR BUILD_WITH_MKL) if(BUILD_WITH_MKL) target_compile_definitions(afopencl PRIVATE USE_MKL) + target_compile_definitions(afopencl PRIVATE AF_MKL_INTERFACE_SIZE=${MKL_INTERFACE_INTEGER_SIZE}) if(MKL_BATCH) target_compile_definitions(afopencl PRIVATE AF_USE_MKL_BATCH) endif() diff --git a/src/backend/opencl/Param.hpp b/src/backend/opencl/Param.hpp index aaf19dea62..879c92c677 100644 --- a/src/backend/opencl/Param.hpp +++ b/src/backend/opencl/Param.hpp @@ -22,6 +22,9 @@ struct Param { Param(const Param& other) = default; Param(Param&& other) = default; + dim_t* dims_ptr() { return info.dims; } + dim_t* strides_ptr() { return info.strides; } + // AF_DEPRECATED("Use Array") Param(); // AF_DEPRECATED("Use Array") diff --git a/src/backend/opencl/all.cpp b/src/backend/opencl/all.cpp index 2d2a1d4717..d81d9def34 100644 --- a/src/backend/opencl/all.cpp +++ b/src/backend/opencl/all.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_and_t, uint, char) INSTANTIATE(af_and_t, intl, char) INSTANTIATE(af_and_t, uintl, char) INSTANTIATE(af_and_t, char, char) +INSTANTIATE(af_and_t, schar, char) INSTANTIATE(af_and_t, uchar, char) INSTANTIATE(af_and_t, short, char) INSTANTIATE(af_and_t, ushort, char) diff --git a/src/backend/opencl/any.cpp b/src/backend/opencl/any.cpp index ce36f8ed90..ee2d16ab63 100644 --- a/src/backend/opencl/any.cpp +++ b/src/backend/opencl/any.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_or_t, uint, char) INSTANTIATE(af_or_t, intl, char) INSTANTIATE(af_or_t, uintl, char) INSTANTIATE(af_or_t, char, char) +INSTANTIATE(af_or_t, schar, char) INSTANTIATE(af_or_t, uchar, char) INSTANTIATE(af_or_t, short, char) INSTANTIATE(af_or_t, ushort, char) diff --git a/src/backend/opencl/assign.cpp b/src/backend/opencl/assign.cpp index 57ceeaab2d..fbe0370dde 100644 --- a/src/backend/opencl/assign.cpp +++ b/src/backend/opencl/assign.cpp @@ -104,6 +104,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/opencl/bilateral.cpp b/src/backend/opencl/bilateral.cpp index 21ec82e2b6..6475377e75 100644 --- a/src/backend/opencl/bilateral.cpp +++ b/src/backend/opencl/bilateral.cpp @@ -34,6 +34,7 @@ INSTANTIATE(float, float) INSTANTIATE(char, float) INSTANTIATE(int, float) INSTANTIATE(uint, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(short, float) INSTANTIATE(ushort, float) diff --git a/src/backend/opencl/binary.hpp b/src/backend/opencl/binary.hpp index 02291d566a..546c5bc085 100644 --- a/src/backend/opencl/binary.hpp +++ b/src/backend/opencl/binary.hpp @@ -1,5 +1,5 @@ /******************************************************* - * Copyright (c) 2014, ArrayFire + * Copyright (c) 2025, ArrayFire * All rights reserved. * * This file is distributed under 3-clause BSD license. @@ -9,6 +9,9 @@ #pragma once #include +#include + +using arrayfire::common::half; namespace arrayfire { namespace opencl { @@ -80,6 +83,11 @@ BINARY_TYPE_2(max) BINARY_TYPE_2(rem) BINARY_TYPE_2(mod) +template<> +struct BinOp { + const char *name() { return "fmod"; } +}; + template struct BinOp { const char *name() { return "__pow"; } @@ -93,6 +101,7 @@ struct BinOp { POW_BINARY_OP(double, "pow") POW_BINARY_OP(float, "pow") +POW_BINARY_OP(half, "pow") POW_BINARY_OP(intl, "__powll") POW_BINARY_OP(uintl, "__powul") POW_BINARY_OP(uint, "__powui") diff --git a/src/backend/opencl/blas.cpp b/src/backend/opencl/blas.cpp index 45b4149599..8010fe555d 100644 --- a/src/backend/opencl/blas.cpp +++ b/src/backend/opencl/blas.cpp @@ -62,13 +62,14 @@ void gemm_fallback(Array & /*out*/, af_mat_prop /*optLhs*/, assert(false && "CPU fallback not implemented for f16"); } -template -void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, - const Array &lhs, const Array &rhs, const T *beta) { +template +void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, + const To *alpha, const Array &lhs, const Array &rhs, + const To *beta) { #if defined(WITH_LINEAR_ALGEBRA) // Do not force offload gemm on OSX Intel devices if (OpenCLCPUOffload(false) && - static_cast(dtype_traits::af_type) != f16) { + static_cast(dtype_traits::af_type) != f16) { gemm_fallback(out, optLhs, optRhs, alpha, lhs, rhs, beta); return; } @@ -114,14 +115,14 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, cl::Event event; if (rDims[bColDim] == 1) { dim_t incr = (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1]; - gpu_blas_gemv_func gemv; + gpu_blas_gemv_func gemv; OPENCL_BLAS_CHECK(gemv(lOpts, lDims[0], lDims[1], *alpha, (*lhs.get())(), lOffset, lStrides[1], (*rhs.get())(), rOffset, incr, *beta, (*out.get())(), oOffset, oStrides[0], 1, &getQueue()(), 0, nullptr, &event())); } else { - gpu_blas_gemm_func gemm; + gpu_blas_gemm_func gemm; OPENCL_BLAS_CHECK(gemm(lOpts, rOpts, M, N, K, *alpha, (*lhs.get())(), lOffset, lStrides[1], (*rhs.get())(), rOffset, rStrides[1], *beta, @@ -131,6 +132,14 @@ void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, } } +template<> +void gemm(Array &out, af_mat_prop optLhs, + af_mat_prop optRhs, const float *alpha, + const Array &lhs, const Array &rhs, + const float *beta) { + TYPE_ERROR(3, af_dtype::s8); +} + template Array dot(const Array &lhs, const Array &rhs, af_mat_prop optLhs, af_mat_prop optRhs) { diff --git a/src/backend/opencl/blas.hpp b/src/backend/opencl/blas.hpp index 4416960f46..fc4571d4b5 100644 --- a/src/backend/opencl/blas.hpp +++ b/src/backend/opencl/blas.hpp @@ -20,9 +20,10 @@ namespace opencl { void initBlas(); void deInitBlas(); -template -void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha, - const Array &lhs, const Array &rhs, const T *beta); +template +void gemm(Array &out, af_mat_prop optLhs, af_mat_prop optRhs, + const To *alpha, const Array &lhs, const Array &rhs, + const To *beta); template Array matmul(const Array &lhs, const Array &rhs, af_mat_prop optLhs, diff --git a/src/backend/opencl/cast.hpp b/src/backend/opencl/cast.hpp index 999d6188d9..cef1d76c0e 100644 --- a/src/backend/opencl/cast.hpp +++ b/src/backend/opencl/cast.hpp @@ -38,6 +38,11 @@ CAST_FN(uchar) CAST_FN(float) CAST_FN(double) +template +struct CastOp { + const char *name() { return "convert_char"; } +}; + #define CAST_CFN(TYPE) \ template \ struct CastOp { \ diff --git a/src/backend/opencl/compile_module.cpp b/src/backend/opencl/compile_module.cpp index 89d382c9c0..f0244b3b0d 100644 --- a/src/backend/opencl/compile_module.cpp +++ b/src/backend/opencl/compile_module.cpp @@ -81,6 +81,9 @@ const static string DEFAULT_MACROS_STR( #else\n \ #define half short\n \ #endif\n \ + #ifndef schar\n \ + #define schar char\n \ + #endif\n \ #ifndef M_PI\n \ #define M_PI 3.1415926535897932384626433832795028841971693993751058209749445923078164\n \ #endif\n \ diff --git a/src/backend/opencl/convolve.cpp b/src/backend/opencl/convolve.cpp index f826102caf..34aa93b642 100644 --- a/src/backend/opencl/convolve.cpp +++ b/src/backend/opencl/convolve.cpp @@ -98,6 +98,7 @@ INSTANTIATE(double, double) INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(char, float) INSTANTIATE(ushort, float) diff --git a/src/backend/opencl/convolve_separable.cpp b/src/backend/opencl/convolve_separable.cpp index 03da468ac4..41b88b6ba8 100644 --- a/src/backend/opencl/convolve_separable.cpp +++ b/src/backend/opencl/convolve_separable.cpp @@ -65,6 +65,7 @@ INSTANTIATE(double, double) INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(char, float) INSTANTIATE(short, float) diff --git a/src/backend/opencl/copy.cpp b/src/backend/opencl/copy.cpp index 970deae518..97d54d432c 100644 --- a/src/backend/opencl/copy.cpp +++ b/src/backend/opencl/copy.cpp @@ -128,6 +128,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) @@ -157,6 +158,8 @@ INSTANTIATE(half) Array const &src); \ template void copyArray(Array & dst, \ Array const &src); \ + template void copyArray(Array & dst, \ + Array const &src); \ template void copyArray(Array & dst, \ Array const &src); \ template void copyArray(Array & dst, \ @@ -170,6 +173,7 @@ INSTANTIATE_COPY_ARRAY(int) INSTANTIATE_COPY_ARRAY(uint) INSTANTIATE_COPY_ARRAY(intl) INSTANTIATE_COPY_ARRAY(uintl) +INSTANTIATE_COPY_ARRAY(schar) INSTANTIATE_COPY_ARRAY(uchar) INSTANTIATE_COPY_ARRAY(char) INSTANTIATE_COPY_ARRAY(short) @@ -201,6 +205,7 @@ INSTANTIATE_GETSCALAR(cfloat) INSTANTIATE_GETSCALAR(cdouble) INSTANTIATE_GETSCALAR(int) INSTANTIATE_GETSCALAR(uint) +INSTANTIATE_GETSCALAR(schar) INSTANTIATE_GETSCALAR(uchar) INSTANTIATE_GETSCALAR(char) INSTANTIATE_GETSCALAR(intl) diff --git a/src/backend/opencl/count.cpp b/src/backend/opencl/count.cpp index 80f12e68cd..fe1b588f89 100644 --- a/src/backend/opencl/count.cpp +++ b/src/backend/opencl/count.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_notzero_t, uint, uint) INSTANTIATE(af_notzero_t, intl, uint) INSTANTIATE(af_notzero_t, uintl, uint) INSTANTIATE(af_notzero_t, char, uint) +INSTANTIATE(af_notzero_t, schar, uint) INSTANTIATE(af_notzero_t, uchar, uint) INSTANTIATE(af_notzero_t, short, uint) INSTANTIATE(af_notzero_t, ushort, uint) diff --git a/src/backend/opencl/device_manager.cpp b/src/backend/opencl/device_manager.cpp index a8ca6e96c9..62c06a21a5 100644 --- a/src/backend/opencl/device_manager.cpp +++ b/src/backend/opencl/device_manager.cpp @@ -171,6 +171,14 @@ static inline bool compare_default(const unique_ptr& ldev, return l_mem > r_mem; } +/// Class to compare two devices for sorting in a map +class deviceLess { + public: + bool operator()(const cl::Device& lhs, const cl::Device& rhs) const { + return lhs() < rhs(); + } +}; + DeviceManager::DeviceManager() : logger(common::loggerFactory("platform")) , mUserDeviceOffset(0) @@ -216,6 +224,7 @@ DeviceManager::DeviceManager() AF_TRACE("Found {} OpenCL platforms", platforms.size()); + std::map mDeviceContextMap; // Iterate through platforms, get all available devices and store them for (auto& platform : platforms) { vector current_devices; @@ -227,11 +236,15 @@ DeviceManager::DeviceManager() } AF_TRACE("Found {} devices on platform {}", current_devices.size(), platform.getInfo()); - for (auto& dev : current_devices) { - mDevices.emplace_back(make_unique(dev)); - AF_TRACE("Found device {} on platform {}", - dev.getInfo(), - platform.getInfo()); + if (!current_devices.empty()) { + cl::Context ctx(current_devices); + for (auto& dev : current_devices) { + mDeviceContextMap[dev] = ctx; + mDevices.emplace_back(make_unique(dev)); + AF_TRACE("Found device {} on platform {}", + dev.getInfo(), + platform.getInfo()); + } } } @@ -248,19 +261,21 @@ DeviceManager::DeviceManager() // Create contexts and queues once the sort is done for (int i = 0; i < nDevices; i++) { - cl_platform_id device_platform = - devices[i]->getInfo(); - cl_context_properties cps[3] = { - CL_CONTEXT_PLATFORM, (cl_context_properties)(device_platform), 0}; + // For OpenCL-HPP >= v2023.12.14 type is cl::Platform instead of + // cl_platform_id + cl::Platform device_platform; + device_platform = devices[i]->getInfo(); + try { - mContexts.push_back(make_unique(*devices[i], cps)); + mContexts.emplace_back( + make_unique(mDeviceContextMap[*devices[i]])); mQueues.push_back(make_unique( *mContexts.back(), *devices[i], cl::QueueProperties::None)); mIsGLSharingOn.push_back(false); mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i])); mPlatforms.push_back( std::make_pair, afcl_platform>( - make_unique(device_platform, true), + make_unique(device_platform(), true), getPlatformEnum(*devices[i]))); mDevices.emplace_back(std::move(devices[i])); diff --git a/src/backend/opencl/device_manager.hpp b/src/backend/opencl/device_manager.hpp index 432758bd87..4b27a8f885 100644 --- a/src/backend/opencl/device_manager.hpp +++ b/src/backend/opencl/device_manager.hpp @@ -105,7 +105,7 @@ class DeviceManager { friend const cl::Context& getContext(); - friend cl::CommandQueue& getQueue(); + friend cl::CommandQueue& getQueue(int device_id); friend cl_command_queue getQueueHandle(int device_id); @@ -139,6 +139,8 @@ class DeviceManager { friend afcl::platform getActivePlatformVendor(); + friend bool isDeviceBufferAccessible(int buf_device_id, int execution_id); + public: static const int MAX_DEVICES = 32; diff --git a/src/backend/opencl/diagonal.cpp b/src/backend/opencl/diagonal.cpp index 094906a77a..2d21b5f461 100644 --- a/src/backend/opencl/diagonal.cpp +++ b/src/backend/opencl/diagonal.cpp @@ -54,6 +54,7 @@ INSTANTIATE_DIAGONAL(uint) INSTANTIATE_DIAGONAL(intl) INSTANTIATE_DIAGONAL(uintl) INSTANTIATE_DIAGONAL(char) +INSTANTIATE_DIAGONAL(schar) INSTANTIATE_DIAGONAL(uchar) INSTANTIATE_DIAGONAL(short) INSTANTIATE_DIAGONAL(ushort) diff --git a/src/backend/opencl/diff.cpp b/src/backend/opencl/diff.cpp index 020365d24c..e152301f0d 100644 --- a/src/backend/opencl/diff.cpp +++ b/src/backend/opencl/diff.cpp @@ -50,6 +50,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/opencl/err_opencl.hpp b/src/backend/opencl/err_opencl.hpp index 2c1187c569..9a24bc2789 100644 --- a/src/backend/opencl/err_opencl.hpp +++ b/src/backend/opencl/err_opencl.hpp @@ -27,6 +27,6 @@ std::string getProgramBuildLog(const cl::Program &prog); #define OPENCL_NOT_SUPPORTED(message) \ do { \ - throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \ - boost::stacktrace::stacktrace()); \ + throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, "OpenCL",\ + message, boost::stacktrace::stacktrace()); \ } while (0) diff --git a/src/backend/opencl/exampleFunction.cpp b/src/backend/opencl/exampleFunction.cpp index 10af977382..87306e329c 100644 --- a/src/backend/opencl/exampleFunction.cpp +++ b/src/backend/opencl/exampleFunction.cpp @@ -57,6 +57,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(cfloat) diff --git a/src/backend/opencl/fast.cpp b/src/backend/opencl/fast.cpp index bfe6c84177..4198cf82ba 100644 --- a/src/backend/opencl/fast.cpp +++ b/src/backend/opencl/fast.cpp @@ -53,6 +53,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/fftconvolve.cpp b/src/backend/opencl/fftconvolve.cpp index f6b243baac..f5a875f41c 100644 --- a/src/backend/opencl/fftconvolve.cpp +++ b/src/backend/opencl/fftconvolve.cpp @@ -137,6 +137,7 @@ INSTANTIATE(double) INSTANTIATE(float) INSTANTIATE(uint) INSTANTIATE(int) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(uintl) diff --git a/src/backend/opencl/flood_fill.cpp b/src/backend/opencl/flood_fill.cpp index b57de824bd..4a759e095d 100644 --- a/src/backend/opencl/flood_fill.cpp +++ b/src/backend/opencl/flood_fill.cpp @@ -34,6 +34,7 @@ Array floodFill(const Array& image, const Array& seedsX, INSTANTIATE(float) INSTANTIATE(uint) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace opencl diff --git a/src/backend/opencl/hist_graphics.cpp b/src/backend/opencl/hist_graphics.cpp index 6c2a06e0b1..a20daeb700 100644 --- a/src/backend/opencl/hist_graphics.cpp +++ b/src/backend/opencl/hist_graphics.cpp @@ -74,6 +74,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace opencl diff --git a/src/backend/opencl/histogram.cpp b/src/backend/opencl/histogram.cpp index 7c3d432228..bbf7e9082e 100644 --- a/src/backend/opencl/histogram.cpp +++ b/src/backend/opencl/histogram.cpp @@ -41,6 +41,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/identity.cpp b/src/backend/opencl/identity.cpp index 9d9ae55718..9aa72fc433 100644 --- a/src/backend/opencl/identity.cpp +++ b/src/backend/opencl/identity.cpp @@ -37,6 +37,7 @@ INSTANTIATE_IDENTITY(uint) INSTANTIATE_IDENTITY(intl) INSTANTIATE_IDENTITY(uintl) INSTANTIATE_IDENTITY(char) +INSTANTIATE_IDENTITY(schar) INSTANTIATE_IDENTITY(uchar) INSTANTIATE_IDENTITY(short) INSTANTIATE_IDENTITY(ushort) diff --git a/src/backend/opencl/image.cpp b/src/backend/opencl/image.cpp index cffc2b8194..663fc63c24 100644 --- a/src/backend/opencl/image.cpp +++ b/src/backend/opencl/image.cpp @@ -78,6 +78,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/opencl/index.cpp b/src/backend/opencl/index.cpp index 0911229936..b1cb238968 100644 --- a/src/backend/opencl/index.cpp +++ b/src/backend/opencl/index.cpp @@ -42,6 +42,16 @@ Array index(const Array& in, const af_index_t idxrs[]) { p.isSeq[i] = idxrs[i].isSeq ? 1 : 0; p.offs[i] = iOffs[i]; p.strds[i] = iStrds[i]; + p.steps[i] = 0; + if (idxrs[i].isSeq) { + af_seq seq = idxrs[i].idx.seq; + // The step for af_span used in the kernel must be 1 + if (seq.begin == af_span.begin && seq.end == af_span.end && + seq.step == af_span.step) + p.steps[i] = 1; + else + p.steps[i] = seq.step; + } } cl::Buffer* bPtrs[4]; @@ -81,6 +91,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/opencl/iota.cpp b/src/backend/opencl/iota.cpp index de69ca6595..87c840b419 100644 --- a/src/backend/opencl/iota.cpp +++ b/src/backend/opencl/iota.cpp @@ -39,6 +39,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/ireduce.cpp b/src/backend/opencl/ireduce.cpp index ca4c916f63..d4b080389c 100644 --- a/src/backend/opencl/ireduce.cpp +++ b/src/backend/opencl/ireduce.cpp @@ -58,6 +58,7 @@ INSTANTIATE(af_min_t, uint) INSTANTIATE(af_min_t, intl) INSTANTIATE(af_min_t, uintl) INSTANTIATE(af_min_t, char) +INSTANTIATE(af_min_t, schar) INSTANTIATE(af_min_t, uchar) INSTANTIATE(af_min_t, short) INSTANTIATE(af_min_t, ushort) @@ -73,6 +74,7 @@ INSTANTIATE(af_max_t, uint) INSTANTIATE(af_max_t, intl) INSTANTIATE(af_max_t, uintl) INSTANTIATE(af_max_t, char) +INSTANTIATE(af_max_t, schar) INSTANTIATE(af_max_t, uchar) INSTANTIATE(af_max_t, short) INSTANTIATE(af_max_t, ushort) diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp index f7ba973032..c0858c3cc5 100644 --- a/src/backend/opencl/jit.cpp +++ b/src/backend/opencl/jit.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,7 @@ using arrayfire::common::Node_map_t; using arrayfire::common::Node_ptr; using arrayfire::common::NodeIterator; using arrayfire::common::saveKernel; +using arrayfire::opencl::jit::ShiftNode; using cl::Kernel; using cl::NDRange; @@ -186,62 +188,77 @@ __kernel void )JIT"; thread_local stringstream outOffsetStream; thread_local stringstream inOffsetsStream; thread_local stringstream opsStream; + thread_local stringstream kerStream; - int oid{0}; - for (size_t i{0}; i < full_nodes.size(); i++) { - const auto& node{full_nodes[i]}; - const auto& ids_curr{full_ids[i]}; - // Generate input parameters, only needs current id - node->genParams(inParamStream, ids_curr.id, is_linear); - // Generate input offsets, only needs current id - node->genOffsets(inOffsetsStream, ids_curr.id, is_linear); - // Generate the core function body, needs children ids as well - node->genFuncs(opsStream, ids_curr); - for (auto outIt{begin(output_ids)}, endIt{end(output_ids)}; - (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) { - // Generate also output parameters - outParamStream << "__global " - << full_nodes[ids_curr.id]->getTypeStr() << " *out" - << oid << ", int offset" << oid << ",\n"; - // Apply output offset - outOffsetStream << "\nout" << oid << " += offset" << oid << ';'; - // Generate code to write the output - opsStream << "out" << oid << "[idx] = val" << ids_curr.id << ";\n"; - ++oid; + string ret; + try { + int oid{0}; + for (size_t i{0}; i < full_nodes.size(); i++) { + const auto& node{full_nodes[i]}; + const auto& ids_curr{full_ids[i]}; + // Generate input parameters, only needs current id + node->genParams(inParamStream, ids_curr.id, is_linear); + // Generate input offsets, only needs current id + node->genOffsets(inOffsetsStream, ids_curr.id, is_linear); + // Generate the core function body, needs children ids as well + node->genFuncs(opsStream, ids_curr); + for (size_t output_idx{0}; output_idx < output_ids.size(); + ++output_idx) { + if (output_ids[output_idx] == ids_curr.id) { + outParamStream + << "__global " << full_nodes[ids_curr.id]->getTypeStr() + << " *out" << oid << ", int offset" << oid << ",\n"; + // Apply output offset + outOffsetStream << "\nout" << oid << " += offset" << oid + << ';'; + // Generate code to write the output + opsStream << "out" << output_idx << "[idx] = val" + << ids_curr.id << ";\n"; + ++oid; + } + } } - } - thread_local stringstream kerStream; - kerStream << kernelVoid << funcName << "(\n" - << inParamStream.str() << outParamStream.str() << dimParams << ")" - << blockStart; - if (is_linear) { - kerStream << linearInit << inOffsetsStream.str() - << outOffsetStream.str() << '\n'; - if (loop0) kerStream << linearLoop0Start; - kerStream << "\n\n" << opsStream.str(); - if (loop0) kerStream << linearLoop0End; - kerStream << linearEnd; - } else { - if (loop0) { - kerStream << stridedLoop0Init << outOffsetStream.str() << '\n' - << stridedLoop0Start; + kerStream << kernelVoid << funcName << "(\n" + << inParamStream.str() << outParamStream.str() << dimParams + << ")" << blockStart; + if (is_linear) { + kerStream << linearInit << inOffsetsStream.str() + << outOffsetStream.str() << '\n'; + if (loop0) kerStream << linearLoop0Start; + kerStream << "\n\n" << opsStream.str(); + if (loop0) kerStream << linearLoop0End; + kerStream << linearEnd; } else { - kerStream << stridedLoopNInit << outOffsetStream.str() << '\n'; - if (loop3) kerStream << stridedLoop3Init; - if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start; - if (loop3) kerStream << stridedLoop3Start; + if (loop0) { + kerStream << stridedLoop0Init << outOffsetStream.str() << '\n' + << stridedLoop0Start; + } else { + kerStream << stridedLoopNInit << outOffsetStream.str() << '\n'; + if (loop3) kerStream << stridedLoop3Init; + if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start; + if (loop3) kerStream << stridedLoop3Start; + } + kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str(); + if (loop3) kerStream << stridedLoop3End; + if (loop1) kerStream << stridedLoop1End; + if (loop0) kerStream << stridedLoop0End; + kerStream << stridedEnd; } - kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str(); - if (loop3) kerStream << stridedLoop3End; - if (loop1) kerStream << stridedLoop1End; - if (loop0) kerStream << stridedLoop0End; - kerStream << stridedEnd; + kerStream << blockEnd; + ret = kerStream.str(); + } catch (...) { + // Prepare for next round + inParamStream.str(""); + outParamStream.str(""); + inOffsetsStream.str(""); + outOffsetStream.str(""); + opsStream.str(""); + kerStream.str(""); + throw; } - kerStream << blockEnd; - const string ret{kerStream.str()}; - // Prepare for next round, limit memory + // Prepare for next round inParamStream.str(""); outParamStream.str(""); inOffsetsStream.str(""); @@ -257,8 +274,9 @@ cl::Kernel getKernel(const vector& output_nodes, const vector& full_nodes, const vector& full_ids, const bool is_linear, const bool loop0, const bool loop1, const bool loop3) { - const string funcName{getFuncName(output_nodes, full_nodes, full_ids, - is_linear, loop0, loop1, false, loop3)}; + const string funcName{getFuncName(output_nodes, output_ids, full_nodes, + full_ids, is_linear, loop0, loop1, false, + loop3)}; // A forward lookup in module cache helps avoid recompiling the JIT // source generated from identical JIT-trees. const auto entry{ @@ -418,21 +436,8 @@ void evalNodes(vector& outputs, const vector& output_nodes) { } } if (emptyColumnsFound) { - const auto isBuffer{ - [](const Node_ptr& ptr) { return ptr->isBuffer(); }}; - for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)}; - (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt; - ++nodeIt) { - BufferNode* buf{static_cast(nodeIt->get())}; - removeEmptyColumns(outDims, ndims, buf->m_param.dims, - buf->m_param.strides); - } - for_each(++begin(outputs), end(outputs), - [outDims, ndims](Param& output) { - removeEmptyColumns(outDims, ndims, output.info.dims, - output.info.strides); - }); - ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides); + common::removeEmptyDimensions( + outputs, node_clones); } full_nodes.clear(); @@ -448,10 +453,11 @@ void evalNodes(vector& outputs, const vector& output_nodes) { int nargs{0}; for (const Node* node : full_nodes) { - nargs = node->setArgs(nargs, is_linear, - [&ker](int id, const void* ptr, size_t arg_size) { - ker.setArg(id, arg_size, ptr); - }); + nargs = node->setArgs( + nargs, is_linear, + [&ker](int id, const void* ptr, size_t arg_size, bool is_buffer) { + ker.setArg(id, arg_size, ptr); + }); } // Set output parameters diff --git a/src/backend/opencl/jit/BufferNode.hpp b/src/backend/opencl/jit/BufferNode.hpp index e188fb429f..14521030f7 100644 --- a/src/backend/opencl/jit/BufferNode.hpp +++ b/src/backend/opencl/jit/BufferNode.hpp @@ -28,7 +28,16 @@ bool BufferNodeBase::operator==( // clang-format off return m_data.get() == other.m_data.get() && m_bytes == other.m_bytes && - m_param.offset == other.m_param.offset; + m_param.offset == other.m_param.offset && + m_linear_buffer == other.m_linear_buffer && + m_param.dims[0] == other.m_param.dims[0] && + m_param.dims[1] == other.m_param.dims[1] && + m_param.dims[2] == other.m_param.dims[2] && + m_param.dims[3] == other.m_param.dims[3] && + m_param.strides[0] == other.m_param.strides[0] && + m_param.strides[1] == other.m_param.strides[1] && + m_param.strides[2] == other.m_param.strides[2] && + m_param.strides[3] == other.m_param.strides[3]; // clang-format on } diff --git a/src/backend/opencl/jit/ShiftNode.hpp b/src/backend/opencl/jit/ShiftNode.hpp new file mode 100644 index 0000000000..8132105faf --- /dev/null +++ b/src/backend/opencl/jit/ShiftNode.hpp @@ -0,0 +1,21 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#include +#include + +namespace arrayfire { +namespace opencl { +namespace jit { + +using ShiftNode = common::ShiftNodeBase; + +} // namespace jit +} // namespace opencl +} // namespace arrayfire diff --git a/src/backend/opencl/jit/kernel_generators.hpp b/src/backend/opencl/jit/kernel_generators.hpp index d4700260c4..0228e7173f 100644 --- a/src/backend/opencl/jit/kernel_generators.hpp +++ b/src/backend/opencl/jit/kernel_generators.hpp @@ -30,17 +30,19 @@ inline void generateParamDeclaration(std::stringstream& kerStream, int id, } /// Calls the setArg function to set the arguments for a kernel call -inline int setKernelArguments( +inline int setBufferKernelArguments( int start_id, bool is_linear, - std::function& setArg, + std::function& setArg, const std::shared_ptr& ptr, const KParam& info) { setArg(start_id + 0, static_cast(&ptr.get()->operator()()), - sizeof(cl_mem)); + sizeof(cl_mem), true); if (is_linear) { setArg(start_id + 1, static_cast(&info.offset), - sizeof(dim_t)); + sizeof(dim_t), true); } else { - setArg(start_id + 1, static_cast(&info), sizeof(KParam)); + setArg(start_id + 1, static_cast(&info), sizeof(KParam), + true); } return start_id + 2; } diff --git a/src/backend/opencl/join.cpp b/src/backend/opencl/join.cpp index 22875d0e61..7975ecfb5a 100644 --- a/src/backend/opencl/join.cpp +++ b/src/backend/opencl/join.cpp @@ -227,6 +227,7 @@ INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(half) @@ -247,6 +248,7 @@ INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(half) diff --git a/src/backend/opencl/kernel/KParam.hpp b/src/backend/opencl/kernel/KParam.hpp index 38a3752760..1f4f1d5ba4 100644 --- a/src/backend/opencl/kernel/KParam.hpp +++ b/src/backend/opencl/kernel/KParam.hpp @@ -17,10 +17,16 @@ #endif // Defines the size and shape of the data in the OpenCL buffer -typedef struct { +typedef struct KParam_t { dim_t dims[4]; dim_t strides[4]; dim_t offset; + +#ifndef __OPENCL_VERSION__ + dim_t *dims_ptr() { return dims; } + dim_t *strides_ptr() { return strides; } +#endif + } KParam; #endif diff --git a/src/backend/opencl/kernel/convolve/conv1.cpp b/src/backend/opencl/kernel/convolve/conv1.cpp index 10ae600888..5bfa9668d6 100644 --- a/src/backend/opencl/kernel/convolve/conv1.cpp +++ b/src/backend/opencl/kernel/convolve/conv1.cpp @@ -58,6 +58,7 @@ INSTANTIATE(double, double) INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(char, float) INSTANTIATE(ushort, float) diff --git a/src/backend/opencl/kernel/convolve/conv2_s8.cpp b/src/backend/opencl/kernel/convolve/conv2_s8.cpp new file mode 100644 index 0000000000..b4b39b3f28 --- /dev/null +++ b/src/backend/opencl/kernel/convolve/conv2_s8.cpp @@ -0,0 +1,20 @@ +/******************************************************* + * Copyright (c) 2023, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#include + +namespace arrayfire { +namespace opencl { +namespace kernel { + +INSTANTIATE(schar, float) + +} // namespace kernel +} // namespace opencl +} // namespace arrayfire diff --git a/src/backend/opencl/kernel/convolve/conv3.cpp b/src/backend/opencl/kernel/convolve/conv3.cpp index 9a1baf9c6b..1383e8f443 100644 --- a/src/backend/opencl/kernel/convolve/conv3.cpp +++ b/src/backend/opencl/kernel/convolve/conv3.cpp @@ -45,6 +45,7 @@ INSTANTIATE(double, double) INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(char, float) INSTANTIATE(ushort, float) diff --git a/src/backend/opencl/kernel/convolve_separable.cpp b/src/backend/opencl/kernel/convolve_separable.cpp index 41bfa55dde..83a9116d72 100644 --- a/src/backend/opencl/kernel/convolve_separable.cpp +++ b/src/backend/opencl/kernel/convolve_separable.cpp @@ -95,6 +95,7 @@ INSTANTIATE(double, double) INSTANTIATE(float, float) INSTANTIATE(uint, float) INSTANTIATE(int, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(char, float) INSTANTIATE(ushort, float) diff --git a/src/backend/opencl/kernel/coo2dense.cl b/src/backend/opencl/kernel/coo2dense.cl index f86c073621..85afbfcd4b 100644 --- a/src/backend/opencl/kernel/coo2dense.cl +++ b/src/backend/opencl/kernel/coo2dense.cl @@ -11,18 +11,15 @@ kernel void coo2Dense(global T *oPtr, const KParam output, global const T *vPtr, const KParam values, global const int *rPtr, const KParam rowIdx, global const int *cPtr, const KParam colIdx) { - const int id = get_group_id(0) * get_local_size(0) * reps + get_local_id(0); - - if (id >= values.dims[0]) return; - const int dimSize = get_local_size(0); for (int i = get_local_id(0); i < reps * dimSize; i += dimSize) { - if (i >= values.dims[0]) return; + const int id = i + get_group_id(0) * dimSize * reps; + if (id >= values.dims[0]) return; - T v = vPtr[i]; - int r = rPtr[i]; - int c = cPtr[i]; + T v = vPtr[id + values.offset]; + int r = rPtr[id + rowIdx.offset]; + int c = cPtr[id + colIdx.offset]; int offset = r + c * output.strides[1]; diff --git a/src/backend/opencl/kernel/cscmv.cl b/src/backend/opencl/kernel/cscmv.cl index fab18301a1..bc56f57e46 100644 --- a/src/backend/opencl/kernel/cscmv.cl +++ b/src/backend/opencl/kernel/cscmv.cl @@ -7,6 +7,10 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ +#if IS_DBL || IS_LONG +#pragma OPENCL EXTENSION cl_khr_int64_base_atomics : enable +#endif + #if IS_CPLX T __cmul(T lhs, T rhs) { T out; @@ -35,100 +39,70 @@ T __ccmul(T lhs, T rhs) { #define CMUL(a, b) (a) * (b) #endif -int binary_search(global const int *ptr, int len, int val) { - int start = 0; - int end = len; - while (end > start) { - int mid = start + (end - start) / 2; - if (val < ptr[mid]) { - end = mid; - } else if (val > ptr[mid]) { - start = mid + 1; - } else { - return mid; - } - } - return start; +#if IS_DBL || IS_LONG +#define U ulong +#define ATOMIC_FN atom_cmpxchg +#else +#define U unsigned +#define ATOMIC_FN atomic_cmpxchg +#endif + +#if IS_CPLX +inline void atomicAdd(volatile __global T *ptr, T val) { + union { + U u[2]; + T t; + } next, expected, current; + current.t = *ptr; + + do { + expected.t.x = current.t.x; + next.t.x = expected.t.x + val.x; + current.u[0] = ATOMIC_FN((volatile __global U *) ptr, expected.u[0], next.u[0]); + } while(current.u[0] != expected.u[0]); + do { + expected.t.y = current.t.y; + next.t.y = expected.t.y + val.y; + current.u[1] = ATOMIC_FN(((volatile __global U *) ptr) + 1, expected.u[1], next.u[1]); + } while(current.u[1] != expected.u[1]); +} +#else +inline void atomicAdd(volatile __global T *ptr, T val) { + union { + U u; + T t; + } next, expected, current; + current.t = *ptr; + + do { + expected.t = current.t; + next.t = expected.t + val; + current.u = ATOMIC_FN((volatile __global U *) ptr, expected.u, next.u); + } while(current.u != expected.u); +} +#endif + +kernel void cscmv_beta(global T *output, const int M, const T beta) { + for(unsigned j = get_global_id(0); j < M; j += THREADS * get_num_groups(0)) + output[j] *= beta; } -// Each thread performs Matrix Vector multiplications for ROWS_PER_GROUP rows -// and (K / THREAD) columns. This generates a local output buffer of size -// ROWS_PER_THREAD for each thread. The outputs from each thread are added up to -// generate the final result. -kernel void cscmv_block( - global T *output, __global const T *values, - global const int *colidx, // rowidx from csr is colidx in csc - global const int *rowidx, // colidx from csr is rowidx in csc - const int M, // K from csr is M in csc +kernel void cscmv_atomic( + global T *output, __global T *values, + global int *colidx, // rowidx from csr is colidx in csc + global int *rowidx, // colidx from csr is rowidx in csc const int K, // M from csr is K in csc - global const T *rhs, const KParam rinfo, const T alpha, const T beta) { - int lid = get_local_id(0); + global const T *rhs, const KParam rinfo, const T alpha) { - // Get the row offset for the current group in the uncompressed matrix - int rowOff = get_group_id(0) * ROWS_PER_GROUP; - int rowLim = min(ROWS_PER_GROUP, M - rowOff); rhs += rinfo.offset; - T l_outvals[ROWS_PER_GROUP]; - for (int i = 0; i < rowLim; i++) { l_outvals[i] = 0; } - - for (int colId = lid; colId < K; colId += THREADS) { - int rowStart = colidx[colId]; - int rowEnd = colidx[colId + 1]; - int nonZeroCount = rowEnd - rowStart; - - // Find the location of the next non zero element after rowOff - int rowPos = binary_search(rowidx + rowStart, nonZeroCount, rowOff); - T rhsval = rhs[colId]; - - // Traversing through nonzero elements in the current chunk - for (int id = rowPos + rowStart; id < rowEnd; id++) { - int rowId = rowidx[id]; - - // Exit if moving past current chunk - if (rowId >= rowOff + ROWS_PER_GROUP) break; - - l_outvals[rowId - rowOff] += CMUL(values[id], rhsval); - } - } - - // s_outvals is used for reduction - local T s_outvals[THREADS]; - - // s_output is used to store the final output into local memory - local T s_output[ROWS_PER_GROUP]; - - // For each row of output, copy registers to local memory, add results, - // write to output. - for (int i = 0; i < rowLim; i++) { - // Copying to local memory - s_outvals[lid] = l_outvals[i]; - barrier(CLK_LOCAL_MEM_FENCE); - - // Adding the results through reduction - for (int n = THREADS / 2; n > 0; n /= 2) { - if (lid < n) s_outvals[lid] += s_outvals[lid + n]; - barrier(CLK_LOCAL_MEM_FENCE); - } - - // Store to another local buffer so it can be written in a coalesced - // manner later - if (lid == 0) { s_output[i] = s_outvals[0]; } - } - barrier(CLK_LOCAL_MEM_FENCE); - - // For each row in output, write output in coalesced manner - for (int i = lid; i < ROWS_PER_GROUP; i += THREADS) { - T outval = s_output[i]; - + for(unsigned j = get_group_id(0); j < K; j += get_num_groups(0)) { + for(unsigned i = get_local_id(0) + colidx[j]; i < colidx[j + 1]; i += THREADS) { + T outval = CMUL(values[i], rhs[j]); #if USE_ALPHA - outval = MUL(alpha, outval); -#endif - -#if USE_BETA - output[rowOff + i] = outval + MUL(beta, output[j * M + rowOff + i]); -#else - output[rowOff + i] = outval; + outval = MUL(alpha, outval); #endif + atomicAdd(output + rowidx[i], outval); + } } } diff --git a/src/backend/opencl/kernel/cscmv.hpp b/src/backend/opencl/kernel/cscmv.hpp index 88008480f8..2ab88b202c 100644 --- a/src/backend/opencl/kernel/cscmv.hpp +++ b/src/backend/opencl/kernel/cscmv.hpp @@ -32,39 +32,64 @@ void cscmv(Param out, const Param &values, const Param &colIdx, bool is_conj) { // TODO: rows_per_group limited by register pressure. Find better way to // handle this. + constexpr int threads_per_g = 64; constexpr int rows_per_group = 64; const bool use_alpha = (alpha != scalar(1.0)); const bool use_beta = (beta != scalar(0.0)); - cl::NDRange local(THREADS_PER_GROUP); + cl::NDRange local(threads_per_g); - std::array targs = { + int K = colIdx.info.dims[0] - 1; + int M = out.info.dims[0]; + + std::array targs = { TemplateTypename(), TemplateArg(use_alpha), - TemplateArg(use_beta), TemplateArg(is_conj), - TemplateArg(rows_per_group), TemplateArg(local[0]), + TemplateArg(is_conj), TemplateArg(rows_per_group), + TemplateArg(local[0]), }; - std::array options = { + std::array options = { DefineKeyValue(T, dtype_traits::getName()), DefineKeyValue(USE_ALPHA, use_alpha), - DefineKeyValue(USE_BETA, use_beta), DefineKeyValue(IS_CONJ, is_conj), DefineKeyValue(THREADS, local[0]), DefineKeyValue(ROWS_PER_GROUP, rows_per_group), DefineKeyValue(IS_CPLX, (iscplx() ? 1 : 0)), + DefineKeyValue(IS_DBL, (isdbl() ? 1 : 0)), + DefineKeyValue(IS_LONG, (islong() ? 1 : 0)), getTypeBuildDefinition()}; - auto cscmvBlock = - common::getKernel("cscmv_block", {{cscmv_cl_src}}, targs, options); + if(use_beta) { + std::array targs_beta = { + TemplateTypename(), TemplateArg(is_conj), + TemplateArg(rows_per_group), TemplateArg(local[0])}; + std::array options_beta = { + DefineKeyValue(T, dtype_traits::getName()), + DefineKeyValue(IS_CONJ, is_conj), + DefineKeyValue(THREADS, local[0]), + DefineKeyValue(ROWS_PER_GROUP, rows_per_group), + DefineKeyValue(IS_CPLX, (iscplx() ? 1 : 0)), + DefineKeyValue(IS_DBL, (isdbl() ? 1 : 0)), + DefineKeyValue(IS_LONG, (islong() ? 1 : 0)), + getTypeBuildDefinition()}; + + int groups_x = divup(M, rows_per_group * threads_per_g); + cl::NDRange global(local[0] * groups_x, 1); + auto cscmvBeta = common::getKernel("cscmv_beta", {{cscmv_cl_src}}, targs_beta, options_beta); + cscmvBeta(cl::EnqueueArgs(getQueue(), global, local), *out.data, M, beta); + + } else { + getQueue().enqueueFillBuffer(*out.data, 0, 0, M * sizeof(T)); + } - int K = colIdx.info.dims[0] - 1; - int M = out.info.dims[0]; int groups_x = divup(M, rows_per_group); cl::NDRange global(local[0] * groups_x, 1); - cscmvBlock(cl::EnqueueArgs(getQueue(), global, local), *out.data, - *values.data, *colIdx.data, *rowIdx.data, M, K, *rhs.data, - rhs.info, alpha, beta); + auto cscmvAtomic = + common::getKernel("cscmv_atomic", {{cscmv_cl_src}}, targs, options); + cscmvAtomic(cl::EnqueueArgs(getQueue(), global, local), *out.data, + *values.data, *colIdx.data, *rowIdx.data, K, *rhs.data, + rhs.info, alpha); CL_DEBUG_FINISH(getQueue()); } } // namespace kernel diff --git a/src/backend/opencl/kernel/csr2dense.cl b/src/backend/opencl/kernel/csr2dense.cl index 15a7c0c60d..e15ef014f3 100644 --- a/src/backend/opencl/kernel/csr2dense.cl +++ b/src/backend/opencl/kernel/csr2dense.cl @@ -9,13 +9,16 @@ kernel void csr2Dense(global T *output, global const T *values, global const int *rowidx, global const int *colidx, - const int M) { + const int M, const int v_off, const int r_off, const int c_off) { + T *v = values + v_off; + int *r = rowidx + r_off; + int *c = colidx + c_off; int lid = get_local_id(0); for (int rowId = get_group_id(0); rowId < M; rowId += get_num_groups(0)) { - int colStart = rowidx[rowId]; - int colEnd = rowidx[rowId + 1]; + int colStart = r[rowId]; + int colEnd = r[rowId + 1]; for (int colId = colStart + lid; colId < colEnd; colId += THREADS) { - output[rowId + colidx[colId] * M] = values[colId]; + output[rowId + c[colId] * M] = v[colId]; } } } diff --git a/src/backend/opencl/kernel/flood_fill.cl b/src/backend/opencl/kernel/flood_fill.cl index 0a7916fd49..58d03b52e8 100644 --- a/src/backend/opencl/kernel/flood_fill.cl +++ b/src/backend/opencl/kernel/flood_fill.cl @@ -23,8 +23,8 @@ kernel void init_seeds(global T *out, KParam oInfo, global const uint *seedsx, KParam syInfo) { uint tid = get_global_id(0); if (tid < sxInfo.dims[0]) { - uint x = seedsx[tid]; - uint y = seedsy[tid]; + uint x = seedsx[tid + sxInfo.offset]; + uint y = seedsy[tid + syInfo.offset]; out[(x * oInfo.strides[0] + y * oInfo.strides[1])] = VALID; } } @@ -42,13 +42,7 @@ int barrierOR(local int *predicates) { barrier(CLK_LOCAL_MEM_FENCE); } int retVal = predicates[0]; -#if AF_IS_PLATFORM_NVIDIA - // Without the extra barrier sync after reading the reduction result, - // the caller's loop is going into infinite loop occasionally which is - // in turn randoms hangs. This doesn't seem to be an issue on non-nvidia - // hardware. Hence, the check. barrier(CLK_LOCAL_MEM_FENCE); -#endif return retVal; } @@ -82,14 +76,15 @@ kernel void flood_step(global T *out, KParam oInfo, global const T *img, T tImgVal = img[(clamp(gx, 0, (int)(iInfo.dims[0] - 1)) * iInfo.strides[0] + - clamp(gy, 0, (int)(iInfo.dims[1] - 1)) * iInfo.strides[1])]; + clamp(gy, 0, (int)(iInfo.dims[1] - 1)) * iInfo.strides[1])+ + iInfo.offset]; const int isPxBtwnThresholds = (tImgVal >= lowValue && tImgVal <= highValue); int tid = lx + get_local_size(0) * ly; barrier(CLK_LOCAL_MEM_FENCE); - + T origOutVal = lmem[j][i]; bool isBorderPxl = (lx == 0 || ly == 0 || lx == (get_local_size(0) - 1) || ly == (get_local_size(1) - 1)); diff --git a/src/backend/opencl/kernel/flood_fill.hpp b/src/backend/opencl/kernel/flood_fill.hpp index 793ae5adcd..8035a61fd6 100644 --- a/src/backend/opencl/kernel/flood_fill.hpp +++ b/src/backend/opencl/kernel/flood_fill.hpp @@ -84,8 +84,6 @@ void floodFill(Param out, const Param image, const Param seedsx, DefineKeyValue(LMEM_WIDTH, (THREADS_X + 2 * RADIUS)), DefineKeyValue(LMEM_HEIGHT, (THREADS_Y + 2 * RADIUS)), DefineKeyValue(GROUP_SIZE, (THREADS_Y * THREADS_X)), - DefineKeyValue(AF_IS_PLATFORM_NVIDIA, (int)(AFCL_PLATFORM_NVIDIA == - getActivePlatformVendor())), getTypeBuildDefinition()}; auto floodStep = diff --git a/src/backend/opencl/kernel/index.cl b/src/backend/opencl/kernel/index.cl index 85e6e10cc0..2cc3cb57fe 100644 --- a/src/backend/opencl/kernel/index.cl +++ b/src/backend/opencl/kernel/index.cl @@ -10,6 +10,7 @@ typedef struct { int offs[4]; int strds[4]; + int steps[4]; char isSeq[4]; } IndexKernelParam_t; @@ -47,14 +48,18 @@ kernel void indexKernel(global T* optr, KParam oInfo, global const T* iptr, if (gx < oInfo.dims[0] && gy < oInfo.dims[1] && gz < oInfo.dims[2] && gw < oInfo.dims[3]) { // calculate pointer offsets for input - int i = p.strds[0] * - trimIndex(s0 ? gx + p.offs[0] : ptr0[gx], iInfo.dims[0]); - int j = p.strds[1] * - trimIndex(s1 ? gy + p.offs[1] : ptr1[gy], iInfo.dims[1]); - int k = p.strds[2] * - trimIndex(s2 ? gz + p.offs[2] : ptr2[gz], iInfo.dims[2]); - int l = p.strds[3] * - trimIndex(s3 ? gw + p.offs[3] : ptr3[gw], iInfo.dims[3]); + int i = + p.strds[0] * trimIndex(s0 ? gx * p.steps[0] + p.offs[0] : ptr0[gx], + iInfo.dims[0]); + int j = + p.strds[1] * trimIndex(s1 ? gy * p.steps[1] + p.offs[1] : ptr1[gy], + iInfo.dims[1]); + int k = + p.strds[2] * trimIndex(s2 ? gz * p.steps[2] + p.offs[2] : ptr2[gz], + iInfo.dims[2]); + int l = + p.strds[3] * trimIndex(s3 ? gw * p.steps[3] + p.offs[3] : ptr3[gw], + iInfo.dims[3]); // offset input and output pointers global const T* src = iptr + (i + j + k + l) + iInfo.offset; global T* dst = optr + (gx * oInfo.strides[0] + gy * oInfo.strides[1] + diff --git a/src/backend/opencl/kernel/index.hpp b/src/backend/opencl/kernel/index.hpp index 9433893b96..5362a8e78b 100644 --- a/src/backend/opencl/kernel/index.hpp +++ b/src/backend/opencl/kernel/index.hpp @@ -26,6 +26,7 @@ namespace kernel { typedef struct { int offs[4]; int strds[4]; + int steps[4]; char isSeq[4]; } IndexKernelParam_t; diff --git a/src/backend/opencl/kernel/ireduce.hpp b/src/backend/opencl/kernel/ireduce.hpp index 1bbcf08d2b..d056fb8fea 100644 --- a/src/backend/opencl/kernel/ireduce.hpp +++ b/src/backend/opencl/kernel/ireduce.hpp @@ -251,13 +251,14 @@ T ireduceAll(uint *loc, Param in) { int in_elements = in.info.dims[0] * in.info.dims[1] * in.info.dims[2] * in.info.dims[3]; + bool is_linear = (in.info.strides[0] == 1); + for (int k = 1; k < 4; k++) { + is_linear &= (in.info.strides[k] == + (in.info.strides[k - 1] * in.info.dims[k - 1])); + } + // FIXME: Use better heuristics to get to the optimum number - if (in_elements > 4096) { - bool is_linear = (in.info.strides[0] == 1); - for (int k = 1; k < 4; k++) { - is_linear &= (in.info.strides[k] == - (in.info.strides[k - 1] * in.info.dims[k - 1])); - } + if (!is_linear || in_elements > 4096) { if (is_linear) { in.info.dims[0] = in_elements; for (int k = 1; k < 4; k++) { diff --git a/src/backend/opencl/kernel/lookup.cl b/src/backend/opencl/kernel/lookup.cl index 622a47e8f6..7ed4bc1cfa 100644 --- a/src/backend/opencl/kernel/lookup.cl +++ b/src/backend/opencl/kernel/lookup.cl @@ -31,7 +31,7 @@ kernel void lookupND(global in_t *out, KParam oInfo, global const in_t *in, int gx = get_local_size(0) * (get_group_id(0) - gz * nBBS0) + lx; int gy = get_local_size(1) * (get_group_id(1) - gw * nBBS1) + ly; - global const idx_t *idxPtr = indices; + global const idx_t *idxPtr = indices + idxInfo.offset; int i = iInfo.strides[0] * (DIM == 0 ? trimIndex((int)idxPtr[gx], iInfo.dims[0]) : gx); diff --git a/src/backend/opencl/kernel/random_engine_write.cl b/src/backend/opencl/kernel/random_engine_write.cl index 8711987e44..c36c5f1d6d 100644 --- a/src/backend/opencl/kernel/random_engine_write.cl +++ b/src/backend/opencl/kernel/random_engine_write.cl @@ -27,6 +27,26 @@ float getFloatNegative11(uint num) { // Writes without boundary checking +void writeOut128Bytes_schar(global char *out, uint index, uint r1, uint r2, + uint r3, uint r4) { + out[index] = r1; + out[index + THREADS] = r1 >> 8; + out[index + 2 * THREADS] = r1 >> 16; + out[index + 3 * THREADS] = r1 >> 24; + out[index + 4 * THREADS] = r2; + out[index + 5 * THREADS] = r2 >> 8; + out[index + 6 * THREADS] = r2 >> 16; + out[index + 7 * THREADS] = r2 >> 24; + out[index + 8 * THREADS] = r3; + out[index + 9 * THREADS] = r3 >> 8; + out[index + 10 * THREADS] = r3 >> 16; + out[index + 11 * THREADS] = r3 >> 24; + out[index + 12 * THREADS] = r4; + out[index + 13 * THREADS] = r4 >> 8; + out[index + 14 * THREADS] = r4 >> 16; + out[index + 15 * THREADS] = r4 >> 24; +} + void writeOut128Bytes_uchar(global uchar *out, uint index, uint r1, uint r2, uint r3, uint r4) { out[index] = r1; @@ -154,6 +174,36 @@ void boxMullerTransform(T *const out1, T *const out2, T r1, T r2) { // Writes with boundary checking +void partialWriteOut128Bytes_schar(global char *out, uint index, uint r1, + uint r2, uint r3, uint r4, uint elements) { + if (index < elements) { out[index] = r1; } + if (index + THREADS < elements) { out[index + THREADS] = r1 >> 8; } + if (index + 2 * THREADS < elements) { out[index + 2 * THREADS] = r1 >> 16; } + if (index + 3 * THREADS < elements) { out[index + 3 * THREADS] = r1 >> 24; } + if (index + 4 * THREADS < elements) { out[index + 4 * THREADS] = r2; } + if (index + 5 * THREADS < elements) { out[index + 5 * THREADS] = r2 >> 8; } + if (index + 6 * THREADS < elements) { out[index + 6 * THREADS] = r2 >> 16; } + if (index + 7 * THREADS < elements) { out[index + 7 * THREADS] = r2 >> 24; } + if (index + 8 * THREADS < elements) { out[index + 8 * THREADS] = r3; } + if (index + 9 * THREADS < elements) { out[index + 9 * THREADS] = r3 >> 8; } + if (index + 10 * THREADS < elements) { + out[index + 10 * THREADS] = r3 >> 16; + } + if (index + 11 * THREADS < elements) { + out[index + 11 * THREADS] = r3 >> 24; + } + if (index + 12 * THREADS < elements) { out[index + 12 * THREADS] = r4; } + if (index + 13 * THREADS < elements) { + out[index + 13 * THREADS] = r4 >> 8; + } + if (index + 14 * THREADS < elements) { + out[index + 14 * THREADS] = r4 >> 16; + } + if (index + 15 * THREADS < elements) { + out[index + 15 * THREADS] = r4 >> 24; + } +} + void partialWriteOut128Bytes_uchar(global uchar *out, uint index, uint r1, uint r2, uint r3, uint r4, uint elements) { if (index < elements) { out[index] = r1; } diff --git a/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl b/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl index 66bbb3e6d2..76941ebbd7 100644 --- a/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl +++ b/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl @@ -82,12 +82,12 @@ kernel void reduce_blocks_by_key_dim(global int *reduced_block_sizes, Tk k; To v; if (gidx < n) { - k = iKeys[gidx]; + k = iKeys[gidx + iKInfo.offset]; const int gid = bidw * iVInfo.strides[dims_ordering[3]] + bidz * iVInfo.strides[dims_ordering[2]] + bidy * iVInfo.strides[dims_ordering[1]] + gidx * iVInfo.strides[DIM]; - v = transform(iVals[gid]); + v = transform(iVals[gid + iVInfo.offset]); if (change_nan) v = IS_NAN(v) ? nanval : v; } else { v = init_val; diff --git a/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl b/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl index f184e94818..c01d3c250d 100644 --- a/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl +++ b/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl @@ -72,10 +72,10 @@ kernel void reduce_blocks_by_key_first(global int *reduced_block_sizes, Tk k; To v; if (gid < n) { - k = iKeys[gid]; + k = iKeys[gid + iKInfo.offset]; const int bOffset = bidw * iVInfo.strides[3] + bidz * iVInfo.strides[2] + bidy * iVInfo.strides[1]; - v = transform(iVals[bOffset + gid]); + v = transform(iVals[bOffset + gid + iVInfo.offset]); if (change_nan) v = IS_NAN(v) ? nanval : v; } else { v = init_val; diff --git a/src/backend/opencl/kernel/reduce_by_key_compact.cl b/src/backend/opencl/kernel/reduce_by_key_compact.cl index c8081e45e9..58b78cd894 100644 --- a/src/backend/opencl/kernel/reduce_by_key_compact.cl +++ b/src/backend/opencl/kernel/reduce_by_key_compact.cl @@ -31,8 +31,8 @@ kernel void compact(global int *reduced_block_sizes, global Tk *oKeys, : (reduced_block_sizes[bid] - reduced_block_sizes[bid - 1]); int writeloc = (bid == 0) ? 0 : reduced_block_sizes[bid - 1]; - k = iKeys[gid]; - v = iVals[bOffset + gid]; + k = iKeys[gid + iKInfo.offset]; + v = iVals[bOffset + gid + iVInfo.offset]; if (lid < nwrite) { oKeys[writeloc + lid] = k; diff --git a/src/backend/opencl/kernel/reduce_by_key_compact_dim.cl b/src/backend/opencl/kernel/reduce_by_key_compact_dim.cl index 285d4cc20c..3d07a63eb7 100644 --- a/src/backend/opencl/kernel/reduce_by_key_compact_dim.cl +++ b/src/backend/opencl/kernel/reduce_by_key_compact_dim.cl @@ -43,8 +43,8 @@ kernel void compact_dim(global int *reduced_block_sizes, global Tk *oKeys, bidz * iVInfo.strides[dim_ordering[2]] + bidy * iVInfo.strides[dim_ordering[1]] + gidx * iVInfo.strides[DIM]; - k = iKeys[gidx]; - v = iVals[tid]; + k = iKeys[gidx + iKInfo.offset]; + v = iVals[tid + iVInfo.offset]; if (lid < nwrite) { oKeys[writeloc + lid] = k; diff --git a/src/backend/opencl/kernel/reduce_by_key_needs_reduction.cl b/src/backend/opencl/kernel/reduce_by_key_needs_reduction.cl index 4b12830aaf..c505689bff 100644 --- a/src/backend/opencl/kernel/reduce_by_key_needs_reduction.cl +++ b/src/backend/opencl/kernel/reduce_by_key_needs_reduction.cl @@ -32,8 +32,8 @@ kernel void test_needs_reduction(global int *needs_another_reduction, // last thread in each block checks if any inter-block keys need further // reduction if (gid == ((bid + 1) * DIMX) - 1 && bid < get_num_groups(0) - 1) { - int k0 = iKeys[gid]; - int k1 = iKeys[gid + 1]; + int k0 = iKeys[gid + iKInfo.offset]; + int k1 = iKeys[gid + 1 + iKInfo.offset]; if (k0 == k1) { atomic_or(needs_block_boundary_reduced, 1); } } } diff --git a/src/backend/opencl/kernel/scan_dim_by_key.cl b/src/backend/opencl/kernel/scan_dim_by_key.cl index 5446b28e29..eacd7f9283 100644 --- a/src/backend/opencl/kernel/scan_dim_by_key.cl +++ b/src/backend/opencl/kernel/scan_dim_by_key.cl @@ -34,7 +34,7 @@ kernel void scanDimByKeyNonfinal( // Hence increment ids[kDim] just after offseting out and before offsetting // in tData += ids[3] * tInfo.strides[3] + ids[2] * tInfo.strides[2] + - ids[1] * tInfo.strides[1] + ids[0]; + ids[1] * tInfo.strides[1] + ids[0] ; tfData += ids[3] * tfInfo.strides[3] + ids[2] * tfInfo.strides[2] + ids[1] * tfInfo.strides[1] + ids[0]; tiData += ids[3] * tiInfo.strides[3] + ids[2] * tiInfo.strides[2] + @@ -45,10 +45,9 @@ kernel void scanDimByKeyNonfinal( oData += ids[3] * oInfo.strides[3] + ids[2] * oInfo.strides[2] + ids[1] * oInfo.strides[1] + ids[0]; iData += ids[3] * iInfo.strides[3] + ids[2] * iInfo.strides[2] + - ids[1] * iInfo.strides[1] + ids[0]; + ids[1] * iInfo.strides[1] + ids[0] + iInfo.offset; kData += ids[3] * kInfo.strides[3] + ids[2] * kInfo.strides[2] + - ids[1] * kInfo.strides[1] + ids[0]; - iData += iInfo.offset; + ids[1] * kInfo.strides[1] + ids[0] + kInfo.offset; int id_dim = ids[kDim]; const int out_dim = oInfo.dims[kDim]; @@ -192,10 +191,9 @@ kernel void scanDimByKeyFinal(global To *oData, KParam oInfo, oData += ids[3] * oInfo.strides[3] + ids[2] * oInfo.strides[2] + ids[1] * oInfo.strides[1] + ids[0]; iData += ids[3] * iInfo.strides[3] + ids[2] * iInfo.strides[2] + - ids[1] * iInfo.strides[1] + ids[0]; + ids[1] * iInfo.strides[1] + ids[0] + iInfo.offset; kData += ids[3] * kInfo.strides[3] + ids[2] * kInfo.strides[2] + - ids[1] * kInfo.strides[1] + ids[0]; - iData += iInfo.offset; + ids[1] * kInfo.strides[1] + ids[0] + kInfo.offset; int id_dim = ids[kDim]; const int out_dim = oInfo.dims[kDim]; diff --git a/src/backend/opencl/kernel/scan_first_by_key.cl b/src/backend/opencl/kernel/scan_first_by_key.cl index 54d572d965..1793f0b293 100644 --- a/src/backend/opencl/kernel/scan_first_by_key.cl +++ b/src/backend/opencl/kernel/scan_first_by_key.cl @@ -39,13 +39,13 @@ kernel void scanFirstByKeyNonfinal(global To *oData, KParam oInfo, yid * kInfo.strides[1] + kInfo.offset; tData += wid * tInfo.strides[3] + zid * tInfo.strides[2] + - yid * tInfo.strides[1] + tInfo.offset; + yid * tInfo.strides[1]; tfData += wid * tfInfo.strides[3] + zid * tfInfo.strides[2] + - yid * tfInfo.strides[1] + tfInfo.offset; + yid * tfInfo.strides[1]; tiData += wid * tiInfo.strides[3] + zid * tiInfo.strides[2] + - yid * tiInfo.strides[1] + tiInfo.offset; + yid * tiInfo.strides[1]; oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] + yid * oInfo.strides[1] + oInfo.offset; @@ -179,7 +179,7 @@ kernel void scanFirstByKeyFinal(global To *oData, KParam oInfo, yid * kInfo.strides[1] + kInfo.offset; oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] + - yid * oInfo.strides[1] + oInfo.offset; + yid * oInfo.strides[1]; local To l_val0[SHARED_MEM_SIZE]; local To l_val1[SHARED_MEM_SIZE]; @@ -283,13 +283,13 @@ kernel void bcastFirstByKey(global To *oData, KParam oInfo, if (cond) { tiData += wid * tiInfo.strides[3] + zid * tiInfo.strides[2] + - yid * tiInfo.strides[1] + tiInfo.offset; + yid * tiInfo.strides[1]; tData += wid * tInfo.strides[3] + zid * tInfo.strides[2] + - yid * tInfo.strides[1] + tInfo.offset; + yid * tInfo.strides[1]; oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] + - yid * oInfo.strides[1] + oInfo.offset; + yid * oInfo.strides[1]; int boundary = tiData[groupId_x]; To accum = tData[groupId_x - 1]; diff --git a/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp b/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp index dd74cccc7e..dd14eee6c5 100644 --- a/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp +++ b/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp @@ -9,7 +9,7 @@ #include -// SBK_TYPES:float double int uint intl uintl short ushort char uchar half +// SBK_TYPES:float double int uint intl uintl short ushort char schar uchar half namespace arrayfire { namespace opencl { diff --git a/src/backend/opencl/kernel/sort_by_key_impl.hpp b/src/backend/opencl/kernel/sort_by_key_impl.hpp index a070a60c67..f03721d01e 100644 --- a/src/backend/opencl/kernel/sort_by_key_impl.hpp +++ b/src/backend/opencl/kernel/sort_by_key_impl.hpp @@ -248,6 +248,7 @@ void sort0ByKey(Param pKey, Param pVal, bool isAscending) { INSTANTIATE(Tk, short) \ INSTANTIATE(Tk, ushort) \ INSTANTIATE(Tk, char) \ + INSTANTIATE(Tk, schar) \ INSTANTIATE(Tk, uchar) \ INSTANTIATE(Tk, intl) \ INSTANTIATE(Tk, uintl) \ diff --git a/src/backend/opencl/kernel/sparse.hpp b/src/backend/opencl/kernel/sparse.hpp index e1b29c986c..4d3a33d14a 100644 --- a/src/backend/opencl/kernel/sparse.hpp +++ b/src/backend/opencl/kernel/sparse.hpp @@ -39,7 +39,7 @@ void coo2dense(Param out, const Param values, const Param rowIdx, }; std::vector compileOpts = { DefineKeyValue(T, dtype_traits::getName()), - DefineKeyValue(resp, REPEAT), + DefineKeyValue(reps, REPEAT), }; compileOpts.emplace_back(getTypeBuildDefinition()); @@ -49,7 +49,8 @@ void coo2dense(Param out, const Param values, const Param rowIdx, cl::NDRange local(THREADS_PER_GROUP, 1, 1); cl::NDRange global( - divup(out.info.dims[0], local[0] * REPEAT) * THREADS_PER_GROUP, 1, 1); + divup(values.info.dims[0], local[0] * REPEAT) * THREADS_PER_GROUP, 1, + 1); coo2dense(cl::EnqueueArgs(getQueue(), global, local), *out.data, out.info, *values.data, values.info, *rowIdx.data, rowIdx.info, @@ -84,7 +85,10 @@ void csr2dense(Param output, const Param values, const Param rowIdx, cl::NDRange global(local[0] * groups_x, 1); csr2dense(cl::EnqueueArgs(getQueue(), global, local), *output.data, - *values.data, *rowIdx.data, *colIdx.data, M); + *values.data, *rowIdx.data, *colIdx.data, M, + static_cast(values.info.offset), + static_cast(rowIdx.info.offset), + static_cast(colIdx.info.offset)); CL_DEBUG_FINISH(getQueue()); } diff --git a/src/backend/opencl/kernel/sparse_arith.hpp b/src/backend/opencl/kernel/sparse_arith.hpp index 313fa902d2..17cd67ca8a 100644 --- a/src/backend/opencl/kernel/sparse_arith.hpp +++ b/src/backend/opencl/kernel/sparse_arith.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include diff --git a/src/backend/opencl/kernel/transform.cl b/src/backend/opencl/kernel/transform.cl index 85c6a293ab..4fae1c05f8 100644 --- a/src/backend/opencl/kernel/transform.cl +++ b/src/backend/opencl/kernel/transform.cl @@ -133,7 +133,7 @@ kernel void transformKernel(global T *d_out, const KParam out, const int transf_len = 6; float tmat[6]; #endif - global const float *tmat_ptr = c_tmat + t_idx * transf_len; + global const float *tmat_ptr = c_tmat + tf.offset + t_idx * transf_len; // We expect a inverse transform matrix by default // If it is an forward transform, then we need its inverse diff --git a/src/backend/opencl/lookup.cpp b/src/backend/opencl/lookup.cpp index 2fee6f6ae0..83bca0ac44 100644 --- a/src/backend/opencl/lookup.cpp +++ b/src/backend/opencl/lookup.cpp @@ -25,8 +25,8 @@ Array lookup(const Array &input, const Array &indices, const dim4 &iDims = input.dims(); dim4 oDims(1); - for (int d = 0; d < 4; ++d) { - oDims[d] = (d == int(dim) ? indices.elements() : iDims[d]); + for (dim_t d = 0; d < 4; ++d) { + oDims[d] = (d == dim ? indices.elements() : iDims[d]); } Array out = createEmptyArray(oDims); @@ -53,6 +53,8 @@ Array lookup(const Array &input, const Array &indices, const unsigned); \ template Array lookup(const Array &, const Array &, \ const unsigned); \ + template Array lookup(const Array &, const Array &, \ + const unsigned); \ template Array lookup(const Array &, const Array &, \ const unsigned); \ template Array lookup(const Array &, const Array &, \ @@ -66,6 +68,7 @@ INSTANTIATE(int); INSTANTIATE(unsigned); INSTANTIATE(intl); INSTANTIATE(uintl); +INSTANTIATE(schar); INSTANTIATE(uchar); INSTANTIATE(char); INSTANTIATE(ushort); diff --git a/src/backend/opencl/match_template.cpp b/src/backend/opencl/match_template.cpp index f97bc6d353..7f02d886b3 100644 --- a/src/backend/opencl/match_template.cpp +++ b/src/backend/opencl/match_template.cpp @@ -37,6 +37,7 @@ INSTANTIATE(float, float) INSTANTIATE(char, float) INSTANTIATE(int, float) INSTANTIATE(uint, float) +INSTANTIATE(schar, float) INSTANTIATE(uchar, float) INSTANTIATE(short, float) INSTANTIATE(ushort, float) diff --git a/src/backend/opencl/math.hpp b/src/backend/opencl/math.hpp index e4745d9e92..f164c3002c 100644 --- a/src/backend/opencl/math.hpp +++ b/src/backend/opencl/math.hpp @@ -18,6 +18,7 @@ #include #include +#include #include #if defined(__GNUC__) || defined(__GNUG__) diff --git a/src/backend/opencl/max.cpp b/src/backend/opencl/max.cpp index b2a2cdfdf0..695415517d 100644 --- a/src/backend/opencl/max.cpp +++ b/src/backend/opencl/max.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_max_t, uint, uint) INSTANTIATE(af_max_t, intl, intl) INSTANTIATE(af_max_t, uintl, uintl) INSTANTIATE(af_max_t, char, char) +INSTANTIATE(af_max_t, schar, schar) INSTANTIATE(af_max_t, uchar, uchar) INSTANTIATE(af_max_t, short, short) INSTANTIATE(af_max_t, ushort, ushort) diff --git a/src/backend/opencl/mean.cpp b/src/backend/opencl/mean.cpp index 7bd586e587..428c2812c3 100644 --- a/src/backend/opencl/mean.cpp +++ b/src/backend/opencl/mean.cpp @@ -59,6 +59,7 @@ INSTANTIATE(intl, double, double); INSTANTIATE(uintl, double, double); INSTANTIATE(short, float, float); INSTANTIATE(ushort, float, float); +INSTANTIATE(schar, float, float); INSTANTIATE(uchar, float, float); INSTANTIATE(char, float, float); INSTANTIATE(cfloat, float, cfloat); diff --git a/src/backend/opencl/meanshift.cpp b/src/backend/opencl/meanshift.cpp index 3c6f140c98..9eaec9db9d 100644 --- a/src/backend/opencl/meanshift.cpp +++ b/src/backend/opencl/meanshift.cpp @@ -38,6 +38,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/medfilt.cpp b/src/backend/opencl/medfilt.cpp index 66a4c6969e..d3025a50b9 100644 --- a/src/backend/opencl/medfilt.cpp +++ b/src/backend/opencl/medfilt.cpp @@ -55,6 +55,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/memory.cpp b/src/backend/opencl/memory.cpp index d2e0190431..7c69b33e24 100644 --- a/src/backend/opencl/memory.cpp +++ b/src/backend/opencl/memory.cpp @@ -162,6 +162,7 @@ INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/opencl/min.cpp b/src/backend/opencl/min.cpp index 9cc6a09272..75c117caa8 100644 --- a/src/backend/opencl/min.cpp +++ b/src/backend/opencl/min.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_min_t, uint, uint) INSTANTIATE(af_min_t, intl, intl) INSTANTIATE(af_min_t, uintl, uintl) INSTANTIATE(af_min_t, char, char) +INSTANTIATE(af_min_t, schar, schar) INSTANTIATE(af_min_t, uchar, uchar) INSTANTIATE(af_min_t, short, short) INSTANTIATE(af_min_t, ushort, ushort) diff --git a/src/backend/opencl/moments.cpp b/src/backend/opencl/moments.cpp index 0b03d203c9..80afc2ece1 100644 --- a/src/backend/opencl/moments.cpp +++ b/src/backend/opencl/moments.cpp @@ -47,6 +47,7 @@ INSTANTIATE(float) INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(ushort) diff --git a/src/backend/opencl/morph.cpp b/src/backend/opencl/morph.cpp index e77b7a063c..a1cb86aa03 100644 --- a/src/backend/opencl/morph.cpp +++ b/src/backend/opencl/morph.cpp @@ -57,6 +57,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/nearest_neighbour.cpp b/src/backend/opencl/nearest_neighbour.cpp index 535be4083f..615165a8e5 100644 --- a/src/backend/opencl/nearest_neighbour.cpp +++ b/src/backend/opencl/nearest_neighbour.cpp @@ -80,6 +80,7 @@ INSTANTIATE(intl, intl) INSTANTIATE(uintl, uintl) INSTANTIATE(short, int) INSTANTIATE(ushort, uint) +INSTANTIATE(schar, int) INSTANTIATE(uchar, uint) INSTANTIATE(uintl, uint) // For Hamming diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp index 165eded95f..b6886c97bb 100644 --- a/src/backend/opencl/platform.cpp +++ b/src/backend/opencl/platform.cpp @@ -278,6 +278,16 @@ afcl::platform getActivePlatformVendor() { return devMngr.mPlatforms[get<1>(devId)].second; } +bool isDeviceBufferAccessible(int buf_device_id, int execution_id) { + DeviceManager& devMngr = DeviceManager::getInstance(); + + common::lock_guard_t lock(devMngr.deviceMutex); + + return buf_device_id == execution_id || + *devMngr.mContexts[buf_device_id] == + *devMngr.mContexts[execution_id]; +} + const Context& getContext() { device_id_t& devId = tlocalActiveDeviceId(); @@ -296,9 +306,10 @@ cl_command_queue getQueueHandle(int device_id) { return (*(devMngr.mQueues[device_id]))(); } -CommandQueue& getQueue() { - device_id_t& devId = tlocalActiveDeviceId(); - +CommandQueue& getQueue(int device_id) { + device_id_t devId = + (device_id = -1) ? tlocalActiveDeviceId() + : make_pair(device_id, device_id); DeviceManager& devMngr = DeviceManager::getInstance(); common::lock_guard_t lock(devMngr.deviceMutex); @@ -326,13 +337,17 @@ const std::string& getActiveDeviceBaseBuildFlags() { } vector getOpenCLCDeviceVersion(const Device& device) { - Platform device_platform(device.getInfo(), false); + // For OpenCL-HPP >= v2023.12.14 type is cl::Platform instead of + // cl_platform_id + Platform device_platform; + device_platform = device.getInfo(); + auto platform_version = device_platform.getInfo(); vector out; - /// The ifdef allows us to support BUILDING ArrayFire with older versions of - /// OpenCL where as the if condition in the ifdef allows us to support older - /// versions of OpenCL at runtime + /// The ifdef allows us to support BUILDING ArrayFire with older + /// versions of OpenCL where as the if condition in the ifdef allows us + /// to support older versions of OpenCL at runtime #ifdef CL_DEVICE_OPENCL_C_ALL_VERSIONS if (platform_version.substr(7).c_str()[0] >= '3') { vector device_versions = @@ -519,24 +534,28 @@ void addDeviceContext(cl_device_id dev, cl_context ctx, cl_command_queue que) { { common::lock_guard_t lock(devMngr.deviceMutex); - auto tDevice = make_unique(dev, true); - auto tContext = make_unique(ctx, true); + cl::Device tDevice(dev, true); + cl::Context tContext(ctx, true); auto tQueue = - (que == NULL ? make_unique(*tContext, *tDevice) + (que == NULL ? make_unique(tContext, tDevice) : make_unique(que, true)); // FIXME: add OpenGL Interop for user provided contexts later devMngr.mIsGLSharingOn.push_back(false); devMngr.mDeviceTypes.push_back( - static_cast(tDevice->getInfo())); + static_cast(tDevice.getInfo())); - auto device_platform = tDevice->getInfo(); + // For OpenCL-HPP >= v2023.12.14 type is cl::Platform instead of + // cl_platform_id + cl::Platform device_platform; + device_platform = tDevice.getInfo(); devMngr.mPlatforms.push_back( std::make_pair, afcl_platform>( - make_unique(device_platform, true), - getPlatformEnum(*tDevice))); + make_unique(device_platform(), true), + getPlatformEnum(tDevice))); - devMngr.mDevices.push_back(move(tDevice)); - devMngr.mContexts.push_back(move(tContext)); + devMngr.mDevices.emplace_back(make_unique(move(tDevice))); + devMngr.mContexts.emplace_back( + make_unique(move(tContext))); devMngr.mQueues.push_back(move(tQueue)); nDevices = static_cast(devMngr.mDevices.size()) - 1; @@ -594,7 +613,8 @@ void removeDeviceContext(cl_device_id dev, cl_context ctx) { common::lock_guard_t lock(devMngr.deviceMutex); const int dCount = static_cast(devMngr.mDevices.size()); - for (int i = 0; i < dCount; ++i) { + for (int i = static_cast(devMngr.mUserDeviceOffset); i < dCount; + ++i) { if (devMngr.mDevices[i]->operator()() == dev && devMngr.mContexts[i]->operator()() == ctx) { deleteIdx = i; diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp index 94ab6dff52..30124d9aa2 100644 --- a/src/backend/opencl/platform.hpp +++ b/src/backend/opencl/platform.hpp @@ -65,7 +65,7 @@ int& getMaxJitSize(); const cl::Context& getContext(); -cl::CommandQueue& getQueue(); +cl::CommandQueue& getQueue(int device_id = -1); /// Return a cl_command_queue handle to the queue for the device. /// @@ -187,5 +187,12 @@ afcl::platform getPlatformEnum(cl::Device dev); void setActiveContext(int device); +/// Returns true if the buffer on device buf_device_id can be accessed by +/// kernels on device execution_id +/// +/// \param[in] buf_device_id The device id of the buffer +/// \param[in] execution_id The device where the buffer will be accessed. +bool isDeviceBufferAccessible(int buf_device_id, int execution_id); + } // namespace opencl } // namespace arrayfire diff --git a/src/backend/opencl/plot.cpp b/src/backend/opencl/plot.cpp index cc7f93262e..5b7dfa69cb 100644 --- a/src/backend/opencl/plot.cpp +++ b/src/backend/opencl/plot.cpp @@ -75,6 +75,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace opencl diff --git a/src/backend/opencl/product.cpp b/src/backend/opencl/product.cpp index f13a9b9ae3..a949f87345 100644 --- a/src/backend/opencl/product.cpp +++ b/src/backend/opencl/product.cpp @@ -24,6 +24,7 @@ INSTANTIATE(af_mul_t, uint, uint) INSTANTIATE(af_mul_t, intl, intl) INSTANTIATE(af_mul_t, uintl, uintl) INSTANTIATE(af_mul_t, char, int) +INSTANTIATE(af_mul_t, schar, int) INSTANTIATE(af_mul_t, uchar, uint) INSTANTIATE(af_mul_t, short, int) INSTANTIATE(af_mul_t, ushort, uint) diff --git a/src/backend/opencl/random_engine.cpp b/src/backend/opencl/random_engine.cpp index f2110c8be0..d307e54c2b 100644 --- a/src/backend/opencl/random_engine.cpp +++ b/src/backend/opencl/random_engine.cpp @@ -138,6 +138,7 @@ INSTANTIATE_UNIFORM(uint) INSTANTIATE_UNIFORM(intl) INSTANTIATE_UNIFORM(uintl) INSTANTIATE_UNIFORM(char) +INSTANTIATE_UNIFORM(schar) INSTANTIATE_UNIFORM(uchar) INSTANTIATE_UNIFORM(short) INSTANTIATE_UNIFORM(ushort) diff --git a/src/backend/opencl/range.cpp b/src/backend/opencl/range.cpp index 92340d34eb..a49ba931c8 100644 --- a/src/backend/opencl/range.cpp +++ b/src/backend/opencl/range.cpp @@ -47,6 +47,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/reorder.cpp b/src/backend/opencl/reorder.cpp index da485911e6..ecacccd677 100644 --- a/src/backend/opencl/reorder.cpp +++ b/src/backend/opencl/reorder.cpp @@ -40,6 +40,7 @@ INSTANTIATE(cfloat) INSTANTIATE(cdouble) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(intl) diff --git a/src/backend/opencl/resize.cpp b/src/backend/opencl/resize.cpp index ee7776b82f..bf3a8497b2 100644 --- a/src/backend/opencl/resize.cpp +++ b/src/backend/opencl/resize.cpp @@ -38,6 +38,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/opencl/rotate.cpp b/src/backend/opencl/rotate.cpp index 46caa65c88..eab0c1da26 100644 --- a/src/backend/opencl/rotate.cpp +++ b/src/backend/opencl/rotate.cpp @@ -49,6 +49,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/opencl/scan.cpp b/src/backend/opencl/scan.cpp index 0fc36366ef..649789ef91 100644 --- a/src/backend/opencl/scan.cpp +++ b/src/backend/opencl/scan.cpp @@ -43,6 +43,7 @@ Array scan(const Array& in, const int dim, bool inclusiveScan) { INSTANTIATE_SCAN(ROp, intl, intl) \ INSTANTIATE_SCAN(ROp, uintl, uintl) \ INSTANTIATE_SCAN(ROp, char, uint) \ + INSTANTIATE_SCAN(ROp, schar, int) \ INSTANTIATE_SCAN(ROp, uchar, uint) \ INSTANTIATE_SCAN(ROp, short, int) \ INSTANTIATE_SCAN(ROp, ushort, uint) diff --git a/src/backend/opencl/select.cpp b/src/backend/opencl/select.cpp index bbafbe989c..20c900007a 100644 --- a/src/backend/opencl/select.cpp +++ b/src/backend/opencl/select.cpp @@ -127,6 +127,7 @@ INSTANTIATE(uint); INSTANTIATE(intl); INSTANTIATE(uintl); INSTANTIATE(char); +INSTANTIATE(schar); INSTANTIATE(uchar); INSTANTIATE(short); INSTANTIATE(ushort); diff --git a/src/backend/opencl/set.cpp b/src/backend/opencl/set.cpp index 195cf23047..1c1b74396c 100644 --- a/src/backend/opencl/set.cpp +++ b/src/backend/opencl/set.cpp @@ -147,6 +147,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/shift.cpp b/src/backend/opencl/shift.cpp index 512c113ed1..19e37286d3 100644 --- a/src/backend/opencl/shift.cpp +++ b/src/backend/opencl/shift.cpp @@ -9,14 +9,15 @@ #include -#include #include +#include #include using af::dim4; using arrayfire::common::Node_ptr; using arrayfire::common::ShiftNodeBase; using arrayfire::opencl::jit::BufferNode; +using arrayfire::opencl::jit::ShiftNode; using std::array; using std::make_shared; using std::static_pointer_cast; @@ -24,7 +25,6 @@ using std::string; namespace arrayfire { namespace opencl { -using ShiftNode = ShiftNodeBase; template Array shift(const Array &in, const int sdims[4]) { @@ -64,6 +64,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/opencl/sobel.cpp b/src/backend/opencl/sobel.cpp index e718021b42..a7651de07d 100644 --- a/src/backend/opencl/sobel.cpp +++ b/src/backend/opencl/sobel.cpp @@ -40,6 +40,7 @@ INSTANTIATE(double, double) INSTANTIATE(int, int) INSTANTIATE(uint, int) INSTANTIATE(char, int) +INSTANTIATE(schar, int) INSTANTIATE(uchar, int) INSTANTIATE(short, int) INSTANTIATE(ushort, int) diff --git a/src/backend/opencl/sort.cpp b/src/backend/opencl/sort.cpp index 8b977316f1..e2bfcaa057 100644 --- a/src/backend/opencl/sort.cpp +++ b/src/backend/opencl/sort.cpp @@ -56,6 +56,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/sort_by_key.cpp b/src/backend/opencl/sort_by_key.cpp index 2e4b2dd616..f1a89aef4d 100644 --- a/src/backend/opencl/sort_by_key.cpp +++ b/src/backend/opencl/sort_by_key.cpp @@ -69,6 +69,7 @@ void sort_by_key(Array &okey, Array &oval, const Array &ikey, INSTANTIATE(Tk, short) \ INSTANTIATE(Tk, ushort) \ INSTANTIATE(Tk, char) \ + INSTANTIATE(Tk, schar) \ INSTANTIATE(Tk, uchar) \ INSTANTIATE(Tk, intl) \ INSTANTIATE(Tk, uintl) @@ -80,6 +81,7 @@ INSTANTIATE1(uint) INSTANTIATE1(short) INSTANTIATE1(ushort) INSTANTIATE1(char) +INSTANTIATE1(schar) INSTANTIATE1(uchar) INSTANTIATE1(intl) INSTANTIATE1(uintl) diff --git a/src/backend/opencl/sort_index.cpp b/src/backend/opencl/sort_index.cpp index 9c92f8406c..afd8bf8413 100644 --- a/src/backend/opencl/sort_index.cpp +++ b/src/backend/opencl/sort_index.cpp @@ -25,6 +25,12 @@ namespace opencl { template void sort_index(Array &okey, Array &oval, const Array &in, const uint dim, bool isAscending) { + + // TODO: fix half implementation of sort0bykey to support this + if (std::is_same_v) { + OPENCL_NOT_SUPPORTED("sort_index with half"); + } + try { // okey contains values, oval contains indices okey = copyArray(in); @@ -70,6 +76,7 @@ INSTANTIATE(double) INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/sum.cpp b/src/backend/opencl/sum.cpp index 890280ba92..1ef26bdb89 100644 --- a/src/backend/opencl/sum.cpp +++ b/src/backend/opencl/sum.cpp @@ -29,6 +29,8 @@ INSTANTIATE(af_add_t, uintl, uintl) INSTANTIATE(af_add_t, uintl, double) INSTANTIATE(af_add_t, char, int) INSTANTIATE(af_add_t, char, float) +INSTANTIATE(af_add_t, schar, int) +INSTANTIATE(af_add_t, schar, float) INSTANTIATE(af_add_t, uchar, uint) INSTANTIATE(af_add_t, uchar, float) INSTANTIATE(af_add_t, short, int) diff --git a/src/backend/opencl/surface.cpp b/src/backend/opencl/surface.cpp index a0de95fb19..7a2e15276b 100644 --- a/src/backend/opencl/surface.cpp +++ b/src/backend/opencl/surface.cpp @@ -78,6 +78,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace opencl diff --git a/src/backend/opencl/susan.cpp b/src/backend/opencl/susan.cpp index 6bd78e2540..91b011120b 100644 --- a/src/backend/opencl/susan.cpp +++ b/src/backend/opencl/susan.cpp @@ -66,6 +66,7 @@ INSTANTIATE(double) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/tile.cpp b/src/backend/opencl/tile.cpp index 14e2d5beac..98c7eb2bfb 100644 --- a/src/backend/opencl/tile.cpp +++ b/src/backend/opencl/tile.cpp @@ -41,6 +41,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/opencl/topk.cpp b/src/backend/opencl/topk.cpp index 18e03d2f0d..201ec06197 100644 --- a/src/backend/opencl/topk.cpp +++ b/src/backend/opencl/topk.cpp @@ -8,12 +8,17 @@ ********************************************************/ #include +#include #include +#include #include #include #include #include #include +#include +#include +#include #include #include @@ -157,12 +162,39 @@ void topk(Array& vals, Array& idxs, const Array& in, vals = values; idxs = indices; } else { - auto values = createEmptyArray(in.dims()); - auto indices = createEmptyArray(in.dims()); - sort_index(values, indices, in, dim, order & AF_TOPK_MIN); - auto indVec = indexForTopK(k); - vals = index(values, indVec.data()); - idxs = index(indices, indVec.data()); + + if (!std::is_same_v) { + auto values = createEmptyArray(in.dims()); + auto indices = createEmptyArray(in.dims()); + sort_index(values, indices, in, dim, order & AF_TOPK_MIN); + auto indVec = indexForTopK(k); + idxs = index(indices, indVec.data()); + vals = index(values, indVec.data()); + } else { + // Temporary implementation for topk due half not being supported in sort_index + // TODO: Fix sort_index and remove this + + auto values = createEmptyArray(in.dims()); + auto indices = createEmptyArray(in.dims()); + sort_index(values, indices, common::cast(in), dim, order & AF_TOPK_MIN); + + auto indVec = indexForTopK(k); + idxs = index(indices, indVec.data()); + + // Index values from original array by using the indices from the previous resuult + auto len = in.elements() / in.dims()[dim]; + auto index_dims = dim4(k, len); + auto new_indices = common::flat(arithOp(arithOp(range(index_dims, 1), createValueArray(index_dims, in.dims()[dim]), index_dims), idxs, index_dims)); + auto indVecVals = indexForTopK(k); + indVecVals[0].idx.arr = getHandle(new_indices); + indVecVals[0].isSeq = false; + indVecVals[0].isBatch = false; + + vals = common::modDims(index(common::flat(in), indVecVals.data()), idxs.dims()); + vals.eval(); + + releaseHandle(indVecVals[0].idx.arr); + } } } diff --git a/src/backend/opencl/traits.hpp b/src/backend/opencl/traits.hpp index 00af1d17b0..2af7257b76 100644 --- a/src/backend/opencl/traits.hpp +++ b/src/backend/opencl/traits.hpp @@ -49,6 +49,36 @@ inline bool iscplx() { return true; } +template +static bool isdbl() { + return false; +} + +template<> +inline bool isdbl() { + return true; +} + +template<> +inline bool isdbl() { + return true; +} + +template +static bool islong() { + return false; +} + +template<> +inline bool islong() { + return true; +} + +template<> +inline bool islong() { + return true; +} + template inline std::string scalar_to_option(const T &val) { using namespace arrayfire::common; diff --git a/src/backend/opencl/transform.cpp b/src/backend/opencl/transform.cpp index 14ee03c962..de99f48a60 100644 --- a/src/backend/opencl/transform.cpp +++ b/src/backend/opencl/transform.cpp @@ -9,6 +9,7 @@ #include +#include #include namespace arrayfire { @@ -18,18 +19,25 @@ template void transform(Array &out, const Array &in, const Array &tf, const af_interp_type method, const bool inverse, const bool perspective) { + // TODO: Temporary Fix, must fix handling subarrays upstream + // tf has to be linear, although offset is allowed. + const Array tf_Lin = tf.isLinear() ? tf : copyArray(tf); + switch (method) { case AF_INTERP_NEAREST: case AF_INTERP_LOWER: - kernel::transform(out, in, tf, inverse, perspective, method, 1); + kernel::transform(out, in, tf_Lin, inverse, perspective, method, + 1); break; case AF_INTERP_BILINEAR: case AF_INTERP_BILINEAR_COSINE: - kernel::transform(out, in, tf, inverse, perspective, method, 2); + kernel::transform(out, in, tf_Lin, inverse, perspective, method, + 2); break; case AF_INTERP_BICUBIC: case AF_INTERP_BICUBIC_SPLINE: - kernel::transform(out, in, tf, inverse, perspective, method, 3); + kernel::transform(out, in, tf_Lin, inverse, perspective, method, + 3); break; default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG); } @@ -49,6 +57,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/opencl/transpose.cpp b/src/backend/opencl/transpose.cpp index a25fa9be28..248de43017 100644 --- a/src/backend/opencl/transpose.cpp +++ b/src/backend/opencl/transpose.cpp @@ -43,6 +43,7 @@ INSTANTIATE(cdouble) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/opencl/transpose_inplace.cpp b/src/backend/opencl/transpose_inplace.cpp index dc23873814..d6b783e5b2 100644 --- a/src/backend/opencl/transpose_inplace.cpp +++ b/src/backend/opencl/transpose_inplace.cpp @@ -39,6 +39,7 @@ INSTANTIATE(cdouble) INSTANTIATE(char) INSTANTIATE(int) INSTANTIATE(uint) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(intl) INSTANTIATE(uintl) diff --git a/src/backend/opencl/triangle.cpp b/src/backend/opencl/triangle.cpp index cb781eeef4..346f8d1af7 100644 --- a/src/backend/opencl/triangle.cpp +++ b/src/backend/opencl/triangle.cpp @@ -47,6 +47,7 @@ INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) INSTANTIATE(char) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/types.cpp b/src/backend/opencl/types.cpp index 35c2b5745a..90393de3f9 100644 --- a/src/backend/opencl/types.cpp +++ b/src/backend/opencl/types.cpp @@ -95,6 +95,7 @@ INSTANTIATE(int); INSTANTIATE(uint); INSTANTIATE(intl); INSTANTIATE(uintl); +INSTANTIATE(schar); INSTANTIATE(uchar); INSTANTIATE(char); INSTANTIATE(half); diff --git a/src/backend/opencl/types.hpp b/src/backend/opencl/types.hpp index 620ab74ca9..48985ab837 100644 --- a/src/backend/opencl/types.hpp +++ b/src/backend/opencl/types.hpp @@ -40,6 +40,7 @@ namespace opencl { using cdouble = cl_double2; using cfloat = cl_float2; using intl = long long; +using schar = cl_char; using uchar = cl_uchar; using uint = cl_uint; using uintl = unsigned long long; @@ -93,6 +94,10 @@ inline const char *shortname(bool caps) { return caps ? "J" : "j"; } template<> +inline const char *shortname(bool caps) { + return caps ? "A" : "a"; // TODO +} +template<> inline const char *shortname(bool caps) { return caps ? "V" : "v"; } @@ -118,6 +123,11 @@ inline const char *getFullName() { return af::dtype_traits::getName(); } +template<> +inline const char *getFullName() { + return "char"; +} + template<> inline const char *getFullName() { return "float2"; diff --git a/src/backend/opencl/unwrap.cpp b/src/backend/opencl/unwrap.cpp index c6c7a12d4f..3fb0d9a14c 100644 --- a/src/backend/opencl/unwrap.cpp +++ b/src/backend/opencl/unwrap.cpp @@ -53,6 +53,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/src/backend/opencl/vector_field.cpp b/src/backend/opencl/vector_field.cpp index e470f73c9a..4d85032602 100644 --- a/src/backend/opencl/vector_field.cpp +++ b/src/backend/opencl/vector_field.cpp @@ -101,6 +101,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(short) INSTANTIATE(ushort) +INSTANTIATE(schar) INSTANTIATE(uchar) } // namespace opencl diff --git a/src/backend/opencl/where.cpp b/src/backend/opencl/where.cpp index c3ac797454..ae86cd8521 100644 --- a/src/backend/opencl/where.cpp +++ b/src/backend/opencl/where.cpp @@ -35,6 +35,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(short) INSTANTIATE(ushort) diff --git a/src/backend/opencl/wrap.cpp b/src/backend/opencl/wrap.cpp index 42d684857a..418dc9bc1f 100644 --- a/src/backend/opencl/wrap.cpp +++ b/src/backend/opencl/wrap.cpp @@ -42,6 +42,7 @@ INSTANTIATE(int) INSTANTIATE(uint) INSTANTIATE(intl) INSTANTIATE(uintl) +INSTANTIATE(schar) INSTANTIATE(uchar) INSTANTIATE(char) INSTANTIATE(short) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0cb3cbfe51..64e1feb777 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2020, ArrayFire +# Copyright (c) 2025, ArrayFire # All rights reserved. # # This file is distributed under 3-clause BSD license. @@ -20,11 +20,11 @@ if(AF_TEST_WITH_MTX_FILES) endif() if(AF_WITH_EXTERNAL_PACKAGES_ONLY) - dependency_check(GTest_FOUND) + dependency_check(GTest_FOUND "Google Tests not found.") elseif(NOT TARGET GTest::gtest) af_dep_check_and_populate(${gtest_prefix} URI https://github.com/google/googletest.git - REF release-1.12.1 + REF v1.16.0 ) if(WIN32) set(gtest_force_shared_crt ON @@ -40,7 +40,9 @@ elseif(NOT TARGET GTest::gtest) target_compile_options(gtest PRIVATE $<$:-fp-model precise>) - add_library(GTest::gtest ALIAS gtest) + if(NOT TARGET GTest::gtest) + add_library(GTest::gtest ALIAS gtest) + endif() # Hide gtest project variables mark_as_advanced( BUILD_SHARED_LIBS @@ -97,8 +99,8 @@ if(${AF_USE_RELATIVE_TEST_DIR}) else(${AF_USE_RELATIVE_TEST_DIR}) af_dep_check_and_populate(${testdata_prefix} URI https://github.com/arrayfire/arrayfire-data.git - #pinv large data set update change - REF 0144a599f913cc67c76c9227031b4100156abc25 + #Add test file for SSAS_LinearSteps + REF 05703a4897c8b89b7a0ece1dbe21ede33d226f44 ) set(TESTDATA_SOURCE_DIR "${${testdata_prefix}_SOURCE_DIR}") endif(${AF_USE_RELATIVE_TEST_DIR}) @@ -257,6 +259,11 @@ function(make_test) MTX_TEST_DIR="${ArrayFire_BINARY_DIR}/extern/matrixmarket/" ) endif() + if(AF_SKIP_UNSUPPORTED_TESTS) + target_compile_definitions(${target} + PRIVATE + SKIP_UNSUPPORTED_TESTS) + endif() if(WIN32) target_compile_definitions(${target} PRIVATE @@ -352,6 +359,7 @@ make_test(SRC moments.cpp) make_test(SRC morph.cpp) make_test(SRC nearest_neighbour.cpp CXX11) make_test(SRC nodevice.cpp CXX11) +make_test(SRC norm.cpp CXX11) if(OpenCL_FOUND) make_test(SRC ocl_ext_context.cpp @@ -370,50 +378,53 @@ if(OpenCL_FOUND) CXX11) endif() -if(CUDA_FOUND) - include(AFcuda_helpers) - foreach(backend ${enabled_backends}) - set(cuda_test_backends "cuda" "unified") - if(${backend} IN_LIST cuda_test_backends) - set(target test_cuda_${backend}) - add_executable(${target} cuda.cu) - target_include_directories(${target} - PRIVATE +if(AF_BUILD_CUDA) + if(CUDA_FOUND) + include(AFcuda_helpers) + foreach(backend ${enabled_backends}) + set(cuda_test_backends "cuda" "unified") + if(${backend} IN_LIST cuda_test_backends) + set(target test_cuda_${backend}) + add_executable(${target} cuda.cu) + target_include_directories(${target} + PRIVATE ${CMAKE_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR} - SYSTEM PRIVATE - ${ArrayFire_SOURCE_DIR}/extern/half/include) - if(${backend} STREQUAL "unified") - target_link_libraries(${target} - ArrayFire::af) - else() + ${CMAKE_CURRENT_SOURCE_DIR}) + target_include_directories(${target} + SYSTEM PRIVATE + ${ArrayFire_SOURCE_DIR}/extern/half/include) + if(${backend} STREQUAL "unified") + target_link_libraries(${target} + ArrayFire::af) + else() + target_link_libraries(${target} + ArrayFire::af${backend}) + endif() target_link_libraries(${target} - ArrayFire::af${backend}) - endif() - target_link_libraries(${target} - mmio - arrayfire_test) - - # Couldn't get Threads::Threads to work with this cuda binary. The import - # target would not add the -pthread flag which is required for this - # executable (on Ubuntu 18.04 anyway) - check_cxx_compiler_flag(-pthread pthread_flag) - if(pthread_flag) - target_link_libraries(${target} -pthread) + mmio + arrayfire_test) + + # Couldn't get Threads::Threads to work with this cuda binary. The import + # target would not add the -pthread flag which is required for this + # executable (on Ubuntu 18.04 anyway) + check_cxx_compiler_flag(-pthread pthread_flag) + if(pthread_flag) + target_link_libraries(${target} -pthread) + endif() + + af_detect_and_set_cuda_architectures(${target}) + + set_target_properties(${target} + PROPERTIES + FOLDER "Tests" + OUTPUT_NAME "cuda_${backend}") + + if(NOT ${backend} STREQUAL "unified") + af_add_test(${target} ${backend} ON) + endif() endif() - - af_detect_and_set_cuda_architectures(${target}) - - set_target_properties(${target} - PROPERTIES - FOLDER "Tests" - OUTPUT_NAME "cuda_${backend}") - - if(NOT ${backend} STREQUAL "unified") - af_add_test(${target} ${backend} ON) - endif() - endif() - endforeach() + endforeach() + endif() endif() diff --git a/test/anisotropic_diffusion.cpp b/test/anisotropic_diffusion.cpp index afeda45d52..a498d4cdd8 100644 --- a/test/anisotropic_diffusion.cpp +++ b/test/anisotropic_diffusion.cpp @@ -29,7 +29,7 @@ using std::vector; template class AnisotropicDiffusion : public ::testing::Test {}; -typedef ::testing::Types +typedef ::testing::Types TestTypes; TYPED_TEST_SUITE(AnisotropicDiffusion, TestTypes); @@ -98,12 +98,12 @@ void imageTest(string pTestFile, const float dt, const float K, if (isCurvatureDiffusion) { ASSERT_SUCCESS(af_anisotropic_diffusion(&_outArray, inArray, dt, K, - iters, fluxKind, - AF_DIFFUSION_MCDE)); + iters, fluxKind, + AF_DIFFUSION_MCDE)); } else { ASSERT_SUCCESS(af_anisotropic_diffusion(&_outArray, inArray, dt, K, - iters, fluxKind, - AF_DIFFUSION_GRAD)); + iters, fluxKind, + AF_DIFFUSION_GRAD)); } double maxima, minima, imag; @@ -142,6 +142,7 @@ void imageTest(string pTestFile, const float dt, const float K, } TYPED_TEST(AnisotropicDiffusion, GradientGrayscale) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); // Numeric values separated by underscore are arguments to fn being tested. // Divide first value by 1000 to get time step `dt` // Divide second value by 100 to get time step `K` @@ -153,6 +154,7 @@ TYPED_TEST(AnisotropicDiffusion, GradientGrayscale) { } TYPED_TEST(AnisotropicDiffusion, GradientColorImage) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); imageTest( string(TEST_DIR "/gradient_diffusion/color_00125_100_2_exp.test"), 0.125f, 1.0, 2, AF_FLUX_EXPONENTIAL); @@ -166,6 +168,7 @@ TEST(AnisotropicDiffusion, GradientInvalidInputArray) { } TYPED_TEST(AnisotropicDiffusion, CurvatureGrayscale) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); // Numeric values separated by underscore are arguments to fn being tested. // Divide first value by 1000 to get time step `dt` // Divide second value by 100 to get time step `K` @@ -177,6 +180,7 @@ TYPED_TEST(AnisotropicDiffusion, CurvatureGrayscale) { } TYPED_TEST(AnisotropicDiffusion, CurvatureColorImage) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); imageTest( string(TEST_DIR "/curvature_diffusion/color_00125_100_2_mcde.test"), 0.125f, 1.0, 2, AF_FLUX_EXPONENTIAL, true); diff --git a/test/array.cpp b/test/array.cpp index 5962797083..c5befe1fdb 100644 --- a/test/array.cpp +++ b/test/array.cpp @@ -21,8 +21,8 @@ using std::vector; template class Array : public ::testing::Test {}; -typedef ::testing::Types TestTypes; @@ -302,6 +302,17 @@ TYPED_TEST(Array, TypeAttributes) { EXPECT_FALSE(one.isbool()); EXPECT_FALSE(one.ishalf()); break; + case s8: + EXPECT_FALSE(one.isfloating()); + EXPECT_FALSE(one.isdouble()); + EXPECT_FALSE(one.issingle()); + EXPECT_FALSE(one.isrealfloating()); + EXPECT_TRUE(one.isinteger()); + EXPECT_TRUE(one.isreal()); + EXPECT_FALSE(one.iscomplex()); + EXPECT_FALSE(one.isbool()); + EXPECT_FALSE(one.ishalf()); + break; case u8: EXPECT_FALSE(one.isfloating()); EXPECT_FALSE(one.isdouble()); @@ -473,7 +484,7 @@ TEST(DeviceId, Same) { TEST(DeviceId, Different) { int ndevices = getDeviceCount(); - if (ndevices < 2) return; + if (ndevices < 2) GTEST_SKIP() << "Skipping mult-GPU test"; int id0 = getDevice(); int id1 = (id0 + 1) % ndevices; @@ -491,7 +502,8 @@ TEST(DeviceId, Different) { af_array c; af_err err = af_matmul(&c, a.get(), b.get(), AF_MAT_NONE, AF_MAT_NONE); - ASSERT_EQ(err, AF_ERR_DEVICE); + af::sync(); + ASSERT_EQ(err, AF_SUCCESS); } setDevice(id1); @@ -500,6 +512,29 @@ TEST(DeviceId, Different) { deviceGC(); } +TEST(Device, MigrateAllDevicesToAllDevices) { + int ndevices = getDeviceCount(); + if (ndevices < 2) GTEST_SKIP() << "Skipping mult-GPU test"; + + for (int i = 0; i < ndevices; i++) { + for (int j = 0; j < ndevices; j++) { + setDevice(i); + array a = constant(i * 255, 10, 10); + a.eval(); + + setDevice(j); + array b = constant(j * 256, 10, 10); + b.eval(); + + array c = a + b; + + std::vector gold(10 * 10, i * 255 + j * 256); + + ASSERT_VEC_ARRAY_EQ(gold, dim4(10, 10), c); + } + } +} + TEST(Device, empty) { array a = array(); ASSERT_EQ(a.device(), nullptr); @@ -657,3 +692,21 @@ TEST(Array, InitializerListFixDim4) { af::array b{dim4(3, 3), data.data()}; ASSERT_ARRAYS_EQ(constant(3.14, 3, 3), b); } + +TEST(Array, OtherDevice) { + if (af::getDeviceCount() == 1) GTEST_SKIP() << "Single device. Skipping"; + af::setDevice(0); + af::info(); + af::array a = constant(3, 5, 5); + a.eval(); + af::setDevice(1); + af::info(); + af::array b = constant(2, 5, 5); + b.eval(); + + af::array c = a + b; + af::eval(c); + af::sync(); + af::setDevice(0); + ASSERT_ARRAYS_EQ(constant(5, 5, 5), c); +} diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp index 4c6e966220..687de09aab 100644 --- a/test/arrayfire_test.cpp +++ b/test/arrayfire_test.cpp @@ -77,6 +77,7 @@ std::ostream &operator<<(std::ostream &os, af::dtype type) { case b8: name = "b8"; break; case s32: name = "s32"; break; case u32: name = "u32"; break; + case s8: name = "s8"; break; case u8: name = "u8"; break; case s64: name = "s64"; break; case u64: name = "u64"; break; @@ -102,14 +103,19 @@ std::string readNextNonEmptyLine(std::ifstream &file) { return result; } -std::string getBackendName() { +std::string getBackendName(bool lower) { af::Backend backend = af::getActiveBackend(); - if (backend == AF_BACKEND_OPENCL) - return std::string("opencl"); - else if (backend == AF_BACKEND_CUDA) - return std::string("cuda"); - - return std::string("cpu"); + switch (backend) { + case AF_BACKEND_CPU: + return lower ? std::string("cpu") : std::string("CPU"); + case AF_BACKEND_CUDA: + return lower ? std::string("cuda") : std::string("CUDA"); + case AF_BACKEND_OPENCL: + return lower ? std::string("opencl") : std::string("OpenCL"); + case AF_BACKEND_ONEAPI: + return lower ? std::string("oneapi") : std::string("oneAPI"); + default: return lower ? std::string("unknown") : std::string("Unknown"); + } } std::string getTestName() { @@ -162,6 +168,9 @@ ::testing::AssertionResult assertArrayEq(std::string aName, std::string bName, case u32: return elemWiseEq(aName, bName, a, b, maxAbsDiff); break; + case s8: + return elemWiseEq(aName, bName, a, b, maxAbsDiff); + break; case u8: return elemWiseEq(aName, bName, a, b, maxAbsDiff); break; @@ -220,9 +229,9 @@ ::testing::AssertionResult imageEq(std::string aName, std::string bName, af::saveImage(result_path.c_str(), b.as(f32)); af::saveImage(diff_path.c_str(), abs(a.as(f32) - b.as(f32))); - std::cout - << "" - << valid_path << "\n"; + std::cout << "" + << valid_path << "\n"; std::cout << "" << result_path << "\n"; @@ -258,6 +267,7 @@ ::testing::AssertionResult assertImageEq(std::string aName, std::string bName, << "Expected: " << aName << "([" << a.dims() << "])"; switch (arrDtype) { + case s8: return imageEq(aName, bName, a, b, maxAbsDiff); case u8: return imageEq(aName, bName, a, b, maxAbsDiff); case b8: return imageEq(aName, bName, a, b, maxAbsDiff); case s32: return imageEq(aName, bName, a, b, maxAbsDiff); @@ -344,6 +354,7 @@ INSTANTIATE(double, float, int); INSTANTIATE(int, float, int); INSTANTIATE(unsigned int, float, int); INSTANTIATE(char, float, int); +INSTANTIATE(signed char, float, int); INSTANTIATE(unsigned char, float, int); INSTANTIATE(short, float, int); INSTANTIATE(unsigned short, float, int); @@ -358,6 +369,7 @@ INSTANTIATE(unsigned int, unsigned int, unsigned int); INSTANTIATE(long long, long long, int); INSTANTIATE(unsigned long long, unsigned long long, int); INSTANTIATE(char, char, int); +INSTANTIATE(signed char, signed char, int); INSTANTIATE(unsigned char, unsigned char, int); INSTANTIATE(short, short, int); INSTANTIATE(unsigned short, unsigned short, int); @@ -366,12 +378,19 @@ INSTANTIATE(af_half, af_half, int); INSTANTIATE(float, int, int); INSTANTIATE(unsigned int, int, int); INSTANTIATE(char, int, int); +INSTANTIATE(signed char, int, int); INSTANTIATE(unsigned char, int, int); INSTANTIATE(short, int, int); INSTANTIATE(unsigned short, int, int); +INSTANTIATE(signed char, unsigned short, int); +INSTANTIATE(signed char, short, int); +INSTANTIATE(signed char, unsigned char, int); +INSTANTIATE(signed char, double, int); + INSTANTIATE(unsigned char, unsigned short, int); INSTANTIATE(unsigned char, short, int); +INSTANTIATE(unsigned char, signed char, int); INSTANTIATE(unsigned char, double, int); INSTANTIATE(long long, unsigned int, unsigned int); @@ -380,6 +399,7 @@ INSTANTIATE(int, unsigned int, unsigned int); INSTANTIATE(short, unsigned int, unsigned int); INSTANTIATE(unsigned short, unsigned int, unsigned int); INSTANTIATE(char, unsigned int, unsigned int); +INSTANTIATE(signed char, unsigned int, unsigned int); INSTANTIATE(unsigned char, unsigned int, unsigned int); INSTANTIATE(float, unsigned int, unsigned int); INSTANTIATE(double, unsigned int, unsigned int); @@ -390,12 +410,14 @@ INSTANTIATE(int, unsigned int, int); INSTANTIATE(long long, unsigned int, int); INSTANTIATE(unsigned long long, unsigned int, int); INSTANTIATE(char, unsigned int, int); +INSTANTIATE(signed char, unsigned int, int); INSTANTIATE(unsigned char, unsigned int, int); INSTANTIATE(short, unsigned int, int); INSTANTIATE(unsigned short, unsigned int, int); INSTANTIATE(float, char, int); INSTANTIATE(double, char, int); +INSTANTIATE(signed char, char, int); INSTANTIATE(unsigned char, char, int); INSTANTIATE(short, char, int); INSTANTIATE(unsigned short, char, int); @@ -406,6 +428,7 @@ INSTANTIATE(char, float, float); INSTANTIATE(int, float, float); INSTANTIATE(unsigned int, float, float); INSTANTIATE(short, float, float); +INSTANTIATE(signed char, float, float); INSTANTIATE(unsigned char, float, float); INSTANTIATE(unsigned short, float, float); INSTANTIATE(double, float, float); @@ -426,6 +449,7 @@ INSTANTIATE(unsigned int, unsigned int, float); INSTANTIATE(long long, long long, float); INSTANTIATE(unsigned long long, unsigned long long, float); INSTANTIATE(char, char, float); +INSTANTIATE(signed char, signed char, float); INSTANTIATE(unsigned char, unsigned char, float); INSTANTIATE(short, short, float); INSTANTIATE(unsigned short, unsigned short, float); @@ -442,6 +466,7 @@ INSTANTIATE(unsigned int, float, double); INSTANTIATE(short, float, double); INSTANTIATE(unsigned short, float, double); INSTANTIATE(char, float, double); +INSTANTIATE(signed char, float, double); INSTANTIATE(unsigned char, float, double); INSTANTIATE(long long, double, double); INSTANTIATE(unsigned long long, double, double); @@ -501,7 +526,8 @@ dim_t ravelIdx(af::dim4 coords, af::dim4 strides) { 0LL); } -// Calculate a linearized index's multi-dimensonal coordinates in an af::array, +// Calculate a linearized index's multi-dimensonal coordinates in an +// af::array, // given its dimension sizes and strides af::dim4 unravelIdx(dim_t idx, af::dim4 dims, af::dim4 strides) { af::dim4 coords; @@ -542,8 +568,9 @@ std::string minimalDim4(af::dim4 coords, af::dim4 dims) { return os.str(); } -// Generates a random array. testWriteToOutputArray expects that it will receive -// the same af_array that this generates after the af_* function is called +// Generates a random array. testWriteToOutputArray expects that it will +// receive the same af_array that this generates after the af_* function is +// called void genRegularArray(TestOutputArrayInfo *metadata, const unsigned ndims, const dim_t *const dims, const af_dtype ty) { metadata->init(ndims, dims, ty); @@ -556,9 +583,9 @@ void genRegularArray(TestOutputArrayInfo *metadata, double val, } // Generates a large, random array, and extracts a subarray for the af_* -// function to use. testWriteToOutputArray expects that the large array that it -// receives is equal to the same large array with the gold array injected on the -// same subarray location +// function to use. testWriteToOutputArray expects that the large array that +// it receives is equal to the same large array with the gold array injected +// on the same subarray location void genSubArray(TestOutputArrayInfo *metadata, const unsigned ndims, const dim_t *const dims, const af_dtype ty) { const dim_t pad_size = 2; @@ -571,8 +598,9 @@ void genSubArray(TestOutputArrayInfo *metadata, const unsigned ndims, } // Calculate index of sub-array. These will be used also by - // testWriteToOutputArray so that the gold sub array will be placed in the - // same location. Currently, this location is the center of the large array + // testWriteToOutputArray so that the gold sub array will be placed in + // the same location. Currently, this location is the center of the + // large array af_seq subarr_idxs[4] = {af_span, af_span, af_span, af_span}; for (uint i = 0; i < ndims; ++i) { af_seq idx = {pad_size, pad_size + dims[i] - 1.0, 1.0}; @@ -595,8 +623,9 @@ void genSubArray(TestOutputArrayInfo *metadata, double val, } // Calculate index of sub-array. These will be used also by - // testWriteToOutputArray so that the gold sub array will be placed in the - // same location. Currently, this location is the center of the large array + // testWriteToOutputArray so that the gold sub array will be placed in + // the same location. Currently, this location is the center of the + // large array af_seq subarr_idxs[4] = {af_span, af_span, af_span, af_span}; for (uint i = 0; i < ndims; ++i) { af_seq idx = {pad_size, pad_size + dims[i] - 1.0, 1.0}; @@ -606,13 +635,14 @@ void genSubArray(TestOutputArrayInfo *metadata, double val, metadata->init(val, ndims, full_arr_dims, ty, &subarr_idxs[0]); } -// Generates a reordered array. testWriteToOutputArray expects that this array -// will still have the correct output values from the af_* function, even though -// the array was initially reordered. +// Generates a reordered array. testWriteToOutputArray expects that this +// array will still have the correct output values from the af_* function, +// even though the array was initially reordered. void genReorderedArray(TestOutputArrayInfo *metadata, const unsigned ndims, const dim_t *const dims, const af_dtype ty) { - // The rest of this function assumes that dims has 4 elements. Just in case - // dims has < 4 elements, use another dims array that is filled with 1s + // The rest of this function assumes that dims has 4 elements. Just in + // case dims has < 4 elements, use another dims array that is filled + // with 1s dim_t all_dims[4] = {1, 1, 1, 1}; for (uint i = 0; i < ndims; ++i) { all_dims[i] = dims[i]; } @@ -623,7 +653,8 @@ void genReorderedArray(TestOutputArrayInfo *metadata, const unsigned ndims, uint reorder_idxs[4] = {0, 2, 1, 3}; // Shape the output array such that the reordered output array will have - // the correct dimensions that the test asks for (i.e. must match dims arg) + // the correct dimensions that the test asks for (i.e. must match dims + // arg) dim_t init_dims[4] = {all_dims[0], all_dims[1], all_dims[2], all_dims[3]}; for (uint i = 0; i < 4; ++i) { init_dims[i] = all_dims[reorder_idxs[i]]; } metadata->init(4, init_dims, ty); @@ -638,8 +669,9 @@ void genReorderedArray(TestOutputArrayInfo *metadata, const unsigned ndims, void genReorderedArray(TestOutputArrayInfo *metadata, double val, const unsigned ndims, const dim_t *const dims, const af_dtype ty) { - // The rest of this function assumes that dims has 4 elements. Just in case - // dims has < 4 elements, use another dims array that is filled with 1s + // The rest of this function assumes that dims has 4 elements. Just in + // case dims has < 4 elements, use another dims array that is filled + // with 1s dim_t all_dims[4] = {1, 1, 1, 1}; for (uint i = 0; i < ndims; ++i) { all_dims[i] = dims[i]; } @@ -650,7 +682,8 @@ void genReorderedArray(TestOutputArrayInfo *metadata, double val, uint reorder_idxs[4] = {0, 2, 1, 3}; // Shape the output array such that the reordered output array will have - // the correct dimensions that the test asks for (i.e. must match dims arg) + // the correct dimensions that the test asks for (i.e. must match dims + // arg) dim_t init_dims[4] = {all_dims[0], all_dims[1], all_dims[2], all_dims[3]}; for (uint i = 0; i < 4; ++i) { init_dims[i] = all_dims[reorder_idxs[i]]; } metadata->init(val, 4, init_dims, ty); @@ -720,8 +753,8 @@ ::testing::AssertionResult testWriteToOutputArray( if (metadata->getOutputArrayType() == SUB_ARRAY) { // There are two full arrays. One will be injected with the gold - // subarray, the other should have already been injected with the af_* - // function's output. Then we compare the two full arrays + // subarray, the other should have already been injected with the + // af_* function's output. Then we compare the two full arrays af_array gold_full_array = metadata->getFullOutputCopy(); af_assign_seq(&gold_full_array, gold_full_array, metadata->getSubArrayNumDims(), @@ -1268,9 +1301,11 @@ ::testing::AssertionResult mtxReadSparseMatrix(af::array &out, return ::testing::AssertionFailure() << "\nEnd of file reached, expected more data, " << "following are some reasons this happens.\n" - << "\t - use of template type that doesn't match data " + << "\t - use of template type that doesn't match " + "data " "type\n" - << "\t - the mtx file itself doesn't have enough data\n"; + << "\t - the mtx file itself doesn't have enough " + "data\n"; } I[i] = r - 1; J[i] = c - 1; @@ -1294,9 +1329,11 @@ ::testing::AssertionResult mtxReadSparseMatrix(af::array &out, return ::testing::AssertionFailure() << "\nEnd of file reached, expected more data, " << "following are some reasons this happens.\n" - << "\t - use of template type that doesn't match data " + << "\t - use of template type that doesn't match " + "data " "type\n" - << "\t - the mtx file itself doesn't have enough data\n"; + << "\t - the mtx file itself doesn't have enough " + "data\n"; } I[i] = r - 1; J[i] = c - 1; @@ -1332,10 +1369,17 @@ af_err conv_image(af_array *out, af_array in) { T *out_data = new T[nElems]; - for (int i = 0; i < (int)nElems; i++) out_data[i] = (T)in_data[i]; + af_dtype out_type = (af_dtype)af::dtype_traits::af_type; + for (int i = 0; i < (int)nElems; i++) { + if (out_type == s8) { + // shift to avoid overflow + out_data[i] = (T)(std::trunc(in_data[i]) - 128.f); + } else { + out_data[i] = (T)in_data[i]; + } + } - af_create_array(&outArray, out_data, idims.ndims(), idims.get(), - (af_dtype)af::dtype_traits::af_type); + af_create_array(&outArray, out_data, idims.ndims(), idims.get(), out_type); std::swap(*out, outArray); @@ -1350,6 +1394,7 @@ af_err conv_image(af_array *out, af_array in) { INSTANTIATE(float); INSTANTIATE(double); +INSTANTIATE(signed char); INSTANTIATE(unsigned char); INSTANTIATE(half_float::half); INSTANTIATE(unsigned int); @@ -1387,6 +1432,7 @@ af::array cpu_randu(const af::dim4 dims) { #define INSTANTIATE(To) template af::array cpu_randu(const af::dim4 dims) INSTANTIATE(float); INSTANTIATE(double); +INSTANTIATE(signed char); INSTANTIATE(unsigned char); INSTANTIATE(half_float::half); INSTANTIATE(unsigned int); @@ -1497,8 +1543,8 @@ vector> toCooVector(const af::array &arr) { } } - // Remove zero elements from result to ensure that only non-zero elements - // are compared + // Remove zero elements from result to ensure that only non-zero + // elements are compared out.erase(std::remove_if(out.begin(), out.end(), isZero), out.end()); std::sort(begin(out), end(out)); return out; @@ -1550,8 +1596,8 @@ std::string printContext(const std::vector &hGold, std::string goldName, // Get dim0 positions and out/reference values for the context window // - // Also get the max string length between the position and out/ref values - // per item so that it can be used later as the field width for + // Also get the max string length between the position and out/ref + // values per item so that it can be used later as the field width for // displaying each item in the context window for (dim_t i = 0; i < ctxElems; ++i) { std::ostringstream tmpOs; @@ -1995,6 +2041,7 @@ ::testing::AssertionResult assertRefEq(std::string hA_name, INSTANTIATE(float); INSTANTIATE(double); +INSTANTIATE(signed char); INSTANTIATE(unsigned char); INSTANTIATE(half_float::half); INSTANTIATE(unsigned int); @@ -2010,6 +2057,170 @@ INSTANTIATE(std::complex); INSTANTIATE(std::complex); #undef INSTANTIATE +af::array toTempFormat(tempFormat form, const af::array &in) { + af::array ret; + const af::dim4 &dims = in.dims(); + switch (form) { + case JIT_FORMAT: + switch (in.type()) { + case b8: ret = !(in); break; + default: ret = in * 2; + } + // Make sure that the base array is <> form original + ret.eval(); + switch (in.type()) { + case b8: ret = !(ret); break; + default: ret /= 2; + } + break; + case SUB_FORMAT_dim0: { + af::dim4 pdims(dims); + pdims[0] *= 2; + af::array parent = af::randu(pdims, in.type()); + const af::seq dim = af::seq(dims[0]) + static_cast(dims[0]); + parent(dim, af::span, af::span, af::span) = in; + ret = parent(dim, af::span, af::span, af::span); + }; break; + case SUB_FORMAT_dim1: { + af::dim4 pdims(dims); + pdims[1] *= 2; + const af::seq dim = af::seq(dims[1]) + static_cast(dims[1]); + af::array parent = af::randu(pdims, in.type()); + parent(af::span, dim, af::span, af::span) = in; + ret = parent(af::span, dim, af::span, af::span); + }; break; + case SUB_FORMAT_dim2: { + af::dim4 pdims(dims); + pdims[2] *= 2; + const af::seq dim = af::seq(dims[2]) + static_cast(dims[2]); + af::array parent = af::randu(pdims, in.type()); + parent(af::span, af::span, dim, af::span) = in; + ret = parent(af::span, af::span, dim, af::span); + }; break; + case SUB_FORMAT_dim3: { + af::dim4 pdims(dims); + pdims[3] *= 2; + const af::seq dim = af::seq(dims[3]) + static_cast(dims[3]); + af::array parent = af::randu(pdims, in.type()); + parent(af::span, af::span, af::span, dim) = in; + ret = parent(af::span, af::span, af::span, dim); + }; break; + case REORDERED_FORMAT: { + const dim_t idxs[4] = {0, 3, 1, 2}; + // idxs[0] has to be 0, to keep the same data in mem + dim_t rev_idxs[4]; + for (dim_t i = 0; i < 4; ++i) { rev_idxs[idxs[i]] = i; }; + ret = af::reorder(in, idxs[0], idxs[1], idxs[2], idxs[3]); + ret = ret.copy(); // make data linear + ret = af::reorder(ret, rev_idxs[0], rev_idxs[1], rev_idxs[2], + rev_idxs[3]); + // ret has same content as in, although data is stored in + // different order + }; break; + case LINEAR_FORMAT: + default: ret = in.copy(); + }; + return ret; +} + +void toTempFormat(tempFormat form, af_array *out, const af_array &in) { + dim_t dims[4]; + af_get_dims(dims, dims + 1, dims + 2, dims + 3, in); + unsigned numdims; + af_get_numdims(&numdims, in); + af_dtype ty; + af_get_type(&ty, in); + switch (form) { + case JIT_FORMAT: { + // af_array one = nullptr, min_one = nullptr, res = nullptr; + af_array res = nullptr, two = nullptr; + ASSERT_SUCCESS(af_constant(&two, 2, numdims, dims, ty)); + switch (ty) { + case b8: af_not(&res, in); break; + default: + // ret = in + af::constant(1, dims, in.type()); + ASSERT_SUCCESS(af_mul(&res, in, two, false)); + } + // Make sure that the base array is <> form original + ASSERT_SUCCESS(af_eval(res)); + switch (ty) { + case b8: af_not(out, res); break; + default: + ASSERT_SUCCESS(af_div(out, res, two, false)); // NO EVAL!! + } + ASSERT_SUCCESS(af_release_array(two)); + two = nullptr; + ASSERT_SUCCESS(af_release_array(res)); + res = nullptr; + }; break; + case SUB_FORMAT_dim0: { + const dim_t pdims[4] = {dims[0] * 2, dims[1], dims[2], dims[3]}; + af_array parent = nullptr; + ASSERT_SUCCESS(af_randu(&parent, 4, pdims, ty)); + const af_seq idxs[4] = {af_make_seq(dims[0], 2. * dims[0] - 1., 1.), + af_span, af_span, af_span}; + ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in)); + ASSERT_SUCCESS(af_index(out, parent, numdims, idxs)); + ASSERT_SUCCESS(af_release_array(parent)); + parent = nullptr; + }; break; + case SUB_FORMAT_dim1: { + const dim_t pdims[4] = {dims[0], dims[1] * 2, dims[2], dims[3]}; + af_array parent = nullptr; + ASSERT_SUCCESS(af_randu(&parent, 4, pdims, ty)); + const af_seq idxs[4] = {af_span, + af_make_seq(dims[1], 2. * dims[1] - 1., 1.), + af_span, af_span}; + ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in)); + ASSERT_SUCCESS(af_index(out, parent, numdims, idxs)); + ASSERT_SUCCESS(af_release_array(parent)); + parent = nullptr; + }; break; + case SUB_FORMAT_dim2: { + const dim_t pdims[4] = {dims[0], dims[1], dims[2] * 2, dims[3]}; + af_array parent = nullptr; + ASSERT_SUCCESS(af_randu(&parent, 4, pdims, ty)); + const af_seq idxs[4] = {af_span, af_span, + af_make_seq(dims[2], 2. * dims[2] - 1., 1.), + af_span}; + ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in)); + ASSERT_SUCCESS(af_index(out, parent, numdims, idxs)); + ASSERT_SUCCESS(af_release_array(parent)); + parent = nullptr; + }; break; + case SUB_FORMAT_dim3: { + const dim_t pdims[4] = {dims[0], dims[1], dims[2], dims[3] * 2}; + af_array parent = nullptr; + ASSERT_SUCCESS(af_randu(&parent, 4, pdims, ty)); + const af_seq idxs[4] = { + af_span, af_span, af_span, + af_make_seq(dims[3], 2. * dims[3] - 1., 1.)}; + ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in)); + ASSERT_SUCCESS(af_index(out, parent, numdims, idxs)); + ASSERT_SUCCESS(af_release_array(parent)); + parent = nullptr; + }; break; + case REORDERED_FORMAT: { + const unsigned idxs[4] = {0, 3, 1, 2}; + // idxs[0] has to be 0, to keep the same data in mem + dim_t rev_idxs[4]; + for (dim_t i = 0; i < 4; ++i) { rev_idxs[idxs[i]] = i; }; + af_array rev = nullptr; + ASSERT_SUCCESS( + af_reorder(&rev, in, idxs[0], idxs[1], idxs[2], idxs[3])); + ASSERT_SUCCESS(af_copy_array(out, rev)); + ASSERT_SUCCESS(af_reorder(out, rev, rev_idxs[0], rev_idxs[1], + rev_idxs[2], rev_idxs[3])); + // ret has same content as in, although data is stored in + // different order + ASSERT_SUCCESS(af_release_array(rev)); + rev = nullptr; + }; break; + case LINEAR_FORMAT: + default: af_copy_array(out, in); + }; +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/test/arrayio.cpp b/test/arrayio.cpp index 00d907a568..ea15165ac4 100644 --- a/test/arrayio.cpp +++ b/test/arrayio.cpp @@ -51,7 +51,8 @@ INSTANTIATE_TEST_SUITE_P( type_params("s32", s32, 11), type_params("u32", u32, 12), type_params("u8", u8, 13), type_params("b8", b8, 1), type_params("s64", s64, 15), type_params("u64", u64, 16), - type_params("s16", s16, 17), type_params("u16", u16, 18)), + type_params("s16", s16, 17), type_params("u16", u16, 18), + type_params("s8", s8, 19)), getTypeName); TEST_P(ArrayIOType, ReadType) { @@ -103,6 +104,7 @@ TEST_P(ArrayIOType, ReadContent) { case c64: checkVals(arr, p.real, p.imag, p.type); break; case s32: checkVals(arr, p.real, p.imag, p.type); break; case u32: checkVals(arr, p.real, p.imag, p.type); break; + case s8: checkVals(arr, p.real, p.imag, p.type); break; case u8: checkVals(arr, p.real, p.imag, p.type); break; case b8: checkVals(arr, p.real, p.imag, p.type); break; case s64: checkVals(arr, p.real, p.imag, p.type); break; diff --git a/test/assign.cpp b/test/assign.cpp index cbfe6359b1..7b94bfa608 100644 --- a/test/assign.cpp +++ b/test/assign.cpp @@ -94,8 +94,8 @@ class ArrayAssign : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list diff --git a/test/bilateral.cpp b/test/bilateral.cpp index f4ff949b55..12b27fc33f 100644 --- a/test/bilateral.cpp +++ b/test/bilateral.cpp @@ -73,7 +73,8 @@ TEST(BilateralOnImage, Color) { template class BilateralOnData : public ::testing::Test {}; -typedef ::testing::Types +typedef ::testing::Types DataTestTypes; // register the type list diff --git a/test/binary.cpp b/test/binary.cpp index ab557f8c9a..7fd47bcfbd 100644 --- a/test/binary.cpp +++ b/test/binary.cpp @@ -1,5 +1,5 @@ /******************************************************* - * Copyright (c) 2014, ArrayFire + * Copyright (c) 2025, ArrayFire * All rights reserved. * * This file is distributed under 3-clause BSD license. @@ -14,6 +14,8 @@ #include #include #include +#include +#include "half.hpp" //note: NOT common. From extern/half/include/half.hpp #include #include @@ -21,6 +23,8 @@ using namespace std; using namespace af; +using half_float_half = half_float::half; + const int num = 10000; #define add(left, right) (left) + (right) @@ -36,6 +40,11 @@ T mod(T a, T b) { return std::fmod(a, b); } +template +T rem(T x, T y) { + return remainder(x, y); +} + af::array randgen(const int num, dtype ty) { af::array tmp = round(1 + 2 * af::randu(num, f32)).as(ty); tmp.eval(); @@ -122,7 +131,7 @@ af::array randgen(const int num, dtype ty) { \ af_dtype ta = (af_dtype)dtype_traits::af_type; \ af::array a = randgen(num, ta); \ - Tb h_b = 0.3; \ + Tb h_b = (Tb)0.3; \ af::array c = func(a, h_b); \ Ta *h_a = a.host(); \ Td *h_d = c.host(); \ @@ -139,7 +148,7 @@ af::array randgen(const int num, dtype ty) { SUPPORTED_TYPE_CHECK(Tc); \ \ af_dtype tb = (af_dtype)dtype_traits::af_type; \ - Ta h_a = 0.3; \ + Ta h_a = (Ta)0.3; \ af::array b = randgen(num, tb); \ af::array c = func(h_a, b); \ Tb *h_b = b.host(); \ @@ -163,6 +172,8 @@ af::array randgen(const int num, dtype ty) { #define BINARY_TESTS_UINT(func) BINARY_TESTS(uint, uint, uint, func) #define BINARY_TESTS_INTL(func) BINARY_TESTS(intl, intl, intl, func) #define BINARY_TESTS_UINTL(func) BINARY_TESTS(uintl, uintl, uintl, func) +#define BINARY_TESTS_NEAR_HALF(func) \ + BINARY_TESTS_NEAR(half_float_half, half_float_half, half_float_half, func, 1e-3) #define BINARY_TESTS_NEAR_FLOAT(func) \ BINARY_TESTS_NEAR(float, float, float, func, 1e-5) #define BINARY_TESTS_NEAR_DOUBLE(func) \ @@ -175,6 +186,7 @@ BINARY_TESTS_NEAR(float, float, float, div, 1e-3) // FIXME BINARY_TESTS_FLOAT(min) BINARY_TESTS_FLOAT(max) BINARY_TESTS_NEAR(float, float, float, mod, 1e-5) // FIXME +BINARY_TESTS_FLOAT(rem) BINARY_TESTS_DOUBLE(add) BINARY_TESTS_DOUBLE(sub) @@ -183,11 +195,16 @@ BINARY_TESTS_DOUBLE(div) BINARY_TESTS_DOUBLE(min) BINARY_TESTS_DOUBLE(max) BINARY_TESTS_DOUBLE(mod) +BINARY_TESTS_DOUBLE(rem) BINARY_TESTS_NEAR_FLOAT(atan2) BINARY_TESTS_NEAR_FLOAT(pow) BINARY_TESTS_NEAR_FLOAT(hypot) +BINARY_TESTS_NEAR_HALF(atan2) +BINARY_TESTS_NEAR_HALF(pow) +BINARY_TESTS_NEAR_HALF(hypot) + BINARY_TESTS_NEAR_DOUBLE(atan2) BINARY_TESTS_NEAR_DOUBLE(pow) BINARY_TESTS_NEAR_DOUBLE(hypot) @@ -195,18 +212,26 @@ BINARY_TESTS_NEAR_DOUBLE(hypot) BINARY_TESTS_INT(add) BINARY_TESTS_INT(sub) BINARY_TESTS_INT(mul) +BINARY_TESTS_INT(div) +BINARY_TESTS_INT(pow) BINARY_TESTS_UINT(add) BINARY_TESTS_UINT(sub) BINARY_TESTS_UINT(mul) +BINARY_TESTS_UINT(div) +BINARY_TESTS_UINT(pow) BINARY_TESTS_INTL(add) BINARY_TESTS_INTL(sub) BINARY_TESTS_INTL(mul) +BINARY_TESTS_INTL(div) +BINARY_TESTS_INTL(pow) BINARY_TESTS_UINTL(add) BINARY_TESTS_UINTL(sub) BINARY_TESTS_UINTL(mul) +BINARY_TESTS_UINTL(div) +BINARY_TESTS_UINTL(pow) BINARY_TESTS_CFLOAT(add) BINARY_TESTS_CFLOAT(sub) @@ -299,6 +324,7 @@ UBITOP(bitnot, int) UBITOP(bitnot, uint) UBITOP(bitnot, intl) UBITOP(bitnot, uintl) +UBITOP(bitnot, schar) UBITOP(bitnot, uchar) UBITOP(bitnot, short) UBITOP(bitnot, ushort) @@ -373,7 +399,12 @@ class PowPrecisionTest : public ::testing::TestWithParam { vector hres(1, 0); \ B.host(&hres[0]); \ std::fesetround(FE_TONEAREST); \ - T gold = (T)std::rint(std::pow((double)param, 2.0)); \ + T gold; \ + if (!af::isDoubleAvailable(af::getDevice())) { \ + gold = (T)std::rint(std::pow((float)param, 2.0f)); \ + } else { \ + gold = (T)std::rint(std::pow((double)param, 2.0)); \ + } \ ASSERT_EQ(hres[0], gold); \ } @@ -384,6 +415,7 @@ DEF_TEST(Int, int) DEF_TEST(UShort, unsigned short) DEF_TEST(Short, short) DEF_TEST(UChar, unsigned char) +DEF_TEST(SChar, signed char) #undef DEF_TEST @@ -401,6 +433,8 @@ INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestShort, testing::Range(1, 180, 50)); INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestUChar, testing::Range(1, 12, 5)); +INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestSChar, + testing::Range(1, 9, 3)); INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestLong, testing::Range(-1e7, 0, 1e6)); @@ -408,6 +442,8 @@ INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestInt, testing::Range(-46340, 0, 10e3)); INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestShort, testing::Range(-180, 0, 50)); +INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestSChar, + testing::Range(-9, 0, 3)); struct result_type_param { af_dtype result_; @@ -466,6 +502,7 @@ INSTANTIATE_TEST_SUITE_P( result_type_param(b8), result_type_param(s32), result_type_param(u32), + result_type_param(s8), result_type_param(u8), result_type_param(s64), result_type_param(u64), @@ -485,6 +522,7 @@ INSTANTIATE_TEST_SUITE_P( result_type_param(f32, b8, f32), result_type_param(f32, s32, f32), result_type_param(f32, u32, f32), + result_type_param(f32, s8, f32), result_type_param(f32, u8, f32), result_type_param(f32, s64, f32), result_type_param(f32, u64, f32), @@ -505,6 +543,7 @@ INSTANTIATE_TEST_SUITE_P( result_type_param(f64, b8, f64), result_type_param(f64, s32, f64), result_type_param(f64, u32, f64), + result_type_param(f64, s8, f64), result_type_param(f64, u8, f64), result_type_param(f64, s64, f64), result_type_param(f64, u64, f64), @@ -537,7 +576,8 @@ class ResultTypeScalar : public ::testing::Test { }; typedef ::testing::Types + unsigned short, char, signed char, unsigned char, + half_float::half> TestTypes; TYPED_TEST_SUITE(ResultTypeScalar, TestTypes); diff --git a/test/blas.cpp b/test/blas.cpp index 6b0590d73b..6f77c10160 100644 --- a/test/blas.cpp +++ b/test/blas.cpp @@ -492,6 +492,36 @@ TEST(MatrixMultiply, half) { } } +TEST(MatrixMultiply, schar) { + array A8 = array(3, 3, h_lhs).as(s8); + array B8 = array(3, 3, h_rhs).as(s8); + array expected32 = array(3, 3, h_gold).as(f32); + + { + af_array C32 = 0; + const float alpha32(1.0f); + const float beta32(0.0f); + af_backend backend; + af_get_active_backend(&backend); + if (backend == AF_BACKEND_CUDA) { + ASSERT_SUCCESS(af_gemm(&C32, AF_MAT_NONE, AF_MAT_NONE, &alpha32, + A8.get(), B8.get(), &beta32)); + } else { + ASSERT_EQ(AF_ERR_TYPE, + af_gemm(&C32, AF_MAT_NONE, AF_MAT_NONE, &alpha32, + A8.get(), B8.get(), &beta32)); + SUCCEED(); + return; + } + af::array C(C32); + ASSERT_ARRAYS_NEAR(expected32, C, 0.00001); + } + { + array C32 = matmul(A8, B8); + ASSERT_ARRAYS_NEAR(expected32, C32, 0.00001); + } +} + struct test_params { af_mat_prop opt_lhs; af_mat_prop opt_rhs; diff --git a/test/canny.cpp b/test/canny.cpp index 7f2fa2918c..0a0fdbc08c 100644 --- a/test/canny.cpp +++ b/test/canny.cpp @@ -28,7 +28,7 @@ class CannyEdgeDetector : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list @@ -53,7 +53,7 @@ void cannyTest(string pTestFile) { (af_dtype)dtype_traits::af_type)); ASSERT_SUCCESS(af_canny(&outArray, sArray, AF_CANNY_THRESHOLD_MANUAL, - 0.4147f, 0.8454f, 3, true)); + 0.4147f, 0.8454f, 3, true)); vector outData(sDims.elements()); @@ -72,10 +72,12 @@ void cannyTest(string pTestFile) { } TYPED_TEST(CannyEdgeDetector, ArraySizeLessThanBlockSize10x10) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); cannyTest(string(TEST_DIR "/CannyEdgeDetector/fast10x10.test")); } TYPED_TEST(CannyEdgeDetector, ArraySizeEqualBlockSize16x16) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); cannyTest(string(TEST_DIR "/CannyEdgeDetector/fast16x16.test")); } @@ -129,8 +131,9 @@ void cannyImageOtsuTest(string pTestFile, bool isColor) { af_load_image_native(&goldArray, outFiles[testId].c_str())); ASSERT_SUCCESS(af_canny(&_outArray, inArray, - AF_CANNY_THRESHOLD_AUTO_OTSU, 0.08, 0.32, 3, - false)); + AF_CANNY_THRESHOLD_AUTO_OTSU, + 0.08, 0.32, 3, false)); + unsigned ndims = 0; dim_t dims[4]; @@ -156,6 +159,7 @@ void cannyImageOtsuTest(string pTestFile, bool isColor) { } TEST(CannyEdgeDetector, OtsuThreshold) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); cannyImageOtsuTest(string(TEST_DIR "/CannyEdgeDetector/gray.test"), false); } @@ -248,7 +252,7 @@ void cannyImageOtsuBatchTest(string pTestFile, const dim_t targetBatchCount) { array inputIm = tile(readImg, 1, 1, targetBatchCount); array outIm = - canny(inputIm, AF_CANNY_THRESHOLD_AUTO_OTSU, 0.08, 0.32, 3, false); + canny(inputIm, AF_CANNY_THRESHOLD_AUTO_OTSU, 0.08, 0.32, 3, false); outIm *= 255.0; ASSERT_IMAGES_NEAR(goldIm, outIm.as(u8), 1.0e-3); @@ -256,6 +260,7 @@ void cannyImageOtsuBatchTest(string pTestFile, const dim_t targetBatchCount) { } TEST(CannyEdgeDetector, BatchofImagesUsingCPPAPI) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); // DO NOT INCREASE BATCH COUNT BEYOND 4 // This is a limitation on the test assert macro that is saving // images to disk which can't handle a batch of images. diff --git a/test/cast.cpp b/test/cast.cpp index cb1f4e3f42..d2b4f95250 100644 --- a/test/cast.cpp +++ b/test/cast.cpp @@ -52,6 +52,7 @@ void cast_test() { REAL_TO_TESTS(Ti, char); \ REAL_TO_TESTS(Ti, int); \ REAL_TO_TESTS(Ti, unsigned); \ + REAL_TO_TESTS(Ti, schar); \ REAL_TO_TESTS(Ti, uchar); \ REAL_TO_TESTS(Ti, intl); \ REAL_TO_TESTS(Ti, uintl); \ @@ -67,6 +68,7 @@ REAL_TEST_INVOKE(double) REAL_TEST_INVOKE(char) REAL_TEST_INVOKE(int) REAL_TEST_INVOKE(unsigned) +REAL_TEST_INVOKE(schar) REAL_TEST_INVOKE(uchar) REAL_TEST_INVOKE(intl) REAL_TEST_INVOKE(uintl) diff --git a/test/clamp.cpp b/test/clamp.cpp index 1e0b04b7c2..c830b06b2b 100644 --- a/test/clamp.cpp +++ b/test/clamp.cpp @@ -125,6 +125,7 @@ INSTANTIATE_TEST_SUITE_P( clamp_params(dim4(10), f16, f16, f16, f16), clamp_params(dim4(10), s32, f32, f32, f32), clamp_params(dim4(10), u32, f32, f32, f32), + clamp_params(dim4(10), s8, f32, f32, f32), clamp_params(dim4(10), u8, f32, f32, f32), clamp_params(dim4(10), b8, f32, f32, f32), clamp_params(dim4(10), s64, f32, f32, f32), diff --git a/test/compare.cpp b/test/compare.cpp index 66d9778039..877c08275f 100644 --- a/test/compare.cpp +++ b/test/compare.cpp @@ -23,8 +23,8 @@ using std::vector; template class Compare : public ::testing::Test {}; -typedef ::testing::Types +typedef ::testing::Types TestTypes; TYPED_TEST_SUITE(Compare, TestTypes); diff --git a/test/confidence_connected.cpp b/test/confidence_connected.cpp index ac5b0bf2bc..39c0f8f0ff 100644 --- a/test/confidence_connected.cpp +++ b/test/confidence_connected.cpp @@ -41,17 +41,6 @@ struct CCCTestParams { double replace; }; -void apiWrapper(af_array *out, const af_array in, const af_array seedx, - const af_array seedy, const CCCTestParams params) { - ASSERT_SUCCESS(af_confidence_cc(out, in, seedx, seedy, params.radius, - params.multiplier, params.iterations, - params.replace)); - - int device = 0; - ASSERT_SUCCESS(af_get_device(&device)); - ASSERT_SUCCESS(af_sync(device)); -} - template void testImage(const std::string pTestFile, const size_t numSeeds, const unsigned *seedx, const unsigned *seedy, @@ -103,7 +92,12 @@ void testImage(const std::string pTestFile, const size_t numSeeds, params.iterations = iter; params.replace = 255.0; - apiWrapper(&outArray, inArray, seedxArr, seedyArr, params); + ASSERT_SUCCESS(af_confidence_cc(&outArray, inArray, seedxArr, seedyArr, + params.radius, params.multiplier, + params.iterations, params.replace)); + int device = 0; + ASSERT_SUCCESS(af_get_device(&device)); + ASSERT_SUCCESS(af_sync(device)); ASSERT_ARRAYS_EQ(outArray, goldArray); @@ -147,7 +141,12 @@ void testData(CCCTestParams params) { (af_dtype)af::dtype_traits::af_type)); af_array outArray = 0; - apiWrapper(&outArray, inArray, seedxArr, seedyArr, params); + ASSERT_SUCCESS(af_confidence_cc(&outArray, inArray, seedxArr, seedyArr, + params.radius, params.multiplier, + params.iterations, params.replace)); + int device = 0; + ASSERT_SUCCESS(af_get_device(&device)); + ASSERT_SUCCESS(af_sync(device)); ASSERT_VEC_ARRAY_EQ(tests[0], dims, outArray); @@ -161,6 +160,7 @@ class ConfidenceConnectedDataTest : public testing::TestWithParam {}; TYPED_TEST(ConfidenceConnectedImageTest, DonutBackgroundExtraction) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); const unsigned seedx = 10; const unsigned seedy = 10; testImage(std::string("donut_background.test"), 1, &seedx, @@ -168,6 +168,7 @@ TYPED_TEST(ConfidenceConnectedImageTest, DonutBackgroundExtraction) { } TYPED_TEST(ConfidenceConnectedImageTest, DonutRingExtraction) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); const unsigned seedx = 132; const unsigned seedy = 132; testImage(std::string("donut_ring.test"), 1, &seedx, &seedy, 3, @@ -175,6 +176,7 @@ TYPED_TEST(ConfidenceConnectedImageTest, DonutRingExtraction) { } TYPED_TEST(ConfidenceConnectedImageTest, DonutKernelExtraction) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); const unsigned seedx = 150; const unsigned seedy = 150; testImage(std::string("donut_core.test"), 1, &seedx, &seedy, 3, @@ -182,6 +184,7 @@ TYPED_TEST(ConfidenceConnectedImageTest, DonutKernelExtraction) { } TEST_P(ConfidenceConnectedDataTest, SegmentARegion) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); testData(GetParam()); } @@ -198,3 +201,50 @@ INSTANTIATE_TEST_SUITE_P( << info.param.iterations << "_replace_" << info.param.replace; return ss.str(); }); + +#define TEST_FORMATS(form) \ + TEST(TEMP_FORMAT, form##_2Dseed) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ + const string filename(string(TEST_DIR) + "/confidence_cc/donut.png"); \ + const af::array image(af::loadImage(filename.c_str())); \ + const af::array seed(dim4(1, 2), {10u, 8u}); \ + \ + const af::array out = \ + af::confidenceCC(toTempFormat(form, image), \ + toTempFormat(form, seed), 3, 3, 25, 255.0); \ + const af::array gold = af::confidenceCC(image, seed, 3, 3, 25, 255.0); \ + \ + EXPECT_ARRAYS_EQ(out, gold); \ + } \ + \ + TEST(TEMP_FORMAT, form##_2xSeed) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ + const string filename(string(TEST_DIR) + "/confidence_cc/donut.png"); \ + const af::array image(af::loadImage(filename.c_str())); \ + const af::array seedx({10u}); \ + const af::array seedy({8u}); \ + \ + const af::array out = af::confidenceCC( \ + toTempFormat(form, image), toTempFormat(form, seedx), \ + toTempFormat(form, seedy), 3, 3, 25, 255.0); \ + const af::array gold = \ + af::confidenceCC(image, seedx, seedy, 3, 3, 25, 255.0); \ + \ + EXPECT_ARRAYS_EQ(out, gold); \ + } \ + TEST(TEMP_FORMAT, form##_vectSeed) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ + const string filename(string(TEST_DIR) + "/confidence_cc/donut.png"); \ + const af::array image(af::loadImage(filename.c_str())); \ + const unsigned seedx[1] = {10u}; \ + const unsigned seedy[1] = {8u}; \ + \ + const af::array out = af::confidenceCC(toTempFormat(form, image), 1, \ + seedx, seedy, 3, 3, 25, 255.0); \ + const af::array gold = \ + af::confidenceCC(image, 1, seedx, seedy, 3, 3, 25, 255.0); \ + \ + EXPECT_ARRAYS_EQ(out, gold); \ + } + +FOREACH_TEMP_FORMAT(TEST_FORMATS) diff --git a/test/constant.cpp b/test/constant.cpp index 0a75e3d974..b1d3e0a5af 100644 --- a/test/constant.cpp +++ b/test/constant.cpp @@ -31,7 +31,8 @@ template class Constant : public ::testing::Test {}; typedef ::testing::Types + schar, uchar, uintl, intl, short, ushort, + half_float::half> TestTypes; TYPED_TEST_SUITE(Constant, TestTypes); diff --git a/test/convolve.cpp b/test/convolve.cpp index 8adeb40fd8..5df8961e1b 100644 --- a/test/convolve.cpp +++ b/test/convolve.cpp @@ -33,8 +33,8 @@ class Convolve : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list @@ -898,7 +898,7 @@ float tolerance(); template<> float tolerance() { - return 2e-3; + return 4e-3; } template<> @@ -1176,4 +1176,10 @@ TEST(ConvolveNN, ZeroPadding_Issue2817) { array convolved = convolve2NN(signal, filter, strides, padding, dilation); ASSERT_EQ(sum(abs(signal(seq(1, 3), seq(1, 3)) - convolved)) < 1E-5, true); + + array incoming_gradient = constant(1 / 9.f, 3, 3); + array convolved_grad = convolve2GradientNN(incoming_gradient, signal, filter, + convolved, strides, padding, dilation, + AF_CONV_GRADIENT_FILTER); + ASSERT_EQ(sum(abs(convolved - convolved_grad)) < 1E-5, true); } diff --git a/test/corrcoef.cpp b/test/corrcoef.cpp index 213a8de092..e9bc5a5616 100644 --- a/test/corrcoef.cpp +++ b/test/corrcoef.cpp @@ -31,7 +31,8 @@ class CorrelationCoefficient : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list @@ -61,9 +62,9 @@ template struct ccOutType { typedef typename cond_type< is_same_type::value || is_same_type::value || - is_same_type::value || is_same_type::value || - is_same_type::value || is_same_type::value || - is_same_type::value, + is_same_type::value || is_same_type::value || + is_same_type::value || is_same_type::value || + is_same_type::value || is_same_type::value, float, typename elseType::type>::type type; }; diff --git a/test/covariance.cpp b/test/covariance.cpp index 4d4e4877f1..f149fbd095 100644 --- a/test/covariance.cpp +++ b/test/covariance.cpp @@ -34,8 +34,8 @@ class Covariance : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list @@ -65,9 +65,9 @@ template struct covOutType { typedef typename cond_type< is_same_type::value || is_same_type::value || - is_same_type::value || is_same_type::value || - is_same_type::value || is_same_type::value || - is_same_type::value, + is_same_type::value || is_same_type::value || + is_same_type::value || is_same_type::value || + is_same_type::value || is_same_type::value, float, typename elseType::type>::type type; }; diff --git a/test/diagonal.cpp b/test/diagonal.cpp index 1eecb883ae..e3031f731c 100644 --- a/test/diagonal.cpp +++ b/test/diagonal.cpp @@ -31,8 +31,8 @@ using std::vector; template class Diagonal : public ::testing::Test {}; -typedef ::testing::Types +typedef ::testing::Types TestTypes; TYPED_TEST_SUITE(Diagonal, TestTypes); diff --git a/test/diff1.cpp b/test/diff1.cpp index a7456fd0a2..9fdf11a91a 100644 --- a/test/diff1.cpp +++ b/test/diff1.cpp @@ -46,7 +46,7 @@ class Diff1 : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + uintl, char, signed char, unsigned char, short, ushort> TestTypes; // register the type list diff --git a/test/diff2.cpp b/test/diff2.cpp index c7c17f333f..cdc2b9909e 100644 --- a/test/diff2.cpp +++ b/test/diff2.cpp @@ -51,7 +51,7 @@ class Diff2 : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + uintl, char, signed char, unsigned char, short, ushort> TestTypes; // register the type list diff --git a/test/dog.cpp b/test/dog.cpp index 0b764f2c06..af76c23f59 100644 --- a/test/dog.cpp +++ b/test/dog.cpp @@ -33,7 +33,8 @@ class DOG : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list diff --git a/test/fast.cpp b/test/fast.cpp index 1d494641ff..c5e3225d0e 100644 --- a/test/fast.cpp +++ b/test/fast.cpp @@ -61,7 +61,7 @@ class FixedFAST : public ::testing::Test { }; typedef ::testing::Types FloatTestTypes; -typedef ::testing::Types FixedTestTypes; +typedef ::testing::Types FixedTestTypes; TYPED_TEST_SUITE(FloatFAST, FloatTestTypes); TYPED_TEST_SUITE(FixedFAST, FixedTestTypes); @@ -158,12 +158,14 @@ void fastTest(string pTestFile, bool nonmax) { #define FLOAT_FAST_INIT(desc, image, nonmax) \ TYPED_TEST(FloatFAST, desc) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ fastTest(string(TEST_DIR "/fast/" #image "_float.test"), \ nonmax); \ } #define FIXED_FAST_INIT(desc, image, nonmax) \ TYPED_TEST(FixedFAST, desc) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ fastTest(string(TEST_DIR "/fast/" #image "_fixed.test"), \ nonmax); \ } @@ -180,6 +182,7 @@ using af::features; using af::loadImage; TEST(FloatFAST, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); IMAGEIO_ENABLED_CHECK(); vector inDims; diff --git a/test/fftconvolve.cpp b/test/fftconvolve.cpp index 57d9398a04..a8f63e2f45 100644 --- a/test/fftconvolve.cpp +++ b/test/fftconvolve.cpp @@ -39,8 +39,8 @@ class FFTConvolveLarge : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; typedef ::testing::Types TestTypesLarge; diff --git a/test/gen_assign.cpp b/test/gen_assign.cpp index 7cfd78ae62..07685108c4 100644 --- a/test/gen_assign.cpp +++ b/test/gen_assign.cpp @@ -455,3 +455,46 @@ TEST(GeneralAssign, CPP_AANN) { freeHost(hIdx0); freeHost(hIdx1); } + +TEST(GeneralAssign, NDimsDoesNotMatchLDims) { + af_err err; + af_array zeros, l1, l2, sevens; + dim_t sevens_size[3] = {5, 1, 1}; + short hsevens[5] = {7, 7, 7, 7, 7}; + + dim_t zeros_size[3] = {5, 6, 1}; + short hzeros[5 * 6] = {0}; + + dim_t hone[1] = {1}; + + ASSERT_SUCCESS(af_create_array(&zeros, hzeros, 3, zeros_size, s16)); + ASSERT_SUCCESS(af_create_array(&sevens, hsevens, 3, sevens_size, s16)); + ASSERT_SUCCESS(af_create_array(&l2, hone, 1, hone, s64)); + + af_index_t *ix; + ASSERT_SUCCESS(af_create_indexers(&ix)); + ASSERT_SUCCESS(af_set_array_indexer(ix, l2, 1)); + + // clang-format off + vector gold = { + 0, 0, 0, 0, 0, + 7, 7, 7, 7, 7, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + }; + // clang-format on + for (int number_of_indices = 2; number_of_indices < 4; + number_of_indices++) { + af_array result = 0; + ASSERT_SUCCESS( + af_assign_gen(&result, zeros, number_of_indices, ix, sevens)); + + ASSERT_VEC_ARRAY_EQ(gold, dim4(3, zeros_size), af::array(result)); + } + ASSERT_SUCCESS(af_release_array(zeros)); + ASSERT_SUCCESS(af_release_array(sevens)); + ASSERT_SUCCESS(af_release_array(l2)); + ASSERT_SUCCESS(af_release_indexers(ix)); +} diff --git a/test/gen_index.cpp b/test/gen_index.cpp index e65d4e48e5..fe684ebd27 100644 --- a/test/gen_index.cpp +++ b/test/gen_index.cpp @@ -108,8 +108,9 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( ::testing::Values(index_test( string(TEST_DIR "/gen_index/s0_3s0_1s1_2a.test"), dim4(4, 2, 2))), - ::testing::Values(f32, f64, c32, c64, u64, s64, u16, s16, u8, b8, f16), - ::testing::Values(f32, f64, u64, s64, u16, s16, u8, f16)), + ::testing::Values(f32, f64, c32, c64, u64, s64, u16, s16, s8, u8, b8, + f16), + ::testing::Values(f32, f64, u64, s64, u16, s16, s8, u8, f16)), testNameGenerator); TEST_P(IndexGeneralizedLegacy, SSSA) { @@ -253,6 +254,56 @@ TEST(GeneralIndex, AASS) { ASSERT_SUCCESS(af_release_array(outArray)); } +TEST(GeneralIndex, SSAS_LinearSteps) { + vector numDims; + vector> in; + vector> tests; // Read tests from file + + readTestsFromFile( + TEST_DIR "/gen_index/s29_9__3s0_9_2as0_n.test", numDims, in, tests); + + af_array outArray = 0; + af_array inArray = 0; + af_array idxArray0 = 0; + dim4 dims0 = numDims[0]; + dim4 dims1 = numDims[1]; + + ASSERT_SUCCESS(af_create_array(&inArray, &(in[0].front()), dims0.ndims(), + dims0.get(), + (af_dtype)dtype_traits::af_type)); + + ASSERT_SUCCESS(af_create_array(&idxArray0, &(in[1].front()), dims1.ndims(), + dims1.get(), + (af_dtype)dtype_traits::af_type)); + + af_index_t indexs[4]; + indexs[0].idx.seq = af_make_seq(29, 9, -3); + indexs[1].idx.seq = af_make_seq(0, 9, 2); + indexs[2].idx.arr = idxArray0; + indexs[3].idx.seq = af_span; + + indexs[0].isSeq = true; + indexs[1].isSeq = true; + indexs[2].isSeq = false; + indexs[3].isSeq = true; + + ASSERT_SUCCESS(af_index_gen(&outArray, inArray, 4, indexs)); + + vector currGoldBar = tests[0]; + size_t nElems = currGoldBar.size(); + vector outData(nElems); + + ASSERT_SUCCESS(af_get_data_ptr((void *)outData.data(), outArray)); + + for (size_t elIter = 0; elIter < nElems; ++elIter) { + ASSERT_EQ(currGoldBar[elIter], outData[elIter]) + << "at: " << elIter << endl; + } + + ASSERT_SUCCESS(af_release_array(inArray)); + ASSERT_SUCCESS(af_release_array(outArray)); +} + using af::array; using af::freeHost; using af::randu; diff --git a/test/gloh.cpp b/test/gloh.cpp index b360ac6a18..4ce2fa547b 100644 --- a/test/gloh.cpp +++ b/test/gloh.cpp @@ -161,8 +161,9 @@ void glohTest(string pTestFile) { af_load_image(&inArray_f32, inFiles[testId].c_str(), false)); ASSERT_SUCCESS(conv_image(&inArray, inArray_f32)); - ASSERT_SUCCESS(af_gloh(&feat, &desc, inArray, 3, 0.04f, 10.0f, 1.6f, - true, 1.f / 256.f, 0.05f)); + ASSERT_SUCCESS(af_gloh(&feat, &desc, inArray, 3, + 0.04f, 10.0f, 1.6f, + true, 1.f / 256.f, 0.05f)); dim_t n = 0; af_array x, y, score, orientation, size; @@ -253,6 +254,7 @@ void glohTest(string pTestFile) { #define GLOH_INIT(desc, image) \ TYPED_TEST(GLOH, desc) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ glohTest(string(TEST_DIR "/gloh/" #image ".test")); \ } @@ -261,6 +263,7 @@ GLOH_INIT(man, man); ///////////////////////////////////// CPP //////////////////////////////// // TEST(GLOH, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); IMAGEIO_ENABLED_CHECK(); vector inDims; diff --git a/test/half.cpp b/test/half.cpp index 7f85950170..8afb6d5f4d 100644 --- a/test/half.cpp +++ b/test/half.cpp @@ -41,6 +41,7 @@ INSTANTIATE_TEST_SUITE_P(ToF16, HalfConvert, convert_params(f64, f16, 10), convert_params(s32, f16, 10), convert_params(u32, f16, 10), + convert_params(s8, f16, 10), convert_params(u8, f16, 10), convert_params(s64, f16, 10), convert_params(u64, f16, 10), @@ -53,6 +54,7 @@ INSTANTIATE_TEST_SUITE_P(FromF16, HalfConvert, convert_params(f16, f64, 10), convert_params(f16, s32, 10), convert_params(f16, u32, 10), + convert_params(f16, s8, 10), convert_params(f16, u8, 10), convert_params(f16, s64, 10), convert_params(f16, u64, 10), diff --git a/test/hamming.cpp b/test/hamming.cpp index b14a33db0a..b8394e36b5 100644 --- a/test/hamming.cpp +++ b/test/hamming.cpp @@ -95,21 +95,25 @@ void hammingMatcherTest(string pTestFile, int feat_dim) { } TYPED_TEST(HammingMatcher8, Hamming_500_5000_Dim0) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); hammingMatcherTest( string(TEST_DIR "/hamming/hamming_500_5000_dim0_u8.test"), 0); } TYPED_TEST(HammingMatcher8, Hamming_500_5000_Dim1) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); hammingMatcherTest( string(TEST_DIR "/hamming/hamming_500_5000_dim1_u8.test"), 1); } TYPED_TEST(HammingMatcher32, Hamming_500_5000_Dim0) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); hammingMatcherTest( string(TEST_DIR "/hamming/hamming_500_5000_dim0_u32.test"), 0); } TYPED_TEST(HammingMatcher32, Hamming_500_5000_Dim1) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); hammingMatcherTest( string(TEST_DIR "/hamming/hamming_500_5000_dim1_u32.test"), 1); } @@ -117,6 +121,7 @@ TYPED_TEST(HammingMatcher32, Hamming_500_5000_Dim1) { ///////////////////////////////////// CPP //////////////////////////////// // TEST(HammingMatcher, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); using af::array; using af::dim4; @@ -155,6 +160,7 @@ TEST(HammingMatcher, CPP) { } TEST(HammingMatcher64bit, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); using af::array; using af::dim4; diff --git a/test/harris.cpp b/test/harris.cpp index 43c0bb6433..f2fd27d47a 100644 --- a/test/harris.cpp +++ b/test/harris.cpp @@ -145,6 +145,7 @@ void harrisTest(string pTestFile, float sigma, unsigned block_size) { #define HARRIS_INIT(desc, image, sigma, block_size) \ TYPED_TEST(Harris, desc) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ harrisTest(string(TEST_DIR "/harris/" #image "_" #sigma \ "_" #block_size ".test"), \ sigma, block_size); \ @@ -167,6 +168,7 @@ using af::harris; using af::loadImage; TEST(FloatHarris, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); IMAGEIO_ENABLED_CHECK(); vector inDims; diff --git a/test/histogram.cpp b/test/histogram.cpp index ca3df72f74..ea9431485c 100644 --- a/test/histogram.cpp +++ b/test/histogram.cpp @@ -33,7 +33,7 @@ class Histogram : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + schar, uchar, short, ushort, intl, uintl> TestTypes; // register the type list diff --git a/test/homography.cpp b/test/homography.cpp index f4c1c75259..bd4809d428 100644 --- a/test/homography.cpp +++ b/test/homography.cpp @@ -69,8 +69,8 @@ void homographyTest(string pTestFile, const af_homography_type htype, ASSERT_SUCCESS(af_load_image(&trainArray_f32, inFiles[0].c_str(), false)); ASSERT_SUCCESS(conv_image(&trainArray, trainArray_f32)); - ASSERT_SUCCESS(af_orb(&train_feat, &train_desc, trainArray, 20.0f, 2000, - 1.2f, 8, true)); + ASSERT_SUCCESS(af_orb(&train_feat, &train_desc, trainArray, + 20.0f, 2000, 1.2f, 8, true)); ASSERT_SUCCESS(af_get_features_xpos(&train_feat_x, train_feat)); ASSERT_SUCCESS(af_get_features_ypos(&train_feat_y, train_feat)); @@ -96,15 +96,16 @@ void homographyTest(string pTestFile, const af_homography_type htype, const dim_t test_d0 = inDims[0][0] * size_ratio; const dim_t test_d1 = inDims[0][1] * size_ratio; const dim_t tDims[] = {test_d0, test_d1}; - if (rotate) + if (rotate) { ASSERT_SUCCESS(af_rotate(&queryArray, trainArray, theta, false, AF_INTERP_NEAREST)); - else + } else { ASSERT_SUCCESS(af_resize(&queryArray, trainArray, test_d0, test_d1, AF_INTERP_BILINEAR)); + } - ASSERT_SUCCESS(af_orb(&query_feat, &query_desc, queryArray, 20.0f, 2000, - 1.2f, 8, true)); + ASSERT_SUCCESS(af_orb(&query_feat, &query_desc, queryArray, + 20.0f, 2000, 1.2f, 8, true)); ASSERT_SUCCESS( af_hamming_matcher(&idx, &dist, train_desc, query_desc, 0, 1)); @@ -144,9 +145,9 @@ void homographyTest(string pTestFile, const af_homography_type htype, int inliers = 0; ASSERT_SUCCESS(af_homography(&H, &inliers, train_feat_x_idx, - train_feat_y_idx, query_feat_x_idx, - query_feat_y_idx, htype, 3.0f, 1000, - (af_dtype)dtype_traits::af_type)); + train_feat_y_idx, query_feat_x_idx, + query_feat_y_idx, htype, 3.0f, 1000, + (af_dtype)dtype_traits::af_type)); array HH(H); @@ -201,6 +202,7 @@ void homographyTest(string pTestFile, const af_homography_type htype, #define HOMOGRAPHY_INIT(desc, image, htype, rotate, size_ratio) \ TYPED_TEST(Homography, desc) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ homographyTest( \ string(TEST_DIR "/homography/" #image ".test"), htype, rotate, \ size_ratio); \ @@ -220,6 +222,7 @@ using af::features; using af::loadImage; TEST(Homography, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); IMAGEIO_ENABLED_CHECK(); vector inDims; @@ -262,7 +265,7 @@ TEST(Homography, CPP) { array H; int inliers = 0; homography(H, inliers, feat_train_x, feat_train_y, feat_query_x, - feat_query_y, AF_HOMOGRAPHY_RANSAC, 3.0f, 1000, f32); + feat_query_y, AF_HOMOGRAPHY_RANSAC, 3.0f, 1000, f32); float* gold_t = new float[8]; for (int i = 0; i < 8; i++) gold_t[i] = 0.f; diff --git a/test/hsv_rgb.cpp b/test/hsv_rgb.cpp index 423fc5fad5..134e56c6c3 100644 --- a/test/hsv_rgb.cpp +++ b/test/hsv_rgb.cpp @@ -38,6 +38,7 @@ TEST(hsv_rgb, InvalidArray) { } TEST(hsv2rgb, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); vector numDims; vector> in; vector> tests; @@ -54,6 +55,7 @@ TEST(hsv2rgb, CPP) { } TEST(rgb2hsv, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); vector numDims; vector> in; vector> tests; @@ -70,6 +72,7 @@ TEST(rgb2hsv, CPP) { } TEST(rgb2hsv, MaxDim) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); vector numDims; vector> in; vector> tests; @@ -108,6 +111,7 @@ TEST(rgb2hsv, MaxDim) { } TEST(hsv2rgb, MaxDim) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); vector numDims; vector> in; vector> tests; diff --git a/test/imageio.cpp b/test/imageio.cpp index 00834fb693..16cead852c 100644 --- a/test/imageio.cpp +++ b/test/imageio.cpp @@ -160,7 +160,7 @@ TEST(ImageIO, SavePNGCPP) { input(9, 0, 2) = 255; input(9, 9, span) = 255; - std::string testname = getTestName() + "_" + getBackendName(); + std::string testname = getTestName() + "_" + getBackendName(true); std::string imagename = "SaveCPP_" + testname + ".png"; saveImage(imagename.c_str(), input); @@ -180,7 +180,7 @@ TEST(ImageIO, SaveBMPCPP) { input(9, 0, 2) = 255; input(9, 9, span) = 255; - std::string testname = getTestName() + "_" + getBackendName(); + std::string testname = getTestName() + "_" + getBackendName(true); std::string imagename = "SaveCPP_" + testname + ".bmp"; saveImage(imagename.c_str(), input); @@ -291,7 +291,7 @@ TEST(ImageIO, SaveImage16CPP) { array input = randu(dims, u16); array input_255 = floor(input.as(f32) / 257); - std::string testname = getTestName() + "_" + getBackendName(); + std::string testname = getTestName() + "_" + getBackendName(true); std::string imagename = "saveImage16CPP_" + testname + ".png"; saveImage(imagename.c_str(), input); @@ -366,7 +366,7 @@ void saveLoadImageNativeCPPTest(dim4 dims) { array input = randu(dims, (af_dtype)dtype_traits::af_type); - std::string imagename = getTestName() + "_" + getBackendName() + ".png"; + std::string imagename = getTestName() + "_" + getBackendName(true) + ".png"; saveImageNative(imagename.c_str(), input); diff --git a/test/index.cpp b/test/index.cpp index c8e1a7ffb9..d5d010ffb1 100644 --- a/test/index.cpp +++ b/test/index.cpp @@ -138,7 +138,7 @@ class Indexing1D : public ::testing::Test { }; typedef ::testing::Types AllTypes; TYPED_TEST_SUITE(Indexing1D, AllTypes); @@ -710,8 +710,9 @@ class lookup : public ::testing::Test { virtual void SetUp() {} }; -typedef ::testing::Types +typedef ::testing::Types ArrIdxTestTypes; TYPED_TEST_SUITE(lookup, ArrIdxTestTypes); @@ -808,6 +809,127 @@ TEST(lookup, Issue2009) { ASSERT_ARRAYS_EQ(a, b); } +TEST(lookup, Issue3613_FirstDimLookupWithOffset) { + dim4 dims(1); + const int selected_dim = 0; // selected span dimension + dims[selected_dim] = 125; // input size + + array a = iota(dims); + array idxs = iota(dim4(5, 4, 3, 2)); + array selected_idx = idxs(af::span, 3, 2, 1); // Offsets in second, third, & fourth dimension + + array expected_selected_idx = range(dim4(5)) * 1 + 3 * 5 + 2 * (5 * 4) + 1 * (5 * 4 * 3); + ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx); + + array b = af::lookup(a, selected_idx, selected_dim); + dim4 output_dims(1); + output_dims[selected_dim] = 5; // output size + ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, output_dims), b); // lookup output should be the same as looked up indices +} + +TEST(lookup, Issue3613_SecondDimLookupWithOffset) { + dim4 dims(1); + const int selected_dim = 1; // selected span dimension + dims[selected_dim] = 125; // input size + + array a = iota(dims); + array idxs = iota(dim4(5, 4, 3, 2)); + array selected_idx = idxs(af::span, 3, 2, 1); // Offsets in second, third, & fourth dimension + + array expected_selected_idx = range(dim4(5)) * 1 + 3 * 5 + 2 * (5 * 4) + 1 * (5 * 4 * 3); + ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx); + + array b = af::lookup(a, selected_idx, selected_dim); + dim4 output_dims(1); + output_dims[selected_dim] = 5; // output size + ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, output_dims), b); // lookup output should be the same as looked up indices +} + + +TEST(lookup, Issue3613_ThirdDimLookupWithOffset) { + dim4 dims(1); + const int selected_dim = 2; // selected span dimension + dims[selected_dim] = 125; // input size + + array a = iota(dims); + array idxs = iota(dim4(5, 4, 3, 2)); + array selected_idx = idxs(af::span, 3, 2, 1); // Offsets in second, third, & fourth dimension + + array expected_selected_idx = range(dim4(5)) * 1 + 3 * 5 + 2 * (5 * 4) + 1 * (5 * 4 * 3); + ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx); + + array b = af::lookup(a, selected_idx, selected_dim); + dim4 output_dims(1); + output_dims[selected_dim] = 5; // output size + ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, output_dims), b); // lookup output should be the same as looked up indices +} + +TEST(lookup, Issue3613_FourthDimLookupWithOffset) { + dim4 dims(1); + const int selected_dim = 3; // selected span dimension + dims[selected_dim] = 125; // input size + + array a = iota(dims); + array idxs = iota(dim4(5, 4, 3, 2)); + array selected_idx = idxs(af::span, 3, 2, 1); // Offsets in second, third, & fourth dimension + + array expected_selected_idx = range(dim4(5)) * 1 + 3 * 5 + 2 * (5 * 4) + 1 * (5 * 4 * 3); + ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx); + + array b = af::lookup(a, selected_idx, selected_dim); + dim4 output_dims(1); + output_dims[selected_dim] = 5; // output size + ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, output_dims), b); // lookup output should be the same as looked up indices +} + +TEST(lookup, IndicesInSecondDimension) { + const int selected_dim = 1; // selected span dimension + dim4 dims(1); + dims[selected_dim] = 3; + + array a = iota(dim4(100)); + array idxs = iota(dim4(3, 3, 3, 3)); + array selected_idx = idxs(0, af::span, 0, 0); // Indices along the second dimension + + array expected_selected_idx = iota(dims) * pow(3, selected_dim); + ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx); + + array b = af::lookup(a, selected_idx); + ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, dim4(3)), b); +} + +TEST(lookup, IndicesInThirdDimension) { + const int selected_dim = 2; // selected span dimension + dim4 dims(1); + dims[selected_dim] = 3; + + array a = iota(dim4(100)); + array idxs = iota(dim4(3, 3, 3, 3)); + array selected_idx = idxs(0, 0, af::span, 0); // Indices along the third dimension + + array expected_selected_idx = iota(dims) * pow(3, selected_dim); + ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx); + + array b = af::lookup(a, selected_idx); + ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, dim4(3)), b); +} + +TEST(lookup, IndicesInFourthDimension) { + const int selected_dim = 3; // selected span dimension + dim4 dims(1); + dims[selected_dim] = 3; + + array a = iota(dim4(100)); + array idxs = iota(dim4(3, 3, 3, 3)); + array selected_idx = idxs(0, 0, 0, af::span); // Indices along the fourth dimension + + array expected_selected_idx = iota(dims) * pow(3, selected_dim); + ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx); + + array b = af::lookup(a, selected_idx); + ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, dim4(3)), b); +} + TEST(lookup, SNIPPET_lookup1d) { //! [ex_index_lookup1d] diff --git a/test/inverse_deconv.cpp b/test/inverse_deconv.cpp index b6db793f4b..86ac2869ab 100644 --- a/test/inverse_deconv.cpp +++ b/test/inverse_deconv.cpp @@ -25,7 +25,7 @@ template class InverseDeconvolution : public ::testing::Test {}; // create a list of types to be tested -typedef ::testing::Types TestTypes; +typedef ::testing::Types TestTypes; // register the type list TYPED_TEST_SUITE(InverseDeconvolution, TestTypes); diff --git a/test/iota.cpp b/test/iota.cpp index c776d7628e..33ff36e3ba 100644 --- a/test/iota.cpp +++ b/test/iota.cpp @@ -39,7 +39,8 @@ class Iota : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + signed char, unsigned char, short, ushort, + half_float::half> TestTypes; // register the type list diff --git a/test/ireduce.cpp b/test/ireduce.cpp index 2ebd951d46..b155512e32 100644 --- a/test/ireduce.cpp +++ b/test/ireduce.cpp @@ -103,6 +103,7 @@ MINMAXOP(min, double) MINMAXOP(min, int) MINMAXOP(min, uint) MINMAXOP(min, char) +MINMAXOP(min, schar) MINMAXOP(min, uchar) MINMAXOP(max, float) @@ -110,6 +111,7 @@ MINMAXOP(max, double) MINMAXOP(max, int) MINMAXOP(max, uint) MINMAXOP(max, char) +MINMAXOP(max, schar) MINMAXOP(max, uchar) TEST(IndexedReduce, MaxIndexedSmall) { @@ -418,3 +420,136 @@ TEST(IndexedReduce, MaxCplxPreferSmallerIdxIfEqual) { ASSERT_EQ(h_max_idx[0], gold_max_idx); } + +#define SUBA_TEST_DATA \ + float test_data[25] = {0.0168, 0.0278, 0.0317, 0.0248, 0.0131, \ + 0.0197, 0.0321, 0.0362, 0.0279, 0.0141, \ + 0.0218, 0.0353, 0.0394, 0.0297, 0.0143, \ + 0.0224, 0.0363, 0.0104, 0.0302, 0.0142, \ + 0.0217, 0.0409, 0.0398, 0.0302, 0.0144}; \ + array a(5, 5, test_data); \ + array a_sub = a(seq(1, 3), seq(2,4)) + +TEST(IndexedReduce, max_subarray_all) { + SUBA_TEST_DATA; + + float gold_max_val = 0.0409; + unsigned gold_max_idx = 6; + + float max_val; + unsigned max_idx; + max(&max_val, &max_idx, a_sub); + + ASSERT_FLOAT_EQ(max_val, gold_max_val); + ASSERT_EQ(max_idx, gold_max_idx); +} + +TEST(IndexedReduce, min_subarray_all) { + SUBA_TEST_DATA; + + float gold_min_val = 0.0104; + unsigned gold_min_idx = 4; + + float min_val; + unsigned min_idx; + min(&min_val, &min_idx, a_sub); + + ASSERT_FLOAT_EQ(min_val, gold_min_val); + ASSERT_EQ(min_idx, gold_min_idx); +} + +TEST(IndexedReduce, max_subarray_0) { + SUBA_TEST_DATA; + + float gold_val[3] = {0.0394, 0.0363, 0.0409}; + unsigned gold_idx[3] = {1, 0, 0}; + + array val; + array idx; + float h_val[3]; + unsigned h_idx[3]; + + max(val, idx, a_sub); + val.host(&h_val); + idx.host(&h_idx); + + for(int i = 0; i < 3; ++i) { + ASSERT_FLOAT_EQ(h_val[i], gold_val[i]); + ASSERT_EQ(h_idx[i], gold_idx[i]); + } +} + +TEST(IndexedReduce, min_subarray_0) { + SUBA_TEST_DATA; + + float gold_val[3] = {0.0297, 0.0104, 0.0302}; + unsigned gold_idx[3] = {2, 1, 2}; + + array val; + array idx; + float h_val[3]; + unsigned h_idx[3]; + + min(val, idx, a_sub); + val.host(&h_val); + idx.host(&h_idx); + + for(int i = 0; i < 3; ++i) { + ASSERT_FLOAT_EQ(h_val[i], gold_val[i]); + ASSERT_EQ(h_idx[i], gold_idx[i]); + } +} + +TEST(IndexedReduce, max_subarray_1) { + SUBA_TEST_DATA; + + float gold_val[3] = {0.0409, 0.0398, 0.0302}; + unsigned gold_idx[3] = {2, 2, 1}; + + array val; + array idx; + float h_val[3]; + unsigned h_idx[3]; + + max(val, idx, a_sub, 1); + val.host(&h_val); + idx.host(&h_idx); + + for(int i = 0; i < 3; ++i) { + ASSERT_FLOAT_EQ(h_val[i], gold_val[i]); + ASSERT_EQ(h_idx[i], gold_idx[i]); + } +} + +TEST(IndexedReduce, min_subarray_1) { + SUBA_TEST_DATA; + + float gold_val[3] = {0.0353, 0.0104, 0.0297}; + unsigned gold_idx[3] = {0, 1, 0}; + + array val; + array idx; + float h_val[3]; + unsigned h_idx[3]; + + min(val, idx, a_sub, 1); + val.host(&h_val); + idx.host(&h_idx); + + for(int i = 0; i < 3; ++i) { + ASSERT_FLOAT_EQ(h_val[i], gold_val[i]); + ASSERT_EQ(h_idx[i], gold_idx[i]); + } +} + +//Ensure that array is evaluated before reducing +TEST(IndexedReduce, reduce_jit_array) { + af::array jit(af::dim4(2),{1.0f, 2.0f}); + jit += af::constant(1.0f, af::dim4(2)); + float val; unsigned idx; + float gold_val = 2.0f; + unsigned gold_idx = 0; + af::min(&val, &idx, jit); + ASSERT_EQ(val, gold_val); + ASSERT_EQ(idx, gold_idx); +} diff --git a/test/iterative_deconv.cpp b/test/iterative_deconv.cpp index e59440b977..290b81f0d6 100644 --- a/test/iterative_deconv.cpp +++ b/test/iterative_deconv.cpp @@ -25,7 +25,7 @@ template class IterativeDeconvolution : public ::testing::Test {}; // create a list of types to be tested -typedef ::testing::Types TestTypes; +typedef ::testing::Types TestTypes; // register the type list TYPED_TEST_SUITE(IterativeDeconvolution, TestTypes); @@ -40,6 +40,11 @@ void iterDeconvImageTest(string pTestFile, const unsigned iters, const float rf, SUPPORTED_TYPE_CHECK(T); IMAGEIO_ENABLED_CHECK(); + if (is_same_type::value && + algo == AF_ITERATIVE_DECONV_RICHARDSONLUCY) { + GTEST_SKIP() << "Incompatible with signed values"; + } + using af::dim4; vector inDims; diff --git a/test/jit.cpp b/test/jit.cpp index 3848a22242..487fdcb6e2 100644 --- a/test/jit.cpp +++ b/test/jit.cpp @@ -814,3 +814,43 @@ TEST(JIT, setKernelCacheDirectory) { // Reset to the old path ASSERT_SUCCESS(af_set_kernel_cache_directory(old_path.c_str(), false)); } + +// Ensure that a correct result is obtained when evaluating an expression +// that contains both an array and its transpose - see ISSUE 3660 +TEST(JIT, evaluateBothArrayAndItsTranspose) { + float X2_ptr[25] = { -1., -1., -1., -1., -1., + -0.5, -0.5, -0.5, -0.5, -0.5, + 0., 0., 0., 0., 0., + 0.5, 0.5, 0.5, 0.5, 0.5, + 1., 1., 1., 1., 1. }; + array X2_gold(5, 5, X2_ptr); + + float Y2_ptr[25] = { -1., -0.5, 0., 0.5, 1., + -1., -0.5, 0., 0.5, 1., + -1., -0.5, 0., 0.5, 1., + -1., -0.5, 0., 0.5, 1., + -1., -0.5, 0., 0.5, 1. }; + array Y2_gold(5, 5, Y2_ptr); + + float X2Y2_ptr[25] = { -2., -1.5, -1., -0.5, 0., + -1.5, -1., -0.5, 0., 0.5, + -1., -0.5, 0., 0.5, 1., + -0.5, 0., 0.5, 1., 1.5, + 0., 0.5, 1., 1.5, 2. }; + array X2Y2_gold(5, 5, X2Y2_ptr); + + int n = 5; + int half = (n - 1) / 2; + double delta = 1.0 / half; + + array coord = delta * (af::range(n) - half); + + array X2 = tile(coord.T(), n, 1); + array Y2 = tile(coord, 1, n); + + array X2Y2 = X2 + Y2; + + ASSERT_ARRAYS_EQ(X2_gold, X2); + ASSERT_ARRAYS_EQ(Y2_gold, Y2); + ASSERT_ARRAYS_EQ(X2Y2_gold, X2Y2); +} diff --git a/test/join.cpp b/test/join.cpp index cf33fccb67..5cd470780f 100644 --- a/test/join.cpp +++ b/test/join.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -47,8 +48,8 @@ class Join : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + intl, uintl, char, signed char, unsigned char, short, + ushort, half_float::half> TestTypes; // register the type list @@ -266,3 +267,67 @@ TEST(Join, ManyEmpty) { ASSERT_ARRAYS_EQ(gold, eace); ASSERT_ARRAYS_EQ(gold, acee); } + +TEST(Join, respect_parameters_order_ISSUE3511) { + const float column_host1[] = {1., 2., 3.}; + const float column_host2[] = {4., 5., 6.}; + const af::array buf1(3, 1, column_host1); + const af::array buf2(3, 1, column_host2); + + // We need to avoid that JIT arrays are evaluated during whatever call, + // so we will have to work with copies for single use + const af::array jit1{buf1 + 1.0}; + const af::array jit2{buf2 + 2.0}; + const std::array cases{jit1, -jit1, jit1 + 1.0, jit2, + -jit2, jit1 + jit2, buf1, buf2}; + const std::array cases_name{"JIT1", "-JIT1", "JIT1+1.0", + "JIT2", "-JIT2", "JIT1+JIT2", + "BUF1", "BUF2"}; + assert(cases.size() == cases_name.size()); + for (size_t cl0{0}; cl0 < cases.size(); ++cl0) { + for (size_t cl1{0}; cl1 < cases.size(); ++cl1) { + printf("Testing: af::join(1,%s,%s)\n", cases_name[cl0], + cases_name[cl1]); + const array col0{cases[cl0]}; + const array col1{cases[cl1]}; + const array result{af::join(1, col0, col1)}; + ASSERT_ARRAYS_EQ(result(af::span, 0), col0); + ASSERT_ARRAYS_EQ(result(af::span, 1), col1); + } + } + // Join of 3 arrays + for (size_t cl0{0}; cl0 < cases.size(); ++cl0) { + for (size_t cl1{0}; cl1 < cases.size(); ++cl1) { + for (size_t cl2{0}; cl2 < cases.size(); ++cl2) { + printf("Testing: af::join(1,%s,%s,%s)\n", cases_name[cl0], + cases_name[cl1], cases_name[cl2]); + const array col0{cases[cl0]}; + const array col1{cases[cl1]}; + const array col2{cases[cl2]}; + const array result{af::join(1, col0, col1, col2)}; + ASSERT_ARRAYS_EQ(result(af::span, 0), col0); + ASSERT_ARRAYS_EQ(result(af::span, 1), col1); + ASSERT_ARRAYS_EQ(result(af::span, 2), col2); + } + } + } +} + +#define TEST_TEMP_FORMAT(form, d) \ + TEST(TEMP_FORMAT, form##_dim##d) { \ + const dim4 dims(2, 2, 2, 2); \ + const array a(randu(dims)); \ + const array b(randu(dims)); \ + \ + array out = join(d, toTempFormat(form, a), toTempFormat(form, b)); \ + array gold = join(d, a, b); \ + EXPECT_ARRAYS_EQ(gold, out); \ + } + +#define TEST_TEMP_FORMATS(form) \ + TEST_TEMP_FORMAT(form, 0) \ + TEST_TEMP_FORMAT(form, 1) \ + TEST_TEMP_FORMAT(form, 2) \ + TEST_TEMP_FORMAT(form, 3) + +FOREACH_TEMP_FORMAT(TEST_TEMP_FORMATS) diff --git a/test/match_template.cpp b/test/match_template.cpp index 33b6096815..f5f6eb4fc7 100644 --- a/test/match_template.cpp +++ b/test/match_template.cpp @@ -31,7 +31,8 @@ class MatchTemplate : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list @@ -84,16 +85,19 @@ void matchTemplateTest(string pTestFile, af_match_type pMatchType) { } TYPED_TEST(MatchTemplate, Matrix_SAD) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); matchTemplateTest( string(TEST_DIR "/MatchTemplate/matrix_sad.test"), AF_SAD); } TYPED_TEST(MatchTemplate, Matrix_SSD) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); matchTemplateTest( string(TEST_DIR "/MatchTemplate/matrix_ssd.test"), AF_SSD); } TYPED_TEST(MatchTemplate, MatrixBatch_SAD) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); matchTemplateTest( string(TEST_DIR "/MatchTemplate/matrix_sad_batch.test"), AF_SAD); } diff --git a/test/math.cpp b/test/math.cpp index 8e2243e13c..ee42a11423 100644 --- a/test/math.cpp +++ b/test/math.cpp @@ -1,5 +1,5 @@ /******************************************************* - * Copyright (c) 2014, ArrayFire + * Copyright (c) 2025, ArrayFire * All rights reserved. * * This file is distributed under 3-clause BSD license. @@ -46,7 +46,7 @@ T rsqrt(T in) { } #define MATH_TEST(T, func, err, lo, hi) \ - TEST(MathTests, Test_##func##_##T) { \ + TEST(Math, func##_##T) { \ try { \ SUPPORTED_TYPE_CHECK(T); \ af_dtype ty = (af_dtype)dtype_traits::af_type; \ @@ -135,7 +135,7 @@ MATH_TESTS_REAL(erf) MATH_TESTS_REAL(erfc) #endif -TEST(MathTests, Not) { +TEST(Math, Not) { array a = randu(5, 5, b8); array b = !a; char *ha = a.host(); @@ -146,3 +146,47 @@ TEST(MathTests, Not) { af_free_host(ha); af_free_host(hb); } + +TEST(Math, Modulus) { + af::dim4 shape(2, 2); + std::vector aData{3, 3, 3, 3}; + std::vector bData{2, 2, 2, 2}; + + auto a = af::array(shape, aData.data(), afHost); + auto b = af::array(shape, bData.data(), afHost); + auto rem = a % b; + auto neg_rem = -a % b; + + ASSERT_ARRAYS_EQ(af::constant(1, shape, s64), rem); + ASSERT_ARRAYS_EQ(af::constant(-1, shape, s64), neg_rem); +} + +TEST(Math, ModulusFloat) { + SUPPORTED_TYPE_CHECK(half_float::half); + af::dim4 shape(2, 2); + + auto a = af::constant(3, shape, af::dtype::f16); + auto b = af::constant(2, shape, af::dtype::f16); + auto a32 = af::constant(3, shape, af::dtype::f32); + auto b32 = af::constant(2, shape, af::dtype::f32); + auto a64 = af::constant(3, shape, af::dtype::f64); + auto b64 = af::constant(2, shape, af::dtype::f64); + + auto rem = a % b; + auto rem32 = a32 % b32; + auto rem64 = a64 % b64; + + auto neg_rem = -a % b; + auto neg_rem32 = -a32 % b32; + auto neg_rem64 = -a64 % b64; + + ASSERT_ARRAYS_EQ(af::constant(1, shape, af::dtype::f16), rem); + ASSERT_ARRAYS_EQ(af::constant(1, shape, af::dtype::f32), rem32); + ASSERT_ARRAYS_EQ(af::constant(1, shape, af::dtype::f64), rem64); + + ASSERT_ARRAYS_EQ(af::constant(-1, shape, af::dtype::f16), neg_rem); + ASSERT_ARRAYS_EQ(af::constant(-1, shape, af::dtype::f32), neg_rem32); + ASSERT_ARRAYS_EQ(af::constant(-1, shape, af::dtype::f64), neg_rem64); + + ASSERT_ARRAYS_EQ(rem32.as(f16), rem); +} diff --git a/test/mean.cpp b/test/mean.cpp index c9c6eb567b..79dd76db2d 100644 --- a/test/mean.cpp +++ b/test/mean.cpp @@ -40,7 +40,7 @@ class Mean : public ::testing::Test { // This list does not allow to cleanly add the af_half/half_float type : at the // moment half tested in some special unittests typedef ::testing::Types + char, schar, uchar, short, ushort, half_float::half> TestTypes; // register the type list @@ -70,9 +70,9 @@ template struct meanOutType { typedef typename cond_type< is_same_type::value || is_same_type::value || - is_same_type::value || is_same_type::value || - is_same_type::value || is_same_type::value || - is_same_type::value, + is_same_type::value || is_same_type::value || + is_same_type::value || is_same_type::value || + is_same_type::value || is_same_type::value, float, typename elseType::type>::type type; }; @@ -228,7 +228,7 @@ TEST(MeanAll, s32) { meanAllTest(2, dim4(5, 5, 2, 2)); } TEST(MeanAll, u32) { meanAllTest(2, dim4(100, 1, 1, 1)); } -TEST(MeanAll, s8) { meanAllTest(2, dim4(5, 5, 2, 2)); } +TEST(MeanAll, s8) { meanAllTest(2, dim4(5, 5, 2, 2)); } TEST(MeanAll, u8) { meanAllTest(2, dim4(100, 1, 1, 1)); } diff --git a/test/meanshift.cpp b/test/meanshift.cpp index 1f0aa697b3..d91648ae52 100644 --- a/test/meanshift.cpp +++ b/test/meanshift.cpp @@ -28,8 +28,8 @@ class Meanshift : public ::testing::Test { virtual void SetUp() {} }; -typedef ::testing::Types +typedef ::testing::Types TestTypes; TYPED_TEST_SUITE(Meanshift, TestTypes); diff --git a/test/meanvar.cpp b/test/meanvar.cpp index 08e4702481..c7eba339a8 100644 --- a/test/meanvar.cpp +++ b/test/meanvar.cpp @@ -40,8 +40,8 @@ struct varOutType { typedef typename cond_type< is_same_type::value || is_same_type::value || is_same_type::value || is_same_type::value || - is_same_type::value || is_same_type::value || - is_same_type::value, + is_same_type::value || is_same_type::value || + is_same_type::value || is_same_type::value, float, typename elseType::type>::type type; }; @@ -377,5 +377,6 @@ TEST_P(MeanVarHalf, TestingCPP) { } // Only test small sizes because the range of the large arrays go out of bounds +MEANVAR_TEST(SignedChar, signed char) MEANVAR_TEST(UnsignedChar, unsigned char) // MEANVAR_TEST(Bool, unsigned char) // TODO(umar): test this type diff --git a/test/medfilt.cpp b/test/medfilt.cpp index 1939379974..5ef951d5b1 100644 --- a/test/medfilt.cpp +++ b/test/medfilt.cpp @@ -35,7 +35,8 @@ class MedianFilter1d : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list @@ -80,24 +81,28 @@ void medfiltTest(string pTestFile, dim_t w_len, dim_t w_wid, } TYPED_TEST(MedianFilter, ZERO_PAD_3x3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); medfiltTest( string(TEST_DIR "/medianfilter/zero_pad_3x3_window.test"), 3, 3, AF_PAD_ZERO); } TYPED_TEST(MedianFilter, SYMMETRIC_PAD_3x3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); medfiltTest( string(TEST_DIR "/medianfilter/symmetric_pad_3x3_window.test"), 3, 3, AF_PAD_SYM); } TYPED_TEST(MedianFilter, BATCH_ZERO_PAD_3x3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); medfiltTest( string(TEST_DIR "/medianfilter/batch_zero_pad_3x3_window.test"), 3, 3, AF_PAD_ZERO); } TYPED_TEST(MedianFilter, BATCH_SYMMETRIC_PAD_3x3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); medfiltTest( string(TEST_DIR "/medianfilter/batch_symmetric_pad_3x3_window.test"), 3, 3, AF_PAD_SYM); @@ -140,24 +145,28 @@ void medfilt1_Test(string pTestFile, dim_t w_wid, af_border_type pad) { } TYPED_TEST(MedianFilter1d, ZERO_PAD_3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); medfilt1_Test( string(TEST_DIR "/medianfilter/zero_pad_3x1_window.test"), 3, AF_PAD_ZERO); } TYPED_TEST(MedianFilter1d, SYMMETRIC_PAD_3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); medfilt1_Test( string(TEST_DIR "/medianfilter/symmetric_pad_3x1_window.test"), 3, AF_PAD_SYM); } TYPED_TEST(MedianFilter1d, BATCH_ZERO_PAD_3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); medfilt1_Test( string(TEST_DIR "/medianfilter/batch_zero_pad_3x1_window.test"), 3, AF_PAD_ZERO); } TYPED_TEST(MedianFilter1d, BATCH_SYMMETRIC_PAD_3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); medfilt1_Test( string(TEST_DIR "/medianfilter/batch_symmetric_pad_3x1_window.test"), 3, AF_PAD_SYM); @@ -338,6 +347,7 @@ TYPED_TEST(MedianFilter1d, InvalidPadType) { medfilt1d_PadTest(); } using af::array; TEST(MedianFilter, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); const dim_t w_len = 3; const dim_t w_wid = 3; @@ -365,6 +375,7 @@ TEST(MedianFilter, CPP) { } TEST(MedianFilter1d, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); const dim_t w_wid = 3; vector numDims; @@ -391,6 +402,7 @@ TEST(MedianFilter1d, CPP) { } TEST(MedianFilter, Docs) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); float input[] = {1.0000, 2.0000, 3.0000, 4.0000, 5.0000, 6.0000, 7.0000, 8.0000, 9.0000, 10.0000, 11.0000, 12.0000, 13.0000, 14.0000, 15.0000, 16.0000}; @@ -431,6 +443,7 @@ using af::seq; using af::span; TEST(MedianFilter, GFOR) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); dim4 dims = dim4(10, 10, 3); array A = iota(dims); array B = constant(0, dims); @@ -445,6 +458,7 @@ TEST(MedianFilter, GFOR) { } TEST(MedianFilter1d, GFOR) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); dim4 dims = dim4(10, 10, 3); array A = iota(dims); array B = constant(0, dims); diff --git a/test/median.cpp b/test/median.cpp index c55251e66c..4f64631c6f 100644 --- a/test/median.cpp +++ b/test/median.cpp @@ -119,6 +119,7 @@ void median_test(int nx, int ny = 1, int nz = 1, int nw = 1) { MEDIAN_FLAT(float, float) MEDIAN_FLAT(float, int) MEDIAN_FLAT(float, uint) +MEDIAN_FLAT(float, schar) MEDIAN_FLAT(float, uchar) MEDIAN_FLAT(float, short) MEDIAN_FLAT(float, ushort) @@ -151,6 +152,7 @@ MEDIAN_FLAT(double, double) MEDIAN(float, float) MEDIAN(float, int) MEDIAN(float, uint) +MEDIAN(float, schar) MEDIAN(float, uchar) MEDIAN(float, short) MEDIAN(float, ushort) diff --git a/test/memory.cpp b/test/memory.cpp index 37a1de87b1..9214ab472c 100644 --- a/test/memory.cpp +++ b/test/memory.cpp @@ -74,7 +74,8 @@ class MemAlloc : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + intl, uintl, char, signed char, unsigned char, short, + ushort> TestTypes; // register the type list @@ -917,6 +918,7 @@ TEST_F(MemoryManagerApi, E2ETest4D) { } TEST_F(MemoryManagerApi, E2ETest4DComplexDouble) { + SUPPORTED_TYPE_CHECK(double); size_t aSize = 8; af::array a = af::array(aSize, aSize, aSize, aSize, af::dtype::c64); @@ -932,6 +934,7 @@ TEST_F(MemoryManagerApi, E2ETest4DComplexDouble) { } TEST_F(MemoryManagerApi, E2ETestMultipleAllocations) { + SUPPORTED_TYPE_CHECK(double); size_t aSize = 8; af::array a = af::array(aSize, af::dtype::c64); diff --git a/test/moddims.cpp b/test/moddims.cpp index a7dea52a00..c8b98f05d1 100644 --- a/test/moddims.cpp +++ b/test/moddims.cpp @@ -36,8 +36,8 @@ class Moddims : public ::testing::Test { // create a list of types to be tested // TODO: complex types tests have to be added -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list diff --git a/test/moments.cpp b/test/moments.cpp index 6b02cb614a..bec90e5b5d 100644 --- a/test/moments.cpp +++ b/test/moments.cpp @@ -158,25 +158,30 @@ void momentsOnImageTest(string pTestFile, string pImageFile, bool isColor) { } TEST(IMAGE, MomentsImage) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); momentsOnImageTest(string(TEST_DIR "/moments/gray_seq_16_moments.test"), string(TEST_DIR "/imageio/gray_seq_16.png"), false); } TEST(Image, MomentsImageBatch) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); momentsTest( string(TEST_DIR "/moments/simple_mat_batch_moments.test")); } TEST(Image, MomentsBatch2D) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); momentsOnImageTest(string(TEST_DIR "/moments/color_seq_16_moments.test"), string(TEST_DIR "/imageio/color_seq_16.png"), true); } TYPED_TEST(Image, MomentsSynthTypes) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); momentsTest(string(TEST_DIR "/moments/simple_mat_moments.test")); } TEST(Image, Moment_Issue1957) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); array A = identity(3, 3, b8); double m00; diff --git a/test/morph.cpp b/test/morph.cpp index 9cc2255fb5..b68d95076f 100644 --- a/test/morph.cpp +++ b/test/morph.cpp @@ -30,7 +30,8 @@ class Morph : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list @@ -59,16 +60,19 @@ void morphTest(string pTestFile) { maskDims.ndims(), maskDims.get(), (af_dtype)dtype_traits::af_type)); + af_err af_stat; if (isDilation) { - if (isVolume) + if (isVolume) { ASSERT_SUCCESS(af_dilate3(&outArray, inArray, maskArray)); - else + } else { ASSERT_SUCCESS(af_dilate(&outArray, inArray, maskArray)); + } } else { - if (isVolume) + if (isVolume) { ASSERT_SUCCESS(af_erode3(&outArray, inArray, maskArray)); - else + } else { ASSERT_SUCCESS(af_erode(&outArray, inArray, maskArray)); + } } for (size_t testIter = 0; testIter < tests.size(); ++testIter) { @@ -83,52 +87,63 @@ void morphTest(string pTestFile) { } TYPED_TEST(Morph, Dilate3x3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest(string(TEST_DIR "/morph/dilate3x3.test")); } TYPED_TEST(Morph, Erode3x3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest(string(TEST_DIR "/morph/erode3x3.test")); } TYPED_TEST(Morph, Dilate4x4) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest(string(TEST_DIR "/morph/dilate4x4.test")); } TYPED_TEST(Morph, Dilate12x12) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest( string(TEST_DIR "/morph/dilate12x12.test")); } TYPED_TEST(Morph, Erode4x4) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest(string(TEST_DIR "/morph/erode4x4.test")); } TYPED_TEST(Morph, Dilate3x3_Batch) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest( string(TEST_DIR "/morph/dilate3x3_batch.test")); } TYPED_TEST(Morph, Erode3x3_Batch) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest( string(TEST_DIR "/morph/erode3x3_batch.test")); } TYPED_TEST(Morph, Dilate3x3x3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest( string(TEST_DIR "/morph/dilate3x3x3.test")); } TYPED_TEST(Morph, Erode3x3x3) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest( string(TEST_DIR "/morph/erode3x3x3.test")); } TYPED_TEST(Morph, Dilate4x4x4) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest( string(TEST_DIR "/morph/dilate4x4x4.test")); } TYPED_TEST(Morph, Erode4x4x4) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphTest( string(TEST_DIR "/morph/erode4x4x4.test")); } @@ -186,10 +201,10 @@ void morphImageTest(string pTestFile, dim_t seLen) { ASSERT_SUCCESS(error_code); ASSERT_IMAGES_NEAR(goldArray, outArray, 0.018f); #else - ASSERT_EQ(error_code, - (targetType != b8 && seLen > 19 ? AF_ERR_NOT_SUPPORTED - : AF_SUCCESS)); - if (!(targetType != b8 && seLen > 19)) { + if (targetType != b8 && seLen > 19) { + ASSERT_EQ(error_code, AF_ERR_NOT_SUPPORTED); + } else { + ASSERT_SUCCESS(error_code); ASSERT_IMAGES_NEAR(goldArray, outArray, 0.018f); } #endif @@ -204,10 +219,12 @@ void morphImageTest(string pTestFile, dim_t seLen) { } TEST(Morph, GrayscaleDilation3x3StructuringElement) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphImageTest(string(TEST_DIR "/morph/gray.test"), 3); } TEST(Morph, ColorImageErosion3x3StructuringElement) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); morphImageTest(string(TEST_DIR "/morph/color.test"), 3); } @@ -428,14 +445,17 @@ void cppMorphImageTest(string pTestFile) { } TEST(Morph, Grayscale_CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); cppMorphImageTest(string(TEST_DIR "/morph/gray.test")); } TEST(Morph, ColorImage_CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); cppMorphImageTest(string(TEST_DIR "/morph/color.test")); } TEST(Morph, GFOR) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); dim4 dims = dim4(10, 10, 3); array A = iota(dims); array B = constant(0, dims); @@ -451,6 +471,7 @@ TEST(Morph, GFOR) { } TEST(Morph, EdgeIssue1564) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); int inputData[10 * 10] = {0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -466,12 +487,13 @@ TEST(Morph, EdgeIssue1564) { array input(10, 10, inputData); int maskData[3 * 3] = {1, 1, 1, 1, 0, 1, 1, 1, 1}; array mask(3, 3, maskData); + array dilated = dilate(input.as(b8), mask.as(b8)); size_t nElems = dilated.elements(); vector outData(nElems); dilated.host((void*)outData.data()); - + for (size_t i = 0; i < nElems; ++i) { ASSERT_EQ((int)outData[i], goldData[i]); } diff --git a/test/nearest_neighbour.cpp b/test/nearest_neighbour.cpp index 01847aea65..82551bc31b 100644 --- a/test/nearest_neighbour.cpp +++ b/test/nearest_neighbour.cpp @@ -34,8 +34,8 @@ class NearestNeighbour : public ::testing::Test { }; // create lists of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; template @@ -53,6 +53,11 @@ struct otype_t { typedef uint otype; }; +template<> +struct otype_t { + typedef int otype; +}; + template<> struct otype_t { typedef uint otype; @@ -117,24 +122,28 @@ void nearestNeighbourTest(string pTestFile, int feat_dim, // SSD ///////////////////////////////////////////////// TYPED_TEST(NearestNeighbour, NN_SSD_100_1000_Dim0) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); nearestNeighbourTest( string(TEST_DIR "/nearest_neighbour/ssd_100_1000_dim0.test"), 0, AF_SSD); } TYPED_TEST(NearestNeighbour, NN_SSD_100_1000_Dim1) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); nearestNeighbourTest( string(TEST_DIR "/nearest_neighbour/ssd_100_1000_dim1.test"), 1, AF_SSD); } TYPED_TEST(NearestNeighbour, NN_SSD_500_5000_Dim0) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); nearestNeighbourTest( string(TEST_DIR "/nearest_neighbour/ssd_500_5000_dim0.test"), 0, AF_SSD); } TYPED_TEST(NearestNeighbour, NN_SSD_500_5000_Dim1) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); nearestNeighbourTest( string(TEST_DIR "/nearest_neighbour/ssd_500_5000_dim1.test"), 1, AF_SSD); @@ -144,24 +153,28 @@ TYPED_TEST(NearestNeighbour, NN_SSD_500_5000_Dim1) { // SAD ///////////////////////////////////////////////// TYPED_TEST(NearestNeighbour, NN_SAD_100_1000_Dim0) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); nearestNeighbourTest( string(TEST_DIR "/nearest_neighbour/sad_100_1000_dim0.test"), 0, AF_SAD); } TYPED_TEST(NearestNeighbour, NN_SAD_100_1000_Dim1) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); nearestNeighbourTest( string(TEST_DIR "/nearest_neighbour/sad_100_1000_dim1.test"), 1, AF_SAD); } TYPED_TEST(NearestNeighbour, NN_SAD_500_5000_Dim0) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); nearestNeighbourTest( string(TEST_DIR "/nearest_neighbour/sad_500_5000_dim0.test"), 0, AF_SAD); } TYPED_TEST(NearestNeighbour, NN_SAD_500_5000_Dim1) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); nearestNeighbourTest( string(TEST_DIR "/nearest_neighbour/sad_500_5000_dim1.test"), 1, AF_SAD); @@ -170,6 +183,7 @@ TYPED_TEST(NearestNeighbour, NN_SAD_500_5000_Dim1) { ///////////////////////////////////// CPP //////////////////////////////// // TEST(NearestNeighbourSSD, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); vector numDims; vector> in; vector> tests; @@ -206,6 +220,7 @@ TEST(NearestNeighbourSSD, CPP) { } TEST(NearestNeighbourSAD, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); vector numDims; vector> in; vector> tests; @@ -242,6 +257,7 @@ TEST(NearestNeighbourSAD, CPP) { } TEST(NearestNeighbourSSD, small) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); const int ntrain = 1; const int nquery = 5; const int nfeat = 2; @@ -272,6 +288,7 @@ TEST(NearestNeighbourSSD, small) { } TEST(KNearestNeighbourSSD, small) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); const int ntrain = 5; const int nquery = 3; const int nfeat = 2; @@ -435,6 +452,7 @@ INSTANTIATE_TEST_SUITE_P(KNearestNeighborsSSD, KNearestNeighborsTest, testNameGenerator); TEST_P(NearestNeighborsTest, SingleQTests) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); nearest_neighbors_params params = GetParam(); array query = array(params.qdims_, params.query_.data()); array train = array(params.tdims_, params.train_.data()); @@ -454,6 +472,7 @@ TEST_P(NearestNeighborsTest, SingleQTests) { } TEST_P(KNearestNeighborsTest, SingleQTests) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); nearest_neighbors_params params = GetParam(); array query = array(params.qdims_, params.query_.data()); @@ -504,6 +523,7 @@ TEST(KNearestNeighbours, InvalidLargeK) { } TEST(NearestNeighbour, DocSnippet1) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); //! [ex_nearest_1] float h_pts[6] = {1.f, 2.f, 3.f, 8.f, 9.f, 10.f}; array pts(dim4(1, 6), h_pts); @@ -537,6 +557,7 @@ TEST(NearestNeighbour, DocSnippet1) { } TEST(NearestNeighbour, DocSnippet2) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); //! [ex_nearest_2] float h_pts[18] = {0.f, 0.f, 0.f, 1.f, 0.f, 0.f, 0.f, 1.f, 0.f, 8.f, 9.f, 1.f, 9.f, 8.f, 1.f, 9.f, 9.f, 1.f}; diff --git a/test/norm.cpp b/test/norm.cpp new file mode 100644 index 0000000000..c795c112c3 --- /dev/null +++ b/test/norm.cpp @@ -0,0 +1,285 @@ +/******************************************************* + * Copyright (c) 2025, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ + +#include +#include +#include +#include + +using af::array; +using af::constant; +using af::dim4; +using std::complex; +using std::stringstream; +using std::vector; + +std::ostream &operator<<(std::ostream &os, af::normType nt) { + switch (nt) { + case AF_NORM_VECTOR_1: os << "AF_NORM_VECTOR_1"; break; + case AF_NORM_VECTOR_INF: os << "AF_NORM_VECTOR_INF"; break; + case AF_NORM_VECTOR_2: os << "AF_NORM_VECTOR_2"; break; + case AF_NORM_VECTOR_P: os << "AF_NORM_VECTOR_P"; break; + case AF_NORM_MATRIX_1: os << "AF_NORM_MATRIX_1"; break; + case AF_NORM_MATRIX_INF: os << "AF_NORM_MATRIX_INF"; break; + case AF_NORM_MATRIX_2: os << "AF_NORM_MATRIX_2"; break; + case AF_NORM_MATRIX_L_PQ: os << "AF_NORM_MATRIX_L_PQ"; break; + } + return os; +} + +template +double cpu_norm1_impl(af::dim4 &dims, std::vector &value) { + int M = dims[0]; + int N = dims[1]; + + double norm1 = std::numeric_limits::lowest(); + for (int n = 0; n < N; n++) { + T *columnN = value.data() + n * M; + double sum = 0; + for (int m = 0; m < M; m++) { sum += abs(columnN[m]); } + norm1 = std::max(norm1, sum); + } + return norm1; +} + +template +double cpu_norm_pq_impl(af::dim4 &dims, std::vector &value, double p, double q) { + int N = dims[0]; + int M = dims[1]; + + double norm = 0; + for (int n = 0; n < N; n++) { + T *columnN = value.data() + n * M; + double sum = 0; + + for (int m = 0; m < M; m++) { sum += std::pow(std::abs(columnN[m]), p); } + + norm += std::pow(sum, q / p); + } + norm = std::pow(norm, 1.0 / q); + + return norm; +} + +double cpu_norm1(af::array &value) { + double norm1; + af::dim4 dims = value.dims(); + if (value.type() == f16) { + vector values(value.elements()); + value.host(values.data()); + norm1 = cpu_norm1_impl(dims, values); + } else if (value.type() == c32 || value.type() == c64) { + vector > values(value.elements()); + value.as(c64).host(values.data()); + norm1 = cpu_norm1_impl >(dims, values); + } else { + vector values(value.elements()); + value.as(f64).host(values.data()); + norm1 = cpu_norm1_impl(dims, values); + } + return norm1; +} + +double cpu_norm_pq(af::array &value, double p, double q) { + double norm2; + af::dim4 dims = value.dims(); + if (value.type() == f16) { + vector values(value.elements()); + value.host(values.data()); + norm2 = cpu_norm_pq_impl(dims, values, p, q); + } else if (value.type() == c32 || value.type() == c64) { + vector > values(value.elements()); + value.as(c64).host(values.data()); + norm2 = cpu_norm_pq_impl >(dims, values, p, q); + } else { + vector values(value.elements()); + value.as(f64).host(values.data()); + norm2 = cpu_norm_pq_impl(dims, values, p, q); + } + return norm2; +} + +template +double cpu_norm_inf_impl(af::dim4 &dims, std::vector &value) { + int M = dims[0]; + int N = dims[1]; + + double norm_inf = std::numeric_limits::lowest(); + for (int m = 0; m < M; m++) { + T *rowM = value.data() + m; + double sum = 0; + for (int n = 0; n < N; n++) { sum += abs(rowM[n * M]); } + norm_inf = std::max(norm_inf, sum); + } + return norm_inf; +} + +double cpu_norm_inf(af::array &value) { + double norm_inf; + af::dim4 dims = value.dims(); + if (value.type() == c32 || value.type() == c64) { + vector > values(value.elements()); + value.as(c64).host(values.data()); + norm_inf = cpu_norm_inf_impl >(dims, values); + } else { + vector values(value.elements()); + value.as(f64).host(values.data()); + norm_inf = cpu_norm_inf_impl(dims, values); + } + return norm_inf; +} + +using norm_params = std::tuple; +class Norm + : public ::testing::TestWithParam > {}; + +INSTANTIATE_TEST_CASE_P( + Norm, Norm, + ::testing::Combine(::testing::Values(dim4(3, 3), dim4(32, 32), dim4(33, 33), + dim4(64, 64), dim4(128, 128), + dim4(129, 129), dim4(256, 256), + dim4(257, 257)), + ::testing::Values(f32, f64, c32, c64, f16)), + [](const ::testing::TestParamInfo info) { + stringstream ss; + using std::get; + ss << "dims_" << get<0>(info.param)[0] << "_" << get<0>(info.param)[1] + << "_dtype_" << get<1>(info.param); + return ss.str(); + }); + +TEST_P(Norm, Identity_AF_NORM_MATRIX_1) { + using std::get; + norm_params param = GetParam(); + if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half); + if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double); + + array identity = af::identity(get<0>(param), get<1>(param)); + double result = norm(identity, AF_NORM_MATRIX_1); + double norm1 = cpu_norm1(identity); + + ASSERT_DOUBLE_EQ(norm1, result); +} + +TEST_P(Norm, Random_AF_NORM_MATRIX_1) { + using std::get; + norm_params param = GetParam(); + if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half); + if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double); + + array in = af::randu(get<0>(param), get<1>(param)) - 0.5f; + double result = norm(in, AF_NORM_MATRIX_1); + double norm1 = cpu_norm1(in); + + ASSERT_NEAR(norm1, result, 2e-4); +} + +TEST_P(Norm, Random_AF_NORM_VECTOR_1) { + using std::get; + norm_params param = GetParam(); + if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half); + if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double); + + af::dim4 dims = get<0>(param); + dims[1] = 1; // Test a vector + + array in = af::randu(dims, get<1>(param)) - 0.5f; + double result = norm(in, AF_NORM_VECTOR_1); + double norm1 = cpu_norm_pq(in, 1, 1); + + ASSERT_NEAR(norm1, result, 2e-4); +} + +TEST_P(Norm, Random_AF_NORM_VECTOR_INF) { + using std::get; + norm_params param = GetParam(); + if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half); + if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double); + + af::dim4 dims = get<0>(param); + dims[1] = 1; // Test a vector + + array in = af::randu(dims, get<1>(param)) - 0.5f; + double result = norm(in, AF_NORM_VECTOR_INF); + double norm_inf = cpu_norm_inf(in); + + ASSERT_NEAR(norm_inf, result, 2e-4); +} + +TEST_P(Norm, Random_AF_NORM_VECTOR_2) { + using std::get; + norm_params param = GetParam(); + if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half); + if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double); + + af::dim4 dims = get<0>(param); + dims[1] = 1; // Test a vector + + array in = af::randu(dims, get<1>(param)) - 0.5f; + double result = norm(in, AF_NORM_VECTOR_2); + double norm2 = cpu_norm_pq(in, 1, 2); // vectors lie in first dims so swap p and q + + ASSERT_NEAR(norm2, result, 3e-4); +} + +TEST_P(Norm, Random_AF_NORM_VECTOR_P_P_EQUAL_3_POINT_5) { + using std::get; + norm_params param = GetParam(); + if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half); + if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double); + + af::dim4 dims = get<0>(param); + dims[1] = 1; // Test a vector + + array in = af::randu(dims, get<1>(param)) - 0.5f; + double result = norm(in, AF_NORM_VECTOR_P, 3.5); + double normp = cpu_norm_pq(in, 1, 3.5); // vectors lie in first dims so swap p and q + + ASSERT_NEAR(normp, result, 3e-4); +} + +TEST_P(Norm, Identity_AF_NORM_MATRIX_2_NOT_SUPPORTED) { + using std::get; + norm_params param = GetParam(); + if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half); + if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double); + try { + double result = + norm(af::identity(get<0>(param), get<1>(param)), AF_NORM_MATRIX_2); + FAIL(); + } catch (af::exception &ex) { + ASSERT_EQ(AF_ERR_NOT_SUPPORTED, ex.err()); + return; + } + FAIL(); +} + +TEST_P(Norm, Identity_AF_NORM_MATRIX_INF) { + using std::get; + norm_params param = GetParam(); + if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half); + if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double); + array in = af::identity(get<0>(param), get<1>(param)); + double result = norm(in, AF_NORM_MATRIX_INF); + double norm_inf = cpu_norm_inf(in); + + ASSERT_DOUBLE_EQ(norm_inf, result); +} + +TEST_P(Norm, Random_AF_NORM_MATRIX_INF) { + using std::get; + norm_params param = GetParam(); + if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half); + if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double); + array in = af::randu(get<0>(param), get<1>(param)); + double result = norm(in, AF_NORM_MATRIX_INF); + double norm_inf = cpu_norm_inf(in); + + ASSERT_NEAR(norm_inf, result, 2e-4); +} diff --git a/test/orb.cpp b/test/orb.cpp index e519fd91dc..3ace1f4b05 100644 --- a/test/orb.cpp +++ b/test/orb.cpp @@ -238,14 +238,19 @@ void orbTest(string pTestFile) { } TYPED_TEST(ORB, Square) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); orbTest(string(TEST_DIR "/orb/square.test")); } -TYPED_TEST(ORB, Lena) { orbTest(string(TEST_DIR "/orb/lena.test")); } +TYPED_TEST(ORB, Lena) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); + orbTest(string(TEST_DIR "/orb/lena.test")); +} ///////////////////////////////////// CPP //////////////////////////////// // TEST(ORB, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); IMAGEIO_ENABLED_CHECK(); vector inDims; diff --git a/test/pad_borders.cpp b/test/pad_borders.cpp index 028c946719..2642ed83ca 100644 --- a/test/pad_borders.cpp +++ b/test/pad_borders.cpp @@ -24,8 +24,8 @@ using std::vector; template class PadBorders : public ::testing::Test {}; -typedef ::testing::Types TestTypes; diff --git a/test/pinverse.cpp b/test/pinverse.cpp index 7258558bc2..13b2151836 100644 --- a/test/pinverse.cpp +++ b/test/pinverse.cpp @@ -124,6 +124,7 @@ TYPED_TEST_SUITE(Pinverse, TestTypes); // Test Moore-Penrose conditions in the following first 4 tests // See https://en.wikipedia.org/wiki/Moore%E2%80%93Penrose_inverse#Definition TYPED_TEST(Pinverse, AApinvA_A) { + SUPPORTED_TYPE_CHECK(TypeParam); array in = readTestInput( string(TEST_DIR "/pinverse/pinverse10x8.test")); array inpinv = pinverse(in); @@ -132,6 +133,7 @@ TYPED_TEST(Pinverse, AApinvA_A) { } TYPED_TEST(Pinverse, ApinvAApinv_Apinv) { + SUPPORTED_TYPE_CHECK(TypeParam); array in = readTestInput( string(TEST_DIR "/pinverse/pinverse10x8.test")); array inpinv = pinverse(in); @@ -140,6 +142,7 @@ TYPED_TEST(Pinverse, ApinvAApinv_Apinv) { } TYPED_TEST(Pinverse, AApinv_IsHermitian) { + SUPPORTED_TYPE_CHECK(TypeParam); array in = readTestInput( string(TEST_DIR "/pinverse/pinverse10x8.test")); array inpinv = pinverse(in); @@ -149,6 +152,7 @@ TYPED_TEST(Pinverse, AApinv_IsHermitian) { } TYPED_TEST(Pinverse, ApinvA_IsHermitian) { + SUPPORTED_TYPE_CHECK(TypeParam); array in = readTestInput( string(TEST_DIR "/pinverse/pinverse10x8.test")); array inpinv = pinverse(in); @@ -158,6 +162,7 @@ TYPED_TEST(Pinverse, ApinvA_IsHermitian) { } TYPED_TEST(Pinverse, Large) { + SUPPORTED_TYPE_CHECK(TypeParam); array in = readTestInput( string(TEST_DIR "/pinverse/pinv_640x480_inputs.test")); array inpinv = pinverse(in); @@ -166,6 +171,7 @@ TYPED_TEST(Pinverse, Large) { } TYPED_TEST(Pinverse, LargeTall) { + SUPPORTED_TYPE_CHECK(TypeParam); array in = readTestInput( string(TEST_DIR "/pinverse/pinv_640x480_inputs.test")) .T(); @@ -227,6 +233,7 @@ TEST(Pinverse, SmallSigValExistsFloat) { } TEST(Pinverse, SmallSigValExistsDouble) { + SUPPORTED_TYPE_CHECK(double); array in = readTestInput(string(TEST_DIR "/pinverse/pinverse10x8.test")); const dim_t dim0 = in.dims(0); diff --git a/test/random.cpp b/test/random.cpp index d0860b70f2..f6fd0dd45f 100644 --- a/test/random.cpp +++ b/test/random.cpp @@ -36,7 +36,7 @@ class Random : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + uintl, signed char, unsigned char, char, af_half> TestTypes; // register the type list @@ -258,15 +258,15 @@ void testSetSeed(const uintl seed0, const uintl seed1) { ASSERT_EQ(h_in0[i], h_in2[i]) << "at : " << i; // Verify different arrays created with different seeds differ - // b8 and u9 can clash because they generate a small set of values - if (ty != b8 && ty != u8) { + // b8, s8 and u8 can clash because they generate a small set of values + if (ty != b8 && ty != s8 && ty != u8) { ASSERT_NE(h_in0[i], h_in1[i]) << "at : " << i; } // Verify different arrays created one after the other with same seed - // differ b8 and u9 can clash because they generate a small set of + // differ b8, s8 and u8 can clash because they generate a small set of // values - if (ty != b8 && ty != u8) { + if (ty != b8 && ty != s8 && ty != u8) { ASSERT_NE(h_in2[i], h_in3[i]) << "at : " << i; } } @@ -394,7 +394,7 @@ void testRandomEngineSeed(randomEngineType type) { for (int i = 0; i < elem; i++) { ASSERT_EQ(h1[i], h3[i]) << "at : " << i; - if (ty != b8 && ty != u8) { + if (ty != b8 && ty != s8 && ty != u8) { ASSERT_NE(h1[i], h2[i]) << "at : " << i; ASSERT_NE(h3[i], h4[i]) << "at : " << i; } diff --git a/test/range.cpp b/test/range.cpp index 35708bde09..0e708160c2 100644 --- a/test/range.cpp +++ b/test/range.cpp @@ -46,12 +46,13 @@ class RangeMax : public Range {}; // create a list of types to be tested typedef ::testing::Types + signed char, unsigned char, short, ushort, + half_float::half> AllTypes; // create a list of types to be tested typedef ::testing::Types + signed char, unsigned char, short, ushort> RegularTypes; // register the type list diff --git a/test/reduce.cpp b/test/reduce.cpp index fc16e60716..c50f95d924 100644 --- a/test/reduce.cpp +++ b/test/reduce.cpp @@ -36,10 +36,14 @@ using std::vector; template class Reduce : public ::testing::Test {}; +template +class ReduceByKey : public ::testing::Test {}; + typedef ::testing::Types + schar, uchar, short, ushort> TestTypes; TYPED_TEST_SUITE(Reduce, TestTypes); +TYPED_TEST_SUITE(ReduceByKey, TestTypes); typedef af_err (*reduceFunc)(af_array *, const af_array, const int); @@ -122,6 +126,10 @@ struct promote_type { // char and uchar are promoted to int for sum and product template<> +struct promote_type { + typedef int type; +}; +template<> struct promote_type { typedef uint type; }; @@ -138,6 +146,10 @@ struct promote_type { typedef uint type; }; template<> +struct promote_type { + typedef int type; +}; +template<> struct promote_type { typedef uint type; }; @@ -154,6 +166,16 @@ struct promote_type { typedef uint type; }; +// float16 is promoted to float32 for sum and product +template<> +struct promote_type { + typedef float type; +}; +template<> +struct promote_type { + typedef float type; +}; + #define REDUCE_TESTS(FN) \ TYPED_TEST(Reduce, Test_##FN) { \ reduceTest::type, \ @@ -375,6 +397,7 @@ array ptrToArray(size_t size, void *ptr, af_dtype type) { case u16: res = array(size, (unsigned short *)ptr); break; case s16: res = array(size, (short *)ptr); break; case b8: res = array(size, (char *)ptr); break; + case s8: res = array(size, (signed char *)ptr); break; case u8: res = array(size, (unsigned char *)ptr); break; case f16: res = array(size, (half_float::half *)ptr); break; } @@ -395,6 +418,7 @@ array ptrToArray(af::dim4 size, void *ptr, af_dtype type) { case u16: res = array(size, (unsigned short *)ptr); break; case s16: res = array(size, (short *)ptr); break; case b8: res = array(size, (char *)ptr); break; + case s8: res = array(size, (signed char *)ptr); break; case u8: res = array(size, (unsigned char *)ptr); break; case f16: res = array(size, (half_float::half *)ptr); break; } @@ -408,7 +432,12 @@ class ReduceByKeyP : public ::testing::TestWithParam { void SetUp() { reduce_by_key_params *params = GetParam(); - if (noHalfTests(params->vType_)) { return; } + if (noHalfTests(params->vType_)) { + GTEST_SKIP() << "Half not supported on this device"; + } + if (noDoubleTests(GetParam()->vType_)) { + GTEST_SKIP() << "Double not supported on this device"; + } keys = ptrToArray(params->iSize, params->iKeys_, params->kType_); vals = ptrToArray(params->iSize, params->iVals_, params->vType_); @@ -425,7 +454,7 @@ template struct generateConsq { T vals; - generateConsq(T v_i = 0) : vals(v_i){}; + generateConsq(T v_i = 0) : vals(v_i) {}; T operator()() { return vals++; } }; @@ -434,7 +463,7 @@ template struct generateConst { T vals; - generateConst(T v_i) : vals(v_i){}; + generateConst(T v_i) : vals(v_i) {}; T operator()() { return vals; } }; @@ -551,7 +580,15 @@ INSTANTIATE_TEST_SUITE_P(UniqueKeyTests, ReduceByKeyP, testNameGenerator); TEST_P(ReduceByKeyP, SumDim0) { - if (noHalfTests(GetParam()->vType_)) { return; } + if (noHalfTests(GetParam()->vType_)) { + GTEST_SKIP() << "Half not supported on this device"; + } + if (noHalfTests(GetParam()->kType_)) { + GTEST_SKIP() << "Half not supported on this device"; + } + if (noDoubleTests(GetParam()->vType_)) { + GTEST_SKIP() << "Double not supported on this device"; + } array keyRes, valsReduced; sumByKey(keyRes, valsReduced, keys, vals, 0, 0); @@ -560,7 +597,15 @@ TEST_P(ReduceByKeyP, SumDim0) { } TEST_P(ReduceByKeyP, SumDim2) { - if (noHalfTests(GetParam()->vType_)) { return; } + if (noHalfTests(GetParam()->vType_)) { + GTEST_SKIP() << "Half not supported on this device"; + } + if (noHalfTests(GetParam()->kType_)) { + GTEST_SKIP() << "Half not supported on this device"; + } + if (noDoubleTests(GetParam()->vType_)) { + GTEST_SKIP() << "Double not supported on this device"; + } const int ntile = 2; vals = tile(vals, 1, ntile, 1, 1); vals = reorder(vals, 1, 2, 0, 3); @@ -577,12 +622,16 @@ TEST_P(ReduceByKeyP, SumDim2) { ASSERT_ARRAYS_NEAR(valsReducedGold, valsReduced, 1e-5); } -TEST(ReduceByKey, MultiBlockReduceSingleval) { +TYPED_TEST(ReduceByKey, MultiBlockReduceSingleval) { + SUPPORTED_TYPE_CHECK(TypeParam); array keys = constant(0, 1024 * 1024, s32); - array vals = constant(1, 1024 * 1024, f32); + array vals = constant(1, 1024 * 1024, + (af_dtype)af::dtype_traits::af_type); array keyResGold = constant(0, 1); - array valsReducedGold = constant(1024 * 1024, 1, f32); + using promoted_t = typename promote_type::type; + array valsReducedGold = constant( + 1024 * 1024, 1, (af_dtype)af::dtype_traits::af_type); array keyRes, valsReduced; sumByKey(keyRes, valsReduced, keys, vals); @@ -680,10 +729,11 @@ TEST(ReduceByKey, MultiBlockReduceByKeyRandom500) { reduce_by_key_test(string(TEST_DIR "/reduce/test_random500_by_key.test")); } -TEST(ReduceByKey, productReduceByKey) { - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; - const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; +TYPED_TEST(ReduceByKey, productReduceByKey) { + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; + const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -692,15 +742,17 @@ TEST(ReduceByKey, productReduceByKey) { productByKey(reduced_keys, reduced_vals, keys, vals, 0, 1); const int goldSz = 5; - const vector gold_reduce{0, 7, 6, 30, 4}; + using promoted_t = typename promote_type::type; + const vector gold_reduce{0, 7, 6, 30, 4}; ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals); } -TEST(ReduceByKey, minReduceByKey) { - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; - const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; +TYPED_TEST(ReduceByKey, minReduceByKey) { + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; + const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -709,14 +761,15 @@ TEST(ReduceByKey, minReduceByKey) { minByKey(reduced_keys, reduced_vals, keys, vals); const int goldSz = 5; - const vector gold_reduce{0, 1, 6, 2, 4}; + const vector gold_reduce{0, 1, 6, 2, 4}; ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals); } -TEST(ReduceByKey, maxReduceByKey) { - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; - const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; +TYPED_TEST(ReduceByKey, maxReduceByKey) { + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; + const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -725,14 +778,15 @@ TEST(ReduceByKey, maxReduceByKey) { maxByKey(reduced_keys, reduced_vals, keys, vals); const int goldSz = 5; - const vector gold_reduce{0, 7, 6, 5, 4}; + const vector gold_reduce{0, 7, 6, 5, 4}; ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals); } -TEST(ReduceByKey, allTrueReduceByKey) { - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; - const float testVals[testSz] = {0, 1, 1, 1, 0, 1, 1, 1}; +TYPED_TEST(ReduceByKey, allTrueReduceByKey) { + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; + const TypeParam testVals[testSz] = {0, 1, 1, 1, 0, 1, 1, 1}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -745,10 +799,11 @@ TEST(ReduceByKey, allTrueReduceByKey) { ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals); } -TEST(ReduceByKey, anyTrueReduceByKey) { - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 8, 8}; - const float testVals[testSz] = {0, 1, 1, 1, 0, 1, 0, 0}; +TYPED_TEST(ReduceByKey, anyTrueReduceByKey) { + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 8, 8}; + const TypeParam testVals[testSz] = {0, 1, 1, 1, 0, 1, 0, 0}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -762,10 +817,11 @@ TEST(ReduceByKey, anyTrueReduceByKey) { ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals); } -TEST(ReduceByKey, countReduceByKey) { - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 5}; - const float testVals[testSz] = {0, 1, 1, 1, 0, 1, 1, 1}; +TYPED_TEST(ReduceByKey, countReduceByKey) { + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 5}; + const TypeParam testVals[testSz] = {0, 1, 1, 1, 0, 1, 1, 1}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -778,11 +834,18 @@ TEST(ReduceByKey, countReduceByKey) { ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals); } -TEST(ReduceByKey, ReduceByKeyNans) { +TYPED_TEST(ReduceByKey, ReduceByKeyNans) { + if (!IsFloatingPoint::value) { + SUCCEED() << "Not a floating point type."; + return; + } + SKIP_IF_FAST_MATH_ENABLED(); - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; - const float testVals[testSz] = {0, 7, NAN, 6, 2, 5, 3, 4}; + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; + const TypeParam nan = std::numeric_limits::quiet_NaN(); + const TypeParam testVals[testSz] = {0, 7, nan, 6, 2, 5, 3, 4}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -791,14 +854,16 @@ TEST(ReduceByKey, ReduceByKeyNans) { productByKey(reduced_keys, reduced_vals, keys, vals, 0, 1); const int goldSz = 5; - const vector gold_reduce{0, 7, 6, 30, 4}; + using promoted_t = typename promote_type::type; + const vector gold_reduce{0, 7, 6, 30, 4}; ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals); } -TEST(ReduceByKey, nDim0ReduceByKey) { - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; - const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; +TYPED_TEST(ReduceByKey, nDim0ReduceByKey) { + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; + const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -812,20 +877,22 @@ TEST(ReduceByKey, nDim0ReduceByKey) { sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval); const dim4 goldSz(5, 2, 2, 2); - const vector gold_reduce{0, 8, 6, 10, 4, 0, 8, 6, 10, 4, + using promoted_t = typename promote_type::type; + const vector gold_reduce{0, 8, 6, 10, 4, 0, 8, 6, 10, 4, - 0, 8, 6, 10, 4, 0, 8, 6, 10, 4, + 0, 8, 6, 10, 4, 0, 8, 6, 10, 4, - 0, 8, 6, 10, 4, 0, 8, 6, 10, 4, + 0, 8, 6, 10, 4, 0, 8, 6, 10, 4, - 0, 8, 6, 10, 4, 0, 8, 6, 10, 4}; + 0, 8, 6, 10, 4, 0, 8, 6, 10, 4}; ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals); } -TEST(ReduceByKey, nDim1ReduceByKey) { - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; - const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; +TYPED_TEST(ReduceByKey, nDim1ReduceByKey) { + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; + const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -839,9 +906,10 @@ TEST(ReduceByKey, nDim1ReduceByKey) { const double nanval = 0.0; sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval); - const int goldSz = 5; - const float gold_reduce[goldSz] = {0, 8, 6, 10, 4}; - vector hreduce(reduced_vals.elements()); + const int goldSz = 5; + using promoted_t = typename promote_type::type; + const promoted_t gold_reduce[goldSz] = {0, 8, 6, 10, 4}; + vector hreduce(reduced_vals.elements()); reduced_vals.host(hreduce.data()); for (int i = 0; i < goldSz * ntile; i++) { @@ -849,10 +917,11 @@ TEST(ReduceByKey, nDim1ReduceByKey) { } } -TEST(ReduceByKey, nDim2ReduceByKey) { - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; - const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; +TYPED_TEST(ReduceByKey, nDim2ReduceByKey) { + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; + const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -866,9 +935,10 @@ TEST(ReduceByKey, nDim2ReduceByKey) { const double nanval = 0.0; sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval); - const int goldSz = 5; - const float gold_reduce[goldSz] = {0, 8, 6, 10, 4}; - vector h_a(reduced_vals.elements()); + const int goldSz = 5; + using promoted_t = typename promote_type::type; + const promoted_t gold_reduce[goldSz] = {0, 8, 6, 10, 4}; + vector h_a(reduced_vals.elements()); reduced_vals.host(h_a.data()); for (int i = 0; i < goldSz * ntile; i++) { @@ -876,10 +946,11 @@ TEST(ReduceByKey, nDim2ReduceByKey) { } } -TEST(ReduceByKey, nDim3ReduceByKey) { - const static int testSz = 8; - const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; - const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; +TYPED_TEST(ReduceByKey, nDim3ReduceByKey) { + SUPPORTED_TYPE_CHECK(TypeParam); + const static int testSz = 8; + const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8}; + const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4}; array keys(testSz, testKeys); array vals(testSz, testVals); @@ -893,9 +964,10 @@ TEST(ReduceByKey, nDim3ReduceByKey) { const double nanval = 0.0; sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval); - const int goldSz = 5; - const float gold_reduce[goldSz] = {0, 8, 6, 10, 4}; - vector h_a(reduced_vals.elements()); + const int goldSz = 5; + using promoted_t = typename promote_type::type; + const promoted_t gold_reduce[goldSz] = {0, 8, 6, 10, 4}; + vector h_a(reduced_vals.elements()); reduced_vals.host(h_a.data()); for (int i = 0; i < goldSz * ntile; i++) { @@ -1946,7 +2018,12 @@ class RaggedReduceMaxRangeP : public ::testing::TestWithParam { void SetUp() { ragged_params *params = GetParam(); - if (noHalfTests(params->vType_)) { return; } + if (noHalfTests(params->vType_)) { + GTEST_SKIP() << "Half not supported on this device"; + } + if (noDoubleTests(GetParam()->vType_)) { + GTEST_SKIP() << "Double not supported on this device"; + } const size_t rdim_size = params->reduceDimLen_; const int dim = params->reduceDim_; @@ -2043,8 +2120,9 @@ INSTANTIATE_TEST_SUITE_P(RaggedReduceTests, RaggedReduceMaxRangeP, testNameGeneratorRagged); TEST_P(RaggedReduceMaxRangeP, rangeMaxTest) { - if (noHalfTests(GetParam()->vType_)) { return; } - + if (noHalfTests(GetParam()->vType_)) { + GTEST_SKIP() << "Half not supported on this device"; + } array ragged_max, idx; const int dim = GetParam()->reduceDim_; max(ragged_max, idx, vals, ragged_lens, dim); @@ -2303,25 +2381,27 @@ TEST(Reduce, Test_Sum_Global_Array_nanval) { TEST(Reduce, nanval_issue_3255) { SKIP_IF_FAST_MATH_ENABLED(); + SUPPORTED_TYPE_CHECK(double); char *info_str; af_array ikeys, ivals, okeys, ovals; dim_t dims[1] = {8}; int ikeys_src[8] = {0, 0, 1, 1, 1, 2, 2, 0}; - af_create_array(&ikeys, ikeys_src, 1, dims, u32); + ASSERT_SUCCESS(af_create_array(&ikeys, ikeys_src, 1, dims, u32)); int i; for (i = 0; i < 8; i++) { double ivals_src[8] = {1, 2, 3, 4, 5, 6, 7, 8}; ivals_src[i] = NAN; - af_create_array(&ivals, ivals_src, 1, dims, f64); + ASSERT_SUCCESS(af_create_array(&ivals, ivals_src, 1, dims, f64)); - af_product_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0); + ASSERT_SUCCESS( + af_product_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0)); af::array ovals_cpp(ovals); ASSERT_FALSE(af::anyTrue(af::isNaN(ovals_cpp))); ASSERT_SUCCESS(af_release_array(okeys)); - af_sum_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0); + ASSERT_SUCCESS(af_sum_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0)); ovals_cpp = af::array(ovals); ASSERT_FALSE(af::anyTrue(af::isNaN(ovals_cpp))); @@ -2340,7 +2420,7 @@ TEST(Reduce, SNIPPET_algorithm_func_sum) { // 1, 3, 5] // Create b by summing across the first dimension - array b = sum(a); // sum across the first dimension, same as sum(a, 0) + array b = sum(a); // sum across the first dimension, same as sum(a,0) // Create c by summing across the second dimension array c = sum(a, 1); // sum across the second dimension @@ -2368,3 +2448,90 @@ TEST(Reduce, SNIPPET_algorithm_func_sum) { ASSERT_VEC_ARRAY_EQ(gold_a, d.dims(), d); ASSERT_VEC_ARRAY_EQ(gold_a, e.dims(), e); } + +#define TEMP_FORMAT_TESTS_reduce(form, op) \ + TEST(TEMP_FORMAT, form##_##op##_array) { \ + const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f}); \ + const array gold = op(in, 3); \ + array out = op(toTempFormat(form, in), 3); \ + EXPECT_ARRAYS_EQ(out, gold); \ + } \ + TEST(TEMP_FORMAT, form##_##op##_value) { \ + const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f}); \ + const float gold = op(in); \ + float out = op(toTempFormat(form, in)); \ + EXPECT_EQ(out, gold); \ + } + +#define TEMP_FORMAT_TESTS_ragged(form, op) \ + TEST(TEMP_FORMAT, form##_##op##_ragged) { \ + const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f}); \ + const array ragged_len(dim4(1), {(unsigned)in.elements()}); \ + array gold_vals, gold_idxs; \ + op(gold_vals, gold_idxs, in, ragged_len, 3); \ + array vals, idxs; \ + op(vals, idxs, toTempFormat(form, in), toTempFormat(form, ragged_len), \ + 3); \ + EXPECT_ARRAYS_EQ(vals, gold_vals); \ + EXPECT_ARRAYS_EQ(idxs, gold_idxs); \ + } + +#define TEMP_FORMAT_TESTS_ByKey(form, op) \ + TEST(TEMP_FORMAT, form##_##op) { \ + const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f}); \ + const array keys(constant(0, in.dims().dims[3], u32)); \ + keys.eval(); \ + array gold_keys, gold_vals; \ + op(gold_keys, gold_vals, keys, in, 3); \ + array out_keys, out_vals; \ + op(out_keys, out_vals, toTempFormat(form, keys), \ + toTempFormat(form, in), 3); \ + EXPECT_ARRAYS_EQ(gold_vals, out_vals); \ + EXPECT_ARRAYS_EQ(gold_keys, out_keys); \ + } + +#define TEMP_FORMAT_TESTS_allTest(form, op) \ + TEST(TEMP_FORMAT, form##_##op##_array) { \ + const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f}); \ + const array gold = op(in > 2.0, 3); \ + array out = op(toTempFormat(form, in) > 2.0, 3); \ + EXPECT_ARRAYS_EQ(gold, out); \ + } \ + TEST(TEMP_FORMAT, form##_##op##_value) { \ + const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f}); \ + const float gold = op(in > 2.0); \ + float out = op(toTempFormat(form, in) > 2.0); \ + EXPECT_EQ(gold, out); \ + } + +#define TEMP_FORMAT_TESTS_allTestByKey(form, op) \ + TEST(TEMP_FORMAT, form##_##op) { \ + const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f}); \ + const array keys(constant(0, in.dims().dims[3], u32)); \ + array gold_vals, gold_keys; \ + op(gold_keys, gold_vals, keys, in > 2.0, 3); \ + array out_vals, out_keys; \ + op(out_keys, out_vals, toTempFormat(form, keys), \ + toTempFormat(form, in) > 2.0, 3); \ + EXPECT_ARRAYS_EQ(gold_vals, out_vals); \ + EXPECT_ARRAYS_EQ(gold_keys, out_keys); \ + } + +#define TEMP_FORMATS_TESTS(form) \ + TEMP_FORMAT_TESTS_reduce(form, min); \ + TEMP_FORMAT_TESTS_reduce(form, max); \ + TEMP_FORMAT_TESTS_reduce(form, sum); \ + TEMP_FORMAT_TESTS_reduce(form, product); \ + TEMP_FORMAT_TESTS_reduce(form, count); \ + TEMP_FORMAT_TESTS_ragged(form, max); \ + TEMP_FORMAT_TESTS_ByKey(form, minByKey); \ + TEMP_FORMAT_TESTS_ByKey(form, maxByKey); \ + TEMP_FORMAT_TESTS_ByKey(form, sumByKey); \ + TEMP_FORMAT_TESTS_ByKey(form, productByKey); \ + TEMP_FORMAT_TESTS_ByKey(form, countByKey); \ + TEMP_FORMAT_TESTS_allTest(form, allTrue); \ + TEMP_FORMAT_TESTS_allTest(form, anyTrue); \ + TEMP_FORMAT_TESTS_allTestByKey(form, allTrueByKey); \ + TEMP_FORMAT_TESTS_allTestByKey(form, anyTrueByKey); + +FOREACH_TEMP_FORMAT(TEMP_FORMATS_TESTS) diff --git a/test/regions.cpp b/test/regions.cpp index 182a22e9b5..a6f14ede81 100644 --- a/test/regions.cpp +++ b/test/regions.cpp @@ -71,7 +71,7 @@ void regionsTest(string pTestFile, af_connectivity connectivity, } ASSERT_SUCCESS(af_regions(&outArray, inArray, connectivity, - (af_dtype)dtype_traits::af_type)); + (af_dtype)dtype_traits::af_type)); // Get result T* outData = new T[idims.elements()]; @@ -97,6 +97,7 @@ void regionsTest(string pTestFile, af_connectivity connectivity, #define REGIONS_INIT(desc, file, conn, conn_type) \ TYPED_TEST(Regions, desc) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ regionsTest( \ string(TEST_DIR "/regions/" #file "_" #conn ".test"), conn_type); \ } @@ -109,6 +110,7 @@ REGIONS_INIT(Regions3, regions_128x128, 8, AF_CONNECTIVITY_8); ///////////////////////////////////// CPP //////////////////////////////// // TEST(Regions, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); vector numDims; vector> in; vector> tests; @@ -139,6 +141,7 @@ TEST(Regions, CPP) { ///////////////////////////////// Documentation Examples /////////////////// TEST(Regions, Docs_8) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); // input data uchar input[64] = {0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, @@ -185,6 +188,7 @@ TEST(Regions, Docs_8) { } TEST(Regions, Docs_4) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); // input data uchar input[64] = {0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, @@ -236,6 +240,7 @@ TEST(Regions, Docs_4) { } TEST(Regions, WholeImageComponent) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); const int dim = 101; const int sz = dim * dim; vector input(sz, 1); @@ -252,6 +257,7 @@ TEST(Regions, WholeImageComponent) { } TEST(Regions, NoComponentImage) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); const int dim = 101; const int sz = dim * dim; vector input(sz, 0); diff --git a/test/reorder.cpp b/test/reorder.cpp index b06f72cdda..3109839786 100644 --- a/test/reorder.cpp +++ b/test/reorder.cpp @@ -44,7 +44,7 @@ class Reorder : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + char, signed char, unsigned char, short, ushort> TestTypes; // register the type list diff --git a/test/replace.cpp b/test/replace.cpp index 6d72cf7fc9..1156731732 100644 --- a/test/replace.cpp +++ b/test/replace.cpp @@ -35,7 +35,7 @@ template class Replace : public ::testing::Test {}; typedef ::testing::Types + int, intl, uintl, schar, uchar, char, short, ushort> TestTypes; TYPED_TEST_SUITE(Replace, TestTypes); diff --git a/test/resize.cpp b/test/resize.cpp index 423bb55416..50c46730f9 100644 --- a/test/resize.cpp +++ b/test/resize.cpp @@ -55,8 +55,8 @@ class ResizeI : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types TestTypesF; -typedef ::testing::Types +typedef ::testing::Types TestTypesI; // register the type list diff --git a/test/rotate.cpp b/test/rotate.cpp index 01675fa1d7..986398f88f 100644 --- a/test/rotate.cpp +++ b/test/rotate.cpp @@ -34,7 +34,8 @@ class Rotate : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list diff --git a/test/rotate_linear.cpp b/test/rotate_linear.cpp index ea19f217e7..1324a59a77 100644 --- a/test/rotate_linear.cpp +++ b/test/rotate_linear.cpp @@ -39,7 +39,8 @@ class RotateLinear : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list @@ -53,6 +54,10 @@ void rotateTest(string pTestFile, const unsigned resultIdx, const float angle, const vector* seqv = NULL) { SUPPORTED_TYPE_CHECK(T); + if (is_same_type::value && (int)angle % 90 != 0) { + GTEST_SKIP() << "Incompatible test data for s8"; + } + vector numDims; vector> in; vector> tests; diff --git a/test/sat.cpp b/test/sat.cpp index 892e2f8f4e..f87b356b85 100644 --- a/test/sat.cpp +++ b/test/sat.cpp @@ -31,8 +31,8 @@ class SAT : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list diff --git a/test/scan.cpp b/test/scan.cpp index a29c6e0e52..afb488278d 100644 --- a/test/scan.cpp +++ b/test/scan.cpp @@ -113,6 +113,7 @@ SCAN_TESTS(accum, cdouble, cdouble, cdouble); SCAN_TESTS(accum, unsigned, unsigned, unsigned); SCAN_TESTS(accum, intl, intl, intl); SCAN_TESTS(accum, uintl, uintl, uintl); +SCAN_TESTS(accum, schar, schar, int); SCAN_TESTS(accum, uchar, uchar, unsigned); SCAN_TESTS(accum, short, short, int); SCAN_TESTS(accum, ushort, ushort, uint); @@ -345,3 +346,22 @@ TEST(Scan, ExclusiveSum2D_Dim3) { ASSERT_ARRAYS_EQ(gold, out); } + +#define TEST_TEMP_FORMAT(form, dim) \ + TEST(TEMP_FORMAT, form##_Dim##dim) { \ + const dim4 dims(2, 2, 2, 2); \ + const array in(af::moddims(range(dim4(dims.elements())), dims)); \ + in.eval(); \ + const array gold = scan(in, dim); \ + \ + array out = scan(toTempFormat(form, in), dim); \ + ASSERT_ARRAYS_EQ(gold, out); \ + } + +#define TEST_TEMP_FORMATS(form) \ + TEST_TEMP_FORMAT(form, 0) \ + TEST_TEMP_FORMAT(form, 1) \ + TEST_TEMP_FORMAT(form, 2) \ + TEST_TEMP_FORMAT(form, 3) + +FOREACH_TEMP_FORMAT(TEST_TEMP_FORMATS) \ No newline at end of file diff --git a/test/scan_by_key.cpp b/test/scan_by_key.cpp index fe4d61d095..08928b5fdc 100644 --- a/test/scan_by_key.cpp +++ b/test/scan_by_key.cpp @@ -127,6 +127,7 @@ void scanByKeyTest(dim4 dims, int scanDim, vector nodeLengths, #define SCAN_BY_KEY_TEST(FN, X, Y, Z, W, Ti, To, INC, DIM, DSTART, DEND, EPS) \ TEST(ScanByKey, Test_Scan_By_Key_##FN##_##Ti##_##INC##_##DIM) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ dim4 dims(X, Y, Z, W); \ int scanDim = DIM; \ int nodel[] = {37, 256}; \ @@ -194,6 +195,7 @@ SCAN_BY_KEY_TEST(AF_BINARY_MAX, 4 * 1024, 512, 1, 1, float, float, false, 1, -5, 5, 1e-3); TEST(ScanByKey, Test_Scan_By_key_Simple_0) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); dim4 dims(16, 8, 2, 1); int scanDim = 0; int nodel[] = {4, 8}; @@ -207,6 +209,7 @@ TEST(ScanByKey, Test_Scan_By_key_Simple_0) { } TEST(ScanByKey, Test_Scan_By_key_Simple_1) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); dim4 dims(8, 256 + 128, 1, 1); int scanDim = 1; int nodel[] = {4, 8}; @@ -220,6 +223,7 @@ TEST(ScanByKey, Test_Scan_By_key_Simple_1) { } TEST(ScanByKey, FixOverflowWrite) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); const int SIZE = 41000; vector keys(SIZE, 0); vector vals(SIZE, 1.0f); @@ -236,3 +240,26 @@ TEST(ScanByKey, FixOverflowWrite) { ASSERT_EQ(prior, valsAF(0).scalar()); } + +#define TEST_TEMP_FORMAT(form, dim) \ + TEST(TEMP_FORMAT, form##_Dim##dim) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ + const dim4 dims(2, 2, 2, 2); \ + const array in(af::moddims(range(dim4(dims.elements())), dims)); \ + in.eval(); \ + const array keys(af::constant(0, dims, u32)); \ + keys.eval(); \ + const array gold = scanByKey(keys, in, dim); \ + \ + array out = \ + scanByKey(toTempFormat(form, keys), toTempFormat(form, in), dim); \ + ASSERT_ARRAYS_EQ(gold, out); \ + } + +#define TEST_TEMP_FORMATS(form) \ + TEST_TEMP_FORMAT(form, 0) \ + TEST_TEMP_FORMAT(form, 1) \ + TEST_TEMP_FORMAT(form, 2) \ + TEST_TEMP_FORMAT(form, 3) + +FOREACH_TEMP_FORMAT(TEST_TEMP_FORMATS) \ No newline at end of file diff --git a/test/select.cpp b/test/select.cpp index 0b6724d8fa..4b4c96dd21 100644 --- a/test/select.cpp +++ b/test/select.cpp @@ -42,7 +42,7 @@ template class Select : public ::testing::Test {}; typedef ::testing::Types + schar, uchar, char, short, ushort, half_float::half> TestTypes; TYPED_TEST_SUITE(Select, TestTypes); diff --git a/test/set.cpp b/test/set.cpp index 97e05d484b..0e1ececadc 100644 --- a/test/set.cpp +++ b/test/set.cpp @@ -77,6 +77,7 @@ UNIQUE_TESTS(float) UNIQUE_TESTS(double) UNIQUE_TESTS(int) UNIQUE_TESTS(uint) +UNIQUE_TESTS(schar) UNIQUE_TESTS(uchar) UNIQUE_TESTS(short) UNIQUE_TESTS(ushort) @@ -149,6 +150,7 @@ SET_TESTS(float) SET_TESTS(double) SET_TESTS(int) SET_TESTS(uint) +SET_TESTS(schar) SET_TESTS(uchar) SET_TESTS(short) SET_TESTS(ushort) diff --git a/test/shift.cpp b/test/shift.cpp index b37385a6f8..c86c43c8e3 100644 --- a/test/shift.cpp +++ b/test/shift.cpp @@ -42,7 +42,8 @@ class Shift : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + intl, uintl, char, signed char, unsigned char, short, + ushort> TestTypes; // register the type list TYPED_TEST_SUITE(Shift, TestTypes); @@ -146,3 +147,12 @@ TEST(Shift, MaxDim) { output = abs(input - output); ASSERT_EQ(1.f, product(output)); } + +TEST(Shift, RowVector) { + const unsigned shift_x = 1; + const unsigned shift_y = 1; + array input = iota(dim4(1, 4)); + array output = shift(input, shift_x, shift_y); + vector gold{3.f, 0.f, 1.f, 2.f}; + EXPECT_VEC_ARRAY_EQ(gold, dim4(1, 4), output); +} diff --git a/test/sift.cpp b/test/sift.cpp index 621659e259..b96325d672 100644 --- a/test/sift.cpp +++ b/test/sift.cpp @@ -162,9 +162,9 @@ void siftTest(string pTestFile, unsigned nLayers, float contrastThr, af_load_image(&inArray_f32, inFiles[testId].c_str(), false)); ASSERT_SUCCESS(conv_image(&inArray, inArray_f32)); - ASSERT_SUCCESS(af_sift(&feat, &desc, inArray, nLayers, contrastThr, - edgeThr, initSigma, doubleInput, 1.f / 256.f, - 0.05f)); + ASSERT_SUCCESS(af_sift(&feat, &desc, inArray, nLayers, + contrastThr, edgeThr, initSigma, + doubleInput, 1.f / 256.f, 0.05f)); dim_t n = 0; af_array x, y, score, orientation, size; @@ -256,6 +256,7 @@ void siftTest(string pTestFile, unsigned nLayers, float contrastThr, #define SIFT_INIT(desc, image, nLayers, contrastThr, edgeThr, initSigma, \ doubleInput) \ TYPED_TEST(SIFT, desc) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ for (int i = 0; i < 1; i++) \ siftTest(string(TEST_DIR "/sift/" #image ".test"), \ nLayers, contrastThr, edgeThr, initSigma, \ @@ -272,6 +273,7 @@ SIFT_INIT(Man_NoDoubleInput, man_nodoubleinput, 3, 0.04f, 10.0f, 1.6f, false); ///////////////////////////////////// CPP //////////////////////////////// // TEST(SIFT, CPP) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); IMAGEIO_ENABLED_CHECK(); vector inDims; diff --git a/test/sobel.cpp b/test/sobel.cpp index 298d36d299..72a70ddde3 100644 --- a/test/sobel.cpp +++ b/test/sobel.cpp @@ -35,7 +35,8 @@ class Sobel_Integer : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types TestTypes; -typedef ::testing::Types +typedef ::testing::Types TestTypesInt; // register the type list @@ -79,11 +80,13 @@ void testSobelDerivatives(string pTestFile) { // border type is set to cv.BORDER_REFLECT_101 in opencv TYPED_TEST(Sobel, Rectangle) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); testSobelDerivatives( string(TEST_DIR "/sobel/rectangle.test")); } TYPED_TEST(Sobel_Integer, Rectangle) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); testSobelDerivatives( string(TEST_DIR "/sobel/rectangle.test")); } diff --git a/test/sort.cpp b/test/sort.cpp index c9da609f93..bd60edb5b5 100644 --- a/test/sort.cpp +++ b/test/sort.cpp @@ -40,8 +40,8 @@ class Sort : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list diff --git a/test/sort_by_key.cpp b/test/sort_by_key.cpp index afd7908660..265ee570b7 100644 --- a/test/sort_by_key.cpp +++ b/test/sort_by_key.cpp @@ -40,8 +40,8 @@ class SortByKey : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list diff --git a/test/sort_index.cpp b/test/sort_index.cpp index f3a10b9084..5e1b88a97d 100644 --- a/test/sort_index.cpp +++ b/test/sort_index.cpp @@ -40,8 +40,8 @@ class SortIndex : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list diff --git a/test/sparse.cpp b/test/sparse.cpp index a130a6bb58..f1e1b67d72 100644 --- a/test/sparse.cpp +++ b/test/sparse.cpp @@ -19,6 +19,7 @@ using af::dtype_traits; using af::identity; using af::randu; using af::span; +using af::seq; #define SPARSE_TESTS(T, eps) \ TEST(Sparse, T##Square) { sparseTester(1000, 1000, 100, 5, eps); } \ @@ -109,6 +110,42 @@ TEST(Sparse, ISSUE_1745) { row_idx.get(), col_idx.get(), AF_STORAGE_CSR)); } +TEST(Sparse, offsets_work_csr_to_dense_ISSUE_1918) { + array reference(2,2); + reference(0, span) = 0; + reference(1, span) = 2; + float value[] = { 1, 1, 2, 2 }; + int row_csr[] = { 0, 2, 2, 0, 0, 2 }; + int col[] = { 0, 1, 0, 1 }; + array values(4, 1, value, afHost); + array rows_csr(6, 1, row_csr, afHost); + array cols(4, 1, col, afHost); + array S_csr; + + S_csr = sparse(2, 2, values(seq(2, 3)), rows_csr(seq(3, 5)), cols(seq(2, 3))); + array output_csr = dense(S_csr); + + EXPECT_ARRAYS_EQ(reference, output_csr); +} + +TEST(Sparse, offsets_work_coo_to_dense_ISSUE_1918) { + array reference(2,2); + reference(0, span) = 0; + reference(1, span) = 2; + float value[] = { 1, 1, 2, 2 }; + int row_coo[] = { 0, 0, 1, 1 }; + int col[] = { 0, 1, 0, 1 }; + array values(4, 1, value, afHost); + array rows_coo(4, 1, row_coo, afHost); + array cols(4, 1, col, afHost); + array S_coo; + + S_coo = sparse(2, 2, values(seq(2, 3)), rows_coo(seq(2, 3)), cols(seq(2, 3)), AF_STORAGE_COO); + array output_coo = dense(S_coo); + + EXPECT_ARRAYS_EQ(reference, output_coo); +} + TEST(Sparse, ISSUE_2134_COO) { int rows[] = {0, 0, 0, 1, 1, 2, 2}; int cols[] = {0, 1, 2, 0, 1, 0, 2}; @@ -416,3 +453,24 @@ TEST(Sparse, CPPDenseToSparseToDenseUsage) { ASSERT_ARRAYS_EQ(in, gold); ASSERT_ARRAYS_EQ(dense, gold); } + +TEST(Sparse, CPPDenseToSparseConversions) { + array in = af::randu(200, 200); + in(in < 0.75) = 0; + + array coo_sparse_arr = af::sparse(in, AF_STORAGE_COO); + array csr_sparse_arr = af::sparse(in, AF_STORAGE_CSR); + + array coo_dense_arr = af::dense(coo_sparse_arr); + array csr_dense_arr = af::dense(csr_sparse_arr); + + ASSERT_ARRAYS_EQ(in, coo_dense_arr); + ASSERT_ARRAYS_EQ(in, csr_dense_arr); + + array non_zero = af::flat(in)(af::where(in)); + array non_zero_T = af::flat(in.T())(af::where(in.T())); + ASSERT_ARRAYS_EQ(non_zero, af::sparseGetValues(coo_sparse_arr)); + ASSERT_ARRAYS_EQ( + non_zero_T, + af::sparseGetValues(csr_sparse_arr)); // csr values are transposed +} diff --git a/test/stdev.cpp b/test/stdev.cpp index 4b93f5b220..bf95801fed 100644 --- a/test/stdev.cpp +++ b/test/stdev.cpp @@ -37,7 +37,8 @@ class StandardDev : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list @@ -67,9 +68,9 @@ template struct sdOutType { typedef typename cond_type< is_same_type::value || is_same_type::value || - is_same_type::value || is_same_type::value || - is_same_type::value || is_same_type::value || - is_same_type::value, + is_same_type::value || is_same_type::value || + is_same_type::value || is_same_type::value || + is_same_type::value || is_same_type::value, float, typename elseType::type>::type type; }; diff --git a/test/susan.cpp b/test/susan.cpp index 34929c22c0..c488bda775 100644 --- a/test/susan.cpp +++ b/test/susan.cpp @@ -59,7 +59,8 @@ class Susan : public ::testing::Test { virtual void SetUp() {} }; -typedef ::testing::Types +typedef ::testing::Types TestTypes; TYPED_TEST_SUITE(Susan, TestTypes); @@ -125,6 +126,7 @@ void susanTest(string pTestFile, float t, float g) { #define SUSAN_TEST(image, tval, gval) \ TYPED_TEST(Susan, image) { \ + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); \ susanTest(string(TEST_DIR "/susan/" #image ".test"), tval, \ gval); \ } diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp index 3f1beb55bb..405f23309d 100644 --- a/test/testHelpers.hpp +++ b/test/testHelpers.hpp @@ -7,13 +7,17 @@ * http://arrayfire.com/licenses/BSD-3-Clause ********************************************************/ #pragma once +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wparentheses" +#endif #include +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif #include #include #include @@ -49,11 +53,20 @@ std::ostream &operator<<(std::ostream &os, const af_half &val); do { (void)(expr); } while (0) namespace aft { +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4996) +#endif typedef intl intl; typedef uintl uintl; +#ifdef __GNUC__ #pragma GCC diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#endif } // namespace aft using aft::intl; @@ -75,11 +88,12 @@ struct dtype_traits { } // namespace af +typedef signed char schar; typedef unsigned char uchar; typedef unsigned int uint; typedef unsigned short ushort; -std::string getBackendName(); +std::string getBackendName(bool lower = false); std::string getTestName(); std::string readNextNonEmptyLine(std::ifstream &file); @@ -229,6 +243,15 @@ bool noHalfTests(af::dtype ty); if (noHalfTests((af_dtype)af::dtype_traits::af_type)) \ GTEST_SKIP() << "Device doesn't support Half" +#ifdef SKIP_UNSUPPORTED_TESTS +#define UNSUPPORTED_BACKEND(backend) \ + if (backend == af::getActiveBackend()) \ + GTEST_SKIP() << "Skipping unsupported function on " + getBackendName() + \ + " backend" +#else +#define UNSUPPORTED_BACKEND(backend) +#endif + #define LAPACK_ENABLED_CHECK() \ if (!af::isLAPACKAvailable()) GTEST_SKIP() << "LAPACK Not Configured." @@ -630,4 +653,30 @@ ::testing::AssertionResult assertArrayEq(std::string aName, std::string bName, const af_array a, const af_array b, TestOutputArrayInfo *metadata); +enum tempFormat { + LINEAR_FORMAT, // Linear array (= default) + JIT_FORMAT, // Array which has JIT operations outstanding + SUB_FORMAT_dim0, // Array where only a subset is allocated for dim0 + SUB_FORMAT_dim1, // Array where only a subset is allocated for dim1 + SUB_FORMAT_dim2, // Array where only a subset is allocated for dim2 + SUB_FORMAT_dim3, // Array where only a subset is allocated for dim3 + REORDERED_FORMAT // Array where the dimensions are reordered +}; +// Calls the function fn for all available formats +#define FOREACH_TEMP_FORMAT(TESTS) \ + TESTS(LINEAR_FORMAT) \ + TESTS(JIT_FORMAT) \ + TESTS(SUB_FORMAT_dim0) \ + TESTS(SUB_FORMAT_dim1) \ + TESTS(SUB_FORMAT_dim2) \ + TESTS(SUB_FORMAT_dim3) \ + TESTS(REORDERED_FORMAT) + +// formats the "in" array according to provided format. The content remains +// unchanged. +af::array toTempFormat(tempFormat form, const af::array &in); +void toTempFormat(tempFormat form, af_array *out, const af_array &in); + +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif diff --git a/test/threading.cpp b/test/threading.cpp index 41c4ebb723..1b71411f0e 100644 --- a/test/threading.cpp +++ b/test/threading.cpp @@ -130,6 +130,7 @@ int nextTargetDeviceId() { void morphTest(const array input, const array mask, const bool isDilation, const array gold, int targetDevice) { + UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI); setDevice(targetDevice); array out; diff --git a/test/tile.cpp b/test/tile.cpp index bc0cdddba7..3a608fa987 100644 --- a/test/tile.cpp +++ b/test/tile.cpp @@ -47,8 +47,8 @@ class Tile : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + intl, uintl, char, signed char, unsigned char, short, + ushort, half_float::half> TestTypes; // register the type list diff --git a/test/transform.cpp b/test/transform.cpp index e3e0efe640..e6026576ba 100644 --- a/test/transform.cpp +++ b/test/transform.cpp @@ -38,7 +38,7 @@ class TransformInt : public ::testing::Test { }; typedef ::testing::Types TestTypes; -typedef ::testing::Types +typedef ::testing::Types TestTypesInt; TYPED_TEST_SUITE(Transform, TestTypes); @@ -620,3 +620,43 @@ TEST(TransformBatching, CPP) { } } } + +#define TEST_TEMP_FORMAT(form, interp) \ + TEST(TEMP_FORMAT, form##_##interp) { \ + IMAGEIO_ENABLED_CHECK(); \ + \ + vector inDims; \ + vector inFiles; \ + vector goldDim; \ + vector goldFiles; \ + \ + vector HDims; \ + vector> HIn; \ + vector> HTests; \ + readTests(TEST_DIR "/transform/tux_tmat.test", \ + HDims, HIn, HTests); \ + \ + readImageTests(string(TEST_DIR "/transform/tux_nearest.test"), inDims, \ + inFiles, goldDim, goldFiles); \ + inFiles[1].insert(0, string(TEST_DIR "/transform/")); \ + const array IH = array(HDims[0][0], HDims[0][1], &(HIn[0].front())); \ + const array scene_img = loadImage(inFiles[1].c_str(), false); \ + \ + const array out = \ + transform(toTempFormat(form, scene_img), toTempFormat(form, IH), \ + inDims[0][0], inDims[0][1], interp, false); \ + const array gold = transform(scene_img, IH, inDims[0][0], \ + inDims[0][1], interp, false); \ + \ + EXPECT_ARRAYS_EQ(out, gold); \ + } + +#define TESTS_TEMP_FORMAT(form) \ + TEST_TEMP_FORMAT(form, AF_INTERP_NEAREST) \ + TEST_TEMP_FORMAT(form, AF_INTERP_BILINEAR) \ + TEST_TEMP_FORMAT(form, AF_INTERP_BILINEAR_COSINE) \ + TEST_TEMP_FORMAT(form, AF_INTERP_BICUBIC) \ + TEST_TEMP_FORMAT(form, AF_INTERP_BICUBIC_SPLINE) \ + TEST_TEMP_FORMAT(form, AF_INTERP_LOWER) + +FOREACH_TEMP_FORMAT(TESTS_TEMP_FORMAT) \ No newline at end of file diff --git a/test/transform_coordinates.cpp b/test/transform_coordinates.cpp index 2875f18c1a..bc5dbed4e9 100644 --- a/test/transform_coordinates.cpp +++ b/test/transform_coordinates.cpp @@ -61,7 +61,7 @@ void transformCoordinatesTest(string pTestFile) { dim_t outEl = 0; ASSERT_SUCCESS(af_get_elements(&outEl, outArray)); vector outData(outEl); - ASSERT_SUCCESS(af_get_data_ptr((void*)&outData.front(), outArray)); + ASSERT_SUCCESS(af_get_data_ptr((void *)&outData.front(), outArray)); ASSERT_SUCCESS(af_release_array(outArray)); const float thr = 1.f; @@ -114,3 +114,26 @@ TEST(TransformCoordinates, CPP) { << "at: " << elIter << endl; } } + +#define TESTS_TEMP_FORMAT(form) \ + TEST(TEMP_FORMAT, form) { \ + vector inDims; \ + vector> in; \ + vector> gold; \ + \ + readTests(TEST_DIR \ + "/transformCoordinates/3d_matrix.test", \ + inDims, in, gold); \ + \ + const array tf(inDims[0][0], inDims[0][1], &(in[0].front())); \ + const float d0 = in[1][0]; \ + const float d1 = in[1][1]; \ + \ + const array out = \ + transformCoordinates(toTempFormat(form, tf), d0, d1); \ + const array gout = transformCoordinates(tf, d0, d1); \ + \ + EXPECT_ARRAYS_EQ(out, gout); \ + } + +FOREACH_TEMP_FORMAT(TESTS_TEMP_FORMAT) \ No newline at end of file diff --git a/test/translate.cpp b/test/translate.cpp index 55fd570ffb..edbab15a2c 100644 --- a/test/translate.cpp +++ b/test/translate.cpp @@ -39,7 +39,7 @@ class TranslateInt : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types TestTypes; -typedef ::testing::Types TestTypesInt; +typedef ::testing::Types TestTypesInt; // register the type list TYPED_TEST_SUITE(Translate, TestTypes); diff --git a/test/transpose.cpp b/test/transpose.cpp index 72a32194fa..420f6d88e3 100644 --- a/test/transpose.cpp +++ b/test/transpose.cpp @@ -44,8 +44,8 @@ class Transpose : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list diff --git a/test/transpose_inplace.cpp b/test/transpose_inplace.cpp index 82b071488a..7e542fd34f 100644 --- a/test/transpose_inplace.cpp +++ b/test/transpose_inplace.cpp @@ -30,8 +30,8 @@ class Transpose : public ::testing::Test { }; // create a list of types to be tested -typedef ::testing::Types +typedef ::testing::Types TestTypes; // register the type list diff --git a/test/triangle.cpp b/test/triangle.cpp index 90b50bb6dc..a7d47832e5 100644 --- a/test/triangle.cpp +++ b/test/triangle.cpp @@ -35,7 +35,8 @@ template class Triangle : public ::testing::Test {}; typedef ::testing::Types + schar, uchar, uintl, intl, short, ushort, + half_float::half> TestTypes; TYPED_TEST_SUITE(Triangle, TestTypes); diff --git a/test/unwrap.cpp b/test/unwrap.cpp index f43b73e7f4..9b97059dac 100644 --- a/test/unwrap.cpp +++ b/test/unwrap.cpp @@ -37,7 +37,8 @@ class Unwrap : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + intl, uintl, char, signed char, unsigned char, short, + ushort> TestTypes; // register the type list diff --git a/test/var.cpp b/test/var.cpp index db846f5d57..b889413646 100644 --- a/test/var.cpp +++ b/test/var.cpp @@ -26,7 +26,7 @@ template class Var : public ::testing::Test {}; typedef ::testing::Types + char, schar, uchar, short, ushort, half_float::half> TestTypes; TYPED_TEST_SUITE(Var, TestTypes); @@ -42,8 +42,8 @@ struct varOutType { typedef typename cond_type< is_same_type::value || is_same_type::value || is_same_type::value || is_same_type::value || - is_same_type::value || is_same_type::value || - is_same_type::value, + is_same_type::value || is_same_type::value || + is_same_type::value || is_same_type::value, float, typename elseType::type>::type type; }; diff --git a/test/where.cpp b/test/where.cpp index bb5375822c..a6c8dcde46 100644 --- a/test/where.cpp +++ b/test/where.cpp @@ -34,7 +34,7 @@ template class Where : public ::testing::Test {}; typedef ::testing::Types + char, schar, uchar, short, ushort> TestTypes; TYPED_TEST_SUITE(Where, TestTypes); @@ -136,3 +136,22 @@ TEST(Where, ISSUE_1259) { array indices = where(a > 2); ASSERT_EQ(indices.elements(), 0); } + +#define TEST_TEMP_FORMAT(form, dim) \ + TEST(TEMP_FORMAT, form##_Dim##dim) { \ + const dim4 dims(2, 3, 4, 5); \ + const array in(af::moddims(range(dim4(dims.elements())), dims)); \ + in.eval(); \ + const array gold = where(in > 3.0); \ + \ + array out = where(toTempFormat(form, in) > 3.0); \ + ASSERT_ARRAYS_EQ(gold, out); \ + } + +#define TEST_TEMP_FORMATS(form) \ + TEST_TEMP_FORMAT(form, 0) \ + TEST_TEMP_FORMAT(form, 1) \ + TEST_TEMP_FORMAT(form, 2) \ + TEST_TEMP_FORMAT(form, 3) + +FOREACH_TEMP_FORMAT(TEST_TEMP_FORMATS) \ No newline at end of file diff --git a/test/wrap.cpp b/test/wrap.cpp index baff77c5b1..4f53d9fd34 100644 --- a/test/wrap.cpp +++ b/test/wrap.cpp @@ -42,7 +42,8 @@ class Wrap : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + intl, uintl, char, signed char, unsigned char, short, + ushort> TestTypes; // register the type list diff --git a/test/write.cpp b/test/write.cpp index 8f18f6e954..db751939ab 100644 --- a/test/write.cpp +++ b/test/write.cpp @@ -34,7 +34,7 @@ class Write : public ::testing::Test { // create a list of types to be tested typedef ::testing::Types + signed char, unsigned char, short, ushort> TestTypes; // register the type list diff --git a/vcpkg.json b/vcpkg.json index 5cf6972ce0..7b8d9bca2f 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,6 +1,6 @@ { "name": "arrayfire", - "version": "3.9.0", + "version": "3.10.0", "homepage": "https://github.com/arrayfire/arrayfire", "description": "ArrayFire is a HPC general-purpose library targeting parallel and massively-parallel architectures such as CPUs, GPUs, etc.", "supports": "x64", @@ -19,6 +19,10 @@ { "name": "spdlog", "version": "1.9.2" + }, + { + "name": "jasper", + "version": "4.2.0" } ], "features": { @@ -67,15 +71,10 @@ "description": "Build OpenCL backend", "dependencies": [ "boost-compute", + "boost-program-options", "opencl" ] }, - "mkl": { - "description": "Build with MKL", - "dependencies": [ - "intel-mkl" - ] - }, "cudnn": { "description": "Build CUDA with support for cuDNN", "dependencies": [ @@ -83,5 +82,5 @@ ] } }, - "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996" + "builtin-baseline": "b02e341c927f16d991edbd915d8ea43eac52096c" }