initial release
This commit is contained in:
civ
@@ -0,0 +1,356 @@
|
||||
find_library(MATH_LIBRARY m)
|
||||
|
||||
# check systems
|
||||
if (NOT UNAME_S)
|
||||
execute_process(COMMAND uname -s OUTPUT_VARIABLE UNAME_S)
|
||||
endif()
|
||||
if (NOT UNAME_P)
|
||||
execute_process(COMMAND uname -p OUTPUT_VARIABLE UNAME_P)
|
||||
endif()
|
||||
if (NOT UNAME_M)
|
||||
execute_process(COMMAND uname -m OUTPUT_VARIABLE UNAME_M)
|
||||
endif()
|
||||
#message(STATUS "UNAME_S: ${UNAME_S} UNAME_P: ${UNAME_P} UNAME_M: ${UNAME_M}")
|
||||
|
||||
# Mac OS + Arm can report x86_64
|
||||
# ref: https://github.com/ggerganov/whisper.cpp/issues/66#issuecomment-1282546789
|
||||
if (UNAME_S MATCHES "Darwin")
|
||||
if (NOT UNAME_P MATCHES "arm")
|
||||
execute_process(COMMAND sysctl -n hw.optional.arm64 OUTPUT_VARIABLE SYSCTL_M)
|
||||
if (SYSCTL_M MATCHES "1")
|
||||
#set(UNAME_P "arm")
|
||||
#set(UNAME_M "arm64")
|
||||
message(WARNING "Your arch is announced as x86_64, but it seems to actually be ARM64. Not fixing that can lea
|
||||
d to bad performance. For more info see: https://github.com/ggerganov/whisper.cpp/issues/66\#issuecomment-#1282546789")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(aarch64|arm.*|ARM64)$")
|
||||
message(STATUS "ARM detected")
|
||||
#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=apple-m1")
|
||||
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
|
||||
message(STATUS "PPC64 detected")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mpower9-vector")
|
||||
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
|
||||
message(STATUS "loongarch64 detected")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mlsx -mlasx")
|
||||
else()
|
||||
message(STATUS "x86 detected")
|
||||
#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx -mavx2 -mfma -mf16c")
|
||||
if (UNAME_S MATCHES "Darwin")
|
||||
execute_process(COMMAND sysctl machdep.cpu.features OUTPUT_VARIABLE AVX1_M)
|
||||
if (AVX1_M MATCHES "AVX1.0")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
|
||||
endif()
|
||||
execute_process(COMMAND sysctl machdep.cpu.leaf7_features OUTPUT_VARIABLE AVX2_M)
|
||||
if (AVX2_M MATCHES "AVX2")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
|
||||
endif()
|
||||
if (AVX1_M MATCHES "FMA")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
|
||||
endif()
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mf16c")
|
||||
elseif (UNAME_S MATCHES "Linux")
|
||||
message(STATUS "Linux detected")
|
||||
# must have to build on ubuntu22 with gcc11:
|
||||
find_package(Threads)
|
||||
set(GGML_EXTRA_LIBS ${GGML_EXTRA_LIBS} Threads::Threads)
|
||||
|
||||
execute_process(COMMAND grep "avx " /proc/cpuinfo OUTPUT_VARIABLE AVX1_M)
|
||||
if (AVX1_M MATCHES "avx")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
|
||||
endif()
|
||||
execute_process(COMMAND grep "avx2 " /proc/cpuinfo OUTPUT_VARIABLE AVX2_M)
|
||||
if (AVX2_M MATCHES "avx2")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
|
||||
endif()
|
||||
execute_process(COMMAND grep "fma " /proc/cpuinfo OUTPUT_VARIABLE FMA_M)
|
||||
if (FMA_M MATCHES "fma")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
|
||||
endif()
|
||||
execute_process(COMMAND grep "f16c " /proc/cpuinfo OUTPUT_VARIABLE F16C_M)
|
||||
if (F16C_M MATCHES "f16c")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mf16c")
|
||||
endif()
|
||||
execute_process(COMMAND grep "sse3 " /proc/cpuinfo OUTPUT_VARIABLE SSE3_M)
|
||||
if (SSE3_M MATCHES "sse3")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse3")
|
||||
endif()
|
||||
elseif (UNAME_S MATCHES "Haiku")
|
||||
message(STATUS "Haiku detected")
|
||||
execute_process(COMMAND sysinfo -cpu COMMAND grep "AVX " OUTPUT_VARIABLE AVX1_M)
|
||||
if (AVX1_M MATCHES "avx")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
|
||||
endif()
|
||||
execute_process(COMMAND sysinfo -cpu COMMAND grep "AVX2 " OUTPUT_VARIABLE AVX2_M)
|
||||
if (AVX2_M MATCHES "avx2")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
|
||||
endif()
|
||||
execute_process(COMMAND sysinfo -cpu COMMAND grep "FMA " OUTPUT_VARIABLE FMA_M)
|
||||
if (FMA_M MATCHES "fma")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
|
||||
endif()
|
||||
execute_process(COMMAND sysinfo -cpu COMMAND grep "F16C " OUTPUT_VARIABLE F16C_M)
|
||||
if (F16C_M MATCHES "f16c")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mf16c")
|
||||
endif()
|
||||
elseif (MSVC)
|
||||
if (GGML_AVX512)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX512")
|
||||
# MSVC has no compile-time flags enabling specific
|
||||
# AVX512 extensions, neither it defines the
|
||||
# macros corresponding to the extensions.
|
||||
# Do it manually.
|
||||
if (GGML_AVX512_VBMI)
|
||||
add_compile_definitions(__AVX512VBMI__)
|
||||
endif()
|
||||
if (GGML_AVX512_VNNI)
|
||||
add_compile_definitions(__AVX512VNNI__)
|
||||
endif()
|
||||
elseif (GGML_AVX2)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX2")
|
||||
elseif (GGML_AVX)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX")
|
||||
endif()
|
||||
else()
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma -mf16c -mavx -mavx2")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# on APPLE - include Accelerate framework
|
||||
if (APPLE AND NOT GGML_NO_ACCELERATE)
|
||||
find_library(ACCELERATE_FRAMEWORK Accelerate)
|
||||
if (ACCELERATE_FRAMEWORK)
|
||||
message(STATUS "Accelerate framework found")
|
||||
|
||||
set(GGML_EXTRA_LIBS ${GGML_EXTRA_LIBS} ${ACCELERATE_FRAMEWORK})
|
||||
set(GGML_EXTRA_FLAGS ${GGML_EXTRA_FLAGS} -DGGML_USE_ACCELERATE)
|
||||
else()
|
||||
message(WARNING "Accelerate framework not found")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (GGML_OPENBLAS)
|
||||
set(OPENBLAS_INCLUDE_SEARCH_PATHS
|
||||
/usr/include
|
||||
/usr/include/openblas
|
||||
/usr/include/openblas-base
|
||||
/usr/local/include
|
||||
/usr/local/include/openblas
|
||||
/usr/local/include/openblas-base
|
||||
/opt/OpenBLAS/include
|
||||
$ENV{OpenBLAS_HOME}
|
||||
$ENV{OpenBLAS_HOME}/include
|
||||
)
|
||||
find_path(OPENBLAS_INC NAMES cblas.h PATHS ${OPENBLAS_INCLUDE_SEARCH_PATHS})
|
||||
find_library(OPENBLAS_LIB NAMES openblas libopenblas)
|
||||
if (OPENBLAS_LIB)
|
||||
message(STATUS "OpenBLAS found")
|
||||
|
||||
set(GGML_EXTRA_LIBS ${GGML_EXTRA_LIBS} ${OPENBLAS_LIB})
|
||||
set(GGML_EXTRA_INCS ${GGML_EXTRA_INCS} ${OPENBLAS_INC})
|
||||
set(GGML_EXTRA_FLAGS ${GGML_EXTRA_FLAGS} -DGGML_USE_OPENBLAS)
|
||||
else()
|
||||
message(WARNING "OpenBLAS not found")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# undefine NDEBUG so asserts don't get disabled in tests
|
||||
add_definitions(-UNDEBUG)
|
||||
|
||||
#
|
||||
# test-backend-ops
|
||||
|
||||
set(TEST_TARGET test-backend-ops)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml Threads::Threads)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
|
||||
if (NOT GGML_BACKEND_DL)
|
||||
#
|
||||
# test-opt
|
||||
set(TEST_TARGET test-opt)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-quantize-fns
|
||||
|
||||
set(TEST_TARGET test-quantize-fns)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-quantize-perf
|
||||
|
||||
set(TEST_TARGET test-quantize-perf)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-pool
|
||||
|
||||
set(TEST_TARGET test-pool)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
if (MSVC)
|
||||
target_link_options(${TEST_TARGET} PRIVATE "/STACK:8388608") # 8MB
|
||||
endif()
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-arange
|
||||
|
||||
set(TEST_TARGET test-arange)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml Threads::Threads)
|
||||
if (MSVC)
|
||||
target_link_options(${TEST_TARGET} PRIVATE "/STACK:8388608") # 8MB
|
||||
endif()
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-timestep_embedding
|
||||
|
||||
set(TEST_TARGET test-timestep_embedding)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
if (MSVC)
|
||||
target_link_options(${TEST_TARGET} PRIVATE "/STACK:8388608") # 8MB
|
||||
endif()
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-pad-reflect-1d
|
||||
|
||||
set(TEST_TARGET test-pad-reflect-1d)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
|
||||
#
|
||||
# test-roll
|
||||
|
||||
set(TEST_TARGET test-roll)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
|
||||
#
|
||||
# test-conv-transpose
|
||||
|
||||
set(TEST_TARGET test-conv-transpose)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
|
||||
# test-conv-transpose-1d
|
||||
|
||||
set(TEST_TARGET test-conv-transpose-1d)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
|
||||
#
|
||||
# test-dup
|
||||
|
||||
set(TEST_TARGET test-dup)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
|
||||
#
|
||||
# test-rel-pos
|
||||
|
||||
set(TEST_TARGET test-rel-pos)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
|
||||
#
|
||||
# test-customop
|
||||
|
||||
set(TEST_TARGET test-customop)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
if (MSVC)
|
||||
target_link_options(${TEST_TARGET} PRIVATE "/STACK:8388608") # 8MB
|
||||
endif()
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-conv1d
|
||||
|
||||
set(TEST_TARGET test-conv1d)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-conv1d-dw-c1
|
||||
|
||||
set(TEST_TARGET test-conv1d-dw-c1)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-conv1d-dw-c2
|
||||
|
||||
set(TEST_TARGET test-conv1d-dw-c2)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-conv2d
|
||||
|
||||
set(TEST_TARGET test-conv2d)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-conv2d-dw
|
||||
|
||||
set(TEST_TARGET test-conv2d-dw)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-cont
|
||||
|
||||
set(TEST_TARGET test-cont)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.c)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
|
||||
#
|
||||
# test-interpolate
|
||||
|
||||
set(TEST_TARGET test-interpolate)
|
||||
add_executable(${TEST_TARGET} ${TEST_TARGET}.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
add_test(NAME ${TEST_TARGET} COMMAND $<TARGET_FILE:${TEST_TARGET}>)
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY ENVIRONMENT "LLVM_PROFILE_FILE=${TEST_TARGET}.profraw")
|
||||
endif()
|
||||
@@ -0,0 +1,100 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
#include "ggml-metal.h"
|
||||
#endif
|
||||
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
int main(int /*argc*/, const char** /*argv*/) {
|
||||
{
|
||||
bool use_gpu = true;
|
||||
GGML_UNUSED(use_gpu);
|
||||
|
||||
ggml_backend_t backend = NULL;
|
||||
//ggml_backend_buffer_t buffer;
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using CUDA backend\n", __func__);
|
||||
backend = ggml_backend_cuda_init(0);
|
||||
if (!backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
if (!backend) {
|
||||
fprintf(stderr, "%s: using Metal backend\n", __func__);
|
||||
backend = ggml_backend_metal_init();
|
||||
if (!backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
const int num_tensors = 2;
|
||||
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ ggml_tensor_overhead() * num_tensors + 2 * 1024 * 1024,
|
||||
/*.mem_size =*/ NULL,
|
||||
/*.mem_size =*/ true,
|
||||
};
|
||||
|
||||
if (!backend) {
|
||||
// fallback to CPU backend
|
||||
backend = ggml_backend_cpu_init();
|
||||
}
|
||||
|
||||
// create context
|
||||
struct ggml_context* ctx = ggml_init(params);
|
||||
struct ggml_tensor * t = ggml_arange(ctx, 0, 3, 1);
|
||||
|
||||
GGML_ASSERT(t->ne[0] == 3);
|
||||
|
||||
ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, t);
|
||||
|
||||
// allocate tensors
|
||||
ggml_gallocr_alloc_graph(galloc, graph);
|
||||
|
||||
int n_threads = 4;
|
||||
|
||||
if (ggml_backend_is_cpu(backend)) {
|
||||
ggml_backend_cpu_set_n_threads(backend, n_threads);
|
||||
}
|
||||
|
||||
ggml_backend_graph_compute(backend, graph);
|
||||
|
||||
float * output = new float[ggml_nelements(t)];
|
||||
ggml_backend_tensor_get(t, output, 0, ggml_nbytes(t));
|
||||
|
||||
for (int i = 0; i < t->ne[0]; i++) {
|
||||
printf("%.2f ", output[i]);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
GGML_ASSERT(output[0] == 0);
|
||||
GGML_ASSERT(output[1] == 1);
|
||||
GGML_ASSERT(output[2] == 2);
|
||||
|
||||
delete[] output;
|
||||
ggml_free(ctx);
|
||||
ggml_gallocr_free(galloc);
|
||||
ggml_backend_free(backend);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,170 @@
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml.h"
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
struct model {
|
||||
struct ggml_context* ctx;
|
||||
struct ggml_context* ctx0;
|
||||
ggml_backend_t backend;
|
||||
ggml_backend_buffer_t buffer;
|
||||
struct ggml_cgraph* gf;
|
||||
ggml_gallocr_t allocr;
|
||||
uint8_t* buf;
|
||||
};
|
||||
|
||||
struct ggml_context* make_ctx(void) {
|
||||
struct ggml_init_params params = {
|
||||
.mem_size = ggml_tensor_overhead() * 3,
|
||||
.mem_buffer = NULL,
|
||||
.no_alloc = true,
|
||||
};
|
||||
return ggml_init(params);
|
||||
}
|
||||
|
||||
ggml_backend_t make_backend(void) {
|
||||
ggml_backend_t backend = NULL;
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
backend = ggml_backend_cuda_init(0);
|
||||
GGML_ASSERT(backend != NULL);
|
||||
#endif
|
||||
|
||||
if (!backend) {
|
||||
backend = ggml_backend_cpu_init();
|
||||
}
|
||||
|
||||
return backend;
|
||||
}
|
||||
|
||||
void model_init(struct model* m) {
|
||||
m->ctx = make_ctx();
|
||||
m->backend = make_backend();
|
||||
|
||||
size_t buf_size = ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
|
||||
m->buf = calloc(buf_size, sizeof(uint8_t));
|
||||
struct ggml_init_params params0 = {
|
||||
.mem_size = buf_size,
|
||||
.mem_buffer = m->buf,
|
||||
.no_alloc = true,
|
||||
};
|
||||
m->ctx0 = ggml_init(params0);
|
||||
m->gf = ggml_new_graph(m->ctx0);
|
||||
}
|
||||
|
||||
void model_alloc(struct model* m) {
|
||||
m->buffer = ggml_backend_alloc_ctx_tensors(m->ctx, m->backend);
|
||||
m->allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(m->backend));
|
||||
}
|
||||
|
||||
void model_compute(struct model* m) {
|
||||
ggml_gallocr_alloc_graph(m->allocr, m->gf);
|
||||
ggml_backend_graph_compute(m->backend, m->gf);
|
||||
}
|
||||
|
||||
void model_free(struct model* m) {
|
||||
ggml_free(m->ctx0);
|
||||
free(m->buf);
|
||||
ggml_gallocr_free(m->allocr);
|
||||
ggml_free(m->ctx);
|
||||
ggml_backend_buffer_free(m->buffer);
|
||||
ggml_backend_free(m->backend);
|
||||
}
|
||||
|
||||
void check_tensor(struct ggml_tensor* t,
|
||||
const float* expected_t_d,
|
||||
const int ne0,
|
||||
const int ne1,
|
||||
const int ne2) {
|
||||
GGML_ASSERT(t->ne[0] == ne0);
|
||||
GGML_ASSERT(t->ne[1] == ne1);
|
||||
GGML_ASSERT(t->ne[2] == ne2);
|
||||
const size_t bsize = ggml_nbytes(t);
|
||||
if (t->type == GGML_TYPE_F32) {
|
||||
float* buffer = malloc(bsize);
|
||||
ggml_backend_tensor_get(t, buffer, 0, bsize);
|
||||
for (int i = 0; i < bsize / sizeof(float); ++i) {
|
||||
float expected = expected_t_d[i];
|
||||
float actual = buffer[i];
|
||||
if (expected != actual) {
|
||||
printf("expected %.1f, got %.1f\n", expected, actual);
|
||||
}
|
||||
GGML_ASSERT(expected == actual);
|
||||
}
|
||||
free(buffer);
|
||||
} else if (t->type == GGML_TYPE_F16) {
|
||||
ggml_fp16_t* buffer = malloc(bsize);
|
||||
ggml_backend_tensor_get(t, buffer, 0, bsize);
|
||||
for (int i = 0; i < bsize / sizeof(ggml_fp16_t); ++i) {
|
||||
float expected = expected_t_d[i];
|
||||
float actual = ggml_fp16_to_fp32(buffer[i]);
|
||||
if (expected != actual) {
|
||||
printf("expected %.1f, got %.1f\n", expected, actual);
|
||||
}
|
||||
GGML_ASSERT(expected == actual);
|
||||
}
|
||||
free(buffer);
|
||||
//} else if (t->type == GGML_TYPE_BF16) {
|
||||
// ggml_bf16_t* buffer = malloc(bsize);
|
||||
// ggml_backend_tensor_get(t, buffer, 0, bsize);
|
||||
// for (int i = 0; i < bsize / sizeof(ggml_bf16_t); ++i) {
|
||||
// float expected = expected_t_d[i];
|
||||
// float actual = ggml_bf16_to_fp32(buffer[i]);
|
||||
// if (expected != actual) {
|
||||
// printf("expected %.1f, got %.1f\n", expected, actual);
|
||||
// }
|
||||
// GGML_ASSERT(expected == actual);
|
||||
// }
|
||||
// free(buffer);
|
||||
} else {
|
||||
GGML_ABORT("unknown type");
|
||||
}
|
||||
}
|
||||
|
||||
void test_cont(void) {
|
||||
float buf_f32[] = {1.0, 2.0};
|
||||
ggml_fp16_t buf_f16[] = {ggml_fp32_to_fp16(buf_f32[0]), ggml_fp32_to_fp16(buf_f32[1])};
|
||||
ggml_bf16_t buf_bf16[] = {ggml_fp32_to_bf16(buf_f32[0]), ggml_fp32_to_bf16(buf_f32[1])};
|
||||
|
||||
float expected_out[] = {1.0, 2.0};
|
||||
|
||||
struct model m;
|
||||
model_init(&m);
|
||||
|
||||
struct ggml_tensor* in_1 = ggml_new_tensor_1d(m.ctx, GGML_TYPE_F32, 2);
|
||||
struct ggml_tensor* in_2 = ggml_new_tensor_1d(m.ctx, GGML_TYPE_F16, 2);
|
||||
//struct ggml_tensor* in_3 = ggml_new_tensor_1d(m.ctx, GGML_TYPE_BF16, 2);
|
||||
|
||||
model_alloc(&m);
|
||||
|
||||
ggml_backend_tensor_set(in_1, buf_f32, 0, ggml_nbytes(in_1));
|
||||
ggml_backend_tensor_set(in_2, buf_f16, 0, ggml_nbytes(in_2));
|
||||
//ggml_backend_tensor_set(in_3, buf_bf16, 0, ggml_nbytes(in_3));
|
||||
|
||||
struct ggml_tensor* out_1 = ggml_cont(m.ctx0, ggml_transpose(m.ctx0, in_1));
|
||||
struct ggml_tensor* out_2 = ggml_cont(m.ctx0, ggml_transpose(m.ctx0, in_2));
|
||||
//struct ggml_tensor* out_3 = ggml_cont(m.ctx0, ggml_transpose(m.ctx0, in_3));
|
||||
|
||||
ggml_build_forward_expand(m.gf, out_1);
|
||||
ggml_build_forward_expand(m.gf, out_2);
|
||||
//ggml_build_forward_expand(m.gf, out_3);
|
||||
|
||||
model_compute(&m);
|
||||
|
||||
check_tensor(out_1, expected_out, 1, 2, 1);
|
||||
check_tensor(out_2, expected_out, 1, 2, 1);
|
||||
//check_tensor(out_3, expected_out, 1, 2, 1);
|
||||
|
||||
model_free(&m);
|
||||
}
|
||||
|
||||
int main(int argc, const char* argv[]) {
|
||||
test_cont();
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,248 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
struct ggml_context* make_ctx(void) {
|
||||
struct ggml_init_params params = {
|
||||
.mem_size = 2 * 1024 * 1024,
|
||||
};
|
||||
|
||||
return ggml_init(params);
|
||||
}
|
||||
|
||||
void printf_tensor(struct ggml_tensor * t) {
|
||||
if (t->type == GGML_TYPE_F32) {
|
||||
const float * t_d = ggml_get_data_f32(t);
|
||||
for (int i = 0; i < t->ne[2]; ++i) {
|
||||
for (int j = 0; j < t->ne[1]; ++j) {
|
||||
for (int k = 0; k < t->ne[0]; ++k) {
|
||||
printf("%.1f ", t_d[i * t->ne[1] * t->ne[0] + j * t->ne[0] + k]);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
printf("---\n");
|
||||
}
|
||||
}
|
||||
else if (t->type == GGML_TYPE_F16) {
|
||||
const ggml_fp16_t * t_d = ggml_get_data(t);
|
||||
for (int i = 0; i < t->ne[2]; ++i) {
|
||||
for (int j = 0; j < t->ne[1]; ++j) {
|
||||
for (int k = 0; k < t->ne[0]; ++k) {
|
||||
printf("%.1f ", ggml_fp16_to_fp32(t_d[i * t->ne[1] * t->ne[0] + j * t->ne[0] + k]));
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
printf("---\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
printf("unknown type\n");
|
||||
}
|
||||
}
|
||||
|
||||
void check_tensor(struct ggml_tensor * t, float * expected_t_d, int ne0, int ne1, int ne2) {
|
||||
GGML_ASSERT(t->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(t->ne[0] == ne0);
|
||||
GGML_ASSERT(t->ne[1] == ne1);
|
||||
GGML_ASSERT(t->ne[2] == ne2);
|
||||
for (int i2 = 0; i2 < ne2; ++i2) {
|
||||
for (int i1 = 0; i1 < ne1; ++i1) {
|
||||
for (int i0 = 0; i0 < ne0; ++i0) {
|
||||
float expected = *(expected_t_d + i2 * ne1 * ne0 + i1 * ne0 + i0);
|
||||
float actual = ggml_get_data_f32(t)[i2 * ne1 * ne0 + i1 * ne0 + i0];
|
||||
if (expected != actual) {
|
||||
printf("expected %.1f, got %.1f\n", expected, actual);
|
||||
}
|
||||
GGML_ASSERT(expected == actual);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void test_conv_transpose_1d(void) {
|
||||
|
||||
float buf_f32[1024];
|
||||
for (int i = 0; i < 1024; ++i) {
|
||||
buf_f32[i] = (float)i;
|
||||
}
|
||||
|
||||
ggml_fp16_t buf_f16[1024];
|
||||
for (int i = 0; i < 1024; ++i) {
|
||||
buf_f16[i] = ggml_fp32_to_fp16((float)i);
|
||||
}
|
||||
|
||||
float expected_out_1[3][4] = {
|
||||
{18.0, 45.0, 59.0, 37.0},
|
||||
{24.0, 61.0, 83.0, 51.0},
|
||||
{30.0, 77.0, 107.0, 65.0},
|
||||
};
|
||||
float expected_out_2[3][6] = {
|
||||
{18.0, 21.0, 24.0, 29.0, 30.0, 37.0},
|
||||
{24.0, 27.0, 34.0, 39.0, 44.0, 51.0},
|
||||
{30.0, 33.0, 44.0, 49.0, 58.0, 65.0},
|
||||
};
|
||||
float expected_out_3[3][8] = {
|
||||
{18.0, 21.0, 0.0, 24.0, 29.0, 0.0, 30.0, 37.0},
|
||||
{24.0, 27.0, 0.0, 34.0, 39.0, 0.0, 44.0, 51.0},
|
||||
{30.0, 33.0, 0.0, 44.0, 49.0, 0.0, 58.0, 65.0},
|
||||
};
|
||||
|
||||
// conv transpose 1d with stride 1, 2 & 3
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
|
||||
struct ggml_tensor * t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 3, 2); // l x cin
|
||||
memcpy(t->data, buf_f32, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * k = ggml_new_tensor_3d(ctx, GGML_TYPE_F16, 2, 3, 2); // k x cout x cin
|
||||
memcpy(k->data, buf_f16, ggml_nbytes(k));
|
||||
|
||||
struct ggml_tensor * out_1 = ggml_conv_transpose_1d(ctx, k, t, 1 /* s0 */, 0 /* p0 */, 1 /* d0 */);
|
||||
struct ggml_tensor * out_2 = ggml_conv_transpose_1d(ctx, k, t, 2 /* s0 */, 0 /* p0 */, 1 /* d0 */);
|
||||
struct ggml_tensor * out_3 = ggml_conv_transpose_1d(ctx, k, t, 3 /* s0 */, 0 /* p0 */, 1 /* d0 */);
|
||||
|
||||
struct ggml_cgraph * gf_1 = ggml_new_graph(ctx);
|
||||
struct ggml_cgraph * gf_2 = ggml_new_graph(ctx);
|
||||
struct ggml_cgraph * gf_3 = ggml_new_graph(ctx);
|
||||
|
||||
ggml_build_forward_expand(gf_1, out_1);
|
||||
ggml_build_forward_expand(gf_2, out_2);
|
||||
ggml_build_forward_expand(gf_3, out_3);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, gf_1, 1);
|
||||
ggml_graph_compute_with_ctx(ctx, gf_2, 1);
|
||||
ggml_graph_compute_with_ctx(ctx, gf_3, 1);
|
||||
|
||||
check_tensor(out_1, (float*)expected_out_1, 4, 3, 1);
|
||||
check_tensor(out_2, (float*)expected_out_2, 6, 3, 1);
|
||||
check_tensor(out_3, (float*)expected_out_3, 8, 3, 1);
|
||||
}
|
||||
}
|
||||
|
||||
void test_conv_transpose_2d(void) {
|
||||
|
||||
float buf_f32[1024];
|
||||
for (int i = 0; i < 1024; ++i) {
|
||||
buf_f32[i] = (float)i;
|
||||
}
|
||||
|
||||
ggml_fp16_t buf_f16[1024];
|
||||
for (int i = 0; i < 1024; ++i) {
|
||||
buf_f16[i] = ggml_fp32_to_fp16((float)i);
|
||||
}
|
||||
|
||||
float expected_out_1[3][3][4] = {
|
||||
{
|
||||
{72.0, 162.0, 188.0, 106.0},
|
||||
{192.0, 430.0, 490.0, 274.0},
|
||||
{132.0, 292.0, 326.0, 180.0},
|
||||
},
|
||||
{
|
||||
{96.0, 218.0, 260.0, 146.0},
|
||||
{264.0, 590.0, 682.0, 378.0},
|
||||
{180.0, 396.0, 446.0, 244.0},
|
||||
},
|
||||
{
|
||||
{120.0, 274.0, 332.0, 186.0},
|
||||
{336.0, 750.0, 874.0, 482.0},
|
||||
{228.0, 500.0, 566.0, 308.0},
|
||||
},
|
||||
};
|
||||
|
||||
float expected_out_2[3][4][6] = {
|
||||
{
|
||||
{72.0, 78.0, 84.0, 92.0, 96.0, 106.0},
|
||||
{84.0, 90.0, 100.0, 108.0, 116.0, 126.0},
|
||||
{108.0, 120.0, 120.0, 134.0, 132.0, 148.0},
|
||||
{132.0, 144.0, 148.0, 162.0, 164.0, 180.0},
|
||||
},
|
||||
{
|
||||
{96.0, 102.0, 116.0, 124.0, 136.0, 146.0},
|
||||
{108.0, 114.0, 132.0, 140.0, 156.0, 166.0},
|
||||
{156.0, 168.0, 176.0, 190.0, 196.0, 212.0},
|
||||
{180.0, 192.0, 204.0, 218.0, 228.0, 244.0},
|
||||
},
|
||||
{
|
||||
{120.0, 126.0, 148.0, 156.0, 176.0, 186.0},
|
||||
{132.0, 138.0, 164.0, 172.0, 196.0, 206.0},
|
||||
{204.0, 216.0, 232.0, 246.0, 260.0, 276.0},
|
||||
{228.0, 240.0, 260.0, 274.0, 292.0, 308.0},
|
||||
},
|
||||
};
|
||||
|
||||
float expected_out_3[3][5][8] = {
|
||||
{
|
||||
{72.0, 78.0, 0.0, 84.0, 92.0, 0.0, 96.0, 106.0},
|
||||
{84.0, 90.0, 0.0, 100.0, 108.0, 0.0, 116.0, 126.0},
|
||||
{0.0, 0.0, 0.0, 0.0, 0.0, 0.0},
|
||||
{108.0, 120.0, 0.0, 120.0, 134.0, 0.0, 132.0, 148.0},
|
||||
{132.0, 144.0, 0.0, 148.0, 162.0, 0.0, 164.0, 180.0},
|
||||
},
|
||||
{
|
||||
{96.0, 102.0, 0.0, 116.0, 124.0, 0.0, 136.0, 146.0},
|
||||
{108.0, 114.0, 0.0, 132.0, 140.0, 0.0, 156.0, 166.0},
|
||||
{0.0, 0.0, 0.0, 0.0, 0.0, 0.0},
|
||||
{156.0, 168.0, 0.0, 176.0, 190.0, 0.0, 196.0, 212.0},
|
||||
{180.0, 192.0, 0.0, 204.0, 218.0, 0.0, 228.0, 244.0},
|
||||
},
|
||||
{
|
||||
{120.0, 126.0, 0.0, 148.0, 156.0, 0.0, 176.0, 186.0},
|
||||
{132.0, 138.0, 0.0, 164.0, 172.0, 0.0, 196.0, 206.0},
|
||||
{0.0, 0.0, 0.0, 0.0, 0.0, 0.0},
|
||||
{204.0, 216.0, 0.0, 232.0, 246.0, 0.0, 260.0, 276.0},
|
||||
{228.0, 240.0, 0.0, 260.0, 274.0, 0.0, 292.0, 308.0},
|
||||
},
|
||||
};
|
||||
|
||||
// conv transpose 2d with stride 1, 2 & 3
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
|
||||
struct ggml_tensor * t = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 3, 2, 2, 1); // w x h x cin
|
||||
memcpy(t->data, buf_f32, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * k = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 2, 2, 3, 2); // w x h cin x cout
|
||||
memcpy(k->data, buf_f16, ggml_nbytes(k));
|
||||
|
||||
struct ggml_tensor * out_1 = ggml_conv_transpose_2d_p0(ctx, k, t, 1);
|
||||
struct ggml_tensor * out_2 = ggml_conv_transpose_2d_p0(ctx, k, t, 2);
|
||||
struct ggml_tensor * out_3 = ggml_conv_transpose_2d_p0(ctx, k, t, 3);
|
||||
|
||||
struct ggml_cgraph * gf_1 = ggml_new_graph(ctx);
|
||||
struct ggml_cgraph * gf_2 = ggml_new_graph(ctx);
|
||||
struct ggml_cgraph * gf_3 = ggml_new_graph(ctx);
|
||||
|
||||
ggml_build_forward_expand(gf_1, out_1);
|
||||
ggml_build_forward_expand(gf_2, out_2);
|
||||
ggml_build_forward_expand(gf_3, out_3);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, gf_1, 1);
|
||||
ggml_graph_compute_with_ctx(ctx, gf_2, 1);
|
||||
ggml_graph_compute_with_ctx(ctx, gf_3, 1);
|
||||
|
||||
// printf("in\n");
|
||||
// printf_tensor(t);
|
||||
// printf("\n\nkernel\n");
|
||||
// printf_tensor(k);
|
||||
// printf("\n\nout\n");
|
||||
// printf_tensor(out);
|
||||
// printf("\n\nout_2\n");
|
||||
// printf_tensor(out_2);
|
||||
// printf("\n\nout_3\n");
|
||||
// printf_tensor(out_3);
|
||||
|
||||
check_tensor(out_1, (float*)expected_out_1, 4, 3, 3);
|
||||
check_tensor(out_2, (float*)expected_out_2, 6, 4, 3);
|
||||
check_tensor(out_3, (float*)expected_out_3, 8, 5, 3);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, const char * argv[]) {
|
||||
test_conv_transpose_1d();
|
||||
test_conv_transpose_2d();
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
#include "ggml-metal.h"
|
||||
#endif
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
|
||||
(void) level;
|
||||
(void) user_data;
|
||||
fputs(text, stderr);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
struct test_model {
|
||||
struct ggml_tensor * weight;
|
||||
struct ggml_tensor * input;
|
||||
ggml_backend_t backend = NULL;
|
||||
ggml_backend_buffer_t buffer;
|
||||
struct ggml_context * ctx;
|
||||
};
|
||||
|
||||
void load_model(test_model & model, bool use_gpu = false) {
|
||||
// create data
|
||||
int K = 3, IC = 2, OC = 2;
|
||||
int IL = 6, N = 1;
|
||||
|
||||
// Initialize adata
|
||||
float weight_data[6] = {10.0f, 20.0f, 30.0f, 0.1f, 0.2f, 0.3f};
|
||||
|
||||
// Convert adata to fp16 format
|
||||
std::vector<ggml_fp16_t> h_weight_data(K * IC);
|
||||
ggml_fp32_to_fp16_row(weight_data, h_weight_data.data(), K * IC);
|
||||
|
||||
// Initialize input data, 2 channels, 6 timesteps, 1 batch
|
||||
float input_data[12] = {
|
||||
1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f,
|
||||
1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f,
|
||||
};
|
||||
|
||||
size_t buffer_size = 0;
|
||||
{
|
||||
buffer_size += K * IC * ggml_type_size(GGML_TYPE_F16); // tensor weight
|
||||
buffer_size += IL * IC * N * ggml_type_size(GGML_TYPE_F32); // tensor input
|
||||
buffer_size += 1024; // overhead
|
||||
}
|
||||
|
||||
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
|
||||
printf("%s: backend buffer size = %0.2f MB\n", __func__, (buffer_size/ 1024.f/ 1024.f));
|
||||
|
||||
ggml_log_set(ggml_log_callback_default, nullptr);
|
||||
|
||||
int num_tensors = 2;
|
||||
struct ggml_init_params params {
|
||||
/*.mem_size =*/ ggml_tensor_overhead() * num_tensors,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
|
||||
// initialize the backend
|
||||
#ifdef GGML_USE_CUDA
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using CUDA backend\n", __func__);
|
||||
model.backend = ggml_backend_cuda_init(0);
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using Metal backend\n", __func__);
|
||||
model.backend = ggml_backend_metal_init();
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if(!model.backend) {
|
||||
// fallback to CPU backend
|
||||
model.backend = ggml_backend_cpu_init();
|
||||
}
|
||||
|
||||
model.buffer = ggml_backend_alloc_buffer(model.backend, buffer_size);
|
||||
|
||||
// create context
|
||||
model.ctx = ggml_init(params);
|
||||
|
||||
// create tensors
|
||||
// A Pytorch grouped Conv1d weight parameter is of shape (out_channels, input_channels/groups, kernel_size)
|
||||
model.weight = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F16, K, 1, IC);
|
||||
model.input = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F32, IL, IC, N);
|
||||
|
||||
// create a allocator
|
||||
ggml_tallocr alloc = ggml_tallocr_new(model.buffer);
|
||||
|
||||
// alloc memory
|
||||
ggml_tallocr_alloc(&alloc, model.weight);
|
||||
|
||||
// load data to buffer
|
||||
if(ggml_backend_is_cpu(model.backend)) {
|
||||
memcpy(model.weight->data, h_weight_data.data(), ggml_nbytes(model.weight));
|
||||
} else {
|
||||
ggml_backend_tensor_set(model.weight, h_weight_data.data(), 0, ggml_nbytes(model.weight));
|
||||
}
|
||||
|
||||
// alloc memory
|
||||
ggml_tallocr_alloc(&alloc, model.input);
|
||||
|
||||
if(ggml_backend_is_cpu(model.backend)
|
||||
#ifdef GGML_USE_METAL
|
||||
|| ggml_backend_is_metal(model.backend)
|
||||
#endif
|
||||
) {
|
||||
memcpy(model.input->data, input_data, ggml_nbytes(model.input));
|
||||
} else {
|
||||
ggml_backend_tensor_set(model.input, input_data, 0, ggml_nbytes(model.input));
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_cgraph * build_graph(const test_model& model) {
|
||||
static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
|
||||
static std::vector<uint8_t> buf(buf_size);
|
||||
|
||||
struct ggml_init_params params0 = {
|
||||
/*.mem_size =*/ buf_size,
|
||||
/*.mem_buffer =*/ buf.data(),
|
||||
/*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()
|
||||
};
|
||||
|
||||
// create a temporally context to build the graph
|
||||
struct ggml_context * ctx0 = ggml_init(params0);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx0);
|
||||
|
||||
int s0 = 1;
|
||||
int p0 = 1;
|
||||
int d0 = 1;
|
||||
|
||||
struct ggml_tensor* conv1d_dw_res = ggml_conv_1d_dw(ctx0, model.weight, model.input, s0, p0, d0);
|
||||
ggml_set_name(conv1d_dw_res, "conv1d_dw_res");
|
||||
ggml_build_forward_expand(gf, conv1d_dw_res);
|
||||
|
||||
// delete the temporally context used to build the graph
|
||||
ggml_free(ctx0);
|
||||
return gf;
|
||||
}
|
||||
|
||||
struct ggml_cgraph* compute_graph(const test_model & model, ggml_gallocr_t allocr) {
|
||||
struct ggml_cgraph * gf = build_graph(model);
|
||||
|
||||
// allocate tensors
|
||||
ggml_gallocr_alloc_graph(allocr, gf);
|
||||
int n_threads = 1;
|
||||
|
||||
if (ggml_backend_is_cpu(model.backend)) {
|
||||
ggml_backend_cpu_set_n_threads(model.backend, n_threads);
|
||||
}
|
||||
|
||||
ggml_backend_graph_compute(model.backend, gf);
|
||||
|
||||
//ggml_graph_print(gf);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
ggml_time_init();
|
||||
|
||||
test_model model;
|
||||
load_model(model, true);
|
||||
|
||||
ggml_gallocr_t allocr = NULL;
|
||||
|
||||
{
|
||||
allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
|
||||
|
||||
//create the worst case graph for memory usage estimation
|
||||
struct ggml_cgraph * gf = build_graph(model);
|
||||
|
||||
// compute the required memory
|
||||
ggml_gallocr_reserve(allocr, gf);
|
||||
size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);
|
||||
fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0f/1024.0f);
|
||||
}
|
||||
|
||||
struct ggml_cgraph * gf_res = compute_graph(model, allocr);
|
||||
|
||||
struct ggml_tensor * conv1d_dw_res = NULL;
|
||||
|
||||
for(int i = 0; i < ggml_graph_n_nodes(gf_res); i++) {
|
||||
if(strcmp(ggml_get_name(ggml_graph_node(gf_res, i)), "conv1d_dw_res") == 0) {
|
||||
conv1d_dw_res = ggml_graph_node(gf_res, i);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<float> conv2d_data(ggml_nelements(conv1d_dw_res));
|
||||
|
||||
ggml_backend_tensor_get(conv1d_dw_res, conv2d_data.data(), 0, ggml_nbytes(conv1d_dw_res));
|
||||
|
||||
const int n_conv1d_dw_test = 12;
|
||||
|
||||
float expected_conv1d_dw[n_conv1d_dw_test] = {
|
||||
50.0f, 60.0f, 60.0f, 60.0f, 60.0f, 30.0f, 0.50f, 0.60f, 0.60f, 0.60f, 0.60f, 0.30f
|
||||
};
|
||||
|
||||
printf("\nPerforming test:\n");
|
||||
|
||||
bool passed = true;
|
||||
passed = true;
|
||||
for(int i = 0; i < n_conv1d_dw_test; i++) {
|
||||
if(std::abs(conv2d_data[i] - expected_conv1d_dw[i]) > 1e-4) {
|
||||
passed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
printf("ggml_conv1d (%d): %s\n", (int) ggml_nelements(conv1d_dw_res), passed && (ggml_nelements(conv1d_dw_res) == n_conv1d_dw_test) ? "\033[32mPASSED\033[0m" : "\033[31mFAILED\033[0m");
|
||||
ggml_free(model.ctx);
|
||||
|
||||
ggml_backend_buffer_free(model.buffer);
|
||||
ggml_backend_free(model.backend);
|
||||
ggml_gallocr_free(allocr);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
#include "ggml-metal.h"
|
||||
#endif
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
|
||||
(void) level;
|
||||
(void) user_data;
|
||||
fputs(text, stderr);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
struct test_model {
|
||||
struct ggml_tensor * weight;
|
||||
struct ggml_tensor * input;
|
||||
ggml_backend_t backend = NULL;
|
||||
ggml_backend_buffer_t buffer;
|
||||
struct ggml_context * ctx;
|
||||
};
|
||||
|
||||
void load_model(test_model & model, bool use_gpu = false) {
|
||||
// create data
|
||||
int K = 3, IC = 2, OC = 2;
|
||||
int IL = 6, N = 1;
|
||||
|
||||
// Initialize adata
|
||||
float weight_data[6] = {10.0f, 20.0f, 30.0f, 0.1f, 0.2f, 0.3f};
|
||||
|
||||
// Convert adata to fp16 format
|
||||
std::vector<ggml_fp16_t> h_weight_data(K * IC);
|
||||
ggml_fp32_to_fp16_row(weight_data, h_weight_data.data(), K * IC);
|
||||
|
||||
// Initialize input data, 2 channels, 6 timesteps, 1 batch
|
||||
float input_data[12] = {
|
||||
1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f,
|
||||
1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f,
|
||||
};
|
||||
|
||||
size_t buffer_size = 0;
|
||||
{
|
||||
buffer_size += K * IC * ggml_type_size(GGML_TYPE_F16); // tensor weight
|
||||
buffer_size += IL * IC * N * ggml_type_size(GGML_TYPE_F32); // tensor input
|
||||
buffer_size += 1024; // overhead
|
||||
}
|
||||
|
||||
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
|
||||
printf("%s: backend buffer size = %0.2f MB\n", __func__, (buffer_size/ 1024.f/ 1024.f));
|
||||
|
||||
ggml_log_set(ggml_log_callback_default, nullptr);
|
||||
|
||||
int num_tensors = 2;
|
||||
struct ggml_init_params params {
|
||||
/*.mem_size =*/ ggml_tensor_overhead() * num_tensors,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
|
||||
// initialize the backend
|
||||
#ifdef GGML_USE_CUDA
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using CUDA backend\n", __func__);
|
||||
model.backend = ggml_backend_cuda_init(0);
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using Metal backend\n", __func__);
|
||||
model.backend = ggml_backend_metal_init();
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if(!model.backend) {
|
||||
// fallback to CPU backend
|
||||
model.backend = ggml_backend_cpu_init();
|
||||
}
|
||||
|
||||
model.buffer = ggml_backend_alloc_buffer(model.backend, buffer_size);
|
||||
|
||||
// create context
|
||||
model.ctx = ggml_init(params);
|
||||
|
||||
// create tensors
|
||||
// A Pytorch grouped Conv1d weight parameter is of shape (out_channels, input_channels/groups, kernel_size)
|
||||
model.weight = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F16, K, 1, IC);
|
||||
model.input = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F32, IL, IC, N);
|
||||
|
||||
// create a allocator
|
||||
ggml_tallocr alloc = ggml_tallocr_new(model.buffer);
|
||||
|
||||
// alloc memory
|
||||
ggml_tallocr_alloc(&alloc, model.weight);
|
||||
|
||||
// load data to buffer
|
||||
if(ggml_backend_is_cpu(model.backend)) {
|
||||
memcpy(model.weight->data, h_weight_data.data(), ggml_nbytes(model.weight));
|
||||
} else {
|
||||
ggml_backend_tensor_set(model.weight, h_weight_data.data(), 0, ggml_nbytes(model.weight));
|
||||
}
|
||||
|
||||
// alloc memory
|
||||
ggml_tallocr_alloc(&alloc, model.input);
|
||||
|
||||
if(ggml_backend_is_cpu(model.backend)
|
||||
#ifdef GGML_USE_METAL
|
||||
|| ggml_backend_is_metal(model.backend)
|
||||
#endif
|
||||
) {
|
||||
memcpy(model.input->data, input_data, ggml_nbytes(model.input));
|
||||
} else {
|
||||
ggml_backend_tensor_set(model.input, input_data, 0, ggml_nbytes(model.input));
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_cgraph * build_graph(const test_model& model) {
|
||||
static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
|
||||
static std::vector<uint8_t> buf(buf_size);
|
||||
|
||||
struct ggml_init_params params0 = {
|
||||
/*.mem_size =*/ buf_size,
|
||||
/*.mem_buffer =*/ buf.data(),
|
||||
/*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()
|
||||
};
|
||||
|
||||
// create a temporally context to build the graph
|
||||
struct ggml_context * ctx0 = ggml_init(params0);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx0);
|
||||
|
||||
int s0 = 3;
|
||||
int p0 = 0;
|
||||
int d0 = 1;
|
||||
|
||||
struct ggml_tensor* conv1d_dw_res = ggml_conv_1d_dw(ctx0, model.weight, model.input, s0, p0, d0);
|
||||
ggml_set_name(conv1d_dw_res, "conv1d_dw_res");
|
||||
ggml_build_forward_expand(gf, conv1d_dw_res);
|
||||
|
||||
// delete the temporally context used to build the graph
|
||||
ggml_free(ctx0);
|
||||
return gf;
|
||||
}
|
||||
|
||||
struct ggml_cgraph* compute_graph(const test_model & model, ggml_gallocr_t allocr) {
|
||||
struct ggml_cgraph * gf = build_graph(model);
|
||||
|
||||
// allocate tensors
|
||||
ggml_gallocr_alloc_graph(allocr, gf);
|
||||
int n_threads = 1;
|
||||
|
||||
if (ggml_backend_is_cpu(model.backend)) {
|
||||
ggml_backend_cpu_set_n_threads(model.backend, n_threads);
|
||||
}
|
||||
|
||||
ggml_backend_graph_compute(model.backend, gf);
|
||||
|
||||
//ggml_graph_print(gf);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
ggml_time_init();
|
||||
|
||||
test_model model;
|
||||
load_model(model, true);
|
||||
|
||||
ggml_gallocr_t allocr = NULL;
|
||||
|
||||
{
|
||||
allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
|
||||
|
||||
//create the worst case graph for memory usage estimation
|
||||
struct ggml_cgraph * gf = build_graph(model);
|
||||
|
||||
// compute the required memory
|
||||
ggml_gallocr_reserve(allocr, gf);
|
||||
size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);
|
||||
fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0f/1024.0f);
|
||||
}
|
||||
|
||||
struct ggml_cgraph * gf_res = compute_graph(model, allocr);
|
||||
|
||||
struct ggml_tensor * conv1d_dw_res = NULL;
|
||||
|
||||
for(int i = 0; i < ggml_graph_n_nodes(gf_res); i++) {
|
||||
if(strcmp(ggml_get_name(ggml_graph_node(gf_res, i)), "conv1d_dw_res") == 0) {
|
||||
conv1d_dw_res = ggml_graph_node(gf_res, i);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<float> conv2d_data(ggml_nelements(conv1d_dw_res));
|
||||
|
||||
ggml_backend_tensor_get(conv1d_dw_res, conv2d_data.data(), 0, ggml_nbytes(conv1d_dw_res));
|
||||
|
||||
const int n_conv1d_dw_test = 4;
|
||||
|
||||
float expected_conv1d_dw[n_conv1d_dw_test] = {
|
||||
60.0f, 60.0f, 0.6f, 0.6f
|
||||
};
|
||||
|
||||
printf("\nPerforming test:\n");
|
||||
|
||||
bool passed = true;
|
||||
passed = true;
|
||||
for(int i = 0; i < n_conv1d_dw_test; i++) {
|
||||
if(std::abs(conv2d_data[i] - expected_conv1d_dw[i]) > 1e-4) {
|
||||
passed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
printf("ggml_conv1d (%d): %s\n", (int) ggml_nelements(conv1d_dw_res), passed && (ggml_nelements(conv1d_dw_res) == n_conv1d_dw_test) ? "\033[32mPASSED\033[0m" : "\033[31mFAILED\033[0m");
|
||||
ggml_free(model.ctx);
|
||||
|
||||
ggml_backend_buffer_free(model.buffer);
|
||||
ggml_backend_free(model.backend);
|
||||
ggml_gallocr_free(allocr);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
#include "ggml-metal.h"
|
||||
#endif
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
|
||||
(void) level;
|
||||
(void) user_data;
|
||||
fputs(text, stderr);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
struct test_model {
|
||||
struct ggml_tensor * a;
|
||||
struct ggml_tensor * b;
|
||||
ggml_backend_t backend = NULL;
|
||||
ggml_backend_buffer_t buffer;
|
||||
struct ggml_context * ctx;
|
||||
};
|
||||
|
||||
void load_model(test_model & model, bool use_gpu = false) {
|
||||
// create data
|
||||
int K = 3, IC = 10, OC = 10;
|
||||
int IL = 8, N = 1;
|
||||
|
||||
// Initialize adata
|
||||
std::vector<float> adata(K * IC * OC);
|
||||
for (int i = 0; i < K * IC * OC; i++) {
|
||||
adata[i] = 4.5f;
|
||||
}
|
||||
|
||||
// Convert adata to fp16 format
|
||||
std::vector<ggml_fp16_t> hadata(K * IC * OC);
|
||||
ggml_fp32_to_fp16_row(adata.data(), hadata.data(), K * IC * OC);
|
||||
|
||||
// Initialize bdata
|
||||
std::vector<float> bdata(IL * IC * N);
|
||||
for (int i = 0; i < IL * IC * N; i++) {
|
||||
bdata[i] = 2.5f;
|
||||
}
|
||||
|
||||
size_t buffer_size = 0;
|
||||
{
|
||||
buffer_size += K * IC * OC * ggml_type_size(GGML_TYPE_F16); // tensor a
|
||||
buffer_size += IL * IC * N * ggml_type_size(GGML_TYPE_F32); // tensor b
|
||||
buffer_size += 1024; // overhead
|
||||
}
|
||||
|
||||
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
|
||||
printf("%s: backend buffer size = %0.2f MB\n", __func__, (buffer_size/ 1024.f/ 1024.f));
|
||||
|
||||
ggml_log_set(ggml_log_callback_default, nullptr);
|
||||
|
||||
int num_tensors = 2;
|
||||
struct ggml_init_params params {
|
||||
/*.mem_size =*/ ggml_tensor_overhead() * num_tensors,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
|
||||
// initialize the backend
|
||||
#ifdef GGML_USE_CUDA
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using CUDA backend\n", __func__);
|
||||
model.backend = ggml_backend_cuda_init(0);
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using Metal backend\n", __func__);
|
||||
model.backend = ggml_backend_metal_init();
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if(!model.backend) {
|
||||
// fallback to CPU backend
|
||||
model.backend = ggml_backend_cpu_init();
|
||||
}
|
||||
|
||||
model.buffer = ggml_backend_alloc_buffer(model.backend, buffer_size);
|
||||
|
||||
// create context
|
||||
model.ctx = ggml_init(params);
|
||||
|
||||
// create tensors
|
||||
model.a = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F16, K, IC, OC);
|
||||
model.b = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F32, IL, IC, N);
|
||||
|
||||
// create a allocator
|
||||
ggml_tallocr alloc = ggml_tallocr_new(model.buffer);
|
||||
|
||||
// alloc memory
|
||||
ggml_tallocr_alloc(&alloc, model.a);
|
||||
|
||||
// load data to buffer
|
||||
if(ggml_backend_is_cpu(model.backend)) {
|
||||
memcpy(model.a->data, hadata.data(), ggml_nbytes(model.a));
|
||||
} else {
|
||||
ggml_backend_tensor_set(model.a, hadata.data(), 0, ggml_nbytes(model.a));
|
||||
}
|
||||
|
||||
// alloc memory
|
||||
ggml_tallocr_alloc(&alloc, model.b);
|
||||
|
||||
if(ggml_backend_is_cpu(model.backend)
|
||||
#ifdef GGML_USE_METAL
|
||||
|| ggml_backend_is_metal(model.backend)
|
||||
#endif
|
||||
) {
|
||||
memcpy(model.b->data, bdata.data(), ggml_nbytes(model.b));
|
||||
} else {
|
||||
ggml_backend_tensor_set(model.b, bdata.data(), 0, ggml_nbytes(model.b));
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_cgraph * build_graph(const test_model& model) {
|
||||
static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
|
||||
static std::vector<uint8_t> buf(buf_size);
|
||||
|
||||
struct ggml_init_params params0 = {
|
||||
/*.mem_size =*/ buf_size,
|
||||
/*.mem_buffer =*/ buf.data(),
|
||||
/*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()
|
||||
};
|
||||
|
||||
// create a temporally context to build the graph
|
||||
struct ggml_context * ctx0 = ggml_init(params0);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx0);
|
||||
|
||||
int s0 = 1;
|
||||
int p0 = 1;
|
||||
int d0 = 1;
|
||||
|
||||
// split conv1d in fundamental methods for test unit
|
||||
struct ggml_tensor* im2col_0 = ggml_im2col(ctx0, model.a, model.b, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F16);
|
||||
ggml_set_name(im2col_0, "im2col_res");
|
||||
ggml_build_forward_expand(gf, im2col_0);
|
||||
|
||||
struct ggml_tensor* conv1d_res = ggml_conv_1d(ctx0, model.a, model.b, s0, p0, d0);
|
||||
ggml_set_name(conv1d_res, "conv1d_res");
|
||||
ggml_build_forward_expand(gf, conv1d_res);
|
||||
|
||||
// delete the temporally context used to build the graph
|
||||
ggml_free(ctx0);
|
||||
return gf;
|
||||
}
|
||||
|
||||
struct ggml_cgraph* compute_graph(const test_model & model, ggml_gallocr_t allocr) {
|
||||
struct ggml_cgraph * gf = build_graph(model);
|
||||
|
||||
// allocate tensors
|
||||
ggml_gallocr_alloc_graph(allocr, gf);
|
||||
int n_threads = 1;
|
||||
|
||||
if (ggml_backend_is_cpu(model.backend)) {
|
||||
ggml_backend_cpu_set_n_threads(model.backend, n_threads);
|
||||
}
|
||||
|
||||
ggml_backend_graph_compute(model.backend, gf);
|
||||
|
||||
//ggml_graph_print(gf);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
ggml_time_init();
|
||||
|
||||
test_model model;
|
||||
load_model(model, true);
|
||||
|
||||
ggml_gallocr_t allocr = NULL;
|
||||
|
||||
{
|
||||
allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
|
||||
|
||||
//create the worst case graph for memory usage estimation
|
||||
struct ggml_cgraph * gf = build_graph(model);
|
||||
|
||||
// compute the required memory
|
||||
ggml_gallocr_reserve(allocr, gf);
|
||||
size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);
|
||||
fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0f/1024.0f);
|
||||
}
|
||||
|
||||
struct ggml_cgraph * gf_res = compute_graph(model, allocr);
|
||||
|
||||
struct ggml_tensor * im2col_res = NULL;
|
||||
struct ggml_tensor * conv1d_res = NULL;
|
||||
|
||||
for(int i = 0; i < ggml_graph_n_nodes(gf_res); i++) {
|
||||
if(strcmp(ggml_get_name(ggml_graph_node(gf_res, i)), "im2col_res") == 0) {
|
||||
im2col_res = ggml_graph_node(gf_res, i);
|
||||
} else if(strcmp(ggml_get_name(ggml_graph_node(gf_res, i)), "conv1d_res") == 0) {
|
||||
conv1d_res = ggml_graph_node(gf_res, i);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<uint16_t> im2col_data(ggml_nelements(im2col_res));
|
||||
std::vector<float> conv2d_data(ggml_nelements(conv1d_res));
|
||||
|
||||
ggml_backend_tensor_get(im2col_res, im2col_data.data(), 0, ggml_nbytes(im2col_res));
|
||||
ggml_backend_tensor_get(conv1d_res, conv2d_data.data(), 0, ggml_nbytes(conv1d_res));
|
||||
|
||||
const int n_conv1d_test = 80;
|
||||
const int n_im2col_test = 240;
|
||||
|
||||
float expected_conv1d[n_conv1d_test] = {
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f
|
||||
};
|
||||
// first im2col test
|
||||
|
||||
uint16_t expected_im2col[n_conv1d_test] = {
|
||||
0, 16640, 16640, 0, 16640, 16640, 0, 16640,
|
||||
16640, 0, 16640, 16640, 0, 16640, 16640, 0,
|
||||
16640, 16640, 0, 16640, 16640, 0, 16640, 16640,
|
||||
0, 16640, 16640, 0, 16640, 16640, 16640, 16640,
|
||||
16640, 16640, 16640, 16640, 16640, 16640, 16640, 16640,
|
||||
16640, 16640, 16640, 16640, 16640, 16640, 16640, 16640,
|
||||
16640, 16640, 16640, 16640, 16640, 16640, 16640, 16640,
|
||||
16640, 16640, 16640, 16640, 16640, 16640, 16640, 16640,
|
||||
16640, 16640, 16640, 16640, 16640, 16640, 16640, 16640,
|
||||
16640, 16640, 16640, 16640, 16640, 16640, 16640, 16640
|
||||
};
|
||||
|
||||
printf("\nPerforming test:\n");
|
||||
|
||||
bool passed = true;
|
||||
for(int i = 0; i < n_conv1d_test; i++) {
|
||||
if(
|
||||
im2col_data[i] != expected_im2col[i]) {
|
||||
passed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
printf("ggml_im2col (%d): %s\n", (int) ggml_nelements(im2col_res), passed && (ggml_nelements(im2col_res) == n_im2col_test) ? "\033[32mPASSED\033[0m" : "\033[31mFAILED\033[0m");
|
||||
|
||||
passed = true;
|
||||
for(int i = 0; i < n_conv1d_test; i++) {
|
||||
if(conv2d_data[i] != expected_conv1d[i]) {
|
||||
passed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
printf("ggml_conv1d (%d): %s\n", (int) ggml_nelements(conv1d_res), passed && (ggml_nelements(conv1d_res) == n_conv1d_test) ? "\033[32mPASSED\033[0m" : "\033[31mFAILED\033[0m");
|
||||
ggml_free(model.ctx);
|
||||
|
||||
ggml_backend_buffer_free(model.buffer);
|
||||
ggml_backend_free(model.backend);
|
||||
ggml_gallocr_free(allocr);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
#include <ggml.h>
|
||||
#include <ggml-cpu.h>
|
||||
#include <ggml-alloc.h>
|
||||
#include <ggml-backend.h>
|
||||
#include <ggml-cpp.h>
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
std::vector<float> f32_range(int n, float start, float end) {
|
||||
std::vector<float> values(n);
|
||||
float step = (end - start) / n;
|
||||
for (int i = 0; i < n; i++) {
|
||||
values[i] = start + i * step;
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
// Most straightforward implementation without any optimizations
|
||||
std::vector<float> conv_2d_dw_reference(
|
||||
int src_w, int src_h, const float * src_data,
|
||||
int knl_w, int knl_h, const float * knl_data,
|
||||
int channels, int batch, int stride, int pad, int dilation) {
|
||||
|
||||
int dst_w = (src_w + 2 * pad - dilation * (knl_w - 1) - 1) / stride + 1;
|
||||
int dst_h = (src_h + 2 * pad - dilation * (knl_h - 1) - 1) / stride + 1;
|
||||
std::vector<float> dst_data(dst_w * dst_h * channels * batch);
|
||||
|
||||
for (int b = 0; b < batch; b++) {
|
||||
const float * src_base = src_data + b * src_w * src_h * channels;
|
||||
float * dst_base = dst_data.data() + b * dst_w * dst_h * channels;
|
||||
for (int c = 0; c < channels; c++) {
|
||||
for (int y = 0; y < dst_h; y++) {
|
||||
for (int x = 0; x < dst_w; x++) {
|
||||
float sum = 0;
|
||||
for (int knl_y = 0; knl_y < knl_h; knl_y++) {
|
||||
for (int knl_x = 0; knl_x < knl_w; knl_x++) {
|
||||
int src_x = x * stride + knl_x * dilation - pad;
|
||||
int src_y = y * stride + knl_y * dilation - pad;
|
||||
if (src_x >= 0 && src_x < src_w && src_y >= 0 && src_y < src_h) {
|
||||
sum += src_base[c * src_w * src_h + src_y * src_w + src_x] *
|
||||
knl_data[c * knl_w * knl_h + knl_y * knl_w + knl_x];
|
||||
}
|
||||
}
|
||||
}
|
||||
dst_base[c * dst_w * dst_h + y * dst_w + x] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return dst_data;
|
||||
}
|
||||
|
||||
bool check_equal(const std::vector<float> & result, const std::vector<float> & expected) {
|
||||
if (result.size() != expected.size()) {
|
||||
printf("result.size() = %d, expected.size() = %d\n", (int)result.size(), (int)expected.size());
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < result.size(); i++) {
|
||||
if(std::abs(result[i] - expected[i]) > 1e-5) {
|
||||
printf("result[%d] %f != %f expected[%d]\n", i, result[i], expected[i], i);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool test_conv_2d_dw(
|
||||
int channels,
|
||||
int kernel_size,
|
||||
int stride,
|
||||
int pad,
|
||||
int dilation,
|
||||
bool contiguous_channels) {
|
||||
ggml_time_init();
|
||||
|
||||
const int batch = 2;
|
||||
const int src_w = 8;
|
||||
const int src_h = 6;
|
||||
const int knl_w = kernel_size;
|
||||
const int knl_h = kernel_size;
|
||||
|
||||
ggml_init_params params {
|
||||
/*.mem_size =*/ 64 * ggml_tensor_overhead() + ggml_graph_overhead(),
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true
|
||||
};
|
||||
|
||||
ggml_context_ptr ctx_ptr{ggml_init(params)};
|
||||
ggml_context * ctx = ctx_ptr.get();
|
||||
ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
|
||||
// Build graph
|
||||
ggml_tensor * src_input = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, src_w, src_h, channels, batch);
|
||||
ggml_tensor * knl_input = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, knl_w, knl_h, 1, channels);
|
||||
ggml_tensor * src = src_input;
|
||||
ggml_tensor * knl = knl_input;
|
||||
if (contiguous_channels) {
|
||||
// Convert tensor to [C, W, H, N] layout in memory, then permute strides back to [W, H, C, N]
|
||||
src = ggml_cont(ctx, ggml_permute(ctx, src, 1, 2, 0, 3));
|
||||
src = ggml_permute(ctx, src, 2, 0, 1, 3);
|
||||
knl = ggml_cont(ctx, ggml_permute(ctx, knl, 2, 3, 1, 0));
|
||||
knl = ggml_permute(ctx, knl, 3, 2, 0, 1);
|
||||
}
|
||||
ggml_tensor * res = ggml_conv_2d_dw_direct(
|
||||
ctx, knl, src, stride, stride, pad, pad, dilation, dilation);
|
||||
if (contiguous_channels) {
|
||||
res = ggml_cont(ctx, res);
|
||||
}
|
||||
ggml_build_forward_expand(gf, res);
|
||||
|
||||
// Create backend & allocate buffers
|
||||
ggml_backend_ptr backend_ptr{ggml_backend_cpu_init()};
|
||||
ggml_backend_t backend = backend_ptr.get();
|
||||
ggml_backend_cpu_set_n_threads(backend, 2);
|
||||
ggml_backend_buffer_ptr buffer{ggml_backend_alloc_ctx_tensors(ctx, backend)};
|
||||
|
||||
std::vector<float> src_values = f32_range(ggml_nelements(src), -1.f, 1.f);
|
||||
std::vector<float> knl_values = f32_range(ggml_nelements(knl), -1.f, 1.f);
|
||||
ggml_backend_tensor_set(src_input, src_values.data(), 0, ggml_nbytes(src));
|
||||
ggml_backend_tensor_set(knl_input, knl_values.data(), 0, ggml_nbytes(knl));
|
||||
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
|
||||
std::vector<float> res_values(ggml_nelements(res));
|
||||
ggml_backend_tensor_get(res, res_values.data(), 0, ggml_nbytes(res));
|
||||
|
||||
std::vector<float> expected = conv_2d_dw_reference(
|
||||
src_w, src_h, src_values.data(),
|
||||
knl_w, knl_h, knl_values.data(),
|
||||
channels, batch, stride, pad, dilation);
|
||||
|
||||
bool passed = check_equal(res_values, expected);
|
||||
|
||||
printf("ggml_conv_2d_dw(channels=%d, kernel=%dx%d, stride=%d, pad=%d, dilation=%d, layout=%s): %s\n",
|
||||
channels, kernel_size, kernel_size, stride, pad, dilation, contiguous_channels ? "CWHN" : "WHCN",
|
||||
passed ? "\033[32mPASSED\033[0m" : "\033[31mFAILED\033[0m");
|
||||
return passed;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
bool passed = true;
|
||||
passed = test_conv_2d_dw(3, 1, 1, 0, 1, false) && passed;
|
||||
passed = test_conv_2d_dw(3, 1, 1, 0, 1, true) && passed;
|
||||
passed = test_conv_2d_dw(42, 3, 2, 1, 1, false) && passed;
|
||||
passed = test_conv_2d_dw(42, 3, 2, 1, 1, true) && passed;
|
||||
passed = test_conv_2d_dw(8, 5, 1, 2, 2, false) && passed;
|
||||
passed = test_conv_2d_dw(8, 5, 1, 2, 2, true) && passed;
|
||||
return passed ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
#include "ggml-metal.h"
|
||||
#endif
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
|
||||
(void) level;
|
||||
(void) user_data;
|
||||
fputs(text, stderr);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
struct test_model {
|
||||
struct ggml_tensor * a;
|
||||
struct ggml_tensor * b;
|
||||
ggml_backend_t backend = NULL;
|
||||
ggml_backend_buffer_t buffer;
|
||||
struct ggml_context * ctx;
|
||||
};
|
||||
|
||||
void load_model(test_model & model, bool use_gpu = false) {
|
||||
// create data
|
||||
int KW = 3, KH = 3, IC = 10, OC = 10;
|
||||
int IW = 8, IH = 6, N = 1;
|
||||
|
||||
// Initialize adata
|
||||
std::vector<float> adata(KW * KH * IC * OC);
|
||||
for (int i = 0; i < KW * KH * IC * OC; i++) {
|
||||
adata[i] = 2.5f;
|
||||
}
|
||||
|
||||
// Convert adata to fp16 format
|
||||
std::vector<ggml_fp16_t> hadata(KW * KH * IC * OC);
|
||||
ggml_fp32_to_fp16_row(adata.data(), hadata.data(), KW * KH * IC * OC);
|
||||
|
||||
// Initialize bdata
|
||||
std::vector<float> bdata(IW * IH * IC * N);
|
||||
for (int i = 0; i < IW * IH * IC * N; i++) {
|
||||
bdata[i] = 1.5f;
|
||||
}
|
||||
|
||||
size_t buffer_size = 0;
|
||||
{
|
||||
buffer_size += KW * KH * IC * OC * ggml_type_size(GGML_TYPE_F16); // tensor a
|
||||
buffer_size += IW * IH * IC * N * ggml_type_size(GGML_TYPE_F32); // tensor b
|
||||
buffer_size += 1024; // overhead
|
||||
}
|
||||
|
||||
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
|
||||
printf("%s: backend buffer size = %0.2f MB\n", __func__, (buffer_size/ 1024.f/ 1024.f));
|
||||
|
||||
int num_tensors = 2;
|
||||
struct ggml_init_params params {
|
||||
/*.mem_size =*/ ggml_tensor_overhead() * num_tensors,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
|
||||
ggml_log_set(ggml_log_callback_default, nullptr);
|
||||
|
||||
// initialize the backend
|
||||
#ifdef GGML_USE_CUDA
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using CUDA backend\n", __func__);
|
||||
model.backend = ggml_backend_cuda_init(0);
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using Metal backend\n", __func__);
|
||||
model.backend = ggml_backend_metal_init();
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if(!model.backend) {
|
||||
// fallback to CPU backend
|
||||
model.backend = ggml_backend_cpu_init();
|
||||
}
|
||||
|
||||
model.buffer = ggml_backend_alloc_buffer(model.backend, buffer_size);
|
||||
|
||||
// create context
|
||||
model.ctx = ggml_init(params);
|
||||
|
||||
// create tensors
|
||||
model.a = ggml_new_tensor_4d(model.ctx, GGML_TYPE_F16, KW, KH, IC, OC);
|
||||
model.b = ggml_new_tensor_4d(model.ctx, GGML_TYPE_F32, IW, IH, IC, N);
|
||||
|
||||
// create a allocator
|
||||
struct ggml_tallocr alloc = ggml_tallocr_new(model.buffer);
|
||||
|
||||
// alloc memory
|
||||
ggml_tallocr_alloc(&alloc, model.a);
|
||||
|
||||
// load data to buffer
|
||||
if(ggml_backend_is_cpu(model.backend)) {
|
||||
memcpy(model.a->data, hadata.data(), ggml_nbytes(model.a));
|
||||
} else {
|
||||
ggml_backend_tensor_set(model.a, hadata.data(), 0, ggml_nbytes(model.a));
|
||||
}
|
||||
|
||||
// alloc memory
|
||||
ggml_tallocr_alloc(&alloc, model.b);
|
||||
|
||||
if(ggml_backend_is_cpu(model.backend)
|
||||
#ifdef GGML_USE_METAL
|
||||
|| ggml_backend_is_metal(model.backend)
|
||||
#endif
|
||||
) {
|
||||
memcpy(model.b->data, bdata.data(), ggml_nbytes(model.b));
|
||||
} else {
|
||||
ggml_backend_tensor_set(model.b, bdata.data(), 0, ggml_nbytes(model.b));
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_cgraph * build_graph(const test_model& model) {
|
||||
static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
|
||||
static std::vector<uint8_t> buf(buf_size);
|
||||
|
||||
struct ggml_init_params params0 = {
|
||||
/*.mem_size =*/ buf_size,
|
||||
/*.mem_buffer =*/ buf.data(),
|
||||
/*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()
|
||||
};
|
||||
|
||||
// create a temporally context to build the graph
|
||||
struct ggml_context * ctx0 = ggml_init(params0);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx0);
|
||||
|
||||
int s0 = 1;
|
||||
int s1 = 1;
|
||||
int p0 = 1;
|
||||
int p1 = 1;
|
||||
int d0 = 1;
|
||||
int d1 = 1;
|
||||
|
||||
// split conv2d in fundamental methods for test unit
|
||||
struct ggml_tensor* im2col_0 = ggml_im2col(ctx0, model.a, model.b, s0, s1, p0, p1, d0, d1, true, GGML_TYPE_F16);
|
||||
ggml_set_name(im2col_0, "im2col_res");
|
||||
ggml_build_forward_expand(gf, im2col_0);
|
||||
|
||||
// recalculate for avoid fragmentation
|
||||
struct ggml_tensor* conv2d_res = ggml_conv_2d(ctx0, model.a, model.b, s0, s1, p0, p1, d0, d1);
|
||||
ggml_set_name(conv2d_res, "conv2d_res");
|
||||
ggml_build_forward_expand(gf, conv2d_res);
|
||||
|
||||
ggml_free(ctx0);
|
||||
return gf;
|
||||
}
|
||||
|
||||
struct ggml_cgraph * compute_graph(const test_model & model, ggml_gallocr_t allocr) {
|
||||
struct ggml_cgraph * gf = build_graph(model);
|
||||
|
||||
// allocate tensors
|
||||
ggml_gallocr_alloc_graph(allocr, gf);
|
||||
int n_threads = 1;
|
||||
|
||||
if (ggml_backend_is_cpu(model.backend)) {
|
||||
ggml_backend_cpu_set_n_threads(model.backend, n_threads);
|
||||
}
|
||||
|
||||
ggml_backend_graph_compute(model.backend, gf);
|
||||
|
||||
//ggml_graph_print(gf);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
ggml_time_init();
|
||||
|
||||
test_model model;
|
||||
load_model(model, true);
|
||||
|
||||
ggml_gallocr_t allocr = NULL;
|
||||
|
||||
{
|
||||
allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
|
||||
|
||||
//create the worst case graph for memory usage estimation
|
||||
struct ggml_cgraph * gf = build_graph(model);
|
||||
|
||||
// compute the required memory
|
||||
ggml_gallocr_reserve(allocr, gf);
|
||||
size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);
|
||||
fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0f/1024.0f);
|
||||
}
|
||||
|
||||
struct ggml_cgraph * gf_res = compute_graph(model, allocr);
|
||||
|
||||
struct ggml_tensor * im2col_res = NULL;
|
||||
struct ggml_tensor * conv2d_res = NULL;
|
||||
|
||||
for(int i = 0; i < ggml_graph_n_nodes(gf_res); ++i) {
|
||||
if(strcmp(ggml_get_name(ggml_graph_node(gf_res, i)), "im2col_res") == 0) {
|
||||
im2col_res = ggml_graph_node(gf_res, i);
|
||||
} else if(strcmp(ggml_get_name(ggml_graph_node(gf_res, i)), "conv2d_res") == 0) {
|
||||
conv2d_res = ggml_graph_node(gf_res, i);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<uint16_t> im2col_data(ggml_nelements(im2col_res));
|
||||
std::vector<float> conv2d_data(ggml_nelements(conv2d_res));
|
||||
|
||||
ggml_backend_tensor_get(im2col_res, im2col_data.data(), 0, ggml_nbytes(im2col_res));
|
||||
ggml_backend_tensor_get(conv2d_res, conv2d_data.data(), 0, ggml_nbytes(conv2d_res));
|
||||
|
||||
const int n_conv2d_test = 480;
|
||||
const int n_im2col_test = 4320;
|
||||
|
||||
float expected_conv2d [n_conv2d_test] = {
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
225.00f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 337.50f, 225.00f,
|
||||
150.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 225.00f, 150.00f };
|
||||
|
||||
uint16_t expected_im2col[n_conv2d_test] = {
|
||||
0, 0, 0, 0, 15872, 15872, 0, 15872,
|
||||
15872, 0, 0, 0, 0, 15872, 15872, 0,
|
||||
15872, 15872, 0, 0, 0, 0, 15872, 15872,
|
||||
0, 15872, 15872, 0, 0, 0, 0, 15872,
|
||||
15872, 0, 15872, 15872, 0, 0, 0, 0,
|
||||
15872, 15872, 0, 15872, 15872, 0, 0, 0,
|
||||
0, 15872, 15872, 0, 15872, 15872, 0, 0,
|
||||
0, 0, 15872, 15872, 0, 15872, 15872, 0,
|
||||
0, 0, 0, 15872, 15872, 0, 15872, 15872,
|
||||
0, 0, 0, 0, 15872, 15872, 0, 15872,
|
||||
15872, 0, 0, 0, 0, 15872, 15872, 0,
|
||||
15872, 15872, 0, 0, 0, 15872, 15872, 15872,
|
||||
15872, 15872, 15872, 0, 0, 0, 15872, 15872,
|
||||
15872, 15872, 15872, 15872, 0, 0, 0, 15872,
|
||||
15872, 15872, 15872, 15872, 15872, 0, 0, 0,
|
||||
15872, 15872, 15872, 15872, 15872, 15872, 0, 0,
|
||||
0, 15872, 15872, 15872, 15872, 15872, 15872, 0,
|
||||
0, 0, 15872, 15872, 15872, 15872, 15872, 15872,
|
||||
0, 0, 0, 15872, 15872, 15872, 15872, 15872,
|
||||
15872, 0, 0, 0, 15872, 15872, 15872, 15872,
|
||||
15872, 15872, 0, 0, 0, 15872, 15872, 15872,
|
||||
15872, 15872, 15872, 0, 0, 0, 15872, 15872,
|
||||
15872, 15872, 15872, 15872, 0, 0, 0, 15872,
|
||||
15872, 15872, 15872, 15872, 15872, 0, 0, 0,
|
||||
15872, 15872, 15872, 15872, 15872, 15872, 0, 0,
|
||||
0, 15872, 15872, 15872, 15872, 15872, 15872, 0,
|
||||
0, 0, 15872, 15872, 15872, 15872, 15872, 15872,
|
||||
0, 0, 0, 15872, 15872, 15872, 15872, 15872,
|
||||
15872, 0, 0, 0, 15872, 15872, 15872, 15872,
|
||||
15872, 15872, 0, 0, 0, 15872, 15872, 15872,
|
||||
15872, 15872, 15872, 0, 0, 0, 15872, 15872,
|
||||
15872, 15872, 15872, 15872, 0, 0, 0, 15872,
|
||||
15872, 15872, 15872, 15872, 15872, 0, 0, 0,
|
||||
15872, 15872, 15872, 15872, 15872, 15872, 0, 0,
|
||||
0, 15872, 15872, 15872, 15872, 15872, 15872, 0,
|
||||
0, 0, 15872, 15872, 15872, 15872, 15872, 15872,
|
||||
0, 0, 0, 15872, 15872, 15872, 15872, 15872,
|
||||
15872, 0, 0, 0, 15872, 15872, 15872, 15872,
|
||||
15872, 15872, 0, 0, 0, 15872, 15872, 15872,
|
||||
15872, 15872, 15872, 0, 0, 0, 15872, 15872,
|
||||
15872, 15872, 15872, 15872, 0, 0, 0, 15872,
|
||||
15872, 15872, 15872, 15872, 15872, 0, 0, 0,
|
||||
15872, 15872, 15872, 15872, 15872, 15872, 0, 0,
|
||||
0, 15872, 15872, 15872, 15872, 15872, 15872, 0,
|
||||
0, 0, 15872, 15872, 15872, 15872, 15872, 15872,
|
||||
0, 0, 0, 15872, 15872, 15872, 15872, 15872,
|
||||
15872, 0, 0, 0, 15872, 15872, 15872, 15872,
|
||||
15872, 15872, 0, 0, 0, 15872, 15872, 15872,
|
||||
15872, 15872, 15872, 0, 0, 0, 15872, 15872,
|
||||
15872, 15872, 15872, 15872, 0, 0, 0, 15872,
|
||||
15872, 15872, 15872, 15872, 15872, 0, 0, 0,
|
||||
15872, 15872, 15872, 15872, 15872, 15872, 0, 0,
|
||||
0, 15872, 15872, 15872, 15872, 15872, 15872, 0,
|
||||
0, 0, 15872, 15872, 15872, 15872, 15872, 15872,
|
||||
0, 0, 0, 15872, 15872, 15872, 15872, 15872,
|
||||
15872, 0, 0, 0, 15872, 15872, 15872, 15872,
|
||||
15872, 15872, 0, 0, 0, 15872, 15872, 15872,
|
||||
15872, 15872, 15872, 0, 0, 0, 15872, 15872,
|
||||
15872, 15872, 15872, 15872, 0, 0, 0, 15872,
|
||||
15872, 15872, 15872, 15872, 15872, 0, 0, 0
|
||||
};
|
||||
|
||||
printf("\nPerforming test:\n");
|
||||
|
||||
bool passed = true;
|
||||
for(int i = 0; i < n_conv2d_test; i++) {
|
||||
if(
|
||||
im2col_data[i] != expected_im2col[i]) {
|
||||
passed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
printf("ggml_im2col (%d): %s\n", (int) ggml_nelements(im2col_res), passed && (ggml_nelements(im2col_res) == n_im2col_test) ? "\033[32mPASSED\033[0m" : "\033[31mFAILED\033[0m");
|
||||
|
||||
passed = true;
|
||||
for(int i = 0; i < n_conv2d_test; i++) {
|
||||
if(conv2d_data[i] != expected_conv2d[i]) {
|
||||
passed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
printf("ggml_conv2d (%d): %s\n", (int) ggml_nelements(conv2d_res), passed && (ggml_nelements(conv2d_res) == n_conv2d_test) ? "\033[32mPASSED\033[0m" : "\033[31mFAILED\033[0m");
|
||||
|
||||
ggml_free(model.ctx);
|
||||
|
||||
ggml_backend_buffer_free(model.buffer);
|
||||
ggml_backend_free(model.backend);
|
||||
ggml_gallocr_free(allocr);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include <windows.h>
|
||||
typedef volatile LONG atomic_int;
|
||||
static LONG atomic_fetch_add(atomic_int * ptr, LONG inc) {
|
||||
return InterlockedExchangeAdd(ptr, inc);
|
||||
}
|
||||
#else
|
||||
#include <stdatomic.h>
|
||||
#endif
|
||||
|
||||
#define MIN(a, b) ((a) < (b) ? (a) : (b))
|
||||
#define MAX(a, b) ((a) > (b) ? (a) : (b))
|
||||
|
||||
struct ggml_context * make_ctx(void) {
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ 1 * 1024 * 1024,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ false,
|
||||
};
|
||||
|
||||
return ggml_init(params);
|
||||
}
|
||||
|
||||
char g_userdata[] = "ggml";
|
||||
atomic_int g_custom1_count = 0;
|
||||
atomic_int g_custom2_count = 0;
|
||||
atomic_int g_custom3_count = 0;
|
||||
|
||||
void custom1(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata) {
|
||||
// check that the userdata is correct
|
||||
GGML_ASSERT(userdata == NULL);
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, a));
|
||||
|
||||
atomic_fetch_add(&g_custom1_count, 1);
|
||||
|
||||
const float * a_data = ggml_get_data_f32(a);
|
||||
float * dst_data = ggml_get_data_f32(dst);
|
||||
|
||||
// this assumes that the tensors are contiguous
|
||||
GGML_ASSERT(ggml_is_contiguous(dst));
|
||||
GGML_ASSERT(ggml_is_contiguous(a));
|
||||
|
||||
// parallelize by elements
|
||||
const int ne = (int)ggml_nelements(dst);
|
||||
const int dr = (ne + nth - 1) / nth;
|
||||
const int ie0 = dr * ith;
|
||||
const int ie1 = MIN(ie0 + dr, ne);
|
||||
|
||||
for (int i = ie0; i < ie1; ++i) {
|
||||
dst_data[i] = a_data[i] * 2;
|
||||
}
|
||||
}
|
||||
|
||||
void custom2(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata) {
|
||||
// check that the userdata is correct
|
||||
GGML_ASSERT(userdata == g_userdata);
|
||||
GGML_ASSERT(strcmp(userdata, "ggml") == 0);
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, a));
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, b));
|
||||
|
||||
atomic_fetch_add(&g_custom2_count, 1);
|
||||
|
||||
const float * a_data = ggml_get_data_f32(a);
|
||||
const float * b_data = ggml_get_data_f32(b);
|
||||
float * dst_data = ggml_get_data_f32(dst);
|
||||
|
||||
// parallelize by rows
|
||||
const int nr = (int)ggml_nrows(dst);
|
||||
// number of rows per thread
|
||||
const int dr = (nr + nth - 1) / nth;
|
||||
// row range for this thread
|
||||
const int ir0 = dr * ith;
|
||||
const int ir1 = MIN(ir0 + dr, nr);
|
||||
|
||||
// number of columns
|
||||
const int nc = (int)dst->ne[0];
|
||||
|
||||
// this assumes that the tensors are contiguous
|
||||
GGML_ASSERT(ggml_is_contiguous(dst));
|
||||
GGML_ASSERT(ggml_is_contiguous(a));
|
||||
GGML_ASSERT(ggml_is_contiguous(b));
|
||||
|
||||
for (int ir = ir0; ir < ir1; ++ir) {
|
||||
for (int ic = 0; ic < nc; ++ic) {
|
||||
const int i = ir * nc + ic;
|
||||
dst_data[i] = a_data[i] + b_data[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void custom3(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata) {
|
||||
// check that the userdata is correct
|
||||
GGML_ASSERT(userdata == g_userdata);
|
||||
GGML_ASSERT(strcmp(userdata, "ggml") == 0);
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, a));
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, b));
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, c));
|
||||
|
||||
atomic_fetch_add(&g_custom3_count, 1);
|
||||
|
||||
const float * a_data = ggml_get_data_f32(a);
|
||||
const float * b_data = ggml_get_data_f32(b);
|
||||
const float * c_data = ggml_get_data_f32(c);
|
||||
float * dst_data = ggml_get_data_f32(dst);
|
||||
|
||||
// dont parallelize
|
||||
GGML_ASSERT(ith == 0);
|
||||
|
||||
// number of elements
|
||||
const int ne = (int)ggml_nelements(dst);
|
||||
|
||||
// this assumes that the tensors are contiguous
|
||||
GGML_ASSERT(ggml_is_contiguous(dst));
|
||||
GGML_ASSERT(ggml_is_contiguous(a));
|
||||
GGML_ASSERT(ggml_is_contiguous(b));
|
||||
GGML_ASSERT(ggml_is_contiguous(c));
|
||||
|
||||
for (int i = 0; i < ne; ++i) {
|
||||
dst_data[i] = a_data[i] + b_data[i] + c_data[i];
|
||||
}
|
||||
}
|
||||
|
||||
void custom(struct ggml_tensor * dst, int ith, int nth, void * userdata) {
|
||||
struct ggml_tensor * src0 = dst->src[0];
|
||||
struct ggml_tensor * src1 = dst->src[1];
|
||||
struct ggml_tensor * src2 = dst->src[2];
|
||||
struct ggml_tensor * src3 = dst->src[3];
|
||||
struct ggml_tensor * src4 = dst->src[4];
|
||||
|
||||
int32_t * dst_data = (int32_t *) ggml_get_data(dst);
|
||||
const float * src0_data = ggml_get_data_f32(src0);
|
||||
const float * src1_data = ggml_get_data_f32(src1);
|
||||
const float * src2_data = ggml_get_data_f32(src2);
|
||||
const float * src3_data = ggml_get_data_f32(src3);
|
||||
const float * src4_data = ggml_get_data_f32(src4);
|
||||
|
||||
// check that the userdata is correct
|
||||
GGML_ASSERT(userdata == g_userdata);
|
||||
GGML_ASSERT(strcmp(userdata, "ggml") == 0);
|
||||
|
||||
// check that the tensors are contiguous
|
||||
GGML_ASSERT(ggml_is_contiguous(dst));
|
||||
GGML_ASSERT(ggml_is_contiguous(src0));
|
||||
GGML_ASSERT(ggml_is_contiguous(src1));
|
||||
GGML_ASSERT(ggml_is_contiguous(src2));
|
||||
GGML_ASSERT(ggml_is_contiguous(src3));
|
||||
GGML_ASSERT(ggml_is_contiguous(src4));
|
||||
|
||||
// check that the shapes are the same
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, src0));
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, src1));
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, src2));
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, src3));
|
||||
GGML_ASSERT(ggml_are_same_shape(dst, src4));
|
||||
|
||||
|
||||
for (int i = ith; i < ggml_nelements(dst); i += nth) {
|
||||
dst_data[i] = src0_data[i] + src1_data[i] * src2_data[i] - src3_data[i] * src4_data[i];
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv) {
|
||||
|
||||
float buf1_f32[1024];
|
||||
for (int i = 0; i < 1024; ++i) {
|
||||
buf1_f32[i] = (float)(i + 1);
|
||||
}
|
||||
float buf2_f32[1024];
|
||||
for (int i = 0; i < 1024; ++i) {
|
||||
buf2_f32[i] = (float)(i + 1) * 2;
|
||||
}
|
||||
float buf3_f32[1024];
|
||||
for (int i = 0; i < 1024; ++i) {
|
||||
buf3_f32[i] = (float)(i + 1) * 3;
|
||||
}
|
||||
|
||||
// map_custom1
|
||||
// 2 tasks, no userdata, parallelized by elements
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
memcpy(t->data, buf1_f32, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * m1 = ggml_map_custom1(ctx, t, custom1, 2, NULL);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, m1);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(m1);
|
||||
|
||||
for (int i = 0; i < ggml_nelements(m1); ++i) {
|
||||
GGML_ASSERT(output[i] == buf1_f32[i] * 2);
|
||||
}
|
||||
GGML_ASSERT(g_custom1_count == 2);
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
// map_custom2
|
||||
// max tasks (4), userdata, parallelized by rows
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
struct ggml_tensor * t2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
memcpy(t1->data, buf1_f32, ggml_nbytes(t1));
|
||||
memcpy(t2->data, buf2_f32, ggml_nbytes(t2));
|
||||
|
||||
struct ggml_tensor * m2 = ggml_map_custom2(ctx, t1, t2, custom2, GGML_N_TASKS_MAX, g_userdata);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, m2);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(m2);
|
||||
|
||||
for (int i = 0; i < ggml_nelements(m2); ++i) {
|
||||
GGML_ASSERT(output[i] == buf1_f32[i] + buf2_f32[i]);
|
||||
}
|
||||
|
||||
GGML_ASSERT(g_custom2_count == 4);
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
// map_custom3
|
||||
// 1 task, userdata, not parallelized
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
struct ggml_tensor * t2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
struct ggml_tensor * t3 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
|
||||
memcpy(t1->data, buf1_f32, ggml_nbytes(t1));
|
||||
memcpy(t2->data, buf2_f32, ggml_nbytes(t2));
|
||||
memcpy(t3->data, buf3_f32, ggml_nbytes(t3));
|
||||
|
||||
struct ggml_tensor * m3 = ggml_map_custom3(ctx, t1, t2, t3, custom3, 1, g_userdata);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, m3);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(m3);
|
||||
|
||||
for (int i = 0; i < ggml_nelements(m3); ++i) {
|
||||
GGML_ASSERT(output[i] == buf1_f32[i] + buf2_f32[i] + buf3_f32[i]);
|
||||
}
|
||||
|
||||
GGML_ASSERT(g_custom3_count == 1);
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
// custom
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
struct ggml_tensor * t2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
struct ggml_tensor * t3 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
struct ggml_tensor * t4 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
struct ggml_tensor * t5 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
memcpy(t1->data, buf1_f32, ggml_nbytes(t1));
|
||||
memcpy(t2->data, buf2_f32, ggml_nbytes(t2));
|
||||
memcpy(t3->data, buf3_f32, ggml_nbytes(t3));
|
||||
memcpy(t4->data, buf1_f32, ggml_nbytes(t4));
|
||||
memcpy(t5->data, buf2_f32, ggml_nbytes(t5));
|
||||
|
||||
struct ggml_tensor * args[] = {
|
||||
t1, t2, t3, t4, t5,
|
||||
};
|
||||
|
||||
struct ggml_tensor * m4 = ggml_custom_4d(ctx, GGML_TYPE_I32, 10, 2, 1, 1, args, sizeof(args)/sizeof(args[0]), custom, GGML_N_TASKS_MAX, g_userdata);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, m4);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const int32_t * output = (const int32_t *) ggml_get_data(m4);
|
||||
|
||||
for (int i = 0; i < ggml_nelements(m4); ++i) {
|
||||
GGML_ASSERT(output[i] == buf1_f32[i] + buf2_f32[i] * buf3_f32[i] - buf1_f32[i] * buf2_f32[i]);
|
||||
}
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
void arange(struct ggml_tensor* tensor) {
|
||||
GGML_ASSERT(ggml_is_contiguous(tensor));
|
||||
for (int i = 0; i < ggml_nelements(tensor); ++i) {
|
||||
ggml_set_i32_1d(tensor, i, i);
|
||||
}
|
||||
}
|
||||
|
||||
void dup_to(struct ggml_tensor* src, struct ggml_tensor* dst) {
|
||||
GGML_ASSERT(dst->op == GGML_OP_VIEW);
|
||||
GGML_ASSERT(ggml_nelements(src) == ggml_nelements(dst));
|
||||
dst->op = GGML_OP_DUP;
|
||||
dst->src[0] = src;
|
||||
}
|
||||
|
||||
bool can_dup(enum ggml_type src_type, enum ggml_type dst_type) {
|
||||
if (src_type == dst_type) return true;
|
||||
if (src_type == GGML_TYPE_F32 && ggml_get_type_traits_cpu(dst_type)->from_float) return true;
|
||||
if (dst_type == GGML_TYPE_F32 && ggml_get_type_traits (src_type)->to_float) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
int main(int argc, const char ** argv) {
|
||||
struct ggml_init_params params = {
|
||||
.mem_size = 128*1024*1024,
|
||||
.mem_buffer = NULL,
|
||||
.no_alloc = false,
|
||||
};
|
||||
|
||||
enum ggml_type type[4] = {GGML_TYPE_I16, GGML_TYPE_I32, GGML_TYPE_F16, GGML_TYPE_F32};
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
enum ggml_type src_type = type[i];
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
enum ggml_type dst_type = type[j];
|
||||
if (!can_dup(src_type, dst_type)) continue;
|
||||
printf("Testing dup on %s -> %s copy\n", ggml_type_name(src_type), ggml_type_name(dst_type));
|
||||
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
struct ggml_tensor * src = ggml_new_tensor_2d(ctx, src_type, 10, 11);
|
||||
arange(src);
|
||||
struct ggml_tensor * dst = ggml_new_tensor_2d(ctx, dst_type, 10, 11);
|
||||
ggml_set_i32(dst, 0);
|
||||
|
||||
// 2nd-row: [20, 21, ..., 29]
|
||||
struct ggml_tensor * src_cont = ggml_view_1d(ctx, src, 10, src->nb[1] * 2);
|
||||
|
||||
// 3rd-col: [03, 13, ..., 93]
|
||||
struct ggml_tensor * src_stride = ggml_view_2d(ctx, src, 1, 10, src->nb[1], src->nb[0] * 3);
|
||||
|
||||
struct ggml_tensor * dst_cont_1 = ggml_view_1d(ctx, dst, 10, dst->nb[1] * 5); // 5nd-row
|
||||
struct ggml_tensor * dst_cont_2 = ggml_view_1d(ctx, dst, 10, dst->nb[1] * 6); // 6rd-row
|
||||
|
||||
struct ggml_tensor * dst_stride_1 = ggml_view_2d(ctx, dst, 1, 10, dst->nb[1], dst->nb[0] * 7); // 7th-col
|
||||
struct ggml_tensor * dst_stride_2 = ggml_view_2d(ctx, dst, 1, 10, dst->nb[1], dst->nb[0] * 8); // 8th-col
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
|
||||
dup_to(src_cont, dst_cont_1);
|
||||
dup_to(src_stride, dst_cont_2);
|
||||
dup_to(src_cont, dst_stride_1);
|
||||
dup_to(src_stride, dst_stride_2);
|
||||
|
||||
ggml_build_forward_expand(gf, dst_cont_1);
|
||||
ggml_build_forward_expand(gf, dst_cont_2);
|
||||
ggml_build_forward_expand(gf, dst_stride_1);
|
||||
ggml_build_forward_expand(gf, dst_stride_2);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, gf, 1);
|
||||
|
||||
// src_cont -> dst_cont_1
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 49) == 0);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 50) == 20);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 51) == 21);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 52) == 22);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 59) == 29);
|
||||
|
||||
// src_stride -> dst_cont_2
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 60) == 3);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 61) == 13);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 62) == 23);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 69) == 93);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 70) == 0);
|
||||
|
||||
// src_cont -> dst_stride_1
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 6) == 0);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 7) == 20);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 17) == 21);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 27) == 22);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 97) == 29);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 107) == 0);
|
||||
|
||||
// src_stride -> dst_stride_2
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 8) == 03);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 18) == 13);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 28) == 23);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 98) == 93);
|
||||
GGML_ASSERT(ggml_get_i32_1d(dst, 108) == 0);
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
#include <ggml.h>
|
||||
#include <ggml-cpu.h>
|
||||
#include <ggml-alloc.h>
|
||||
#include <ggml-backend.h>
|
||||
#include <ggml-cpp.h>
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <array>
|
||||
#include <vector>
|
||||
|
||||
bool check_equal(const float * result, const float * expected, int64_t n) {
|
||||
for (int i = 0; i < n; i++) {
|
||||
if(std::abs(result[i] - expected[i]) > 1e-4) {
|
||||
printf("result[%d] %f != %f expected[%d]\n", i, result[i], expected[i], i);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool test_interpolate(char const* name,
|
||||
std::array<int64_t, 4> src_ne, const float * src_data,
|
||||
std::array<int32_t, 4> dst_ne, const float * expected,
|
||||
uint32_t mode) {
|
||||
ggml_time_init();
|
||||
|
||||
ggml_init_params params {
|
||||
/*.mem_size =*/ 64 * ggml_tensor_overhead() + ggml_graph_overhead(),
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true
|
||||
};
|
||||
|
||||
ggml_context_ptr ctx_ptr{ggml_init(params)};
|
||||
ggml_context * ctx = ctx_ptr.get();
|
||||
ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
|
||||
// Build graph
|
||||
ggml_tensor * src = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, src_ne.data());
|
||||
ggml_tensor * res = ggml_interpolate(ctx, src, dst_ne[0], dst_ne[1], dst_ne[2], dst_ne[3], mode);
|
||||
ggml_build_forward_expand(gf, res);
|
||||
|
||||
// Create backend & allocate buffers
|
||||
ggml_backend_ptr backend_ptr{ggml_backend_cpu_init()};
|
||||
ggml_backend_t backend = backend_ptr.get();
|
||||
ggml_backend_cpu_set_n_threads(backend, 2);
|
||||
ggml_backend_buffer_ptr buffer{ggml_backend_alloc_ctx_tensors(ctx, backend)};
|
||||
|
||||
// Execute and compare results
|
||||
ggml_backend_tensor_set(src, src_data, 0, ggml_nbytes(src));
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
|
||||
std::vector<float> res_values(ggml_nelements(res));
|
||||
ggml_backend_tensor_get(res, res_values.data(), 0, ggml_nbytes(res));
|
||||
|
||||
bool passed = check_equal(res_values.data(), expected, ggml_nelements(res));
|
||||
|
||||
printf("ggml_interpolate(%s): %s\n", name, passed ? "\033[32mPASSED\033[0m" : "\033[31mFAILED\033[0m");
|
||||
return passed;
|
||||
}
|
||||
|
||||
const float input_upscale[] = {
|
||||
0.0f, 1.0f,
|
||||
2.0f, 4.0f
|
||||
};
|
||||
|
||||
const float expected_upscale_x2_nearest[] = {
|
||||
0.0f, 0.0f, 1.0f, 1.0f,
|
||||
0.0f, 0.0f, 1.0f, 1.0f,
|
||||
2.0f, 2.0f, 4.0f, 4.0f,
|
||||
2.0f, 2.0f, 4.0f, 4.0f
|
||||
};
|
||||
|
||||
const float expected_upscale_x2_bilinear[] = {
|
||||
0.0f, 0.2500f, 0.7500f, 1.00f,
|
||||
0.5f, 0.8125f, 1.4375f, 1.75f,
|
||||
1.5f, 1.9375f, 2.8125f, 3.25f,
|
||||
2.0f, 2.5000f, 3.5000f, 4.00f
|
||||
};
|
||||
|
||||
const float expected_upscale_x2_bilinear_align_corners[] = {
|
||||
0.0000f, 0.3333f, 0.6667f, 1.0000f,
|
||||
0.6667f, 1.1111f, 1.5556f, 2.0000f,
|
||||
1.3333f, 1.8889f, 2.4444f, 3.0000f,
|
||||
2.0000f, 2.6667f, 3.3333f, 4.0000f
|
||||
};
|
||||
|
||||
const float expected_upscale_x1_5_bilinear_align_corners[] = {
|
||||
0.0f, 1.0f,
|
||||
1.0f, 2.5f,
|
||||
2.0f, 4.0f
|
||||
};
|
||||
|
||||
const float input_downscale[] = {
|
||||
0.0f, -1.0f, -2.0f, 0.0f,
|
||||
1.0f, 2.0f , 4.0f , 4.0f,
|
||||
2.0f, 2.0f , 1.0f , 1.0f,
|
||||
|
||||
1.0f, 2.0f , 3.0f , 4.0f,
|
||||
2.0f, 2.0f , 2.0f , 2.0f,
|
||||
-2.0f, 2.0f, -4.0f, 4.0f
|
||||
};
|
||||
|
||||
const float expected_downscale_nearest[] = {
|
||||
0.0f, -2.0f,
|
||||
|
||||
1.0f, 3.0f
|
||||
};
|
||||
|
||||
const float expected_downscale_bilinear[] = {
|
||||
0.1667f, -0.3750f, 0.7500f,
|
||||
1.7917f, 1.8750f, 1.7500f,
|
||||
|
||||
1.3750f, 2.3750f, 3.3750f,
|
||||
-0.5000f, -0.2500f, 2.5000f
|
||||
};
|
||||
|
||||
const float expected_downscale_bilinear_align_corners[] = {
|
||||
0.0f , -1.5f, 0.0f,
|
||||
2.0f , 1.5f, 1.0f,
|
||||
|
||||
1.0f , 2.5f, 4.0f,
|
||||
-2.0f, -1.0f, 4.0f
|
||||
};
|
||||
|
||||
int main() {
|
||||
bool passed = true;
|
||||
|
||||
passed &= test_interpolate("upscale_x2_nearest",
|
||||
{2, 2, 1, 1}, input_upscale,
|
||||
{4, 4, 1, 1}, expected_upscale_x2_nearest,
|
||||
GGML_SCALE_MODE_NEAREST);
|
||||
|
||||
passed &= test_interpolate("upscale_x2_bilinear",
|
||||
{2, 2, 1, 1}, input_upscale,
|
||||
{4, 4, 1, 1}, expected_upscale_x2_bilinear,
|
||||
GGML_SCALE_MODE_BILINEAR);
|
||||
|
||||
passed &= test_interpolate("upscale_x2_bilinear_align_corners",
|
||||
{2, 2, 1, 1}, input_upscale,
|
||||
{4, 4, 1, 1}, expected_upscale_x2_bilinear_align_corners,
|
||||
GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS);
|
||||
|
||||
passed &= test_interpolate("upscale_x1_5_bilinear_align_corners",
|
||||
{2, 2, 1, 1}, input_upscale,
|
||||
{2, 3, 1, 1}, expected_upscale_x1_5_bilinear_align_corners,
|
||||
GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS);
|
||||
|
||||
passed &= test_interpolate("downscale_nearest",
|
||||
{4, 3, 2, 1}, input_downscale,
|
||||
{2, 1, 2, 1}, expected_downscale_nearest,
|
||||
GGML_SCALE_MODE_NEAREST);
|
||||
|
||||
passed &= test_interpolate("downscale_bilinear",
|
||||
{4, 3, 2, 1}, input_downscale,
|
||||
{3, 2, 2, 1}, expected_downscale_bilinear,
|
||||
GGML_SCALE_MODE_BILINEAR);
|
||||
|
||||
passed &= test_interpolate("downscale_bilinear_align_corners",
|
||||
{4, 3, 2, 1}, input_downscale,
|
||||
{3, 2, 2, 1}, expected_downscale_bilinear_align_corners,
|
||||
GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS);
|
||||
|
||||
return passed ? 0 : 1;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,213 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
#include "ggml-metal.h"
|
||||
#endif
|
||||
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
|
||||
static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
|
||||
(void) level;
|
||||
(void) user_data;
|
||||
fputs(text, stderr);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
struct ggml_context* make_ctx(void) {
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ 2 * 1024 * 1024,
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc. =*/ false
|
||||
};
|
||||
return ggml_init(params);
|
||||
}
|
||||
|
||||
void check_tensor(struct ggml_tensor * t, float * expected_t_d, int ne0, int ne1, int ne2) {
|
||||
GGML_ASSERT(t->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(t->ne[0] == ne0);
|
||||
GGML_ASSERT(t->ne[1] == ne1);
|
||||
GGML_ASSERT(t->ne[2] == ne2);
|
||||
for (int i2 = 0; i2 < ne2; ++i2) {
|
||||
for (int i1 = 0; i1 < ne1; ++i1) {
|
||||
for (int i0 = 0; i0 < ne0; ++i0) {
|
||||
float expected = *(expected_t_d + i2 * ne1 * ne0 + i1 * ne0 + i0);
|
||||
float actual = ggml_get_data_f32(t)[i2 * ne1 * ne0 + i1 * ne0 + i0];
|
||||
if (expected != actual) {
|
||||
printf("expected %.1f, got %.1f at (%d,%d,%d)\n", expected, actual, i0, i1, i2);
|
||||
}
|
||||
GGML_ASSERT(expected == actual);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void test_pad_reflect_1d(bool use_gpu) {
|
||||
ggml_backend_t backend = NULL;
|
||||
struct ggml_init_params params;
|
||||
ggml_backend_buffer_t buffer;
|
||||
struct ggml_context * ctx;
|
||||
struct ggml_tallocr tallocr;
|
||||
ggml_gallocr_t gallocr;
|
||||
|
||||
// initialize the backend
|
||||
#ifdef GGML_USE_CUDA
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using CUDA backend\n", __func__);
|
||||
backend = ggml_backend_cuda_init(0);
|
||||
if (!backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using Metal backend\n", __func__);
|
||||
backend = ggml_backend_metal_init();
|
||||
if (!backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (!backend) {
|
||||
fprintf(stderr, "%s: using CPU backend\n", __func__);
|
||||
backend = ggml_backend_cpu_init();
|
||||
}
|
||||
|
||||
// Test cases for different padding configurations
|
||||
{
|
||||
params = ggml_init_params{
|
||||
/*.mem_size =*/ 16*1024*1024,
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc. =*/ true
|
||||
};
|
||||
|
||||
ggml_log_set(ggml_log_callback_default, nullptr);
|
||||
|
||||
ctx = ggml_init(params);
|
||||
buffer = ggml_backend_alloc_buffer(backend, 16*1024*1024);
|
||||
tallocr = ggml_tallocr_new(buffer);
|
||||
gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));
|
||||
|
||||
// Create a simple 1D input tensor [1, 2, 3, 4]
|
||||
struct ggml_tensor * t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4);
|
||||
float input_data[] = {1.0f, 2.0f, 3.0f, 4.0f};
|
||||
ggml_tallocr_alloc(&tallocr, t);
|
||||
|
||||
// load data to buffer
|
||||
if(ggml_backend_is_cpu(backend)) {
|
||||
memcpy(t->data, input_data, ggml_nbytes(t));
|
||||
} else {
|
||||
ggml_backend_tensor_set(t, input_data, 0, ggml_nbytes(t));
|
||||
}
|
||||
|
||||
// Test case 1: pad left=1, right=1
|
||||
// Expected: [2, 1, 2, 3, 4, 3]
|
||||
float expected_1[] = {2.0f, 1.0f, 2.0f, 3.0f, 4.0f, 3.0f};
|
||||
struct ggml_tensor * out_1 = ggml_pad_reflect_1d(ctx, t, 1, 1);
|
||||
|
||||
// Test case 2: pad left=2, right=1
|
||||
// Expected: [3, 2, 1, 2, 3, 4, 3]
|
||||
float expected_2[] = {3.0f, 2.0f, 1.0f, 2.0f, 3.0f, 4.0f, 3.0f};
|
||||
struct ggml_tensor * out_2 = ggml_pad_reflect_1d(ctx, t, 2, 1);
|
||||
|
||||
// Test case 3: pad left=1, right=2
|
||||
// Expected: [2, 1, 2, 3, 4, 3, 2]
|
||||
float expected_3[] = {2.0f, 1.0f, 2.0f, 3.0f, 4.0f, 3.0f, 2.0f};
|
||||
struct ggml_tensor * out_3 = ggml_pad_reflect_1d(ctx, t, 1, 2);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(gf, out_1);
|
||||
ggml_build_forward_expand(gf, out_2);
|
||||
ggml_build_forward_expand(gf, out_3);
|
||||
|
||||
ggml_gallocr_alloc_graph(gallocr, gf);
|
||||
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
|
||||
check_tensor(out_1, expected_1, 6, 1, 1);
|
||||
check_tensor(out_2, expected_2, 7, 1, 1);
|
||||
check_tensor(out_3, expected_3, 7, 1, 1);
|
||||
|
||||
ggml_free(ctx);
|
||||
ggml_backend_buffer_free(buffer);
|
||||
ggml_gallocr_free(gallocr);
|
||||
}
|
||||
|
||||
{
|
||||
params = ggml_init_params{
|
||||
/*.mem_size =*/ 16*1024*1024,
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc. =*/ true
|
||||
};
|
||||
|
||||
ggml_log_set(ggml_log_callback_default, nullptr);
|
||||
|
||||
ctx = ggml_init(params);
|
||||
buffer = ggml_backend_alloc_buffer(backend, 16*1024*1024);
|
||||
tallocr = ggml_tallocr_new(buffer);
|
||||
gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));
|
||||
|
||||
// Create a 2D input tensor (5 columns × 4 rows)
|
||||
struct ggml_tensor * t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 5, 4);
|
||||
float input_data[] = {
|
||||
1.0f, 2.0f, 3.0f, 4.0f, 5.0f, // row 1
|
||||
6.0f, 7.0f, 8.0f, 9.0f, 10.0f, // row 2
|
||||
11.0f, 12.0f, 13.0f, 14.0f, 15.0f, // row 3
|
||||
16.0f, 17.0f, 18.0f, 19.0f, 20.0f // row 4
|
||||
};
|
||||
ggml_tallocr_alloc(&tallocr, t);
|
||||
|
||||
// load data to buffer
|
||||
if(ggml_backend_is_cpu(backend)) {
|
||||
memcpy(t->data, input_data, ggml_nbytes(t));
|
||||
} else {
|
||||
ggml_backend_tensor_set(t, input_data, 0, ggml_nbytes(t));
|
||||
}
|
||||
|
||||
// Test case 4: pad left=3, right=2 on a 2D tensor
|
||||
// Each row should be padded independently
|
||||
float expected_4[] = {
|
||||
4.0f, 3.0f, 2.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 4.0f, 3.0f, // row 1
|
||||
9.0f, 8.0f, 7.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 9.0f, 8.0f, // row 2
|
||||
14.0f, 13.0f, 12.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 14.0f, 13.0f, // row 3
|
||||
19.0f, 18.0f, 17.0f, 16.0f, 17.0f, 18.0f, 19.0f, 20.0f, 19.0f, 18.0f // row 4
|
||||
};
|
||||
struct ggml_tensor * out_4 = ggml_pad_reflect_1d(ctx, t, 3, 2);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(gf, out_4);
|
||||
|
||||
ggml_gallocr_alloc_graph(gallocr, gf);
|
||||
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
|
||||
check_tensor(out_4, expected_4, 10, 4, 1);
|
||||
|
||||
ggml_free(ctx);
|
||||
ggml_gallocr_free(gallocr);
|
||||
ggml_backend_buffer_free(buffer);
|
||||
}
|
||||
|
||||
ggml_backend_free(backend);
|
||||
}
|
||||
|
||||
int main(int argc, const char * argv[]) {
|
||||
bool use_gpu = false;
|
||||
if (argc > 1) {
|
||||
use_gpu = strcmp(argv[1], "--gpu") == 0;
|
||||
}
|
||||
test_pad_reflect_1d(use_gpu);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
struct ggml_context* make_ctx(void) {
|
||||
struct ggml_init_params params = {
|
||||
.mem_size = 2 * 1024 * 1024,
|
||||
};
|
||||
|
||||
return ggml_init(params);
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv) {
|
||||
|
||||
float buf_f32[1024];
|
||||
ggml_fp16_t buf_f16[1024];
|
||||
for (int i = 0; i < 1024; ++i) {
|
||||
buf_f32[i] = (float)(i + 1);
|
||||
buf_f16[i] = ggml_fp32_to_fp16(buf_f32[i]);
|
||||
}
|
||||
|
||||
// avg pool 1d - Float 32
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
memcpy(t->data, buf_f32, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * t_pooled = ggml_pool_1d(ctx, t, GGML_OP_POOL_AVG, 3, 3, 0);
|
||||
GGML_ASSERT(t_pooled->ne[0] == 3);
|
||||
GGML_ASSERT(t_pooled->ne[1] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[2] == 1);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, t_pooled);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(t_pooled);
|
||||
|
||||
GGML_ASSERT(output[0] == 2);
|
||||
GGML_ASSERT(output[1] == 5);
|
||||
GGML_ASSERT(output[2] == 8);
|
||||
GGML_ASSERT(output[3] == 12);
|
||||
GGML_ASSERT(output[4] == 15);
|
||||
GGML_ASSERT(output[5] == 18);
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
// avg pool 1d - Float 16
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, 10, 2);
|
||||
memcpy(t->data, buf_f16, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * t_pooled = ggml_pool_1d(ctx, t, GGML_OP_POOL_AVG, 3, 3, 0);
|
||||
GGML_ASSERT(t_pooled->ne[0] == 3);
|
||||
GGML_ASSERT(t_pooled->ne[1] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[2] == 1);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, t_pooled);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(t_pooled);
|
||||
|
||||
GGML_ASSERT(output[0] == 2);
|
||||
GGML_ASSERT(output[1] == 5);
|
||||
GGML_ASSERT(output[2] == 8);
|
||||
GGML_ASSERT(output[3] == 12);
|
||||
GGML_ASSERT(output[4] == 15);
|
||||
GGML_ASSERT(output[5] == 18);
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
// max pool 1d - Float 32
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 10, 2);
|
||||
memcpy(t->data, buf_f32, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * t_pooled = ggml_pool_1d(ctx, t, GGML_OP_POOL_MAX, 3, 3, 0);
|
||||
GGML_ASSERT(t_pooled->ne[0] == 3);
|
||||
GGML_ASSERT(t_pooled->ne[1] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[2] == 1);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, t_pooled);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(t_pooled);
|
||||
GGML_ASSERT(output[0] == 3);
|
||||
GGML_ASSERT(output[1] == 6);
|
||||
GGML_ASSERT(output[2] == 9);
|
||||
GGML_ASSERT(output[3] == 13);
|
||||
GGML_ASSERT(output[4] == 16);
|
||||
GGML_ASSERT(output[5] == 19);
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
// max pool 1d - Float 16
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, 10, 2);
|
||||
memcpy(t->data, buf_f16, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * t_pooled = ggml_pool_1d(ctx, t, GGML_OP_POOL_MAX, 3, 3, 0);
|
||||
GGML_ASSERT(t_pooled->ne[0] == 3);
|
||||
GGML_ASSERT(t_pooled->ne[1] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[2] == 1);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, t_pooled);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(t_pooled);
|
||||
GGML_ASSERT(output[0] == 3);
|
||||
GGML_ASSERT(output[1] == 6);
|
||||
GGML_ASSERT(output[2] == 9);
|
||||
GGML_ASSERT(output[3] == 13);
|
||||
GGML_ASSERT(output[4] == 16);
|
||||
GGML_ASSERT(output[5] == 19);
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
// avg pool 2d - Float 32
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 10, 10, 2);
|
||||
memcpy(t->data, buf_f32, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * t_pooled = ggml_pool_2d(ctx, t, GGML_OP_POOL_AVG, 3, 4, 3, 4, 0, 0);
|
||||
GGML_ASSERT(t_pooled->ne[0] == 3);
|
||||
GGML_ASSERT(t_pooled->ne[1] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[2] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[3] == 1);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, t_pooled);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(t_pooled);
|
||||
GGML_ASSERT(output[0] == 17);
|
||||
GGML_ASSERT(output[1] == 20);
|
||||
GGML_ASSERT(output[2] == 23);
|
||||
GGML_ASSERT(output[3] == 57);
|
||||
GGML_ASSERT(output[4] == 60);
|
||||
GGML_ASSERT(output[5] == 63);
|
||||
GGML_ASSERT(output[6] == 117);
|
||||
GGML_ASSERT(output[7] == 120);
|
||||
GGML_ASSERT(output[8] == 123);
|
||||
GGML_ASSERT(output[9] == 157);
|
||||
GGML_ASSERT(output[10] == 160);
|
||||
GGML_ASSERT(output[11] == 163);
|
||||
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
// avg pool 2d - Float 16
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t = ggml_new_tensor_3d(ctx, GGML_TYPE_F16, 10, 10, 2);
|
||||
memcpy(t->data, buf_f16, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * t_pooled = ggml_pool_2d(ctx, t, GGML_OP_POOL_AVG, 3, 4, 3, 4, 0, 0);
|
||||
GGML_ASSERT(t_pooled->ne[0] == 3);
|
||||
GGML_ASSERT(t_pooled->ne[1] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[2] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[3] == 1);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, t_pooled);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(t_pooled);
|
||||
GGML_ASSERT(output[0] == 17);
|
||||
GGML_ASSERT(output[1] == 20);
|
||||
GGML_ASSERT(output[2] == 23);
|
||||
GGML_ASSERT(output[3] == 57);
|
||||
GGML_ASSERT(output[4] == 60);
|
||||
GGML_ASSERT(output[5] == 63);
|
||||
GGML_ASSERT(output[6] == 117);
|
||||
GGML_ASSERT(output[7] == 120);
|
||||
GGML_ASSERT(output[8] == 123);
|
||||
GGML_ASSERT(output[9] == 157);
|
||||
GGML_ASSERT(output[10] == 160);
|
||||
GGML_ASSERT(output[11] == 163);
|
||||
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
// max pool 2d - Float 32
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 10, 10, 2);
|
||||
memcpy(t->data, buf_f32, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * t_pooled = ggml_pool_2d(ctx, t, GGML_OP_POOL_MAX, 3, 4, 3, 4, 0, 0);
|
||||
GGML_ASSERT(t_pooled->ne[0] == 3);
|
||||
GGML_ASSERT(t_pooled->ne[1] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[2] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[3] == 1);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, t_pooled);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(t_pooled);
|
||||
GGML_ASSERT(output[0] == 33);
|
||||
GGML_ASSERT(output[1] == 36);
|
||||
GGML_ASSERT(output[2] == 39);
|
||||
GGML_ASSERT(output[3] == 73);
|
||||
GGML_ASSERT(output[4] == 76);
|
||||
GGML_ASSERT(output[5] == 79);
|
||||
GGML_ASSERT(output[6] == 133);
|
||||
GGML_ASSERT(output[7] == 136);
|
||||
GGML_ASSERT(output[8] == 139);
|
||||
GGML_ASSERT(output[9] == 173);
|
||||
GGML_ASSERT(output[10] == 176);
|
||||
GGML_ASSERT(output[11] == 179);
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
// max pool 2d - Float 16
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
struct ggml_tensor * t = ggml_new_tensor_3d(ctx, GGML_TYPE_F16, 10, 10, 2);
|
||||
memcpy(t->data, buf_f16, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * t_pooled = ggml_pool_2d(ctx, t, GGML_OP_POOL_MAX, 3, 4, 3, 4, 0, 0);
|
||||
GGML_ASSERT(t_pooled->ne[0] == 3);
|
||||
GGML_ASSERT(t_pooled->ne[1] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[2] == 2);
|
||||
GGML_ASSERT(t_pooled->ne[3] == 1);
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, t_pooled);
|
||||
|
||||
ggml_graph_compute_with_ctx(ctx, graph, 4);
|
||||
|
||||
const float * output = ggml_get_data_f32(t_pooled);
|
||||
GGML_ASSERT(output[0] == 33);
|
||||
GGML_ASSERT(output[1] == 36);
|
||||
GGML_ASSERT(output[2] == 39);
|
||||
GGML_ASSERT(output[3] == 73);
|
||||
GGML_ASSERT(output[4] == 76);
|
||||
GGML_ASSERT(output[5] == 79);
|
||||
GGML_ASSERT(output[6] == 133);
|
||||
GGML_ASSERT(output[7] == 136);
|
||||
GGML_ASSERT(output[8] == 139);
|
||||
GGML_ASSERT(output[9] == 173);
|
||||
GGML_ASSERT(output[10] == 176);
|
||||
GGML_ASSERT(output[11] == 179);
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
// Unit tests for quantization specific functions - quantize, dequantize and dot product
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#undef NDEBUG
|
||||
#include <assert.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
constexpr float MAX_QUANTIZATION_REFERENCE_ERROR = 0.0001f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR = 0.002f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_BINARY = 0.025f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_TERNARY = 0.01f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_2BITS = 0.0075f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS = 0.0040f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS = 0.0050f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_FP4 = 0.0030f;
|
||||
constexpr float MAX_DOT_PRODUCT_ERROR = 0.02f;
|
||||
constexpr float MAX_DOT_PRODUCT_ERROR_LOWBIT = 0.04f;
|
||||
constexpr float MAX_DOT_PRODUCT_ERROR_FP4 = 0.03f;
|
||||
constexpr float MAX_DOT_PRODUCT_ERROR_BINARY = 0.40f;
|
||||
constexpr float MAX_DOT_PRODUCT_ERROR_TERNARY = 0.15f;
|
||||
|
||||
static const char* RESULT_STR[] = {"ok", "FAILED"};
|
||||
|
||||
|
||||
// Generate synthetic data
|
||||
static void generate_data(float offset, size_t n, float * dst) {
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
dst[i] = 0.1 + 2*cosf(i + offset);
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate RMSE between two float arrays
|
||||
static float array_rmse(const float * a1, const float * a2, size_t n) {
|
||||
double sum = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
double diff = a1[i] - a2[i];
|
||||
sum += diff * diff;
|
||||
}
|
||||
return sqrtf(sum) / n;
|
||||
}
|
||||
|
||||
// Total quantization error on test data
|
||||
static float total_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {
|
||||
std::vector<uint8_t> tmp_q(2*test_size);
|
||||
std::vector<float> tmp_out(test_size);
|
||||
|
||||
qfns_cpu->from_float(test_data, tmp_q.data(), test_size);
|
||||
qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);
|
||||
return array_rmse(test_data, tmp_out.data(), test_size);
|
||||
}
|
||||
|
||||
// Total quantization error on test data
|
||||
static float reference_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {
|
||||
std::vector<uint8_t> tmp_q(2*test_size);
|
||||
std::vector<float> tmp_out(test_size);
|
||||
std::vector<float> tmp_out_ref(test_size);
|
||||
|
||||
// FIXME: why is done twice?
|
||||
qfns_cpu->from_float(test_data, tmp_q.data(), test_size);
|
||||
qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);
|
||||
|
||||
qfns->from_float_ref(test_data, tmp_q.data(), test_size);
|
||||
qfns->to_float(tmp_q.data(), tmp_out_ref.data(), test_size);
|
||||
|
||||
return array_rmse(tmp_out.data(), tmp_out_ref.data(), test_size);
|
||||
}
|
||||
|
||||
static float dot_product(const float * a1, const float * a2, size_t test_size) {
|
||||
double sum = 0;
|
||||
for (size_t i = 0; i < test_size; i++) {
|
||||
sum += a1[i] * a2[i];
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// Total dot product error
|
||||
static float dot_product_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data1, const float * test_data2) {
|
||||
GGML_UNUSED(qfns);
|
||||
|
||||
std::vector<uint8_t> tmp_q1(2*test_size);
|
||||
std::vector<uint8_t> tmp_q2(2*test_size);
|
||||
|
||||
const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type);
|
||||
|
||||
qfns_cpu->from_float(test_data1, tmp_q1.data(), test_size);
|
||||
vdot->from_float(test_data2, tmp_q2.data(), test_size);
|
||||
|
||||
float result = INFINITY;
|
||||
qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1);
|
||||
|
||||
const float dot_ref = dot_product(test_data1, test_data2, test_size);
|
||||
|
||||
return fabsf(result - dot_ref) / test_size;
|
||||
}
|
||||
|
||||
int main(int argc, char * argv[]) {
|
||||
bool verbose = false;
|
||||
const size_t test_size = 32 * 128;
|
||||
|
||||
std::string arg;
|
||||
for (int i = 1; i < argc; i++) {
|
||||
arg = argv[i];
|
||||
|
||||
if (arg == "-v") {
|
||||
verbose = true;
|
||||
} else {
|
||||
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<float> test_data(test_size);
|
||||
std::vector<float> test_data2(test_size);
|
||||
|
||||
generate_data(0.0, test_data.size(), test_data.data());
|
||||
generate_data(1.0, test_data2.size(), test_data2.data());
|
||||
|
||||
ggml_cpu_init();
|
||||
|
||||
int num_failed = 0;
|
||||
bool failed = false;
|
||||
|
||||
for (int i = 0; i < GGML_TYPE_COUNT; i++) {
|
||||
ggml_type type = (ggml_type) i;
|
||||
const auto * qfns = ggml_get_type_traits(type);
|
||||
const auto * qfns_cpu = ggml_get_type_traits_cpu(type);
|
||||
|
||||
// deprecated - skip
|
||||
if (qfns->blck_size == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const ggml_type ei = (ggml_type)i;
|
||||
|
||||
printf("Testing %s\n", ggml_type_name((ggml_type) i));
|
||||
ggml_quantize_init(ei);
|
||||
|
||||
if (qfns_cpu->from_float && qfns->to_float) {
|
||||
const float total_error = total_quantization_error(qfns, qfns_cpu, test_size, test_data.data());
|
||||
const float max_quantization_error =
|
||||
type == GGML_TYPE_Q1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_BINARY :
|
||||
type == GGML_TYPE_TQ1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
|
||||
type == GGML_TYPE_TQ2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
|
||||
type == GGML_TYPE_Q2_K ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :
|
||||
type == GGML_TYPE_IQ2_S ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :
|
||||
type == GGML_TYPE_Q3_K ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :
|
||||
type == GGML_TYPE_IQ3_S ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :
|
||||
type == GGML_TYPE_IQ3_XXS ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS :
|
||||
type == GGML_TYPE_NVFP4 ? MAX_QUANTIZATION_TOTAL_ERROR_FP4 : MAX_QUANTIZATION_TOTAL_ERROR;
|
||||
failed = !(total_error < max_quantization_error);
|
||||
num_failed += failed;
|
||||
if (failed || verbose) {
|
||||
printf("%5s absolute quantization error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error);
|
||||
}
|
||||
|
||||
const float reference_error = reference_quantization_error(qfns, qfns_cpu, test_size, test_data.data());
|
||||
failed = !(reference_error < MAX_QUANTIZATION_REFERENCE_ERROR);
|
||||
num_failed += failed;
|
||||
if (failed || verbose) {
|
||||
printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error);
|
||||
}
|
||||
|
||||
const float vec_dot_error = dot_product_error(qfns, qfns_cpu, test_size, test_data.data(), test_data2.data());
|
||||
const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS ||
|
||||
type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S
|
||||
? MAX_DOT_PRODUCT_ERROR_LOWBIT
|
||||
: type == GGML_TYPE_Q1_0
|
||||
? MAX_DOT_PRODUCT_ERROR_BINARY
|
||||
: type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0
|
||||
? MAX_DOT_PRODUCT_ERROR_TERNARY
|
||||
: type == GGML_TYPE_NVFP4
|
||||
? MAX_DOT_PRODUCT_ERROR_FP4
|
||||
: MAX_DOT_PRODUCT_ERROR;
|
||||
failed = !(vec_dot_error < max_allowed_error);
|
||||
num_failed += failed;
|
||||
if (failed || verbose) {
|
||||
printf("%5s dot product error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (num_failed || verbose) {
|
||||
printf("%d tests failed\n", num_failed);
|
||||
}
|
||||
|
||||
return num_failed > 0;
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
// Benchmark quantization specific functions on synthetic data
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#undef NDEBUG
|
||||
#include <algorithm>
|
||||
#include <assert.h>
|
||||
#include <functional>
|
||||
#include <math.h>
|
||||
#include <memory>
|
||||
#include <stdio.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
#define MAX_ALIGNMENT 64
|
||||
#define QK 32
|
||||
#define WARMUP 5
|
||||
#define ITERATIONS 10
|
||||
#define MAX_ITERATIONS 100000000
|
||||
|
||||
#define L1_SIZE 32*128
|
||||
#define L2_SIZE 32*2048
|
||||
#define L3_SIZE 32*20480
|
||||
#define MEM_SIZE 32*2048000
|
||||
|
||||
struct quantize_perf_params {
|
||||
std::vector<std::string> include_types;
|
||||
std::vector<size_t> test_sizes;
|
||||
size_t alignment_offset = 0;
|
||||
bool op_quantize_row_q_reference = false;
|
||||
bool op_quantize_row_q = false;
|
||||
bool op_dequantize_row_q = false;
|
||||
bool op_quantize_row_q_dot = false;
|
||||
bool op_vec_dot_q = false;
|
||||
int64_t iterations = ITERATIONS;
|
||||
};
|
||||
|
||||
#if defined(__x86_64__) || defined(__i386__)
|
||||
|
||||
#include <x86intrin.h>
|
||||
inline int64_t cpu_cycles() {
|
||||
// Rough way to detect new-ish CPUs
|
||||
#ifdef __POPCNT__
|
||||
unsigned int dummy;
|
||||
return __rdtscp(&dummy);
|
||||
#else
|
||||
return __rdtsc();
|
||||
#endif
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#define cpu_cycles() 0
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
// Generate synthetic data
|
||||
static void generate_data(float offset, size_t n, float * dst) {
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
dst[i] = 0.1 + 2*cosf(i + offset);
|
||||
}
|
||||
}
|
||||
|
||||
static float gigabytes_per_second(size_t bytes, int64_t usecs) {
|
||||
return bytes / (float) usecs * 1000000 / (1024*1024*1024);
|
||||
}
|
||||
|
||||
static void * align_with_offset(void * ptr, int offset) {
|
||||
size_t dummy_size = MAX_ALIGNMENT * 4;
|
||||
return (char *) std::align(MAX_ALIGNMENT, MAX_ALIGNMENT, ptr, dummy_size) + offset;
|
||||
}
|
||||
|
||||
static void benchmark_function(size_t size, size_t q_size, int64_t iterations, const std::function<float(void)> & func) {
|
||||
int64_t min_time_us = INT64_MAX;
|
||||
int64_t total_time_us = 0;
|
||||
int64_t min_time_cycles = INT64_MAX;
|
||||
int64_t total_time_cycles = 0;
|
||||
|
||||
for (int i = 0; i < WARMUP; i++) {
|
||||
func();
|
||||
}
|
||||
|
||||
for (int i = 0; i < iterations; i++) {
|
||||
const int64_t start_time = ggml_time_us();
|
||||
const int64_t start_cycles = cpu_cycles();
|
||||
|
||||
func();
|
||||
|
||||
const int64_t end_cycles = cpu_cycles();
|
||||
const int64_t end_time = ggml_time_us();
|
||||
|
||||
total_time_cycles += end_cycles - start_cycles;
|
||||
min_time_cycles = std::min(min_time_cycles, end_cycles - start_cycles);
|
||||
total_time_us += end_time - start_time;
|
||||
min_time_us = std::min(min_time_us, end_time - start_time);
|
||||
}
|
||||
|
||||
printf(" min cycles/%d vals : %9.2f\n", QK, QK * min_time_cycles / (float) size);
|
||||
printf(" avg cycles/%d vals : %9.2f\n", QK, QK * total_time_cycles / (float) (size * iterations));
|
||||
printf(" float32 throughput : %9.2f GB/s\n", gigabytes_per_second(4 * size * iterations, total_time_us));
|
||||
printf(" quantized throughput : %9.2f GB/s\n", gigabytes_per_second(q_size * iterations, total_time_us));
|
||||
}
|
||||
|
||||
static void usage(char * argv[]) {
|
||||
printf("Benchmark quantization specific functions on synthetic data\n");
|
||||
printf("\n");
|
||||
printf("usage: %s [options]\n", argv[0]);
|
||||
printf("\n");
|
||||
printf("options: (default)\n");
|
||||
printf(" -h, --help show this help message and exit\n");
|
||||
printf(" --size SIZE set test size, divisible by 32 (L1_SIZE:%d)\n", L1_SIZE);
|
||||
printf(" -3 use size as L1, L2, L3 sizes (L1:%d L2:%d L3:%d)\n", L1_SIZE, L2_SIZE, L3_SIZE);
|
||||
printf(" -4 use size as L1, L2, L3, MEM sizes (L1:%d L2:%d L3:%d MEM:%d)\n", L1_SIZE, L2_SIZE, L3_SIZE, MEM_SIZE);
|
||||
printf(" --op OP set test operation as quantize_row_q_reference, quantize_row_q, dequantize_row_q,\n");
|
||||
printf(" quantize_row_q_dot, vec_dot_q (all)\n");
|
||||
printf(" --type TYPE set test type as");
|
||||
for (int i = 0; i < GGML_TYPE_COUNT; i++) {
|
||||
ggml_type type = (ggml_type) i;
|
||||
const auto * qfns = ggml_get_type_traits(type);
|
||||
const auto * qfns_cpu = ggml_get_type_traits_cpu(type);
|
||||
if (ggml_type_name(type) != NULL) {
|
||||
if (qfns_cpu->from_float && qfns->to_float) {
|
||||
printf(" %s", ggml_type_name(type));
|
||||
}
|
||||
}
|
||||
}
|
||||
printf(" (all)\n");
|
||||
printf(" --alignment-offset OFFSET\n");
|
||||
printf(" set alignment offset as OFFSET (0)\n");
|
||||
printf(" -i NUM, --iterations NUM\n");
|
||||
printf(" set test iteration number (%d)\n", ITERATIONS);
|
||||
}
|
||||
|
||||
int main(int argc, char * argv[]) {
|
||||
quantize_perf_params params {};
|
||||
|
||||
// read command line
|
||||
|
||||
bool invalid_param = false;
|
||||
std::string arg;
|
||||
for (int i = 1; i < argc; i++) {
|
||||
arg = argv[i];
|
||||
|
||||
if (arg == "--size") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
size_t size = std::stoi(argv[i]);
|
||||
if (size % 32 != 0) {
|
||||
fprintf(stderr, "error: size %zu not divisible by 32\n", size);
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.test_sizes.push_back(size);
|
||||
} else if (arg == "-3") {
|
||||
// quick select sizes that probably fit in CPU caches
|
||||
params.test_sizes.push_back(L1_SIZE);
|
||||
params.test_sizes.push_back(L2_SIZE);
|
||||
params.test_sizes.push_back(L3_SIZE);
|
||||
} else if (arg == "-4") {
|
||||
// quick select cache sizes + memory
|
||||
params.test_sizes.push_back(L1_SIZE);
|
||||
params.test_sizes.push_back(L2_SIZE);
|
||||
params.test_sizes.push_back(L3_SIZE);
|
||||
params.test_sizes.push_back(MEM_SIZE);
|
||||
} else if (arg == "--op") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
std::string op {argv[i]};
|
||||
if (op == "quantize_row_q_reference") {
|
||||
params.op_quantize_row_q_reference = true;
|
||||
} else if (op == "quantize_row_q") {
|
||||
params.op_quantize_row_q = true;
|
||||
} else if (op == "dequantize_row_q") {
|
||||
params.op_dequantize_row_q = true;
|
||||
} else if (op == "quantize_row_q_dot") {
|
||||
params.op_quantize_row_q_dot = true;
|
||||
} else if (op == "vec_dot_q") {
|
||||
params.op_vec_dot_q = true;
|
||||
} else {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
} else if (arg == "--type") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.include_types.push_back(argv[i]);
|
||||
} else if (arg == "--alignment-offset") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
int alignment = std::stoi(argv[i]);
|
||||
if (alignment < 0 || alignment > MAX_ALIGNMENT) {
|
||||
fprintf(stderr, "error: alignment-offset must be less than %d\n", MAX_ALIGNMENT);
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.alignment_offset = alignment;
|
||||
} else if ((arg == "-i") || (arg == "--iterations")) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
int number = std::stoi(argv[i]);
|
||||
if (number < 0 || number > MAX_ITERATIONS) {
|
||||
fprintf(stderr, "error: iterations must be less than %d\n", MAX_ITERATIONS);
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.iterations = number;
|
||||
} else if ((arg == "-h") || (arg == "--help")) {
|
||||
usage(argv);
|
||||
return 1;
|
||||
} else {
|
||||
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (invalid_param) {
|
||||
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (params.test_sizes.empty()) {
|
||||
params.test_sizes.push_back(L1_SIZE);
|
||||
}
|
||||
if (!(params.op_quantize_row_q_reference || params.op_quantize_row_q || params.op_dequantize_row_q || params.op_quantize_row_q_dot || params.op_vec_dot_q)) {
|
||||
params.op_quantize_row_q_reference = params.op_quantize_row_q = params.op_dequantize_row_q = params.op_quantize_row_q_dot = params.op_vec_dot_q = true;
|
||||
}
|
||||
|
||||
std::sort(params.test_sizes.begin(), params.test_sizes.end());
|
||||
size_t largest = params.test_sizes.back();
|
||||
|
||||
std::vector<uint8_t> test_data1_v(largest*4 + MAX_ALIGNMENT*2);
|
||||
std::vector<uint8_t> test_data2_v(largest*4 + MAX_ALIGNMENT*2);
|
||||
std::vector<uint8_t> test_q1_v (largest*4 + MAX_ALIGNMENT*2);
|
||||
std::vector<uint8_t> test_q2_v (largest*4 + MAX_ALIGNMENT*2);
|
||||
std::vector<uint8_t> test_out_v (largest*4 + MAX_ALIGNMENT*2);
|
||||
|
||||
float * test_data1 = (float *) align_with_offset(test_data1_v.data(), params.alignment_offset);
|
||||
float * test_data2 = (float *) align_with_offset(test_data2_v.data(), params.alignment_offset);
|
||||
float * test_q1 = (float *) align_with_offset(test_q1_v.data(), params.alignment_offset);
|
||||
float * test_q2 = (float *) align_with_offset(test_q2_v.data(), params.alignment_offset);
|
||||
float * test_out = (float *) align_with_offset(test_out_v.data(), params.alignment_offset);
|
||||
|
||||
generate_data(0, largest, test_data1);
|
||||
generate_data(1, largest, test_data2);
|
||||
|
||||
int64_t iterations = params.iterations;
|
||||
|
||||
ggml_cpu_init();
|
||||
|
||||
for (int i = 0; i < GGML_TYPE_COUNT; i++) {
|
||||
ggml_type type = (ggml_type) i;
|
||||
const auto * qfns = ggml_get_type_traits(type);
|
||||
const auto * qfns_cpu = ggml_get_type_traits_cpu(type);
|
||||
if (!params.include_types.empty() && ggml_type_name(type) && std::find(params.include_types.begin(), params.include_types.end(), ggml_type_name(type)) == params.include_types.end()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (qfns_cpu->from_float && qfns->to_float) {
|
||||
printf("%s\n", ggml_type_name(type));
|
||||
|
||||
ggml_quantize_init(type);
|
||||
|
||||
if (params.op_quantize_row_q_reference) {
|
||||
printf(" quantize_row_q_reference\n");
|
||||
for (size_t size : params.test_sizes) {
|
||||
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
|
||||
auto quantize_fn = [&](void) -> float {
|
||||
qfns->from_float_ref(test_data1, test_q1, size);
|
||||
return test_q1[0];
|
||||
};
|
||||
size_t quantized_size = ggml_row_size(type, size);
|
||||
benchmark_function(size, quantized_size, iterations, quantize_fn);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
if (params.op_quantize_row_q) {
|
||||
printf(" quantize_row_q\n");
|
||||
for (size_t size : params.test_sizes) {
|
||||
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
|
||||
auto quantize_fn = [&](void) -> float {
|
||||
qfns_cpu->from_float(test_data1, test_q1, size);
|
||||
return test_q1[0];
|
||||
};
|
||||
size_t quantized_size = ggml_row_size(type, size);
|
||||
benchmark_function(size, quantized_size, iterations, quantize_fn);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
if (params.op_dequantize_row_q) {
|
||||
printf(" dequantize_row_q\n");
|
||||
qfns_cpu->from_float(test_data1, test_q1, largest);
|
||||
for (size_t size : params.test_sizes) {
|
||||
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
|
||||
auto quantize_fn = [&](void) -> float {
|
||||
qfns->to_float(test_q1, test_out, size);
|
||||
return test_out[0];
|
||||
};
|
||||
size_t quantized_size = ggml_row_size(type, size);
|
||||
benchmark_function(size, quantized_size, iterations, quantize_fn);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
if (params.op_quantize_row_q_dot) {
|
||||
printf(" quantize_row_q_dot\n");
|
||||
for (size_t size : params.test_sizes) {
|
||||
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
|
||||
auto quantize_fn = [&](void) -> float {
|
||||
const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type);
|
||||
vdot->from_float(test_data1, test_q1, size);
|
||||
return test_q1[0];
|
||||
};
|
||||
size_t quantized_size = ggml_row_size(type, size);
|
||||
benchmark_function(size, quantized_size, iterations, quantize_fn);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
if (params.op_vec_dot_q) {
|
||||
printf(" vec_dot_q\n");
|
||||
qfns_cpu->from_float(test_data1, test_q1, largest);
|
||||
qfns_cpu->from_float(test_data2, test_q2, largest);
|
||||
for (size_t size : params.test_sizes) {
|
||||
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
|
||||
auto quantize_fn = [&](void) -> float {
|
||||
float result;
|
||||
qfns_cpu->vec_dot(size, &result, 0, test_q1, 0, test_q2, 0, 1);
|
||||
return result;
|
||||
};
|
||||
size_t quantized_size = ggml_row_size(type, size);
|
||||
benchmark_function(size, quantized_size, iterations, quantize_fn);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
struct ggml_context* make_ctx(void) {
|
||||
struct ggml_init_params params = {
|
||||
.mem_size = 2 * 1024 * 1024,
|
||||
};
|
||||
|
||||
return ggml_init(params);
|
||||
}
|
||||
|
||||
void check_tensor(struct ggml_tensor * t, float * expected_t_d, int ne0, int ne1, int ne2) {
|
||||
GGML_ASSERT(t->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(t->ne[0] == ne0);
|
||||
GGML_ASSERT(t->ne[1] == ne1);
|
||||
GGML_ASSERT(t->ne[2] == ne2);
|
||||
for (int i2 = 0; i2 < ne2; ++i2) {
|
||||
for (int i1 = 0; i1 < ne1; ++i1) {
|
||||
for (int i0 = 0; i0 < ne0; ++i0) {
|
||||
float expected = *(expected_t_d + i2 * ne1 * ne0 + i1 * ne0 + i0);
|
||||
float actual = ggml_get_data_f32(t)[i2 * ne1 * ne0 + i1 * ne0 + i0];
|
||||
GGML_ASSERT(expected == actual);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv) {
|
||||
ggml_fp16_t buf_f16[1024];
|
||||
for (int i = 0; i < 1024; ++i) {
|
||||
buf_f16[i] = ggml_fp32_to_fp16((float)i);
|
||||
}
|
||||
|
||||
float expected_out[4][9] = {
|
||||
{ 8.0, 9.0, 10.0, 9.0, 10.0, 11.0, 10.0, 11.0, 12.0 },
|
||||
{ 2.0, 3.0, 4.0, 3.0, 4.0, 5.0, 4.0, 5.0, 6.0 },
|
||||
{ 14.0, 15.0, 16.0, 15.0, 16.0, 17.0, 16.0, 17.0, 18.0 },
|
||||
{ 8.0, 9.0, 10.0, 9.0, 10.0, 11.0, 10.0, 11.0, 12.0 },
|
||||
};
|
||||
|
||||
{
|
||||
struct ggml_context * ctx = make_ctx();
|
||||
|
||||
|
||||
struct ggml_tensor * t = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, 3, 3);
|
||||
ggml_fp16_t* t_d = (ggml_fp16_t*)t->data;
|
||||
memcpy(t_d, buf_f16, ggml_nbytes(t));
|
||||
|
||||
struct ggml_tensor * t_2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, 3, 3);
|
||||
ggml_fp16_t* t_d_2 = (ggml_fp16_t*)t_2->data;
|
||||
memcpy(t_d_2, buf_f16 + 1, ggml_nbytes(t_2));
|
||||
|
||||
struct ggml_tensor * rw = ggml_get_rel_pos(ctx, t, 2, 2);
|
||||
struct ggml_tensor * rh = ggml_get_rel_pos(ctx, t_2, 2, 2);
|
||||
|
||||
struct ggml_tensor * rw_f32 = ggml_cpy(ctx, rw, ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 3, 2, 2));
|
||||
struct ggml_tensor * rh_f32 = ggml_cpy(ctx, rh, ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 3, 2, 2));
|
||||
|
||||
struct ggml_tensor * in = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 9, 4);
|
||||
struct ggml_tensor * out_inplace = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 9, 4);
|
||||
float * in_d = (float*)in->data;
|
||||
float * out_inplace_d = (float*)out_inplace->data;
|
||||
for (int i = 0; i < ggml_nelements(in); ++i) {
|
||||
in_d[i] = 1.f;
|
||||
out_inplace_d[i] = 1.f;
|
||||
}
|
||||
|
||||
struct ggml_tensor * out = ggml_add_rel_pos(ctx, in, rw_f32, rh_f32);
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(gf, out);
|
||||
ggml_graph_compute_with_ctx(ctx, gf, 1);
|
||||
|
||||
out_inplace = ggml_add_rel_pos_inplace(ctx, out_inplace, rw_f32, rh_f32);
|
||||
struct ggml_cgraph * gf_2 = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(gf_2, out_inplace);
|
||||
ggml_graph_compute_with_ctx(ctx, gf_2, 1);
|
||||
|
||||
check_tensor(out, (float*)expected_out, 9, 4, 1);
|
||||
check_tensor(out_inplace, (float*)expected_out, 9, 4, 1);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
#include <ggml.h>
|
||||
#include <ggml-cpu.h>
|
||||
#include <ggml-alloc.h>
|
||||
#include <ggml-backend.h>
|
||||
#include <ggml-cpp.h>
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <array>
|
||||
#include <numeric>
|
||||
#include <vector>
|
||||
|
||||
int64_t wrap(int64_t i, int64_t ne) {
|
||||
if (i < 0) {
|
||||
return i + ne;
|
||||
} else if (i >= ne) {
|
||||
return i - ne;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
std::vector<float> roll_reference(
|
||||
const float * src, std::array<int64_t, 4> ne, std::array<int, 4> shift) {
|
||||
|
||||
const int64_t ne0 = ne[0], ne1 = ne[1], ne2 = ne[2], ne3 = ne[3];
|
||||
std::vector<float> dst(ne0 * ne1 * ne2 * ne3);
|
||||
|
||||
for (int64_t i3 = 0; i3 < ne3; ++i3) {
|
||||
for (int64_t i2 = 0; i2 < ne2; ++i2) {
|
||||
for (int64_t i1 = 0; i1 < ne1; ++i1) {
|
||||
for (int64_t i0 = 0; i0 < ne0; ++i0) {
|
||||
const int64_t i03 = wrap(i3 - shift[3], ne3);
|
||||
const int64_t i02 = wrap(i2 - shift[2], ne2);
|
||||
const int64_t i01 = wrap(i1 - shift[1], ne1);
|
||||
const int64_t i00 = wrap(i0 - shift[0], ne0);
|
||||
|
||||
dst[i3 * (ne2*ne1*ne0) + i2 * (ne1*ne0) + i1 * ne0 + i0] =
|
||||
src[i03 * (ne2*ne1*ne0) + i02 * (ne1*ne0) + i01 * ne0 + i00];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return dst;
|
||||
}
|
||||
|
||||
std::vector<float> f32_range(int64_t n) {
|
||||
std::vector<float> values(n);
|
||||
std::iota(values.begin(), values.end(), 0.f);
|
||||
return values;
|
||||
}
|
||||
|
||||
bool check_equal(const std::vector<float> & result, const std::vector<float> & expected) {
|
||||
if (result.size() != expected.size()) {
|
||||
printf("result.size() = %d, expected.size() = %d\n", (int)result.size(), (int)expected.size());
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < result.size(); i++) {
|
||||
if(std::abs(result[i] - expected[i]) > 1e-5) {
|
||||
printf("result[%d] %f != %f expected[%d]\n", i, result[i], expected[i], i);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool test_roll(std::array<int64_t, 4> ne, std::array<int, 4> shift, bool permute) {
|
||||
ggml_time_init();
|
||||
|
||||
ggml_init_params params {
|
||||
/*.mem_size =*/ 64 * ggml_tensor_overhead() + ggml_graph_overhead(),
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true
|
||||
};
|
||||
|
||||
ggml_context_ptr ctx_ptr{ggml_init(params)};
|
||||
ggml_context * ctx = ctx_ptr.get();
|
||||
ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
|
||||
// Build graph
|
||||
ggml_tensor * src = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
|
||||
ggml_tensor * res;
|
||||
if (!permute) {
|
||||
res = ggml_roll(ctx, src, shift[0], shift[1], shift[2], shift[3]);
|
||||
} else {
|
||||
ggml_tensor * p = ggml_permute(ctx, src, 0, 3, 1, 2);
|
||||
res = ggml_roll(ctx, p, shift[0], shift[2], shift[3], shift[1]);
|
||||
res = ggml_cont(ctx, ggml_permute(ctx, res, 0, 2, 3, 1));
|
||||
}
|
||||
ggml_build_forward_expand(gf, res);
|
||||
|
||||
// Create backend & allocate buffers
|
||||
ggml_backend_ptr backend_ptr{ggml_backend_cpu_init()};
|
||||
ggml_backend_t backend = backend_ptr.get();
|
||||
ggml_backend_cpu_set_n_threads(backend, 2);
|
||||
ggml_backend_buffer_ptr buffer{ggml_backend_alloc_ctx_tensors(ctx, backend)};
|
||||
|
||||
std::vector<float> src_values = f32_range(ggml_nelements(src));
|
||||
ggml_backend_tensor_set(src, src_values.data(), 0, ggml_nbytes(src));
|
||||
|
||||
// Execute and compare results
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
|
||||
std::vector<float> res_values(ggml_nelements(res));
|
||||
ggml_backend_tensor_get(res, res_values.data(), 0, ggml_nbytes(res));
|
||||
|
||||
std::vector<float> expected = roll_reference(src_values.data(), ne, shift);
|
||||
|
||||
bool passed = check_equal(res_values, expected);
|
||||
|
||||
printf("ggml_roll(%d(%d), %d(%d), %d(%d), %d(%d), %s): %s\n",
|
||||
int(ne[0]), int(shift[0]),
|
||||
int(ne[1]), int(shift[1]),
|
||||
int(ne[2]), int(shift[2]),
|
||||
int(ne[3]), int(shift[3]),
|
||||
permute ? "permuted" : "contiguous",
|
||||
passed ? "\033[32mPASSED\033[0m" : "\033[31mFAILED\033[0m");
|
||||
return passed;
|
||||
}
|
||||
|
||||
int main() {
|
||||
bool passed = true;
|
||||
passed &= test_roll({3, 7, 4, 2}, {1, 0, -1, 0}, false);
|
||||
passed &= test_roll({37, 42, 59, 2}, {-4, 3, -7, 1}, false);
|
||||
passed &= test_roll({37, 42, 59, 2}, {-4, 3, -7, 1}, true);
|
||||
return passed ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
#include "ggml-metal.h"
|
||||
#endif
|
||||
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <vector>
|
||||
#include <cmath>
|
||||
|
||||
void ggml_tensor_set_f32(struct ggml_tensor* tensor, float value, int l, int k = 0, int j = 0, int i = 0) {
|
||||
GGML_ASSERT(tensor->nb[0] == sizeof(float));
|
||||
*(float*)((char*)(tensor->data) + i * tensor->nb[3] + j * tensor->nb[2] + k * tensor->nb[1] + l * tensor->nb[0]) = value;
|
||||
}
|
||||
|
||||
void set_timestep_embedding(struct ggml_tensor* timesteps, struct ggml_tensor* embedding, int dim, int max_period = 10000) {
|
||||
// timesteps: [N,]
|
||||
// embedding: [dim, N]
|
||||
int half = dim / 2;
|
||||
std::vector<float> freqs(half);
|
||||
for (int i = 0; i < half; ++i) {
|
||||
freqs[i] = (float)std::exp(-std::log(max_period) * i / half);
|
||||
}
|
||||
for (int i = 0; i < timesteps->ne[0]; ++i) {
|
||||
for (int j = 0; j < half; ++j) {
|
||||
float arg = ggml_get_f32_1d(timesteps, i) * freqs[j];
|
||||
ggml_tensor_set_f32(embedding, std::cos(arg), j, i);
|
||||
ggml_tensor_set_f32(embedding, std::sin(arg), j + half, i);
|
||||
}
|
||||
if (dim % 2 != 0) {
|
||||
*(float*)((char*)embedding->data + i * embedding->nb[1] + dim * embedding->nb[0]) = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool equalsf(float v1, float v2) {
|
||||
if (fabs(v1 - v2) <= 0.00001) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
struct ggml_tensor* new_timestep_embedding(struct ggml_context* ctx,
|
||||
struct ggml_tensor* timesteps,
|
||||
int dim,
|
||||
int max_period = 10000) {
|
||||
// timesteps: [N,]
|
||||
// embedding: [dim, N]
|
||||
int actual_dim = dim;
|
||||
struct ggml_tensor* embedding = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, actual_dim, timesteps->ne[0]);
|
||||
set_timestep_embedding(timesteps, embedding, dim, max_period);
|
||||
return embedding;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv) {
|
||||
std::vector<float> ts = {12, 24};
|
||||
int dim = 15;
|
||||
int max_period = 10000;
|
||||
std::vector<float> expected_result;
|
||||
{
|
||||
struct ggml_init_params params;
|
||||
params.mem_size = 16 * 1024 * 1024;
|
||||
params.mem_buffer = NULL;
|
||||
params.no_alloc = false;
|
||||
// memory allocation happens here
|
||||
struct ggml_context* ctx = ggml_init(params);
|
||||
|
||||
struct ggml_tensor* timesteps = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ts.size());
|
||||
memcpy(timesteps->data, ts.data(), ggml_nbytes(timesteps));
|
||||
struct ggml_tensor* embedding = new_timestep_embedding(ctx, timesteps, dim, max_period);
|
||||
expected_result.resize(ggml_nelements(embedding));
|
||||
|
||||
float* vec1 = ggml_get_data_f32(embedding);
|
||||
for (int i = 0; i < ggml_nelements(embedding); i++) {
|
||||
float value = vec1[i];
|
||||
expected_result[i] = value;
|
||||
printf("%.4f ", value);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
printf("-----------------------------------\n");
|
||||
{
|
||||
bool use_gpu = true; GGML_UNUSED(use_gpu);
|
||||
|
||||
ggml_backend_t backend = NULL;
|
||||
ggml_backend_buffer_t params_buffer = NULL;
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using CUDA backend\n", __func__);
|
||||
backend = ggml_backend_cuda_init(0);
|
||||
if (!backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
if (use_gpu) {
|
||||
fprintf(stderr, "%s: using Metal backend\n", __func__);
|
||||
backend = ggml_backend_metal_init();
|
||||
if (!backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
const int num_tensors = 2;
|
||||
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ ggml_tensor_overhead() * num_tensors + 2 * 1024 * 1024,
|
||||
/*.mem_size =*/ NULL,
|
||||
/*.mem_size =*/ true,
|
||||
};
|
||||
|
||||
if (!backend) {
|
||||
// fallback to CPU backend
|
||||
backend = ggml_backend_cpu_init();
|
||||
}
|
||||
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
|
||||
struct ggml_tensor * timesteps = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ts.size());
|
||||
|
||||
params_buffer = ggml_backend_alloc_ctx_tensors(ctx, backend);
|
||||
|
||||
// load data to buffer
|
||||
if (ggml_backend_is_cpu(backend)) {
|
||||
memcpy(timesteps->data, ts.data(), ggml_nbytes(timesteps));
|
||||
} else {
|
||||
ggml_backend_tensor_set(timesteps, ts.data(), 0, ggml_nbytes(timesteps));
|
||||
}
|
||||
|
||||
struct ggml_tensor * t = ggml_timestep_embedding(ctx, timesteps, dim, max_period);
|
||||
|
||||
ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));
|
||||
|
||||
struct ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, t);
|
||||
|
||||
ggml_gallocr_alloc_graph(galloc, graph);
|
||||
|
||||
int n_threads = 4;
|
||||
|
||||
if (ggml_backend_is_cpu(backend)) {
|
||||
ggml_backend_cpu_set_n_threads(backend, n_threads);
|
||||
}
|
||||
|
||||
ggml_backend_graph_compute(backend, graph);
|
||||
|
||||
float * output = new float[ggml_nelements(t)];
|
||||
ggml_backend_tensor_get(t, output, 0, ggml_nbytes(t));
|
||||
|
||||
GGML_ASSERT((size_t)ggml_nelements(t) == expected_result.size());
|
||||
|
||||
for (int i = 0; i < ggml_nelements(t); i++) {
|
||||
printf("%.4f ", output[i]);
|
||||
GGML_ASSERT(equalsf(output[i], expected_result[i]));
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
delete[] output;
|
||||
ggml_free(ctx);
|
||||
ggml_backend_buffer_free(params_buffer);
|
||||
ggml_backend_free(backend);
|
||||
ggml_gallocr_free(galloc);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user