initial release
This commit is contained in:
civ
@@ -0,0 +1,3 @@
|
||||
data/
|
||||
*.gguf
|
||||
*.ggml
|
||||
@@ -0,0 +1,58 @@
|
||||
#
|
||||
# mnist-common
|
||||
|
||||
set(TEST_TARGET mnist-common)
|
||||
add_library(${TEST_TARGET} STATIC mnist-common.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common)
|
||||
|
||||
#
|
||||
# mnist-eval
|
||||
|
||||
set(TEST_TARGET mnist-eval)
|
||||
add_executable(${TEST_TARGET} mnist-eval.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common)
|
||||
|
||||
#
|
||||
# mnist-train
|
||||
|
||||
set(TEST_TARGET mnist-train)
|
||||
add_executable(${TEST_TARGET} mnist-train.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common)
|
||||
|
||||
|
||||
#
|
||||
# mnist-wasm
|
||||
if (EMSCRIPTEN)
|
||||
set(TARGET mnist)
|
||||
|
||||
add_executable(${TARGET} mnist-common.cpp)
|
||||
target_link_libraries(${TARGET} PRIVATE ggml ggml-cpu)
|
||||
|
||||
set_target_properties(${TARGET} PROPERTIES LINK_FLAGS " \
|
||||
--bind \
|
||||
-s FORCE_FILESYSTEM=1 \
|
||||
-s USE_PTHREADS=1 \
|
||||
-s PTHREAD_POOL_SIZE=10 \
|
||||
-s ASSERTIONS=1 \
|
||||
-s WASM=1 \
|
||||
-s EXPORTED_RUNTIME_METHODS=\"['ccall', 'cwrap', 'setValue', 'getValue']\" \
|
||||
-s EXPORTED_FUNCTIONS=\"['_wasm_eval','_wasm_random_digit','_malloc','_free']\" \
|
||||
-s ALLOW_MEMORY_GROWTH=1 \
|
||||
--preload-file ${CMAKE_CURRENT_SOURCE_DIR}/mnist-f32.gguf@/ \
|
||||
--preload-file ${CMAKE_CURRENT_SOURCE_DIR}/t10k-images-idx3-ubyte@/ \
|
||||
")
|
||||
|
||||
# Copy output to web directory
|
||||
add_custom_command(
|
||||
TARGET ${TARGET} POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${CMAKE_BINARY_DIR}/bin/mnist.js
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.js
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${CMAKE_BINARY_DIR}/bin/mnist.wasm
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.wasm
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${CMAKE_BINARY_DIR}/bin/mnist.worker.js
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.worker.js
|
||||
)
|
||||
endif()
|
||||
@@ -0,0 +1,206 @@
|
||||
# MNIST Examples for GGML
|
||||
|
||||
This directory contains simple examples of how to use GGML for training and inference using the [MNIST dataset](https://yann.lecun.com/exdb/mnist/).
|
||||
All commands listed in this README assume the working directory to be `examples/mnist`.
|
||||
Please note that training in GGML is a work-in-progress and not production ready.
|
||||
|
||||
## Obtaining the data
|
||||
|
||||
A description of the dataset can be found on [Yann LeCun's website](https://yann.lecun.com/exdb/mnist/).
|
||||
While it is also in principle possible to download the dataset from this website these downloads are frequently throttled and
|
||||
it is recommended to use [HuggingFace](https://huggingface.co/datasets/ylecun/mnist) instead.
|
||||
The dataset will be downloaded automatically when running `mnist-train-fc.py`.
|
||||
|
||||
## Fully connected network
|
||||
|
||||
For our first example we will train a fully connected network.
|
||||
To train a fully connected model in PyTorch and save it as a GGUF file, run:
|
||||
|
||||
```bash
|
||||
$ python3 mnist-train-fc.py mnist-fc-f32.gguf
|
||||
|
||||
...
|
||||
|
||||
Test loss: 0.066377+-0.010468, Test accuracy: 97.94+-0.14%
|
||||
|
||||
Model tensors saved to mnist-fc-f32.gguf:
|
||||
fc1.weight (500, 784)
|
||||
fc1.bias (500,)
|
||||
fc2.weight (10, 500)
|
||||
fc2.bias (10,)
|
||||
```
|
||||
|
||||
The training script includes an evaluation of the model on the test set.
|
||||
To evaluate the model on the CPU using GGML, run:
|
||||
|
||||
```bash
|
||||
$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte
|
||||
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
__________________________________####__________________
|
||||
______________________________########__________________
|
||||
__________________________##########____________________
|
||||
______________________##############____________________
|
||||
____________________######________####__________________
|
||||
__________________________________####__________________
|
||||
__________________________________####__________________
|
||||
________________________________####____________________
|
||||
______________________________####______________________
|
||||
________________________##########______________________
|
||||
______________________########__####____________________
|
||||
________________________##__________##__________________
|
||||
____________________________________##__________________
|
||||
__________________________________##____________________
|
||||
__________________________________##____________________
|
||||
________________________________##______________________
|
||||
____________________________####________________________
|
||||
__________##____________######__________________________
|
||||
__________##############________________________________
|
||||
________________####____________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
|
||||
ggml_cuda_init: found 1 CUDA devices:
|
||||
Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
|
||||
mnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend
|
||||
mnist_model: unsupported operations will be executed on the following fallback backends (in order of priority):
|
||||
mnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor)
|
||||
mnist_model_init_from_file: loading model weights from 'mnist-fc-f32.gguf'
|
||||
mnist_model_init_from_file: model arch is mnist-fc
|
||||
mnist_model_init_from_file: successfully loaded weights from mnist-fc-f32.gguf
|
||||
main: loaded model in 109.44 ms
|
||||
mnist_model_eval: model evaluation on 10000 images took 76.92 ms, 7.69 us/image
|
||||
main: predicted digit is 3
|
||||
main: test_loss=0.066379+-0.009101
|
||||
main: test_acc=97.94+-0.14%
|
||||
```
|
||||
|
||||
In addition to the evaluation on the test set the GGML evaluation also prints a random image from the test set as well as the model prediction for said image.
|
||||
To train a fully connected model on the CPU using GGML run:
|
||||
|
||||
``` bash
|
||||
$ ../../build/bin/mnist-train mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte
|
||||
```
|
||||
|
||||
It can then be evaluated with the same binary as above.
|
||||
|
||||
## Convolutional network
|
||||
|
||||
To train a convolutional network using TensorFlow run:
|
||||
|
||||
```bash
|
||||
$ python3 mnist-train-cnn.py mnist-cnn-f32.gguf
|
||||
|
||||
...
|
||||
|
||||
Test loss: 0.047947
|
||||
Test accuracy: 98.46%
|
||||
GGUF model saved to 'mnist-cnn-f32.gguf'
|
||||
```
|
||||
|
||||
The saved model can be evaluated on the CPU using the `mnist-eval` binary:
|
||||
|
||||
```bash
|
||||
$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte
|
||||
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
______________________________________##________________
|
||||
______________________________________##________________
|
||||
______________________________________##________________
|
||||
____________________________________##__________________
|
||||
__________________________________####__________________
|
||||
__________________________________##____________________
|
||||
________________________________##______________________
|
||||
______________________________##________________________
|
||||
____________________________####________________________
|
||||
____________________________##__________________________
|
||||
__________________________##____________________________
|
||||
________________________##______________________________
|
||||
______________________##________________________________
|
||||
____________________####________________________________
|
||||
____________________##__________________________________
|
||||
__________________##____________________________________
|
||||
________________##______________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
|
||||
ggml_cuda_init: found 1 CUDA devices:
|
||||
Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
|
||||
mnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend
|
||||
mnist_model: unsupported operations will be executed on the following fallback backends (in order of priority):
|
||||
mnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor)
|
||||
mnist_model_init_from_file: loading model weights from 'mnist-cnn-f32.gguf'
|
||||
mnist_model_init_from_file: model arch is mnist-cnn
|
||||
mnist_model_init_from_file: successfully loaded weights from mnist-cnn-f32.gguf
|
||||
main: loaded model in 91.99 ms
|
||||
mnist_model_eval: model evaluation on 10000 images took 267.61 ms, 26.76 us/image
|
||||
main: predicted digit is 1
|
||||
main: test_loss=0.047955+-0.007029
|
||||
main: test_acc=98.46+-0.12%
|
||||
```
|
||||
|
||||
Like with the fully connected network the convolutional network can also be trained using GGML:
|
||||
|
||||
``` bash
|
||||
$ ../../build/bin/mnist-train mnist-cnn mnist-cnn-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte
|
||||
```
|
||||
|
||||
As always, the evaluation is done using `mnist-eval` and like with the fully connected network the GGML graph is exported to `mnist-cnn-f32.ggml`.
|
||||
|
||||
## Hardware Acceleration
|
||||
|
||||
Both the training and evaluation code is agnostic in terms of hardware as long as the corresponding GGML backend has implemented the necessary operations.
|
||||
A specific backend can be selected by appending the above commands with a backend name.
|
||||
The compute graphs then schedule the operations to preferentially use the specified backend.
|
||||
Note that if a backend does not implement some of the necessary operations a CPU fallback is used instead which may result in bad performance.
|
||||
|
||||
## Web demo
|
||||
|
||||
The evaluation code can be compiled to WebAssembly using [Emscripten](https://emscripten.org/) (may need to re-login to update `$PATH` after installation).
|
||||
First, copy the GGUF file of either of the trained models to `examples/mnist` and name it `mnist-f32.gguf`.
|
||||
Copy the test set to `examples/mnist` and name it `t10k-images-idx3-ubyte`.
|
||||
Symlinking these files will *not* work!
|
||||
Compile the code like so:
|
||||
|
||||
```bash
|
||||
$ cd ../../
|
||||
$ mkdir -p build-em
|
||||
$ emcmake cmake .. -DGGML_BUILD_EXAMPLES=ON \
|
||||
-DCMAKE_C_FLAGS="-pthread -matomics -mbulk-memory" \
|
||||
-DCMAKE_CXX_FLAGS="-pthread -matomics -mbulk-memory"
|
||||
$ make mnist
|
||||
```
|
||||
|
||||
The compilation output is copied into `examples/mnist/web`.
|
||||
To run it, you need an HTTP server.
|
||||
For example:
|
||||
|
||||
``` bash
|
||||
$ python3 examples/mnist/server.py
|
||||
|
||||
Serving directory '/home/danbev/work/ai/ggml/examples/mnist/web' at http://localhost:8000
|
||||
Application context root: http://localhost:8000/
|
||||
```
|
||||
|
||||
The web demo can then be accessed via the link printed on the console.
|
||||
Simply draw a digit on the canvas and the model will try to predict what it's supposed to be.
|
||||
Alternatively, click the "Random" button to retrieve a random digit from the test set.
|
||||
Be aware that like all neural networks the one we trained is susceptible to distributional shift:
|
||||
if the numbers you draw look different than the ones in the training set
|
||||
(e.g. because they're not centered) the model will perform comparatively worse.
|
||||
An online demo can be accessed [here](https://mnist.ggerganov.com).
|
||||
@@ -0,0 +1,496 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml-opt.h"
|
||||
|
||||
#include "mnist-common.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
bool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset) {
|
||||
auto fin = std::ifstream(fname, std::ios::binary);
|
||||
if (!fin) {
|
||||
fprintf(stderr, "failed to open images file %s\n", fname.c_str());
|
||||
return false;
|
||||
}
|
||||
fin.seekg(16);
|
||||
|
||||
uint8_t image[MNIST_NINPUT];
|
||||
struct ggml_tensor * images = ggml_opt_dataset_data(dataset);
|
||||
float * buf = ggml_get_data_f32(images);
|
||||
|
||||
GGML_ASSERT(images->ne[0] == MNIST_NINPUT);
|
||||
for (int64_t iex = 0; iex < images->ne[1]; ++iex) {
|
||||
fin.read((char *) image, sizeof(image));
|
||||
|
||||
for (int64_t i = 0; i < MNIST_NINPUT; ++i) {
|
||||
buf[iex*MNIST_NINPUT + i] = image[i] / 255.0f; // Normalize to [0, 1]
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void mnist_image_print(FILE * stream, ggml_opt_dataset_t dataset, const int iex) {
|
||||
struct ggml_tensor * images = ggml_opt_dataset_data(dataset);
|
||||
GGML_ASSERT(images->ne[0] == MNIST_NINPUT);
|
||||
GGML_ASSERT(iex < images->ne[1]);
|
||||
const float * image = ggml_get_data_f32(images) + iex*MNIST_NINPUT;
|
||||
|
||||
for (int64_t row = 0; row < MNIST_HW; row++) {
|
||||
for (int64_t col = 0; col < MNIST_HW; col++) {
|
||||
const int rgb = roundf(255.0f * image[row*MNIST_HW + col]);
|
||||
#ifdef _WIN32
|
||||
fprintf(stream, "%s", rgb >= 220 ? "##" : "__"); // Represented via text.
|
||||
#else
|
||||
fprintf(stream, "\033[48;2;%d;%d;%dm \033[0m", rgb, rgb, rgb); // Represented via colored blocks.
|
||||
#endif // _WIN32
|
||||
}
|
||||
fprintf(stream, "\n");
|
||||
}
|
||||
}
|
||||
|
||||
bool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset) {
|
||||
auto fin = std::ifstream(fname, std::ios::binary);
|
||||
if (!fin) {
|
||||
fprintf(stderr, "failed to open labels file %s\n", fname.c_str());
|
||||
return 0;
|
||||
}
|
||||
fin.seekg(8);
|
||||
|
||||
uint8_t label;
|
||||
struct ggml_tensor * labels = ggml_opt_dataset_labels(dataset);
|
||||
float * buf = ggml_get_data_f32(labels);
|
||||
|
||||
GGML_ASSERT(labels->ne[0] == MNIST_NCLASSES);
|
||||
for (int64_t iex = 0; iex < labels->ne[1]; ++iex) {
|
||||
fin.read((char *) &label, sizeof(label));
|
||||
|
||||
for (int64_t i = 0; i < MNIST_NCLASSES; ++i) {
|
||||
buf[iex*MNIST_NCLASSES + i] = i == label ? 1.0f : 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Temporary util function for loading data from GGUF to a backend != CPU until GGML itself provides this functionality:
|
||||
bool load_from_gguf(const char * fname, struct ggml_context * ctx_ggml, struct gguf_context * ctx_gguf) {
|
||||
FILE * f = ggml_fopen(fname, "rb");
|
||||
if (!f) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const size_t buf_size = 4*1024*1024;
|
||||
void * buf = malloc(buf_size);
|
||||
|
||||
const int n_tensors = gguf_get_n_tensors(ctx_gguf);
|
||||
for (int i = 0; i < n_tensors; i++) {
|
||||
const char * name = gguf_get_tensor_name(ctx_gguf, i);
|
||||
|
||||
struct ggml_tensor * tensor = ggml_get_tensor(ctx_ggml, name);
|
||||
if (!tensor) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i);
|
||||
|
||||
if (fseek(f, offs, SEEK_SET) != 0) {
|
||||
fclose(f);
|
||||
free(buf);
|
||||
return false;
|
||||
}
|
||||
|
||||
const size_t nbytes = ggml_nbytes(tensor);
|
||||
for (size_t pos = 0; pos < nbytes; pos += buf_size) {
|
||||
const size_t nbytes_cpy = buf_size < nbytes - pos ? buf_size : nbytes - pos;
|
||||
|
||||
if (fread(buf, 1, nbytes_cpy, f) != nbytes_cpy) {
|
||||
fclose(f);
|
||||
free(buf);
|
||||
return false;
|
||||
}
|
||||
|
||||
ggml_backend_tensor_set(tensor, buf, pos, nbytes_cpy);
|
||||
}
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
free(buf);
|
||||
return true;
|
||||
}
|
||||
|
||||
mnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical) {
|
||||
mnist_model model(backend, nbatch_logical, nbatch_physical);
|
||||
fprintf(stderr, "%s: loading model weights from '%s'\n", __func__, fname.c_str());
|
||||
|
||||
struct gguf_context * ctx;
|
||||
{
|
||||
struct gguf_init_params params = {
|
||||
/*.no_alloc =*/ true,
|
||||
/*.ctx =*/ &model.ctx_gguf,
|
||||
};
|
||||
ctx = gguf_init_from_file(fname.c_str(), params);
|
||||
if (!ctx) {
|
||||
fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
model.arch = gguf_get_val_str(ctx, gguf_find_key(ctx, "general.architecture"));
|
||||
fprintf(stderr, "%s: model arch is %s\n", __func__, model.arch.c_str());
|
||||
|
||||
if (model.arch == "mnist-fc") {
|
||||
model.fc1_weight = ggml_get_tensor(model.ctx_gguf, "fc1.weight");
|
||||
GGML_ASSERT(model.fc1_weight->ne[0] == MNIST_NINPUT);
|
||||
GGML_ASSERT(model.fc1_weight->ne[1] == MNIST_NHIDDEN);
|
||||
GGML_ASSERT(model.fc1_weight->ne[2] == 1);
|
||||
GGML_ASSERT(model.fc1_weight->ne[3] == 1);
|
||||
|
||||
model.fc1_bias = ggml_get_tensor(model.ctx_gguf, "fc1.bias");
|
||||
GGML_ASSERT(model.fc1_bias->ne[0] == MNIST_NHIDDEN);
|
||||
GGML_ASSERT(model.fc1_bias->ne[1] == 1);
|
||||
GGML_ASSERT(model.fc1_bias->ne[2] == 1);
|
||||
GGML_ASSERT(model.fc1_bias->ne[3] == 1);
|
||||
|
||||
model.fc2_weight = ggml_get_tensor(model.ctx_gguf, "fc2.weight");
|
||||
GGML_ASSERT(model.fc2_weight->ne[0] == MNIST_NHIDDEN);
|
||||
GGML_ASSERT(model.fc2_weight->ne[1] == MNIST_NCLASSES);
|
||||
GGML_ASSERT(model.fc2_weight->ne[2] == 1);
|
||||
GGML_ASSERT(model.fc2_weight->ne[3] == 1);
|
||||
|
||||
model.fc2_bias = ggml_get_tensor(model.ctx_gguf, "fc2.bias");
|
||||
GGML_ASSERT(model.fc2_bias->ne[0] == MNIST_NCLASSES);
|
||||
GGML_ASSERT(model.fc2_bias->ne[1] == 1);
|
||||
GGML_ASSERT(model.fc2_bias->ne[2] == 1);
|
||||
GGML_ASSERT(model.fc2_bias->ne[3] == 1);
|
||||
} else if (model.arch == "mnist-cnn") {
|
||||
model.conv1_kernel = ggml_get_tensor(model.ctx_gguf, "conv1.kernel");
|
||||
GGML_ASSERT(model.conv1_kernel->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.conv1_kernel->ne[0] == 3);
|
||||
GGML_ASSERT(model.conv1_kernel->ne[1] == 3);
|
||||
GGML_ASSERT(model.conv1_kernel->ne[2] == 1);
|
||||
GGML_ASSERT(model.conv1_kernel->ne[3] == MNIST_CNN_NCB);
|
||||
|
||||
model.conv1_bias = ggml_get_tensor(model.ctx_gguf, "conv1.bias");
|
||||
GGML_ASSERT(model.conv1_bias->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.conv1_bias->ne[0] == 1);
|
||||
GGML_ASSERT(model.conv1_bias->ne[1] == 1);
|
||||
GGML_ASSERT(model.conv1_bias->ne[2] == MNIST_CNN_NCB);
|
||||
GGML_ASSERT(model.conv1_bias->ne[3] == 1);
|
||||
|
||||
model.conv2_kernel = ggml_get_tensor(model.ctx_gguf, "conv2.kernel");
|
||||
GGML_ASSERT(model.conv2_kernel->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.conv2_kernel->ne[0] == 3);
|
||||
GGML_ASSERT(model.conv2_kernel->ne[1] == 3);
|
||||
GGML_ASSERT(model.conv2_kernel->ne[2] == MNIST_CNN_NCB);
|
||||
GGML_ASSERT(model.conv2_kernel->ne[3] == MNIST_CNN_NCB*2);
|
||||
|
||||
model.conv2_bias = ggml_get_tensor(model.ctx_gguf, "conv2.bias");
|
||||
GGML_ASSERT(model.conv2_bias->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.conv2_bias->ne[0] == 1);
|
||||
GGML_ASSERT(model.conv2_bias->ne[1] == 1);
|
||||
GGML_ASSERT(model.conv2_bias->ne[2] == MNIST_CNN_NCB*2);
|
||||
GGML_ASSERT(model.conv2_bias->ne[3] == 1);
|
||||
|
||||
model.dense_weight = ggml_get_tensor(model.ctx_gguf, "dense.weight");
|
||||
GGML_ASSERT(model.dense_weight->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.dense_weight->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2));
|
||||
GGML_ASSERT(model.dense_weight->ne[1] == MNIST_NCLASSES);
|
||||
GGML_ASSERT(model.dense_weight->ne[2] == 1);
|
||||
GGML_ASSERT(model.dense_weight->ne[3] == 1);
|
||||
|
||||
model.dense_bias = ggml_get_tensor(model.ctx_gguf, "dense.bias");
|
||||
GGML_ASSERT(model.dense_bias->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.dense_bias->ne[0] == MNIST_NCLASSES);
|
||||
GGML_ASSERT(model.dense_bias->ne[1] == 1);
|
||||
GGML_ASSERT(model.dense_bias->ne[2] == 1);
|
||||
GGML_ASSERT(model.dense_bias->ne[3] == 1);
|
||||
} else {
|
||||
fprintf(stderr, "%s: unknown model arch: %s\n", __func__, model.arch.c_str());
|
||||
}
|
||||
|
||||
model.buf_gguf = ggml_backend_alloc_ctx_tensors(model.ctx_gguf, model.backends[0]);
|
||||
|
||||
if(!load_from_gguf(fname.c_str(), model.ctx_gguf, ctx)) {
|
||||
fprintf(stderr, "%s: loading weights from %s failed\n", __func__, fname.c_str());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// The space in ctx_gguf exactly fits the model weights,
|
||||
// the images (which also need to be statically allocated) need to be put in a different context.
|
||||
|
||||
model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, nbatch_physical);
|
||||
|
||||
ggml_set_name(model.images, "images");
|
||||
ggml_set_input(model.images);
|
||||
|
||||
model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]);
|
||||
|
||||
fprintf(stderr, "%s: successfully loaded weights from %s\n", __func__, fname.c_str());
|
||||
return model;
|
||||
}
|
||||
|
||||
mnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical) {
|
||||
mnist_model model(backend, nbatch_logical, nbatch_physical);
|
||||
model.arch = arch;
|
||||
|
||||
std::random_device rd{};
|
||||
std::mt19937 gen{rd()};
|
||||
std::normal_distribution<float> nd{0.0f, 1e-2f};
|
||||
std::vector<ggml_tensor *> init_tensors;
|
||||
|
||||
if (model.arch == "mnist-fc") {
|
||||
fprintf(stderr, "%s: initializing random weights for a fully connected model\n", __func__);
|
||||
|
||||
model.fc1_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NHIDDEN);
|
||||
model.fc1_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN);
|
||||
model.fc2_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN, MNIST_NCLASSES);
|
||||
model.fc2_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES);
|
||||
|
||||
ggml_set_name(model.fc1_weight, "fc1.weight");
|
||||
ggml_set_name(model.fc1_bias, "fc1.bias");
|
||||
ggml_set_name(model.fc2_weight, "fc2.weight");
|
||||
ggml_set_name(model.fc2_bias, "fc2.bias");
|
||||
|
||||
init_tensors.push_back(model.fc1_weight);
|
||||
init_tensors.push_back(model.fc1_bias);
|
||||
init_tensors.push_back(model.fc2_weight);
|
||||
init_tensors.push_back(model.fc2_bias);
|
||||
} else if (model.arch == "mnist-cnn") {
|
||||
model.conv1_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, 1, MNIST_CNN_NCB);
|
||||
model.conv1_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB);
|
||||
model.conv2_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, MNIST_CNN_NCB, MNIST_CNN_NCB*2);
|
||||
model.conv2_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB*2);
|
||||
model.dense_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), MNIST_NCLASSES);
|
||||
model.dense_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES);
|
||||
|
||||
ggml_set_name(model.conv1_kernel, "conv1.kernel");
|
||||
ggml_set_name(model.conv1_bias, "conv1.bias");
|
||||
ggml_set_name(model.conv2_kernel, "conv2.kernel");
|
||||
ggml_set_name(model.conv2_bias, "conv2.bias");
|
||||
ggml_set_name(model.dense_weight, "dense.weight");
|
||||
ggml_set_name(model.dense_bias, "dense.bias");
|
||||
|
||||
init_tensors.push_back(model.conv1_kernel);
|
||||
init_tensors.push_back(model.conv1_bias);
|
||||
init_tensors.push_back(model.conv2_kernel);
|
||||
init_tensors.push_back(model.conv2_bias);
|
||||
init_tensors.push_back(model.dense_weight);
|
||||
init_tensors.push_back(model.dense_bias);
|
||||
} else {
|
||||
fprintf(stderr, "%s: unknown model arch: %s\n", __func__, model.arch.c_str());
|
||||
}
|
||||
|
||||
model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NBATCH_PHYSICAL);
|
||||
ggml_set_name(model.images, "images");
|
||||
ggml_set_input(model.images);
|
||||
|
||||
model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]);
|
||||
|
||||
for (ggml_tensor * t : init_tensors) {
|
||||
GGML_ASSERT(t->type == GGML_TYPE_F32);
|
||||
const int64_t ne = ggml_nelements(t);
|
||||
std::vector<float> tmp(ne);
|
||||
|
||||
for (int64_t i = 0; i < ne; ++i) {
|
||||
tmp[i] = nd(gen);
|
||||
}
|
||||
ggml_backend_tensor_set(t, tmp.data(), 0, ggml_nbytes(t));
|
||||
}
|
||||
|
||||
return model;
|
||||
}
|
||||
|
||||
void mnist_model_build(mnist_model & model) {
|
||||
if (model.arch == "mnist-fc") {
|
||||
ggml_set_param(model.fc1_weight);
|
||||
ggml_set_param(model.fc1_bias);
|
||||
ggml_set_param(model.fc2_weight);
|
||||
ggml_set_param(model.fc2_bias);
|
||||
|
||||
ggml_tensor * fc1 = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
|
||||
ggml_mul_mat(model.ctx_compute, model.fc1_weight, model.images),
|
||||
model.fc1_bias));
|
||||
model.logits = ggml_add(model.ctx_compute,
|
||||
ggml_mul_mat(model.ctx_compute, model.fc2_weight, fc1),
|
||||
model.fc2_bias);
|
||||
} else if (model.arch == "mnist-cnn") {
|
||||
ggml_set_param(model.conv1_kernel);
|
||||
ggml_set_param(model.conv1_bias);
|
||||
ggml_set_param(model.conv2_kernel);
|
||||
ggml_set_param(model.conv2_bias);
|
||||
ggml_set_param(model.dense_weight);
|
||||
ggml_set_param(model.dense_bias);
|
||||
|
||||
struct ggml_tensor * images_2D = ggml_reshape_4d(model.ctx_compute, model.images, MNIST_HW, MNIST_HW, 1, model.images->ne[1]);
|
||||
|
||||
struct ggml_tensor * conv1_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
|
||||
ggml_conv_2d(model.ctx_compute, model.conv1_kernel, images_2D, 1, 1, 1, 1, 1, 1),
|
||||
model.conv1_bias));
|
||||
GGML_ASSERT(conv1_out->ne[0] == MNIST_HW);
|
||||
GGML_ASSERT(conv1_out->ne[1] == MNIST_HW);
|
||||
GGML_ASSERT(conv1_out->ne[2] == MNIST_CNN_NCB);
|
||||
GGML_ASSERT(conv1_out->ne[3] == model.nbatch_physical);
|
||||
|
||||
struct ggml_tensor * conv2_in = ggml_pool_2d(model.ctx_compute, conv1_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);
|
||||
GGML_ASSERT(conv2_in->ne[0] == MNIST_HW/2);
|
||||
GGML_ASSERT(conv2_in->ne[1] == MNIST_HW/2);
|
||||
GGML_ASSERT(conv2_in->ne[2] == MNIST_CNN_NCB);
|
||||
GGML_ASSERT(conv2_in->ne[3] == model.nbatch_physical);
|
||||
|
||||
struct ggml_tensor * conv2_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
|
||||
ggml_conv_2d(model.ctx_compute, model.conv2_kernel, conv2_in, 1, 1, 1, 1, 1, 1),
|
||||
model.conv2_bias));
|
||||
GGML_ASSERT(conv2_out->ne[0] == MNIST_HW/2);
|
||||
GGML_ASSERT(conv2_out->ne[1] == MNIST_HW/2);
|
||||
GGML_ASSERT(conv2_out->ne[2] == MNIST_CNN_NCB*2);
|
||||
GGML_ASSERT(conv2_out->ne[3] == model.nbatch_physical);
|
||||
|
||||
struct ggml_tensor * dense_in = ggml_pool_2d(model.ctx_compute, conv2_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);
|
||||
GGML_ASSERT(dense_in->ne[0] == MNIST_HW/4);
|
||||
GGML_ASSERT(dense_in->ne[1] == MNIST_HW/4);
|
||||
GGML_ASSERT(dense_in->ne[2] == MNIST_CNN_NCB*2);
|
||||
GGML_ASSERT(dense_in->ne[3] == model.nbatch_physical);
|
||||
|
||||
dense_in = ggml_reshape_2d(model.ctx_compute,
|
||||
ggml_cont(model.ctx_compute, ggml_permute(model.ctx_compute, dense_in, 1, 2, 0, 3)),
|
||||
(MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), model.nbatch_physical);
|
||||
GGML_ASSERT(dense_in->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2));
|
||||
GGML_ASSERT(dense_in->ne[1] == model.nbatch_physical);
|
||||
GGML_ASSERT(dense_in->ne[2] == 1);
|
||||
GGML_ASSERT(dense_in->ne[3] == 1);
|
||||
|
||||
model.logits = ggml_add(model.ctx_compute, ggml_mul_mat(model.ctx_compute, model.dense_weight, dense_in), model.dense_bias);
|
||||
} else {
|
||||
GGML_ASSERT(false);
|
||||
}
|
||||
|
||||
ggml_set_name(model.logits, "logits");
|
||||
ggml_set_output(model.logits);
|
||||
GGML_ASSERT(model.logits->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.logits->ne[0] == MNIST_NCLASSES);
|
||||
GGML_ASSERT(model.logits->ne[1] == model.nbatch_physical);
|
||||
GGML_ASSERT(model.logits->ne[2] == 1);
|
||||
GGML_ASSERT(model.logits->ne[3] == 1);
|
||||
}
|
||||
|
||||
ggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset) {
|
||||
ggml_opt_result_t result = ggml_opt_result_init();
|
||||
|
||||
ggml_opt_params params = ggml_opt_default_params(model.backend_sched, GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);
|
||||
params.ctx_compute = model.ctx_compute;
|
||||
params.inputs = model.images;
|
||||
params.outputs = model.logits;
|
||||
params.build_type = GGML_OPT_BUILD_TYPE_FORWARD;
|
||||
ggml_opt_context_t opt_ctx = ggml_opt_init(params);
|
||||
|
||||
{
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
ggml_opt_epoch(opt_ctx, dataset, nullptr, result, /*idata_split =*/ 0, nullptr, nullptr);
|
||||
|
||||
const int64_t t_total_us = ggml_time_us() - t_start_us;
|
||||
const double t_total_ms = 1e-3*t_total_us;
|
||||
const int nex = ggml_opt_dataset_data(dataset)->ne[1];
|
||||
fprintf(stderr, "%s: model evaluation on %d images took %.2lf ms, %.2lf us/image\n",
|
||||
__func__, nex, t_total_ms, (double) t_total_us/nex);
|
||||
}
|
||||
|
||||
ggml_opt_free(opt_ctx);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split) {
|
||||
ggml_opt_fit(model.backend_sched, model.ctx_compute, model.images, model.logits, dataset,
|
||||
GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, GGML_OPT_OPTIMIZER_TYPE_ADAMW, ggml_opt_get_default_optimizer_params, nepoch, model.nbatch_logical, val_split, false);
|
||||
}
|
||||
|
||||
void mnist_model_save(mnist_model & model, const std::string & fname) {
|
||||
printf("%s: saving model to '%s'\n", __func__, fname.c_str());
|
||||
|
||||
struct ggml_context * ggml_ctx;
|
||||
{
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ 100 * 1024*1024,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ false,
|
||||
};
|
||||
ggml_ctx = ggml_init(params);
|
||||
}
|
||||
|
||||
gguf_context * gguf_ctx = gguf_init_empty();
|
||||
gguf_set_val_str(gguf_ctx, "general.architecture", model.arch.c_str());
|
||||
|
||||
std::vector<struct ggml_tensor *> weights;
|
||||
if (model.arch == "mnist-fc") {
|
||||
weights = {model.fc1_weight, model.fc1_bias, model.fc2_weight, model.fc2_bias};
|
||||
} else if (model.arch == "mnist-cnn") {
|
||||
weights = {model.conv1_kernel, model.conv1_bias, model.conv2_kernel, model.conv2_bias, model.dense_weight, model.dense_bias};
|
||||
} else {
|
||||
GGML_ASSERT(false);
|
||||
}
|
||||
for (struct ggml_tensor * t : weights) {
|
||||
struct ggml_tensor * copy = ggml_dup_tensor(ggml_ctx, t);
|
||||
ggml_set_name(copy, t->name);
|
||||
ggml_backend_tensor_get(t, copy->data, 0, ggml_nbytes(t));
|
||||
gguf_add_tensor(gguf_ctx, copy);
|
||||
}
|
||||
gguf_write_to_file(gguf_ctx, fname.c_str(), false);
|
||||
|
||||
ggml_free(ggml_ctx);
|
||||
gguf_free(gguf_ctx);
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
int wasm_eval(uint8_t * digitPtr) {
|
||||
std::vector<float> digit(digitPtr, digitPtr + MNIST_NINPUT);
|
||||
|
||||
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, 1, 1);
|
||||
struct ggml_tensor * data = ggml_opt_dataset_data(dataset);
|
||||
|
||||
float * buf = ggml_get_data_f32(data);
|
||||
for (int i = 0; i < MNIST_NINPUT; ++i) {
|
||||
buf[i] = digitPtr[i] / 255.0f;
|
||||
}
|
||||
ggml_set_zero(ggml_opt_dataset_labels(dataset)); // The labels are not needed.
|
||||
|
||||
mnist_model model = mnist_model_init_from_file("mnist-f32.gguf", "CPU", /*nbatch_logical =*/ 1, /*nbatch_physical =*/ 1);
|
||||
mnist_model_build(model);
|
||||
ggml_opt_result_t result = mnist_model_eval(model, dataset);
|
||||
|
||||
int32_t pred;
|
||||
ggml_opt_result_pred(result, &pred);
|
||||
|
||||
return pred;
|
||||
}
|
||||
|
||||
int wasm_random_digit(char * digitPtr) {
|
||||
auto fin = std::ifstream("t10k-images-idx3-ubyte", std::ios::binary);
|
||||
if (!fin) {
|
||||
fprintf(stderr, "failed to open digits file\n");
|
||||
return 0;
|
||||
}
|
||||
srand(time(NULL));
|
||||
|
||||
// Seek to a random digit: 16-byte header + 28*28 * (random 0 - 10000)
|
||||
fin.seekg(16 + MNIST_NINPUT * (rand() % MNIST_NTEST));
|
||||
fin.read(digitPtr, MNIST_NINPUT);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,166 @@
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml.h"
|
||||
#include "gguf.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-opt.h"
|
||||
|
||||
#define MNIST_NTRAIN 60000
|
||||
#define MNIST_NTEST 10000
|
||||
|
||||
// Gradient accumulation can be achieved by setting the logical batch size to a multiple of the physical one.
|
||||
// The logical batch size determines how many datapoints are used for a gradient update.
|
||||
// The physical batch size determines how many datapoints are processed in parallel, larger values utilize compute better but need more memory.
|
||||
#define MNIST_NBATCH_LOGICAL 1000
|
||||
#define MNIST_NBATCH_PHYSICAL 500
|
||||
|
||||
static_assert(MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL == 0, "MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL != 0");
|
||||
static_assert(MNIST_NTRAIN % MNIST_NBATCH_LOGICAL == 0, "MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0");
|
||||
static_assert(MNIST_NTEST % MNIST_NBATCH_LOGICAL == 0, "MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0");
|
||||
|
||||
#define MNIST_HW 28
|
||||
#define MNIST_NINPUT (MNIST_HW*MNIST_HW)
|
||||
#define MNIST_NCLASSES 10
|
||||
|
||||
#define MNIST_NHIDDEN 500
|
||||
|
||||
// NCB = number of channels base
|
||||
#define MNIST_CNN_NCB 8
|
||||
|
||||
struct mnist_model {
|
||||
std::string arch;
|
||||
ggml_backend_sched_t backend_sched;
|
||||
std::vector<ggml_backend_t> backends;
|
||||
const int nbatch_logical;
|
||||
const int nbatch_physical;
|
||||
|
||||
struct ggml_tensor * images = nullptr;
|
||||
struct ggml_tensor * logits = nullptr;
|
||||
|
||||
struct ggml_tensor * fc1_weight = nullptr;
|
||||
struct ggml_tensor * fc1_bias = nullptr;
|
||||
struct ggml_tensor * fc2_weight = nullptr;
|
||||
struct ggml_tensor * fc2_bias = nullptr;
|
||||
|
||||
struct ggml_tensor * conv1_kernel = nullptr;
|
||||
struct ggml_tensor * conv1_bias = nullptr;
|
||||
struct ggml_tensor * conv2_kernel = nullptr;
|
||||
struct ggml_tensor * conv2_bias = nullptr;
|
||||
struct ggml_tensor * dense_weight = nullptr;
|
||||
struct ggml_tensor * dense_bias = nullptr;
|
||||
|
||||
struct ggml_context * ctx_gguf = nullptr;
|
||||
struct ggml_context * ctx_static = nullptr;
|
||||
struct ggml_context * ctx_compute = nullptr;
|
||||
ggml_backend_buffer_t buf_gguf = nullptr;
|
||||
ggml_backend_buffer_t buf_static = nullptr;
|
||||
|
||||
mnist_model(const std::string & backend_name, const int nbatch_logical, const int nbatch_physical)
|
||||
: nbatch_logical(nbatch_logical), nbatch_physical(nbatch_physical) {
|
||||
std::vector<ggml_backend_dev_t> devices;
|
||||
const int ncores_logical = std::thread::hardware_concurrency();
|
||||
const int nthreads = std::min(ncores_logical, (ncores_logical + 4) / 2);
|
||||
|
||||
// Add primary backend:
|
||||
if (!backend_name.empty()) {
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_by_name(backend_name.c_str());
|
||||
if (dev == nullptr) {
|
||||
fprintf(stderr, "%s: ERROR: backend %s not found, available:\n", __func__, backend_name.c_str());
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
|
||||
ggml_backend_dev_t dev_i = ggml_backend_dev_get(i);
|
||||
fprintf(stderr, " - %s (%s)\n", ggml_backend_dev_name(dev_i), ggml_backend_dev_description(dev_i));
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
|
||||
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
|
||||
GGML_ASSERT(backend);
|
||||
|
||||
if (ggml_backend_is_cpu(backend)) {
|
||||
ggml_backend_cpu_set_n_threads(backend, nthreads);
|
||||
}
|
||||
|
||||
backends.push_back(backend);
|
||||
devices.push_back(dev);
|
||||
}
|
||||
|
||||
// Add all available backends as fallback.
|
||||
// A "backend" is a stream on a physical device so there is no problem with adding multiple backends for the same device.
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
|
||||
|
||||
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
|
||||
GGML_ASSERT(backend);
|
||||
|
||||
if (ggml_backend_is_cpu(backend)) {
|
||||
ggml_backend_cpu_set_n_threads(backend, nthreads);
|
||||
}
|
||||
|
||||
backends.push_back(backend);
|
||||
devices.push_back(dev);
|
||||
}
|
||||
|
||||
// The order of the backends passed to ggml_backend_sched_new determines which backend is given priority.
|
||||
backend_sched = ggml_backend_sched_new(backends.data(), nullptr, backends.size(), GGML_DEFAULT_GRAPH_SIZE, false, true);
|
||||
fprintf(stderr, "%s: using %s (%s) as primary backend\n",
|
||||
__func__, ggml_backend_name(backends[0]), ggml_backend_dev_description(devices[0]));
|
||||
if (backends.size() >= 2) {
|
||||
fprintf(stderr, "%s: unsupported operations will be executed on the following fallback backends (in order of priority):\n", __func__);
|
||||
for (size_t i = 1; i < backends.size(); ++i) {
|
||||
fprintf(stderr, "%s: - %s (%s)\n", __func__, ggml_backend_name(backends[i]), ggml_backend_dev_description(devices[i]));
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const size_t size_meta = 1024*ggml_tensor_overhead();
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ size_meta,
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ctx_static = ggml_init(params);
|
||||
}
|
||||
|
||||
{
|
||||
// The compute context needs a total of 3 compute graphs: forward pass + backwards pass (with/without optimizer step).
|
||||
const size_t size_meta = GGML_DEFAULT_GRAPH_SIZE*ggml_tensor_overhead() + 3*ggml_graph_overhead();
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ size_meta,
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ctx_compute = ggml_init(params);
|
||||
}
|
||||
}
|
||||
|
||||
~mnist_model() {
|
||||
ggml_free(ctx_gguf);
|
||||
ggml_free(ctx_static);
|
||||
ggml_free(ctx_compute);
|
||||
|
||||
ggml_backend_buffer_free(buf_gguf);
|
||||
ggml_backend_buffer_free(buf_static);
|
||||
ggml_backend_sched_free(backend_sched);
|
||||
for (ggml_backend_t backend : backends) {
|
||||
ggml_backend_free(backend);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
bool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset);
|
||||
void mnist_image_print(FILE * f, ggml_opt_dataset_t dataset, const int iex);
|
||||
bool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset);
|
||||
|
||||
mnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical);
|
||||
mnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical);
|
||||
void mnist_model_build(mnist_model & model);
|
||||
ggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset);
|
||||
void mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split);
|
||||
void mnist_model_save(mnist_model & model, const std::string & fname);
|
||||
@@ -0,0 +1,67 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-opt.h"
|
||||
|
||||
#include "mnist-common.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
srand(time(NULL));
|
||||
ggml_time_init();
|
||||
|
||||
if (argc != 4 && argc != 5) {
|
||||
fprintf(stderr, "Usage: %s mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTEST, MNIST_NBATCH_PHYSICAL);
|
||||
|
||||
if (!mnist_image_load(argv[2], dataset)) {
|
||||
return 1;
|
||||
}
|
||||
if (!mnist_label_load(argv[3], dataset)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const int iex = rand() % MNIST_NTEST;
|
||||
mnist_image_print(stdout, dataset, iex);
|
||||
|
||||
const std::string backend = argc >= 5 ? argv[4] : "";
|
||||
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
mnist_model model = mnist_model_init_from_file(argv[1], backend, MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);
|
||||
mnist_model_build(model);
|
||||
const int64_t t_load_us = ggml_time_us() - t_start_us;
|
||||
fprintf(stdout, "%s: loaded model in %.2lf ms\n", __func__, t_load_us / 1000.0);
|
||||
|
||||
ggml_opt_result_t result_eval = mnist_model_eval(model, dataset);
|
||||
|
||||
std::vector<int32_t> pred(MNIST_NTEST);
|
||||
ggml_opt_result_pred(result_eval, pred.data());
|
||||
fprintf(stdout, "%s: predicted digit is %d\n", __func__, pred[iex]);
|
||||
|
||||
double loss;
|
||||
double loss_unc;
|
||||
ggml_opt_result_loss(result_eval, &loss, &loss_unc);
|
||||
fprintf(stdout, "%s: test_loss=%.6lf+-%.6lf\n", __func__, loss, loss_unc);
|
||||
|
||||
double accuracy;
|
||||
double accuracy_unc;
|
||||
ggml_opt_result_accuracy(result_eval, &accuracy, &accuracy_unc);
|
||||
fprintf(stdout, "%s: test_acc=%.2lf+-%.2lf%%\n", __func__, 100.0*accuracy, 100.0*accuracy_unc);
|
||||
|
||||
ggml_opt_result_free(result_eval);
|
||||
|
||||
return 0;
|
||||
}
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from time import time
|
||||
import gguf
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
from tensorflow import keras
|
||||
from tensorflow.keras import layers
|
||||
|
||||
|
||||
def train(model_path):
|
||||
# Model / data parameters
|
||||
num_classes = 10
|
||||
input_shape = (28, 28, 1)
|
||||
|
||||
# Load the data and split it between train and test sets
|
||||
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
|
||||
|
||||
# Scale images to the [0, 1] range
|
||||
x_train = x_train.astype("float32") / 255
|
||||
x_test = x_test.astype("float32") / 255
|
||||
x_train = np.expand_dims(x_train, -1)
|
||||
x_test = np.expand_dims(x_test, -1)
|
||||
print("x_train shape:", x_train.shape)
|
||||
print(x_train.shape[0], "train samples")
|
||||
print(x_test.shape[0], "test samples")
|
||||
|
||||
# convert class vectors to binary class matrices
|
||||
y_train = keras.utils.to_categorical(y_train, num_classes)
|
||||
y_test = keras.utils.to_categorical(y_test, num_classes)
|
||||
|
||||
model = keras.Sequential(
|
||||
[
|
||||
keras.Input(shape=input_shape, dtype=tf.float32),
|
||||
layers.Conv2D(8, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32),
|
||||
layers.MaxPooling2D(pool_size=(2, 2)),
|
||||
layers.Conv2D(16, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32),
|
||||
layers.MaxPooling2D(pool_size=(2, 2)),
|
||||
layers.Flatten(),
|
||||
layers.Dense(num_classes, activation="softmax", dtype=tf.float32),
|
||||
]
|
||||
)
|
||||
|
||||
model.summary()
|
||||
batch_size = 1000
|
||||
epochs = 30
|
||||
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
|
||||
|
||||
t_start = time()
|
||||
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)
|
||||
print(f"Training took {time()-t_start:.2f}s")
|
||||
|
||||
score = model.evaluate(x_test, y_test, verbose=0)
|
||||
print(f"Test loss: {score[0]:.6f}")
|
||||
print(f"Test accuracy: {100*score[1]:.2f}%")
|
||||
|
||||
gguf_writer = gguf.GGUFWriter(model_path, "mnist-cnn")
|
||||
|
||||
conv1_kernel = model.layers[0].weights[0].numpy()
|
||||
conv1_kernel = np.moveaxis(conv1_kernel, [2, 3], [0, 1])
|
||||
gguf_writer.add_tensor("conv1.kernel", conv1_kernel, raw_shape=(8, 1, 3, 3))
|
||||
|
||||
conv1_bias = model.layers[0].weights[1].numpy()
|
||||
gguf_writer.add_tensor("conv1.bias", conv1_bias, raw_shape=(1, 8, 1, 1))
|
||||
|
||||
conv2_kernel = model.layers[2].weights[0].numpy()
|
||||
conv2_kernel = np.moveaxis(conv2_kernel, [0, 1, 2, 3], [2, 3, 1, 0])
|
||||
gguf_writer.add_tensor("conv2.kernel", conv2_kernel, raw_shape=(16, 8, 3, 3))
|
||||
|
||||
conv2_bias = model.layers[2].weights[1].numpy()
|
||||
gguf_writer.add_tensor("conv2.bias", conv2_bias, raw_shape=(1, 16, 1, 1))
|
||||
|
||||
dense_weight = model.layers[-1].weights[0].numpy()
|
||||
dense_weight = dense_weight.transpose()
|
||||
gguf_writer.add_tensor("dense.weight", dense_weight, raw_shape=(10, 7*7*16))
|
||||
|
||||
dense_bias = model.layers[-1].weights[1].numpy()
|
||||
gguf_writer.add_tensor("dense.bias", dense_bias)
|
||||
|
||||
gguf_writer.write_header_to_file()
|
||||
gguf_writer.write_kv_data_to_file()
|
||||
gguf_writer.write_tensors_to_file()
|
||||
gguf_writer.close()
|
||||
print(f"GGUF model saved to '{model_path}'")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2:
|
||||
print(f"Usage: {sys.argv[0]} <model_path>")
|
||||
sys.exit(1)
|
||||
train(sys.argv[1])
|
||||
@@ -0,0 +1,131 @@
|
||||
import gguf
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torchvision.datasets as dsets
|
||||
import torchvision.transforms as transforms
|
||||
from torch.autograd import Variable
|
||||
|
||||
import sys
|
||||
from time import time
|
||||
|
||||
input_size = 784 # img_size = (28,28) ---> 28*28=784 in total
|
||||
hidden_size = 500 # number of nodes at hidden layer
|
||||
num_classes = 10 # number of output classes discrete range [0,9]
|
||||
num_epochs = 30 # number of times which the entire dataset is passed throughout the model
|
||||
batch_size = 1000 # the size of input data used for one iteration
|
||||
lr = 1e-3 # size of step
|
||||
|
||||
|
||||
class Net(nn.Module):
|
||||
def __init__(self, input_size, hidden_size, num_classes):
|
||||
super(Net, self).__init__()
|
||||
self.fc1 = nn.Linear(input_size, hidden_size)
|
||||
self.relu = nn.ReLU()
|
||||
self.fc2 = nn.Linear(hidden_size, num_classes)
|
||||
|
||||
def forward(self, x):
|
||||
out = self.fc1(x)
|
||||
out = self.relu(out)
|
||||
out = self.fc2(out)
|
||||
return out
|
||||
|
||||
|
||||
def train(model_path):
|
||||
train_data = dsets.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download=True)
|
||||
test_data = dsets.MNIST(root='./data', train=False, transform=transforms.ToTensor())
|
||||
|
||||
assert len(train_data) == 60000
|
||||
assert len(test_data) == 10000
|
||||
|
||||
kwargs_train_test = dict(batch_size=batch_size, num_workers=4, pin_memory=True)
|
||||
train_gen = torch.utils.data.DataLoader(dataset=train_data, shuffle=True, **kwargs_train_test)
|
||||
test_gen = torch.utils.data.DataLoader(dataset=test_data, shuffle=False, **kwargs_train_test)
|
||||
|
||||
net = Net(input_size, hidden_size, num_classes)
|
||||
|
||||
if torch.cuda.is_available():
|
||||
net.cuda()
|
||||
|
||||
loss_function = nn.CrossEntropyLoss()
|
||||
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
|
||||
|
||||
t_start = time()
|
||||
for epoch in range(num_epochs):
|
||||
loss_history = []
|
||||
ncorrect = 0
|
||||
|
||||
for i, (images, labels) in enumerate(train_gen):
|
||||
images = Variable(images.view(-1, 28*28))
|
||||
labels = Variable(labels)
|
||||
|
||||
if torch.cuda.is_available():
|
||||
images = images.cuda()
|
||||
labels = labels.cuda()
|
||||
|
||||
optimizer.zero_grad()
|
||||
outputs = net(images)
|
||||
loss = loss_function(outputs, labels)
|
||||
|
||||
loss_history.append(loss.cpu().data)
|
||||
_, predictions = torch.max(outputs, 1)
|
||||
ncorrect += (predictions == labels).sum()
|
||||
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
if (i + 1)*batch_size % 10000 == 0:
|
||||
loss_mean = np.mean(loss_history)
|
||||
accuracy = ncorrect / ((i + 1) * batch_size)
|
||||
print(
|
||||
f"Epoch [{epoch+1:02d}/{num_epochs}], "
|
||||
f"Step [{(i+1)*batch_size:05d}/{len(train_data)}], "
|
||||
f"Loss: {loss_mean:.4f}, Accuracy: {100*accuracy:.2f}%")
|
||||
print()
|
||||
print(f"Training took {time()-t_start:.2f}s")
|
||||
|
||||
loss_history = []
|
||||
ncorrect = 0
|
||||
|
||||
for i, (images, labels) in enumerate(test_gen):
|
||||
images = Variable(images.view(-1, 28*28))
|
||||
labels = Variable(labels)
|
||||
|
||||
if torch.cuda.is_available():
|
||||
images = images.cuda()
|
||||
labels = labels.cuda()
|
||||
|
||||
outputs = net(images)
|
||||
loss = loss_function(outputs, labels)
|
||||
|
||||
loss_history.append(loss.cpu().data)
|
||||
_, predictions = torch.max(outputs, 1)
|
||||
ncorrect += (predictions == labels).sum().cpu().numpy()
|
||||
|
||||
loss_mean = np.mean(loss_history)
|
||||
loss_uncertainty = np.std(loss_history) / np.sqrt(len(loss_history) - 1)
|
||||
accuracy_mean = ncorrect / (len(test_gen) * batch_size)
|
||||
accuracy_uncertainty = np.sqrt(accuracy_mean * (1.0 - accuracy_mean) / (len(test_gen) * batch_size))
|
||||
print()
|
||||
print(f"Test loss: {loss_mean:.6f}+-{loss_uncertainty:.6f}, Test accuracy: {100*accuracy_mean:.2f}+-{100*accuracy_uncertainty:.2f}%")
|
||||
|
||||
gguf_writer = gguf.GGUFWriter(model_path, "mnist-fc")
|
||||
|
||||
print()
|
||||
print(f"Model tensors saved to {model_path}:")
|
||||
for tensor_name in net.state_dict().keys():
|
||||
data = net.state_dict()[tensor_name].squeeze().cpu().numpy()
|
||||
print(tensor_name, "\t", data.shape)
|
||||
gguf_writer.add_tensor(tensor_name, data)
|
||||
|
||||
gguf_writer.write_header_to_file()
|
||||
gguf_writer.write_kv_data_to_file()
|
||||
gguf_writer.write_tensors_to_file()
|
||||
gguf_writer.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2:
|
||||
print(f"Usage: {sys.argv[0]} <model_path>")
|
||||
sys.exit(1)
|
||||
train(sys.argv[1])
|
||||
@@ -0,0 +1,39 @@
|
||||
#include "ggml-opt.h"
|
||||
#include "mnist-common.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
#include <string>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
if (argc != 5 && argc != 6) {
|
||||
fprintf(stderr, "Usage: %s mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// The MNIST model is so small that the overhead from data shuffling is non-negligible, especially with CUDA.
|
||||
// With a shard size of 10 this overhead is greatly reduced at the cost of less shuffling (does not seem to have a significant impact).
|
||||
// A batch of 500 images then consists of 50 random shards of size 10 instead of 500 random shards of size 1.
|
||||
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTRAIN, /*ndata_shard =*/ 10);
|
||||
|
||||
if (!mnist_image_load(argv[3], dataset)) {
|
||||
return 1;
|
||||
}
|
||||
if (!mnist_label_load(argv[4], dataset)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
mnist_model model = mnist_model_init_random(argv[1], argc >= 6 ? argv[5] : "", MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);
|
||||
|
||||
mnist_model_build(model);
|
||||
|
||||
mnist_model_train(model, dataset, /*nepoch =*/ 30, /*val_split =*/ 0.05f);
|
||||
|
||||
mnist_model_save(model, argv[2]);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import http.server
|
||||
import socketserver
|
||||
import os
|
||||
import sys
|
||||
|
||||
DIRECTORY = os.path.abspath(os.path.join(os.path.dirname(__file__), 'web'))
|
||||
PORT = 8000
|
||||
|
||||
class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, directory=DIRECTORY, **kwargs)
|
||||
|
||||
def end_headers(self):
|
||||
# Add required headers for SharedArrayBuffer
|
||||
self.send_header("Cross-Origin-Opener-Policy", "same-origin")
|
||||
self.send_header("Cross-Origin-Embedder-Policy", "require-corp")
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
super().end_headers()
|
||||
|
||||
# Enable address reuse
|
||||
class CustomServer(socketserver.TCPServer):
|
||||
allow_reuse_address = True
|
||||
|
||||
try:
|
||||
with CustomServer(("", PORT), CustomHTTPRequestHandler) as httpd:
|
||||
print(f"Serving directory '{DIRECTORY}' at http://localhost:{PORT}")
|
||||
print(f"Application context root: http://localhost:{PORT}/")
|
||||
try:
|
||||
httpd.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\nServer stopped.")
|
||||
# Force complete exit
|
||||
sys.exit(0)
|
||||
except OSError as e:
|
||||
print(f"Error: {e}")
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user