civ/omnivoice-cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial release

Browse Source
This commit is contained in:
civ
2026-07-05 18:11:23 +07:00
commit 8fb29dac70
2105 changed files with 499091 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/examples/CMakeLists.txt
+34
View File
@@ -0,0 +1,34 @@
if (GGML_ALL_WARNINGS)
if (NOT MSVC)
set(cxx_flags
# TODO(marella): Add other warnings.
-Wpedantic
-Wunused-variable
-Wno-unused-function
-Wno-multichar
)
add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:${cxx_flags}>")
endif()
endif()
add_library(common STATIC common.cpp)
target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
add_library(common-ggml STATIC common-ggml.cpp)
target_link_libraries(common-ggml PRIVATE ggml)
target_include_directories(common-ggml PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
add_subdirectory(yolo)
if (NOT GGML_BACKEND_DL)
add_subdirectory(gpt-2)
add_subdirectory(gpt-j)
add_subdirectory(mnist)
add_subdirectory(sam)
add_subdirectory(simple)
add_subdirectory(magika)
endif()
if (GGML_METAL)
add_subdirectory(perf-metal)
endif()
ggml/examples/common-ggml.cpp
+244
View File
@@ -0,0 +1,244 @@
#include "common-ggml.h"
#include <regex>
#include <map>
static const std::map<std::string, enum ggml_ftype> GGML_FTYPE_MAP = {
{"q4_0", GGML_FTYPE_MOSTLY_Q4_0},
{"q4_1", GGML_FTYPE_MOSTLY_Q4_1},
{"q5_0", GGML_FTYPE_MOSTLY_Q5_0},
{"q5_1", GGML_FTYPE_MOSTLY_Q5_1},
{"q8_0", GGML_FTYPE_MOSTLY_Q8_0},
{"q2_k", GGML_FTYPE_MOSTLY_Q2_K},
{"q3_k", GGML_FTYPE_MOSTLY_Q3_K},
{"q4_k", GGML_FTYPE_MOSTLY_Q4_K},
{"q5_k", GGML_FTYPE_MOSTLY_Q5_K},
{"q6_k", GGML_FTYPE_MOSTLY_Q6_K},
};
void ggml_print_ftypes(FILE * fp) {
for (auto it = GGML_FTYPE_MAP.begin(); it != GGML_FTYPE_MAP.end(); it++) {
fprintf(fp, " type = \"%s\" or %d\n", it->first.c_str(), it->second);
}
}
enum ggml_ftype ggml_parse_ftype(const char * str) {
enum ggml_ftype ftype;
if (str[0] == 'q') {
const auto it = GGML_FTYPE_MAP.find(str);
if (it == GGML_FTYPE_MAP.end()) {
fprintf(stderr, "%s: unknown ftype '%s'\n", __func__, str);
return GGML_FTYPE_UNKNOWN;
}
ftype = it->second;
} else {
ftype = (enum ggml_ftype) atoi(str);
}
return ftype;
}
bool ggml_common_quantize_0(
std::ifstream & finp,
std::ofstream & fout,
const ggml_ftype ftype,
const std::vector<std::string> & to_quant,
const std::vector<std::string> & to_skip) {
ggml_type qtype = GGML_TYPE_F32;
switch (ftype) {
case GGML_FTYPE_MOSTLY_Q4_0: qtype = GGML_TYPE_Q4_0; break;
case GGML_FTYPE_MOSTLY_Q4_1: qtype = GGML_TYPE_Q4_1; break;
case GGML_FTYPE_MOSTLY_Q5_0: qtype = GGML_TYPE_Q5_0; break;
case GGML_FTYPE_MOSTLY_Q5_1: qtype = GGML_TYPE_Q5_1; break;
case GGML_FTYPE_MOSTLY_Q8_0: qtype = GGML_TYPE_Q8_0; break;
case GGML_FTYPE_MOSTLY_Q2_K: qtype = GGML_TYPE_Q2_K; break;
case GGML_FTYPE_MOSTLY_Q3_K: qtype = GGML_TYPE_Q3_K; break;
case GGML_FTYPE_MOSTLY_Q4_K: qtype = GGML_TYPE_Q4_K; break;
case GGML_FTYPE_MOSTLY_Q5_K: qtype = GGML_TYPE_Q5_K; break;
case GGML_FTYPE_MOSTLY_Q6_K: qtype = GGML_TYPE_Q6_K; break;
case GGML_FTYPE_UNKNOWN:
case GGML_FTYPE_ALL_F32:
case GGML_FTYPE_MOSTLY_F16:
case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16:
case GGML_FTYPE_MOSTLY_IQ2_XXS:
case GGML_FTYPE_MOSTLY_IQ2_XS:
case GGML_FTYPE_MOSTLY_IQ2_S:
case GGML_FTYPE_MOSTLY_IQ3_XXS:
case GGML_FTYPE_MOSTLY_IQ3_S:
case GGML_FTYPE_MOSTLY_IQ1_S:
case GGML_FTYPE_MOSTLY_IQ4_NL:
case GGML_FTYPE_MOSTLY_IQ4_XS:
case GGML_FTYPE_MOSTLY_IQ1_M:
case GGML_FTYPE_MOSTLY_BF16:
case GGML_FTYPE_MOSTLY_MXFP4:
case GGML_FTYPE_MOSTLY_NVFP4:
case GGML_FTYPE_MOSTLY_Q1_0:
{
fprintf(stderr, "%s: invalid model type %d\n", __func__, ftype);
return false;
}
};
if (!ggml_is_quantized(qtype)) {
fprintf(stderr, "%s: invalid quantization type %d (%s)\n", __func__, qtype, ggml_type_name(qtype));
return false;
}
size_t total_size_org = 0;
size_t total_size_new = 0;
std::vector<float> work;
std::vector<uint8_t> data_u8;
std::vector<ggml_fp16_t> data_f16;
std::vector<float> data_f32;
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
finp.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
finp.read(reinterpret_cast<char *>(&length), sizeof(length));
finp.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (finp.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[4] = { 1, 1, 1, 1 };
for (int i = 0; i < n_dims; ++i) {
finp.read (reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
finp.read (&name[0], length);
printf("%64s - [%5d, %5d, %5d], type = %6s ", name.data(), ne[0], ne[1], ne[2], ggml_type_name((ggml_type) ttype));
bool quantize = false;
// check if we should quantize this tensor
for (const auto & s : to_quant) {
if (std::regex_match(name, std::regex(s))) {
quantize = true;
break;
}
}
// check if we should skip this tensor
for (const auto & s : to_skip) {
if (std::regex_match(name, std::regex(s))) {
quantize = false;
break;
}
}
// quantize only 2D tensors
quantize &= (n_dims == 2);
if (quantize) {
if (ttype != GGML_TYPE_F32 && ttype != GGML_TYPE_F16) {
fprintf(stderr, "%s: unsupported ttype %d (%s) for integer quantization\n", __func__, ttype, ggml_type_name((ggml_type) ttype));
return false;
}
if (ttype == GGML_TYPE_F16) {
data_f16.resize(nelements);
finp.read(reinterpret_cast<char *>(data_f16.data()), nelements * sizeof(ggml_fp16_t));
data_f32.resize(nelements);
for (int i = 0; i < nelements; ++i) {
data_f32[i] = ggml_fp16_to_fp32(data_f16[i]);
}
} else {
data_f32.resize(nelements);
finp.read(reinterpret_cast<char *>(data_f32.data()), nelements * sizeof(float));
}
ttype = qtype;
} else {
const int bpe = (ttype == 0) ? sizeof(float) : sizeof(uint16_t);
data_u8.resize(nelements*bpe);
finp.read(reinterpret_cast<char *>(data_u8.data()), nelements * bpe);
}
fout.write(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fout.write(reinterpret_cast<char *>(&length), sizeof(length));
fout.write(reinterpret_cast<char *>(&ttype), sizeof(ttype));
for (int i = 0; i < n_dims; ++i) {
fout.write(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
}
fout.write(&name[0], length);
if (quantize) {
work.resize(nelements); // for quantization
size_t cur_size = 0;
switch ((ggml_type) ttype) {
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q2_K:
case GGML_TYPE_Q3_K:
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q5_K:
case GGML_TYPE_Q6_K:
{
cur_size = ggml_quantize_chunk((ggml_type) ttype, data_f32.data(), work.data(), 0, nelements/ne[0], ne[0], nullptr);
} break;
case GGML_TYPE_F32:
case GGML_TYPE_F16:
case GGML_TYPE_I8:
case GGML_TYPE_I16:
case GGML_TYPE_I32:
case GGML_TYPE_I64:
case GGML_TYPE_F64:
case GGML_TYPE_Q8_1:
case GGML_TYPE_Q8_K:
case GGML_TYPE_IQ2_XXS:
case GGML_TYPE_IQ2_XS:
case GGML_TYPE_IQ2_S:
case GGML_TYPE_IQ3_XXS:
case GGML_TYPE_IQ3_S:
case GGML_TYPE_IQ1_S:
case GGML_TYPE_IQ4_NL:
case GGML_TYPE_IQ4_XS:
case GGML_TYPE_IQ1_M:
case GGML_TYPE_BF16:
case GGML_TYPE_TQ1_0:
case GGML_TYPE_TQ2_0:
case GGML_TYPE_MXFP4:
case GGML_TYPE_NVFP4:
case GGML_TYPE_Q1_0:
case GGML_TYPE_COUNT:
{
fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype));
return false;
}
}
fout.write(reinterpret_cast<char *>(work.data()), cur_size);
total_size_new += cur_size;
printf("size = %8.2f MB -> %8.2f MB\n", nelements * sizeof(float)/1024.0/1024.0, cur_size/1024.0/1024.0);
} else {
printf("size = %8.3f MB\n", data_u8.size()/1024.0/1024.0);
fout.write(reinterpret_cast<char *>(data_u8.data()), data_u8.size());
total_size_new += data_u8.size();
}
total_size_org += nelements * sizeof(float);
}
printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0);
printf("%s: quant size = %8.2f MB | ftype = %d (%s)\n", __func__, total_size_new/1024.0/1024.0, ftype, ggml_type_name(qtype));
return true;
}
ggml/examples/common-ggml.h
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "ggml.h"
#include <fstream>
#include <vector>
#include <string>
enum ggml_ftype ggml_parse_ftype(const char * str);
void ggml_print_ftypes(FILE * fp = stderr);
bool ggml_common_quantize_0(
std::ifstream & finp,
std::ofstream & fout,
const ggml_ftype ftype,
const std::vector<std::string> & to_quant,
const std::vector<std::string> & to_skip);
ggml/examples/common.cpp
+675
View File
@@ -0,0 +1,675 @@
#define _USE_MATH_DEFINES // for M_PI
#include "common.h"
#include <cmath>
#include <codecvt>
#include <cstring>
#include <fstream>
#include <locale>
#include <regex>
#include <sstream>
// Function to check if the next argument exists
static std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) {
if (i + 1 < argc && argv[i + 1][0] != '-') {
return argv[++i];
} else {
fprintf(stderr, "error: %s requires one argument.\n", flag.c_str());
gpt_print_usage(argc, argv, params);
exit(0);
}
}
bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
for (int i = 1; i < argc; i++) {
std::string arg = argv[i];
if (arg == "-s" || arg == "--seed") {
params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-t" || arg == "--threads") {
params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-p" || arg == "--prompt") {
params.prompt = get_next_arg(i, argc, argv, arg, params);
} else if (arg == "-n" || arg == "--n_predict") {
params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-np" || arg == "--n_parallel") {
params.n_parallel = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--top_k") {
params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--top_p") {
params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--temp") {
params.temp = std::stof(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--repeat-last-n") {
params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--repeat-penalty") {
params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-b" || arg == "--batch_size") {
params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-c" || arg == "--context") {
params.n_ctx= std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") {
params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--ignore-eos") {
params.ignore_eos = true;
} else if (arg == "-m" || arg == "--model") {
params.model = get_next_arg(i, argc, argv, arg, params);
} else if (arg == "-i" || arg == "--interactive") {
params.interactive = true;
} else if (arg == "-ip" || arg == "--interactive-port") {
params.interactive = true;
params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-h" || arg == "--help") {
gpt_print_usage(argc, argv, params);
exit(0);
} else if (arg == "-f" || arg == "--file") {
get_next_arg(i, argc, argv, arg, params);
std::ifstream file(argv[i]);
if (!file) {
fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
break;
}
std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
if (params.prompt.back() == '\n') {
params.prompt.pop_back();
}
} else if (arg == "-tt" || arg == "--token_test") {
params.token_test = get_next_arg(i, argc, argv, arg, params);
}
else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
gpt_print_usage(argc, argv, params);
exit(0);
}
}
return true;
}
void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
fprintf(stderr, "usage: %s [options]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, "options:\n");
fprintf(stderr, " -h, --help show this help message and exit\n");
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n");
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");
fprintf(stderr, " prompt to start generation with (default: random)\n");
fprintf(stderr, " -f FNAME, --file FNAME\n");
fprintf(stderr, " load prompt from a file\n");
fprintf(stderr, " -tt TOKEN_TEST, --token_test TOKEN_TEST\n");
fprintf(stderr, " test tokenization\n");
fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d)\n", params.n_predict);
fprintf(stderr, " --top_k N top-k sampling (default: %d)\n", params.top_k);
fprintf(stderr, " --top_p N top-p sampling (default: %.1f)\n", params.top_p);
fprintf(stderr, " --temp N temperature (default: %.1f)\n", params.temp);
fprintf(stderr, " --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled)\n", params.repeat_last_n);
fprintf(stderr, " --repeat-penalty N penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\n", (double)params.repeat_penalty);
fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch);
fprintf(stderr, " -c N, --context N context / KV cache size (default: %d)\n", params.n_ctx);
fprintf(stderr, " --ignore-eos ignore EOS token during generation\n");
fprintf(stderr, " -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers);
fprintf(stderr, " -m FNAME, --model FNAME\n");
fprintf(stderr, " model path (default: %s)\n", params.model.c_str());
fprintf(stderr, "\n");
}
std::string gpt_random_prompt(std::mt19937 & rng) {
const int r = rng() % 10;
switch (r) {
case 0: return "So";
case 1: return "Once upon a time";
case 2: return "When";
case 3: return "The";
case 4: return "After";
case 5: return "If";
case 6: return "import";
case 7: return "He";
case 8: return "She";
case 9: return "They";
}
return "The";
}
std::string trim(const std::string & s) {
std::regex e("^\\s+|\\s+$");
return std::regex_replace(s, e, "");
}
std::string replace(const std::string & s, const std::string & from, const std::string & to) {
std::string result = s;
size_t pos = 0;
while ((pos = result.find(from, pos)) != std::string::npos) {
result.replace(pos, from.length(), to);
pos += to.length();
}
return result;
}
void gpt_vocab::add_special_token(const std::string & token) {
special_tokens.push_back(token);
}
std::map<std::string, int32_t> json_parse(const std::string & fname) {
std::map<std::string, int32_t> result;
// read file into string
std::string json;
{
std::ifstream ifs(fname);
if (!ifs) {
fprintf(stderr, "Failed to open %s\n", fname.c_str());
exit(1);
}
json = std::string((std::istreambuf_iterator<char>(ifs)),
(std::istreambuf_iterator<char>()));
}
if (json[0] != '{') {
return result;
}
// parse json
{
bool has_key = false;
bool in_token = false;
std::string str_key = "";
std::string str_val = "";
int n = json.size();
for (int i = 1; i < n; ++i) {
if (!in_token) {
if (json[i] == ' ') continue;
if (json[i] == '"') {
in_token = true;
continue;
}
} else {
if (json[i] == '\\' && i+1 < n) {
if (has_key == false) {
str_key += json[i];
} else {
str_val += json[i];
}
++i;
} else if (json[i] == '"') {
if (has_key == false) {
has_key = true;
++i;
while (json[i] == ' ') ++i;
++i; // :
while (json[i] == ' ') ++i;
if (json[i] != '\"') {
while (json[i] != ',' && json[i] != '}') {
str_val += json[i++];
}
has_key = false;
} else {
in_token = true;
continue;
}
} else {
has_key = false;
}
str_key = ::replace(str_key, "\\u0120", " " ); // \u0120 -> space
str_key = ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line
str_key = ::replace(str_key, "\\\"", "\""); // \\\" -> "
try {
result[str_key] = std::stoi(str_val);
} catch (...) {
//fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str());
}
str_key = "";
str_val = "";
in_token = false;
continue;
}
if (has_key == false) {
str_key += json[i];
} else {
str_val += json[i];
}
}
}
}
return result;
}
void gpt_split_words(std::string str, std::vector<std::string>& words) {
const std::string pattern = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)";
const std::regex re(pattern);
std::smatch m;
while (std::regex_search(str, m, re)) {
for (auto x : m) {
words.push_back(x);
}
str = m.suffix();
}
}
std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {
std::vector<std::string> words;
// first split the text into words
{
std::string str = text;
// Generate the subpattern from the special_tokens vector if it's not empty
if (!vocab.special_tokens.empty()) {
const std::regex escape(R"([\[\\\^\$\.\|\?\*\+\(\)\{\}])");
std::string special_tokens_subpattern;
for (const auto & token : vocab.special_tokens) {
if (!special_tokens_subpattern.empty()) {
special_tokens_subpattern += "|";
}
special_tokens_subpattern += std::regex_replace(token, escape, R"(\$&)");
}
std::regex re(special_tokens_subpattern);
std::smatch m;
// Split the text by special tokens.
while (std::regex_search(str, m, re)) {
// Split the substrings in-between special tokens into words.
gpt_split_words(m.prefix(), words);
// Add matched special tokens as words.
for (auto x : m) {
words.push_back(x);
}
str = m.suffix();
}
// Remaining text without special tokens will be handled below.
}
gpt_split_words(str, words);
}
// find the longest token that forms each word in words:
std::vector<gpt_vocab::id> tokens;
for (const auto & word : words) {
for (int i = 0; i < (int) word.size(); ){
for (int j = word.size() - 1; j >= i; j--){
auto cand = word.substr(i, j-i+1);
auto it = vocab.token_to_id.find(cand);
if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab
tokens.push_back(it->second);
i = j + 1;
break;
}
else if (j == i){ // word.substr(i, 1) has no matching
fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());
i++;
}
}
}
}
return tokens;
}
static std::vector<gpt_vocab::id> parse_tokens_from_string(const std::string& input, char delimiter) {
std::vector<gpt_vocab::id> output;
std::stringstream ss(input);
std::string token;
while (std::getline(ss, token, delimiter)) {
output.push_back(std::stoi(token));
}
return output;
}
static std::map<std::string, std::vector<gpt_vocab::id>> extract_tests_from_file(const std::string & fpath_test){
if (fpath_test.empty()){
fprintf(stderr, "%s : No test file found.\n", __func__);
return std::map<std::string, std::vector<gpt_vocab::id>>();
}
std::map<std::string, std::vector<gpt_vocab::id>> tests;
auto fin = std::ifstream(fpath_test, std::ios_base::in);
const char * delimeter = " => ";
const char del_tok = ',';
std::string line;
while (std::getline(fin, line)) {
size_t delimiterPos = line.find(delimeter);
if (delimiterPos != std::string::npos) {
std::string text = line.substr(0, delimiterPos);
std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter));
tests[text] = parse_tokens_from_string(s_tokens, del_tok);
}
}
return tests;
}
void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){
std::map<std::string, std::vector<gpt_vocab::id>> tests = extract_tests_from_file(fpath_test);
size_t n_fails = 0;
for (const auto & test : tests) {
std::vector<gpt_vocab::id> tokens = gpt_tokenize(vocab, test.first);
if (tokens != test.second){
n_fails++;
// print out failure cases
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test.first.c_str());
fprintf(stderr, "%s : tokens in hf: ", __func__);
for (const auto & t : test.second) {
fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t);
}
fprintf(stderr, "\n");
fprintf(stderr, "%s : tokens in ggml: ", __func__);
for (const auto & t : tokens) {
fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t);
}
fprintf(stderr, "\n");
}
}
fprintf(stderr, "%s : %zu tests failed out of %zu tests.\n", __func__, n_fails, tests.size());
}
bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {
printf("%s: loading vocab from '%s'\n", __func__, fname.c_str());
vocab.token_to_id = ::json_parse(fname);
for (const auto & kv : vocab.token_to_id) {
vocab.id_to_token[kv.second] = kv.first;
}
printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size());
// print the vocabulary
//for (auto kv : vocab.token_to_id) {
// printf("'%s' -> %d\n", kv.first.data(), kv.second);
//}
return true;
}
gpt_vocab::id gpt_sample_top_k_top_p(
const gpt_vocab & vocab,
const float * logits,
int top_k,
double top_p,
double temp,
std::mt19937 & rng) {
int n_logits = vocab.id_to_token.size();
std::vector<std::pair<double, gpt_vocab::id>> logits_id;
logits_id.reserve(n_logits);
{
const double scale = 1.0/temp;
for (int i = 0; i < n_logits; ++i) {
logits_id.push_back(std::make_pair(logits[i]*scale, i));
}
}
// find the top K tokens
std::partial_sort(
logits_id.begin(),
logits_id.begin() + top_k, logits_id.end(),
[](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
return a.first > b.first;
});
logits_id.resize(top_k);
double maxl = -INFINITY;
for (const auto & kv : logits_id) {
maxl = std::max(maxl, kv.first);
}
// compute probs for the top K tokens
std::vector<double> probs;
probs.reserve(logits_id.size());
double sum = 0.0;
for (const auto & kv : logits_id) {
double p = exp(kv.first - maxl);
probs.push_back(p);
sum += p;
}
// normalize the probs
for (auto & p : probs) {
p /= sum;
}
if (top_p < 1.0f) {
double cumsum = 0.0f;
for (int i = 0; i < top_k; i++) {
cumsum += probs[i];
if (cumsum >= top_p) {
top_k = i + 1;
probs.resize(top_k);
logits_id.resize(top_k);
break;
}
}
cumsum = 1.0/cumsum;
for (int i = 0; i < (int) probs.size(); i++) {
probs[i] *= cumsum;
}
}
//printf("\n");
//for (int i = 0; i < (int) probs.size(); i++) {
// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
//}
//exit(0);
std::discrete_distribution<> dist(probs.begin(), probs.end());
int idx = dist(rng);
return logits_id[idx].second;
}
gpt_vocab::id gpt_sample_top_k_top_p_repeat(
const gpt_vocab & vocab,
const float * logits,
const int32_t * last_n_tokens_data,
size_t last_n_tokens_data_size,
int top_k,
double top_p,
double temp,
int repeat_last_n,
float repeat_penalty,
std::mt19937 & rng) {
int n_logits = vocab.id_to_token.size();
const auto * plogits = logits;
const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size);
if (temp <= 0) {
// select the token with the highest logit directly
float max_logit = plogits[0];
gpt_vocab::id max_id = 0;
for (int i = 1; i < n_logits; ++i) {
if (plogits[i] > max_logit) {
max_logit = plogits[i];
max_id = i;
}
}
return max_id;
}
std::vector<std::pair<double, gpt_vocab::id>> logits_id;
logits_id.reserve(n_logits);
{
const float scale = 1.0f/temp;
for (int i = 0; i < n_logits; ++i) {
// repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858)
// credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main
if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) {
// if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
if (plogits[i] < 0.0f) {
logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i));
} else {
logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i));
}
} else {
logits_id.push_back(std::make_pair(plogits[i]*scale, i));
}
}
}
// find the top K tokens
std::partial_sort(
logits_id.begin(),
logits_id.begin() + top_k, logits_id.end(),
[](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
return a.first > b.first;
});
logits_id.resize(top_k);
double maxl = -INFINITY;
for (const auto & kv : logits_id) {
maxl = std::max(maxl, kv.first);
}
// compute probs for the top K tokens
std::vector<double> probs;
probs.reserve(logits_id.size());
double sum = 0.0;
for (const auto & kv : logits_id) {
double p = exp(kv.first - maxl);
probs.push_back(p);
sum += p;
}
// normalize the probs
for (auto & p : probs) {
p /= sum;
}
if (top_p < 1.0f) {
double cumsum = 0.0f;
for (int i = 0; i < top_k; i++) {
cumsum += probs[i];
if (cumsum >= top_p) {
top_k = i + 1;
probs.resize(top_k);
logits_id.resize(top_k);
break;
}
}
cumsum = 1.0/cumsum;
for (int i = 0; i < (int) probs.size(); i++) {
probs[i] *= cumsum;
}
}
// printf("\n");
// for (int i = 0; i < (int) probs.size(); i++) {
// for (int i = 0; i < 10; i++) {
// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
// }
std::discrete_distribution<> dist(probs.begin(), probs.end());
int idx = dist(rng);
return logits_id[idx].second;
}
void high_pass_filter(std::vector<float> & data, float cutoff, float sample_rate) {
const float rc = 1.0f / (2.0f * M_PI * cutoff);
const float dt = 1.0f / sample_rate;
const float alpha = dt / (rc + dt);
float y = data[0];
for (size_t i = 1; i < data.size(); i++) {
y = alpha * (y + data[i] - data[i - 1]);
data[i] = y;
}
}
bool vad_simple(std::vector<float> & pcmf32, int sample_rate, int last_ms, float vad_thold, float freq_thold, bool verbose) {
const int n_samples = pcmf32.size();
const int n_samples_last = (sample_rate * last_ms) / 1000;
if (n_samples_last >= n_samples) {
// not enough samples - assume no speech
return false;
}
if (freq_thold > 0.0f) {
high_pass_filter(pcmf32, freq_thold, sample_rate);
}
float energy_all = 0.0f;
float energy_last = 0.0f;
for (int i = 0; i < n_samples; i++) {
energy_all += fabsf(pcmf32[i]);
if (i >= n_samples - n_samples_last) {
energy_last += fabsf(pcmf32[i]);
}
}
energy_all /= n_samples;
energy_last /= n_samples_last;
if (verbose) {
fprintf(stderr, "%s: energy_all: %f, energy_last: %f, vad_thold: %f, freq_thold: %f\n", __func__, energy_all, energy_last, vad_thold, freq_thold);
}
if (energy_last > vad_thold*energy_all) {
return false;
}
return true;
}
float similarity(const std::string & s0, const std::string & s1) {
const size_t len0 = s0.size() + 1;
const size_t len1 = s1.size() + 1;
std::vector<int> col(len1, 0);
std::vector<int> prevCol(len1, 0);
for (size_t i = 0; i < len1; i++) {
prevCol[i] = i;
}
for (size_t i = 0; i < len0; i++) {
col[0] = i;
for (size_t j = 1; j < len1; j++) {
col[j] = std::min(std::min(1 + col[j - 1], 1 + prevCol[j]), prevCol[j - 1] + (i > 0 && s0[i - 1] == s1[j - 1] ? 0 : 1));
}
col.swap(prevCol);
}
const float dist = prevCol[len1 - 1];
return 1.0f - (dist / std::max(s0.size(), s1.size()));
}
bool is_file_exist(const char * filename) {
std::ifstream infile(filename);
return infile.good();
}
ggml/examples/common.h
+322
View File
@@ -0,0 +1,322 @@
// Various helper functions and utilities
#pragma once
#include <string>
#include <map>
#include <vector>
#include <random>
#include <thread>
#include <ctime>
#include <fstream>
#include <sstream>
//
// GPT CLI argument parsing
//
struct gpt_params {
int32_t seed = -1; // RNG seed
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
int32_t n_predict = 200; // new tokens to predict
int32_t n_parallel = 1; // number of parallel streams
int32_t n_batch = 32; // batch size for prompt processing
int32_t n_ctx = 2048; // context size (this is the KV cache max size)
int32_t n_gpu_layers = 0; // number of layers to offlload to the GPU
bool ignore_eos = false; // ignore EOS token when generating text
// sampling parameters
int32_t top_k = 40;
float top_p = 0.9f;
float temp = 0.9f;
int32_t repeat_last_n = 64;
float repeat_penalty = 1.00f;
std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path
std::string prompt = "";
std::string token_test = "";
bool interactive = false;
int32_t interactive_port = -1;
};
bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
std::string gpt_random_prompt(std::mt19937 & rng);
//
// Vocab utils
//
std::string trim(const std::string & s);
std::string replace(
const std::string & s,
const std::string & from,
const std::string & to);
struct gpt_vocab {
using id = int32_t;
using token = std::string;
std::map<token, id> token_to_id;
std::map<id, token> id_to_token;
std::vector<std::string> special_tokens;
void add_special_token(const std::string & token);
};
// poor-man's JSON parsing
std::map<std::string, int32_t> json_parse(const std::string & fname);
std::string convert_to_utf8(const std::wstring & input);
std::wstring convert_to_wstring(const std::string & input);
void gpt_split_words(std::string str, std::vector<std::string>& words);
// split text into tokens
//
// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53
//
// Regex (Python):
// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
//
// Regex (C++):
// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"
//
std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text);
// test outputs of gpt_tokenize
//
// - compare with tokens generated by the huggingface tokenizer
// - test cases are chosen based on the model's main language (under 'prompt' directory)
// - if all sentences are tokenized identically, print 'All tests passed.'
// - otherwise, print sentence, huggingface tokens, ggml tokens
//
void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test);
// load the tokens from encoder.json
bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab);
// sample next token given probabilities for each embedding
//
// - consider only the top K tokens
// - from them, consider only the top tokens with cumulative probability > P
//
// TODO: not sure if this implementation is correct
// TODO: temperature is not implemented
//
gpt_vocab::id gpt_sample_top_k_top_p(
const gpt_vocab & vocab,
const float * logits,
int top_k,
double top_p,
double temp,
std::mt19937 & rng);
gpt_vocab::id gpt_sample_top_k_top_p_repeat(
const gpt_vocab & vocab,
const float * logits,
const int32_t * last_n_tokens_data,
size_t last_n_tokens_data_size,
int top_k,
double top_p,
double temp,
int repeat_last_n,
float repeat_penalty,
std::mt19937 & rng);
//
// Audio utils
//
// Write PCM data into WAV audio file
class wav_writer {
private:
std::ofstream file;
uint32_t dataSize = 0;
std::string wav_filename;
bool write_header(const uint32_t sample_rate,
const uint16_t bits_per_sample,
const uint16_t channels) {
file.write("RIFF", 4);
file.write("\0\0\0\0", 4); // Placeholder for file size
file.write("WAVE", 4);
file.write("fmt ", 4);
const uint32_t sub_chunk_size = 16;
const uint16_t audio_format = 1; // PCM format
const uint32_t byte_rate = sample_rate * channels * bits_per_sample / 8;
const uint16_t block_align = channels * bits_per_sample / 8;
file.write(reinterpret_cast<const char *>(&sub_chunk_size), 4);
file.write(reinterpret_cast<const char *>(&audio_format), 2);
file.write(reinterpret_cast<const char *>(&channels), 2);
file.write(reinterpret_cast<const char *>(&sample_rate), 4);
file.write(reinterpret_cast<const char *>(&byte_rate), 4);
file.write(reinterpret_cast<const char *>(&block_align), 2);
file.write(reinterpret_cast<const char *>(&bits_per_sample), 2);
file.write("data", 4);
file.write("\0\0\0\0", 4); // Placeholder for data size
return true;
}
// It is assumed that PCM data is normalized to a range from -1 to 1
bool write_audio(const float * data, size_t length) {
for (size_t i = 0; i < length; ++i) {
const int16_t intSample = int16_t(data[i] * 32767);
file.write(reinterpret_cast<const char *>(&intSample), sizeof(int16_t));
dataSize += sizeof(int16_t);
}
if (file.is_open()) {
file.seekp(4, std::ios::beg);
uint32_t fileSize = 36 + dataSize;
file.write(reinterpret_cast<char *>(&fileSize), 4);
file.seekp(40, std::ios::beg);
file.write(reinterpret_cast<char *>(&dataSize), 4);
file.seekp(0, std::ios::end);
}
return true;
}
bool open_wav(const std::string & filename) {
if (filename != wav_filename) {
if (file.is_open()) {
file.close();
}
}
if (!file.is_open()) {
file.open(filename, std::ios::binary);
wav_filename = filename;
dataSize = 0;
}
return file.is_open();
}
public:
bool open(const std::string & filename,
const uint32_t sample_rate,
const uint16_t bits_per_sample,
const uint16_t channels) {
if (open_wav(filename)) {
write_header(sample_rate, bits_per_sample, channels);
} else {
return false;
}
return true;
}
bool close() {
file.close();
return true;
}
bool write(const float * data, size_t length) {
return write_audio(data, length);
}
~wav_writer() {
if (file.is_open()) {
file.close();
}
}
};
// Apply a high-pass frequency filter to PCM audio
// Suppresses frequencies below cutoff Hz
void high_pass_filter(
std::vector<float> & data,
float cutoff,
float sample_rate);
// Basic voice activity detection (VAD) using audio energy adaptive threshold
bool vad_simple(
std::vector<float> & pcmf32,
int sample_rate,
int last_ms,
float vad_thold,
float freq_thold,
bool verbose);
// compute similarity between two strings using Levenshtein distance
float similarity(const std::string & s0, const std::string & s1);
//
// Terminal utils
//
#define SQR(X) ((X) * (X))
#define UNCUBE(x) x < 48 ? 0 : x < 115 ? 1 : (x - 35) / 40
/**
* Quantizes 24-bit RGB to xterm256 code range [16,256).
*/
static int rgb2xterm256(int r, int g, int b) {
unsigned char cube[] = {0, 0137, 0207, 0257, 0327, 0377};
int av, ir, ig, ib, il, qr, qg, qb, ql;
av = r * .299 + g * .587 + b * .114 + .5;
ql = (il = av > 238 ? 23 : (av - 3) / 10) * 10 + 8;
qr = cube[(ir = UNCUBE(r))];
qg = cube[(ig = UNCUBE(g))];
qb = cube[(ib = UNCUBE(b))];
if (SQR(qr - r) + SQR(qg - g) + SQR(qb - b) <=
SQR(ql - r) + SQR(ql - g) + SQR(ql - b))
return ir * 36 + ig * 6 + ib + 020;
return il + 0350;
}
static std::string set_xterm256_foreground(int r, int g, int b) {
int x = rgb2xterm256(r, g, b);
std::ostringstream oss;
oss << "\033[38;5;" << x << "m";
return oss.str();
}
// Lowest is red, middle is yellow, highest is green. Color scheme from
// Paul Tol; it is colorblind friendly https://sronpersonalpages.nl/~pault
const std::vector<std::string> k_colors = {
set_xterm256_foreground(220, 5, 12),
set_xterm256_foreground(232, 96, 28),
set_xterm256_foreground(241, 147, 45),
set_xterm256_foreground(246, 193, 65),
set_xterm256_foreground(247, 240, 86),
set_xterm256_foreground(144, 201, 135),
set_xterm256_foreground( 78, 178, 101),
};
// ANSI formatting codes
static std::string set_inverse() {
return "\033[7m";
}
static std::string set_underline() {
return "\033[4m";
}
static std::string set_dim() {
return "\033[2m";
}
// Style scheme for different confidence levels
const std::vector<std::string> k_styles = {
set_inverse(), // Low confidence - inverse (highlighted)
set_underline(), // Medium confidence - underlined
set_dim(), // High confidence - dim
};
//
// Other utils
//
// check if file exists using ifstream
bool is_file_exist(const char * filename);
ggml/examples/gpt-2/CMakeLists.txt
+32
View File
@@ -0,0 +1,32 @@
#
# gpt-2
set(TEST_TARGET gpt-2-ctx)
add_executable(${TEST_TARGET} main-ctx.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
set(TEST_TARGET gpt-2-alloc)
add_executable(${TEST_TARGET} main-alloc.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
set(TEST_TARGET gpt-2-backend)
add_executable(${TEST_TARGET} main-backend.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
set(TEST_TARGET gpt-2-sched)
add_executable(${TEST_TARGET} main-sched.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
#
# gpt-2-quantize
set(TEST_TARGET gpt-2-quantize)
add_executable(${TEST_TARGET} quantize.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
#
# gpt-2-batched
set(TEST_TARGET gpt-2-batched)
add_executable(${TEST_TARGET} main-batched.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
ggml/examples/gpt-2/README.md
+225
View File
@@ -0,0 +1,225 @@
# gpt-2
This is a C++ example running GPT-2 inference using the [ggml](https://github.com/ggerganov/ggml) library.
The program runs on the CPU - no video card is required.
The [Cerebras-GPT](https://huggingface.co/cerebras) models are also supported.
The example supports the following GPT-2 models:
| Model | Description | Disk Size |
| --- | --- | --- |
| 117M | Small model | 240 MB |
| 345M | Medium model | 680 MB |
| 774M | Large model | 1.5 GB |
| 1558M | XL model | 3.0 GB |
Sample performance on MacBook M1 Pro:
| Model | Size | Time / Token |
| --- | --- | --- |
| GPT-2 | 117M | 5 ms |
| GPT-2 | 345M | 12 ms |
| GPT-2 | 774M | 23 ms |
| GPT-2 | 1558M | 42 ms |
*TODO: add tables for Cerebras-GPT models*
Sample output:
```bash
$ ./bin/gpt-2 -h
usage: ./bin/gpt-2 [options]
options:
-h, --help show this help message and exit
-s SEED, --seed SEED RNG seed (default: -1)
-t N, --threads N number of threads to use during computation (default: 8)
-p PROMPT, --prompt PROMPT
prompt to start generation with (default: random)
-n N, --n_predict N number of tokens to predict (default: 200)
--top_k N top-k sampling (default: 40)
--top_p N top-p sampling (default: 0.9)
--temp N temperature (default: 1.0)
-b N, --batch_size N batch size for prompt processing (default: 8)
-m FNAME, --model FNAME
model path (default: models/gpt-2-117M/ggml-model.bin)
$ ./bin/gpt-2
gpt2_model_load: loading model from 'models/gpt-2-117M/ggml-model.bin'
gpt2_model_load: n_vocab = 50257
gpt2_model_load: n_ctx = 1024
gpt2_model_load: n_embd = 768
gpt2_model_load: n_head = 12
gpt2_model_load: n_layer = 12
gpt2_model_load: f16 = 1
gpt2_model_load: ggml ctx size = 311.12 MB
gpt2_model_load: memory size = 72.00 MB, n_mem = 12288
gpt2_model_load: model size = 239.08 MB
main: number of tokens in prompt = 1
So this is going to be the end of the line for us.
If the Dolphins continue to do their business, it's possible that the team could make a bid to bring in new defensive coordinator Scott Linehan.
Linehan's job is a little daunting, but he's a great coach and an excellent coach. I don't believe we're going to make the playoffs.
We're going to have to work hard to keep our heads down and get ready to go.<|endoftext|>
main: mem per token = 2048612 bytes
main: load time = 106.32 ms
main: sample time = 7.10 ms
main: predict time = 506.40 ms / 5.06 ms per token
main: total time = 629.84 ms
```
## Downloading and converting the original models (GPT-2)
You can download the original model files using the [download-model.sh](download-model.sh) Bash script. The models are
in Tensorflow format, so in order to use them with ggml, you need to convert them to appropriate format. This is done
via the [convert-ckpt-to-ggml.py](convert-ckpt-to-ggml.py) python script.
Here is the entire process for the GPT-2 117M model (download from official site + conversion):
```bash
cd ggml/build
../examples/gpt-2/download-model.sh 117M
Downloading model 117M ...
models/gpt-2-117M/checkpoint 100%[=============================>] 77 --.-KB/s in 0s
models/gpt-2-117M/encoder.json 100%[=============================>] 1018K 1.20MB/s in 0.8s
models/gpt-2-117M/hparams.json 100%[=============================>] 90 --.-KB/s in 0s
models/gpt-2-117M/model.ckpt.data-00000-of-00001 100%[=============================>] 474.70M 1.21MB/s in 8m 39s
models/gpt-2-117M/model.ckpt.index 100%[=============================>] 5.09K --.-KB/s in 0s
models/gpt-2-117M/model.ckpt.meta 100%[=============================>] 460.11K 806KB/s in 0.6s
models/gpt-2-117M/vocab.bpe 100%[=============================>] 445.62K 799KB/s in 0.6s
Done! Model '117M' saved in 'models/gpt-2-117M/'
Run the convert-ckpt-to-ggml.py script to convert the model to ggml format.
python /Users/john/ggml/examples/gpt-2/convert-ckpt-to-ggml.py models/gpt-2-117M/ 1
```
This conversion requires that you have python and Tensorflow installed on your computer. Still, if you want to avoid
this, you can download the already converted ggml models as described below.
## Downloading and converting the original models (Cerebras-GPT)
Clone the respective repository from here: https://huggingface.co/cerebras
Use the [convert-cerebras-to-ggml.py](convert-cerebras-to-ggml.py) script to convert the model to `ggml` format:
```bash
cd ggml/build
git clone https://huggingface.co/cerebras/Cerebras-GPT-111M models/
python ../examples/gpt-2/convert-cerebras-to-ggml.py models/Cerebras-GPT-111M/
```
## Downloading the ggml model directly (GPT-2)
For convenience, I will be hosting the converted ggml model files in order to make it easier to run the examples. This
way, you can directly download a single binary file and start using it. No python or Tensorflow is required.
Here is how to get the 117M ggml model:
```bash
cd ggml/build
../examples/gpt-2/download-ggml-model.sh 117M
Downloading ggml model 117M ...
models/gpt-2-117M/ggml-model.bin 100%[===============================>] 239.58M 8.52MB/s in 28s
Done! Model '117M' saved in 'models/gpt-2-117M/ggml-model.bin'
You can now use it like this:
$ ./bin/gpt-2 -m models/gpt-2-117M/ggml-model.bin -p "This is an example"
```
At some point, I might decide to stop hosting these models. So in that case, simply revert to the manual process above.
## Quantizing the models
You can also try to quantize the `ggml` models via 4-bit integer quantization.
Keep in mind that for smaller models, this will render them completely useless.
You generally want to quantize larger models.
```bash
# quantize GPT-2 F16 to Q4_0 (faster but less precise)
./bin/gpt-2-quantize models/gpt-2-1558M/ggml-model-f16.bin models/gpt-2-1558M/ggml-model-q4_0.bin 2
./bin/gpt-2 -m models/gpt-2-1558M/ggml-model-q4_0.bin -p "This is an example"
# quantize Cerebras F16 to Q4_1 (slower but more precise)
./bin/gpt-2-quantize models/Cerebras-GPT-6.7B/ggml-model-f16.bin models/Cerebras-GPT-6.7B/ggml-model-q4_1.bin 3
./bin/gpt-2 -m models/Cerebras-GPT-6.7B/ggml-model-q4_1.bin -p "This is an example"
```
## Batched generation example
You can try the batched generation from a given prompt using the gpt-2-batched binary.
Sample output:
```bash
$ gpt-2-batched -np 5 -m models/gpt-2-117M/ggml-model.bin -p "Hello my name is" -n 50
main: seed = 1697037431
gpt2_model_load: loading model from 'models/gpt-2-117M/ggml-model.bin'
gpt2_model_load: n_vocab = 50257
gpt2_model_load: n_ctx = 1024
gpt2_model_load: n_embd = 768
gpt2_model_load: n_head = 12
gpt2_model_load: n_layer = 12
gpt2_model_load: ftype = 1
gpt2_model_load: qntvr = 0
gpt2_model_load: ggml tensor size = 320 bytes
gpt2_model_load: backend buffer size = 312.72 MB
ggml_init_cublas: found 1 CUDA devices:
Device 0: NVIDIA GeForce GTX 1660, compute capability 7.5
gpt2_model_load: using CPU backend
gpt2_model_load: memory size = 72.00 MB, n_mem = 12288
gpt2_model_load: model size = 239.08 MB
extract_tests_from_file : No test file found.
test_gpt_tokenizer : 0 tests failed out of 0 tests.
main: compute buffer size: 3.26 MB
main: generating 5 sequences ...
main: prompt: 'Hello my name is'
main: number of tokens in prompt = 4, first 8 tokens: 15496 616 1438 318
sequence 0:
Hello my name is John. You can call me any way you want, if you want, but for my very first date, I will be on the phone with you. We're both in our early 20s, but I feel like it's all
sequence 1:
Hello my name is Robert, and I want to say that we're proud to have your company here on the world's largest platform for sharing your stories with us. This is a huge opportunity for our community. We have hundreds of people on this team and
sequence 2:
Hello my name is Jack. I'm the one who created you.
Jack is a boy with a big smile and a big heart. He is a handsome guy. He loves the outdoors and loves the people he meets. He wants to be a
sequence 3:
Hello my name is John. I am a Canadian citizen with a large number of family in Quebec and I am interested in studying. My aim is to take up a post in the Journal of the International Academy of Sciences of Canada which I am currently finishing.
sequence 4:
Hello my name is Dan. I am an entrepreneur. I am a great father. I am a great husband. I am a great husband. I am a great dad. And I am a great husband.
I love my life. I love
main: load time = 880.80 ms
main: sample time = 91.43 ms
main: predict time = 2518.29 ms
main: total time = 3544.32 ms
```
ggml/examples/gpt-2/convert-cerebras-to-ggml.py
+183
View File
@@ -0,0 +1,183 @@
# Convert Cerebras models to ggml format
#
# ref: https://www.cerebras.net/blog/cerebras-gpt-a-family-of-open-compute-efficient-large-language-models/
#
import sys
import struct
import json
import torch
import numpy as np
import re
from transformers import AutoModelForCausalLM
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
if len(sys.argv) < 2:
print("Usage: convert-cerebras-to-ggml.py dir-model [use-f32]\n")
sys.exit(1)
# output in the same directory as the model
dir_model = sys.argv[1]
fname_out = sys.argv[1] + "/ggml-model-f16.bin"
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
encoder = json.load(f)
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
# use 16-bit or 32-bit floats
use_f16 = True
if len(sys.argv) > 2:
use_f16 = False
fname_out = sys.argv[1] + "/ggml-model-f32.bin"
model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True)
#print (model)
list_vars = model.state_dict()
#print (list_vars)
print(hparams)
fout = open(fname_out, "wb")
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
fout.write(struct.pack("i", hparams["vocab_size"]))
fout.write(struct.pack("i", hparams["n_positions"]))
fout.write(struct.pack("i", hparams["n_embd"]))
fout.write(struct.pack("i", hparams["n_head"]))
fout.write(struct.pack("i", hparams["n_layer"]))
fout.write(struct.pack("i", use_f16))
byte_encoder = bytes_to_unicode()
byte_decoder = {v:k for k, v in byte_encoder.items()}
fout.write(struct.pack("i", len(encoder)))
for key in encoder:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for name in list_vars.keys():
data = list_vars[name].squeeze().numpy()
print("Processing variable: " + name + " with shape: ", data.shape)
# rename headers to keep compatibility
if name == "transformer.ln_f.weight":
name = "model/ln_f/g"
elif name == "transformer.ln_f.bias":
name = "model/ln_f/b"
elif name == "transformer.wte.weight":
name = "model/wte"
elif name == "transformer.wpe.weight":
name = "model/wpe"
elif name == "lm_head.weight":
name = "model/lm_head"
elif re.match(r"transformer.h\.\d+\.ln_1\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_1/g"
elif re.match(r"transformer.h\.\d+\.ln_1\.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_1/b"
elif re.match(r"transformer.h\.\d+\.attn\.c_attn\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_attn/w"
elif re.match(r"transformer.h\.\d+\.attn\.c_attn\.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_attn/b"
elif re.match(r"transformer.h\.\d+\.attn\.c_proj\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_proj/w"
elif re.match(r"transformer.h.\d+.attn.c_proj.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_proj/b"
elif re.match(r"transformer.h.\d+.ln_2.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_2/g"
elif re.match(r"transformer.h.\d+.ln_2.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_2/b"
elif re.match(r"transformer.h.\d+.mlp.c_fc.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_fc/w"
elif re.match(r"transformer.h.\d+.mlp.c_fc.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_fc/b"
elif re.match(r"transformer.h.\d+.mlp.c_proj.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_proj/w"
elif re.match(r"transformer.h.\d+.mlp.c_proj.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_proj/b"
else:
print("Unrecognized variable name. %s", name)
# we don't need these
if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
print(" Skipping variable: " + name)
continue
n_dims = len(data.shape);
# ftype == 0 -> float32, ftype == 1 -> float16
ftype = 0;
if use_f16:
if (name == "model/wte" or name == "model/lm_head" or name[-2:] == "/g" or name[-2:] == "/w") and n_dims == 2:
print(" Converting to float16")
data = data.astype(np.float16)
ftype = 1
else:
print(" Converting to float32")
data = data.astype(np.float32)
ftype = 0
# for efficiency - transpose the projection matrices
# "model/h.*/attn/c_attn/w"
# "model/h.*/attn/c_proj/w"
# "model/h.*/mlp/c_fc/w"
# "model/h.*/mlp/c_proj/w"
if name[-14:] == "/attn/c_attn/w" or \
name[-14:] == "/attn/c_proj/w" or \
name[-11:] == "/mlp/c_fc/w" or \
name[-13:] == "/mlp/c_proj/w":
print(" Transposing")
data = data.transpose()
# header
str = name.encode('utf-8')
fout.write(struct.pack("iii", n_dims, len(str), ftype))
for i in range(n_dims):
fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
fout.write(str);
# data
data.tofile(fout)
fout.close()
print("Done. Output file: " + fname_out)
print("")
ggml/examples/gpt-2/convert-ckpt-to-ggml.py
+159
View File
@@ -0,0 +1,159 @@
# Convert a model checkpoint to a ggml compatible file
#
# Load the model using TensorFlow.
# Iterate over all variables and write them to a binary file.
#
# For each variable, write the following:
# - Number of dimensions (int)
# - Name length (int)
# - Dimensions (int[n_dims])
# - Name (char[name_length])
# - Data (float[n_dims])
#
# By default, the bigger matrices are converted to 16-bit floats.
# This can be disabled by adding the "use-f32" CLI argument.
#
# At the start of the ggml file we write the model parameters
# and vocabulary.
#
import sys
import json
import struct
import numpy as np
import tensorflow as tf
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
# helper method to convert a numpy array to different float types
def convert_to_ftype(data, ftype):
# fp16
if ftype == 1:
return data.astype(np.float16)
assert False, "Invalid ftype: " + str(ftype)
if len(sys.argv) < 3:
print("Usage: convert-ckpt-to-ggml.py dir-model ftype\n")
print(" ftype == 0 -> float32")
print(" ftype == 1 -> float16")
sys.exit(1)
# output in the same directory as the model
dir_model = sys.argv[1]
fname_out = sys.argv[1] + "/ggml-model.bin"
with open(dir_model + "/encoder.json", "r", encoding="utf-8") as f:
encoder = json.load(f)
with open(dir_model + "/hparams.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
# possible data types
# ftype == 0 -> float32
# ftype == 1 -> float16
#
# map from ftype to string
ftype_str = ["f32", "f16"]
ftype = 1
if len(sys.argv) > 2:
ftype = int(sys.argv[2])
if ftype < 0 or ftype > 1:
print("Invalid ftype: " + str(ftype))
sys.exit(1)
fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
list_vars = tf.train.list_variables(dir_model)
fout = open(fname_out, "wb")
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
fout.write(struct.pack("i", hparams["n_vocab"]))
fout.write(struct.pack("i", hparams["n_ctx"]))
fout.write(struct.pack("i", hparams["n_embd"]))
fout.write(struct.pack("i", hparams["n_head"]))
fout.write(struct.pack("i", hparams["n_layer"]))
fout.write(struct.pack("i", ftype))
byte_encoder = bytes_to_unicode()
byte_decoder = {v:k for k, v in byte_encoder.items()}
fout.write(struct.pack("i", len(encoder)))
for key in encoder:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for name, shape in list_vars:
print("Processing variable: " + name + " with shape: ", shape)
data = tf.train.load_variable(dir_model, name).squeeze()
n_dims = len(data.shape);
# for efficiency - transpose the projection matrices
# "model/h.*/attn/c_attn/w"
# "model/h.*/attn/c_proj/w"
# "model/h.*/mlp/c_fc/w"
# "model/h.*/mlp/c_proj/w"
if name[-14:] == "/attn/c_attn/w" or \
name[-14:] == "/attn/c_proj/w" or \
name[-11:] == "/mlp/c_fc/w" or \
name[-13:] == "/mlp/c_proj/w":
print(" Transposing")
data = data.transpose()
dshape = data.shape
ftype_cur = 0
if ftype != 0:
# match name:
# "model/wte"
# "model/h.*/attn/c_attn/w"
# "model/h.*/attn/c_proj/w"
# "model/h.*/mlp/c_fc/w"
# "model/h.*/mlp/c_proj/w"
if name == "model/wte" or name[-2:] == "/w":
print(" Converting to " + ftype_str[ftype])
data = convert_to_ftype(data, ftype)
ftype_cur = ftype
else:
print(" Converting to float32")
data = data.astype(np.float32)
ftype_cur = 0
# header
str = name.encode('utf-8')
fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
for i in range(n_dims):
fout.write(struct.pack("i", dshape[n_dims - 1 - i]))
fout.write(str);
# data
data.tofile(fout)
fout.close()
print("Done. Output file: " + fname_out)
print("")
ggml/examples/gpt-2/convert-h5-to-ggml.py
+195
View File
@@ -0,0 +1,195 @@
# Convert GPT-2 h5 transformer model to ggml format
#
# Load the model using GPT2Model.
# Iterate over all variables and write them to a binary file.
#
# For each variable, write the following:
# - Number of dimensions (int)
# - Name length (int)
# - Dimensions (int[n_dims])
# - Name (char[name_length])
# - Data (float[n_dims])
#
# By default, the bigger matrices are converted to 16-bit floats.
# This can be disabled by adding the "use-f32" CLI argument.
#
# At the start of the ggml file we write the model parameters
# and vocabulary.
#
import sys
import struct
import json
import numpy as np
import re
from transformers import GPT2Model
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
if len(sys.argv) < 2:
print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
sys.exit(1)
# output in the same directory as the model
dir_model = sys.argv[1]
fname_out = sys.argv[1] + "/ggml-model.bin"
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
encoder = json.load(f)
with open(dir_model + "/added_tokens.json", "r", encoding="utf-8") as f:
encoder_added = json.load(f)
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
# use 16-bit or 32-bit floats
use_f16 = True
if len(sys.argv) > 2:
use_f16 = False
fname_out = sys.argv[1] + "/ggml-model-f32.bin"
model = GPT2Model.from_pretrained(dir_model, low_cpu_mem_usage=True)
#print (model)
list_vars = model.state_dict()
#print (list_vars)
fout = open(fname_out, "wb")
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
fout.write(struct.pack("i", hparams["vocab_size"]))
fout.write(struct.pack("i", hparams["n_positions"]))
fout.write(struct.pack("i", hparams["n_embd"]))
fout.write(struct.pack("i", hparams["n_head"]))
fout.write(struct.pack("i", hparams["n_layer"]))
#fout.write(struct.pack("i", hparams["rotary_dim"]))
fout.write(struct.pack("i", use_f16))
byte_encoder = bytes_to_unicode()
byte_decoder = {v:k for k, v in byte_encoder.items()}
fout.write(struct.pack("i", len(encoder) + len(encoder_added)))
for key in encoder:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for key in encoder_added:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for name in list_vars.keys():
data = list_vars[name].squeeze().numpy()
print("Processing variable: " + name + " with shape: ", data.shape)
# we don't need these
if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
print(" Skipping variable: " + name)
continue
n_dims = len(data.shape);
# ftype == 0 -> float32, ftype == 1 -> float16
ftype = 0;
if use_f16:
if name[-7:] == ".weight" and n_dims == 2:
print(" Converting to float16")
data = data.astype(np.float16)
ftype = 1
else:
print(" Converting to float32")
data = data.astype(np.float32)
ftype = 0
# for efficiency - transpose these matrices:
# "transformer.h.*.mlp.c_proj.weight
if name.endswith(".mlp.c_proj.weight"):
print(" Transposing")
data = data.transpose()
# rename headers to keep compatibility
if name == "ln_f.weight":
name = "model/ln_f/g"
elif name == "ln_f.bias":
name = "model/ln_f/b"
elif name == "wte.weight":
name = "model/wte"
elif name == "wpe.weight":
name = "model/wpe"
elif re.match(r"h\.\d+\.ln_1\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_1/g"
elif re.match(r"h\.\d+\.ln_1\.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_1/b"
elif re.match(r"h\.\d+\.attn\.c_attn\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_attn/w"
elif re.match(r"h\.\d+\.attn\.c_attn\.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_attn/b"
elif re.match(r"h\.\d+\.attn\.c_proj\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_proj/w"
elif re.match(r"h.\d+.attn.c_proj.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_proj/b"
elif re.match(r"h.\d+.ln_2.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_2/g"
elif re.match(r"h.\d+.ln_2.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_2/b"
elif re.match(r"h.\d+.mlp.c_fc.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_fc/w"
elif re.match(r"h.\d+.mlp.c_fc.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_fc/b"
elif re.match(r"h.\d+.mlp.c_proj.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_proj/w"
elif re.match(r"h.\d+.mlp.c_proj.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_proj/b"
else:
print("Unrecognized variable name. %s", name)
str = name.encode('utf-8')
fout.write(struct.pack("iii", n_dims, len(str), ftype))
for i in range(n_dims):
fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
fout.write(str);
# data
data.tofile(fout)
fout.close()
print("Done. Output file: " + fname_out)
print("")
ggml/examples/gpt-2/download-ggml-model.sh
Executable
+69
View File
@@ -0,0 +1,69 @@
#!/bin/bash
# This script downloads GPT-2 model files that have already been converted to ggml format.
# This way you don't have to convert them yourself.
#
# If you want to download the original GPT-2 model files, use the "download-model.sh" script instead.
#src="https://ggml.ggerganov.com"
#pfx="ggml-model-gpt-2"
src="https://huggingface.co/ggerganov/ggml"
pfx="resolve/main/ggml-model-gpt-2"
ggml_path=$(dirname $(realpath $0))
# GPT-2 models
models=( "117M" "345M" "774M" "1558M" )
# list available models
function list_models {
printf "\n"
printf " Available models:"
for model in "${models[@]}"; do
printf " $model"
done
printf "\n\n"
}
if [ "$#" -ne 1 ]; then
printf "Usage: $0 <model>\n"
list_models
exit 1
fi
model=$1
if [[ ! " ${models[@]} " =~ " ${model} " ]]; then
printf "Invalid model: $model\n"
list_models
exit 1
fi
# download ggml model
printf "Downloading ggml model $model ...\n"
mkdir -p models/gpt-2-$model
if [ -x "$(command -v wget)" ]; then
wget --quiet --show-progress -O models/gpt-2-$model/ggml-model.bin $src/$pfx-$model.bin
elif [ -x "$(command -v curl)" ]; then
curl -L --output models/gpt-2-$model/ggml-model.bin $src/$pfx-$model.bin
else
printf "Either wget or curl is required to download models.\n"
exit 1
fi
if [ $? -ne 0 ]; then
printf "Failed to download ggml model $model \n"
printf "Please try again later or download the original GPT-2 model files and convert them yourself.\n"
exit 1
fi
printf "Done! Model '$model' saved in 'models/gpt-2-$model/ggml-model.bin'\n"
printf "You can now use it like this:\n\n"
printf " $ ./bin/gpt-2 -m models/gpt-2-$model/ggml-model.bin -p \"This is an example\"\n"
printf "\n"
ggml/examples/gpt-2/download-model.sh
Executable
+48
View File
@@ -0,0 +1,48 @@
#!/bin/bash
ggml_path=$(dirname $(realpath $0))
# GPT-2 models
models=( "117M" "345M" "774M" "1558M" )
# list available models
function list_models {
printf "\n"
printf " Available models:"
for model in "${models[@]}"; do
printf " $model"
done
printf "\n\n"
}
if [ "$#" -ne 1 ]; then
printf "Usage: $0 <model>\n"
list_models
exit 1
fi
model=$1
if [[ ! " ${models[@]} " =~ " ${model} " ]]; then
printf "Invalid model: $model\n"
list_models
exit 1
fi
# download model
printf "Downloading model $model ...\n"
mkdir -p models/gpt-2-$model
for file in checkpoint encoder.json hparams.json model.ckpt.data-00000-of-00001 model.ckpt.index model.ckpt.meta vocab.bpe; do
wget --quiet --show-progress -O models/gpt-2-$model/$file https://openaipublic.blob.core.windows.net/gpt-2/models/$model/$file
done
printf "Done! Model '$model' saved in 'models/gpt-2-$model/'\n\n"
printf "Run the convert-ckpt-to-ggml.py script to convert the model to ggml format.\n"
printf "\n"
printf " python $ggml_path/convert-ckpt-to-ggml.py models/gpt-2-$model/\n"
printf "\n"
ggml/examples/gpt-2/main-alloc.cpp
+880
View File
@@ -0,0 +1,880 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
// default hparams (GPT-2 117M)
struct gpt2_hparams {
int32_t n_vocab = 50257;
int32_t n_ctx = 1024;
int32_t n_embd = 768;
int32_t n_head = 12;
int32_t n_layer = 12;
int32_t ftype = 1;
float eps = 1e-5f;
};
struct gpt2_layer {
// normalization
struct ggml_tensor * ln_1_g;
struct ggml_tensor * ln_1_b;
struct ggml_tensor * ln_2_g;
struct ggml_tensor * ln_2_b;
// attention
struct ggml_tensor * c_attn_attn_w;
struct ggml_tensor * c_attn_attn_b;
struct ggml_tensor * c_attn_proj_w;
struct ggml_tensor * c_attn_proj_b;
// mlp
struct ggml_tensor * c_mlp_fc_w;
struct ggml_tensor * c_mlp_fc_b;
struct ggml_tensor * c_mlp_proj_w;
struct ggml_tensor * c_mlp_proj_b;
};
struct gpt2_model {
gpt2_hparams hparams;
// normalization
struct ggml_tensor * ln_f_g;
struct ggml_tensor * ln_f_b;
struct ggml_tensor * wte; // token embedding
struct ggml_tensor * wpe; // position embedding
struct ggml_tensor * lm_head; // language model head
std::vector<gpt2_layer> layers;
// key + value memory
struct ggml_tensor * memory_k;
struct ggml_tensor * memory_v;
//
struct ggml_context * ctx_w;
std::map<std::string, struct ggml_tensor *> tensors;
};
// load the model's weights from a file
bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab) {
printf("%s: loading model from '%s'\n", __func__, fname.c_str());
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
return false;
}
// verify magic
{
uint32_t magic;
fin.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
return false;
}
}
// load hparams
{
auto & hparams = model.hparams;
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: ftype = %d\n", __func__, hparams.ftype);
printf("%s: qntvr = %d\n", __func__, qntvr);
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
}
// load vocab
{
int32_t n_vocab = 0;
fin.read((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != model.hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
return false;
}
std::string word;
std::vector<char> buf(128);
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
fin.read((char *) &len, sizeof(len));
buf.resize(len);
fin.read((char *) buf.data(), len);
word.assign(buf.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
// in order to save memory and also to speed up the computation
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
if (wtype == GGML_TYPE_COUNT) {
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
__func__, fname.c_str(), model.hparams.ftype);
return false;
}
auto & ctx = model.ctx_w;
size_t ctx_size = 0;
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // wte
ctx_size += ggml_row_size(GGML_TYPE_F32 , n_ctx*n_embd); // wpe
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b
ctx_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_k
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_v
ctx_size += (6 + 12*n_layer)*512; // object overhead
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
}
// create the ggml context
{
struct ggml_init_params params = {
/*.mem_size =*/ ctx_size,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ false,
};
model.ctx_w = ggml_init(params);
if (!model.ctx_w) {
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
return false;
}
}
// prepare memory for the weights
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
model.layers.resize(n_layer);
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);
model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
// map by name
model.tensors["model/ln_f/g"] = model.ln_f_g;
model.tensors["model/ln_f/b"] = model.ln_f_b;
model.tensors["model/wte"] = model.wte;
model.tensors["model/wpe"] = model.wpe;
model.tensors["model/lm_head"] = model.lm_head;
for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);
layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
// map by name
model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g;
model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b;
model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g;
model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b;
}
}
// key + value memory
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_mem = n_layer*n_ctx;
const int n_elements = n_embd*n_mem;
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
}
// load weights
{
size_t total_size = 0;
bool has_lm_head = false;
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (fin.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[2] = { 1, 1 };
for (int i = 0; i < n_dims; ++i) {
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
fin.read(&name[0], length);
if (model.tensors.find(name) == model.tensors.end()) {
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
return false;
}
auto tensor = model.tensors[name];
if (ggml_nelements(tensor) != nelements) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
return false;
}
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
return false;
}
// for debugging
if (0) {
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
}
const size_t bpe = ggml_type_size(ggml_type(ttype));
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
return false;
}
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
// GPT-2 models share the WTE tensor as the LM head
if (name == "model/wte" && has_lm_head == false) {
memcpy(model.lm_head->data, tensor->data, ggml_nbytes(tensor));
}
if (name == "model/lm_head") {
has_lm_head = true;
}
total_size += ggml_nbytes(tensor);
}
printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0);
}
fin.close();
return true;
}
// build the computation graph
struct ggml_cgraph * gpt2_graph(
const gpt2_model & model,
const int n_past,
const int n_tokens) {
const int N = n_tokens;
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_head = hparams.n_head;
// since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data
static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
static std::vector<uint8_t> buf(buf_size);
struct ggml_init_params params = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ buf.data(),
/*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()
};
struct ggml_context * ctx = ggml_init(params);
struct ggml_cgraph * gf = ggml_new_graph(ctx);
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
// at this point, the tensor data is not allocated yet and cannot be set
// we will find the tensor after the graph is allocated by its name, and set the data then
ggml_set_name(embd, "embd");
// setting a tensor as an input will ensure that it is allocated at the beginning of the graph
// this is important to ensure that the input tensors are not overwritten before they are used
ggml_set_input(embd);
struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
ggml_set_name(position, "position");
ggml_set_input(position);
// wte + wpe
struct ggml_tensor * inpL =
ggml_add(ctx,
ggml_get_rows(ctx, model.wte, embd),
ggml_get_rows(ctx, model.wpe, position));
for (int il = 0; il < n_layer; ++il) {
struct ggml_tensor * cur;
// norm
{
// [ 768, N]
cur = ggml_norm(ctx, inpL, hparams.eps);
// cur = ln_1_g*cur + ln_1_b
// [ 768, N]
cur = ggml_add(ctx,
ggml_mul(ctx,
ggml_repeat(ctx, model.layers[il].ln_1_g, cur),
cur),
ggml_repeat(ctx, model.layers[il].ln_1_b, cur));
}
// attn
// [2304, 768] - model.layers[il].c_attn_attn_w
// [2304, 1] - model.layers[il].c_attn_attn_b
// [ 768, N] - cur (in)
// [2304, N] - cur (out)
//
// cur = attn_w*cur + attn_b
// [2304, N]
{
cur = ggml_mul_mat(ctx,
model.layers[il].c_attn_attn_w,
cur);
cur = ggml_add(ctx,
ggml_repeat(ctx, model.layers[il].c_attn_attn_b, cur),
cur);
}
// self-attention
{
struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);
struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);
struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);
// store key and value to memory
if (N >= 1) {
struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));
ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k));
ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v));
}
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
// [64, N, 12]
struct ggml_tensor * Q =
ggml_permute(ctx,
ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N),
0, 2, 1, 3);
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
// [64, n_past + N, 12]
struct ggml_tensor * K =
ggml_permute(ctx,
ggml_reshape_3d(ctx,
ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
n_embd/n_head, n_head, n_past + N),
0, 2, 1, 3);
// GG: flash attention
//struct ggml_tensor * V =
// ggml_cpy(ctx0,
// ggml_permute(ctx0,
// ggml_reshape_3d(ctx0,
// ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
// n_embd/n_head, n_head, n_past + N),
// 1, 2, 0, 3),
// ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));
//struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);
// K * Q
// [n_past + N, N, 12]
struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q);
// KQ_scaled = KQ / sqrt(n_embd/n_head)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_scaled =
ggml_scale(ctx,
KQ,
1.0f/sqrtf(float(n_embd)/n_head));
// KQ_masked = mask_past(KQ_scaled)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past);
// KQ = soft_max(KQ_masked)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked);
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
// [n_past + N, 64, 12]
struct ggml_tensor * V_trans =
ggml_cont_3d(ctx,
ggml_permute(ctx,
ggml_reshape_3d(ctx,
ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
n_embd/n_head, n_head, n_past + N),
1, 2, 0, 3),
n_past + N, n_embd/n_head, n_head);
// KQV = transpose(V) * KQ_soft_max
// [64, N, 12]
struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max);
// KQV_merged = KQV.permute(0, 2, 1, 3)
// [64, 12, N]
struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3);
// cur = KQV_merged.contiguous().view(n_embd, N)
// [768, N]
cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N);
}
// projection
// [ 768, 768] - model.layers[il].c_attn_proj_w
// [ 768, 1] - model.layers[il].c_attn_proj_b
// [ 768, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
{
cur = ggml_mul_mat(ctx,
model.layers[il].c_attn_proj_w,
cur);
cur = ggml_add(ctx,
ggml_repeat(ctx, model.layers[il].c_attn_proj_b, cur),
cur);
}
// add the input
cur = ggml_add(ctx, cur, inpL);
struct ggml_tensor * inpFF = cur;
// feed-forward network
{
// norm
{
cur = ggml_norm(ctx, inpFF, hparams.eps);
// cur = ln_2_g*cur + ln_2_b
// [ 768, N]
cur = ggml_add(ctx,
ggml_mul(ctx,
ggml_repeat(ctx, model.layers[il].ln_2_g, cur),
cur),
ggml_repeat(ctx, model.layers[il].ln_2_b, cur));
}
// fully connected
// [3072, 768] - model.layers[il].c_mlp_fc_w
// [3072, 1] - model.layers[il].c_mlp_fc_b
// [ 768, N] - cur (in)
// [3072, N] - cur (out)
//
// cur = fc_w*cur + fc_b
// [3072, N]
cur = ggml_mul_mat(ctx,
model.layers[il].c_mlp_fc_w,
cur);
cur = ggml_add(ctx,
ggml_repeat(ctx, model.layers[il].c_mlp_fc_b, cur),
cur);
// GELU activation
// [3072, N]
cur = ggml_gelu(ctx, cur);
// projection
// [ 768, 3072] - model.layers[il].c_mlp_proj_w
// [ 768, 1] - model.layers[il].c_mlp_proj_b
// [3072, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
cur = ggml_mul_mat(ctx,
model.layers[il].c_mlp_proj_w,
cur);
cur = ggml_add(ctx,
ggml_repeat(ctx, model.layers[il].c_mlp_proj_b, cur),
cur);
}
// input for next layer
inpL = ggml_add(ctx, cur, inpFF);
}
// norm
{
// [ 768, N]
inpL = ggml_norm(ctx, inpL, hparams.eps);
// inpL = ln_f_g*inpL + ln_f_b
// [ 768, N]
inpL = ggml_add(ctx,
ggml_mul(ctx,
ggml_repeat(ctx, model.ln_f_g, inpL),
inpL),
ggml_repeat(ctx, model.ln_f_b, inpL));
}
// inpL = WTE * inpL
// [ 768, 50257] - model.lm_head
// [ 768, N] - inpL
inpL = ggml_mul_mat(ctx, model.lm_head, inpL);
ggml_set_name(inpL, "logits");
// setting a tensor as the output will ensure that it is not overwritten by subsequent operations
ggml_set_output(inpL);
// logits -> probs
//inpL = ggml_soft_max(ctx0, inpL);
ggml_build_forward_expand(gf, inpL);
ggml_free(ctx);
return gf;
}
// evaluate the transformer
//
// - model: the model
// - allocr: ggml_gallocr to use to allocate the compute buffer
// - n_threads: number of threads to use
// - n_past: the context size so far
// - embd_inp: the embeddings of the tokens in the context
// - embd_w: the predicted logits for the next token
//
bool gpt2_eval(
const gpt2_model & model,
ggml_gallocr_t allocr,
const int n_threads,
const int n_past,
const std::vector<gpt_vocab::id> & embd_inp,
std::vector<float> & embd_w) {
const int N = embd_inp.size();
const auto & hparams = model.hparams;
const int n_vocab = hparams.n_vocab;
struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp.size());
// allocate the graph tensors
ggml_gallocr_alloc_graph(allocr, gf);
// set the graph inputs
struct ggml_tensor * embd = ggml_graph_get_tensor(gf, "embd");
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
struct ggml_tensor * position = ggml_graph_get_tensor(gf, "position");
for (int i = 0; i < N; ++i) {
((int32_t *) position->data)[i] = n_past + i;
}
// run the computation
struct ggml_cplan plan = ggml_graph_plan(gf, n_threads, nullptr);
static std::vector<uint8_t> work_buffer;
work_buffer.resize(plan.work_size);
plan.work_data = work_buffer.data();
ggml_graph_compute(gf, &plan);
//if (n_past%100 == 0) {
// ggml_graph_print (&gf);
// ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
//}
// get the graph outputs
struct ggml_tensor * logits = ggml_graph_get_tensor(gf, "logits");
//embd_w.resize(n_vocab*N);
//memcpy(embd_w.data(), ggml_get_data(logits), sizeof(float)*n_vocab*N);
// return result just for the last token
embd_w.resize(n_vocab);
memcpy(embd_w.data(), (float *) ggml_get_data(logits) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
return true;
}
int main(int argc, char ** argv) {
ggml_time_init();
const int64_t t_main_start_us = ggml_time_us();
gpt_params params;
params.model = "models/gpt-2-117M/ggml-model.bin";
if (gpt_params_parse(argc, argv, params) == false) {
return 1;
}
if (params.seed < 0) {
params.seed = time(NULL);
}
printf("%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed);
if (params.prompt.empty()) {
params.prompt = gpt_random_prompt(rng);
}
int64_t t_load_us = 0;
gpt_vocab vocab;
gpt2_model model;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gpt2_model_load(params.model, model, vocab)) {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
return 1;
}
t_load_us = ggml_time_us() - t_start_us;
test_gpt_tokenizer(vocab, params.token_test);
}
ggml_gallocr_t allocr = NULL;
// allocate the compute buffer
{
allocr = ggml_gallocr_new(ggml_backend_cpu_buffer_type());
// create the worst case graph for memory usage estimation
int n_tokens = std::min(model.hparams.n_ctx, params.n_batch);
int n_past = model.hparams.n_ctx - n_tokens;
struct ggml_cgraph * gf = gpt2_graph(model, n_past, n_tokens);
// pre-allocate the compute buffer for the worst case (optional)
ggml_gallocr_reserve(allocr, gf);
size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);
fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0/1024.0);
}
int n_past = 0;
int64_t t_sample_us = 0;
int64_t t_predict_us = 0;
std::vector<float> logits;
// tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());
printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size());
for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {
printf("%d ", embd_inp[i]);
}
printf("\n\n");
// submit the input prompt token-by-token
// this reduces the memory usage during inference, at the cost of a bit of speed at the beginning
std::vector<gpt_vocab::id> embd;
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
// predict
if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us();
if (!gpt2_eval(model, allocr, params.n_threads, n_past, embd, logits)) {
printf("Failed to predict\n");
return 1;
}
t_predict_us += ggml_time_us() - t_start_us;
}
n_past += embd.size();
embd.clear();
if (i >= embd_inp.size()) {
// sample next token
const int top_k = params.top_k;
const float top_p = params.top_p;
const float temp = params.temp;
const int n_vocab = model.hparams.n_vocab;
gpt_vocab::id id = 0;
{
const int64_t t_start_sample_us = ggml_time_us();
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
t_sample_us += ggml_time_us() - t_start_sample_us;
}
// add it to the context
embd.push_back(id);
} else {
// if here, it means we are still processing the input prompt
for (size_t k = i; k < embd_inp.size(); k++) {
embd.push_back(embd_inp[k]);
if (int32_t(embd.size()) >= params.n_batch) {
break;
}
}
i += embd.size() - 1;
}
// display text
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
// end of text token
if (embd.back() == 50256) {
break;
}
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n\n");
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
ggml_free(model.ctx_w);
return 0;
}
ggml/examples/gpt-2/main-backend.cpp
+946
View File
@@ -0,0 +1,946 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#ifdef GGML_USE_CUDA
#include "ggml-cuda.h"
#endif
#ifdef GGML_USE_METAL
#include "ggml-metal.h"
#endif
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
#define GPT2_MAX_NODES 4096
static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
(void) level;
(void) user_data;
fputs(text, stderr);
fflush(stderr);
}
// default hparams (GPT-2 117M)
struct gpt2_hparams {
int32_t n_vocab = 50257;
int32_t n_ctx = 1024;
int32_t n_embd = 768;
int32_t n_head = 12;
int32_t n_layer = 12;
int32_t ftype = 1;
float eps = 1e-5f;
};
struct gpt2_layer {
// normalization
struct ggml_tensor * ln_1_g;
struct ggml_tensor * ln_1_b;
struct ggml_tensor * ln_2_g;
struct ggml_tensor * ln_2_b;
// attention
struct ggml_tensor * c_attn_attn_w;
struct ggml_tensor * c_attn_attn_b;
struct ggml_tensor * c_attn_proj_w;
struct ggml_tensor * c_attn_proj_b;
// mlp
struct ggml_tensor * c_mlp_fc_w;
struct ggml_tensor * c_mlp_fc_b;
struct ggml_tensor * c_mlp_proj_w;
struct ggml_tensor * c_mlp_proj_b;
};
struct gpt2_model {
gpt2_hparams hparams;
// normalization
struct ggml_tensor * ln_f_g;
struct ggml_tensor * ln_f_b;
struct ggml_tensor * wte; // token embedding
struct ggml_tensor * wpe; // position embedding
struct ggml_tensor * lm_head; // language model head
std::vector<gpt2_layer> layers;
// key + value memory
struct ggml_tensor * memory_k;
struct ggml_tensor * memory_v;
//
struct ggml_context * ctx_w;
struct ggml_context * ctx_kv;
ggml_backend_t backend = NULL;
ggml_backend_buffer_t buffer_w;
ggml_backend_buffer_t buffer_kv;
std::map<std::string, struct ggml_tensor *> tensors;
};
// load the model's weights from a file
bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab, int n_ctx, int n_gpu_layers) {
printf("%s: loading model from '%s'\n", __func__, fname.c_str());
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
return false;
}
// verify magic
{
uint32_t magic;
fin.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
return false;
}
}
// load hparams
{
auto & hparams = model.hparams;
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: ftype = %d\n", __func__, hparams.ftype);
printf("%s: qntvr = %d\n", __func__, qntvr);
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
}
// load vocab
{
int32_t n_vocab = 0;
fin.read((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != model.hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
return false;
}
std::string word;
std::vector<char> buf(128);
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
fin.read((char *) &len, sizeof(len));
buf.resize(len);
fin.read((char *) buf.data(), len);
word.assign(buf.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
// in order to save memory and also to speed up the computation
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
if (wtype == GGML_TYPE_COUNT) {
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
__func__, fname.c_str(), model.hparams.ftype);
return false;
}
ggml_log_set(ggml_log_callback_default, nullptr);
auto & ctx = model.ctx_w;
// create the ggml context
{
size_t n_tensors = 2 + 6 + 12*model.hparams.n_layer;
struct ggml_init_params params = {
/*.mem_size =*/ ggml_tensor_overhead() * n_tensors,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
ctx = ggml_init(params);
if (!ctx) {
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
return false;
}
}
// initialize the backend
#ifdef GGML_USE_CUDA
if (n_gpu_layers > 0) {
fprintf(stderr, "%s: using CUDA backend\n", __func__);
model.backend = ggml_backend_cuda_init(0);
if (!model.backend) {
fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
}
}
#endif
#ifdef GGML_USE_METAL
if (n_gpu_layers > 0) {
fprintf(stderr, "%s: using Metal backend\n", __func__);
model.backend = ggml_backend_metal_init();
if (!model.backend) {
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
}
}
#endif
if (!model.backend) {
// fallback to CPU backend
fprintf(stderr, "%s: using CPU backend\n", __func__);
model.backend = ggml_backend_cpu_init();
}
if (!model.backend) {
fprintf(stderr, "%s: ggml_backend_cpu_init() failed\n", __func__);
return false;
}
// create the tensors for the model
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
model.layers.resize(n_layer);
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);
model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
// map by name
model.tensors["model/ln_f/g"] = model.ln_f_g;
model.tensors["model/ln_f/b"] = model.ln_f_b;
model.tensors["model/wte"] = model.wte;
model.tensors["model/wpe"] = model.wpe;
model.tensors["model/lm_head"] = model.lm_head;
for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);
layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
// map by name
model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g;
model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b;
model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g;
model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b;
}
}
// allocate the model tensors in a backend buffer
model.buffer_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend);
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
printf("%s: backend buffer size = %6.2f MB\n", __func__, ggml_backend_buffer_get_size(model.buffer_w)/(1024.0*1024.0));
// override the default training context with the user-provided
model.hparams.n_ctx = n_ctx;
// key + value memory
{
auto * ctx = model.ctx_kv;
// create the ggml context
{
size_t n_tensors = 2;
struct ggml_init_params params = {
/*.mem_size =*/ ggml_tensor_overhead() * n_tensors,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
ctx = ggml_init(params);
if (!ctx) {
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
return false;
}
}
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_mem = n_layer*n_ctx;
const int n_elements = n_embd*n_mem;
// k and v here can also be GGML_TYPE_F16 to save memory and speed up the computation
// if backend supports it
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
// allocate the KV memory in a backend buffer
model.buffer_kv = ggml_backend_alloc_ctx_tensors(ctx, model.backend);
const size_t memory_size = ggml_backend_buffer_get_size(model.buffer_kv);
printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
}
// load weights
{
size_t total_size = 0;
bool has_lm_head = false;
std::vector<char> read_buf;
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (fin.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[2] = { 1, 1 };
for (int i = 0; i < n_dims; ++i) {
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
fin.read(&name[0], length);
if (model.tensors.find(name) == model.tensors.end()) {
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
return false;
}
auto tensor = model.tensors[name];
ggml_set_name(tensor, name.c_str());
if (ggml_nelements(tensor) != nelements) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
return false;
}
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
return false;
}
// for debugging
if (0) {
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
}
const size_t bpe = ggml_type_size(ggml_type(ttype));
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
return false;
}
if (ggml_backend_buffer_is_host(model.buffer_w)) {
// for some backends such as CPU and Metal, the tensor data is in system memory and we can read directly into it
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
} else {
// read into a temporary buffer first, then copy to device memory
read_buf.resize(ggml_nbytes(tensor));
fin.read(read_buf.data(), ggml_nbytes(tensor));
ggml_backend_tensor_set(tensor, read_buf.data(), 0, ggml_nbytes(tensor));
}
// GPT-2 models share the WTE tensor as the LM head
if (name == "model/wte" && has_lm_head == false) {
//ggml_backend_tensor_copy(tensor, model.lm_head);
model.lm_head = tensor;
}
if (name == "model/lm_head") {
has_lm_head = true;
}
total_size += ggml_nbytes(tensor);
}
printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0);
}
fin.close();
return true;
}
// build the computation graph
struct ggml_cgraph * gpt2_graph(
const gpt2_model & model,
const int n_past,
const int n_tokens) {
const int N = n_tokens;
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_head = hparams.n_head;
// since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data
static size_t buf_size = ggml_tensor_overhead()*GPT2_MAX_NODES + ggml_graph_overhead_custom(GPT2_MAX_NODES, false);
static std::vector<uint8_t> buf(buf_size);
struct ggml_init_params params = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ buf.data(),
/*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()
};
struct ggml_context * ctx = ggml_init(params);
struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, GPT2_MAX_NODES, false);
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
// at this point, the tensor data is not allocated yet and cannot be set
// we will find the tensor after the graph is allocated by its name, and set the data then
ggml_set_name(embd, "embd");
// setting a tensor as an input will ensure that it is allocated at the beginning of the graph
// this is important to ensure that the input tensors are not overwritten before they are used
ggml_set_input(embd);
struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
ggml_set_name(position, "position");
ggml_set_input(position);
// wte + wpe
struct ggml_tensor * inpL =
ggml_add(ctx,
ggml_get_rows(ctx, model.wte, embd),
ggml_get_rows(ctx, model.wpe, position));
for (int il = 0; il < n_layer; ++il) {
struct ggml_tensor * cur;
// norm
{
// [ 768, N]
cur = ggml_norm(ctx, inpL, hparams.eps);
// cur = ln_1_g*cur + ln_1_b
// [ 768, N]
cur = ggml_add(ctx,
ggml_mul(ctx,
cur,
model.layers[il].ln_1_g),
model.layers[il].ln_1_b);
}
// attn
// [2304, 768] - model.layers[il].c_attn_attn_w
// [2304, 1] - model.layers[il].c_attn_attn_b
// [ 768, N] - cur (in)
// [2304, N] - cur (out)
//
// cur = attn_w*cur + attn_b
// [2304, N]
{
cur = ggml_mul_mat(ctx,
model.layers[il].c_attn_attn_w,
cur);
cur = ggml_add(ctx,
cur,
model.layers[il].c_attn_attn_b);
}
// self-attention
{
struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);
struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);
struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);
// store key and value to memory
if (N >= 1) {
struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));
ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k));
ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v));
}
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
// [64, N, 12]
struct ggml_tensor * Q =
ggml_permute(ctx,
ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N),
0, 2, 1, 3);
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
// [64, n_past + N, 12]
struct ggml_tensor * K =
ggml_permute(ctx,
ggml_reshape_3d(ctx,
ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
n_embd/n_head, n_head, n_past + N),
0, 2, 1, 3);
// GG: flash attention
//struct ggml_tensor * V =
// ggml_cpy(ctx0,
// ggml_permute(ctx0,
// ggml_reshape_3d(ctx0,
// ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
// n_embd/n_head, n_head, n_past + N),
// 1, 2, 0, 3),
// ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));
//struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);
// K * Q
// [n_past + N, N, 12]
struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q);
// KQ_scaled = KQ / sqrt(n_embd/n_head)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_scaled =
ggml_scale(ctx,
KQ,
1.0f/sqrtf(float(n_embd)/n_head));
// KQ_masked = mask_past(KQ_scaled)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past);
// KQ = soft_max(KQ_masked)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked);
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
// [n_past + N, 64, 12]
struct ggml_tensor * V_trans =
ggml_cont_3d(ctx,
ggml_permute(ctx,
ggml_reshape_3d(ctx,
ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
n_embd/n_head, n_head, n_past + N),
1, 2, 0, 3),
n_past + N, n_embd/n_head, n_head);
// KQV = transpose(V) * KQ_soft_max
// [64, N, 12]
struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max);
// KQV_merged = KQV.permute(0, 2, 1, 3)
// [64, 12, N]
struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3);
// cur = KQV_merged.contiguous().view(n_embd, N)
// [768, N]
cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N);
}
// projection
// [ 768, 768] - model.layers[il].c_attn_proj_w
// [ 768, 1] - model.layers[il].c_attn_proj_b
// [ 768, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
{
cur = ggml_mul_mat(ctx,
model.layers[il].c_attn_proj_w,
cur);
cur = ggml_add(ctx,
cur,
model.layers[il].c_attn_proj_b);
}
// add the input
cur = ggml_add(ctx, cur, inpL);
struct ggml_tensor * inpFF = cur;
// feed-forward network
{
// norm
{
cur = ggml_norm(ctx, inpFF, hparams.eps);
// cur = ln_2_g*cur + ln_2_b
// [ 768, N]
cur = ggml_add(ctx,
ggml_mul(ctx,
cur,
model.layers[il].ln_2_g),
model.layers[il].ln_2_b);
}
// fully connected
// [3072, 768] - model.layers[il].c_mlp_fc_w
// [3072, 1] - model.layers[il].c_mlp_fc_b
// [ 768, N] - cur (in)
// [3072, N] - cur (out)
//
// cur = fc_w*cur + fc_b
// [3072, N]
cur = ggml_mul_mat(ctx,
model.layers[il].c_mlp_fc_w,
cur);
cur = ggml_add(ctx,
cur,
model.layers[il].c_mlp_fc_b);
// GELU activation
// [3072, N]
cur = ggml_gelu(ctx, cur);
// projection
// [ 768, 3072] - model.layers[il].c_mlp_proj_w
// [ 768, 1] - model.layers[il].c_mlp_proj_b
// [3072, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
cur = ggml_mul_mat(ctx,
model.layers[il].c_mlp_proj_w,
cur);
cur = ggml_add(ctx,
cur,
model.layers[il].c_mlp_proj_b);
}
// input for next layer
inpL = ggml_add(ctx, cur, inpFF);
}
// norm
{
// [ 768, N]
inpL = ggml_norm(ctx, inpL, hparams.eps);
// inpL = ln_f_g*inpL + ln_f_b
// [ 768, N]
inpL = ggml_add(ctx,
ggml_mul(ctx,
inpL,
model.ln_f_g),
model.ln_f_b);
}
// inpL = WTE * inpL
// [ 768, 50257] - model.lm_head
// [ 768, N] - inpL
inpL = ggml_mul_mat(ctx, model.lm_head, inpL);
ggml_set_name(inpL, "logits");
// setting a tensor as the output will ensure that it is not overwritten by subsequent operations
ggml_set_output(inpL);
// logits -> probs
//inpL = ggml_soft_max(ctx0, inpL);
ggml_build_forward_expand(gf, inpL);
ggml_free(ctx);
return gf;
}
// evaluate the transformer
//
// - model: the model
// - allocr: ggml_gallocr to use to allocate the compute buffer
// - n_threads: number of threads to use
// - n_past: the context size so far
// - embd_inp: the embeddings of the tokens in the context
// - embd_w: the predicted logits for the next token
//
bool gpt2_eval(
const gpt2_model & model,
ggml_gallocr_t allocr,
const int n_threads,
const int n_past,
const std::vector<gpt_vocab::id> & embd_inp,
std::vector<float> & embd_w) {
const int N = embd_inp.size();
const auto & hparams = model.hparams;
const int n_vocab = hparams.n_vocab;
struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp.size());
// allocate the graph tensors
ggml_gallocr_alloc_graph(allocr, gf);
// set the graph inputs
struct ggml_tensor * embd = ggml_graph_get_tensor(gf, "embd");
ggml_backend_tensor_set(embd, embd_inp.data(), 0, N*ggml_element_size(embd));
struct ggml_tensor * position = ggml_graph_get_tensor(gf, "position");
for (int i = 0; i < N; ++i) {
int32_t v = n_past + i;
ggml_backend_tensor_set(position, &v, i*sizeof(int32_t), sizeof(v));
}
// set backend options
if (ggml_backend_is_cpu(model.backend)) {
ggml_backend_cpu_set_n_threads(model.backend, n_threads);
}
// run the computation
ggml_backend_graph_compute(model.backend, gf);
//if (n_past%100 == 0) {
// ggml_graph_print (&gf);
// ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
//}
// get the graph outputs
struct ggml_tensor * logits = ggml_graph_get_tensor(gf, "logits");
//embd_w.resize(n_vocab*N);
//ggml_backend_tensor_get(logits, embd_w.data(), 0, sizeof(float)*n_vocab*N);
// return result just for the last token
embd_w.resize(n_vocab);
ggml_backend_tensor_get(logits, embd_w.data(), (n_vocab*(N-1))*sizeof(float), sizeof(float)*n_vocab);
return true;
}
int main(int argc, char ** argv) {
ggml_time_init();
const int64_t t_main_start_us = ggml_time_us();
gpt_params params;
params.model = "models/gpt-2-117M/ggml-model.bin";
if (gpt_params_parse(argc, argv, params) == false) {
return 1;
}
if (params.seed < 0) {
params.seed = time(NULL);
}
printf("%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed);
if (params.prompt.empty()) {
params.prompt = gpt_random_prompt(rng);
}
int64_t t_load_us = 0;
gpt_vocab vocab;
gpt2_model model;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gpt2_model_load(params.model, model, vocab, params.n_ctx, params.n_gpu_layers)) {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
return 1;
}
t_load_us = ggml_time_us() - t_start_us;
test_gpt_tokenizer(vocab, params.token_test);
}
ggml_gallocr_t allocr = NULL;
// allocate the compute buffer
{
// create a graph allocator with the backend's default buffer type
allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
// create the worst case graph for memory usage estimation
int n_tokens = std::min(model.hparams.n_ctx, params.n_batch);
int n_past = model.hparams.n_ctx - n_tokens;
struct ggml_cgraph * gf = gpt2_graph(model, n_past, n_tokens);
// pre-allocate the compute buffer for the worst case (optional)
ggml_gallocr_reserve(allocr, gf);
size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);
fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0/1024.0);
}
int n_past = 0;
int64_t t_sample_us = 0;
int64_t t_predict_us = 0;
std::vector<float> logits;
// tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());
printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size());
for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {
printf("%d ", embd_inp[i]);
}
printf("\n\n");
// submit the input prompt token-by-token
// this reduces the memory usage during inference, at the cost of a bit of speed at the beginning
std::vector<gpt_vocab::id> embd;
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
// predict
if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us();
if (!gpt2_eval(model, allocr, params.n_threads, n_past, embd, logits)) {
printf("Failed to predict\n");
return 1;
}
t_predict_us += ggml_time_us() - t_start_us;
}
n_past += embd.size();
embd.clear();
if (i >= embd_inp.size()) {
// sample next token
const int top_k = params.top_k;
const float top_p = params.top_p;
const float temp = params.temp;
const int n_vocab = model.hparams.n_vocab;
gpt_vocab::id id = 0;
{
const int64_t t_start_sample_us = ggml_time_us();
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
t_sample_us += ggml_time_us() - t_start_sample_us;
}
// add it to the context
embd.push_back(id);
} else {
// if here, it means we are still processing the input prompt
for (size_t k = i; k < embd_inp.size(); k++) {
embd.push_back(embd_inp[k]);
if (int32_t(embd.size()) >= params.n_batch) {
break;
}
}
i += embd.size() - 1;
}
// display text
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
// end of text token
if (!params.ignore_eos && embd.back() == 50256) {
break;
}
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n\n");
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
ggml_free(model.ctx_w);
ggml_gallocr_free(allocr);
ggml_backend_buffer_free(model.buffer_w);
ggml_backend_buffer_free(model.buffer_kv);
ggml_backend_free(model.backend);
return 0;
}
ggml/examples/gpt-2/main-batched.cpp
+1210
View File
File diff suppressed because it is too large Load Diff
ggml/examples/gpt-2/main-ctx.cpp
+840
View File
@@ -0,0 +1,840 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
// default hparams (GPT-2 117M)
struct gpt2_hparams {
int32_t n_vocab = 50257;
int32_t n_ctx = 1024;
int32_t n_embd = 768;
int32_t n_head = 12;
int32_t n_layer = 12;
int32_t ftype = 1;
float eps = 1e-5f;
};
struct gpt2_layer {
// normalization
struct ggml_tensor * ln_1_g;
struct ggml_tensor * ln_1_b;
struct ggml_tensor * ln_2_g;
struct ggml_tensor * ln_2_b;
// attention
struct ggml_tensor * c_attn_attn_w;
struct ggml_tensor * c_attn_attn_b;
struct ggml_tensor * c_attn_proj_w;
struct ggml_tensor * c_attn_proj_b;
// mlp
struct ggml_tensor * c_mlp_fc_w;
struct ggml_tensor * c_mlp_fc_b;
struct ggml_tensor * c_mlp_proj_w;
struct ggml_tensor * c_mlp_proj_b;
};
struct gpt2_model {
gpt2_hparams hparams;
// normalization
struct ggml_tensor * ln_f_g;
struct ggml_tensor * ln_f_b;
struct ggml_tensor * wte; // token embedding
struct ggml_tensor * wpe; // position embedding
struct ggml_tensor * lm_head; // language model head
std::vector<gpt2_layer> layers;
// key + value memory
struct ggml_tensor * memory_k;
struct ggml_tensor * memory_v;
//
struct ggml_context * ctx_w;
std::map<std::string, struct ggml_tensor *> tensors;
};
// load the model's weights from a file
bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab) {
printf("%s: loading model from '%s'\n", __func__, fname.c_str());
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
return false;
}
// verify magic
{
uint32_t magic;
fin.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
return false;
}
}
// load hparams
{
auto & hparams = model.hparams;
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: ftype = %d\n", __func__, hparams.ftype);
printf("%s: qntvr = %d\n", __func__, qntvr);
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
}
// load vocab
{
int32_t n_vocab = 0;
fin.read((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != model.hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
return false;
}
std::string word;
std::vector<char> buf(128);
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
fin.read((char *) &len, sizeof(len));
buf.resize(len);
fin.read((char *) buf.data(), len);
word.assign(buf.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
// in order to save memory and also to speed up the computation
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
if (wtype == GGML_TYPE_COUNT) {
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
__func__, fname.c_str(), model.hparams.ftype);
return false;
}
auto & ctx = model.ctx_w;
size_t ctx_size = 0;
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // wte
ctx_size += ggml_row_size(GGML_TYPE_F32, n_ctx*n_embd); // wpe
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b
ctx_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_k
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_v
ctx_size += (6 + 12*n_layer)*512; // object overhead
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
}
// create the ggml context
{
struct ggml_init_params params = {
/*.mem_size =*/ ctx_size,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ false,
};
model.ctx_w = ggml_init(params);
if (!model.ctx_w) {
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
return false;
}
}
// prepare memory for the weights
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
model.layers.resize(n_layer);
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);
model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
// map by name
model.tensors["model/ln_f/g"] = model.ln_f_g;
model.tensors["model/ln_f/b"] = model.ln_f_b;
model.tensors["model/wte"] = model.wte;
model.tensors["model/wpe"] = model.wpe;
model.tensors["model/lm_head"] = model.lm_head;
for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);
layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
// map by name
model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g;
model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b;
model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g;
model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b;
}
}
// key + value memory
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_mem = n_layer*n_ctx;
const int n_elements = n_embd*n_mem;
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
}
// load weights
{
size_t total_size = 0;
bool has_lm_head = false;
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (fin.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[2] = { 1, 1 };
for (int i = 0; i < n_dims; ++i) {
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
fin.read(&name[0], length);
if (model.tensors.find(name) == model.tensors.end()) {
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
return false;
}
auto tensor = model.tensors[name];
if (ggml_nelements(tensor) != nelements) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
return false;
}
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
return false;
}
// for debugging
if (0) {
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
}
const size_t bpe = ggml_type_size(ggml_type(ttype));
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
return false;
}
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
// GPT-2 models share the WTE tensor as the LM head
if (name == "model/wte" && has_lm_head == false) {
memcpy(model.lm_head->data, tensor->data, ggml_nbytes(tensor));
}
if (name == "model/lm_head") {
has_lm_head = true;
}
total_size += ggml_nbytes(tensor);
}
printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0);
}
fin.close();
return true;
}
// evaluate the transformer
//
// - model: the model
// - n_threads: number of threads to use
// - n_past: the context size so far
// - embd_inp: the embeddings of the tokens in the context
// - embd_w: the predicted logits for the next token
//
bool gpt2_eval(
const gpt2_model & model,
const int n_threads,
const int n_past,
const std::vector<gpt_vocab::id> & embd_inp,
std::vector<float> & embd_w,
size_t & mem_per_token) {
const int N = embd_inp.size();
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_head = hparams.n_head;
const int n_vocab = hparams.n_vocab;
static size_t buf_size = 256u*1024*1024;
static void * buf = malloc(buf_size);
if (mem_per_token > 0 && mem_per_token*N > buf_size) {
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
// reallocate
buf_size = buf_size_new;
buf = realloc(buf, buf_size);
if (buf == nullptr) {
fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size);
return false;
}
}
struct ggml_init_params params = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ buf,
/*.no_alloc =*/ false,
};
struct ggml_context * ctx0 = ggml_init(params);
struct ggml_cgraph * gf = ggml_new_graph(ctx0);
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
struct ggml_tensor * position = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
for (int i = 0; i < N; ++i) {
((int32_t *) position->data)[i] = n_past + i;
}
// wte + wpe
struct ggml_tensor * inpL =
ggml_add(ctx0,
ggml_get_rows(ctx0, model.wte, embd),
ggml_get_rows(ctx0, model.wpe, position));
for (int il = 0; il < n_layer; ++il) {
struct ggml_tensor * cur;
// norm
{
// [ 768, N]
cur = ggml_norm(ctx0, inpL, hparams.eps);
// cur = ln_1_g*cur + ln_1_b
// [ 768, N]
cur = ggml_add(ctx0,
ggml_mul(ctx0,
ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),
cur),
ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));
}
// attn
// [2304, 768] - model.layers[il].c_attn_attn_w
// [2304, 1] - model.layers[il].c_attn_attn_b
// [ 768, N] - cur (in)
// [2304, N] - cur (out)
//
// cur = attn_w*cur + attn_b
// [2304, N]
{
cur = ggml_mul_mat(ctx0,
model.layers[il].c_attn_attn_w,
cur);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_attn_attn_b, cur),
cur);
}
// self-attention
{
struct ggml_tensor * Qcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);
struct ggml_tensor * Kcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);
struct ggml_tensor * Vcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);
// store key and value to memory
if (N >= 1) {
struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
struct ggml_tensor * v = ggml_view_1d(ctx0, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
}
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
// [64, N, 12]
struct ggml_tensor * Q =
ggml_permute(ctx0,
ggml_cpy(ctx0,
Qcur,
ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
0, 2, 1, 3);
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
// [64, n_past + N, 12]
struct ggml_tensor * K =
ggml_permute(ctx0,
ggml_reshape_3d(ctx0,
ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
n_embd/n_head, n_head, n_past + N),
0, 2, 1, 3);
// GG: flash attention
//struct ggml_tensor * V =
// ggml_cpy(ctx0,
// ggml_permute(ctx0,
// ggml_reshape_3d(ctx0,
// ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
// n_embd/n_head, n_head, n_past + N),
// 1, 2, 0, 3),
// ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));
//struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);
// K * Q
// [n_past + N, N, 12]
struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
// KQ_scaled = KQ / sqrt(n_embd/n_head)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head));
// KQ_masked = mask_past(KQ_scaled)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
// KQ = soft_max(KQ_masked)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
// [n_past + N, 64, 12]
struct ggml_tensor * V_trans =
ggml_cpy(ctx0,
ggml_permute(ctx0,
ggml_reshape_3d(ctx0,
ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
n_embd/n_head, n_head, n_past + N),
1, 2, 0, 3),
ggml_new_tensor_3d(ctx0, model.memory_v->type, n_past + N, n_embd/n_head, n_head));
// KQV = transpose(V) * KQ_soft_max
// [64, N, 12]
struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);
// KQV_merged = KQV.permute(0, 2, 1, 3)
// [64, 12, N]
struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
// cur = KQV_merged.contiguous().view(n_embd, N)
// [768, N]
cur = ggml_cpy(ctx0,
KQV_merged,
ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
}
// projection
// [ 768, 768] - model.layers[il].c_attn_proj_w
// [ 768, 1] - model.layers[il].c_attn_proj_b
// [ 768, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
{
cur = ggml_mul_mat(ctx0,
model.layers[il].c_attn_proj_w,
cur);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_attn_proj_b, cur),
cur);
}
// add the input
cur = ggml_add(ctx0, cur, inpL);
struct ggml_tensor * inpFF = cur;
// feed-forward network
{
// norm
{
cur = ggml_norm(ctx0, inpFF, hparams.eps);
// cur = ln_2_g*cur + ln_2_b
// [ 768, N]
cur = ggml_add(ctx0,
ggml_mul(ctx0,
ggml_repeat(ctx0, model.layers[il].ln_2_g, cur),
cur),
ggml_repeat(ctx0, model.layers[il].ln_2_b, cur));
}
// fully connected
// [3072, 768] - model.layers[il].c_mlp_fc_w
// [3072, 1] - model.layers[il].c_mlp_fc_b
// [ 768, N] - cur (in)
// [3072, N] - cur (out)
//
// cur = fc_w*cur + fc_b
// [3072, N]
cur = ggml_mul_mat(ctx0,
model.layers[il].c_mlp_fc_w,
cur);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
cur);
// GELU activation
// [3072, N]
cur = ggml_gelu(ctx0, cur);
// projection
// [ 768, 3072] - model.layers[il].c_mlp_proj_w
// [ 768, 1] - model.layers[il].c_mlp_proj_b
// [3072, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
cur = ggml_mul_mat(ctx0,
model.layers[il].c_mlp_proj_w,
cur);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),
cur);
}
// input for next layer
inpL = ggml_add(ctx0, cur, inpFF);
}
// norm
{
// [ 768, N]
inpL = ggml_norm(ctx0, inpL, hparams.eps);
// inpL = ln_f_g*inpL + ln_f_b
// [ 768, N]
inpL = ggml_add(ctx0,
ggml_mul(ctx0,
ggml_repeat(ctx0, model.ln_f_g, inpL),
inpL),
ggml_repeat(ctx0, model.ln_f_b, inpL));
}
// inpL = WTE * inpL
// [ 768, 50257] - model.lm_head
// [ 768, N] - inpL
inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);
// logits -> probs
//inpL = ggml_soft_max_inplace(ctx0, inpL);
// run the computation
ggml_build_forward_expand(gf, inpL);
ggml_graph_compute_with_ctx(ctx0, gf, n_threads);
//if (n_past%100 == 0) {
// ggml_graph_print (&gf);
// ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
//}
//embd_w.resize(n_vocab*N);
//memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
// return result just for the last token
embd_w.resize(n_vocab);
memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
if (mem_per_token == 0) {
mem_per_token = ggml_used_mem(ctx0)/N;
}
//printf("used_mem = %zu\n", ggml_used_mem(ctx0));
ggml_free(ctx0);
return true;
}
int main(int argc, char ** argv) {
ggml_time_init();
const int64_t t_main_start_us = ggml_time_us();
gpt_params params;
params.model = "models/gpt-2-117M/ggml-model.bin";
if (gpt_params_parse(argc, argv, params) == false) {
return 1;
}
if (params.seed < 0) {
params.seed = time(NULL);
}
printf("%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed);
if (params.prompt.empty()) {
params.prompt = gpt_random_prompt(rng);
}
int64_t t_load_us = 0;
gpt_vocab vocab;
gpt2_model model;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gpt2_model_load(params.model, model, vocab)) {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
return 1;
}
t_load_us = ggml_time_us() - t_start_us;
test_gpt_tokenizer(vocab, params.token_test);
}
int n_past = 0;
int64_t t_sample_us = 0;
int64_t t_predict_us = 0;
std::vector<float> logits;
// tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());
printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size());
for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {
printf("%d ", embd_inp[i]);
}
printf("\n\n");
// submit the input prompt token-by-token
// this reduces the memory usage during inference, at the cost of a bit of speed at the beginning
std::vector<gpt_vocab::id> embd;
// determine the required inference memory per token:
size_t mem_per_token = 0;
gpt2_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
// predict
if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us();
if (!gpt2_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {
printf("Failed to predict\n");
return 1;
}
t_predict_us += ggml_time_us() - t_start_us;
}
n_past += embd.size();
embd.clear();
if (i >= embd_inp.size()) {
// sample next token
const int top_k = params.top_k;
const float top_p = params.top_p;
const float temp = params.temp;
const int n_vocab = model.hparams.n_vocab;
gpt_vocab::id id = 0;
{
const int64_t t_start_sample_us = ggml_time_us();
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
t_sample_us += ggml_time_us() - t_start_sample_us;
}
// add it to the context
embd.push_back(id);
} else {
// if here, it means we are still processing the input prompt
for (size_t k = i; k < embd_inp.size(); k++) {
embd.push_back(embd_inp[k]);
if (int32_t(embd.size()) >= params.n_batch) {
break;
}
}
i += embd.size() - 1;
}
// display text
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
// end of text token
if (embd.back() == 50256) {
break;
}
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n\n");
printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token);
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
ggml_free(model.ctx_w);
return 0;
}
ggml/examples/gpt-2/main-sched.cpp
+1079
View File
File diff suppressed because it is too large Load Diff
ggml/examples/gpt-2/quantize.cpp
+184
View File
@@ -0,0 +1,184 @@
#include "ggml.h"
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#include <regex>
// default hparams (GPT-2 117M)
struct gpt2_hparams {
int32_t n_vocab = 50257;
int32_t n_ctx = 1024;
int32_t n_embd = 768;
int32_t n_head = 12;
int32_t n_layer = 12;
int32_t ftype = 1;
};
// quantize a model
bool gpt2_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {
gpt_vocab vocab;
printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str());
auto finp = std::ifstream(fname_inp, std::ios::binary);
if (!finp) {
fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str());
return false;
}
auto fout = std::ofstream(fname_out, std::ios::binary);
if (!fout) {
fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str());
return false;
}
// verify magic
{
uint32_t magic;
finp.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname_inp.c_str());
return false;
}
fout.write((char *) &magic, sizeof(magic));
}
gpt2_hparams hparams;
// load hparams
{
finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
finp.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
finp.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
finp.read((char *) &hparams.n_head, sizeof(hparams.n_head));
finp.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
finp.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR;
const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: ftype (src) = %d\n", __func__, hparams.ftype);
printf("%s: qntvr (src) = %d\n", __func__, qntvr_src);
printf("%s: ftype (dst) = %d\n", __func__, ftype_dst);
printf("%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION);
fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fout.write((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fout.write((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fout.write((char *) &hparams.n_head, sizeof(hparams.n_head));
fout.write((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fout.write((char *) &ftype_dst, sizeof(ftype_dst));
}
// load vocab
{
int32_t n_vocab = 0;
finp.read ((char *) &n_vocab, sizeof(n_vocab));
fout.write((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname_inp.c_str(), n_vocab, hparams.n_vocab);
return false;
}
std::string word;
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
finp.read ((char *) &len, sizeof(len));
fout.write((char *) &len, sizeof(len));
word.resize(len);
finp.read ((char *) word.data(), len);
fout.write((char *) word.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// regexes of tensor names to be quantized
const std::vector<std::string> to_quant = {
"model/wte",
"model/lm_head",
"model/h.*/attn/c_attn/w",
"model/h.*/attn/c_proj/w",
"model/h.*/mlp/c_fc/w",
"model/h.*/mlp/c_proj/w",
};
if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, {})) {
fprintf(stderr, "%s: failed to quantize model '%s'\n", __func__, fname_inp.c_str());
return false;
}
finp.close();
fout.close();
return true;
}
// usage:
// ./gpt-2-quantize models/gpt-2-117M/ggml-model.bin models/gpt-2-117M/ggml-model-quant.bin type
//
int main(int argc, char ** argv) {
if (argc != 4) {
fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]);
ggml_print_ftypes(stderr);
return 1;
}
// needed to initialize f16 tables
{
struct ggml_init_params params = { 0, NULL, false };
struct ggml_context * ctx = ggml_init(params);
ggml_free(ctx);
}
const std::string fname_inp = argv[1];
const std::string fname_out = argv[2];
const ggml_ftype ftype = ggml_parse_ftype(argv[3]);
const int64_t t_main_start_us = ggml_time_us();
int64_t t_quantize_us = 0;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gpt2_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) {
fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());
return 1;
}
t_quantize_us = ggml_time_us() - t_start_us;
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n");
printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0f);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
return 0;
}
ggml/examples/gpt-j/CMakeLists.txt
+13
View File
@@ -0,0 +1,13 @@
#
# gpt-j
set(TEST_TARGET gpt-j)
add_executable(${TEST_TARGET} main.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
#
# gpt-j-quantize
set(TEST_TARGET gpt-j-quantize)
add_executable(${TEST_TARGET} quantize.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
ggml/examples/gpt-j/README.md
+239
View File
@@ -0,0 +1,239 @@
# gpt-j
Local GPT-J inference on your computer using C/C++
No video card required. You just need to have 16 GB of RAM.
## Motivation
The GPT-J 6B model is the open-source alternative to OpenAI's GPT-3. It's basically a neural network that allows you to
generate coherent, human-like text given a certain context (prompt).
The GPT-J model is quite big - the compact version of the model uses 16-bit floating point representation of the weights
and is still 12 GB big. This means that in order to run inference on your computer, you would need to have a video card
with at least 12 GB of video RAM. Alternatively, you can try to run the python implementations on the CPU, but that
would probably not be very efficient as they are primarily optimized for running on a GPU (or at least this is my guess -
I don't have much experience with python).
I wanted to try and run the model on my MacBook, so I decided to implement the model inference from scratch using my own
custom build tensor library. The tensor library (called [ggml](https://github.com/ggerganov/ggml), written in C) is in
early development stage, but it already allows me to run the GPT-J model.
On my 32GB MacBook M1 Pro, I achieve an inference speed of about `125 ms/token` or about ~6 words per second (1 word
typically consists of 1 or 2 tokens).
Here is a sample run with prompt `int main(int argc, char ** argv) {`:
```bash
$ time ./bin/gpt-j -p "int main(int argc, char ** argv) {"
gptj_model_load: loading model from 'models/gpt-j-6B/ggml-model.bin' - please wait ...
gptj_model_load: n_vocab = 50400
gptj_model_load: n_ctx = 2048
gptj_model_load: n_embd = 4096
gptj_model_load: n_head = 16
gptj_model_load: n_layer = 28
gptj_model_load: n_rot = 64
gptj_model_load: f16 = 1
gptj_model_load: ggml ctx size = 13334.86 MB
gptj_model_load: memory_size = 1792.00 MB, n_mem = 57344
gptj_model_load: ................................... done
gptj_model_load: model size = 11542.79 MB / num tensors = 285
main: number of tokens in prompt = 13
int main(int argc, char ** argv) {
(void)argc;
(void)argv;
{
struct sockaddr_in addr;
int addrlen;
char * ip = "192.168.1.4";
int i;
if ( (addrlen = sizeof(addr)) == -1 )
return -1;
for (i = 0; i < 10; ++i) {
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = inet_addr(ip);
main: mem per token = 16430420 bytes
main: load time = 6211.48 ms
main: sample time = 13.74 ms
main: predict time = 26420.34 ms / 124.62 ms per token
main: total time = 33035.37 ms
real 0m33.171s
user 3m32.269s
sys 0m3.686s
$
```
It took ~6.2 seconds to load the model to memory. After that, it took ~26.4 seconds to generate 200 tokens of what
looks like to be the beginning of a networking program in C. Pretty cool!
Here is another run, just for fun:
```bash
time ./bin/gpt-j -n 500 -t 8 -p "Ask HN: Inherited the worst code and tech team I have ever seen. How to fix it?
"
gptj_model_load: loading model from 'models/gpt-j-6B/ggml-model.bin' - please wait ...
gptj_model_load: n_vocab = 50400
gptj_model_load: n_ctx = 2048
gptj_model_load: n_embd = 4096
gptj_model_load: n_head = 16
gptj_model_load: n_layer = 28
gptj_model_load: n_rot = 64
gptj_model_load: f16 = 1
gptj_model_load: ggml ctx size = 13334.86 MB
gptj_model_load: memory_size = 1792.00 MB, n_mem = 57344
gptj_model_load: ................................... done
gptj_model_load: model size = 11542.79 MB / num tensors = 285
main: number of tokens in prompt = 24
Ask HN: Inherited the worst code and tech team I have ever seen. How to fix it?
I've inherited a team with some very strange and un-documented practices, one of them is that they use an old custom
application with a very slow tech stack written in Python that the team doesn't want to touch but also doesn't want to
throw away as it has some "legacy" code in it.
The problem is, the tech stack is very very slow.
They have a single web server on a VM that is slow.
The server is a little bit busy (not very busy though) and they have a lot of processes (30+ that are constantly being
spawned by the application)
They have an application that is single threaded and was written in Python and the team don't want to touch this, and
the application is very slow.
My task as a new member of the team is to fix this.
I'm a senior dev on the team (3 years on the project) and have been told that I will take the lead on this task. I know
next to nothing about Python. So here is what I have so far.
What I have done is I've been trying to debug the processes with the "ps" command. This way I can see what is running
and where. From what I see, the application spawns 10 processes a minute and some of them are used for nothing.
I have also started to look for the code. The application source is not in GitHub or any other repository, it is only on
our internal GitLab.
What I've found so far:
The application uses a custom SQLAlchemy implementation to interact with the data. I've looked at the source, it looks
like an object cache or something like that. But from what I've seen, the cache gets full every 20 minutes and then gets
cleared with a special command.
Another strange thing is that the application creates a file for every entry in the database (even if the entry already
exists). I've looked at the file to see if it contains something, but it seems to be a JSON file with lots of records.
The other strange thing is that I can only find the database tables in the GitLab repository and not the code. So I
can't really understand how the application is supposed to interact with the database.
I also found a "log" directory, but the code is encrypted with AES. From what I've found, it is in
main: mem per token = 16430420 bytes
main: load time = 3900.10 ms
main: sample time = 32.58 ms
main: predict time = 68049.91 ms / 130.11 ms per token
main: total time = 73020.05 ms
real 1m13.156s
user 9m1.328s
sys. 0m7.103s
```
## Implementation details
The high level implementation of the model is contained in the [main.cpp](main.cpp) file. The core computations are
performed by the [ggml](https://github.com/ggerganov/ggml/blob/master/include/ggml.h) library.
#### Matrix multiplication
The most performance critical part of the implementation is of course the matrix multiplication routine. 99% of the time
is spent here, so it was important to optimize this as much as possible.
On Arm64, I utilize the 128-bit NEON intrinsics for 16-bit floating point operations:
https://github.com/ggerganov/ggml/blob/fb558f78d905f85c54813602649ddd628ffe0f3a/src/ggml.c#L187-L243
These instructions allow each core to operate simultaneously on 64 16-bit floats. I'm no expert in SIMD, but after quite
some trials this was the most efficient code for dot product of a row and column that I could come up with. Combined
with the parallel computation on 8 CPU threads, I believe I'm close to the maximum performance that one could possibly
get on the M1 CPU. Still, I'm curious to know if there is a more efficient way to implement this.
#### Attempt to use the M1 GPU
One interesting property of the GPT-J transformer architecture is that it allows you to perform part of the inference in
parallel - i.e. the Feed-forward network can be computed in parallel to the Self-attention layer:
https://github.com/ggerganov/ggml/blob/fb558f78d905f85c54813602649ddd628ffe0f3a/examples/gpt-j/main.cpp#L507-L531
So I thought why not try and bring in the M1 GPU to compute half of the neural network in parallel to the CPU and
potentially gain some extra performance. Thanks to the M1's shared memory model, it was relatively easy to offload part
of the computation to the GPU using Apple's [Metal Performance
Shaders](https://developer.apple.com/documentation/metalperformanceshaders). The GPU shares the host memory, so there is
no need to copy the data back and forth as you would normally do with Cuda or OpenCL. The weight matrices are directly
available to be used by the GPU.
However, to my surprise, using MPS together with the CPU did not lead to any performance improvement at all. My
conclusion was that the 8-thread NEON CPU computation is already saturating the memory bandwidth of the M1 and since
the CPU and the GPU on the MacBook are sharing that bandwidth, it does not help to offload the computation to the GPU.
Another observation was that the MPS GPU matrix multiplication using 16-bit floats had the same performance as the
8-thread NEON CPU implementation. Again, I explain this with a saturated memory channel. But of course, my explanation
could be totally wrong and somehow the implementation wasn't utilizing the resources correctly.
In the end, I decided to not use MPS or the GPU all together.
### Zero memory allocations
Another property of my implementation is that it does not perform any memory allocations once the model is loaded into
memory. All required memory is allocated at the start of the program with a single `malloc` (technically 2 calls, but
that is not important).
## Usage
If you want to give this a try and you are on Linux or Mac OS, simply follow these instructions:
```bash
# Download the ggml-compatible GPT-J 6B model (requires 12GB disk space)
../examples/gpt-j/download-ggml-model.sh 6B
# Run the inference (requires 16GB of CPU RAM)
./bin/gpt-j -m models/gpt-j-6B/ggml-model.bin -p "This is an example"
# Input prompt through pipe and run the inference.
echo "This is an example" > prompt.txt
cat prompt.txt | ./bin/gpt-j -m models/gpt-j-6B/ggml-model.bin
```
To run the `gpt-j` tool, you need the 12GB `ggml-model.bin` file which contains the GPT-J model in
[ggml](https://github.com/ggerganov/ggml) compatible format. In the instructions above, the binary file
is downloaded from my repository on Hugging Face using the [download-ggml-model.sh](download-ggml-model.sh) script.
You can also, download the file manually from this link:
https://huggingface.co/ggerganov/ggml/tree/main
---
Alternatively, if you don't want to download the 12GB ggml model file, you can perform the conversion yourself using
python.
First, you need to download the full GPT-J model from here: https://huggingface.co/EleutherAI/gpt-j-6B
Note that the full model is quite big - about 72 GB. After you download it, you need to convert it to ggml format using
the [convert-h5-to-ggml.py](convert-h5-to-ggml.py) script. This will generate the `ggml-model.bin` file, which you can
then use with the `gpt-j` program.
## GPT-2
I also implemented a tool for CPU inference using the smaller GPT-2 models. They have worse quality compared to GPT-J,
but are much faster to execute.
For example, the Small GPT-2 model is only 240 MB big and the inference speed on my MacBook is about 200 tokens/sec.
For more details, checkout the GPT-2 example here: [gpt-2](https://github.com/ggerganov/ggml/tree/master/examples/gpt-2)
ggml/examples/gpt-j/convert-h5-to-ggml.py
+173
View File
@@ -0,0 +1,173 @@
# Convert GPT-J-6B h5 transformer model to ggml format
#
# Load the model using GPTJForCausalLM.
# Iterate over all variables and write them to a binary file.
#
# For each variable, write the following:
# - Number of dimensions (int)
# - Name length (int)
# - Dimensions (int[n_dims])
# - Name (char[name_length])
# - Data (float[n_dims])
#
# By default, the bigger matrices are converted to 16-bit floats.
# This can be disabled by adding the "use-f32" CLI argument.
#
# At the start of the ggml file we write the model parameters
# and vocabulary.
#
import sys
import struct
import json
import torch
import numpy as np
from transformers import GPTJForCausalLM
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
if len(sys.argv) < 3:
print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
print(" ftype == 0 -> float32")
print(" ftype == 1 -> float16")
sys.exit(1)
# output in the same directory as the model
dir_model = sys.argv[1]
fname_out = sys.argv[1] + "/ggml-model.bin"
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
encoder = json.load(f)
with open(dir_model + "/added_tokens.json", "r", encoding="utf-8") as f:
encoder_added = json.load(f)
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
# possible data types
# ftype == 0 -> float32
# ftype == 1 -> float16
#
# map from ftype to string
ftype_str = ["f32", "f16"]
ftype = 1
if len(sys.argv) > 2:
ftype = int(sys.argv[2])
if ftype < 0 or ftype > 1:
print("Invalid ftype: " + str(ftype))
sys.exit(1)
fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
model = GPTJForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True)
#print (model)
list_vars = model.state_dict()
#print (list_vars)
fout = open(fname_out, "wb")
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
fout.write(struct.pack("i", hparams["vocab_size"]))
fout.write(struct.pack("i", hparams["n_positions"]))
fout.write(struct.pack("i", hparams["n_embd"]))
fout.write(struct.pack("i", hparams["n_head"]))
fout.write(struct.pack("i", hparams["n_layer"]))
fout.write(struct.pack("i", hparams["rotary_dim"]))
fout.write(struct.pack("i", ftype))
byte_encoder = bytes_to_unicode()
byte_decoder = {v:k for k, v in byte_encoder.items()}
fout.write(struct.pack("i", len(encoder) + len(encoder_added)))
for key in encoder:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for key in encoder_added:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for name in list_vars.keys():
data = list_vars[name].squeeze().numpy()
print("Processing variable: " + name + " with shape: ", data.shape)
# we don't need these
if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
print(" Skipping variable: " + name)
continue
n_dims = len(data.shape);
# ftype == 0 -> float32, ftype == 1 -> float16
ftype_cur = 0;
if ftype != 0:
if name[-7:] == ".weight" and n_dims == 2:
print(" Converting to float16")
data = data.astype(np.float16)
ftype_cur = 1
else:
print(" Converting to float32")
data = data.astype(np.float32)
ftype_cur = 0
else:
if data.dtype != np.float32:
print(" Converting to float32")
data = data.astype(np.float32)
ftype_cur = 0
# for efficiency - transpose these matrices:
# (note - with latest ggml this is no longer more efficient, so disabling it)
# "transformer.h.*.mlp.fc_in.weight"
# "transformer.h.*.attn.out_proj.weight"
# "transformer.h.*.attn.q_proj.weight"
# "transformer.h.*.attn.k_proj.weight"
# "transformer.h.*.attn.v_proj.weight"
#if name.endswith(".mlp.fc_in.weight") or \
# name.endswith(".attn.out_proj.weight") or \
# name.endswith(".attn.q_proj.weight") or \
# name.endswith(".attn.k_proj.weight") or \
# name.endswith(".attn.v_proj.weight"):
# print(" Transposing")
# data = data.transpose()
# header
str = name.encode('utf-8')
fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
for i in range(n_dims):
fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
fout.write(str);
# data
data.tofile(fout)
fout.close()
print("Done. Output file: " + fname_out)
print("")
ggml/examples/gpt-j/download-ggml-model.sh
Executable
+69
View File
@@ -0,0 +1,69 @@
#!/bin/bash
# This script downloads GPT-J model files that have already been converted to ggml format.
# This way you don't have to convert them yourself.
#
# If you want to download the original GPT-J model files, use the "download-model.sh" script instead.
#src="https://ggml.ggerganov.com"
#pfx="ggml-model-gpt-j"
src="https://huggingface.co/ggerganov/ggml"
pfx="resolve/main/ggml-model-gpt-j"
ggml_path=$(dirname $(realpath $0))
# GPT-J models
models=( "6B" )
# list available models
function list_models {
printf "\n"
printf " Available models:"
for model in "${models[@]}"; do
printf " $model"
done
printf "\n\n"
}
if [ "$#" -ne 1 ]; then
printf "Usage: $0 <model>\n"
list_models
exit 1
fi
model=$1
if [[ ! " ${models[@]} " =~ " ${model} " ]]; then
printf "Invalid model: $model\n"
list_models
exit 1
fi
# download ggml model
printf "Downloading ggml model $model ...\n"
mkdir -p models/gpt-j-$model
if [ -x "$(command -v wget)" ]; then
wget --quiet --show-progress -O models/gpt-j-$model/ggml-model.bin $src/$pfx-$model.bin
elif [ -x "$(command -v curl)" ]; then
curl -L --output models/gpt-j-$model/ggml-model.bin $src/$pfx-$model.bin
else
printf "Either wget or curl is required to download models.\n"
exit 1
fi
if [ $? -ne 0 ]; then
printf "Failed to download ggml model $model \n"
printf "Please try again later or download the original GPT-J model files and convert them yourself.\n"
exit 1
fi
printf "Done! Model '$model' saved in 'models/gpt-j-$model/ggml-model.bin'\n"
printf "You can now use it like this:\n\n"
printf " $ ./bin/gpt-j -m models/gpt-j-$model/ggml-model.bin -p \"This is an example\"\n"
printf "\n"
ggml/examples/gpt-j/download-model.sh
Executable
+11
View File
@@ -0,0 +1,11 @@
#!/bin/bash
printf "To obtain the GPT-J 6B model files, please visit: https://huggingface.co/EleutherAI/gpt-j-6B\n\n"
printf "The model is very big. For example, the reposirory above is 72GB in size.\n"
printf "If you are sure that you want to clone it, simply run the following command:\n\n"
printf " $ git clone https://huggingface.co/EleutherAI/gpt-j-6B models/gpt-j-6B\n\n"
printf "Alternatively, use the 'download-ggml-model.sh' script to download a 12GB ggml version of the model.\n"
printf "This version is enough to run inference using the ggml library.\n\n"
ggml/examples/gpt-j/main.cpp
+755
View File
@@ -0,0 +1,755 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
// default hparams (GPT-J 6B)
struct gptj_hparams {
int32_t n_vocab = 50400;
int32_t n_ctx = 2048;
int32_t n_embd = 4096;
int32_t n_head = 16;
int32_t n_layer = 28;
int32_t n_rot = 64;
int32_t ftype = 1;
float eps = 1e-5f;
};
struct gptj_layer {
// normalization
struct ggml_tensor * ln_1_g;
struct ggml_tensor * ln_1_b;
// attention
struct ggml_tensor * c_attn_q_proj_w;
struct ggml_tensor * c_attn_k_proj_w;
struct ggml_tensor * c_attn_v_proj_w;
struct ggml_tensor * c_attn_proj_w;
// ff
struct ggml_tensor * c_mlp_fc_w;
struct ggml_tensor * c_mlp_fc_b;
struct ggml_tensor * c_mlp_proj_w;
struct ggml_tensor * c_mlp_proj_b;
};
struct gptj_model {
gptj_hparams hparams;
// normalization
struct ggml_tensor * ln_f_g;
struct ggml_tensor * ln_f_b;
struct ggml_tensor * wte; // token embedding
struct ggml_tensor * lmh_g; // language model head
struct ggml_tensor * lmh_b; // language model bias
std::vector<gptj_layer> layers;
// key + value memory
struct ggml_tensor * memory_k;
struct ggml_tensor * memory_v;
//
struct ggml_context * ctx;
std::map<std::string, struct ggml_tensor *> tensors;
};
// load the model's weights from a file
bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {
printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
return false;
}
// verify magic
{
uint32_t magic;
fin.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
return false;
}
}
// load hparams
{
auto & hparams = model.hparams;
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fin.read((char *) &hparams.n_rot, sizeof(hparams.n_rot));
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: n_rot = %d\n", __func__, hparams.n_rot);
printf("%s: ftype = %d\n", __func__, hparams.ftype);
printf("%s: qntvr = %d\n", __func__, qntvr);
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
}
// load vocab
{
int32_t n_vocab = 0;
fin.read((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != model.hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
return false;
}
std::string word;
std::vector<char> buf(128);
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
fin.read((char *) &len, sizeof(len));
buf.resize(len);
fin.read((char *) buf.data(), len);
word.assign(buf.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
// in order to save memory and also to speed up the computation
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
if (wtype == GGML_TYPE_COUNT) {
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
__func__, fname.c_str(), model.hparams.ftype);
return false;
}
auto & ctx = model.ctx;
size_t ctx_size = 0;
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b
ctx_size += ggml_row_size(wtype, n_embd*n_vocab); // wte
ctx_size += ggml_row_size(wtype, n_embd*n_vocab); // lmh_g
ctx_size += ggml_row_size(GGML_TYPE_F32, n_vocab); // lmh_b
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_q_proj_w
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_k_proj_w
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_v_proj_w
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_mlp_proj_b
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F16, n_embd); // memory_k
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F16, n_embd); // memory_v
ctx_size += (5 + 10*n_layer)*512; // object overhead
printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
}
// create the ggml context
{
struct ggml_init_params params = {
/*.mem_size =*/ ctx_size,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ false,
};
model.ctx = ggml_init(params);
if (!model.ctx) {
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
return false;
}
}
// prepare memory for the weights
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_vocab = hparams.n_vocab;
model.layers.resize(n_layer);
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.lmh_g = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
model.lmh_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);
// map by name
model.tensors["transformer.wte.weight"] = model.wte;
model.tensors["transformer.ln_f.weight"] = model.ln_f_g;
model.tensors["transformer.ln_f.bias"] = model.ln_f_b;
model.tensors["lm_head.weight"] = model.lmh_g;
model.tensors["lm_head.bias"] = model.lmh_b;
for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
// map by name
model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"] = layer.ln_1_g;
model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"] = layer.ln_1_b;
model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"] = layer.c_attn_q_proj_w;
model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"] = layer.c_attn_k_proj_w;
model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"] = layer.c_attn_v_proj_w;
model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w;
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"] = layer.c_mlp_fc_w;
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"] = layer.c_mlp_fc_b;
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"] = layer.c_mlp_proj_w;
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"] = layer.c_mlp_proj_b;
}
}
// key + value memory
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_mem = n_layer*n_ctx;
const int n_elements = n_embd*n_mem;
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, n_elements);
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, n_elements);
const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
printf("%s: memory_size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
}
// load weights
{
int n_tensors = 0;
size_t total_size = 0;
printf("%s: ", __func__);
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (fin.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[2] = { 1, 1 };
for (int i = 0; i < n_dims; ++i) {
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
fin.read(&name[0], length);
if (model.tensors.find(name) == model.tensors.end()) {
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
return false;
}
auto tensor = model.tensors[name];
if (ggml_nelements(tensor) != nelements) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
return false;
}
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
return false;
}
// for debugging
if (0) {
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
}
const size_t bpe = ggml_type_size(ggml_type(ttype));
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
return false;
}
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
//printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.c_str(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
total_size += ggml_nbytes(tensor);
if (++n_tensors % 8 == 0) {
printf(".");
fflush(stdout);
}
}
printf(" done\n");
printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
}
fin.close();
return true;
}
// evaluate the transformer
//
// - model: the model
// - n_threads: number of threads to use
// - n_past: the context size so far
// - embd_inp: the embeddings of the tokens in the context
// - embd_w: the predicted logits for the next token
//
// The GPT-J model requires about 16MB of memory per input token.
//
bool gptj_eval(
const gptj_model & model,
const int n_threads,
const int n_past,
const std::vector<gpt_vocab::id> & embd_inp,
std::vector<float> & embd_w,
size_t & mem_per_token) {
const int N = embd_inp.size();
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_head = hparams.n_head;
const int n_vocab = hparams.n_vocab;
const int n_rot = hparams.n_rot;
static size_t buf_size = 256u*1024*1024;
static void * buf = malloc(buf_size);
if (mem_per_token > 0 && mem_per_token*N > buf_size) {
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
// reallocate
buf_size = buf_size_new;
buf = realloc(buf, buf_size);
if (buf == nullptr) {
fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size);
return false;
}
}
struct ggml_init_params params = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ buf,
/*.no_alloc =*/ false,
};
struct ggml_context * ctx0 = ggml_init(params);
struct ggml_cgraph * gf = ggml_new_graph(ctx0);
// KQ_pos - contains the positions
struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
int * data = (int *) KQ_pos->data;
for (int i = 0; i < N; ++i) {
data[i] = n_past + i;
}
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
// wte
struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
for (int il = 0; il < n_layer; ++il) {
struct ggml_tensor * cur;
// norm
{
cur = ggml_norm(ctx0, inpL, hparams.eps);
// cur = ln_1_g*cur + ln_1_b
cur = ggml_add(ctx0,
ggml_mul(ctx0,
ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),
cur),
ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));
}
struct ggml_tensor * inpSA = cur;
// self-attention
{
struct ggml_tensor * Qcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N), KQ_pos, n_rot, 0);
struct ggml_tensor * Kcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N), KQ_pos, n_rot, 0);
// store key and value to memory
{
struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));
struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
struct ggml_tensor * v = ggml_view_2d(ctx0, model.memory_v, N, n_embd,
( n_ctx)*ggml_element_size(model.memory_v),
(il*n_ctx)*ggml_element_size(model.memory_v)*n_embd + n_past*ggml_element_size(model.memory_v));
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
}
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
struct ggml_tensor * Q =
ggml_permute(ctx0,
Qcur,
0, 2, 1, 3);
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
struct ggml_tensor * K =
ggml_permute(ctx0,
ggml_reshape_3d(ctx0,
ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
n_embd/n_head, n_head, n_past + N),
0, 2, 1, 3);
// K * Q
struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
// KQ_scaled = KQ / sqrt(n_embd/n_head)
struct ggml_tensor * KQ_scaled =
ggml_scale_inplace(ctx0,
KQ,
1.0f/sqrt(float(n_embd)/n_head));
// KQ_masked = mask_past(KQ_scaled)
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
// KQ = soft_max(KQ_masked)
struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
struct ggml_tensor * V =
ggml_view_3d(ctx0, model.memory_v,
n_past + N, n_embd/n_head, n_head,
n_ctx*ggml_element_size(model.memory_v),
n_ctx*ggml_element_size(model.memory_v)*n_embd/n_head,
il*n_ctx*ggml_element_size(model.memory_v)*n_embd);
// KQV = transpose(V) * KQ_soft_max
struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
// KQV_merged = KQV.permute(0, 2, 1, 3)
struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
// cur = KQV_merged.contiguous().view(n_embd, N)
cur = ggml_cpy(ctx0,
KQV_merged,
ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
// projection (no bias)
cur = ggml_mul_mat(ctx0,
model.layers[il].c_attn_proj_w,
cur);
}
struct ggml_tensor * inpFF = cur;
// feed-forward network
// this is independent of the self-attention result, so it could be done in parallel to the self-attention
{
// note here we pass inpSA instead of cur
cur = ggml_mul_mat(ctx0,
model.layers[il].c_mlp_fc_w,
inpSA);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
cur);
// GELU activation
cur = ggml_gelu(ctx0, cur);
// projection
// cur = proj_w*cur + proj_b
cur = ggml_mul_mat(ctx0,
model.layers[il].c_mlp_proj_w,
cur);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),
cur);
}
// self-attention + FF
cur = ggml_add(ctx0, cur, inpFF);
// input for next layer
inpL = ggml_add(ctx0, cur, inpL);
}
// norm
{
inpL = ggml_norm(ctx0, inpL, hparams.eps);
// inpL = ln_f_g*inpL + ln_f_b
inpL = ggml_add(ctx0,
ggml_mul(ctx0,
ggml_repeat(ctx0, model.ln_f_g, inpL),
inpL),
ggml_repeat(ctx0, model.ln_f_b, inpL));
}
// lm_head
{
inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);
inpL = ggml_add(ctx0,
ggml_repeat(ctx0, model.lmh_b, inpL),
inpL);
}
// logits -> probs
//inpL = ggml_soft_max_inplace(ctx0, inpL);
// run the computation
ggml_build_forward_expand(gf, inpL);
ggml_graph_compute_with_ctx(ctx0, gf, n_threads);
//if (n_past%100 == 0) {
// ggml_graph_print (&gf);
// ggml_graph_dump_dot(&gf, NULL, "gpt-j.dot");
//}
//embd_w.resize(n_vocab*N);
//memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
// return result for just the last token
embd_w.resize(n_vocab);
memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
if (mem_per_token == 0) {
mem_per_token = ggml_used_mem(ctx0)/N;
}
//printf("used_mem = %zu\n", ggml_used_mem(ctx0));
ggml_free(ctx0);
return true;
}
int main(int argc, char ** argv) {
ggml_time_init();
const int64_t t_main_start_us = ggml_time_us();
gpt_params params;
params.model = "models/gpt-j-6B/ggml-model.bin";
if (gpt_params_parse(argc, argv, params) == false) {
return 1;
}
if (params.seed < 0) {
params.seed = time(NULL);
}
printf("%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed);
if (params.prompt.empty()) {
params.prompt = gpt_random_prompt(rng);
}
int64_t t_load_us = 0;
gpt_vocab vocab;
gptj_model model;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gptj_model_load(params.model, model, vocab)) {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
return 1;
}
t_load_us = ggml_time_us() - t_start_us;
test_gpt_tokenizer(vocab, params.token_test);
}
int n_past = 0;
int64_t t_sample_us = 0;
int64_t t_predict_us = 0;
std::vector<float> logits;
// tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
printf("%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
printf("\n");
std::vector<gpt_vocab::id> embd;
// determine the required inference memory per token:
size_t mem_per_token = 0;
gptj_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
// predict
if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us();
if (!gptj_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {
printf("Failed to predict\n");
return 1;
}
t_predict_us += ggml_time_us() - t_start_us;
}
n_past += embd.size();
embd.clear();
if (i >= embd_inp.size()) {
// sample next token
const int top_k = params.top_k;
const float top_p = params.top_p;
const float temp = params.temp;
const int n_vocab = model.hparams.n_vocab;
gpt_vocab::id id = 0;
{
const int64_t t_start_sample_us = ggml_time_us();
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
t_sample_us += ggml_time_us() - t_start_sample_us;
}
// add it to the context
embd.push_back(id);
} else {
// if here, it means we are still processing the input prompt
for (size_t k = i; k < embd_inp.size(); k++) {
embd.push_back(embd_inp[k]);
if (int32_t(embd.size()) > params.n_batch) {
break;
}
}
i += embd.size() - 1;
}
// display text
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
// end of text token
if (embd.back() == 50256) {
break;
}
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n\n");
printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token);
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
ggml_free(model.ctx);
return 0;
}
ggml/examples/gpt-j/quantize.cpp
+182
View File
@@ -0,0 +1,182 @@
#include "ggml.h"
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#include <regex>
// default hparams (GPT-J 6B)
struct gptj_hparams {
int32_t n_vocab = 50400;
int32_t n_ctx = 2048;
int32_t n_embd = 4096;
int32_t n_head = 16;
int32_t n_layer = 28;
int32_t n_rot = 64;
int32_t ftype = 1;
};
// quantize a model
bool gptj_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {
gpt_vocab vocab;
printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str());
auto finp = std::ifstream(fname_inp, std::ios::binary);
if (!finp) {
fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str());
return false;
}
auto fout = std::ofstream(fname_out, std::ios::binary);
if (!fout) {
fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str());
return false;
}
// verify magic
{
uint32_t magic;
finp.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname_inp.c_str());
return false;
}
fout.write((char *) &magic, sizeof(magic));
}
gptj_hparams hparams;
// load hparams
{
finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
finp.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
finp.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
finp.read((char *) &hparams.n_head, sizeof(hparams.n_head));
finp.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
finp.read((char *) &hparams.n_rot, sizeof(hparams.n_rot));
finp.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR;
const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: ftype (src) = %d\n", __func__, hparams.ftype);
printf("%s: qntvr (src) = %d\n", __func__, qntvr_src);
printf("%s: ftype (dst) = %d\n", __func__, ftype_dst);
printf("%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION);
fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fout.write((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fout.write((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fout.write((char *) &hparams.n_head, sizeof(hparams.n_head));
fout.write((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fout.write((char *) &hparams.n_rot, sizeof(hparams.n_rot));
fout.write((char *) &ftype_dst, sizeof(ftype_dst));
}
// load vocab
{
int32_t n_vocab = 0;
finp.read ((char *) &n_vocab, sizeof(n_vocab));
fout.write((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname_inp.c_str(), n_vocab, hparams.n_vocab);
return false;
}
std::string word;
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
finp.read ((char *) &len, sizeof(len));
fout.write((char *) &len, sizeof(len));
word.resize(len);
finp.read ((char *) word.data(), len);
fout.write((char *) word.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// regexes of tensor names to be quantized
const std::vector<std::string> to_quant = {
".*weight",
};
if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, {})) {
fprintf(stderr, "%s: failed to quantize model '%s'\n", __func__, fname_inp.c_str());
return false;
}
finp.close();
fout.close();
return true;
}
// usage:
// ./gpt-2-quantize models/gpt-2-117M/ggml-model.bin models/gpt-2-117M/ggml-model-quant.bin type
//
int main(int argc, char ** argv) {
if (argc != 4) {
fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]);
ggml_print_ftypes(stderr);
return 1;
}
// needed to initialize f16 tables
{
struct ggml_init_params params = { 0, NULL, false };
struct ggml_context * ctx = ggml_init(params);
ggml_free(ctx);
}
const std::string fname_inp = argv[1];
const std::string fname_out = argv[2];
const ggml_ftype ftype = ggml_parse_ftype(argv[3]);
const int64_t t_main_start_us = ggml_time_us();
int64_t t_quantize_us = 0;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gptj_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) {
fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());
return 1;
}
t_quantize_us = ggml_time_us() - t_start_us;
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n");
printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0f);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
return 0;
}
ggml/examples/magika/CMakeLists.txt
+17
View File
@@ -0,0 +1,17 @@
#
# magika
set(TEST_TARGET magika)
add_executable(${TEST_TARGET} main.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
#
# For GPU offloading
if (GGML_CUDA)
add_compile_definitions(GGML_USE_CUDA)
endif()
if (GGML_METAL)
add_compile_definitions(GGML_USE_METAL)
endif()
ggml/examples/magika/README.md
+23
View File
@@ -0,0 +1,23 @@
# Google Magika inference
Simple example that shows how to use GGML for inference with the [Google Magika](https://github.com/google/magika) file type detection model.
### Usage
- Obtain the Magika model in H5 format
- Pinned version: https://github.com/google/magika/blob/4460acb5d3f86807c3b53223229dee2afa50c025/assets_generation/models/standard_v1/model.h5
- Use `convert.py` to convert the model to gguf format:
```bash
$ python examples/magika/convert.py /path/to/model.h5
```
- Invoke the program with the model file and a list of files to identify:
```bash
$ build/bin/magika model.h5.gguf examples/sam/example.jpg examples/magika/convert.py README.md src/ggml.c /bin/gcc write.exe jfk.wav
examples/sam/example.jpg : jpeg (100.00%) pptx (0.00%) smali (0.00%) shell (0.00%) sevenzip (0.00%)
examples/magika/convert.py : python (99.99%) javascript (0.00%) txt (0.00%) asm (0.00%) scala (0.00%)
README.md : markdown (100.00%) txt (0.00%) yaml (0.00%) ppt (0.00%) shell (0.00%)
src/ggml.c : c (99.95%) txt (0.04%) asm (0.01%) yaml (0.00%) html (0.00%)
/bin/gcc : elf (99.98%) odex (0.02%) pptx (0.00%) smali (0.00%) shell (0.00%)
write.exe : pebin (100.00%) ppt (0.00%) smali (0.00%) shell (0.00%) sevenzip (0.00%)
jfk.wav : wav (100.00%) ppt (0.00%) shell (0.00%) sevenzip (0.00%) scala (0.00%)
```
ggml/examples/magika/convert.py
+32
View File
@@ -0,0 +1,32 @@
import sys
from tensorflow import keras
import gguf
def convert(model_name):
model = keras.models.load_model(model_name, compile=False)
gguf_model_name = model_name + ".gguf"
gguf_writer = gguf.GGUFWriter(gguf_model_name, "magika")
for layer in model.layers:
# export layers with weights
if layer.weights:
for weight in layer.weights:
print(f" [{weight.name}] {weight.shape} {weight.dtype}")
weight_data = weight.numpy()
gguf_writer.add_tensor(weight.name, weight_data.T)
gguf_writer.write_header_to_file()
gguf_writer.write_kv_data_to_file()
gguf_writer.write_tensors_to_file()
gguf_writer.close()
print("Model converted and saved to '{}'".format(gguf_model_name))
if __name__ == '__main__':
if len(sys.argv) > 1:
model_file = sys.argv[1]
else:
model_file = "model.h5"
convert(model_file)
ggml/examples/magika/main.cpp
+374
View File
@@ -0,0 +1,374 @@
#include "ggml.h"
#include "gguf.h"
#include "ggml-cpu.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include <algorithm>
#include <cmath>
#include <numeric>
#include <stdexcept>
#include <string>
#include <vector>
static const char * magika_labels[] = {
"ai", "apk", "appleplist", "asm", "asp",
"batch", "bmp", "bzip", "c", "cab",
"cat", "chm", "coff", "crx", "cs",
"css", "csv", "deb", "dex", "dmg",
"doc", "docx", "elf", "emf", "eml",
"epub", "flac", "gif", "go", "gzip",
"hlp", "html", "ico", "ini", "internetshortcut",
"iso", "jar", "java", "javabytecode", "javascript",
"jpeg", "json", "latex", "lisp", "lnk",
"m3u", "macho", "makefile", "markdown", "mht",
"mp3", "mp4", "mscompress", "msi", "mum",
"odex", "odp", "ods", "odt", "ogg",
"outlook", "pcap", "pdf", "pebin", "pem",
"perl", "php", "png", "postscript", "powershell",
"ppt", "pptx", "python", "pythonbytecode", "rar",
"rdf", "rpm", "rst", "rtf", "ruby",
"rust", "scala", "sevenzip", "shell", "smali",
"sql", "squashfs", "svg", "swf", "symlinktext",
"tar", "tga", "tiff", "torrent", "ttf",
"txt", "unknown", "vba", "wav", "webm",
"webp", "winregistry", "wmf", "xar", "xls",
"xlsb", "xlsx", "xml", "xpi", "xz",
"yaml", "zip", "zlibstream"
};
struct magika_hparams {
const int block_size = 4096;
const int beg_size = 512;
const int mid_size = 512;
const int end_size = 512;
const int min_file_size_for_dl = 16;
const int n_label = 113;
const float f_norm_eps = 0.001f;
const int padding_token = 256;
};
struct magika_model {
~magika_model() {
ggml_backend_buffer_free(buf_w);
ggml_backend_free(backend);
ggml_free(ctx_w);
}
magika_hparams hparams;
struct ggml_tensor * dense_w;
struct ggml_tensor * dense_b;
struct ggml_tensor * layer_norm_gamma;
struct ggml_tensor * layer_norm_beta;
struct ggml_tensor * dense_1_w;
struct ggml_tensor * dense_1_b;
struct ggml_tensor * dense_2_w;
struct ggml_tensor * dense_2_b;
struct ggml_tensor * layer_norm_1_gamma;
struct ggml_tensor * layer_norm_1_beta;
struct ggml_tensor * target_label_w;
struct ggml_tensor * target_label_b;
ggml_backend_t backend = ggml_backend_cpu_init();
ggml_backend_buffer_t buf_w = nullptr;
struct ggml_context * ctx_w = nullptr;
};
struct ggml_tensor * checked_get_tensor(struct ggml_context * ctx, const char * name) {
struct ggml_tensor * tensor = ggml_get_tensor(ctx, name);
if (!tensor) {
fprintf(stderr, "%s: tensor '%s' not found\n", __func__, name);
throw std::runtime_error("ggml_get_tensor() failed");
}
return tensor;
}
bool magika_model_load(const std::string & fname, magika_model & model) {
auto & ctx = model.ctx_w;
struct gguf_init_params params = {
/*.no_alloc =*/ true,
/*.ctx =*/ &ctx,
};
struct gguf_context * ctx_gguf = gguf_init_from_file(fname.c_str(), params);
if (!ctx_gguf) {
fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
return false;
}
model.buf_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend);
if (!model.buf_w) {
fprintf(stderr, "%s: ggml_backend_alloc_ctx_tensors() failed\n", __func__);
gguf_free(ctx_gguf);
return false;
}
try {
model.dense_w = checked_get_tensor(ctx, "dense/kernel:0");
model.dense_b = checked_get_tensor(ctx, "dense/bias:0");
model.layer_norm_gamma = checked_get_tensor(ctx, "layer_normalization/gamma:0");
model.layer_norm_beta = checked_get_tensor(ctx, "layer_normalization/beta:0");
model.dense_1_w = checked_get_tensor(ctx, "dense_1/kernel:0");
model.dense_1_b = checked_get_tensor(ctx, "dense_1/bias:0");
model.dense_2_w = checked_get_tensor(ctx, "dense_2/kernel:0");
model.dense_2_b = checked_get_tensor(ctx, "dense_2/bias:0");
model.layer_norm_1_gamma = checked_get_tensor(ctx, "layer_normalization_1/gamma:0");
model.layer_norm_1_beta = checked_get_tensor(ctx, "layer_normalization_1/beta:0");
model.target_label_w = checked_get_tensor(ctx, "target_label/kernel:0");
model.target_label_b = checked_get_tensor(ctx, "target_label/bias:0");
} catch (const std::exception & e) {
fprintf(stderr, "%s: %s\n", __func__, e.what());
gguf_free(ctx_gguf);
return false;
}
FILE * f = fopen(fname.c_str(), "rb");
if (!f) {
fprintf(stderr, "%s: fopen() failed\n", __func__);
gguf_free(ctx_gguf);
return false;
}
const int n_tensors = gguf_get_n_tensors(ctx_gguf);
for (int i = 0; i < n_tensors; i++) {
const char * name = gguf_get_tensor_name(ctx_gguf, i);
struct ggml_tensor * tensor = ggml_get_tensor(ctx, name);
size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i);
//printf("%-30s: [%3ld, %3ld, %3ld, %3ld] %s\n",
// name,
// tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3],
// ggml_type_name(tensor->type));
std::vector<uint8_t> buf(ggml_nbytes(tensor));
if (fseek(f, offs, SEEK_SET) != 0) {
fprintf(stderr, "%s: fseek() failed\n", __func__);
gguf_free(ctx_gguf);
fclose(f);
return false;
}
if (fread(buf.data(), 1, buf.size(), f) != buf.size()) {
fprintf(stderr, "%s: fread() failed\n", __func__);
gguf_free(ctx_gguf);
fclose(f);
return false;
}
ggml_backend_tensor_set(tensor, buf.data(), 0, buf.size());
}
fclose(f);
gguf_free(ctx_gguf);
return true;
}
struct ggml_cgraph * magika_graph(
const magika_model & model,
const int n_files) {
const auto & hparams = model.hparams;
static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
static std::vector<uint8_t> buf(buf_size);
struct ggml_init_params params = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ buf.data(),
/*.no_alloc =*/ true,
};
struct ggml_context * ctx = ggml_init(params);
struct ggml_cgraph * gf = ggml_new_graph(ctx);
struct ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 257, 1536, n_files); // one-hot
ggml_set_name(input, "input");
ggml_set_input(input);
struct ggml_tensor * cur;
// dense
cur = ggml_mul_mat(ctx, model.dense_w, input);
cur = ggml_add(ctx, cur, model.dense_b); // [128, 1536, n_files]
cur = ggml_gelu(ctx, cur);
// reshape
cur = ggml_reshape_3d(ctx, cur, 512, 384, n_files); // [384, 512, n_files]
cur = ggml_cont(ctx, ggml_transpose(ctx, cur));
// layer normalization
cur = ggml_norm(ctx, cur, hparams.f_norm_eps);
cur = ggml_mul(ctx, cur, model.layer_norm_gamma); // [384, 512, n_files]
cur = ggml_add(ctx, cur, model.layer_norm_beta); // [384, 512, n_files]
// dense_1
cur = ggml_cont(ctx, ggml_transpose(ctx, cur));
cur = ggml_mul_mat(ctx, model.dense_1_w, cur);
cur = ggml_add(ctx, cur, model.dense_1_b); // [256, 384, n_files]
cur = ggml_gelu(ctx, cur);
// dense_2
cur = ggml_mul_mat(ctx, model.dense_2_w, cur);
cur = ggml_add(ctx, cur, model.dense_2_b); // [256, 384, n_files]
cur = ggml_gelu(ctx, cur);
// global_max_pooling1d
cur = ggml_cont(ctx, ggml_transpose(ctx, cur)); // [384, 256, n_files]
cur = ggml_pool_1d(ctx, cur, GGML_OP_POOL_MAX, 384, 384, 0); // [1, 256, n_files]
cur = ggml_reshape_2d(ctx, cur, 256, n_files); // [256, n_files]
// layer normalization 1
cur = ggml_norm(ctx, cur, hparams.f_norm_eps);
cur = ggml_mul(ctx, cur, model.layer_norm_1_gamma); // [256, n_files]
cur = ggml_add(ctx, cur, model.layer_norm_1_beta); // [256, n_files]
// target_label
cur = ggml_mul_mat(ctx, model.target_label_w, cur);
cur = ggml_add(ctx, cur, model.target_label_b); // [n_label, n_files]
cur = ggml_soft_max(ctx, cur); // [n_label, n_files]
ggml_set_name(cur, "target_label_probs");
ggml_set_output(cur);
ggml_build_forward_expand(gf, cur);
return gf;
}
bool magika_eval(
struct magika_model & model,
const std::vector<std::string> & fnames) {
const auto & hparams = model.hparams;
static ggml_gallocr_t alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
struct ggml_cgraph * gf = magika_graph(model, fnames.size());
if (!ggml_gallocr_alloc_graph(alloc, gf)) {
fprintf(stderr, "%s: ggml_gallocr_alloc_graph() failed\n", __func__);
return false;
}
struct ggml_tensor * input = ggml_graph_get_tensor(gf, "input");
for (size_t i = 0; i < fnames.size(); i++) {
FILE * f = fopen(fnames[i].c_str(), "rb");
if (!f) {
fprintf(stderr, "%s: fopen() failed\n", __func__);
return false;
}
fseek(f, 0, SEEK_END);
long fsize = ftell(f);
// the buffer is padded with the padding_token if the file is smaller than the block size
std::vector<int> buf(1536, hparams.padding_token);
std::vector<uint8_t> read_buf(std::max(hparams.beg_size, std::max(hparams.mid_size, hparams.end_size)));
// read beg
fseek(f, 0, SEEK_SET);
int n_read = fread(read_buf.data(), 1, hparams.beg_size, f);
for (int j = 0; j < n_read; j++) {
// pad at the end
buf[j] = read_buf[j];
}
// read mid
long mid_offs = std::max(0L, (fsize - hparams.mid_size) / 2);
fseek(f, mid_offs, SEEK_SET);
n_read = fread(read_buf.data(), 1, hparams.mid_size, f);
for (int j = 0; j < n_read; j++) {
// pad at both ends
long mid_idx = hparams.beg_size + (hparams.mid_size / 2) - n_read / 2 + j;
buf[mid_idx] = read_buf[j];
}
// read end
long end_offs = std::max(0L, fsize - hparams.end_size);
fseek(f, end_offs, SEEK_SET);
n_read = fread(read_buf.data(), 1, hparams.end_size, f);
for (int j = 0; j < n_read; j++) {
// pad at the beginning
int end_idx = hparams.beg_size + hparams.mid_size + hparams.end_size - n_read + j;
buf[end_idx] = read_buf[j];
}
fclose(f);
const size_t inp_bytes = hparams.beg_size + hparams.mid_size + hparams.end_size;
// convert to one-hot
std::vector<float> one_hot(257*inp_bytes);
for (size_t j = 0; j < inp_bytes; j++) {
one_hot[257*j + buf[j]] = 1.0f;
}
ggml_backend_tensor_set(input, one_hot.data(), 257*inp_bytes*i*sizeof(float), 257*inp_bytes*sizeof(float));
}
if (ggml_backend_graph_compute(model.backend, gf) != GGML_STATUS_SUCCESS) {
fprintf(stderr, "%s: ggml_backend_graph_compute() failed\n", __func__);
return false;
}
struct ggml_tensor * target_label_probs = ggml_graph_get_tensor(gf, "target_label_probs");
// print probabilities for the top labels of each file
for (size_t i = 0; i < fnames.size(); i++) {
std::vector<float> probs(hparams.n_label);
ggml_backend_tensor_get(target_label_probs, probs.data(), hparams.n_label*i*sizeof(float), hparams.n_label*sizeof(float));
// sort the probabilities
std::vector<int> idx(hparams.n_label);
std::iota(idx.begin(), idx.end(), 0);
std::sort(idx.begin(), idx.end(), [&probs](int i1, int i2) { return probs[i1] > probs[i2]; });
// print the top labels
const int top_n = 5;
printf("%-30s: ", fnames[i].c_str());
for (int j = 0; j < top_n; j++) {
printf("%s (%.2f%%) ", magika_labels[idx[j]], probs[idx[j]]*100);
}
printf("\n");
}
return true;
}
int main(int argc, const char ** argv) {
if (argc < 3) {
fprintf(stderr, "usage: %s <model> <file1> [<file2> ...]\n", argv[0]);
return 1;
}
const char * model_fname = argv[1];
std::vector<std::string> fnames;
for (int i = 2; i < argc; i++) {
fnames.push_back(argv[i]);
}
magika_model model;
if (!magika_model_load(model_fname, model)) {
fprintf(stderr, "magika_model_load() failed\n");
return 1;
}
magika_eval(model, fnames);
return 0;
}
ggml/examples/mnist/.gitignore
+3
View File
@@ -0,0 +1,3 @@
data/
*.gguf
*.ggml
ggml/examples/mnist/CMakeLists.txt
+58
View File
@@ -0,0 +1,58 @@
#
# mnist-common
set(TEST_TARGET mnist-common)
add_library(${TEST_TARGET} STATIC mnist-common.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common)
#
# mnist-eval
set(TEST_TARGET mnist-eval)
add_executable(${TEST_TARGET} mnist-eval.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common)
#
# mnist-train
set(TEST_TARGET mnist-train)
add_executable(${TEST_TARGET} mnist-train.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common)
#
# mnist-wasm
if (EMSCRIPTEN)
set(TARGET mnist)
add_executable(${TARGET} mnist-common.cpp)
target_link_libraries(${TARGET} PRIVATE ggml ggml-cpu)
set_target_properties(${TARGET} PROPERTIES LINK_FLAGS " \
--bind \
-s FORCE_FILESYSTEM=1 \
-s USE_PTHREADS=1 \
-s PTHREAD_POOL_SIZE=10 \
-s ASSERTIONS=1 \
-s WASM=1 \
-s EXPORTED_RUNTIME_METHODS=\"['ccall', 'cwrap', 'setValue', 'getValue']\" \
-s EXPORTED_FUNCTIONS=\"['_wasm_eval','_wasm_random_digit','_malloc','_free']\" \
-s ALLOW_MEMORY_GROWTH=1 \
--preload-file ${CMAKE_CURRENT_SOURCE_DIR}/mnist-f32.gguf@/ \
--preload-file ${CMAKE_CURRENT_SOURCE_DIR}/t10k-images-idx3-ubyte@/ \
")
# Copy output to web directory
add_custom_command(
TARGET ${TARGET} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy
${CMAKE_BINARY_DIR}/bin/mnist.js
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.js
COMMAND ${CMAKE_COMMAND} -E copy
${CMAKE_BINARY_DIR}/bin/mnist.wasm
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.wasm
COMMAND ${CMAKE_COMMAND} -E copy
${CMAKE_BINARY_DIR}/bin/mnist.worker.js
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.worker.js
)
endif()
ggml/examples/mnist/README.md
+206
View File
@@ -0,0 +1,206 @@
# MNIST Examples for GGML
This directory contains simple examples of how to use GGML for training and inference using the [MNIST dataset](https://yann.lecun.com/exdb/mnist/).
All commands listed in this README assume the working directory to be `examples/mnist`.
Please note that training in GGML is a work-in-progress and not production ready.
## Obtaining the data
A description of the dataset can be found on [Yann LeCun's website](https://yann.lecun.com/exdb/mnist/).
While it is also in principle possible to download the dataset from this website these downloads are frequently throttled and
it is recommended to use [HuggingFace](https://huggingface.co/datasets/ylecun/mnist) instead.
The dataset will be downloaded automatically when running `mnist-train-fc.py`.
## Fully connected network
For our first example we will train a fully connected network.
To train a fully connected model in PyTorch and save it as a GGUF file, run:
```bash
$ python3 mnist-train-fc.py mnist-fc-f32.gguf
...
Test loss: 0.066377+-0.010468, Test accuracy: 97.94+-0.14%
Model tensors saved to mnist-fc-f32.gguf:
fc1.weight (500, 784)
fc1.bias (500,)
fc2.weight (10, 500)
fc2.bias (10,)
```
The training script includes an evaluation of the model on the test set.
To evaluate the model on the CPU using GGML, run:
```bash
$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
__________________________________####__________________
______________________________########__________________
__________________________##########____________________
______________________##############____________________
____________________######________####__________________
__________________________________####__________________
__________________________________####__________________
________________________________####____________________
______________________________####______________________
________________________##########______________________
______________________########__####____________________
________________________##__________##__________________
____________________________________##__________________
__________________________________##____________________
__________________________________##____________________
________________________________##______________________
____________________________####________________________
__________##____________######__________________________
__________##############________________________________
________________####____________________________________
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
ggml_cuda_init: found 1 CUDA devices:
Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
mnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend
mnist_model: unsupported operations will be executed on the following fallback backends (in order of priority):
mnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor)
mnist_model_init_from_file: loading model weights from 'mnist-fc-f32.gguf'
mnist_model_init_from_file: model arch is mnist-fc
mnist_model_init_from_file: successfully loaded weights from mnist-fc-f32.gguf
main: loaded model in 109.44 ms
mnist_model_eval: model evaluation on 10000 images took 76.92 ms, 7.69 us/image
main: predicted digit is 3
main: test_loss=0.066379+-0.009101
main: test_acc=97.94+-0.14%
```
In addition to the evaluation on the test set the GGML evaluation also prints a random image from the test set as well as the model prediction for said image.
To train a fully connected model on the CPU using GGML run:
``` bash
$ ../../build/bin/mnist-train mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte
```
It can then be evaluated with the same binary as above.
## Convolutional network
To train a convolutional network using TensorFlow run:
```bash
$ python3 mnist-train-cnn.py mnist-cnn-f32.gguf
...
Test loss: 0.047947
Test accuracy: 98.46%
GGUF model saved to 'mnist-cnn-f32.gguf'
```
The saved model can be evaluated on the CPU using the `mnist-eval` binary:
```bash
$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
______________________________________##________________
______________________________________##________________
______________________________________##________________
____________________________________##__________________
__________________________________####__________________
__________________________________##____________________
________________________________##______________________
______________________________##________________________
____________________________####________________________
____________________________##__________________________
__________________________##____________________________
________________________##______________________________
______________________##________________________________
____________________####________________________________
____________________##__________________________________
__________________##____________________________________
________________##______________________________________
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
ggml_cuda_init: found 1 CUDA devices:
Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
mnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend
mnist_model: unsupported operations will be executed on the following fallback backends (in order of priority):
mnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor)
mnist_model_init_from_file: loading model weights from 'mnist-cnn-f32.gguf'
mnist_model_init_from_file: model arch is mnist-cnn
mnist_model_init_from_file: successfully loaded weights from mnist-cnn-f32.gguf
main: loaded model in 91.99 ms
mnist_model_eval: model evaluation on 10000 images took 267.61 ms, 26.76 us/image
main: predicted digit is 1
main: test_loss=0.047955+-0.007029
main: test_acc=98.46+-0.12%
```
Like with the fully connected network the convolutional network can also be trained using GGML:
``` bash
$ ../../build/bin/mnist-train mnist-cnn mnist-cnn-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte
```
As always, the evaluation is done using `mnist-eval` and like with the fully connected network the GGML graph is exported to `mnist-cnn-f32.ggml`.
## Hardware Acceleration
Both the training and evaluation code is agnostic in terms of hardware as long as the corresponding GGML backend has implemented the necessary operations.
A specific backend can be selected by appending the above commands with a backend name.
The compute graphs then schedule the operations to preferentially use the specified backend.
Note that if a backend does not implement some of the necessary operations a CPU fallback is used instead which may result in bad performance.
## Web demo
The evaluation code can be compiled to WebAssembly using [Emscripten](https://emscripten.org/) (may need to re-login to update `$PATH` after installation).
First, copy the GGUF file of either of the trained models to `examples/mnist` and name it `mnist-f32.gguf`.
Copy the test set to `examples/mnist` and name it `t10k-images-idx3-ubyte`.
Symlinking these files will *not* work!
Compile the code like so:
```bash
$ cd ../../
$ mkdir -p build-em
$ emcmake cmake .. -DGGML_BUILD_EXAMPLES=ON \
-DCMAKE_C_FLAGS="-pthread -matomics -mbulk-memory" \
-DCMAKE_CXX_FLAGS="-pthread -matomics -mbulk-memory"
$ make mnist
```
The compilation output is copied into `examples/mnist/web`.
To run it, you need an HTTP server.
For example:
``` bash
$ python3 examples/mnist/server.py
Serving directory '/home/danbev/work/ai/ggml/examples/mnist/web' at http://localhost:8000
Application context root: http://localhost:8000/
```
The web demo can then be accessed via the link printed on the console.
Simply draw a digit on the canvas and the model will try to predict what it's supposed to be.
Alternatively, click the "Random" button to retrieve a random digit from the test set.
Be aware that like all neural networks the one we trained is susceptible to distributional shift:
if the numbers you draw look different than the ones in the training set
(e.g. because they're not centered) the model will perform comparatively worse.
An online demo can be accessed [here](https://mnist.ggerganov.com).
ggml/examples/mnist/mnist-common.cpp
+496
View File
@@ -0,0 +1,496 @@
#include "ggml.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "ggml-opt.h"
#include "mnist-common.h"
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <cstdint>
#include <fstream>
#include <random>
#include <string>
#include <utility>
bool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset) {
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "failed to open images file %s\n", fname.c_str());
return false;
}
fin.seekg(16);
uint8_t image[MNIST_NINPUT];
struct ggml_tensor * images = ggml_opt_dataset_data(dataset);
float * buf = ggml_get_data_f32(images);
GGML_ASSERT(images->ne[0] == MNIST_NINPUT);
for (int64_t iex = 0; iex < images->ne[1]; ++iex) {
fin.read((char *) image, sizeof(image));
for (int64_t i = 0; i < MNIST_NINPUT; ++i) {
buf[iex*MNIST_NINPUT + i] = image[i] / 255.0f; // Normalize to [0, 1]
}
}
return true;
}
void mnist_image_print(FILE * stream, ggml_opt_dataset_t dataset, const int iex) {
struct ggml_tensor * images = ggml_opt_dataset_data(dataset);
GGML_ASSERT(images->ne[0] == MNIST_NINPUT);
GGML_ASSERT(iex < images->ne[1]);
const float * image = ggml_get_data_f32(images) + iex*MNIST_NINPUT;
for (int64_t row = 0; row < MNIST_HW; row++) {
for (int64_t col = 0; col < MNIST_HW; col++) {
const int rgb = roundf(255.0f * image[row*MNIST_HW + col]);
#ifdef _WIN32
fprintf(stream, "%s", rgb >= 220 ? "##" : "__"); // Represented via text.
#else
fprintf(stream, "\033[48;2;%d;%d;%dm \033[0m", rgb, rgb, rgb); // Represented via colored blocks.
#endif // _WIN32
}
fprintf(stream, "\n");
}
}
bool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset) {
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "failed to open labels file %s\n", fname.c_str());
return 0;
}
fin.seekg(8);
uint8_t label;
struct ggml_tensor * labels = ggml_opt_dataset_labels(dataset);
float * buf = ggml_get_data_f32(labels);
GGML_ASSERT(labels->ne[0] == MNIST_NCLASSES);
for (int64_t iex = 0; iex < labels->ne[1]; ++iex) {
fin.read((char *) &label, sizeof(label));
for (int64_t i = 0; i < MNIST_NCLASSES; ++i) {
buf[iex*MNIST_NCLASSES + i] = i == label ? 1.0f : 0.0f;
}
}
return true;
}
// Temporary util function for loading data from GGUF to a backend != CPU until GGML itself provides this functionality:
bool load_from_gguf(const char * fname, struct ggml_context * ctx_ggml, struct gguf_context * ctx_gguf) {
FILE * f = ggml_fopen(fname, "rb");
if (!f) {
return false;
}
const size_t buf_size = 4*1024*1024;
void * buf = malloc(buf_size);
const int n_tensors = gguf_get_n_tensors(ctx_gguf);
for (int i = 0; i < n_tensors; i++) {
const char * name = gguf_get_tensor_name(ctx_gguf, i);
struct ggml_tensor * tensor = ggml_get_tensor(ctx_ggml, name);
if (!tensor) {
continue;
}
const size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i);
if (fseek(f, offs, SEEK_SET) != 0) {
fclose(f);
free(buf);
return false;
}
const size_t nbytes = ggml_nbytes(tensor);
for (size_t pos = 0; pos < nbytes; pos += buf_size) {
const size_t nbytes_cpy = buf_size < nbytes - pos ? buf_size : nbytes - pos;
if (fread(buf, 1, nbytes_cpy, f) != nbytes_cpy) {
fclose(f);
free(buf);
return false;
}
ggml_backend_tensor_set(tensor, buf, pos, nbytes_cpy);
}
}
fclose(f);
free(buf);
return true;
}
mnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical) {
mnist_model model(backend, nbatch_logical, nbatch_physical);
fprintf(stderr, "%s: loading model weights from '%s'\n", __func__, fname.c_str());
struct gguf_context * ctx;
{
struct gguf_init_params params = {
/*.no_alloc =*/ true,
/*.ctx =*/ &model.ctx_gguf,
};
ctx = gguf_init_from_file(fname.c_str(), params);
if (!ctx) {
fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
exit(1);
}
}
model.arch = gguf_get_val_str(ctx, gguf_find_key(ctx, "general.architecture"));
fprintf(stderr, "%s: model arch is %s\n", __func__, model.arch.c_str());
if (model.arch == "mnist-fc") {
model.fc1_weight = ggml_get_tensor(model.ctx_gguf, "fc1.weight");
GGML_ASSERT(model.fc1_weight->ne[0] == MNIST_NINPUT);
GGML_ASSERT(model.fc1_weight->ne[1] == MNIST_NHIDDEN);
GGML_ASSERT(model.fc1_weight->ne[2] == 1);
GGML_ASSERT(model.fc1_weight->ne[3] == 1);
model.fc1_bias = ggml_get_tensor(model.ctx_gguf, "fc1.bias");
GGML_ASSERT(model.fc1_bias->ne[0] == MNIST_NHIDDEN);
GGML_ASSERT(model.fc1_bias->ne[1] == 1);
GGML_ASSERT(model.fc1_bias->ne[2] == 1);
GGML_ASSERT(model.fc1_bias->ne[3] == 1);
model.fc2_weight = ggml_get_tensor(model.ctx_gguf, "fc2.weight");
GGML_ASSERT(model.fc2_weight->ne[0] == MNIST_NHIDDEN);
GGML_ASSERT(model.fc2_weight->ne[1] == MNIST_NCLASSES);
GGML_ASSERT(model.fc2_weight->ne[2] == 1);
GGML_ASSERT(model.fc2_weight->ne[3] == 1);
model.fc2_bias = ggml_get_tensor(model.ctx_gguf, "fc2.bias");
GGML_ASSERT(model.fc2_bias->ne[0] == MNIST_NCLASSES);
GGML_ASSERT(model.fc2_bias->ne[1] == 1);
GGML_ASSERT(model.fc2_bias->ne[2] == 1);
GGML_ASSERT(model.fc2_bias->ne[3] == 1);
} else if (model.arch == "mnist-cnn") {
model.conv1_kernel = ggml_get_tensor(model.ctx_gguf, "conv1.kernel");
GGML_ASSERT(model.conv1_kernel->type == GGML_TYPE_F32);
GGML_ASSERT(model.conv1_kernel->ne[0] == 3);
GGML_ASSERT(model.conv1_kernel->ne[1] == 3);
GGML_ASSERT(model.conv1_kernel->ne[2] == 1);
GGML_ASSERT(model.conv1_kernel->ne[3] == MNIST_CNN_NCB);
model.conv1_bias = ggml_get_tensor(model.ctx_gguf, "conv1.bias");
GGML_ASSERT(model.conv1_bias->type == GGML_TYPE_F32);
GGML_ASSERT(model.conv1_bias->ne[0] == 1);
GGML_ASSERT(model.conv1_bias->ne[1] == 1);
GGML_ASSERT(model.conv1_bias->ne[2] == MNIST_CNN_NCB);
GGML_ASSERT(model.conv1_bias->ne[3] == 1);
model.conv2_kernel = ggml_get_tensor(model.ctx_gguf, "conv2.kernel");
GGML_ASSERT(model.conv2_kernel->type == GGML_TYPE_F32);
GGML_ASSERT(model.conv2_kernel->ne[0] == 3);
GGML_ASSERT(model.conv2_kernel->ne[1] == 3);
GGML_ASSERT(model.conv2_kernel->ne[2] == MNIST_CNN_NCB);
GGML_ASSERT(model.conv2_kernel->ne[3] == MNIST_CNN_NCB*2);
model.conv2_bias = ggml_get_tensor(model.ctx_gguf, "conv2.bias");
GGML_ASSERT(model.conv2_bias->type == GGML_TYPE_F32);
GGML_ASSERT(model.conv2_bias->ne[0] == 1);
GGML_ASSERT(model.conv2_bias->ne[1] == 1);
GGML_ASSERT(model.conv2_bias->ne[2] == MNIST_CNN_NCB*2);
GGML_ASSERT(model.conv2_bias->ne[3] == 1);
model.dense_weight = ggml_get_tensor(model.ctx_gguf, "dense.weight");
GGML_ASSERT(model.dense_weight->type == GGML_TYPE_F32);
GGML_ASSERT(model.dense_weight->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2));
GGML_ASSERT(model.dense_weight->ne[1] == MNIST_NCLASSES);
GGML_ASSERT(model.dense_weight->ne[2] == 1);
GGML_ASSERT(model.dense_weight->ne[3] == 1);
model.dense_bias = ggml_get_tensor(model.ctx_gguf, "dense.bias");
GGML_ASSERT(model.dense_bias->type == GGML_TYPE_F32);
GGML_ASSERT(model.dense_bias->ne[0] == MNIST_NCLASSES);
GGML_ASSERT(model.dense_bias->ne[1] == 1);
GGML_ASSERT(model.dense_bias->ne[2] == 1);
GGML_ASSERT(model.dense_bias->ne[3] == 1);
} else {
fprintf(stderr, "%s: unknown model arch: %s\n", __func__, model.arch.c_str());
}
model.buf_gguf = ggml_backend_alloc_ctx_tensors(model.ctx_gguf, model.backends[0]);
if(!load_from_gguf(fname.c_str(), model.ctx_gguf, ctx)) {
fprintf(stderr, "%s: loading weights from %s failed\n", __func__, fname.c_str());
exit(1);
}
// The space in ctx_gguf exactly fits the model weights,
// the images (which also need to be statically allocated) need to be put in a different context.
model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, nbatch_physical);
ggml_set_name(model.images, "images");
ggml_set_input(model.images);
model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]);
fprintf(stderr, "%s: successfully loaded weights from %s\n", __func__, fname.c_str());
return model;
}
mnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical) {
mnist_model model(backend, nbatch_logical, nbatch_physical);
model.arch = arch;
std::random_device rd{};
std::mt19937 gen{rd()};
std::normal_distribution<float> nd{0.0f, 1e-2f};
std::vector<ggml_tensor *> init_tensors;
if (model.arch == "mnist-fc") {
fprintf(stderr, "%s: initializing random weights for a fully connected model\n", __func__);
model.fc1_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NHIDDEN);
model.fc1_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN);
model.fc2_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN, MNIST_NCLASSES);
model.fc2_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES);
ggml_set_name(model.fc1_weight, "fc1.weight");
ggml_set_name(model.fc1_bias, "fc1.bias");
ggml_set_name(model.fc2_weight, "fc2.weight");
ggml_set_name(model.fc2_bias, "fc2.bias");
init_tensors.push_back(model.fc1_weight);
init_tensors.push_back(model.fc1_bias);
init_tensors.push_back(model.fc2_weight);
init_tensors.push_back(model.fc2_bias);
} else if (model.arch == "mnist-cnn") {
model.conv1_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, 1, MNIST_CNN_NCB);
model.conv1_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB);
model.conv2_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, MNIST_CNN_NCB, MNIST_CNN_NCB*2);
model.conv2_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB*2);
model.dense_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), MNIST_NCLASSES);
model.dense_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES);
ggml_set_name(model.conv1_kernel, "conv1.kernel");
ggml_set_name(model.conv1_bias, "conv1.bias");
ggml_set_name(model.conv2_kernel, "conv2.kernel");
ggml_set_name(model.conv2_bias, "conv2.bias");
ggml_set_name(model.dense_weight, "dense.weight");
ggml_set_name(model.dense_bias, "dense.bias");
init_tensors.push_back(model.conv1_kernel);
init_tensors.push_back(model.conv1_bias);
init_tensors.push_back(model.conv2_kernel);
init_tensors.push_back(model.conv2_bias);
init_tensors.push_back(model.dense_weight);
init_tensors.push_back(model.dense_bias);
} else {
fprintf(stderr, "%s: unknown model arch: %s\n", __func__, model.arch.c_str());
}
model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NBATCH_PHYSICAL);
ggml_set_name(model.images, "images");
ggml_set_input(model.images);
model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]);
for (ggml_tensor * t : init_tensors) {
GGML_ASSERT(t->type == GGML_TYPE_F32);
const int64_t ne = ggml_nelements(t);
std::vector<float> tmp(ne);
for (int64_t i = 0; i < ne; ++i) {
tmp[i] = nd(gen);
}
ggml_backend_tensor_set(t, tmp.data(), 0, ggml_nbytes(t));
}
return model;
}
void mnist_model_build(mnist_model & model) {
if (model.arch == "mnist-fc") {
ggml_set_param(model.fc1_weight);
ggml_set_param(model.fc1_bias);
ggml_set_param(model.fc2_weight);
ggml_set_param(model.fc2_bias);
ggml_tensor * fc1 = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
ggml_mul_mat(model.ctx_compute, model.fc1_weight, model.images),
model.fc1_bias));
model.logits = ggml_add(model.ctx_compute,
ggml_mul_mat(model.ctx_compute, model.fc2_weight, fc1),
model.fc2_bias);
} else if (model.arch == "mnist-cnn") {
ggml_set_param(model.conv1_kernel);
ggml_set_param(model.conv1_bias);
ggml_set_param(model.conv2_kernel);
ggml_set_param(model.conv2_bias);
ggml_set_param(model.dense_weight);
ggml_set_param(model.dense_bias);
struct ggml_tensor * images_2D = ggml_reshape_4d(model.ctx_compute, model.images, MNIST_HW, MNIST_HW, 1, model.images->ne[1]);
struct ggml_tensor * conv1_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
ggml_conv_2d(model.ctx_compute, model.conv1_kernel, images_2D, 1, 1, 1, 1, 1, 1),
model.conv1_bias));
GGML_ASSERT(conv1_out->ne[0] == MNIST_HW);
GGML_ASSERT(conv1_out->ne[1] == MNIST_HW);
GGML_ASSERT(conv1_out->ne[2] == MNIST_CNN_NCB);
GGML_ASSERT(conv1_out->ne[3] == model.nbatch_physical);
struct ggml_tensor * conv2_in = ggml_pool_2d(model.ctx_compute, conv1_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);
GGML_ASSERT(conv2_in->ne[0] == MNIST_HW/2);
GGML_ASSERT(conv2_in->ne[1] == MNIST_HW/2);
GGML_ASSERT(conv2_in->ne[2] == MNIST_CNN_NCB);
GGML_ASSERT(conv2_in->ne[3] == model.nbatch_physical);
struct ggml_tensor * conv2_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
ggml_conv_2d(model.ctx_compute, model.conv2_kernel, conv2_in, 1, 1, 1, 1, 1, 1),
model.conv2_bias));
GGML_ASSERT(conv2_out->ne[0] == MNIST_HW/2);
GGML_ASSERT(conv2_out->ne[1] == MNIST_HW/2);
GGML_ASSERT(conv2_out->ne[2] == MNIST_CNN_NCB*2);
GGML_ASSERT(conv2_out->ne[3] == model.nbatch_physical);
struct ggml_tensor * dense_in = ggml_pool_2d(model.ctx_compute, conv2_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);
GGML_ASSERT(dense_in->ne[0] == MNIST_HW/4);
GGML_ASSERT(dense_in->ne[1] == MNIST_HW/4);
GGML_ASSERT(dense_in->ne[2] == MNIST_CNN_NCB*2);
GGML_ASSERT(dense_in->ne[3] == model.nbatch_physical);
dense_in = ggml_reshape_2d(model.ctx_compute,
ggml_cont(model.ctx_compute, ggml_permute(model.ctx_compute, dense_in, 1, 2, 0, 3)),
(MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), model.nbatch_physical);
GGML_ASSERT(dense_in->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2));
GGML_ASSERT(dense_in->ne[1] == model.nbatch_physical);
GGML_ASSERT(dense_in->ne[2] == 1);
GGML_ASSERT(dense_in->ne[3] == 1);
model.logits = ggml_add(model.ctx_compute, ggml_mul_mat(model.ctx_compute, model.dense_weight, dense_in), model.dense_bias);
} else {
GGML_ASSERT(false);
}
ggml_set_name(model.logits, "logits");
ggml_set_output(model.logits);
GGML_ASSERT(model.logits->type == GGML_TYPE_F32);
GGML_ASSERT(model.logits->ne[0] == MNIST_NCLASSES);
GGML_ASSERT(model.logits->ne[1] == model.nbatch_physical);
GGML_ASSERT(model.logits->ne[2] == 1);
GGML_ASSERT(model.logits->ne[3] == 1);
}
ggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset) {
ggml_opt_result_t result = ggml_opt_result_init();
ggml_opt_params params = ggml_opt_default_params(model.backend_sched, GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);
params.ctx_compute = model.ctx_compute;
params.inputs = model.images;
params.outputs = model.logits;
params.build_type = GGML_OPT_BUILD_TYPE_FORWARD;
ggml_opt_context_t opt_ctx = ggml_opt_init(params);
{
const int64_t t_start_us = ggml_time_us();
ggml_opt_epoch(opt_ctx, dataset, nullptr, result, /*idata_split =*/ 0, nullptr, nullptr);
const int64_t t_total_us = ggml_time_us() - t_start_us;
const double t_total_ms = 1e-3*t_total_us;
const int nex = ggml_opt_dataset_data(dataset)->ne[1];
fprintf(stderr, "%s: model evaluation on %d images took %.2lf ms, %.2lf us/image\n",
__func__, nex, t_total_ms, (double) t_total_us/nex);
}
ggml_opt_free(opt_ctx);
return result;
}
void mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split) {
ggml_opt_fit(model.backend_sched, model.ctx_compute, model.images, model.logits, dataset,
GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, GGML_OPT_OPTIMIZER_TYPE_ADAMW, ggml_opt_get_default_optimizer_params, nepoch, model.nbatch_logical, val_split, false);
}
void mnist_model_save(mnist_model & model, const std::string & fname) {
printf("%s: saving model to '%s'\n", __func__, fname.c_str());
struct ggml_context * ggml_ctx;
{
struct ggml_init_params params = {
/*.mem_size =*/ 100 * 1024*1024,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ false,
};
ggml_ctx = ggml_init(params);
}
gguf_context * gguf_ctx = gguf_init_empty();
gguf_set_val_str(gguf_ctx, "general.architecture", model.arch.c_str());
std::vector<struct ggml_tensor *> weights;
if (model.arch == "mnist-fc") {
weights = {model.fc1_weight, model.fc1_bias, model.fc2_weight, model.fc2_bias};
} else if (model.arch == "mnist-cnn") {
weights = {model.conv1_kernel, model.conv1_bias, model.conv2_kernel, model.conv2_bias, model.dense_weight, model.dense_bias};
} else {
GGML_ASSERT(false);
}
for (struct ggml_tensor * t : weights) {
struct ggml_tensor * copy = ggml_dup_tensor(ggml_ctx, t);
ggml_set_name(copy, t->name);
ggml_backend_tensor_get(t, copy->data, 0, ggml_nbytes(t));
gguf_add_tensor(gguf_ctx, copy);
}
gguf_write_to_file(gguf_ctx, fname.c_str(), false);
ggml_free(ggml_ctx);
gguf_free(gguf_ctx);
}
#ifdef __cplusplus
extern "C" {
#endif
int wasm_eval(uint8_t * digitPtr) {
std::vector<float> digit(digitPtr, digitPtr + MNIST_NINPUT);
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, 1, 1);
struct ggml_tensor * data = ggml_opt_dataset_data(dataset);
float * buf = ggml_get_data_f32(data);
for (int i = 0; i < MNIST_NINPUT; ++i) {
buf[i] = digitPtr[i] / 255.0f;
}
ggml_set_zero(ggml_opt_dataset_labels(dataset)); // The labels are not needed.
mnist_model model = mnist_model_init_from_file("mnist-f32.gguf", "CPU", /*nbatch_logical =*/ 1, /*nbatch_physical =*/ 1);
mnist_model_build(model);
ggml_opt_result_t result = mnist_model_eval(model, dataset);
int32_t pred;
ggml_opt_result_pred(result, &pred);
return pred;
}
int wasm_random_digit(char * digitPtr) {
auto fin = std::ifstream("t10k-images-idx3-ubyte", std::ios::binary);
if (!fin) {
fprintf(stderr, "failed to open digits file\n");
return 0;
}
srand(time(NULL));
// Seek to a random digit: 16-byte header + 28*28 * (random 0 - 10000)
fin.seekg(16 + MNIST_NINPUT * (rand() % MNIST_NTEST));
fin.read(digitPtr, MNIST_NINPUT);
return 1;
}
#ifdef __cplusplus
}
#endif
ggml/examples/mnist/mnist-common.h
+166
View File
@@ -0,0 +1,166 @@
#include <algorithm>
#include <cstdint>
#include <random>
#include <string>
#include <thread>
#include <vector>
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "ggml.h"
#include "gguf.h"
#include "ggml-cpu.h"
#include "ggml-opt.h"
#define MNIST_NTRAIN 60000
#define MNIST_NTEST 10000
// Gradient accumulation can be achieved by setting the logical batch size to a multiple of the physical one.
// The logical batch size determines how many datapoints are used for a gradient update.
// The physical batch size determines how many datapoints are processed in parallel, larger values utilize compute better but need more memory.
#define MNIST_NBATCH_LOGICAL 1000
#define MNIST_NBATCH_PHYSICAL 500
static_assert(MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL == 0, "MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL != 0");
static_assert(MNIST_NTRAIN % MNIST_NBATCH_LOGICAL == 0, "MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0");
static_assert(MNIST_NTEST % MNIST_NBATCH_LOGICAL == 0, "MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0");
#define MNIST_HW 28
#define MNIST_NINPUT (MNIST_HW*MNIST_HW)
#define MNIST_NCLASSES 10
#define MNIST_NHIDDEN 500
// NCB = number of channels base
#define MNIST_CNN_NCB 8
struct mnist_model {
std::string arch;
ggml_backend_sched_t backend_sched;
std::vector<ggml_backend_t> backends;
const int nbatch_logical;
const int nbatch_physical;
struct ggml_tensor * images = nullptr;
struct ggml_tensor * logits = nullptr;
struct ggml_tensor * fc1_weight = nullptr;
struct ggml_tensor * fc1_bias = nullptr;
struct ggml_tensor * fc2_weight = nullptr;
struct ggml_tensor * fc2_bias = nullptr;
struct ggml_tensor * conv1_kernel = nullptr;
struct ggml_tensor * conv1_bias = nullptr;
struct ggml_tensor * conv2_kernel = nullptr;
struct ggml_tensor * conv2_bias = nullptr;
struct ggml_tensor * dense_weight = nullptr;
struct ggml_tensor * dense_bias = nullptr;
struct ggml_context * ctx_gguf = nullptr;
struct ggml_context * ctx_static = nullptr;
struct ggml_context * ctx_compute = nullptr;
ggml_backend_buffer_t buf_gguf = nullptr;
ggml_backend_buffer_t buf_static = nullptr;
mnist_model(const std::string & backend_name, const int nbatch_logical, const int nbatch_physical)
: nbatch_logical(nbatch_logical), nbatch_physical(nbatch_physical) {
std::vector<ggml_backend_dev_t> devices;
const int ncores_logical = std::thread::hardware_concurrency();
const int nthreads = std::min(ncores_logical, (ncores_logical + 4) / 2);
// Add primary backend:
if (!backend_name.empty()) {
ggml_backend_dev_t dev = ggml_backend_dev_by_name(backend_name.c_str());
if (dev == nullptr) {
fprintf(stderr, "%s: ERROR: backend %s not found, available:\n", __func__, backend_name.c_str());
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev_i = ggml_backend_dev_get(i);
fprintf(stderr, " - %s (%s)\n", ggml_backend_dev_name(dev_i), ggml_backend_dev_description(dev_i));
}
exit(1);
}
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
GGML_ASSERT(backend);
if (ggml_backend_is_cpu(backend)) {
ggml_backend_cpu_set_n_threads(backend, nthreads);
}
backends.push_back(backend);
devices.push_back(dev);
}
// Add all available backends as fallback.
// A "backend" is a stream on a physical device so there is no problem with adding multiple backends for the same device.
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
GGML_ASSERT(backend);
if (ggml_backend_is_cpu(backend)) {
ggml_backend_cpu_set_n_threads(backend, nthreads);
}
backends.push_back(backend);
devices.push_back(dev);
}
// The order of the backends passed to ggml_backend_sched_new determines which backend is given priority.
backend_sched = ggml_backend_sched_new(backends.data(), nullptr, backends.size(), GGML_DEFAULT_GRAPH_SIZE, false, true);
fprintf(stderr, "%s: using %s (%s) as primary backend\n",
__func__, ggml_backend_name(backends[0]), ggml_backend_dev_description(devices[0]));
if (backends.size() >= 2) {
fprintf(stderr, "%s: unsupported operations will be executed on the following fallback backends (in order of priority):\n", __func__);
for (size_t i = 1; i < backends.size(); ++i) {
fprintf(stderr, "%s: - %s (%s)\n", __func__, ggml_backend_name(backends[i]), ggml_backend_dev_description(devices[i]));
}
}
{
const size_t size_meta = 1024*ggml_tensor_overhead();
struct ggml_init_params params = {
/*.mem_size =*/ size_meta,
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ctx_static = ggml_init(params);
}
{
// The compute context needs a total of 3 compute graphs: forward pass + backwards pass (with/without optimizer step).
const size_t size_meta = GGML_DEFAULT_GRAPH_SIZE*ggml_tensor_overhead() + 3*ggml_graph_overhead();
struct ggml_init_params params = {
/*.mem_size =*/ size_meta,
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ctx_compute = ggml_init(params);
}
}
~mnist_model() {
ggml_free(ctx_gguf);
ggml_free(ctx_static);
ggml_free(ctx_compute);
ggml_backend_buffer_free(buf_gguf);
ggml_backend_buffer_free(buf_static);
ggml_backend_sched_free(backend_sched);
for (ggml_backend_t backend : backends) {
ggml_backend_free(backend);
}
}
};
bool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset);
void mnist_image_print(FILE * f, ggml_opt_dataset_t dataset, const int iex);
bool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset);
mnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical);
mnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical);
void mnist_model_build(mnist_model & model);
ggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset);
void mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split);
void mnist_model_save(mnist_model & model, const std::string & fname);
ggml/examples/mnist/mnist-eval.cpp
+67
View File
@@ -0,0 +1,67 @@
#include "ggml.h"
#include "ggml-opt.h"
#include "mnist-common.h"
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <ctime>
#include <string>
#include <thread>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
int main(int argc, char ** argv) {
srand(time(NULL));
ggml_time_init();
if (argc != 4 && argc != 5) {
fprintf(stderr, "Usage: %s mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]);
exit(1);
}
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTEST, MNIST_NBATCH_PHYSICAL);
if (!mnist_image_load(argv[2], dataset)) {
return 1;
}
if (!mnist_label_load(argv[3], dataset)) {
return 1;
}
const int iex = rand() % MNIST_NTEST;
mnist_image_print(stdout, dataset, iex);
const std::string backend = argc >= 5 ? argv[4] : "";
const int64_t t_start_us = ggml_time_us();
mnist_model model = mnist_model_init_from_file(argv[1], backend, MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);
mnist_model_build(model);
const int64_t t_load_us = ggml_time_us() - t_start_us;
fprintf(stdout, "%s: loaded model in %.2lf ms\n", __func__, t_load_us / 1000.0);
ggml_opt_result_t result_eval = mnist_model_eval(model, dataset);
std::vector<int32_t> pred(MNIST_NTEST);
ggml_opt_result_pred(result_eval, pred.data());
fprintf(stdout, "%s: predicted digit is %d\n", __func__, pred[iex]);
double loss;
double loss_unc;
ggml_opt_result_loss(result_eval, &loss, &loss_unc);
fprintf(stdout, "%s: test_loss=%.6lf+-%.6lf\n", __func__, loss, loss_unc);
double accuracy;
double accuracy_unc;
ggml_opt_result_accuracy(result_eval, &accuracy, &accuracy_unc);
fprintf(stdout, "%s: test_acc=%.2lf+-%.2lf%%\n", __func__, 100.0*accuracy, 100.0*accuracy_unc);
ggml_opt_result_free(result_eval);
return 0;
}
ggml/examples/mnist/mnist-train-cnn.py
Executable
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
import sys
from time import time
import gguf
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
def train(model_path):
# Model / data parameters
num_classes = 10
input_shape = (28, 28, 1)
# Load the data and split it between train and test sets
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
# Scale images to the [0, 1] range
x_train = x_train.astype("float32") / 255
x_test = x_test.astype("float32") / 255
x_train = np.expand_dims(x_train, -1)
x_test = np.expand_dims(x_test, -1)
print("x_train shape:", x_train.shape)
print(x_train.shape[0], "train samples")
print(x_test.shape[0], "test samples")
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = keras.Sequential(
[
keras.Input(shape=input_shape, dtype=tf.float32),
layers.Conv2D(8, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32),
layers.MaxPooling2D(pool_size=(2, 2)),
layers.Conv2D(16, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32),
layers.MaxPooling2D(pool_size=(2, 2)),
layers.Flatten(),
layers.Dense(num_classes, activation="softmax", dtype=tf.float32),
]
)
model.summary()
batch_size = 1000
epochs = 30
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
t_start = time()
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)
print(f"Training took {time()-t_start:.2f}s")
score = model.evaluate(x_test, y_test, verbose=0)
print(f"Test loss: {score[0]:.6f}")
print(f"Test accuracy: {100*score[1]:.2f}%")
gguf_writer = gguf.GGUFWriter(model_path, "mnist-cnn")
conv1_kernel = model.layers[0].weights[0].numpy()
conv1_kernel = np.moveaxis(conv1_kernel, [2, 3], [0, 1])
gguf_writer.add_tensor("conv1.kernel", conv1_kernel, raw_shape=(8, 1, 3, 3))
conv1_bias = model.layers[0].weights[1].numpy()
gguf_writer.add_tensor("conv1.bias", conv1_bias, raw_shape=(1, 8, 1, 1))
conv2_kernel = model.layers[2].weights[0].numpy()
conv2_kernel = np.moveaxis(conv2_kernel, [0, 1, 2, 3], [2, 3, 1, 0])
gguf_writer.add_tensor("conv2.kernel", conv2_kernel, raw_shape=(16, 8, 3, 3))
conv2_bias = model.layers[2].weights[1].numpy()
gguf_writer.add_tensor("conv2.bias", conv2_bias, raw_shape=(1, 16, 1, 1))
dense_weight = model.layers[-1].weights[0].numpy()
dense_weight = dense_weight.transpose()
gguf_writer.add_tensor("dense.weight", dense_weight, raw_shape=(10, 7*7*16))
dense_bias = model.layers[-1].weights[1].numpy()
gguf_writer.add_tensor("dense.bias", dense_bias)
gguf_writer.write_header_to_file()
gguf_writer.write_kv_data_to_file()
gguf_writer.write_tensors_to_file()
gguf_writer.close()
print(f"GGUF model saved to '{model_path}'")
if __name__ == '__main__':
if len(sys.argv) != 2:
print(f"Usage: {sys.argv[0]} <model_path>")
sys.exit(1)
train(sys.argv[1])
ggml/examples/mnist/mnist-train-fc.py
+131
View File
@@ -0,0 +1,131 @@
import gguf
import numpy as np
import torch
import torch.nn as nn
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.autograd import Variable
import sys
from time import time
input_size = 784 # img_size = (28,28) ---> 28*28=784 in total
hidden_size = 500 # number of nodes at hidden layer
num_classes = 10 # number of output classes discrete range [0,9]
num_epochs = 30 # number of times which the entire dataset is passed throughout the model
batch_size = 1000 # the size of input data used for one iteration
lr = 1e-3 # size of step
class Net(nn.Module):
def __init__(self, input_size, hidden_size, num_classes):
super(Net, self).__init__()
self.fc1 = nn.Linear(input_size, hidden_size)
self.relu = nn.ReLU()
self.fc2 = nn.Linear(hidden_size, num_classes)
def forward(self, x):
out = self.fc1(x)
out = self.relu(out)
out = self.fc2(out)
return out
def train(model_path):
train_data = dsets.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download=True)
test_data = dsets.MNIST(root='./data', train=False, transform=transforms.ToTensor())
assert len(train_data) == 60000
assert len(test_data) == 10000
kwargs_train_test = dict(batch_size=batch_size, num_workers=4, pin_memory=True)
train_gen = torch.utils.data.DataLoader(dataset=train_data, shuffle=True, **kwargs_train_test)
test_gen = torch.utils.data.DataLoader(dataset=test_data, shuffle=False, **kwargs_train_test)
net = Net(input_size, hidden_size, num_classes)
if torch.cuda.is_available():
net.cuda()
loss_function = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
t_start = time()
for epoch in range(num_epochs):
loss_history = []
ncorrect = 0
for i, (images, labels) in enumerate(train_gen):
images = Variable(images.view(-1, 28*28))
labels = Variable(labels)
if torch.cuda.is_available():
images = images.cuda()
labels = labels.cuda()
optimizer.zero_grad()
outputs = net(images)
loss = loss_function(outputs, labels)
loss_history.append(loss.cpu().data)
_, predictions = torch.max(outputs, 1)
ncorrect += (predictions == labels).sum()
loss.backward()
optimizer.step()
if (i + 1)*batch_size % 10000 == 0:
loss_mean = np.mean(loss_history)
accuracy = ncorrect / ((i + 1) * batch_size)
print(
f"Epoch [{epoch+1:02d}/{num_epochs}], "
f"Step [{(i+1)*batch_size:05d}/{len(train_data)}], "
f"Loss: {loss_mean:.4f}, Accuracy: {100*accuracy:.2f}%")
print()
print(f"Training took {time()-t_start:.2f}s")
loss_history = []
ncorrect = 0
for i, (images, labels) in enumerate(test_gen):
images = Variable(images.view(-1, 28*28))
labels = Variable(labels)
if torch.cuda.is_available():
images = images.cuda()
labels = labels.cuda()
outputs = net(images)
loss = loss_function(outputs, labels)
loss_history.append(loss.cpu().data)
_, predictions = torch.max(outputs, 1)
ncorrect += (predictions == labels).sum().cpu().numpy()
loss_mean = np.mean(loss_history)
loss_uncertainty = np.std(loss_history) / np.sqrt(len(loss_history) - 1)
accuracy_mean = ncorrect / (len(test_gen) * batch_size)
accuracy_uncertainty = np.sqrt(accuracy_mean * (1.0 - accuracy_mean) / (len(test_gen) * batch_size))
print()
print(f"Test loss: {loss_mean:.6f}+-{loss_uncertainty:.6f}, Test accuracy: {100*accuracy_mean:.2f}+-{100*accuracy_uncertainty:.2f}%")
gguf_writer = gguf.GGUFWriter(model_path, "mnist-fc")
print()
print(f"Model tensors saved to {model_path}:")
for tensor_name in net.state_dict().keys():
data = net.state_dict()[tensor_name].squeeze().cpu().numpy()
print(tensor_name, "\t", data.shape)
gguf_writer.add_tensor(tensor_name, data)
gguf_writer.write_header_to_file()
gguf_writer.write_kv_data_to_file()
gguf_writer.write_tensors_to_file()
gguf_writer.close()
if __name__ == '__main__':
if len(sys.argv) != 2:
print(f"Usage: {sys.argv[0]} <model_path>")
sys.exit(1)
train(sys.argv[1])
ggml/examples/mnist/mnist-train.cpp
+39
View File
@@ -0,0 +1,39 @@
#include "ggml-opt.h"
#include "mnist-common.h"
#include <cmath>
#include <cstdio>
#include <cstring>
#include <ctime>
#include <string>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
int main(int argc, char ** argv) {
if (argc != 5 && argc != 6) {
fprintf(stderr, "Usage: %s mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]);
exit(0);
}
// The MNIST model is so small that the overhead from data shuffling is non-negligible, especially with CUDA.
// With a shard size of 10 this overhead is greatly reduced at the cost of less shuffling (does not seem to have a significant impact).
// A batch of 500 images then consists of 50 random shards of size 10 instead of 500 random shards of size 1.
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTRAIN, /*ndata_shard =*/ 10);
if (!mnist_image_load(argv[3], dataset)) {
return 1;
}
if (!mnist_label_load(argv[4], dataset)) {
return 1;
}
mnist_model model = mnist_model_init_random(argv[1], argc >= 6 ? argv[5] : "", MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);
mnist_model_build(model);
mnist_model_train(model, dataset, /*nepoch =*/ 30, /*val_split =*/ 0.05f);
mnist_model_save(model, argv[2]);
}
ggml/examples/mnist/server.py
+36
View File
@@ -0,0 +1,36 @@
import http.server
import socketserver
import os
import sys
DIRECTORY = os.path.abspath(os.path.join(os.path.dirname(__file__), 'web'))
PORT = 8000
class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
def __init__(self, *args, **kwargs):
super().__init__(*args, directory=DIRECTORY, **kwargs)
def end_headers(self):
# Add required headers for SharedArrayBuffer
self.send_header("Cross-Origin-Opener-Policy", "same-origin")
self.send_header("Cross-Origin-Embedder-Policy", "require-corp")
self.send_header("Access-Control-Allow-Origin", "*")
super().end_headers()
# Enable address reuse
class CustomServer(socketserver.TCPServer):
allow_reuse_address = True
try:
with CustomServer(("", PORT), CustomHTTPRequestHandler) as httpd:
print(f"Serving directory '{DIRECTORY}' at http://localhost:{PORT}")
print(f"Application context root: http://localhost:{PORT}/")
try:
httpd.serve_forever()
except KeyboardInterrupt:
print("\nServer stopped.")
# Force complete exit
sys.exit(0)
except OSError as e:
print(f"Error: {e}")
sys.exit(1)
ggml/examples/perf-metal/CMakeLists.txt
+7
View File
@@ -0,0 +1,7 @@
#
# perf-metal
set(TEST_TARGET perf-metal)
add_executable(${TEST_TARGET} perf-metal.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
ggml/examples/perf-metal/perf-metal.cpp
+152
View File
@@ -0,0 +1,152 @@
// basic tool to experiment with the Metal backend
//
// 1. Get GPU trace of a dummy graph:
//
// rm -rf /tmp/perf-metal.gputrace
// make -j perf-metal && METAL_CAPTURE_ENABLED=1 ./bin/perf-metal
// open /tmp/perf-metal.gputrace
//
// https://github.com/ggerganov/llama.cpp/issues/9507
//
#include "ggml.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "ggml-metal.h"
#include <cstdio>
#include <vector>
#include <thread>
int main(int argc, char ** argv) {
int n_op = 1024;
int n_iter = 128;
if (argc > 1) {
n_op = std::atoi(argv[1]);
}
if (argc > 2) {
n_iter = std::atoi(argv[2]);
}
printf("%s: n_op = %d, n_iter = %d\n", __func__, n_op, n_iter);
const int ne00 = 8;
const int ne01 = 8;
const int ne11 = 8;
std::vector<float> data0(ne00*ne01, 1.0f);
std::vector<float> data1(ne00*ne01, 1.0f/ne00);
ggml_backend_t backend = ggml_backend_metal_init();
if (!backend) {
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
return 1;
}
const size_t ctx_size = 2 * ggml_tensor_overhead();
struct ggml_init_params params = {
/*.mem_size =*/ ctx_size,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
struct ggml_context * ctx = ggml_init(params);
struct ggml_tensor * t0 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ne00, ne01);
struct ggml_tensor * t1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ne00, ne11);
ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend);
ggml_backend_tensor_set(t0, data0.data(), 0, ggml_nbytes(t0));
ggml_backend_tensor_set(t1, data1.data(), 0, ggml_nbytes(t1));
struct ggml_cgraph * gf = NULL;
struct ggml_context * ctx_cgraph = NULL;
// create a dummy compute graph:
//
// x = mul_mat(t0, t1)
// x = x * 1.0f
// x = mul_mat(x, t1)
// x = x * 1.0f
// ... repeat n_op times ...
//
{
struct ggml_init_params params0 = {
/*.mem_size =*/ 4*n_op*ggml_tensor_overhead() + ggml_graph_overhead(),
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
ctx_cgraph = ggml_init(params0);
gf = ggml_new_graph_custom(ctx_cgraph, 4*n_op, false);
struct ggml_tensor * cur = ggml_mul_mat(ctx_cgraph, t0, t1);
cur = ggml_scale(ctx_cgraph, cur, 1.0f);
for (int i = 0; i < n_op - 1; i++) {
cur = ggml_mul_mat(ctx_cgraph, cur, t1);
cur = ggml_scale(ctx_cgraph, cur, 1.0f);
}
cur = ggml_scale(ctx_cgraph, cur, 42.0f);
ggml_build_forward_expand(gf, cur);
}
printf("%s: graph nodes = %d\n", __func__, ggml_graph_n_nodes(gf));
ggml_gallocr_t allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));
ggml_gallocr_alloc_graph(allocr, gf);
{
// warm-up
ggml_backend_graph_compute(backend, gf);
const int64_t t_start = ggml_time_us();
for (int iter = 0; iter < n_iter; iter++) {
ggml_backend_graph_compute(backend, gf);
}
const int64_t t_end = ggml_time_us();
// actual trace
ggml_backend_metal_capture_next_compute(backend);
ggml_backend_graph_compute(backend, gf);
//std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // NOTE: these intervals do not appear in the XCode trace!
ggml_backend_metal_capture_next_compute(backend);
ggml_backend_graph_compute(backend, gf);
//std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // NOTE: these intervals do not appear in the XCode trace!
ggml_backend_metal_capture_next_compute(backend);
ggml_backend_graph_compute(backend, gf);
printf("%s: time = %f ms\n", __func__, (t_end - t_start) / 1000.0 / n_iter);
}
{
struct ggml_tensor * res = ggml_graph_node(gf, -1);
std::vector<float> data(res->ne[0] * res->ne[1], 0.0f);
ggml_backend_tensor_get(res, data.data(), 0, ggml_nbytes(res));
for (int i1 = 0; i1 < res->ne[1]; i1++) {
for (int i0 = 0; i0 < res->ne[0]; i0++) {
printf("%f ", data[i1*res->ne[0] + i0]);
}
printf("\n");
}
}
ggml_free(ctx_cgraph);
ggml_gallocr_free(allocr);
ggml_free(ctx);
ggml_backend_buffer_free(buffer);
ggml_backend_free(backend);
return 0;
}
ggml/examples/prompts/dolly-v2.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 12092,3645,2
I can't believe it's already Friday!" => 42,476,626,2868,352,434,2168,6794,1476
The URL for the website is https://www.example.com." => 510,10611,323,253,4422,310,5987,1358,2700,15,11667,15,681,449
"She said, 'I love to travel.'" => 3,2993,753,13,686,42,2389,281,4288,18574
'The temperature is 25.5°C.' => 8,510,3276,310,2030,15,22,3272,36,2464
"Let's meet at 2:30 p.m. in the park." => 3,1466,434,2525,387,374,27,1229,268,15,78,15,275,253,5603,449
The book costs $19.99 => 510,1984,4815,370,746,15,1525
"John's favorite color is blue." => 3,8732,434,7583,3295,310,4797,449
Th@nk y0u f0r y0ur h3lp! => 1044,33,30664,340,17,86,269,17,83,340,17,321,288,20,24343,2
C@n I g3t a c0ffee, pl3@se? => 36,33,79,309,305,20,85,247,260,17,71,6851,13,499,20,33,339,32
W0w! Th@t's @m@zing! => 56,17,88,2,596,33,85,434,1214,78,33,8537,2
H0w 4re y0u t0d@y? => 41,17,88,577,250,340,17,86,246,17,69,33,90,32
I l0ve t0 tr@vel @r0und the w0rld. => 42,298,17,306,246,17,492,33,652,1214,83,17,1504,253,259,17,83,392,15
Wh@t's y0ur f@v0rite m0vie? => 3152,33,85,434,340,17,321,269,33,87,17,3852,278,17,25858,32
The cat is sleeping on the mat. => 510,5798,310,14343,327,253,1111,15
I need to buy some groceries for dinner. => 42,878,281,4489,690,45160,447,323,8955,15
The sun is shining brightly in the sky. => 510,5101,310,28115,43925,275,253,8467,15
She is reading a book in the park. => 2993,310,4361,247,1984,275,253,5603,15
We went for a walk on the beach yesterday. => 1231,2427,323,247,2940,327,253,11600,11066,15
He plays the guitar like a pro. => 1328,7120,253,12609,751,247,354,15
They are going to the movies tonight. => 3726,403,1469,281,253,11321,11608,15
The flowers are blooming in the garden. => 510,12405,403,30601,272,275,253,10329,15
I enjoy listening to classical music. => 42,4264,11298,281,8946,3440,15
We need to buy groceries for the week. => 1231,878,281,4489,45160,447,323,253,2129,15
The dog is chasing its tail in circles. => 510,4370,310,31702,697,8105,275,14240,15
She is wearing a beautiful red dress. => 2993,310,9398,247,5389,2502,7619,15
He is a talented actor in Hollywood. => 1328,310,247,21220,12353,275,14759,15
The children are playing in the playground. => 510,2151,403,4882,275,253,41008,15
I'm going to visit my grandparents this weekend. => 42,1353,1469,281,4143,619,37186,436,8849,15
The coffee tastes bitter without sugar. => 510,8574,27491,17123,1293,8618,15
They are planning a surprise party for her. => 3726,403,7219,247,9326,3128,323,617,15
She sings like an angel on stage. => 2993,44718,751,271,23087,327,3924,15
We should take a vacation to relax. => 1231,943,1379,247,18125,281,7921,15
He is studying medicine at the university. => 1328,310,12392,9921,387,253,9835,15
The rain is pouring heavily outside. => 510,9313,310,31226,11306,3345,15
I enjoy watching romantic movies. => 42,4264,7487,18109,11321,15
They are celebrating their anniversary today. => 3726,403,28765,616,19054,3063,15
She dances gracefully to the music. => 2993,47078,14426,2920,281,253,3440,15
He is an excellent basketball player. => 1328,310,271,7126,14648,4760,15
The baby is sleeping soundly in the crib. => 510,6858,310,14343,3590,314,275,253,260,725,15
I need to finish my homework before dinner. => 42,878,281,8416,619,32110,1078,8955,15
They are organizing a charity event next month. => 3726,403,26169,247,19489,2362,1735,1770,15
She is cooking a delicious meal for us. => 2993,310,12398,247,17319,11484,323,441,15
We should go hiking in the mountains. => 1231,943,564,33061,275,253,14700,15
The car broke down on the way to work. => 510,1113,9377,1066,327,253,1039,281,789,15
He loves playing video games in his free time. => 1328,14528,4882,3492,3958,275,521,1959,673,15
The birds are chirping in the trees. => 510,11260,403,36494,14650,275,253,7139,15
I want to learn how to play the piano. => 42,971,281,3037,849,281,1132,253,18542,15
They are building a new shopping mall in the city. => 3726,403,3652,247,747,12701,28974,275,253,2846,15
She is writing a novel in her spare time. => 2993,310,4028,247,4460,275,617,18345,673,15
We are going to the zoo this Saturday. => 1231,403,1469,281,253,41089,436,7814,15
The cake looks delicious with chocolate frosting. => 510,15221,4453,17319,342,14354,34724,272,15
He is a talented painter who sells his artwork. => 1328,310,247,21220,27343,665,27924,521,28227,15
The students are studying for their exams. => 510,3484,403,12392,323,616,34666,15
I enjoy swimming in the ocean. => 42,4264,17120,275,253,12927,15
They are renovating their house. => 3726,403,30074,839,616,2419,15
She is practicing yoga to stay healthy. => 2993,310,25815,25551,281,3297,5875,15
We should plant flowers in the garden. => 1231,943,4444,12405,275,253,10329,15
The traffic is heavy during rush hour. => 510,7137,310,5536,1309,16949,4964,15
He is a skilled chef who creates amazing dishes. => 1328,310,247,18024,26540,665,10513,8644,17114,15
The baby is crawling on the floor. => 510,6858,310,44922,327,253,5254,15
I need to buy a new pair of shoes. => 42,878,281,4489,247,747,4667,273,12682,15
They are going on a road trip across the country. => 3726,403,1469,327,247,3971,7408,2439,253,2586,15
She is playing the piano beautifully. => 2993,310,4882,253,18542,27839,15
We are going to a concert tomorrow night. => 1231,403,1469,281,247,12699,10873,2360,15
The cake tastes delicious with vanilla frosting. => 510,15221,27491,17319,342,26724,34724,272,15
He is a dedicated teacher who inspires his students. => 1328,310,247,9940,9732,665,6381,2731,521,3484,15
The students are participating in a science fair. => 510,3484,403,15299,275,247,5859,4344,15
I enjoy hiking in the mountains. => 42,4264,33061,275,253,14700,15
They are organizing a beach cleanup next weekend. => 3726,403,26169,247,11600,34709,1735,8849,15
She is taking photographs of nature. => 2993,310,3192,15928,273,3753,15
We should try a new restaurant in town. => 1231,943,1611,247,747,10301,275,3874,15
The traffic is moving slowly on the highway. => 510,7137,310,4886,7808,327,253,17657,15
He is a talented singer with a beautiful voice. => 1328,310,247,21220,16057,342,247,5389,4318,15
The baby is laughing and giggling. => 510,6858,310,17053,285,41542,1981,15
I need to do laundry and wash my clothes. => 42,878,281,513,29023,285,14841,619,10015,15
They are planning a trip to Europe. => 3726,403,7219,247,7408,281,3060,15
She is learning how to play the guitar. => 2993,310,4715,849,281,1132,253,12609,15
We are going to a museum this Sunday. => 1231,403,1469,281,247,16064,436,6926,15
The coffee smells amazing in the morning. => 510,8574,34247,8644,275,253,4131,15
He is a hardworking farmer who grows crops. => 1328,310,247,1892,21107,24718,665,17202,19492,15
The students are presenting their research projects. => 510,3484,403,15250,616,2561,6493,15
I enjoy playing soccer with my friends. => 42,4264,4882,20391,342,619,3858,15
They are volunteering at a local shelter. => 3726,403,10057,2158,387,247,1980,17824,15
She is practicing martial arts for self-defense. => 2993,310,25815,29731,14635,323,1881,14,29337,15
We should try a new recipe for dinner. => 1231,943,1611,247,747,13612,323,8955,15
The traffic is congest => 510,7137,310,25801
The sun is shining brightly today. => 510,5101,310,28115,43925,3063,15
I enjoy reading books in my free time. => 42,4264,4361,5098,275,619,1959,673,15
She plays the piano beautifully. => 2993,7120,253,18542,27839,15
The cat chased the mouse around the room. => 510,5798,40754,253,6521,1475,253,2316,15
I love eating pizza with extra cheese. => 42,2389,9123,22534,342,4465,12173,15
He always wears a hat wherever he goes. => 1328,1900,31394,247,7856,20312,344,4566,15
The flowers in the garden are blooming. => 510,12405,275,253,10329,403,30601,272,15
She danced gracefully on the stage. => 2993,39860,14426,2920,327,253,3924,15
The dog barked loudly in the park. => 510,4370,21939,264,31311,275,253,5603,15
We went swimming in the ocean yesterday. => 1231,2427,17120,275,253,12927,11066,15
He speaks fluent French and Spanish. => 1328,16544,2938,290,5112,285,9883,15
The train arrived at the station on time. => 510,6194,7244,387,253,4660,327,673,15
She cooked a delicious meal for her family. => 2993,18621,247,17319,11484,323,617,2021,15
ggml/examples/prompts/gpt-2-chinese.txt
+1
View File
@@ -0,0 +1 @@
请问洗手间在哪里？ => 6435,7309,3819,2797,7313,1762,1525,7027,8043
ggml/examples/prompts/gpt-2.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 15496,2159,0
I can't believe it's already Friday!" => 40,460,470,1975,340,338,1541,3217,2474
The URL for the website is https://www.example.com." => 464,10289,329,262,3052,318,3740,1378,2503,13,20688,13,785,526
"She said, 'I love to travel.'" => 1,3347,531,11,705,40,1842,284,3067,11496
'The temperature is 25.5°C.' => 6,464,5951,318,1679,13,20,7200,34,2637
"Let's meet at 2:30 p.m. in the park." => 1,5756,338,1826,379,362,25,1270,279,13,76,13,287,262,3952,526
The book costs $19.99 => 464,1492,3484,720,1129,13,2079
"John's favorite color is blue." => 1,7554,338,4004,3124,318,4171,526
Th@nk y0u f0r y0ur h3lp! => 817,31,77,74,331,15,84,277,15,81,331,15,333,289,18,34431,0
C@n I g3t a c0ffee, pl3@se? => 34,31,77,314,308,18,83,257,269,15,5853,11,458,18,31,325,30
W0w! Th@t's @m@zing! => 54,15,86,0,536,31,83,338,2488,76,31,9510,0
H0w 4re y0u t0d@y? => 39,15,86,604,260,331,15,84,256,15,67,31,88,30
I l0ve t0 tr@vel @r0und the w0rld. => 40,300,15,303,256,15,491,31,626,2488,81,15,917,262,266,15,81,335,13
Wh@t's y0ur f@v0rite m0vie? => 1199,31,83,338,331,15,333,277,31,85,15,6525,285,15,85,494,30
The cat is sleeping on the mat. => 464,3797,318,11029,319,262,2603,13
I need to buy some groceries for dinner. => 40,761,284,2822,617,38464,329,8073,13
The sun is shining brightly in the sky. => 464,4252,318,22751,35254,287,262,6766,13
She is reading a book in the park. => 3347,318,3555,257,1492,287,262,3952,13
We went for a walk on the beach yesterday. => 1135,1816,329,257,2513,319,262,10481,7415,13
He plays the guitar like a pro. => 1544,5341,262,10047,588,257,386,13
They are going to the movies tonight. => 2990,389,1016,284,262,6918,9975,13
The flowers are blooming in the garden. => 464,12734,389,24924,3383,287,262,11376,13
I enjoy listening to classical music. => 40,2883,8680,284,15993,2647,13
We need to buy groceries for the week. => 1135,761,284,2822,38464,329,262,1285,13
The dog is chasing its tail in circles. => 464,3290,318,20023,663,7894,287,13332,13
She is wearing a beautiful red dress. => 3347,318,5762,257,4950,2266,6576,13
He is a talented actor in Hollywood. => 1544,318,257,12356,8674,287,8502,13
The children are playing in the playground. => 464,1751,389,2712,287,262,24817,13
I'm going to visit my grandparents this weekend. => 40,1101,1016,284,3187,616,28571,428,5041,13
The coffee tastes bitter without sugar. => 464,6891,18221,12922,1231,7543,13
They are planning a surprise party for her. => 2990,389,5410,257,5975,2151,329,607,13
She sings like an angel on stage. => 3347,33041,588,281,18304,319,3800,13
We should take a vacation to relax. => 1135,815,1011,257,14600,284,8960,13
He is studying medicine at the university. => 1544,318,11065,9007,379,262,6403,13
The rain is pouring heavily outside. => 464,6290,318,23147,7272,2354,13
I enjoy watching romantic movies. => 40,2883,4964,14348,6918,13
They are celebrating their anniversary today. => 2990,389,17499,511,11162,1909,13
She dances gracefully to the music. => 3347,38207,11542,2759,284,262,2647,13
He is an excellent basketball player. => 1544,318,281,6275,9669,2137,13
The baby is sleeping soundly in the crib. => 464,5156,318,11029,2128,306,287,262,48083,13
I need to finish my homework before dinner. => 40,761,284,5461,616,26131,878,8073,13
They are organizing a charity event next month. => 2990,389,16924,257,11016,1785,1306,1227,13
She is cooking a delicious meal for us. => 3347,318,10801,257,12625,9799,329,514,13
We should go hiking in the mountains. => 1135,815,467,24522,287,262,12269,13
The car broke down on the way to work. => 464,1097,6265,866,319,262,835,284,670,13
He loves playing video games in his free time. => 1544,10408,2712,2008,1830,287,465,1479,640,13
The birds are chirping in the trees. => 464,10087,389,442,343,13886,287,262,7150,13
I want to learn how to play the piano. => 40,765,284,2193,703,284,711,262,19132,13
They are building a new shopping mall in the city. => 2990,389,2615,257,649,9735,17374,287,262,1748,13
She is writing a novel in her spare time. => 3347,318,3597,257,5337,287,607,13952,640,13
We are going to the zoo this Saturday. => 1135,389,1016,284,262,26626,428,3909,13
The cake looks delicious with chocolate frosting. => 464,12187,3073,12625,351,11311,21682,278,13
He is a talented painter who sells his artwork. => 1544,318,257,12356,34537,508,16015,465,16257,13
The students are studying for their exams. => 464,2444,389,11065,329,511,26420,13
I enjoy swimming in the ocean. => 40,2883,14899,287,262,9151,13
They are renovating their house. => 2990,389,24317,803,511,2156,13
She is practicing yoga to stay healthy. => 3347,318,18207,20351,284,2652,5448,13
We should plant flowers in the garden. => 1135,815,4618,12734,287,262,11376,13
The traffic is heavy during rush hour. => 464,4979,318,4334,1141,10484,1711,13
He is a skilled chef who creates amazing dishes. => 1544,318,257,14297,21221,508,8075,4998,16759,13
The baby is crawling on the floor. => 464,5156,318,34499,319,262,4314,13
I need to buy a new pair of shoes. => 40,761,284,2822,257,649,5166,286,10012,13
They are going on a road trip across the country. => 2990,389,1016,319,257,2975,5296,1973,262,1499,13
She is playing the piano beautifully. => 3347,318,2712,262,19132,21104,13
We are going to a concert tomorrow night. => 1135,389,1016,284,257,10010,9439,1755,13
The cake tastes delicious with vanilla frosting. => 464,12187,18221,12625,351,16858,21682,278,13
He is a dedicated teacher who inspires his students. => 1544,318,257,7256,4701,508,38934,465,2444,13
The students are participating in a science fair. => 464,2444,389,11983,287,257,3783,3148,13
I enjoy hiking in the mountains. => 40,2883,24522,287,262,12269,13
They are organizing a beach cleanup next weekend. => 2990,389,16924,257,10481,27425,1306,5041,13
She is taking photographs of nature. => 3347,318,2263,12566,286,3450,13
We should try a new restaurant in town. => 1135,815,1949,257,649,7072,287,3240,13
The traffic is moving slowly on the highway. => 464,4979,318,3867,6364,319,262,12763,13
He is a talented singer with a beautiful voice. => 1544,318,257,12356,14015,351,257,4950,3809,13
The baby is laughing and giggling. => 464,5156,318,14376,290,30442,1359,13
I need to do laundry and wash my clothes. => 40,761,284,466,25724,290,13502,616,8242,13
They are planning a trip to Europe. => 2990,389,5410,257,5296,284,2031,13
She is learning how to play the guitar. => 3347,318,4673,703,284,711,262,10047,13
We are going to a museum this Sunday. => 1135,389,1016,284,257,13257,428,3502,13
The coffee smells amazing in the morning. => 464,6891,25760,4998,287,262,3329,13
He is a hardworking farmer who grows crops. => 1544,318,257,1327,16090,18739,508,13676,14450,13
The students are presenting their research projects. => 464,2444,389,17728,511,2267,4493,13
I enjoy playing soccer with my friends. => 40,2883,2712,11783,351,616,2460,13
They are volunteering at a local shelter. => 2990,389,41434,379,257,1957,11772,13
She is practicing martial arts for self-defense. => 3347,318,18207,15618,10848,329,2116,12,19774,13
We should try a new recipe for dinner. => 1135,815,1949,257,649,8364,329,8073,13
The traffic is congest => 464,4979,318,22791
The sun is shining brightly today. => 464,4252,318,22751,35254,1909,13
I enjoy reading books in my free time. => 40,2883,3555,3835,287,616,1479,640,13
She plays the piano beautifully. => 3347,5341,262,19132,21104,13
The cat chased the mouse around the room. => 464,3797,26172,262,10211,1088,262,2119,13
I love eating pizza with extra cheese. => 40,1842,6600,14256,351,3131,9891,13
He always wears a hat wherever he goes. => 1544,1464,17326,257,6877,14530,339,2925,13
The flowers in the garden are blooming. => 464,12734,287,262,11376,389,24924,3383,13
She danced gracefully on the stage. => 3347,39480,11542,2759,319,262,3800,13
The dog barked loudly in the park. => 464,3290,21405,276,23112,287,262,3952,13
We went swimming in the ocean yesterday. => 1135,1816,14899,287,262,9151,7415,13
He speaks fluent French and Spanish. => 1544,9209,43472,4141,290,7897,13
The train arrived at the station on time. => 464,4512,5284,379,262,4429,319,640,13
She cooked a delicious meal for her family. => 3347,15847,257,12625,9799,329,607,1641,13
ggml/examples/prompts/gpt-j.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 15496,2159,0
I can't believe it's already Friday!" => 40,460,470,1975,340,338,1541,3217,2474
The URL for the website is https://www.example.com." => 464,10289,329,262,3052,318,3740,1378,2503,13,20688,13,785,526
"She said, 'I love to travel.'" => 1,3347,531,11,705,40,1842,284,3067,11496
'The temperature is 25.5°C.' => 6,464,5951,318,1679,13,20,7200,34,2637
"Let's meet at 2:30 p.m. in the park." => 1,5756,338,1826,379,362,25,1270,279,13,76,13,287,262,3952,526
The book costs $19.99 => 464,1492,3484,720,1129,13,2079
"John's favorite color is blue." => 1,7554,338,4004,3124,318,4171,526
Th@nk y0u f0r y0ur h3lp! => 817,31,77,74,331,15,84,277,15,81,331,15,333,289,18,34431,0
C@n I g3t a c0ffee, pl3@se? => 34,31,77,314,308,18,83,257,269,15,5853,11,458,18,31,325,30
W0w! Th@t's @m@zing! => 54,15,86,0,536,31,83,338,2488,76,31,9510,0
H0w 4re y0u t0d@y? => 39,15,86,604,260,331,15,84,256,15,67,31,88,30
I l0ve t0 tr@vel @r0und the w0rld. => 40,300,15,303,256,15,491,31,626,2488,81,15,917,262,266,15,81,335,13
Wh@t's y0ur f@v0rite m0vie? => 1199,31,83,338,331,15,333,277,31,85,15,6525,285,15,85,494,30
The cat is sleeping on the mat. => 464,3797,318,11029,319,262,2603,13
I need to buy some groceries for dinner. => 40,761,284,2822,617,38464,329,8073,13
The sun is shining brightly in the sky. => 464,4252,318,22751,35254,287,262,6766,13
She is reading a book in the park. => 3347,318,3555,257,1492,287,262,3952,13
We went for a walk on the beach yesterday. => 1135,1816,329,257,2513,319,262,10481,7415,13
He plays the guitar like a pro. => 1544,5341,262,10047,588,257,386,13
They are going to the movies tonight. => 2990,389,1016,284,262,6918,9975,13
The flowers are blooming in the garden. => 464,12734,389,24924,3383,287,262,11376,13
I enjoy listening to classical music. => 40,2883,8680,284,15993,2647,13
We need to buy groceries for the week. => 1135,761,284,2822,38464,329,262,1285,13
The dog is chasing its tail in circles. => 464,3290,318,20023,663,7894,287,13332,13
She is wearing a beautiful red dress. => 3347,318,5762,257,4950,2266,6576,13
He is a talented actor in Hollywood. => 1544,318,257,12356,8674,287,8502,13
The children are playing in the playground. => 464,1751,389,2712,287,262,24817,13
I'm going to visit my grandparents this weekend. => 40,1101,1016,284,3187,616,28571,428,5041,13
The coffee tastes bitter without sugar. => 464,6891,18221,12922,1231,7543,13
They are planning a surprise party for her. => 2990,389,5410,257,5975,2151,329,607,13
She sings like an angel on stage. => 3347,33041,588,281,18304,319,3800,13
We should take a vacation to relax. => 1135,815,1011,257,14600,284,8960,13
He is studying medicine at the university. => 1544,318,11065,9007,379,262,6403,13
The rain is pouring heavily outside. => 464,6290,318,23147,7272,2354,13
I enjoy watching romantic movies. => 40,2883,4964,14348,6918,13
They are celebrating their anniversary today. => 2990,389,17499,511,11162,1909,13
She dances gracefully to the music. => 3347,38207,11542,2759,284,262,2647,13
He is an excellent basketball player. => 1544,318,281,6275,9669,2137,13
The baby is sleeping soundly in the crib. => 464,5156,318,11029,2128,306,287,262,48083,13
I need to finish my homework before dinner. => 40,761,284,5461,616,26131,878,8073,13
They are organizing a charity event next month. => 2990,389,16924,257,11016,1785,1306,1227,13
She is cooking a delicious meal for us. => 3347,318,10801,257,12625,9799,329,514,13
We should go hiking in the mountains. => 1135,815,467,24522,287,262,12269,13
The car broke down on the way to work. => 464,1097,6265,866,319,262,835,284,670,13
He loves playing video games in his free time. => 1544,10408,2712,2008,1830,287,465,1479,640,13
The birds are chirping in the trees. => 464,10087,389,442,343,13886,287,262,7150,13
I want to learn how to play the piano. => 40,765,284,2193,703,284,711,262,19132,13
They are building a new shopping mall in the city. => 2990,389,2615,257,649,9735,17374,287,262,1748,13
She is writing a novel in her spare time. => 3347,318,3597,257,5337,287,607,13952,640,13
We are going to the zoo this Saturday. => 1135,389,1016,284,262,26626,428,3909,13
The cake looks delicious with chocolate frosting. => 464,12187,3073,12625,351,11311,21682,278,13
He is a talented painter who sells his artwork. => 1544,318,257,12356,34537,508,16015,465,16257,13
The students are studying for their exams. => 464,2444,389,11065,329,511,26420,13
I enjoy swimming in the ocean. => 40,2883,14899,287,262,9151,13
They are renovating their house. => 2990,389,24317,803,511,2156,13
She is practicing yoga to stay healthy. => 3347,318,18207,20351,284,2652,5448,13
We should plant flowers in the garden. => 1135,815,4618,12734,287,262,11376,13
The traffic is heavy during rush hour. => 464,4979,318,4334,1141,10484,1711,13
He is a skilled chef who creates amazing dishes. => 1544,318,257,14297,21221,508,8075,4998,16759,13
The baby is crawling on the floor. => 464,5156,318,34499,319,262,4314,13
I need to buy a new pair of shoes. => 40,761,284,2822,257,649,5166,286,10012,13
They are going on a road trip across the country. => 2990,389,1016,319,257,2975,5296,1973,262,1499,13
She is playing the piano beautifully. => 3347,318,2712,262,19132,21104,13
We are going to a concert tomorrow night. => 1135,389,1016,284,257,10010,9439,1755,13
The cake tastes delicious with vanilla frosting. => 464,12187,18221,12625,351,16858,21682,278,13
He is a dedicated teacher who inspires his students. => 1544,318,257,7256,4701,508,38934,465,2444,13
The students are participating in a science fair. => 464,2444,389,11983,287,257,3783,3148,13
I enjoy hiking in the mountains. => 40,2883,24522,287,262,12269,13
They are organizing a beach cleanup next weekend. => 2990,389,16924,257,10481,27425,1306,5041,13
She is taking photographs of nature. => 3347,318,2263,12566,286,3450,13
We should try a new restaurant in town. => 1135,815,1949,257,649,7072,287,3240,13
The traffic is moving slowly on the highway. => 464,4979,318,3867,6364,319,262,12763,13
He is a talented singer with a beautiful voice. => 1544,318,257,12356,14015,351,257,4950,3809,13
The baby is laughing and giggling. => 464,5156,318,14376,290,30442,1359,13
I need to do laundry and wash my clothes. => 40,761,284,466,25724,290,13502,616,8242,13
They are planning a trip to Europe. => 2990,389,5410,257,5296,284,2031,13
She is learning how to play the guitar. => 3347,318,4673,703,284,711,262,10047,13
We are going to a museum this Sunday. => 1135,389,1016,284,257,13257,428,3502,13
The coffee smells amazing in the morning. => 464,6891,25760,4998,287,262,3329,13
He is a hardworking farmer who grows crops. => 1544,318,257,1327,16090,18739,508,13676,14450,13
The students are presenting their research projects. => 464,2444,389,17728,511,2267,4493,13
I enjoy playing soccer with my friends. => 40,2883,2712,11783,351,616,2460,13
They are volunteering at a local shelter. => 2990,389,41434,379,257,1957,11772,13
She is practicing martial arts for self-defense. => 3347,318,18207,15618,10848,329,2116,12,19774,13
We should try a new recipe for dinner. => 1135,815,1949,257,649,8364,329,8073,13
The traffic is congest => 464,4979,318,22791
The sun is shining brightly today. => 464,4252,318,22751,35254,1909,13
I enjoy reading books in my free time. => 40,2883,3555,3835,287,616,1479,640,13
She plays the piano beautifully. => 3347,5341,262,19132,21104,13
The cat chased the mouse around the room. => 464,3797,26172,262,10211,1088,262,2119,13
I love eating pizza with extra cheese. => 40,1842,6600,14256,351,3131,9891,13
He always wears a hat wherever he goes. => 1544,1464,17326,257,6877,14530,339,2925,13
The flowers in the garden are blooming. => 464,12734,287,262,11376,389,24924,3383,13
She danced gracefully on the stage. => 3347,39480,11542,2759,319,262,3800,13
The dog barked loudly in the park. => 464,3290,21405,276,23112,287,262,3952,13
We went swimming in the ocean yesterday. => 1135,1816,14899,287,262,9151,7415,13
He speaks fluent French and Spanish. => 1544,9209,43472,4141,290,7897,13
The train arrived at the station on time. => 464,4512,5284,379,262,4429,319,640,13
She cooked a delicious meal for her family. => 3347,15847,257,12625,9799,329,607,1641,13
ggml/examples/prompts/gpt-neox-japanese.txt
+1
View File
@@ -0,0 +1 @@
明日の天気はどうですか。 => 263,7353,268,18461,271,1722,18405,265
ggml/examples/prompts/gpt-neox.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 12092,3645,2
I can't believe it's already Friday!" => 42,476,626,2868,352,434,2168,6794,1476
The URL for the website is https://www.example.com." => 510,10611,323,253,4422,310,5987,1358,2700,15,11667,15,681,449
"She said, 'I love to travel.'" => 3,2993,753,13,686,42,2389,281,4288,18574
'The temperature is 25.5°C.' => 8,510,3276,310,2030,15,22,3272,36,2464
"Let's meet at 2:30 p.m. in the park." => 3,1466,434,2525,387,374,27,1229,268,15,78,15,275,253,5603,449
The book costs $19.99 => 510,1984,4815,370,746,15,1525
"John's favorite color is blue." => 3,8732,434,7583,3295,310,4797,449
Th@nk y0u f0r y0ur h3lp! => 1044,33,30664,340,17,86,269,17,83,340,17,321,288,20,24343,2
C@n I g3t a c0ffee, pl3@se? => 36,33,79,309,305,20,85,247,260,17,71,6851,13,499,20,33,339,32
W0w! Th@t's @m@zing! => 56,17,88,2,596,33,85,434,1214,78,33,8537,2
H0w 4re y0u t0d@y? => 41,17,88,577,250,340,17,86,246,17,69,33,90,32
I l0ve t0 tr@vel @r0und the w0rld. => 42,298,17,306,246,17,492,33,652,1214,83,17,1504,253,259,17,83,392,15
Wh@t's y0ur f@v0rite m0vie? => 3152,33,85,434,340,17,321,269,33,87,17,3852,278,17,25858,32
The cat is sleeping on the mat. => 510,5798,310,14343,327,253,1111,15
I need to buy some groceries for dinner. => 42,878,281,4489,690,45160,447,323,8955,15
The sun is shining brightly in the sky. => 510,5101,310,28115,43925,275,253,8467,15
She is reading a book in the park. => 2993,310,4361,247,1984,275,253,5603,15
We went for a walk on the beach yesterday. => 1231,2427,323,247,2940,327,253,11600,11066,15
He plays the guitar like a pro. => 1328,7120,253,12609,751,247,354,15
They are going to the movies tonight. => 3726,403,1469,281,253,11321,11608,15
The flowers are blooming in the garden. => 510,12405,403,30601,272,275,253,10329,15
I enjoy listening to classical music. => 42,4264,11298,281,8946,3440,15
We need to buy groceries for the week. => 1231,878,281,4489,45160,447,323,253,2129,15
The dog is chasing its tail in circles. => 510,4370,310,31702,697,8105,275,14240,15
She is wearing a beautiful red dress. => 2993,310,9398,247,5389,2502,7619,15
He is a talented actor in Hollywood. => 1328,310,247,21220,12353,275,14759,15
The children are playing in the playground. => 510,2151,403,4882,275,253,41008,15
I'm going to visit my grandparents this weekend. => 42,1353,1469,281,4143,619,37186,436,8849,15
The coffee tastes bitter without sugar. => 510,8574,27491,17123,1293,8618,15
They are planning a surprise party for her. => 3726,403,7219,247,9326,3128,323,617,15
She sings like an angel on stage. => 2993,44718,751,271,23087,327,3924,15
We should take a vacation to relax. => 1231,943,1379,247,18125,281,7921,15
He is studying medicine at the university. => 1328,310,12392,9921,387,253,9835,15
The rain is pouring heavily outside. => 510,9313,310,31226,11306,3345,15
I enjoy watching romantic movies. => 42,4264,7487,18109,11321,15
They are celebrating their anniversary today. => 3726,403,28765,616,19054,3063,15
She dances gracefully to the music. => 2993,47078,14426,2920,281,253,3440,15
He is an excellent basketball player. => 1328,310,271,7126,14648,4760,15
The baby is sleeping soundly in the crib. => 510,6858,310,14343,3590,314,275,253,260,725,15
I need to finish my homework before dinner. => 42,878,281,8416,619,32110,1078,8955,15
They are organizing a charity event next month. => 3726,403,26169,247,19489,2362,1735,1770,15
She is cooking a delicious meal for us. => 2993,310,12398,247,17319,11484,323,441,15
We should go hiking in the mountains. => 1231,943,564,33061,275,253,14700,15
The car broke down on the way to work. => 510,1113,9377,1066,327,253,1039,281,789,15
He loves playing video games in his free time. => 1328,14528,4882,3492,3958,275,521,1959,673,15
The birds are chirping in the trees. => 510,11260,403,36494,14650,275,253,7139,15
I want to learn how to play the piano. => 42,971,281,3037,849,281,1132,253,18542,15
They are building a new shopping mall in the city. => 3726,403,3652,247,747,12701,28974,275,253,2846,15
She is writing a novel in her spare time. => 2993,310,4028,247,4460,275,617,18345,673,15
We are going to the zoo this Saturday. => 1231,403,1469,281,253,41089,436,7814,15
The cake looks delicious with chocolate frosting. => 510,15221,4453,17319,342,14354,34724,272,15
He is a talented painter who sells his artwork. => 1328,310,247,21220,27343,665,27924,521,28227,15
The students are studying for their exams. => 510,3484,403,12392,323,616,34666,15
I enjoy swimming in the ocean. => 42,4264,17120,275,253,12927,15
They are renovating their house. => 3726,403,30074,839,616,2419,15
She is practicing yoga to stay healthy. => 2993,310,25815,25551,281,3297,5875,15
We should plant flowers in the garden. => 1231,943,4444,12405,275,253,10329,15
The traffic is heavy during rush hour. => 510,7137,310,5536,1309,16949,4964,15
He is a skilled chef who creates amazing dishes. => 1328,310,247,18024,26540,665,10513,8644,17114,15
The baby is crawling on the floor. => 510,6858,310,44922,327,253,5254,15
I need to buy a new pair of shoes. => 42,878,281,4489,247,747,4667,273,12682,15
They are going on a road trip across the country. => 3726,403,1469,327,247,3971,7408,2439,253,2586,15
She is playing the piano beautifully. => 2993,310,4882,253,18542,27839,15
We are going to a concert tomorrow night. => 1231,403,1469,281,247,12699,10873,2360,15
The cake tastes delicious with vanilla frosting. => 510,15221,27491,17319,342,26724,34724,272,15
He is a dedicated teacher who inspires his students. => 1328,310,247,9940,9732,665,6381,2731,521,3484,15
The students are participating in a science fair. => 510,3484,403,15299,275,247,5859,4344,15
I enjoy hiking in the mountains. => 42,4264,33061,275,253,14700,15
They are organizing a beach cleanup next weekend. => 3726,403,26169,247,11600,34709,1735,8849,15
She is taking photographs of nature. => 2993,310,3192,15928,273,3753,15
We should try a new restaurant in town. => 1231,943,1611,247,747,10301,275,3874,15
The traffic is moving slowly on the highway. => 510,7137,310,4886,7808,327,253,17657,15
He is a talented singer with a beautiful voice. => 1328,310,247,21220,16057,342,247,5389,4318,15
The baby is laughing and giggling. => 510,6858,310,17053,285,41542,1981,15
I need to do laundry and wash my clothes. => 42,878,281,513,29023,285,14841,619,10015,15
They are planning a trip to Europe. => 3726,403,7219,247,7408,281,3060,15
She is learning how to play the guitar. => 2993,310,4715,849,281,1132,253,12609,15
We are going to a museum this Sunday. => 1231,403,1469,281,247,16064,436,6926,15
The coffee smells amazing in the morning. => 510,8574,34247,8644,275,253,4131,15
He is a hardworking farmer who grows crops. => 1328,310,247,1892,21107,24718,665,17202,19492,15
The students are presenting their research projects. => 510,3484,403,15250,616,2561,6493,15
I enjoy playing soccer with my friends. => 42,4264,4882,20391,342,619,3858,15
They are volunteering at a local shelter. => 3726,403,10057,2158,387,247,1980,17824,15
She is practicing martial arts for self-defense. => 2993,310,25815,29731,14635,323,1881,14,29337,15
We should try a new recipe for dinner. => 1231,943,1611,247,747,13612,323,8955,15
The traffic is congest => 510,7137,310,25801
The sun is shining brightly today. => 510,5101,310,28115,43925,3063,15
I enjoy reading books in my free time. => 42,4264,4361,5098,275,619,1959,673,15
She plays the piano beautifully. => 2993,7120,253,18542,27839,15
The cat chased the mouse around the room. => 510,5798,40754,253,6521,1475,253,2316,15
I love eating pizza with extra cheese. => 42,2389,9123,22534,342,4465,12173,15
He always wears a hat wherever he goes. => 1328,1900,31394,247,7856,20312,344,4566,15
The flowers in the garden are blooming. => 510,12405,275,253,10329,403,30601,272,15
She danced gracefully on the stage. => 2993,39860,14426,2920,327,253,3924,15
The dog barked loudly in the park. => 510,4370,21939,264,31311,275,253,5603,15
We went swimming in the ocean yesterday. => 1231,2427,17120,275,253,12927,11066,15
He speaks fluent French and Spanish. => 1328,16544,2938,290,5112,285,9883,15
The train arrived at the station on time. => 510,6194,7244,387,253,4660,327,673,15
She cooked a delicious meal for her family. => 2993,18621,247,17319,11484,323,617,2021,15
ggml/examples/prompts/polyglot-ko.txt
+3
View File
@@ -0,0 +1,3 @@
이것은 테스트 이다. => 12271,296,6474,28037,17
걱정할 필요 없다. => 18311,482,1062,550,267,17
버그는 언젠가 고쳐진다. => 6904,272,8575,10381,1765,17
ggml/examples/prompts/replit.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 6466,147,2317,350
I can't believe it's already Friday!" => 286,512,172,185,13392,393,172,155,3239,147,29249,8537
The URL for the website is https://www.example.com." => 505,5635,250,170,11745,235,147,303,262,552,148,811,148,241,148,161
"She said, 'I love to travel.'" => 161,10386,4089,150,206,286,8440,194,147,12363,148,172,161
'The temperature is 25.5°C.' => 172,505,147,9502,235,147,20022,8516,228,148,172
"Let's meet at 2:30 p.m. in the park." => 161,8997,172,155,17120,536,147,162,5245,147,207,148,204,148,219,170,147,17664,148,161
The book costs $19.99 => 505,147,2277,17494,236,166,11824
"John's favorite color is blue." => 161,7475,172,155,147,11105,147,349,235,17046,148,161
Th@nk y0u f0r y0ur h3lp! => 6309,240,9019,147,237,159,247,147,202,159,223,147,237,159,2458,147,226,171,3899,350
C@n I g3t a c0ffee, pl3@se? => 228,240,211,398,147,267,171,185,216,147,196,159,13360,163,150,147,1287,171,240,155,163,272
W0w! Th@t's @m@zing! => 450,159,274,350,147,6309,240,185,172,155,268,204,240,301,248,350
H0w 4re y0u t0d@y? => 304,159,274,320,440,147,237,159,247,147,185,159,182,240,237,272
I l0ve t0 tr@vel @r0und the w0rld. => 286,997,159,1290,147,185,159,147,490,240,3893,268,223,159,3981,170,147,274,159,223,2833,148
Wh@t's y0ur f@v0rite m0vie? => 450,226,240,185,172,155,147,237,159,2458,147,202,240,252,159,5961,163,147,204,159,24373,272
The cat is sleeping on the mat. => 505,147,1604,235,147,3987,248,347,170,147,1297,148
I need to buy some groceries for dinner. => 286,1645,194,147,8068,1499,147,10022,1037,10023,250,147,182,2749,148
The sun is shining brightly in the sky. => 505,147,5852,235,147,7304,2967,147,215,649,391,219,170,147,7310,148
She is reading a book in the park. => 10386,235,9838,216,147,2277,219,170,147,17664,148
We went for a walk on the beach yesterday. => 3250,10825,250,216,147,8156,347,170,294,5371,147,28830,148
He plays the guitar like a pro. => 5301,7084,155,170,147,4604,2214,1425,216,3474,148
They are going to the movies tonight. => 18815,429,6552,194,170,147,15877,194,7907,148
The flowers are blooming in the garden. => 505,147,22953,155,429,147,10411,2799,248,219,170,147,22140,148
I enjoy listening to classical music. => 286,23162,15876,248,194,239,4251,147,7395,148
We need to buy groceries for the week. => 3250,1645,194,147,8068,147,10022,1037,10023,250,170,9238,148
The dog is chasing its tail in circles. => 505,147,6540,235,147,196,916,248,1602,147,5129,219,147,4095,155,148
She is wearing a beautiful red dress. => 10386,235,147,16427,248,216,147,23447,147,1160,147,14592,148
He is a talented actor in Hollywood. => 5301,235,216,147,29750,246,147,5112,219,147,16924,391,10477,148
The children are playing in the playground. => 505,7934,429,7084,248,219,170,7084,12055,148
I'm going to visit my grandparents this weekend. => 286,172,204,6552,194,9939,1247,147,11806,12019,291,9238,314,148
The coffee tastes bitter without sugar. => 505,147,21526,147,20931,155,5145,1430,1988,147,28759,148
They are planning a surprise party for her. => 18815,429,147,23661,216,147,29240,147,7344,250,1869,148
She sings like an angel on stage. => 10386,147,155,6502,1425,426,147,26028,347,12685,148
We should take a vacation to relax. => 3250,936,4654,216,147,15388,946,194,1998,2744,148
He is studying medicine at the university. => 5301,235,7959,248,147,20742,1668,536,170,147,8025,148
The rain is pouring heavily outside. => 505,147,6885,235,5306,248,1189,5451,391,8096,148
I enjoy watching romantic movies. => 286,23162,147,3355,248,147,26080,4140,147,15877,148
They are celebrating their anniversary today. => 18815,429,147,30000,5841,1669,147,24734,5464,1770,13386,148
She dances gracefully to the music. => 10386,147,182,1626,155,147,267,8771,8001,194,170,147,7395,148
He is an excellent basketball player. => 5301,235,426,147,12300,675,185,147,26646,5132,6294,148
The baby is sleeping soundly in the crib. => 505,147,23597,235,147,3987,248,12642,391,219,170,147,7696,215,148
I need to finish my homework before dinner. => 286,1645,194,147,6717,1247,147,1071,2722,2643,147,182,2749,148
They are organizing a charity event next month. => 18815,429,147,16442,248,216,1054,1511,1663,2399,12821,148
She is cooking a delicious meal for us. => 10386,235,147,20453,248,216,3936,23455,147,26658,250,147,539,148
We should go hiking in the mountains. => 3250,936,4242,147,2254,5357,219,170,147,204,18028,155,148
The car broke down on the way to work. => 505,7553,147,510,10036,4288,347,170,3699,194,1916,148
He loves playing video games in his free time. => 5301,8440,155,7084,248,8722,147,11281,219,1439,4002,801,148
The birds are chirping in the trees. => 505,147,13043,155,429,147,3904,223,4639,219,170,5311,155,148
I want to learn how to play the piano. => 286,1857,194,14167,2496,194,7084,170,147,207,23635,148
They are building a new shopping mall in the city. => 18815,429,11038,216,277,147,22184,147,204,609,219,170,147,2416,148
She is writing a novel in her spare time. => 10386,235,3242,216,147,25814,219,1869,6772,2382,801,148
We are going to the zoo this Saturday. => 3250,429,6552,194,170,147,25101,291,147,31426,148
The cake looks delicious with chocolate frosting. => 505,147,24422,16303,3936,23455,312,147,5619,533,2239,147,202,3973,3431,148
He is a talented painter who sells his artwork. => 5301,235,216,147,29750,246,147,9226,279,2888,13004,155,1439,12234,2722,148
The students are studying for their exams. => 505,15707,429,7959,248,250,1669,147,12398,155,148
I enjoy swimming in the ocean. => 286,23162,147,4729,8528,248,219,170,147,26193,148
They are renovating their house. => 18815,429,991,10724,3643,1669,13788,148
She is practicing yoga to stay healthy. => 10386,235,147,18453,248,147,5063,1186,194,15344,147,28550,148
We should plant flowers in the garden. => 3250,936,147,9212,147,22953,155,219,170,147,22140,148
The traffic is heavy during rush hour. => 505,147,11097,235,147,22232,4340,147,22319,147,5686,148
He is a skilled chef who creates amazing dishes. => 5301,235,216,147,8891,246,9784,202,2888,13720,147,28880,147,23852,383,148
The baby is crawling on the floor. => 505,147,23597,235,147,22120,248,347,170,147,5895,148
I need to buy a new pair of shoes. => 286,1645,194,147,8068,216,277,12632,210,147,155,21953,155,148
They are going on a road trip across the country. => 18815,429,6552,347,216,147,6362,147,11395,9762,170,11305,148
She is playing the piano beautifully. => 10386,235,7084,248,170,147,207,23635,147,23447,391,148
We are going to a concert tomorrow night. => 3250,429,6552,194,216,1710,4391,29524,12716,148
The cake tastes delicious with vanilla frosting. => 505,147,24422,147,20931,155,3936,23455,312,5535,7476,147,202,3973,3431,148
He is a dedicated teacher who inspires his students. => 5301,235,216,326,8298,3460,147,9675,2888,147,28801,155,1439,15707,148
The students are participating in a science fair. => 505,15707,429,147,30961,3643,219,216,147,10587,147,7636,148
I enjoy hiking in the mountains. => 286,23162,147,2254,5357,219,170,147,204,18028,155,148
They are organizing a beach cleanup next weekend. => 18815,429,147,16442,248,216,294,5371,147,10401,2399,9238,314,148
She is taking photographs of nature. => 10386,235,147,12345,147,4709,1547,155,210,147,211,8603,148
We should try a new restaurant in town. => 3250,936,147,746,216,277,147,11007,219,147,10200,148
The traffic is moving slowly on the highway. => 505,147,11097,235,147,8601,147,9880,391,347,170,5976,3330,148
He is a talented singer with a beautiful voice. => 5301,235,216,147,29750,246,147,155,248,279,312,216,147,23447,147,9316,148
The baby is laughing and giggling. => 505,147,23597,235,147,23066,248,221,147,2341,3631,2869,148
I need to do laundry and wash my clothes. => 286,1645,194,543,960,3981,2154,221,147,27589,1247,147,22141,383,148
They are planning a trip to Europe. => 18815,429,147,23661,216,147,11395,194,13131,148
She is learning how to play the guitar. => 10386,235,11754,2496,194,7084,170,147,4604,2214,148
We are going to a museum this Sunday. => 3250,429,6552,194,216,147,204,433,1177,291,147,29111,148
The coffee smells amazing in the morning. => 505,147,21526,31454,155,147,28880,219,170,20701,148
He is a hardworking farmer who grows crops. => 5301,235,216,8524,14992,147,16679,279,2888,147,6044,155,147,8650,155,148
The students are presenting their research projects. => 505,15707,429,5130,248,1669,13217,14235,148
I enjoy playing soccer with my friends. => 286,23162,7084,248,147,9351,5318,312,1247,147,5347,155,148
They are volunteering at a local shelter. => 18815,429,147,5238,7478,163,12798,536,216,2491,2905,1359,279,148
She is practicing martial arts for self-defense. => 10386,235,147,18453,248,147,3261,185,4381,12234,155,250,623,153,29896,148
We should try a new recipe for dinner. => 3250,936,147,746,216,277,147,9851,250,147,182,2749,148
The traffic is congest => 505,147,11097,235,1710,14169
The sun is shining brightly today. => 505,147,5852,235,147,7304,2967,147,215,649,391,13386,148
I enjoy reading books in my free time. => 286,23162,9838,147,9670,219,1247,4002,801,148
She plays the piano beautifully. => 10386,7084,155,170,147,207,23635,147,23447,391,148
The cat chased the mouse around the room. => 505,147,1604,147,196,916,246,170,12551,6890,170,9654,148
I love eating pizza with extra cheese. => 286,8440,147,163,3643,147,207,8403,312,8230,9784,383,163,148
He always wears a hat wherever he goes. => 5301,5418,147,16427,155,216,147,4879,2171,2433,1189,16177,148
The flowers in the garden are blooming. => 505,147,22953,155,219,170,147,22140,429,147,10411,2799,248,148
She danced gracefully on the stage. => 10386,13378,12408,147,267,8771,8001,347,170,12685,148
The dog barked loudly in the park. => 505,147,6540,147,973,293,246,147,30182,391,219,170,147,17664,148
We went swimming in the ocean yesterday. => 3250,10825,147,4729,8528,248,219,170,147,26193,147,28830,148
He speaks fluent French and Spanish. => 5301,147,13285,155,147,21677,147,254,17590,221,147,31519,148
The train arrived at the station on time. => 505,147,872,147,20712,182,536,170,147,7184,347,801,148
She cooked a delicious meal for her family. => 10386,147,20453,246,216,3936,23455,147,26658,250,1869,147,2002,148
ggml/examples/prompts/starcoder.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 8279,10896,19
I can't believe it's already Friday!" => 59,883,1330,13710,561,1182,3425,506,25674,11555
The URL for the website is https://www.example.com." => 1318,3834,436,322,9575,438,1678,555,1499,32,2763,32,508,3107
"She said, 'I love to travel.'" => 20,25387,9884,30,330,59,14290,372,25283,29329
'The temperature is 25.5°C.' => 25,1318,13587,438,225,36,39,32,39,23767,53,4564
"Let's meet at 2:30 p.m. in the park." => 20,9809,1182,18450,821,225,36,44,37,34,298,32,95,32,328,322,880,93,3107
The book costs $19.99 => 1318,7618,25950,398,35,43,32,43,43
"John's favorite color is blue." => 20,19693,1182,27448,1963,438,10087,3107
Th@nk y0u f0r y0ur h3lp! => 1027,50,19877,533,34,103,296,34,100,533,34,305,420,37,1915,19
C@n I g3t a c0ffee, pl3@se? => 53,50,96,439,485,37,102,312,281,34,21298,30,1278,37,50,277,49
W0w! Th@t's @m@zing! => 73,34,105,19,947,50,102,1182,477,95,50,26768,19
H0w 4re y0u t0d@y? => 58,34,105,225,38,268,533,34,103,273,34,86,50,107,49
I l0ve t0 tr@vel @r0und the w0rld. => 59,456,34,587,273,34,554,50,1203,477,100,34,642,322,341,34,100,1381,32
Wh@t's y0ur f@v0rite m0vie? => 2444,50,102,1182,533,34,305,296,50,104,34,1049,345,34,104,1075,49
The cat is sleeping on the mat. => 1318,10501,438,9368,299,544,322,2491,32
I need to buy some groceries for dinner. => 59,1849,372,16968,1629,20234,85,6958,436,343,3369,32
The sun is shining brightly in the sky. => 1318,15323,438,787,19068,38231,631,328,322,26718,32
She is reading a book in the park. => 25387,438,9175,312,7618,328,322,880,93,32
We went for a walk on the beach yesterday. => 3122,14236,436,312,13503,544,322,526,867,39485,32
He plays the guitar like a pro. => 1331,41271,322,3932,19931,2124,312,534,32
They are going to the movies tonight. => 31805,884,6783,372,322,27889,26076,694,32
The flowers are blooming in the garden. => 1318,7290,483,884,323,18466,299,328,322,485,22461,32
I enjoy listening to classical music. => 59,31567,20498,372,443,1578,17522,32
We need to buy groceries for the week. => 3122,1849,372,16968,20234,85,6958,436,322,8209,32
The dog is chasing its tail in circles. => 1318,27435,438,663,9949,2819,13203,328,46428,32
She is wearing a beautiful red dress. => 25387,438,996,6992,312,36493,3346,343,714,32
He is a talented actor in Hollywood. => 1331,438,312,273,9556,318,16038,328,48228,631,21118,32
The children are playing in the playground. => 1318,5713,884,19788,328,322,4654,1749,32
I'm going to visit my grandparents this weekend. => 59,3464,6783,372,7725,1672,33162,19277,458,40618,32
The coffee tastes bitter without sugar. => 1318,36917,273,633,307,3493,391,2876,309,18628,32
They are planning a surprise party for her. => 31805,884,26116,312,6178,9251,15270,436,7791,32
She sings like an angel on stage. => 25387,309,2052,2124,600,600,17691,544,10019,32
We should take a vacation to relax. => 3122,1395,4818,312,29164,367,372,41972,32
He is studying medicine at the university. => 1331,438,14866,299,32388,482,821,322,707,9190,32
The rain is pouring heavily outside. => 1318,36987,438,9202,299,46003,2801,11127,32
I enjoy watching romantic movies. => 59,31567,37652,26045,7268,27889,32
They are celebrating their anniversary today. => 31805,884,48278,839,1741,3623,23921,5810,672,11610,32
She dances gracefully to the music. => 25387,343,3151,31376,4938,372,322,17522,32
He is an excellent basketball player. => 1331,438,600,39203,48400,11653,4362,32
The baby is sleeping soundly in the crib. => 1318,323,17156,438,9368,299,9934,631,328,322,281,7972,32
I need to finish my homework before dinner. => 59,1849,372,11361,1672,6765,1007,2670,343,3369,32
They are organizing a charity event next month. => 31805,884,10558,6183,312,1351,543,1692,2354,6811,32
She is cooking a delicious meal for us. => 25387,438,23682,299,312,409,406,2406,597,279,436,1770,32
We should go hiking in the mountains. => 3122,1395,1983,420,1546,299,328,322,10874,1907,32
The car broke down on the way to work. => 1318,6346,43289,2835,544,322,3352,372,1389,32
He loves playing video games in his free time. => 1331,598,4954,19788,6027,19705,328,6697,3741,1133,32
The birds are chirping in the trees. => 1318,8424,3210,884,663,476,7075,328,322,23453,32
I want to learn how to play the piano. => 59,2637,372,7350,2624,372,4654,322,298,25757,32
They are building a new shopping mall in the city. => 31805,884,9038,312,537,40692,345,464,328,322,11297,32
She is writing a novel in her spare time. => 25387,438,4127,312,32913,328,7791,1869,586,1133,32
We are going to the zoo this Saturday. => 3122,884,6783,372,322,1288,604,458,358,30288,32
The cake looks delicious with chocolate frosting. => 1318,281,1062,7780,409,406,2406,623,10408,27589,296,20932,299,32
He is a talented painter who sells his artwork. => 1331,438,312,273,9556,318,42300,6560,10800,101,6697,5549,1007,32
The students are studying for their exams. => 1318,16512,884,14866,299,436,3623,538,1462,32
I enjoy swimming in the ocean. => 59,31567,2535,449,6714,328,322,337,18857,32
They are renovating their house. => 31805,884,316,15007,1741,3623,17075,32
She is practicing yoga to stay healthy. => 25387,438,11808,11636,533,40067,372,20005,44538,32
We should plant flowers in the garden. => 3122,1395,26795,7290,483,328,322,485,22461,32
The traffic is heavy during rush hour. => 1318,16391,438,32389,5929,540,1372,12021,32
He is a skilled chef who creates amazing dishes. => 1331,438,312,3001,12088,44051,6560,9585,36986,1214,4279,32
The baby is crawling on the floor. => 1318,323,17156,438,281,1294,2920,544,322,17648,32
I need to buy a new pair of shoes. => 59,1849,372,16968,312,537,6092,432,787,37764,32
They are going on a road trip across the country. => 31805,884,6783,544,312,24122,19337,10160,322,10769,32
She is playing the piano beautifully. => 25387,438,19788,322,298,25757,526,4846,325,514,107,32
We are going to a concert tomorrow night. => 3122,884,6783,372,312,457,6989,31841,19212,32
The cake tastes delicious with vanilla frosting. => 1318,281,1062,273,633,307,409,406,2406,623,44653,296,20932,299,32
He is a dedicated teacher who inspires his students. => 1331,438,312,23112,30877,6560,26194,8017,6697,16512,32
The students are participating in a science fair. => 1318,16512,884,24623,1741,328,312,27536,19375,32
I enjoy hiking in the mountains. => 59,31567,420,1546,299,328,322,10874,1907,32
They are organizing a beach cleanup next weekend. => 31805,884,10558,6183,312,526,867,13144,2354,40618,32
She is taking photographs of nature. => 25387,438,15137,15110,23626,432,24406,32
We should try a new restaurant in town. => 3122,1395,1596,312,537,43719,328,38212,32
The traffic is moving slowly on the highway. => 1318,16391,438,14089,12899,631,544,322,3857,3073,32
He is a talented singer with a beautiful voice. => 1331,438,312,273,9556,318,309,10118,623,312,36493,20309,32
The baby is laughing and giggling. => 1318,323,17156,438,2317,2943,299,461,485,365,36088,32
I need to do laundry and wash my clothes. => 59,1849,372,745,2317,642,994,461,341,917,1672,7375,46948,32
They are planning a trip to Europe. => 31805,884,26116,312,19337,372,27268,32
She is learning how to play the guitar. => 25387,438,9608,2624,372,4654,322,3932,19931,32
We are going to a museum this Sunday. => 3122,884,6783,372,312,345,539,378,458,358,28036,32
The coffee smells amazing in the morning. => 1318,36917,309,42153,101,36986,328,322,33768,32
He is a hardworking farmer who grows crops. => 1331,438,312,6784,13578,9019,2302,6560,485,2138,25170,1069,32
The students are presenting their research projects. => 1318,16512,884,5024,299,3623,13234,8528,32
I enjoy playing soccer with my friends. => 59,31567,19788,22682,10035,623,1672,22523,32
They are volunteering at a local shelter. => 31805,884,3920,45585,8637,821,312,2196,309,2542,391,32
She is practicing martial arts for self-defense. => 25387,438,11808,11636,345,502,564,5549,101,436,630,31,43694,32
We should try a new recipe for dinner. => 3122,1395,1596,312,537,15233,436,343,3369,32
The traffic is congest => 1318,16391,438,457,2776
The sun is shining brightly today. => 1318,15323,438,787,19068,38231,631,11610,32
I enjoy reading books in my free time. => 59,31567,9175,21739,328,1672,3741,1133,32
She plays the piano beautifully. => 25387,41271,322,298,25757,526,4846,325,514,107,32
The cat chased the mouse around the room. => 1318,10501,663,16109,322,8459,6835,322,8355,32
I love eating pizza with extra cheese. => 59,14290,484,1741,47630,623,6717,8277,30315,32
He always wears a hat wherever he goes. => 1331,5182,996,4177,312,25793,2154,424,938,13107,32
The flowers in the garden are blooming. => 1318,7290,483,328,322,485,22461,884,323,18466,299,32
She danced gracefully on the stage. => 25387,343,6087,31376,4938,544,322,10019,32
The dog barked loudly in the park. => 1318,27435,323,1087,318,598,836,631,328,322,880,93,32
We went swimming in the ocean yesterday. => 3122,14236,2535,449,6714,328,322,337,18857,39485,32
He speaks fluent French and Spanish. => 1331,24498,101,38055,43652,461,14911,1708,32
The train arrived at the station on time. => 1318,5683,2099,32114,821,322,18662,544,1133,32
She cooked a delicious meal for her family. => 25387,23682,318,312,409,406,2406,597,279,436,7791,13872,32
ggml/examples/prompts/test-cases.txt
+110
View File
@@ -0,0 +1,110 @@
# test case format
# <language>: <sentence>
English: Hello World!
English: I can't believe it's already Friday!"
English: The URL for the website is https://www.example.com."
English: "She said, 'I love to travel.'"
English: 'The temperature is 25.5°C.'
English: "Let's meet at 2:30 p.m. in the park."
English: The book costs $19.99
English: "John's favorite color is blue."
English: Th@nk y0u f0r y0ur h3lp!
English: C@n I g3t a c0ffee, pl3@se?
English: W0w! Th@t's @m@zing!
English: H0w 4re y0u t0d@y?
English: I l0ve t0 tr@vel @r0und the w0rld.
English: Wh@t's y0ur f@v0rite m0vie?
English: The cat is sleeping on the mat.
English: I need to buy some groceries for dinner.
English: The sun is shining brightly in the sky.
English: She is reading a book in the park.
English: We went for a walk on the beach yesterday.
English: He plays the guitar like a pro.
English: They are going to the movies tonight.
English: The flowers are blooming in the garden.
English: I enjoy listening to classical music.
English: We need to buy groceries for the week.
English: The dog is chasing its tail in circles.
English: She is wearing a beautiful red dress.
English: He is a talented actor in Hollywood.
English: The children are playing in the playground.
English: I'm going to visit my grandparents this weekend.
English: The coffee tastes bitter without sugar.
English: They are planning a surprise party for her.
English: She sings like an angel on stage.
English: We should take a vacation to relax.
English: He is studying medicine at the university.
English: The rain is pouring heavily outside.
English: I enjoy watching romantic movies.
English: They are celebrating their anniversary today.
English: She dances gracefully to the music.
English: He is an excellent basketball player.
English: The baby is sleeping soundly in the crib.
English: I need to finish my homework before dinner.
English: They are organizing a charity event next month.
English: She is cooking a delicious meal for us.
English: We should go hiking in the mountains.
English: The car broke down on the way to work.
English: He loves playing video games in his free time.
English: The birds are chirping in the trees.
English: I want to learn how to play the piano.
English: They are building a new shopping mall in the city.
English: She is writing a novel in her spare time.
English: We are going to the zoo this Saturday.
English: The cake looks delicious with chocolate frosting.
English: He is a talented painter who sells his artwork.
English: The students are studying for their exams.
English: I enjoy swimming in the ocean.
English: They are renovating their house.
English: She is practicing yoga to stay healthy.
English: We should plant flowers in the garden.
English: The traffic is heavy during rush hour.
English: He is a skilled chef who creates amazing dishes.
English: The baby is crawling on the floor.
English: I need to buy a new pair of shoes.
English: They are going on a road trip across the country.
English: She is playing the piano beautifully.
English: We are going to a concert tomorrow night.
English: The cake tastes delicious with vanilla frosting.
English: He is a dedicated teacher who inspires his students.
English: The students are participating in a science fair.
English: I enjoy hiking in the mountains.
English: They are organizing a beach cleanup next weekend.
English: She is taking photographs of nature.
English: We should try a new restaurant in town.
English: The traffic is moving slowly on the highway.
English: He is a talented singer with a beautiful voice.
English: The baby is laughing and giggling.
English: I need to do laundry and wash my clothes.
English: They are planning a trip to Europe.
English: She is learning how to play the guitar.
English: We are going to a museum this Sunday.
English: The coffee smells amazing in the morning.
English: He is a hardworking farmer who grows crops.
English: The students are presenting their research projects.
English: I enjoy playing soccer with my friends.
English: They are volunteering at a local shelter.
English: She is practicing martial arts for self-defense.
English: We should try a new recipe for dinner.
English: The traffic is congest
English: The sun is shining brightly today.
English: I enjoy reading books in my free time.
English: She plays the piano beautifully.
English: The cat chased the mouse around the room.
English: I love eating pizza with extra cheese.
English: He always wears a hat wherever he goes.
English: The flowers in the garden are blooming.
English: She danced gracefully on the stage.
English: The dog barked loudly in the park.
English: We went swimming in the ocean yesterday.
English: He speaks fluent French and Spanish.
English: The train arrived at the station on time.
English: She cooked a delicious meal for her family.
Korean: 이것은 테스트 이다.
Korean: 걱정할 필요 없다.
Korean: 버그는 언젠가 고쳐진다.
Japanese: 明日の天気はどうですか。
Chinese: 请问洗手间在哪里？
Emoji: I'm feeling 😄 today!
Unicode: ◑ ▢ ▣ ◱
ggml/examples/prompts/tokenize_huggingface.py
+65
View File
@@ -0,0 +1,65 @@
import os
from transformers import AutoTokenizer
os.environ['TOKENIZERS_PARALLELISM'] = "false"
list_repo_hf = ["databricks/dolly-v2-3b", # dolly-v2 (3b, 7b, 12b models share the same tokenizer)
"gpt2", # gpt-2 (gpt2-xl, gpt2-large share the same tokenizer)
"uer/gpt2-chinese-cluecorpussmall", # gpt-2-chinese
"EleutherAI/gpt-j-6b", # gpt-j
"EleutherAI/gpt-neox-20b", # gpt-neox
"EleutherAI/polyglot-ko-1.3b", # gpt-neox (polyglot-ko 5.8b and 12.8b share the same tokenizer")
"rinna/japanese-gpt-neox-3.6b", # gpt-neox
# mpt-7b (uses gpt-neox-20b tokenizer)
"replit/replit-code-v1-3b", # replit
"bigcode/starcoder", # starcoder (huggingface-cli login required)
"openai/whisper-tiny" # whisper (base, large, large-v2 share the same tokenizer)
]
repo2ggml = {"databricks/dolly-v2-3b" : "dolly-v2",
"gpt2" : "gpt-2",
"uer/gpt2-chinese-cluecorpussmall" : "gpt-2-chinese",
"EleutherAI/gpt-j-6b" : "gpt-j",
"EleutherAI/gpt-neox-20b" : "gpt-neox",
"EleutherAI/polyglot-ko-1.3b" : "polyglot-ko",
"rinna/japanese-gpt-neox-3.6b" : "gpt-neox-japanese",
"replit/replit-code-v1-3b" : "replit",
"bigcode/starcoder" : "starcoder",
"openai/whisper-tiny" : "whisper"}
repo2language = {"databricks/dolly-v2-3b" : "english",
"gpt2" : "english",
"uer/gpt2-chinese-cluecorpussmall" : "chinese",
"EleutherAI/gpt-j-6b" : "english",
"EleutherAI/gpt-neox-20b" : "english",
"EleutherAI/polyglot-ko-1.3b" : "korean",
"rinna/japanese-gpt-neox-3.6b" : "japanese",
"replit/replit-code-v1-3b" : "english",
"bigcode/starcoder" : "english",
"openai/whisper-tiny" : "english"}
delimeter = ": "
test_sentences = []
with open("test-cases.txt", "r") as f:
lines = [l.rstrip() for l in f.readlines()]
for l in lines:
if delimeter in l:
language = l[:l.index(delimeter)]
sentence = l[l.index(delimeter) + len(delimeter):]
test_sentences.append((language.lower(), sentence))
for repo in list_repo_hf:
target_language = repo2language[repo]
tokenizer = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)
tokens_hf = []
for language, sentence in test_sentences:
if language == target_language:
tokens = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentence))
tokens_hf.append((sentence, tokens))
save_txt = repo2ggml[repo] + ".txt"
with open(save_txt, "w") as f:
f.writelines([sentence + " => " + ",".join(str(t) for t in tokens) + "\n" for sentence, tokens in tokens_hf])
ggml/examples/prompts/whisper.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 15947,3937,0
I can't believe it's already Friday!" => 40,393,380,1697,309,311,1217,6984,2963
The URL for the website is https://www.example.com." => 2278,12905,337,220,3322,3144,307,34426,21492,17919,13,3121,335,781,13,1112,889
"She said, 'I love to travel.'" => 1,9526,848,11,922,40,959,220,1353,220,17227,779,28763
'The temperature is 25.5°C.' => 6,2278,220,18275,610,1503,307,3552,13,20,11782,34,4443
"Let's meet at 2:30 p.m. in the park." => 1,8373,311,1677,412,568,25,3446,280,13,76,13,294,220,3322,3884,889
The book costs $19.99 => 2278,1446,5497,1848,3405,13,8494
"John's favorite color is blue." => 1,16938,311,2954,2017,307,3344,889
Th@nk y0u f0r y0ur h3lp! => 2434,31,77,74,288,15,84,283,15,81,288,15,374,276,18,75,79,0
C@n I g3t a c0ffee, pl3@se? => 34,31,77,286,290,18,83,257,269,15,4617,11,499,18,31,405,30
W0w! Th@t's @m@zing! => 54,15,86,0,334,31,83,311,10428,76,31,8781,0
H0w 4re y0u t0d@y? => 39,15,86,1017,265,288,15,84,220,83,15,67,31,88,30
I l0ve t0 tr@vel @r0und the w0rld. => 40,287,15,303,220,83,15,220,6903,31,779,10428,81,15,997,220,3322,261,15,81,348,13
Wh@t's y0ur f@v0rite m0vie? => 2471,31,83,311,288,15,374,283,31,85,15,35002,275,15,12702,30
The cat is sleeping on the mat. => 2278,3857,307,8296,322,220,3322,3803,13
I need to buy some groceries for dinner. => 40,643,220,1353,2256,512,31391,337,6148,13
The sun is shining brightly in the sky. => 2278,3295,307,18269,47418,294,220,3322,5443,13
She is reading a book in the park. => 9526,307,3760,257,1446,294,220,3322,3884,13
We went for a walk on the beach yesterday. => 4360,1437,337,257,1792,322,220,3322,7534,5186,13
He plays the guitar like a pro. => 5205,5749,220,3322,7531,411,257,447,13
They are going to the movies tonight. => 8829,366,516,220,1353,220,3322,6233,220,1756,397,13
The flowers are blooming in the garden. => 2278,8085,366,45294,294,220,3322,7431,13
I enjoy listening to classical music. => 40,2103,4764,220,1353,13735,1318,13
We need to buy groceries for the week. => 4360,643,220,1353,2256,31391,337,220,3322,1243,13
The dog is chasing its tail in circles. => 2278,3000,307,17876,1080,220,14430,294,13040,13
She is wearing a beautiful red dress. => 9526,307,4769,257,2238,2182,5231,13
He is a talented actor in Hollywood. => 5205,307,257,220,32831,6003,8747,294,11628,13
The children are playing in the playground. => 2278,2227,366,2433,294,220,3322,24646,13
I'm going to visit my grandparents this weekend. => 40,478,516,220,1353,3441,452,21876,220,11176,6711,13
The coffee tastes bitter without sugar. => 2278,4982,220,83,40246,13871,1553,5076,13
They are planning a surprise party for her. => 8829,366,5038,257,6365,3595,337,720,13
She sings like an angel on stage. => 9526,23250,411,364,14250,322,3233,13
We should take a vacation to relax. => 4360,820,220,27612,257,12830,220,1353,5789,13
He is studying medicine at the university. => 5205,307,7601,7195,412,220,3322,5454,13
The rain is pouring heavily outside. => 2278,4830,307,20450,10950,2380,13
I enjoy watching romantic movies. => 40,2103,1976,13590,6233,13
They are celebrating their anniversary today. => 8829,366,15252,220,3322,347,12962,220,83,378,320,13
She dances gracefully to the music. => 9526,28322,10042,2277,220,1353,220,3322,1318,13
He is an excellent basketball player. => 5205,307,364,7103,11767,4256,13
The baby is sleeping soundly in the crib. => 2278,3186,307,8296,1626,356,294,220,3322,47163,13
I need to finish my homework before dinner. => 40,643,220,1353,2413,452,14578,949,6148,13
They are organizing a charity event next month. => 8829,366,17608,257,16863,2280,958,1618,13
She is cooking a delicious meal for us. => 9526,307,6361,257,4809,6791,337,505,13
We should go hiking in the mountains. => 4360,820,352,23784,294,220,3322,10233,13
The car broke down on the way to work. => 2278,1032,6902,760,322,220,3322,636,220,1353,589,13
He loves playing video games in his free time. => 5205,6752,2433,960,2813,294,702,1737,220,3766,13
The birds are chirping in the trees. => 2278,9009,366,36682,294,220,3322,220,3599,279,13
I want to learn how to play the piano. => 40,528,220,1353,1466,577,220,1353,862,220,3322,9211,13
They are building a new shopping mall in the city. => 8829,366,2390,257,777,8688,16026,294,220,3322,2307,13
She is writing a novel in her spare time. => 9526,307,3579,257,7613,294,720,13798,220,3766,13
We are going to the zoo this Saturday. => 4360,366,516,220,1353,220,3322,25347,220,11176,8803,13
The cake looks delicious with chocolate frosting. => 2278,5908,1542,4809,365,6215,37048,13
He is a talented painter who sells his artwork. => 5205,307,257,220,32831,6003,26619,567,20897,702,15829,13
The students are studying for their exams. => 2278,1731,366,7601,337,220,3322,347,20514,13
I enjoy swimming in the ocean. => 40,2103,11989,294,220,3322,7810,13
They are renovating their house. => 8829,366,18845,990,220,3322,347,1782,13
She is practicing yoga to stay healthy. => 9526,307,11350,15128,220,1353,1754,4627,13
We should plant flowers in the garden. => 4360,820,3709,8085,294,220,3322,7431,13
The traffic is heavy during rush hour. => 2278,220,17227,3341,307,4676,1830,9300,1773,13
He is a skilled chef who creates amazing dishes. => 5205,307,257,19690,10530,567,7829,2243,10814,13
The baby is crawling on the floor. => 2278,3186,307,32979,322,220,3322,4123,13
I need to buy a new pair of shoes. => 40,643,220,1353,2256,257,777,6119,295,6654,13
They are going on a road trip across the country. => 8829,366,516,322,257,3060,220,83,8400,2108,220,3322,1941,13
She is playing the piano beautifully. => 9526,307,2433,220,3322,9211,16525,13
We are going to a concert tomorrow night. => 4360,366,516,220,1353,257,8543,220,83,298,3162,1818,13
The cake tastes delicious with vanilla frosting. => 2278,5908,220,83,40246,4809,365,17528,37048,13
He is a dedicated teacher who inspires his students. => 5205,307,257,8374,220,975,4062,567,32566,702,1731,13
The students are participating in a science fair. => 2278,1731,366,13950,294,257,3497,3143,13
I enjoy hiking in the mountains. => 40,2103,23784,294,220,3322,10233,13
They are organizing a beach cleanup next weekend. => 8829,366,17608,257,7534,40991,958,6711,13
She is taking photographs of nature. => 9526,307,220,48625,17649,295,3687,13
We should try a new restaurant in town. => 4360,820,220,83,627,257,777,6383,294,220,30401,13
The traffic is moving slowly on the highway. => 2278,220,17227,3341,307,2684,5692,322,220,3322,17205,13
He is a talented singer with a beautiful voice. => 5205,307,257,220,32831,6003,11564,365,257,2238,3177,13
The baby is laughing and giggling. => 2278,3186,307,5059,293,290,24542,13
I need to do laundry and wash my clothes. => 40,643,220,1353,360,19811,293,5675,452,5534,13
They are planning a trip to Europe. => 8829,366,5038,257,220,83,8400,220,1353,3315,13
She is learning how to play the guitar. => 9526,307,2539,577,220,1353,862,220,3322,7531,13
We are going to a museum this Sunday. => 4360,366,516,220,1353,257,8441,220,11176,7776,13
The coffee smells amazing in the morning. => 2278,4982,10036,2243,294,220,3322,2446,13
He is a hardworking farmer who grows crops. => 5205,307,257,1152,22475,17891,567,13156,16829,13
The students are presenting their research projects. => 2278,1731,366,15578,220,3322,347,2132,4455,13
I enjoy playing soccer with my friends. => 40,2103,2433,15469,365,452,1855,13
They are volunteering at a local shelter. => 8829,366,33237,412,257,2654,13341,13
She is practicing martial arts for self-defense. => 9526,307,11350,20755,8609,337,2698,12,49268,13
We should try a new recipe for dinner. => 4360,820,220,83,627,257,777,6782,337,6148,13
The traffic is congest => 2278,220,17227,3341,307,31871
The sun is shining brightly today. => 2278,3295,307,18269,47418,220,83,378,320,13
I enjoy reading books in my free time. => 40,2103,3760,3642,294,452,1737,220,3766,13
She plays the piano beautifully. => 9526,5749,220,3322,9211,16525,13
The cat chased the mouse around the room. => 2278,3857,33091,220,3322,9719,926,220,3322,1808,13
I love eating pizza with extra cheese. => 40,959,3936,8298,365,2857,5399,13
He always wears a hat wherever he goes. => 5205,1009,20877,257,2385,8660,415,1709,13
The flowers in the garden are blooming. => 2278,8085,294,220,3322,7431,366,45294,13
She danced gracefully on the stage. => 9526,32909,10042,2277,322,220,3322,3233,13
The dog barked loudly in the park. => 2278,3000,16202,292,22958,294,220,3322,3884,13
We went swimming in the ocean yesterday. => 4360,1437,11989,294,220,3322,7810,5186,13
He speaks fluent French and Spanish. => 5205,10789,40799,5522,293,8058,13
The train arrived at the station on time. => 2278,220,83,7146,6678,412,220,3322,5214,322,220,3766,13
She cooked a delicious meal for her family. => 9526,9267,257,4809,6791,337,720,1605,13
ggml/examples/python/README.md
+115
View File
@@ -0,0 +1,115 @@
# Simple autogenerated Python bindings for ggml
This folder contains:
- Scripts to generate full Python bindings from ggml headers (+ stubs for autocompletion in IDEs)
- Some barebones utils (see [ggml/utils.py](./ggml/utils.py)):
- `ggml.utils.init` builds a context that's freed automatically when the pointer gets GC'd
- `ggml.utils.copy` **copies between same-shaped tensors (numpy or ggml), w/ automatic (de/re)quantization**
- `ggml.utils.numpy` returns a numpy view over a ggml tensor; if it's quantized, it returns a copy (requires `allow_copy=True`)
- Very basic examples (anyone wants to port [llama2.c](https://github.com/karpathy/llama2.c)?)
Provided you set `GGML_LIBRARY=.../path/to/libggml_shared.so` (see instructions below), it's trivial to do some operations on quantized tensors:
```python
# Make sure libllama.so is in your [DY]LD_LIBRARY_PATH, or set GGML_LIBRARY=.../libggml_shared.so
from ggml import lib, ffi
from ggml.utils import init, copy, numpy
import numpy as np
ctx = init(mem_size=12*1024*1024)
n = 256
n_threads = 4
a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) # Can't both be quantized
sum = lib.ggml_add(ctx, a, b) # all zeroes for now. Will be quantized too!
gf = ffi.new('struct ggml_cgraph*')
lib.ggml_build_forward_expand(gf, sum)
copy(np.array([i for i in range(n)], np.float32), a)
copy(np.array([i*100 for i in range(n)], np.float32), b)
lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)
print(numpy(a, allow_copy=True))
# 0. 1.0439453 2.0878906 3.131836 4.1757812 5.2197266. ...
print(numpy(b))
# 0. 100. 200. 300. 400. 500. ...
print(numpy(sum, allow_copy=True))
# 0. 105.4375 210.875 316.3125 421.75 527.1875 ...
```
### Prerequisites
You'll need a shared library of ggml to use the bindings.
#### Build libggml_shared.so or libllama.so
As of this writing the best is to use [ggerganov/llama.cpp](https://github.com/ggerganov/llama.cpp)'s generated `libggml_shared.so` or `libllama.so`, which you can build as follows:
```bash
git clone https://github.com/ggerganov/llama.cpp
# On a CUDA-enabled system add -DLLAMA_CUDA=1
# On a Mac add -DLLAMA_METAL=1
cmake llama.cpp \
-B llama_build \
-DCMAKE_C_FLAGS=-Ofast \
-DLLAMA_NATIVE=1 \
-DLLAMA_LTO=1 \
-DBUILD_SHARED_LIBS=1 \
-DLLAMA_MPI=1 \
-DLLAMA_BUILD_TESTS=0 \
-DLLAMA_BUILD_EXAMPLES=0
( cd llama_build && make -j )
# On Mac, this will be libggml_shared.dylib instead
export GGML_LIBRARY=$PWD/llama_build/libggml_shared.so
# Alternatively, you can just copy it to your system's lib dir, e.g /usr/local/lib
```
#### (Optional) Regenerate the bindings and stubs
If you added or changed any signatures of the C API, you'll want to regenerate the bindings ([ggml/cffi.py](./ggml/cffi.py)) and stubs ([ggml/__init__.pyi](./ggml/__init__.pyi)).
Luckily it's a one-liner using [regenerate.py](./regenerate.py):
```bash
pip install -q cffi
python regenerate.py
```
By default it assumes `llama.cpp` was cloned in ../../../llama.cpp (alongside the ggml folder). You can override this with:
```bash
C_INCLUDE_DIR=$LLAMA_CPP_DIR python regenerate.py
```
You can also edit [api.h](./api.h) to control which files should be included in the generated bindings (defaults to `llama.cpp/ggml*.h`)
In fact, if you wanted to only generate bindings for the current version of the `ggml` repo itself (instead of `llama.cpp`; you'd loose support for k-quants), you could run:
```bash
API=../../include/ggml.h python regenerate.py
```
## Develop
Run tests:
```bash
pytest
```
### Alternatives
This example's goal is to showcase [cffi](https://cffi.readthedocs.io/)-generated bindings that are trivial to use and update, but there are already alternatives in the wild:
- https://github.com/abetlen/ggml-python: these bindings seem to be hand-written and use [ctypes](https://docs.python.org/3/library/ctypes.html). It has [high-quality API reference docs](https://ggml-python.readthedocs.io/en/latest/api-reference/#ggml.ggml) that can be used with these bindings too, but it doesn't expose Metal, CUDA, MPI or OpenCL calls, doesn't support transparent (de/re)quantization like this example does (see [ggml.utils](./ggml/utils.py) module), and won't pick up your local changes.
- https://github.com/abetlen/llama-cpp-python: these expose the C++ `llama.cpp` interface, which this example cannot easily be extended to support (`cffi` only generates bindings of C libraries)
- [pybind11](https://github.com/pybind/pybind11) and [nanobind](https://github.com/wjakob/nanobind) are two alternatives to cffi that support binding C++ libraries, but it doesn't seem either of them have an automatic generator (writing bindings is rather time-consuming).
ggml/examples/python/api.h
+14
View File
@@ -0,0 +1,14 @@
/*
List here all the headers you want to expose in the Python bindings,
then run `python regenerate.py` (see details in README.md)
*/
#include "ggml.h"
#include "ggml-metal.h"
#include "ggml-opencl.h"
// Headers below are currently only present in the llama.cpp repository, comment them out if you don't have them.
#include "k_quants.h"
#include "ggml-alloc.h"
#include "ggml-cuda.h"
#include "ggml-mpi.h"
ggml/examples/python/example_add_quant.py
+25
View File
@@ -0,0 +1,25 @@
from ggml import lib, ffi
from ggml.utils import init, copy, numpy
import numpy as np
ctx = init(mem_size=12*1024*1024) # automatically freed when pointer is GC'd
n = 256
n_threads = 4
a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) # can't both be quantized
sum = lib.ggml_add(ctx, a, b) # all zeroes for now. Will be quantized too!
# See cffi's doc on how to allocate native memory: it's very simple!
# https://cffi.readthedocs.io/en/latest/ref.html#ffi-interface
gf = ffi.new('struct ggml_cgraph*')
lib.ggml_build_forward_expand(gf, sum)
copy(np.array([i for i in range(n)], np.float32), a)
copy(np.array([i*100 for i in range(n)], np.float32), b)
lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)
print(numpy(a, allow_copy=True))
print(numpy(b))
print(numpy(sum, allow_copy=True))
ggml/examples/python/example_test_all_quants.py
+68
View File
@@ -0,0 +1,68 @@
from ggml import ffi, lib
from ggml.utils import init, numpy, copy
import numpy as np
from math import pi, cos, sin, ceil
import matplotlib.pyplot as plt
ctx = init(mem_size=100*1024*1024) # Will be auto-GC'd
n = 256
orig = np.array([
[
cos(j * 2 * pi / n) * (sin(i * 2 * pi / n))
for j in range(n)
]
for i in range(n)
], np.float32)
orig_tensor = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, n, n)
copy(orig, orig_tensor)
quants = [
type for type in range(lib.GGML_TYPE_COUNT)
if lib.ggml_is_quantized(type) and
type not in [lib.GGML_TYPE_Q8_1, lib.GGML_TYPE_Q8_K] # Apparently not supported
]
# quants = [lib.GGML_TYPE_Q2_K] # Test a single one
def get_name(type):
name = lib.ggml_type_name(type)
return ffi.string(name).decode('utf-8') if name else '?'
quants.sort(key=get_name)
quants.insert(0, None)
print(quants)
ncols=4
nrows = ceil(len(quants) / ncols)
plt.figure(figsize=(ncols * 5, nrows * 5), layout='tight')
for i, type in enumerate(quants):
plt.subplot(nrows, ncols, i + 1)
try:
if type == None:
plt.title('Original')
plt.imshow(orig)
else:
quantized_tensor = lib.ggml_new_tensor_2d(ctx, type, n, n)
copy(orig_tensor, quantized_tensor)
quantized = numpy(quantized_tensor, allow_copy=True)
d = quantized - orig
results = {
"l2": np.linalg.norm(d, 2),
"linf": np.linalg.norm(d, np.inf),
"compression":
round(lib.ggml_nbytes(orig_tensor) /
lib.ggml_nbytes(quantized_tensor), 1)
}
name = get_name(type)
print(f'{name}: {results}')
plt.title(f'{name} ({results["compression"]}x smaller)')
plt.imshow(quantized, interpolation='nearest')
except Exception as e:
print(f'Error: {e}')
plt.show()
ggml/examples/python/ggml/__init__.py
+58
View File
@@ -0,0 +1,58 @@
"""
Python bindings for the ggml library.
Usage example:
from ggml import lib, ffi
from ggml.utils import init, copy, numpy
import numpy as np
ctx = init(mem_size=10*1024*1024)
n = 1024
n_threads = 4
a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)
sum = lib.ggml_add(ctx, a, b)
gf = ffi.new('struct ggml_cgraph*')
lib.ggml_build_forward_expand(gf, sum)
copy(np.array([i for i in range(n)], np.float32), a)
copy(np.array([i*100 for i in range(n)], np.float32), b)
lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)
print(numpy(sum, allow_copy=True))
See https://cffi.readthedocs.io/en/latest/cdef.html for more on cffi.
"""
try:
from ggml.cffi import ffi as ffi
except ImportError as e:
raise ImportError(f"Couldn't find ggml bindings ({e}). Run `python regenerate.py` or check your PYTHONPATH.")
import os, platform
__exact_library = os.environ.get("GGML_LIBRARY")
if __exact_library:
__candidates = [__exact_library]
elif platform.system() == "Windows":
__candidates = ["ggml_shared.dll", "llama.dll"]
else:
__candidates = ["libggml_shared.so", "libllama.so"]
if platform.system() == "Darwin":
__candidates += ["libggml_shared.dylib", "libllama.dylib"]
for i, name in enumerate(__candidates):
try:
# This is where all the functions, enums and constants are defined
lib = ffi.dlopen(name)
except OSError:
if i < len(__candidates) - 1:
continue
raise OSError(f"Couldn't find ggml's shared library (tried names: {__candidates}). Add its directory to DYLD_LIBRARY_PATH (on Mac) or LD_LIBRARY_PATH, or define GGML_LIBRARY.")
# This contains the cffi helpers such as new, cast, string, etc.
# https://cffi.readthedocs.io/en/latest/ref.html#ffi-interface
ffi = ffi
ggml/examples/python/ggml/__init__.pyi
+2406
View File
File diff suppressed because it is too large Load Diff
ggml/examples/python/ggml/cffi.py
+11
View File
File diff suppressed because one or more lines are too long
ggml/examples/python/ggml/ffi/__init__.pyi
+7
View File
@@ -0,0 +1,7 @@
# Phony stubs.
class CData:
pass
class CType:
pass
ggml/examples/python/ggml/utils.py
+182
View File
@@ -0,0 +1,182 @@
"""
Common helpers for working with ggml + numpy
"""
from ggml import ffi, lib
from typing import Union, Optional
import numpy as np
def init(mem_size: int, mem_buffer: ffi.CData = ffi.NULL, no_alloc: bool = False) -> ffi.CData:
"""
Initialize a ggml context, which will be freed automatically when the pointer is garbage collected.
"""
params = ffi.new('struct ggml_init_params*')
params.mem_size = mem_size
params.mem_buffer = mem_buffer
params.no_alloc = no_alloc
return ffi.gc(lib.ggml_init(params[0]), lib.ggml_free)
TensorLike = Union[ffi.CData, np.ndarray]
def copy(from_tensor: TensorLike, to_tensor: TensorLike, allow_requantize: bool = True):
"""
Copy the contents of one tensor to another, doing any necessary (de/re)quantization transparently.
Works across numpy & ggml tensors, but they must have the same shape (and be contiguous).
Parameters
----------
from_tensor : TensorLike
The tensor to copy from (a numpy array or possibly-quantized ggml tensor)
to_tensor : TensorLike
The tensor to copy to (a numpy array or possibly-quantized ggml tensor)
allow_requantize : bool
If False, will throw an error if requantization is required (i.e. both from_tensor
and to_tensor are quantized with different quantization types)
"""
if id(from_tensor) == id(to_tensor):
return
__expect_same_layout("source", from_tensor, "destination", to_tensor)
__check_shape_consistent_with_type(from_tensor)
__check_shape_consistent_with_type(to_tensor)
from_type = __get_type(from_tensor)
to_type = __get_type(to_tensor)
if from_type == to_type:
ffi.memmove(__get_data(to_tensor), __get_data(from_tensor), __get_nbytes(from_tensor))
else:
assert allow_requantize or not lib.ggml_is_quantized(from_type) or not lib.ggml_is_quantized(to_type), \
f"Requantizing from {__type_name(from_type)} to {__type_name(to_type)} is disabled. Force with allow_requantize=True"
__set_floats(to_tensor, __get_floats(from_tensor))
def numpy(tensor: ffi.CData, allow_copy: Union[bool, np.ndarray] = False, allow_requantize=False) -> np.ndarray:
"""
Convert a ggml tensor to a numpy array.
If the tensor isn't quantized, the returned numpy array will be a view over its data.
If it is quantized (and allow_copy is True), the copy will involve dequantization and the returned array will
be a copy of the original tensor (any changes to the numpy array won't then be reflected back to the tensor).
Parameters
----------
tensor : ffi.CData
The tensor to convert to a numpy array
allow_copy : bool or np.ndarray
If False, will throw an error if the tensor is quantized (since dequantization requires extra memory).
If True, will dequantize the tensor and return a copy of the data in a new float32 numpy array.
If an np.ndarray, will copy the data into the given array (which must be the same shape as the tensor) when dequantization is needed
allow_requantize : bool
If allow_copy is a tensor with a different quantization type than the source tensor, will throw an error unless allow_requantize is True.
"""
shape = __get_shape(tensor)
if lib.ggml_is_quantized(tensor.type):
if allow_copy == False:
raise ValueError(f"{__describe(tensor)} is quantized, conversion to numpy requires a copy (pass allow_copy=True; changes to the numpy array won't affect the original).")
elif isinstance(allow_copy, np.ndarray):
__expect_same_layout("source tensor", tensor, "dequantization output tensor", allow_copy)
destination = allow_copy
else:
destination = np.empty(shape, dtype=np.float32)
copy(tensor, destination, allow_requantize=allow_requantize)
return destination
else:
dtype = __type_to_dtype(tensor.type)
if not dtype:
raise NotImplementedError(f'Cannot convert {__describe(tensor)} to numpy')
assert __is_contiguous(tensor), f"Cannot convert {__describe(tensor)} to numpy (support contiguous tensors only)"
nbytes = lib.ggml_nelements(tensor) * lib.ggml_type_size(tensor.type)
array = np.frombuffer(ffi.buffer(lib.ggml_get_data(tensor), nbytes), dtype=dtype)
array.shape = shape
return array
def __type_name(type: int) -> str:
name = lib.ggml_type_name(type)
return ffi.string(name).decode('utf-8') if name else None
__k_quant_types = set([
lib.GGML_TYPE_Q2_K,
lib.GGML_TYPE_Q3_K,
lib.GGML_TYPE_Q4_K,
lib.GGML_TYPE_Q5_K,
lib.GGML_TYPE_Q6_K,
lib.GGML_TYPE_Q8_K,
])
__type_to_dtype_dict = {
lib.GGML_TYPE_I8: np.int8,
lib.GGML_TYPE_I16: np.int16,
lib.GGML_TYPE_I32: np.int32,
lib.GGML_TYPE_F16: np.float16,
lib.GGML_TYPE_F32: np.float32,
}
def __type_to_dtype(type: int) -> Optional[np.dtype]: return __type_to_dtype_dict.get(type)
def __dtype_to_type(dtype: np.dtype):
if dtype == np.float32: return lib.GGML_TYPE_F32
elif dtype == np.float16: return lib.GGML_TYPE_F16
elif dtype == np.int32: return lib.GGML_TYPE_I32
elif dtype == np.int16: return lib.GGML_TYPE_I16
elif dtype == np.int8: return lib.GGML_TYPE_I8
else: raise ValueError(f"Unsupported dtype: {dtype}")
def __describe(tensor: ffi.CType): return f'Tensor[{__type_name(__get_type(tensor))}, {__get_shape(tensor)}]'
def __get_type(tensor: TensorLike): return __dtype_to_type(tensor.dtype) if isinstance(tensor, np.ndarray) else tensor.type
def __get_shape(x: TensorLike): return x.shape if isinstance(x, np.ndarray) else tuple([x.ne[i] for i in range(x.n_dims)])
def __get_strides(x: TensorLike): return x.strides if isinstance(x, np.ndarray) else tuple([x.nb[i] for i in range(x.n_dims)])
def __get_data(x: TensorLike) -> ffi.CData: return ffi.from_buffer(x) if isinstance(x, np.ndarray) else lib.ggml_get_data(x)
def __get_nbytes(tensor: TensorLike): return tensor.nbytes if isinstance(tensor, np.ndarray) else lib.ggml_nbytes(tensor)
def __get_nelements(tensor: TensorLike): return tensor.size if isinstance(tensor, np.ndarray) else lib.ggml_nelements(tensor)
def __is_contiguous(tensor: TensorLike): return tensor.flags['C_CONTIGUOUS'] if isinstance(tensor, np.ndarray) else lib.ggml_is_contiguous(tensor)
def __get_floats(tensor: TensorLike) -> ffi.CData:
data, type = __get_data(tensor), __get_type(tensor)
if type == lib.GGML_TYPE_F32:
return ffi.cast('float*', data)
else:
nelements = __get_nelements(tensor)
floats = ffi.new('float[]', nelements)
if type == lib.GGML_TYPE_F16:
lib.ggml_fp16_to_fp32_row(ffi.cast('uint16_t*', data), floats, nelements)
elif lib.ggml_is_quantized(type):
qtype = lib.ggml_internal_get_type_traits(type)
assert qtype.to_float, f"Type {__type_name(type)} is not supported by ggml"
qtype.to_float(data, floats, nelements)
else:
raise NotImplementedError(f'Cannot read floats from {__describe(tensor)}')
return floats
def __set_floats(tensor: TensorLike, f32_data: ffi.CData) -> None:
data, type, nbytes = __get_data(tensor), __get_type(tensor), __get_nbytes(tensor)
if type == lib.GGML_TYPE_F32:
ffi.memmove(data, f32_data, nbytes)
else:
nelements = __get_nelements(tensor)
if type == lib.GGML_TYPE_F16:
lib.ggml_fp32_to_fp16_row(f32_data, ffi.cast('uint16_t*', data), nelements)
elif lib.ggml_is_quantized(type):
qtype = lib.ggml_internal_get_type_traits(type)
assert qtype.from_float, f"Type {__type_name(type)} is not supported by ggml"
qtype.from_float(f32_data, data, nelements)
else:
raise NotImplementedError(f'Cannot write floats to {__describe(tensor)}')
def __expect_same_layout(name1: str, tensor1: TensorLike, name2: str, tensor2: TensorLike):
shape1, shape2 = __get_shape(tensor1), __get_shape(tensor2)
assert shape1 == shape2, f"Shape mismatch: {name1} has {shape1} but {name2} has {shape2}"
assert __is_contiguous(tensor1) and __is_contiguous(tensor2), f"Only contiguous tensors are supported (got {name1} with strides {__get_strides(tensor1)} and {name2} with strides {__get_strides(tensor2)})"
def __check_shape_consistent_with_type(tensor: TensorLike):
type = __get_type(tensor)
if not lib.ggml_is_quantized(type):
return
shape = __get_shape(tensor)
block_size = lib.ggml_blck_size(type)
assert not (block_size == 0 and type in __k_quant_types), f"Can't quantize, native library was not compiled with USE_K_QUANTS!"
assert block_size > 0, f"Invalid block size {block_size} for type {__type_name(type)}"
for i, d in enumerate(shape):
assert d % block_size == 0, f"Dimension {i} of {__describe(tensor)} is not divisible by {block_size}, required for quantization."
ggml/examples/python/regenerate.py
+42
View File
@@ -0,0 +1,42 @@
# Generates bindings for the ggml library.
#
# cffi requires prior C preprocessing of the headers, and it uses pycparser which chokes on a couple of things
# so we help it a bit (e.g. replace sizeof expressions with their value, remove exotic syntax found in Darwin headers).
import os, sys, re, subprocess
import cffi
from stubs import generate_stubs
API = os.environ.get('API', 'api.h')
CC = os.environ.get('CC') or 'gcc'
C_INCLUDE_DIR = os.environ.get('C_INCLUDE_DIR', '../../../llama.cpp')
CPPFLAGS = [
"-I", C_INCLUDE_DIR,
'-D__fp16=uint16_t', # pycparser doesn't support __fp16
'-D__attribute__(x)=',
'-D_Static_assert(x, m)=',
] + [x for x in os.environ.get('CPPFLAGS', '').split(' ') if x != '']
try: header = subprocess.run([CC, "-E", *CPPFLAGS, API], capture_output=True, text=True, check=True).stdout
except subprocess.CalledProcessError as e: print(f'{e.stderr}\n{e}', file=sys.stderr); raise
header = '\n'.join([l for l in header.split('\n') if '__darwin_va_list' not in l]) # pycparser hates this
# Replace constant size expressions w/ their value (compile & run a mini exe for each, because why not).
# First, extract anyting *inside* square brackets and anything that looks like a sizeof call.
for expr in set(re.findall(f'(?<=\\[)[^\\]]+(?=])|sizeof\\s*\\([^()]+\\)', header)):
if re.match(r'^(\d+|\s*)$', expr): continue # skip constants and empty bracket contents
subprocess.run([CC, "-o", "eval_size_expr", *CPPFLAGS, "-x", "c", "-"], text=True, check=True,
input=f'''#include <stdio.h>
#include "{API}"
int main() {{ printf("%lu", (size_t)({expr})); }}''')
size = subprocess.run(["./eval_size_expr"], capture_output=True, text=True, check=True).stdout
print(f'Computed constexpr {expr} = {size}')
header = header.replace(expr, size)
ffibuilder = cffi.FFI()
ffibuilder.cdef(header)
ffibuilder.set_source(f'ggml.cffi', None) # we're not compiling a native extension, as this quickly gets hairy
ffibuilder.compile(verbose=True)
with open("ggml/__init__.pyi", "wt") as f:
f.write(generate_stubs(header))
ggml/examples/python/stubs.py
+128
View File
@@ -0,0 +1,128 @@
"""
This generates .pyi stubs for the cffi Python bindings generated by regenerate.py
"""
import sys, re, itertools
sys.path.extend(['.', '..']) # for pycparser
from pycparser import c_ast, parse_file, CParser
import pycparser.plyparser
from pycparser.c_ast import PtrDecl, TypeDecl, FuncDecl, EllipsisParam, IdentifierType, Struct, Enum, Typedef
from typing import Tuple
__c_type_to_python_type = {
'void': 'None', '_Bool': 'bool',
'char': 'int', 'short': 'int', 'int': 'int', 'long': 'int',
'ptrdiff_t': 'int', 'size_t': 'int',
'int8_t': 'int', 'uint8_t': 'int',
'int16_t': 'int', 'uint16_t': 'int',
'int32_t': 'int', 'uint32_t': 'int',
'int64_t': 'int', 'uint64_t': 'int',
'float': 'float', 'double': 'float',
'ggml_fp16_t': 'np.float16',
}
def format_type(t: TypeDecl):
if isinstance(t, PtrDecl) or isinstance(t, Struct):
return 'ffi.CData'
if isinstance(t, Enum):
return 'int'
if isinstance(t, TypeDecl):
return format_type(t.type)
if isinstance(t, IdentifierType):
assert len(t.names) == 1, f'Expected a single name, got {t.names}'
return __c_type_to_python_type.get(t.names[0]) or 'ffi.CData'
return t.name
class PythonStubFuncDeclVisitor(c_ast.NodeVisitor):
def __init__(self):
self.sigs = {}
self.sources = {}
def get_source_snippet_lines(self, coord: pycparser.plyparser.Coord) -> Tuple[list[str], list[str]]:
if coord.file not in self.sources:
with open(coord.file, 'rt') as f:
self.sources[coord.file] = f.readlines()
source_lines = self.sources[coord.file]
ncomment_lines = len(list(itertools.takewhile(lambda i: re.search(r'^\s*(//|/\*)', source_lines[i]), range(coord.line - 2, -1, -1))))
comment_lines = [l.strip() for l in source_lines[coord.line - 1 - ncomment_lines:coord.line - 1]]
decl_lines = []
for line in source_lines[coord.line - 1:]:
decl_lines.append(line.rstrip())
if (';' in line) or ('{' in line): break
return (comment_lines, decl_lines)
def visit_Enum(self, node: Enum):
if node.values is not None:
for e in node.values.enumerators:
self.sigs[e.name] = f' @property\n def {e.name}(self) -> int: ...'
def visit_Typedef(self, node: Typedef):
pass
def visit_FuncDecl(self, node: FuncDecl):
ret_type = node.type
is_ptr = False
while isinstance(ret_type, PtrDecl):
ret_type = ret_type.type
is_ptr = True
fun_name = ret_type.declname
if fun_name.startswith('__'):
return
args = []
argnames = []
def gen_name(stem):
i = 1
while True:
new_name = stem if i == 1 else f'{stem}{i}'
if new_name not in argnames: return new_name
i += 1
for a in node.args.params:
if isinstance(a, EllipsisParam):
arg_name = gen_name('args')
argnames.append(arg_name)
args.append('*' + gen_name('args'))
elif format_type(a.type) == 'None':
continue
else:
arg_name = a.name or gen_name('arg')
argnames.append(arg_name)
args.append(f'{arg_name}: {format_type(a.type)}')
ret = format_type(ret_type if not is_ptr else node.type)
comment_lines, decl_lines = self.get_source_snippet_lines(node.coord)
lines = [f' def {fun_name}({", ".join(args)}) -> {ret}:']
if len(comment_lines) == 0 and len(decl_lines) == 1:
lines += [f' """{decl_lines[0]}"""']
else:
lines += [' """']
lines += [f' {c.lstrip("/* ")}' for c in comment_lines]
if len(comment_lines) > 0:
lines += ['']
lines += [f' {d}' for d in decl_lines]
lines += [' """']
lines += [' ...']
self.sigs[fun_name] = '\n'.join(lines)
def generate_stubs(header: str):
"""
Generates a .pyi Python stub file for the GGML API using C header files.
"""
v = PythonStubFuncDeclVisitor()
v.visit(CParser().parse(header, "<input>"))
keys = list(v.sigs.keys())
keys.sort()
return '\n'.join([
'# auto-generated file',
'import ggml.ffi as ffi',
'import numpy as np',
'class lib:',
*[v.sigs[k] for k in keys]
])
ggml/examples/python/test_tensor.py
+258
View File
@@ -0,0 +1,258 @@
import pytest
from pytest import raises
from ggml import lib, ffi
from ggml.utils import init, copy, numpy
import numpy as np
import numpy.testing as npt
@pytest.fixture()
def ctx():
print("setup")
yield init(mem_size=10*1024*1024)
print("teardown")
class TestNumPy:
# Single element
def test_set_get_single_i32(self, ctx):
i = lib.ggml_new_i32(ctx, 42)
assert lib.ggml_get_i32_1d(i, 0) == 42
assert numpy(i) == np.array([42], dtype=np.int32)
def test_set_get_single_f32(self, ctx):
i = lib.ggml_new_f32(ctx, 4.2)
epsilon = 0.000001 # Not sure why so large a difference??
pytest.approx(lib.ggml_get_f32_1d(i, 0), 4.2, epsilon)
pytest.approx(numpy(i), np.array([4.2], dtype=np.float32), epsilon)
def _test_copy_np_to_ggml(self, a: np.ndarray, t: ffi.CData):
a2 = a.copy() # Clone original
copy(a, t)
npt.assert_array_equal(numpy(t), a2)
# I32
def test_copy_np_to_ggml_1d_i32(self, ctx):
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_I32, 10)
a = np.arange(10, dtype=np.int32)
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_2d_i32(self, ctx):
t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_I32, 2, 3)
a = np.arange(2 * 3, dtype=np.int32).reshape((2, 3))
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_3d_i32(self, ctx):
t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_I32, 2, 3, 4)
a = np.arange(2 * 3 * 4, dtype=np.int32).reshape((2, 3, 4))
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_4d_i32(self, ctx):
t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_I32, 2, 3, 4, 5)
a = np.arange(2 * 3 * 4 * 5, dtype=np.int32).reshape((2, 3, 4, 5))
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_4d_n_i32(self, ctx):
dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash
pdims = ffi.new('int64_t[]', len(dims))
for i, d in enumerate(dims): pdims[i] = d
t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_I32, len(dims), pdims)
a = np.arange(np.prod(dims), dtype=np.int32).reshape(tuple(pdims))
self._test_copy_np_to_ggml(a, t)
# F32
def test_copy_np_to_ggml_1d_f32(self, ctx):
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)
a = np.arange(10, dtype=np.float32)
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_2d_f32(self, ctx):
t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, 2, 3)
a = np.arange(2 * 3, dtype=np.float32).reshape((2, 3))
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_3d_f32(self, ctx):
t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F32, 2, 3, 4)
a = np.arange(2 * 3 * 4, dtype=np.float32).reshape((2, 3, 4))
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_4d_f32(self, ctx):
t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F32, 2, 3, 4, 5)
a = np.arange(2 * 3 * 4 * 5, dtype=np.float32).reshape((2, 3, 4, 5))
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_4d_n_f32(self, ctx):
dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash
pdims = ffi.new('int64_t[]', len(dims))
for i, d in enumerate(dims): pdims[i] = d
t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_F32, len(dims), pdims)
a = np.arange(np.prod(dims), dtype=np.float32).reshape(tuple(pdims))
self._test_copy_np_to_ggml(a, t)
# F16
def test_copy_np_to_ggml_1d_f16(self, ctx):
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F16, 10)
a = np.arange(10, dtype=np.float16)
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_2d_f16(self, ctx):
t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F16, 2, 3)
a = np.arange(2 * 3, dtype=np.float16).reshape((2, 3))
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_3d_f16(self, ctx):
t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F16, 2, 3, 4)
a = np.arange(2 * 3 * 4, dtype=np.float16).reshape((2, 3, 4))
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_4d_f16(self, ctx):
t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F16, 2, 3, 4, 5)
a = np.arange(2 * 3 * 4 * 5, dtype=np.float16).reshape((2, 3, 4, 5))
self._test_copy_np_to_ggml(a, t)
def test_copy_np_to_ggml_4d_n_f16(self, ctx):
dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash
pdims = ffi.new('int64_t[]', len(dims))
for i, d in enumerate(dims): pdims[i] = d
t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_F16, len(dims), pdims)
a = np.arange(np.prod(dims), dtype=np.float16).reshape(tuple(pdims))
self._test_copy_np_to_ggml(a, t)
# Mismatching shapes
def test_copy_mismatching_shapes_1d(self, ctx):
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)
a = np.arange(10, dtype=np.float32)
copy(a, t) # OK
a = a.reshape((5, 2))
with raises(AssertionError): copy(a, t)
with raises(AssertionError): copy(t, a)
def test_copy_mismatching_shapes_2d(self, ctx):
t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, 2, 3)
a = np.arange(6, dtype=np.float32)
copy(a.reshape((2, 3)), t) # OK
a = a.reshape((3, 2))
with raises(AssertionError): copy(a, t)
with raises(AssertionError): copy(t, a)
def test_copy_mismatching_shapes_3d(self, ctx):
t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F32, 2, 3, 4)
a = np.arange(24, dtype=np.float32)
copy(a.reshape((2, 3, 4)), t) # OK
a = a.reshape((2, 4, 3))
with raises(AssertionError): copy(a, t)
with raises(AssertionError): copy(t, a)
def test_copy_mismatching_shapes_4d(self, ctx):
t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F32, 2, 3, 4, 5)
a = np.arange(24*5, dtype=np.float32)
copy(a.reshape((2, 3, 4, 5)), t) # OK
a = a.reshape((2, 3, 5, 4))
with raises(AssertionError): copy(a, t)
with raises(AssertionError): copy(t, a)
def test_copy_f16_to_f32(self, ctx):
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 1)
a = np.array([123.45], dtype=np.float16)
copy(a, t)
np.testing.assert_allclose(lib.ggml_get_f32_1d(t, 0), 123.45, rtol=1e-3)
def test_copy_f32_to_f16(self, ctx):
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F16, 1)
a = np.array([123.45], dtype=np.float32)
copy(a, t)
np.testing.assert_allclose(lib.ggml_get_f32_1d(t, 0), 123.45, rtol=1e-3)
def test_copy_f16_to_Q5_K(self, ctx):
n = 256
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
a = np.arange(n, dtype=np.float16)
copy(a, t)
np.testing.assert_allclose(a, numpy(t, allow_copy=True), rtol=0.05)
def test_copy_Q5_K_to_f16(self, ctx):
n = 256
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
copy(np.arange(n, dtype=np.float32), t)
a = np.arange(n, dtype=np.float16)
copy(t, a)
np.testing.assert_allclose(a, numpy(t, allow_copy=True), rtol=0.05)
def test_copy_i16_f32_mismatching_types(self, ctx):
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 1)
a = np.arange(1, dtype=np.int16)
with raises(NotImplementedError): copy(a, t)
with raises(NotImplementedError): copy(t, a)
class TestTensorCopy:
def test_copy_self(self, ctx):
t = lib.ggml_new_i32(ctx, 42)
copy(t, t)
assert lib.ggml_get_i32_1d(t, 0) == 42
def test_copy_1d(self, ctx):
t1 = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)
t2 = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)
a = np.arange(10, dtype=np.float32)
copy(a, t1)
copy(t1, t2)
assert np.allclose(a, numpy(t2))
assert np.allclose(numpy(t1), numpy(t2))
class TestGraph:
def test_add(self, ctx):
n = 256
ta = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)
tb = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)
tsum = lib.ggml_add(ctx, ta, tb)
assert tsum.type == lib.GGML_TYPE_F32
gf = ffi.new('struct ggml_cgraph*')
lib.ggml_build_forward_expand(gf, tsum)
a = np.arange(0, n, dtype=np.float32)
b = np.arange(n, 0, -1, dtype=np.float32)
copy(a, ta)
copy(b, tb)
lib.ggml_graph_compute_with_ctx(ctx, gf, 1)
assert np.allclose(numpy(tsum, allow_copy=True), a + b)
class TestQuantization:
def test_quantized_add(self, ctx):
n = 256
ta = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
tb = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)
tsum = lib.ggml_add(ctx, ta, tb)
assert tsum.type == lib.GGML_TYPE_Q5_K
gf = ffi.new('struct ggml_cgraph*')
lib.ggml_build_forward_expand(gf, tsum)
a = np.arange(0, n, dtype=np.float32)
b = np.arange(n, 0, -1, dtype=np.float32)
copy(a, ta)
copy(b, tb)
lib.ggml_graph_compute_with_ctx(ctx, gf, 1)
unquantized_sum = a + b
sum = numpy(tsum, allow_copy=True)
diff = np.linalg.norm(unquantized_sum - sum, np.inf)
assert diff > 4
assert diff < 5
ggml/examples/sam/CMakeLists.txt
+13
View File
@@ -0,0 +1,13 @@
#
# sam
set(TEST_TARGET sam)
add_executable(${TEST_TARGET} sam.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common)
#
# sam-quantize
#set(TEST_TARGET sam-quantize)
#add_executable(${TEST_TARGET} quantize.cpp)
#target_link_libraries(${TEST_TARGET} PRIVATE ggml common)
ggml/examples/sam/README.md
+95
View File
@@ -0,0 +1,95 @@
# SAM.cpp
Inference of Meta's [Segment Anything Model](https://github.com/facebookresearch/segment-anything/) in pure C/C++
## Description
The example currently supports only the [ViT-B SAM model checkpoint](https://huggingface.co/facebook/sam-vit-base).
## Next steps
- [X] Reduce memory usage by utilizing the new ggml-alloc
- [X] Remove redundant graph nodes
- [ ] Make inference faster
- [X] Fix the difference in output masks compared to the PyTorch implementation
- [X] Filter masks based on stability score
- [ ] Add support for user input
- [ ] Support F16 for heavy F32 ops
- [ ] Test quantization
- [X] Support bigger model checkpoints
- [ ] GPU support
## Quick start
Setup Python and build examples according to main README.
```bash
# Download PTH model
wget -P examples/sam/ https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth
# Convert PTH model to ggml
python examples/sam/convert-pth-to-ggml.py examples/sam/sam_vit_b_01ec64.pth examples/sam/ 1
# run inference
./bin/sam -t 16 -i ../examples/sam/example.jpg -m ../examples/sam/ggml-model-f16.bin
```
## Downloading and converting the model checkpoints
You can download a [model checkpoint](https://github.com/facebookresearch/segment-anything/tree/main#model-checkpoints) and convert it to `ggml` format using the script `convert-pth-to-ggml.py`:
## Example output on M2 Ultra
```
$ ▶ make -j sam && time ./bin/sam -t 8 -i img.jpg
[ 28%] Built target common
[ 71%] Built target ggml
[100%] Built target sam
main: seed = 1693224265
main: loaded image 'img.jpg' (680 x 453)
sam_image_preprocess: scale = 0.664062
main: preprocessed image (1024 x 1024)
sam_model_load: loading model from 'models/sam-vit-b/ggml-model-f16.bin' - please wait ...
sam_model_load: n_enc_state = 768
sam_model_load: n_enc_layer = 12
sam_model_load: n_enc_head = 12
sam_model_load: n_enc_out_chans = 256
sam_model_load: n_pt_embd = 4
sam_model_load: ftype = 1
sam_model_load: qntvr = 0
operator(): ggml ctx size = 202.32 MB
sam_model_load: ...................................... done
sam_model_load: model size = 185.05 MB / num tensors = 304
embd_img
dims: 64 64 256 1 f32
First & Last 10 elements:
-0.05117 -0.06408 -0.07154 -0.06991 -0.07212 -0.07690 -0.07508 -0.07281 -0.07383 -0.06779
0.01589 0.01775 0.02250 0.01675 0.01766 0.01661 0.01811 0.02051 0.02103 0.03382
sum: 12736.272313
Skipping mask 0 with iou 0.705935 below threshold 0.880000
Skipping mask 1 with iou 0.762136 below threshold 0.880000
Mask 2: iou = 0.947081, stability_score = 0.955437, bbox (371, 436), (144, 168)
main: load time = 51.28 ms
main: total time = 2047.49 ms
real 0m2.068s
user 0m16.343s
sys 0m0.214s
```
Input point is (414.375, 162.796875) (currently hardcoded)
Input image:
![llamas](https://user-images.githubusercontent.com/8558655/261301565-37b7bf4b-bf91-40cf-8ec1-1532316e1612.jpg)
Output mask (mask_out_2.png in build folder):
![mask_glasses](https://user-images.githubusercontent.com/8558655/263706800-47eeea30-1457-4c87-938b-8f11536c5aa7.png)
## References
- [ggml](https://github.com/ggerganov/ggml)
- [SAM](https://segment-anything.com/)
- [SAM demo](https://segment-anything.com/demo)
ggml/examples/sam/convert-pth-to-ggml.py
+147
View File
@@ -0,0 +1,147 @@
# Convert a SAM model checkpoint to a ggml compatible file
#
import sys
import torch
import struct
import numpy as np
if len(sys.argv) < 3:
print("Usage: convert-pth-to-ggml.py file-model dir-output [ftype]\n")
print(" ftype == 0 -> float32")
print(" ftype == 1 -> float16")
sys.exit(1)
# output in the same directory as the model
fname_model = sys.argv[1]
dir_out = sys.argv[2]
fname_out = dir_out + "/ggml-model.bin"
# possible data types
# ftype == 0 -> float32
# ftype == 1 -> float16
#
# map from ftype to string
ftype_str = ["f32", "f16"]
ftype = 1
if len(sys.argv) > 3:
ftype = int(sys.argv[3])
if ftype < 0 or ftype > 1:
print("Invalid ftype: " + str(ftype))
sys.exit(1)
fname_out = fname_out.replace(".bin", "-" + ftype_str[ftype] + ".bin")
# Default params are set to sam_vit_b checkpoint
n_enc_state = 768
n_enc_layers = 12
n_enc_heads = 12
n_enc_out_chans = 256
n_pt_embd = 4
model = torch.load(fname_model, map_location="cpu")
for k, v in model.items():
print(k, v.shape)
if k == "image_encoder.blocks.0.norm1.weight":
n_enc_state = v.shape[0]
if n_enc_state == 1024: # sam_vit_l
n_enc_layers = 24
n_enc_heads = 16
elif n_enc_state == 1280: # sam_vit_h
n_enc_layers = 32
n_enc_heads = 16
hparams = {
"n_enc_state": n_enc_state,
"n_enc_layers": n_enc_layers,
"n_enc_heads": n_enc_heads,
"n_enc_out_chans": n_enc_out_chans,
"n_pt_embd": n_pt_embd,
}
print(hparams)
for k, v in model.items():
print(k, v.shape)
#exit()
#code.interact(local=locals())
fout = open(fname_out, "wb")
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
fout.write(struct.pack("i", hparams["n_enc_state"]))
fout.write(struct.pack("i", hparams["n_enc_layers"]))
fout.write(struct.pack("i", hparams["n_enc_heads"]))
fout.write(struct.pack("i", hparams["n_enc_out_chans"]))
fout.write(struct.pack("i", hparams["n_pt_embd"]))
fout.write(struct.pack("i", ftype))
for k, v in model.items():
name = k
shape = v.shape
if name[:19] == "prompt_encoder.mask":
continue
print("Processing variable: " + name + " with shape: ", shape, " and type: ", v.dtype)
#data = tf.train.load_variable(dir_model, name).squeeze()
#data = v.numpy().squeeze()
data = v.numpy()
n_dims = len(data.shape)
# for efficiency - transpose some matrices
# "model/h.*/attn/c_attn/w"
# "model/h.*/attn/c_proj/w"
# "model/h.*/mlp/c_fc/w"
# "model/h.*/mlp/c_proj/w"
#if name[-14:] == "/attn/c_attn/w" or \
# name[-14:] == "/attn/c_proj/w" or \
# name[-11:] == "/mlp/c_fc/w" or \
# name[-13:] == "/mlp/c_proj/w":
# print(" Transposing")
# data = data.transpose()
dshape = data.shape
# default type is fp16
ftype_cur = 1
if ftype == 0 or n_dims == 1 or \
name == "image_encoder.pos_embed" or \
name.startswith("prompt_encoder") or \
name.startswith("mask_decoder.iou_token") or \
name.startswith("mask_decoder.mask_tokens"):
print(" Converting to float32")
data = data.astype(np.float32)
ftype_cur = 0
else:
print(" Converting to float16")
data = data.astype(np.float16)
# reshape the 1D bias into a 4D tensor so we can use ggml_repeat
# keep it in F32 since the data is small
if name == "image_encoder.patch_embed.proj.bias":
data = data.reshape(1, data.shape[0], 1, 1)
n_dims = len(data.shape)
dshape = data.shape
print(" New shape: ", dshape)
# header
str = name.encode('utf-8')
fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
for i in range(n_dims):
fout.write(struct.pack("i", dshape[n_dims - 1 - i]))
fout.write(str)
# data
data.tofile(fout)
fout.close()
print("Done. Output file: " + fname_out)
print("")
ggml/examples/sam/example.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 77 KiB

ggml/examples/sam/sam.cpp
+2370
View File
File diff suppressed because it is too large Load Diff
ggml/examples/simple/CMakeLists.txt
+21
View File
@@ -0,0 +1,21 @@
#
# simple-ctx
set(TEST_TARGET simple-ctx)
add_executable(${TEST_TARGET} simple-ctx.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
#
# simple-backend
set(TEST_TARGET simple-backend)
add_executable(${TEST_TARGET} simple-backend.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
if (GGML_CUDA)
add_compile_definitions(GGML_USE_CUDA)
endif()
if (GGML_METAL)
add_compile_definitions(GGML_USE_METAL)
endif()
ggml/examples/simple/README.md
+61
View File
@@ -0,0 +1,61 @@
## Simple
This example simply performs a matrix multiplication, solely for the purpose of demonstrating a basic usage of ggml and backend handling. The code is commented to help understand what each part does.
Traditional matrix multiplication goes like this (multiply row-by-column):
$$
A \times B = C
$$
$$
\begin{bmatrix}
2 & 8 \\
5 & 1 \\
4 & 2 \\
8 & 6 \\
\end{bmatrix}
\times
\begin{bmatrix}
10 & 9 & 5 \\
5 & 9 & 4 \\
\end{bmatrix}
\=
\begin{bmatrix}
60 & 90 & 42 \\
55 & 54 & 29 \\
50 & 54 & 28 \\
110 & 126 & 64 \\
\end{bmatrix}
$$
In `ggml`, we pass the matrix $B$ in transposed form and multiply row-by-row. The result $C$ is also transposed:
$$
ggml\\_mul\\_mat(A, B^T) = C^T
$$
$$
ggml\\_mul\\_mat(
\begin{bmatrix}
2 & 8 \\
5 & 1 \\
4 & 2 \\
8 & 6 \\
\end{bmatrix}
,
\begin{bmatrix}
10 & 5 \\
9 & 9 \\
5 & 4 \\
\end{bmatrix}
)
\=
\begin{bmatrix}
60 & 55 & 50 & 110 \\
90 & 54 & 54 & 126 \\
42 & 29 & 28 & 64 \\
\end{bmatrix}
$$
The `simple-ctx` doesn't support gpu acceleration. `simple-backend` demonstrates how to use other backends like CUDA and Metal.
ggml/examples/simple/simple-backend.cpp
+153
View File
@@ -0,0 +1,153 @@
#include "ggml.h"
#include "ggml-backend.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <vector>
static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
(void) level;
(void) user_data;
fputs(text, stderr);
fflush(stderr);
}
// This is a simple model with two tensors a and b
struct simple_model {
struct ggml_tensor * a {};
struct ggml_tensor * b {};
// the backend to perform the computation (CPU, CUDA, METAL)
ggml_backend_t backend {};
ggml_backend_t cpu_backend {};
ggml_backend_sched_t sched {};
// storage for the graph and tensors
std::vector<uint8_t> buf;
};
// initialize data of matrices to perform matrix multiplication
const int rows_A = 4, cols_A = 2;
float matrix_A[rows_A * cols_A] = {
2, 8,
5, 1,
4, 2,
8, 6
};
const int rows_B = 3, cols_B = 2;
/* Transpose([
10, 9, 5,
5, 9, 4
]) 2 rows, 3 cols */
float matrix_B[rows_B * cols_B] = {
10, 5,
9, 9,
5, 4
};
// initialize the tensors of the model in this case two matrices 2x2
void init_model(simple_model & model) {
ggml_log_set(ggml_log_callback_default, nullptr);
ggml_backend_load_all();
model.backend = ggml_backend_init_best();
model.cpu_backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
ggml_backend_t backends[2] = { model.backend, model.cpu_backend };
model.sched = ggml_backend_sched_new(backends, nullptr, 2, GGML_DEFAULT_GRAPH_SIZE, false, true);
}
// build the compute graph to perform a matrix multiplication
struct ggml_cgraph * build_graph(simple_model& model) {
size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
model.buf.resize(buf_size);
struct ggml_init_params params0 = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ model.buf.data(),
/*.no_alloc =*/ true, // the tensors will be allocated later
};
// create a context to build the graph
struct ggml_context * ctx = ggml_init(params0);
struct ggml_cgraph * gf = ggml_new_graph(ctx);
// create tensors
model.a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cols_A, rows_A);
model.b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cols_B, rows_B);
// result = a*b^T
struct ggml_tensor * result = ggml_mul_mat(ctx, model.a, model.b);
// build operations nodes
ggml_build_forward_expand(gf, result);
ggml_free(ctx);
return gf;
}
// compute with backend
struct ggml_tensor * compute(simple_model & model, struct ggml_cgraph * gf) {
ggml_backend_sched_reset(model.sched);
ggml_backend_sched_alloc_graph(model.sched, gf);
// load data from cpu memory to backend buffer
ggml_backend_tensor_set(model.a, matrix_A, 0, ggml_nbytes(model.a));
ggml_backend_tensor_set(model.b, matrix_B, 0, ggml_nbytes(model.b));
// compute the graph
ggml_backend_sched_graph_compute(model.sched, gf);
// in this case, the output tensor is the last one in the graph
return ggml_graph_node(gf, -1);
}
int main(void) {
ggml_time_init();
simple_model model;
init_model(model);
struct ggml_cgraph * gf = build_graph(model);
// perform computation
struct ggml_tensor * result = compute(model, gf);
// create a array to print result
std::vector<float> out_data(ggml_nelements(result));
// bring the data from the backend memory
ggml_backend_tensor_get(result, out_data.data(), 0, ggml_nbytes(result));
// expected result:
// [ 60.00 55.00 50.00 110.00
// 90.00 54.00 54.00 126.00
// 42.00 29.00 28.00 64.00 ]
printf("mul mat (%d x %d) (transposed result):\n[", (int) result->ne[0], (int) result->ne[1]);
for (int j = 0; j < result->ne[1] /* rows */; j++) {
if (j > 0) {
printf("\n");
}
for (int i = 0; i < result->ne[0] /* cols */; i++) {
printf(" %.2f", out_data[j * result->ne[0] + i]);
}
}
printf(" ]\n");
// release backend memory and free backend
ggml_backend_sched_free(model.sched);
ggml_backend_free(model.backend);
ggml_backend_free(model.cpu_backend);
return 0;
}
ggml/examples/simple/simple-ctx.cpp
+127
View File
@@ -0,0 +1,127 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
// This is a simple model with two tensors a and b
struct simple_model {
struct ggml_tensor * a;
struct ggml_tensor * b;
// the context to define the tensor information (dimensions, size, memory data)
struct ggml_context * ctx;
};
// initialize the tensors of the model in this case two matrices 2x2
void load_model(simple_model & model, float * a, float * b, int rows_A, int cols_A, int rows_B, int cols_B) {
size_t ctx_size = 0;
{
ctx_size += rows_A * cols_A * ggml_type_size(GGML_TYPE_F32); // tensor a
ctx_size += rows_B * cols_B * ggml_type_size(GGML_TYPE_F32); // tensor b
ctx_size += 2 * ggml_tensor_overhead(), // tensors
ctx_size += ggml_graph_overhead(); // compute graph
ctx_size += 1024; // some overhead
}
struct ggml_init_params params {
/*.mem_size =*/ ctx_size,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ false, // NOTE: this should be false when using the legacy API
};
// create context
model.ctx = ggml_init(params);
// create tensors
model.a = ggml_new_tensor_2d(model.ctx, GGML_TYPE_F32, cols_A, rows_A);
model.b = ggml_new_tensor_2d(model.ctx, GGML_TYPE_F32, cols_B, rows_B);
memcpy(model.a->data, a, ggml_nbytes(model.a));
memcpy(model.b->data, b, ggml_nbytes(model.b));
}
// build the compute graph to perform a matrix multiplication
struct ggml_cgraph * build_graph(const simple_model& model) {
struct ggml_cgraph * gf = ggml_new_graph(model.ctx);
// result = a*b^T
struct ggml_tensor * result = ggml_mul_mat(model.ctx, model.a, model.b);
ggml_build_forward_expand(gf, result);
return gf;
}
// compute with backend
struct ggml_tensor * compute(const simple_model & model) {
struct ggml_cgraph * gf = build_graph(model);
int n_threads = 1; // number of threads to perform some operations with multi-threading
ggml_graph_compute_with_ctx(model.ctx, gf, n_threads);
// in this case, the output tensor is the last one in the graph
return ggml_graph_node(gf, -1);
}
int main(void) {
ggml_time_init();
// initialize data of matrices to perform matrix multiplication
const int rows_A = 4, cols_A = 2;
float matrix_A[rows_A * cols_A] = {
2, 8,
5, 1,
4, 2,
8, 6
};
const int rows_B = 3, cols_B = 2;
/* Transpose([
10, 9, 5,
5, 9, 4
]) 2 rows, 3 cols */
float matrix_B[rows_B * cols_B] = {
10, 5,
9, 9,
5, 4
};
simple_model model;
load_model(model, matrix_A, matrix_B, rows_A, cols_A, rows_B, cols_B);
// perform computation in cpu
struct ggml_tensor * result = compute(model);
// get the result data pointer as a float array to print
std::vector<float> out_data(ggml_nelements(result));
memcpy(out_data.data(), result->data, ggml_nbytes(result));
// expected result:
// [ 60.00 55.00 50.00 110.00
// 90.00 54.00 54.00 126.00
// 42.00 29.00 28.00 64.00 ]
printf("mul mat (%d x %d) (transposed result):\n[", (int) result->ne[0], (int) result->ne[1]);
for (int j = 0; j < result->ne[1] /* rows */; j++) {
if (j > 0) {
printf("\n");
}
for (int i = 0; i < result->ne[0] /* cols */; i++) {
printf(" %.2f", out_data[j * result->ne[0] + i]);
}
}
printf(" ]\n");
// free memory
ggml_free(model.ctx);
return 0;
}
ggml/examples/stb_image.h
+7987
View File
File diff suppressed because it is too large Load Diff
ggml/examples/stb_image_write.h
+1724
View File
File diff suppressed because it is too large Load Diff
ggml/examples/test-cmake/CMakeLists.txt
+10
View File
@@ -0,0 +1,10 @@
cmake_minimum_required(VERSION 3.14)
project(ggml-simple)
set(CMAKE_CXX_STANDARD 17)
find_package(ggml CONFIG REQUIRED)
set(TEST_TARGET test-cmake)
add_executable(test-cmake test-cmake.cpp)
target_link_libraries(test-cmake PRIVATE ggml::ggml)
ggml/examples/test-cmake/README.md
+3
View File
@@ -0,0 +1,3 @@
## cmake-test
This directory can be built as a separate project with an installed ggml.
ggml/examples/test-cmake/test-cmake.cpp
+6
View File
@@ -0,0 +1,6 @@
#include "ggml-backend.h"
int main(void) {
ggml_backend_load_all();
return 0;
}
ggml/examples/yolo/CMakeLists.txt
+6
View File
@@ -0,0 +1,6 @@
#
# yolov3-tiny
set(TEST_TARGET yolov3-tiny)
add_executable(${TEST_TARGET} yolov3-tiny.cpp yolo-image.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common)
ggml/examples/yolo/README.md
+59
View File
@@ -0,0 +1,59 @@
This example shows how to implement YOLO object detection with ggml using pretrained model.
# YOLOv3-tiny
Download the model weights:
```bash
$ wget https://pjreddie.com/media/files/yolov3-tiny.weights
$ sha1sum yolov3-tiny.weights
40f3c11883bef62fd850213bc14266632ed4414f yolov3-tiny.weights
```
Convert the weights to GGUF format:
```bash
$ ./convert-yolov3-tiny.py yolov3-tiny.weights
yolov3-tiny.weights converted to yolov3-tiny.gguf
```
Alternatively, you can download the converted model from [HuggingFace](https://huggingface.co/rgerganov/yolo-gguf/resolve/main/yolov3-tiny.gguf)
Object detection:
```bash
$ wget https://raw.githubusercontent.com/pjreddie/darknet/master/data/dog.jpg
$ ./yolov3-tiny -m yolov3-tiny.gguf -i dog.jpg
load_model: using CUDA backend
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
ggml_cuda_init: found 1 CUDA devices:
Device 0: NVIDIA T1200 Laptop GPU, compute capability 7.5, VMM: yes
Layer 0 output shape: 416 x 416 x 16 x 1
Layer 1 output shape: 208 x 208 x 16 x 1
Layer 2 output shape: 208 x 208 x 32 x 1
Layer 3 output shape: 104 x 104 x 32 x 1
Layer 4 output shape: 104 x 104 x 64 x 1
Layer 5 output shape: 52 x 52 x 64 x 1
Layer 6 output shape: 52 x 52 x 128 x 1
Layer 7 output shape: 26 x 26 x 128 x 1
Layer 8 output shape: 26 x 26 x 256 x 1
Layer 9 output shape: 13 x 13 x 256 x 1
Layer 10 output shape: 13 x 13 x 512 x 1
Layer 11 output shape: 13 x 13 x 512 x 1
Layer 12 output shape: 13 x 13 x 1024 x 1
Layer 13 output shape: 13 x 13 x 256 x 1
Layer 14 output shape: 13 x 13 x 512 x 1
Layer 15 output shape: 13 x 13 x 255 x 1
Layer 18 output shape: 13 x 13 x 128 x 1
Layer 19 output shape: 26 x 26 x 128 x 1
Layer 20 output shape: 26 x 26 x 384 x 1
Layer 21 output shape: 26 x 26 x 256 x 1
Layer 22 output shape: 26 x 26 x 255 x 1
dog: 57%
car: 52%
truck: 56%
car: 62%
bicycle: 59%
Detected objects saved in 'predictions.jpg' (time: 0.057000 sec.)
```
ggml/examples/yolo/convert-yolov3-tiny.py
Executable
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env python3
import sys
import gguf
import numpy as np
def save_conv2d_layer(f, gguf_writer, prefix, inp_c, filters, size, batch_normalize=True):
biases = np.fromfile(f, dtype=np.float32, count=filters)
gguf_writer.add_tensor(prefix + "_biases", biases, raw_shape=(1, filters, 1, 1))
if batch_normalize:
scales = np.fromfile(f, dtype=np.float32, count=filters)
gguf_writer.add_tensor(prefix + "_scales", scales, raw_shape=(1, filters, 1, 1))
rolling_mean = np.fromfile(f, dtype=np.float32, count=filters)
gguf_writer.add_tensor(prefix + "_rolling_mean", rolling_mean, raw_shape=(1, filters, 1, 1))
rolling_variance = np.fromfile(f, dtype=np.float32, count=filters)
gguf_writer.add_tensor(prefix + "_rolling_variance", rolling_variance, raw_shape=(1, filters, 1, 1))
weights_count = filters * inp_c * size * size
l0_weights = np.fromfile(f, dtype=np.float32, count=weights_count)
## ggml doesn't support f32 convolution yet, use f16 instead
l0_weights = l0_weights.astype(np.float16)
gguf_writer.add_tensor(prefix + "_weights", l0_weights, raw_shape=(filters, inp_c, size, size))
if __name__ == '__main__':
if len(sys.argv) != 2:
print("Usage: %s <yolov3-tiny.weights>" % sys.argv[0])
sys.exit(1)
outfile = 'yolov3-tiny.gguf'
gguf_writer = gguf.GGUFWriter(outfile, 'yolov3-tiny')
f = open(sys.argv[1], 'rb')
f.read(20) # skip header
save_conv2d_layer(f, gguf_writer, "l0", 3, 16, 3)
save_conv2d_layer(f, gguf_writer, "l1", 16, 32, 3)
save_conv2d_layer(f, gguf_writer, "l2", 32, 64, 3)
save_conv2d_layer(f, gguf_writer, "l3", 64, 128, 3)
save_conv2d_layer(f, gguf_writer, "l4", 128, 256, 3)
save_conv2d_layer(f, gguf_writer, "l5", 256, 512, 3)
save_conv2d_layer(f, gguf_writer, "l6", 512, 1024, 3)
save_conv2d_layer(f, gguf_writer, "l7", 1024, 256, 1)
save_conv2d_layer(f, gguf_writer, "l8", 256, 512, 3)
save_conv2d_layer(f, gguf_writer, "l9", 512, 255, 1, batch_normalize=False)
save_conv2d_layer(f, gguf_writer, "l10", 256, 128, 1)
save_conv2d_layer(f, gguf_writer, "l11", 384, 256, 3)
save_conv2d_layer(f, gguf_writer, "l12", 256, 255, 1, batch_normalize=False)
f.close()
gguf_writer.write_header_to_file()
gguf_writer.write_kv_data_to_file()
gguf_writer.write_tensors_to_file()
gguf_writer.close()
print("{} converted to {}".format(sys.argv[1], outfile))
ggml/examples/yolo/data/coco.names
+80
View File
@@ -0,0 +1,80 @@
person
bicycle
car
motorbike
aeroplane
bus
train
truck
boat
traffic light
fire hydrant
stop sign
parking meter
bench
bird
cat
dog
horse
sheep
cow
elephant
bear
zebra
giraffe
backpack
umbrella
handbag
tie
suitcase
frisbee
skis
snowboard
sports ball
kite
baseball bat
baseball glove
skateboard
surfboard
tennis racket
bottle
wine glass
cup
fork
knife
spoon
bowl
banana
apple
sandwich
orange
broccoli
carrot
hot dog
pizza
donut
cake
chair
sofa
pottedplant
bed
diningtable
toilet
tvmonitor
laptop
mouse
remote
keyboard
cell phone
microwave
oven
toaster
sink
refrigerator
book
clock
vase
scissors
teddy bear
hair drier
toothbrush
ggml/examples/yolo/data/labels/100_0.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 320 B

ggml/examples/yolo/data/labels/100_1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 377 B

ggml/examples/yolo/data/labels/100_2.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 451 B

ggml/examples/yolo/data/labels/100_3.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 508 B

ggml/examples/yolo/data/labels/100_4.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 577 B

ggml/examples/yolo/data/labels/100_5.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 631 B

ggml/examples/yolo/data/labels/100_6.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 697 B

ggml/examples/yolo/data/labels/100_7.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 753 B

ggml/examples/yolo/data/labels/101_0.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 321 B

ggml/examples/yolo/data/labels/101_1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 388 B

ggml/examples/yolo/data/labels/101_2.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 458 B

ggml/examples/yolo/data/labels/101_3.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 514 B

ggml/examples/yolo/data/labels/101_4.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 581 B

ggml/examples/yolo/data/labels/101_5.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 654 B

ggml/examples/yolo/data/labels/101_6.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 726 B

ggml/examples/yolo/data/labels/101_7.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 804 B

ggml/examples/yolo/data/labels/102_0.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 305 B

Some files were not shown because too many files have changed in this diff Show More