option(WHISPER_NO_FMA "whisper: disable FMA" OFF)
option(WHISPER_NO_F16C "whisper: disable F16c" OFF)
+option(WHISPER_OPENVINO "whisper: support for OpenVINO" OFF)
+
if (APPLE)
option(WHISPER_NO_ACCELERATE "whisper: disable Accelerate framework" OFF)
option(WHISPER_COREML "whisper: enable Core ML framework" OFF)
endif()
endif()
+if( WHISPER_OPENVINO )
+ find_package(OpenVINO REQUIRED COMPONENTS Runtime)
+endif()
+
# compiler flags
if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
)
endif()
+if (WHISPER_OPENVINO)
+ set(TARGET whisper.openvino)
+
+ add_library(${TARGET} OBJECT
+ openvino/whisper-openvino-encoder.h
+ openvino/whisper-openvino-encoder.cpp
+ )
+
+ target_include_directories(${TARGET} PUBLIC
+ .
+ )
+
+ set_property(TARGET ${TARGET} PROPERTY POSITION_INDEPENDENT_CODE ON)
+ set(WHISPER_EXTRA_FLAGS ${WHISPER_EXTRA_FLAGS} -DWHISPER_USE_OPENVINO)
+
+ target_link_libraries(${TARGET} PRIVATE openvino::runtime)
+endif()
+
#
# whisper - this is the main library of the project
#
target_link_libraries(${TARGET} PRIVATE whisper.coreml)
endif()
+if (WHISPER_OPENVINO)
+ target_link_libraries(${TARGET} PRIVATE whisper.openvino)
+endif()
+
if (MSVC)
target_link_libraries(${TARGET} PRIVATE ${WHISPER_EXTRA_LIBS} ${CMAKE_THREAD_LIBS_INIT})
// [TDRZ] speaker turn string
std::string tdrz_speaker_turn = " [SPEAKER_TURN]"; // TODO: set from command line
+ std::string openvino_encode_device = "CPU";
+
std::vector<std::string> fname_inp = {};
std::vector<std::string> fname_out = {};
};
else if ( arg == "--prompt") { params.prompt = argv[++i]; }
else if (arg == "-m" || arg == "--model") { params.model = argv[++i]; }
else if (arg == "-f" || arg == "--file") { params.fname_inp.emplace_back(argv[++i]); }
+ else if (arg == "-oved" || arg == "--ov-e-device") { params.openvino_encode_device = argv[++i]; }
else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
whisper_print_usage(argc, argv, params);
fprintf(stderr, " --prompt PROMPT [%-7s] initial prompt\n", params.prompt.c_str());
fprintf(stderr, " -m FNAME, --model FNAME [%-7s] model path\n", params.model.c_str());
fprintf(stderr, " -f FNAME, --file FNAME [%-7s] input WAV file path\n", "");
+ fprintf(stderr, " -oved D, --ov-e-device DNAME [%-7s] the OpenVINO device used for encode inference\n", params.openvino_encode_device.c_str());
fprintf(stderr, "\n");
}
return 3;
}
+ // initialize openvino encoder. This has no effect on whisper.cpp builds that don't have OpenVINO configured.
+ whisper_ctx_init_openvino_encoder(ctx, nullptr, params.openvino_encode_device.c_str(), nullptr);
+
for (int f = 0; f < (int) params.fname_inp.size(); ++f) {
const auto fname_inp = params.fname_inp[f];
const auto fname_out = f < (int) params.fname_out.size() && !params.fname_out[f].empty() ? params.fname_out[f] : params.fname_inp[f];
--- /dev/null
+import argparse
+import torch
+from whisper import load_model
+import os
+from openvino.tools import mo
+from openvino.runtime import serialize
+import shutil
+
+def convert_encoder(hparams, encoder, mname):
+ encoder.eval()
+
+ mel = torch.zeros((1, 80, 3000))
+
+ onnx_folder=os.path.join(os.path.dirname(__file__),"onnx_encoder")
+
+ #create a directory to store the onnx model, and other collateral that is saved during onnx export procedure
+ if not os.path.isdir(onnx_folder):
+ os.makedirs(onnx_folder)
+
+ onnx_path = os.path.join(onnx_folder, "whisper_encoder.onnx")
+
+ torch.onnx.export(
+ encoder,
+ mel,
+ onnx_path,
+ input_names=["mel"],
+ output_names=["output_features"]
+ )
+
+ # use model optimizer to convert onnx to OpenVINO IR format
+ encoder_model = mo.convert_model(onnx_path, compress_to_fp16=True)
+ serialize(encoder_model, xml_path='ggml-' + mname + '-encoder-openvino.xml')
+
+ #cleanup
+ if os.path.isdir(onnx_folder):
+ shutil.rmtree(onnx_folder)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--model", type=str, help="model to convert (e.g. tiny, tiny.en, base, base.en, small, small.en, medium, medium.en, large, large-v1)", required=True)
+ args = parser.parse_args()
+
+ if args.model not in ["tiny", "tiny.en", "base", "base.en", "small", "small.en", "medium", "medium.en", "large", "large-v1"]:
+ raise ValueError("Invalid model name")
+
+ whisper = load_model(args.model).cpu()
+ hparams = whisper.dims
+
+ encoder = whisper.encoder
+
+ # Convert encoder to onnx
+ convert_encoder(hparams, encoder, args.model)
--- /dev/null
+openvino-dev[pytorch,onnx]
+openai-whisper
\ No newline at end of file
--- /dev/null
+#include "openvino/whisper-openvino-encoder.h"
+#include "ggml.h"
+#include <openvino/openvino.hpp>
+#include <iostream>
+
+struct whisper_openvino_context {
+ ov::InferRequest inferRequest;
+};
+
+struct whisper_openvino_context * whisper_openvino_init(const char* path_model,
+ const char* device,
+ const char* cache_dir)
+{
+ if (!path_model || !device) {
+ fprintf(stderr, "%s: path_model and/or device is null\n", __func__);
+ return nullptr;
+ }
+
+ fprintf(stderr, "%s: path_model = %s, device = %s, cache_dir = %s\n",
+ __func__, path_model, device, cache_dir ? cache_dir : "(not set)");
+
+ whisper_openvino_context *context = new whisper_openvino_context;
+ try {
+ ov::Core core;
+
+ if (cache_dir) {
+ // enables caching of device-specific 'blobs' during core.compile_model
+ // routine. This speeds up calls to compile_model for successive runs.
+ core.set_property(ov::cache_dir(cache_dir));
+ }
+
+ //Read the OpenVINO encoder IR (.xml/.bin) from disk, producing an ov::Model object.
+ std::shared_ptr<ov::Model> model = core.read_model(path_model);
+
+ // Produce a compiled-model object, given the device ("CPU", "GPU", etc.)
+ auto compiledModel = core.compile_model(model, device);
+
+ // From the compiled model object, create an infer request. This is the thing that we
+ // we will use later on to trigger inference execution.
+ context->inferRequest = compiledModel.create_infer_request();
+ }
+ catch (const std::exception& error) {
+ std::cout << "in openvino encoder compile routine: exception: " << error.what() << std::endl;
+ delete context;
+ context = nullptr;
+ }
+
+ return context;
+}
+
+void whisper_openvino_free(struct whisper_openvino_context * ctx) {
+ if( ctx ) {
+ delete ctx;
+ }
+}
+
+int whisper_openvino_encode(
+ whisper_openvino_context* ctx,
+ ggml_tensor* mel,
+ ggml_tensor* out) {
+
+ if (!ctx || !mel || !out) {
+ fprintf(stderr, "%s: Error! ctx / mel / out is null\n", __func__);
+ return 0;
+ }
+
+ if (mel->n_dims != 2) {
+ fprintf(stderr, "%s: Error! mel ggml_tensor expected to have n_dims=2, but it has n_dims=%d\n",
+ __func__, mel->n_dims);
+ return 0;
+ }
+
+ if (out->n_dims != 2) {
+ fprintf(stderr, "%s: Error! out ggml_tensor expected to have n_dims=2, but it has n_dims=%d\n",
+ __func__, out->n_dims);
+ return 0;
+ }
+
+ try {
+
+ //wrap the passed-in mel ggml_tensor as an OpenVINO Tensor object, and set as input tensor to infer request
+ {
+ // note, we populate shape & stride dimensions in opposite order from how they are listed in ne / nb arrays
+ ov::Shape input_shape = { 1, (unsigned long long)mel->ne[1], (unsigned long long)mel->ne[0] };
+ ov::Strides input_strides = { mel->nb[2], mel->nb[1], mel->nb[0] };
+ ov::Tensor input_tensor(ov::element::f32, input_shape, mel->data, input_strides);
+ ctx->inferRequest.set_input_tensor(input_tensor);
+ }
+
+ //wrap the passed-in out ggml_tensor as an OpenVINO Tensor object, and set as output tensor to infer request
+ {
+ // note, we populate shape & stride dimensions in opposite order from how they are listed in ne / nb arrays
+ ov::Shape output_shape = { 1, (unsigned long long)out->ne[1], (unsigned long long)out->ne[0] };
+ ov::Strides output_strides = { out->nb[2], out->nb[1], out->nb[0] };
+ ov::Tensor out_tensor(ov::element::f32, output_shape, out->data, output_strides);
+ ctx->inferRequest.set_output_tensor(out_tensor);
+ }
+
+ //run inference
+ ctx->inferRequest.infer();
+ }
+ catch (const std::exception& error) {
+ std::cout << "in openvino encode inference execution routine: exception: " << error.what() << std::endl;
+ return 0;
+ }
+
+ return 1;
+}
\ No newline at end of file
--- /dev/null
+// Wrapper of the OpenVINO Whisper Encoder model
+//
+
+#if __cplusplus
+extern "C" {
+#endif
+
+struct whisper_openvino_context;
+
+// initialize openvino encoder, given path to model xml, device ("CPU", "GPU", etc.), and
+// path to cache_dir. Returns null upon failure.
+struct whisper_openvino_context * whisper_openvino_init(const char * path_model,
+ const char * device,
+ const char * cache_dir);
+
+// clean up a ctx previously returned from whisper_openvino_init()
+void whisper_openvino_free(struct whisper_openvino_context * ctx);
+
+struct ggml_tensor;
+
+// Perform encode using OpenVINO.
+// Returns 1 on success
+// Returns 0 on failure
+int whisper_openvino_encode(
+ whisper_openvino_context* ctx,
+ ggml_tensor* mel,
+ ggml_tensor* out);
+
+#if __cplusplus
+}
+#endif
#include "coreml/whisper-encoder.h"
#endif
+#if WHISPER_USE_OPENVINO
+#include "openvino/whisper-openvino-encoder.h"
+#endif
+
#include "ggml.h"
#include <algorithm>
whisper_coreml_context * ctx_coreml = nullptr;
#endif
+#ifdef WHISPER_USE_OPENVINO
+ whisper_openvino_context * ctx_openvino = nullptr;
+#endif
+
// [EXPERIMENTAL] token-level timestamps data
int64_t t_beg = 0;
int64_t t_last = 0;
const bool use_coreml = wstate.ctx_coreml != nullptr;
#endif
- if (!use_coreml) {
+#ifndef WHISPER_USE_OPENVINO
+ const bool use_openvino = false;
+#else
+ const bool use_openvino = wstate.ctx_openvino != nullptr;
+#endif
+
+ if (!use_coreml && !use_openvino) {
// convolution + gelu
{
wstate.use_buf(ctx0, 1);
}
}
#ifdef WHISPER_USE_COREML
- else
- {
+ else if (use_coreml) {
wstate.use_buf(ctx0, -1);
cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_state, n_ctx);
whisper_coreml_encode(wstate.ctx_coreml, (float *) mel->data, (float *) cur->data);
}
#endif
+#ifdef WHISPER_USE_OPENVINO
+ else if (use_openvino) {
+ wstate.use_buf(ctx0, -1);
+
+ cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_state, n_ctx);
+
+ if (!whisper_openvino_encode(wstate.ctx_openvino, mel, cur)) {
+ return false;
+ }
+ }
+#endif
// cur
//{
}
#endif
+#ifdef WHISPER_USE_OPENVINO
+// replace .bin with-encoder-openvino.xml
+static std::string whisper_get_openvino_path_encoder(std::string path_bin) {
+ auto pos = path_bin.rfind('.');
+ if (pos != std::string::npos) {
+ path_bin = path_bin.substr(0, pos);
+ }
+
+ path_bin += "-encoder-openvino.xml";
+
+ return path_bin;
+}
+
+static std::string whisper_get_openvino_path_cache(std::string path_bin) {
+ auto pos = path_bin.rfind('.');
+ if (pos != std::string::npos) {
+ path_bin = path_bin.substr(0, pos);
+ }
+
+ path_bin += "-encoder-openvino-cache";
+
+ return path_bin;
+}
+#endif
+
struct whisper_state * whisper_init_state(whisper_context * ctx) {
whisper_state * state = new whisper_state;
return state;
}
+int whisper_ctx_init_openvino_encoder(struct whisper_context* ctx,
+ const char* openvino_model_path,
+ const char* openvino_device,
+ const char* openvino_cache_dir)
+{
+#ifndef WHISPER_USE_OPENVINO
+ (void)(ctx);
+ (void)(openvino_model_path);
+ (void)(openvino_device);
+ (void)(openvino_cache_dir);
+ return 0;
+#else
+ if (!openvino_model_path && ctx->path_model.empty())
+ {
+ fprintf(stderr, "%s: openvino_model_path is nullptr, and ctx has no model_path set.\n", __func__);
+ return 0;
+ }
+
+ std::string path_openvino;
+ if (!openvino_model_path) {
+ //if openvino_model_path is not set, attempt to find it in the same directory as ggml-<model>.bin model
+ path_openvino = whisper_get_openvino_path_encoder(ctx->path_model);
+ }
+ else {
+ path_openvino = openvino_model_path;
+ }
+
+ std::string path_openvino_cache_dir;
+ if (!openvino_cache_dir) {
+ //if openvino_cache_dir is not set, set it as a dir residing next to ggml-<model>.bin
+ path_openvino_cache_dir = whisper_get_openvino_path_cache(ctx->path_model);
+ }
+ else {
+ path_openvino_cache_dir = openvino_cache_dir;
+ }
+
+ fprintf(stderr, "%s: loading OpenVINO model from '%s'\n", __func__, path_openvino.c_str());
+ fprintf(stderr, "%s: first run on a device may take a while ...\n", __func__);
+
+ ctx->state->ctx_openvino = whisper_openvino_init(path_openvino.c_str(), openvino_device, path_openvino_cache_dir.c_str());
+ if (!ctx->state->ctx_openvino) {
+ fprintf(stderr, "%s: failed to init OpenVINO encoder from '%s'\n", __func__, path_openvino.c_str());
+ return 0;
+ }
+ else {
+ fprintf(stderr, "%s: OpenVINO model loaded\n", __func__);
+ }
+
+ return 1;
+#endif
+}
+
struct whisper_context * whisper_init_from_file_no_state(const char * path_model) {
fprintf(stderr, "%s: loading model from '%s'\n", __func__, path_model);
}
#endif
+#ifdef WHISPER_USE_OPENVINO
+ if (state->ctx_openvino != nullptr) {
+ whisper_openvino_free(state->ctx_openvino);
+ state->ctx_openvino = nullptr;
+ }
+#endif
+
delete state;
}
}
#endif
}
+static int whisper_has_openvino(void) {
+#ifdef WHISPER_USE_OPENVINO
+ return 1;
+#else
+ return 0;
+#endif
+}
+
const char * whisper_print_system_info(void) {
static std::string s;
s += "SSE3 = " + std::to_string(ggml_cpu_has_sse3()) + " | ";
s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | ";
s += "COREML = " + std::to_string(whisper_has_coreml()) + " | ";
+ s += "OPENVINO = " + std::to_string(whisper_has_openvino()) + " | ";
return s.c_str();
}
WHISPER_API struct whisper_state * whisper_init_state(struct whisper_context * ctx);
+ // Given a context, enable use of OpenVINO for encode inference.
+ // model_path: Optional path to OpenVINO encoder IR model. If set to nullptr,
+ // the path will be generated from the ggml model path that was passed
+ // in to whisper_init_from_file. For example, if 'path_model' was
+ // "/path/to/ggml-base.en.bin", then OpenVINO IR model path will be
+ // assumed to be "/path/to/ggml-base.en-encoder-openvino.xml".
+ // device: OpenVINO device to run inference on ("CPU", "GPU", etc.)
+ // cache_dir: Optional cache directory that can speed up init time, especially for
+ // GPU, by caching compiled 'blobs' there.
+ // Set to nullptr if not used.
+ // Returns 1 on success. If OpenVINO is not enabled in build, this
+ // simply returns 0.
+ WHISPER_API int whisper_ctx_init_openvino_encoder(
+ struct whisper_context * ctx,
+ const char * model_path,
+ const char * device,
+ const char * cache_dir);
+
// Frees all allocated memory
WHISPER_API void whisper_free (struct whisper_context * ctx);
WHISPER_API void whisper_free_state(struct whisper_state * state);
overview / pkg / ggml / sources / whisper.cpp / commitdiff