#define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256
-#define VK_CHECK(err, msg) \
+#define VK_CHECK(err, msg, dev) \
do { \
- vk::Result err_ = (err); \
+ vk::Result err_; \
+ try { \
+ err_ = (err); \
+ } catch (vk::DeviceLostError &) { \
+ ggml_vk_print_device_lost_info(dev); \
+ GGML_LOG_ERROR("ggml_vulkan: %s at %s:%d\n", \
+ #err, __FILE__, __LINE__); \
+ throw; \
+ } \
if (err_ != vk::Result::eSuccess) { \
- fprintf(stderr, "ggml_vulkan: %s error %s at %s:%d\n", \
+ GGML_LOG_ERROR("ggml_vulkan: %s error %s at %s:%d\n", \
#err, to_string(err_).c_str(), __FILE__, __LINE__); \
- exit(1); \
+ throw vk::SystemError(vk::make_error_code(err_), \
+ "ggml_vulkan: " msg); \
} \
} while (0)
}
};
+static void ggml_vk_print_device_fault_info(const vk_device& device);
+static void ggml_vk_print_device_lost_info(const vk_device& device);
+
// Prevent simultaneous submissions to the same queue.
struct vk_queue_handle {
vk::Queue queue;
+ vk_device_ref device;
virtual void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) = 0;
virtual void lock() {} // no-op by default (internally synchronized case)
virtual void unlock() {}
std::mutex mutex;
void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {
std::lock_guard<std::mutex> guard(mutex);
- queue.submit(submits, fence);
+ try {
+ queue.submit(submits, fence);
+ } catch (vk::DeviceLostError &) {
+ if (auto dev = device.lock()) {
+ ggml_vk_print_device_lost_info(dev);
+ }
+ throw;
+ }
}
void lock() override { mutex.lock(); }
void unlock() override { mutex.unlock(); }
struct vk_queue_handle_unsynchronized : vk_queue_handle {
void submit(vk::ArrayProxy<const vk::SubmitInfo> submits, vk::Fence fence) override {
// Driver guarantees internal synchronization via VK_KHR_internally_synchronized_queues
- queue.submit(submits, fence);
+ try {
+ queue.submit(submits, fence);
+ } catch (vk::DeviceLostError &) {
+ if (auto dev = device.lock()) {
+ ggml_vk_print_device_lost_info(dev);
+ }
+ throw;
+ }
}
// lock()/unlock() inherited no-ops
};
bool pipeline_executable_properties_support {};
+ bool device_fault {};
+ PFN_vkGetDeviceFaultInfoEXT pfn_vkGetDeviceFaultInfoEXT {};
+
+ bool serialize_submissions {};
+
+ const ggml_cgraph * diag_cgraph {};
+ int diag_prev_start = -1;
+ int diag_prev_end = -1;
+
size_t idx;
bool mul_mat_l[GGML_TYPE_COUNT];
cmd_buffers.clear();
}
+static void ggml_vk_print_device_fault_info(const vk_device& device) {
+ if (!device->device_fault || !device->pfn_vkGetDeviceFaultInfoEXT) {
+ return;
+ }
+
+ VkDeviceFaultCountsEXT fault_counts {};
+ fault_counts.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT;
+ VkResult res = device->pfn_vkGetDeviceFaultInfoEXT(device->device, &fault_counts, nullptr);
+ if (res != VK_SUCCESS) {
+ GGML_LOG_ERROR("ggml_vulkan: vkGetDeviceFaultInfoEXT (counts) failed: %d\n", res);
+ return;
+ }
+
+ std::vector<VkDeviceFaultAddressInfoEXT> address_infos(fault_counts.addressInfoCount);
+ std::vector<VkDeviceFaultVendorInfoEXT> vendor_infos(fault_counts.vendorInfoCount);
+
+ VkDeviceFaultInfoEXT fault_info {};
+ fault_info.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT;
+ fault_info.pAddressInfos = address_infos.data();
+ fault_info.pVendorInfos = vendor_infos.data();
+
+ res = device->pfn_vkGetDeviceFaultInfoEXT(device->device, &fault_counts, &fault_info);
+ if (res != VK_SUCCESS) {
+ GGML_LOG_ERROR("ggml_vulkan: vkGetDeviceFaultInfoEXT (info) failed: %d\n", res);
+ return;
+ }
+
+ if (fault_counts.addressInfoCount == 0 && fault_counts.vendorInfoCount == 0 && fault_info.description[0] == '\0') {
+ return;
+ }
+
+ if (fault_info.description[0] != '\0') {
+ GGML_LOG_ERROR("ggml_vulkan: device fault on %s: %s\n", device->name.c_str(), fault_info.description);
+ }
+
+ for (uint32_t i = 0; i < fault_counts.addressInfoCount; i++) {
+ const auto& info = address_infos[i];
+ GGML_LOG_CONT(" address fault %u: type=%d address=0x%llx precision=0x%llx\n",
+ i, (int)info.addressType,
+ (unsigned long long)info.reportedAddress,
+ (unsigned long long)info.addressPrecision);
+ }
+ for (uint32_t i = 0; i < fault_counts.vendorInfoCount; i++) {
+ const auto& info = vendor_infos[i];
+ GGML_LOG_CONT(" vendor fault %u: %s (code=0x%llx data=0x%llx)\n",
+ i, info.description,
+ (unsigned long long)info.vendorFaultCode,
+ (unsigned long long)info.vendorFaultData);
+ }
+}
+
struct vk_buffer_struct {
vk::Buffer buffer = VK_NULL_HANDLE;
vk::DeviceMemory device_memory = VK_NULL_HANDLE;
return 0;
}
+static void ggml_vk_print_node_list(const ggml_cgraph * cgraph, int start, int end) {
+ uint64_t total_flops = 0;
+ int n_ops = 0;
+ for (int j = start; j <= end && j < cgraph->n_nodes; j++) {
+ uint64_t flops = ggml_vk_get_node_flops(cgraph->nodes[j]);
+ total_flops += flops;
+ n_ops++;
+ if (flops > 0) {
+ GGML_LOG_CONT(" node %d: %s (%s) [%.2f GFLOP]\n",
+ j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op),
+ flops / 1e9);
+ } else {
+ GGML_LOG_CONT(" node %d: %s (%s)\n",
+ j, cgraph->nodes[j]->name, ggml_op_name(cgraph->nodes[j]->op));
+ }
+ }
+ GGML_LOG_CONT(" total: %d ops, %.2f GFLOP\n", n_ops, total_flops / 1e9);
+}
+
+static void ggml_vk_print_device_lost_info(const vk_device& device) {
+ ggml_vk_print_device_fault_info(device);
+ if (device->serialize_submissions && device->diag_cgraph != nullptr && device->diag_prev_start >= 0) {
+ GGML_LOG_ERROR("ggml_vulkan: device lost on %s, likely caused by previous submission (nodes %d to %d):\n",
+ device->name.c_str(), device->diag_prev_start, device->diag_prev_end);
+ ggml_vk_print_node_list(device->diag_cgraph, device->diag_prev_start, device->diag_prev_end);
+ } else {
+ GGML_LOG_ERROR("ggml_vulkan: device lost on %s\n", device->name.c_str());
+ }
+}
+
class vk_perf_logger {
public:
void print_timings(bool force = false) {
// Use waitForFences while most of the graph executes. Hopefully the CPU can sleep
// during this wait.
if (ctx->almost_ready_fence_pending) {
- VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence");
+ VK_CHECK(ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX), "almost_ready_fence", ctx->device);
ctx->device->device.resetFences({ ctx->almost_ready_fence });
ctx->almost_ready_fence_pending = false;
}
// Spin (w/pause) waiting for the graph to finish executing.
vk::Result result;
- while ((result = ctx->device->device.getFenceStatus(ctx->fence)) != vk::Result::eSuccess) {
+ for (;;) {
+ try {
+ result = ctx->device->device.getFenceStatus(ctx->fence);
+ } catch (vk::DeviceLostError &) {
+ ggml_vk_print_device_lost_info(ctx->device);
+ GGML_LOG_ERROR("ggml_vulkan: getFenceStatus at %s:%d\n", __FILE__, __LINE__);
+ throw;
+ }
+ if (result == vk::Result::eSuccess) {
+ break;
+ }
if (result != vk::Result::eNotReady) {
- fprintf(stderr, "ggml_vulkan: error %s at %s:%d\n", to_string(result).c_str(), __FILE__, __LINE__);
- exit(1);
+ GGML_LOG_ERROR("ggml_vulkan: error %s at %s:%d\n", to_string(result).c_str(), __FILE__, __LINE__);
+ throw vk::SystemError(vk::make_error_code(result), "ggml_vulkan: getFenceStatus");
}
for (uint32_t i = 0; i < 100; ++i) {
YIELD();
}
h->queue = device->device.getQueue2(queue_info2);
+ h->device = device;
q->handle = h;
q->cmd_pool.init(device, q.get());
#endif
} else if (strcmp(VK_KHR_INTERNALLY_SYNCHRONIZED_QUEUES_EXTENSION_NAME, properties.extensionName) == 0) {
internally_sync_support = true;
+ } else if (strcmp("VK_EXT_device_fault", properties.extensionName) == 0) {
+ device->device_fault = true;
}
}
}
#endif
+ VkPhysicalDeviceFaultFeaturesEXT fault_features {};
+ fault_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FAULT_FEATURES_EXT;
+ if (device->device_fault) {
+ last_struct->pNext = (VkBaseOutStructure *)&fault_features;
+ last_struct = (VkBaseOutStructure *)&fault_features;
+ device_extensions.push_back("VK_EXT_device_fault");
+ }
+
vkGetPhysicalDeviceFeatures2(device->physical_device, &device_features2);
+ device->device_fault = device->device_fault && fault_features.deviceFault;
+
device->has_internally_synchronized_queues = internally_synchronized_queues_features.internallySynchronizedQueues;
// Build queue create infos only after querying whether internally synchronized queues are enabled.
device_create_info.setPNext(&device_features2);
device->device = device->physical_device.createDevice(device_create_info);
+ if (device->device_fault) {
+ device->pfn_vkGetDeviceFaultInfoEXT = (PFN_vkGetDeviceFaultInfoEXT)
+ vkGetDeviceProcAddr(device->device, "vkGetDeviceFaultInfoEXT");
+ }
+
// Queues
device->compute_queue = ggml_vk_create_queue(device, compute_queue_family_index, 0, { vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer }, false);
device->idx = idx;
+ device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr;
+
device->disable_fusion = getenv("GGML_VK_DISABLE_FUSION") != nullptr;
device->add_rms_fusion = !device->disable_fusion &&
}
ggml_vk_submit(subctx, dst->device->fence);
- VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences");
+ VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_buffer_write_2d waitForFences", dst->device);
dst->device->device.resetFences({ dst->device->fence });
ggml_vk_queue_command_pools_cleanup(dst->device);
}
ggml_vk_ctx_end(subctx);
ggml_vk_submit(subctx, src->device->fence);
VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX),
- "vk_buffer_read_2d uma waitForFences");
+ "vk_buffer_read_2d uma waitForFences", src->device);
src->device->device.resetFences({ src->device->fence });
ggml_vk_queue_command_pools_cleanup(src->device);
ggml_vk_ctx_end(subctx);
ggml_vk_submit(subctx, src->device->fence);
- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences");
+ VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_read_2d waitForFences", src->device);
src->device->device.resetFences({ src->device->fence });
ggml_vk_queue_command_pools_cleanup(src->device);
ggml_vk_buffer_copy_async(subctx, dst, dst_offset, src, src_offset, size);
ggml_vk_ctx_end(subctx);
ggml_vk_submit(subctx, src->device->fence);
- VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences");
+ VK_CHECK(src->device->device.waitForFences({ src->device->fence }, true, UINT64_MAX), "vk_buffer_copy waitForFences", src->device);
src->device->device.resetFences({ src->device->fence });
ggml_vk_queue_command_pools_cleanup(src->device);
} else {
ggml_vk_ctx_end(subctx);
ggml_vk_submit(subctx, dst->device->fence);
- VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences");
+ VK_CHECK(dst->device->device.waitForFences({ dst->device->fence }, true, UINT64_MAX), "vk_memset waitForFences", dst->device);
dst->device->device.resetFences({ dst->device->fence });
ggml_vk_queue_command_pools_cleanup(dst->device);
}
auto begin = std::chrono::high_resolution_clock::now();
ggml_vk_submit(subctx, ctx->fence);
- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences");
+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_matmul waitForFences", ctx->device);
ctx->device->device.resetFences({ ctx->fence });
ggml_vk_queue_command_pools_cleanup(ctx->device);
auto begin = std::chrono::high_resolution_clock::now();
ggml_vk_submit(subctx, ctx->fence);
- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences");
+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);
ctx->device->device.resetFences({ ctx->fence });
ggml_vk_queue_command_pools_cleanup(ctx->device);
auto begin = std::chrono::high_resolution_clock::now();
ggml_vk_submit(subctx, ctx->fence);
- VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences");
+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "ggml_vk_test_dequant waitForFences", ctx->device);
ctx->device->device.resetFences({ ctx->fence });
ggml_vk_queue_command_pools_cleanup(ctx->device);
memset(mset.dst, mset.val, mset.n);
}
- if (almost_ready && !ctx->almost_ready_fence_pending) {
+ if (ctx->device->serialize_submissions) {
+ ggml_vk_submit(subctx, ctx->fence);
+ } else if (almost_ready && !ctx->almost_ready_fence_pending) {
ggml_vk_submit(subctx, ctx->almost_ready_fence);
ctx->almost_ready_fence_pending = true;
} else {
memcpy(cpy.dst, cpy.src, cpy.n);
}
- ggml_vk_submit(compute_ctx, {});
+ if (ctx->device->serialize_submissions) {
+ ggml_vk_submit(compute_ctx, ctx->fence);
+ VK_CHECK(ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX), "synchronize waitForFences", ctx->device);
+ ctx->device->device.resetFences({ ctx->fence });
+ } else {
+ ggml_vk_submit(compute_ctx, {});
+ }
ctx->submit_pending = true;
}
if (ctx->submit_pending) {
- if (ctx->device->async_use_transfer_queue && ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) {
+ if (ctx->device->serialize_submissions) {
+ ctx->submit_pending = false;
+ } else if (ctx->device->async_use_transfer_queue && ctx->transfer_semaphore_last_submitted < ctx->transfer_semaphore.value) {
vk::TimelineSemaphoreSubmitInfo tl_info{
1, &ctx->transfer_semaphore.value,
0, nullptr,
} else {
ctx->device->compute_queue->handle->submit({}, ctx->fence);
}
- ggml_vk_wait_for_fence(ctx);
+ if (!ctx->device->serialize_submissions) {
+ ggml_vk_wait_for_fence(ctx);
+ }
ctx->submit_pending = false;
if (cmd_buf) {
cmd_buf->in_use = false;
VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
+ ctx->device->diag_cgraph = nullptr;
+ ctx->device->diag_prev_start = -1;
+ ctx->device->diag_prev_end = -1;
+
if (vk_instance.debug_utils_support) {
vk::DebugUtilsLabelEXT dul = {};
dul.pLabelName = "ggml_backend_vk_graph_compute";
}
uint64_t flops_per_submit = std::min(flops_cap, ctx->last_total_flops / 40u);
+ auto const submit_after = [&](int start, int end) {
+ if (ctx->device->serialize_submissions) {
+ try {
+ auto res = ctx->device->device.waitForFences({ ctx->fence }, true, UINT64_MAX);
+ if (res != vk::Result::eSuccess) {
+ GGML_LOG_ERROR("ggml_vulkan: waitForFences error during serialized submission\n");
+ throw vk::SystemError(vk::make_error_code(res), "ggml_vulkan: waitForFences during serialized submission");
+ }
+ } catch (vk::DeviceLostError &) {
+ ggml_vk_print_device_fault_info(ctx->device);
+ GGML_LOG_ERROR("ggml_vulkan: device lost on %s waiting for submission (nodes %d to %d):\n",
+ ctx->device->name.c_str(), start, end);
+ ggml_vk_print_node_list(cgraph, start, end);
+ throw;
+ }
+ ctx->device->device.resetFences({ ctx->fence });
+ ctx->submit_pending = false;
+ ctx->device->diag_cgraph = cgraph;
+ ctx->device->diag_prev_start = start;
+ ctx->device->diag_prev_end = end;
+ }
+ first_node_in_batch = true;
+ submitted_nodes = 0;
+ batch_flops = 0;
+ if (submit_count < 3) {
+ flops_per_submit *= 2;
+ }
+ submit_count++;
+ };
+
for (int i = 0; i < cgraph->n_nodes; i++) {
if (first_node_in_batch) {
submit_node_idx = i;
{
auto node_flops = ggml_vk_get_node_flops(cgraph->nodes[i]);
- batch_flops += node_flops;
total_flops += node_flops;
+
+ // Flush the current batch before recording a node that would push it over the flop threshold
+ if (flops_per_submit != 0 && submitted_nodes > 0 && batch_flops + node_flops >= flops_per_submit) {
+ vk_context flush_ctx = ggml_vk_get_compute_ctx(ctx);
+ ggml_vk_ctx_end(flush_ctx);
+ flush_ctx->exit_tensor_idx = -1;
+ ctx->compute_ctx.reset();
+ ggml_vk_compute_forward(ctx, cgraph, cgraph->nodes[submit_node_idx], submit_node_idx, false);
+ submit_after(submit_node_idx, i - 1);
+ submit_node_idx = i;
+ }
+
+ batch_flops += node_flops;
}
// op_srcs_fused_elementwise indicates whether an op's srcs all contribute to
}
if (submit && enqueued) {
- first_node_in_batch = true;
- submitted_nodes = 0;
- batch_flops = 0;
- if (submit_count < 3) {
- flops_per_submit *= 2;
- }
- submit_count++;
+ submit_after(submit_node_idx, i + (int)ctx->num_additional_fused_ops);
}
i += ctx->num_additional_fused_ops;
ctx->num_additional_fused_ops = 0;
ggml_vk_ctx_end(compute_ctx);
ggml_vk_submit(compute_ctx, ctx->device->fence);
- VK_CHECK(ctx->device->device.waitForFences({ ctx->device->fence }, true, UINT64_MAX), "GGML_VULKAN_PERF waitForFences");
+ VK_CHECK(ctx->device->device.waitForFences({ ctx->device->fence }, true, UINT64_MAX), "GGML_VULKAN_PERF waitForFences", ctx->device);
ctx->device->device.resetFences({ ctx->device->fence });
ctx->compute_ctx.reset();
// Get the results and pass them to the logger
std::vector<uint64_t> timestamps(cgraph->n_nodes + 1);
- VK_CHECK(ctx->device->device.getQueryPoolResults(ctx->query_pool, 0, ctx->query_idx, (cgraph->n_nodes + 1)*sizeof(uint64_t), timestamps.data(), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait), "get timestamp results");
+ VK_CHECK(ctx->device->device.getQueryPoolResults(ctx->query_pool, 0, ctx->query_idx, (cgraph->n_nodes + 1)*sizeof(uint64_t), timestamps.data(), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait), "get timestamp results", ctx->device);
if (!vk_perf_logger_concurrent) {
// Log each op separately
for (int i = 1; i < ctx->query_idx; i++) {
vk::Semaphore sem = vkev->tl_semaphore.s;
uint64_t val = vkev->tl_semaphore.value;
vk::SemaphoreWaitInfo swi{vk::SemaphoreWaitFlags{}, sem, val};
- VK_CHECK(device->device.waitSemaphores(swi, UINT64_MAX), "event_synchronize");
+ VK_CHECK(device->device.waitSemaphores(swi, UINT64_MAX), "event_synchronize", device);
// Reset and move submitted events
for (auto& event : vkev->events_submitted) {
overview / pkg / ggml / sources / llama.cpp / commitdiff