Mesa (main): vulkan: Add a common VkPipelineCache implementation
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Fri Apr 22 20:10:56 UTC 2022
Module: Mesa
Branch: main
Commit: 591da9877900c56434f9e23e0ad0058b118b0be8
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=591da9877900c56434f9e23e0ad0058b118b0be8
Author: Jason Ekstrand <jason at jlekstrand.net>
Date: Sat Oct 2 03:38:28 2021 -0500
vulkan: Add a common VkPipelineCache implementation
This is partly copied+pasted from ANV but is mostly new code with lots
of reference counting bugs fixed (I hope!). The new cache caches
"object" which derive from a base vk_pipeline_class_object struct. It
uses a kernel-style "ops" interface for virtual methods on these objects
to allow for easy destruction (when the reference count hits zero) as
well as serialization an deserialization interfaces. This should allow
drivers to cache basically whatever they want without having to think
too hard about the details.
Reviewed-by: Connor Abbott <cwabbott0 at gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/13184>
---
src/vulkan/runtime/meson.build | 2 +
src/vulkan/runtime/vk_physical_device.c | 3 +
src/vulkan/runtime/vk_physical_device.h | 16 +
src/vulkan/runtime/vk_pipeline_cache.c | 731 ++++++++++++++++++++++++++++++++
src/vulkan/runtime/vk_pipeline_cache.h | 243 +++++++++++
5 files changed, 995 insertions(+)
diff --git a/src/vulkan/runtime/meson.build b/src/vulkan/runtime/meson.build
index af134efb04e..196997490d1 100644
--- a/src/vulkan/runtime/meson.build
+++ b/src/vulkan/runtime/meson.build
@@ -54,6 +54,8 @@ vulkan_runtime_files = files(
'vk_nir.h',
'vk_object.c',
'vk_object.h',
+ 'vk_pipeline_cache.c',
+ 'vk_pipeline_cache.h',
'vk_physical_device.c',
'vk_physical_device.h',
'vk_queue.c',
diff --git a/src/vulkan/runtime/vk_physical_device.c b/src/vulkan/runtime/vk_physical_device.c
index 8915e73fa7c..2b747a0cb60 100644
--- a/src/vulkan/runtime/vk_physical_device.c
+++ b/src/vulkan/runtime/vk_physical_device.c
@@ -45,6 +45,9 @@ vk_physical_device_init(struct vk_physical_device *pdevice,
vk_physical_device_dispatch_table_from_entrypoints(
&pdevice->dispatch_table, &vk_common_physical_device_entrypoints, false);
+ /* TODO */
+ pdevice->disk_cache = NULL;
+
return VK_SUCCESS;
}
diff --git a/src/vulkan/runtime/vk_physical_device.h b/src/vulkan/runtime/vk_physical_device.h
index f8be3deaf4c..eac023df917 100644
--- a/src/vulkan/runtime/vk_physical_device.h
+++ b/src/vulkan/runtime/vk_physical_device.h
@@ -31,8 +31,10 @@
extern "C" {
#endif
+struct disk_cache;
struct wsi_device;
struct vk_sync_type;
+struct vk_pipeline_cache_object_ops;
/** Base struct for all VkPhysicalDevice implementations
*/
@@ -57,6 +59,9 @@ struct vk_physical_device {
/** Physical-device-level dispatch table */
struct vk_physical_device_dispatch_table dispatch_table;
+ /** Disk cache, or NULL */
+ struct disk_cache *disk_cache;
+
/** WSI device, or NULL */
struct wsi_device *wsi_device;
@@ -70,6 +75,17 @@ struct vk_physical_device {
* considered just one more criterion.
*/
const struct vk_sync_type *const *supported_sync_types;
+
+ /** A null-terminated array of supported pipeline cache object types
+ *
+ * The common implementation of VkPipelineCache uses this to remember the
+ * type of objects stored in the cache and deserialize them immediately
+ * when importing the cache. If an object type isn't in this list, then it
+ * will be loaded as a raw data object and then deserialized when we first
+ * look it up. Deserializing immediately avoids a copy but may be more
+ * expensive for objects that aren't hit.
+ */
+ const struct vk_pipeline_cache_object_ops *const *pipeline_cache_import_ops;
};
VK_DEFINE_HANDLE_CASTS(vk_physical_device, base, VkPhysicalDevice,
diff --git a/src/vulkan/runtime/vk_pipeline_cache.c b/src/vulkan/runtime/vk_pipeline_cache.c
new file mode 100644
index 00000000000..2214b27751c
--- /dev/null
+++ b/src/vulkan/runtime/vk_pipeline_cache.c
@@ -0,0 +1,731 @@
+/*
+ * Copyright © 2021 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "vk_pipeline_cache.h"
+
+#include "vk_alloc.h"
+#include "vk_common_entrypoints.h"
+#include "vk_device.h"
+#include "vk_log.h"
+#include "vk_physical_device.h"
+
+#include "util/blob.h"
+#include "util/debug.h"
+#include "util/disk_cache.h"
+#include "util/hash_table.h"
+#include "util/set.h"
+
+struct raw_data_object {
+ struct vk_pipeline_cache_object base;
+
+ const void *data;
+ size_t data_size;
+};
+
+static bool
+raw_data_object_serialize(struct vk_pipeline_cache_object *object,
+ struct blob *blob)
+{
+ struct raw_data_object *data_obj =
+ container_of(object, struct raw_data_object, base);
+
+ blob_write_bytes(blob, data_obj->data, data_obj->data_size);
+
+ return true;
+}
+
+static void
+raw_data_object_destroy(struct vk_pipeline_cache_object *object)
+{
+ struct raw_data_object *data_obj =
+ container_of(object, struct raw_data_object, base);
+
+ vk_free(&data_obj->base.device->alloc, data_obj);
+}
+
+static const struct vk_pipeline_cache_object_ops raw_data_object_ops = {
+ .serialize = raw_data_object_serialize,
+ .destroy = raw_data_object_destroy,
+};
+
+static struct raw_data_object *
+raw_data_object_create(struct vk_device *device,
+ const void *key_data, size_t key_size,
+ const void *data, size_t data_size)
+{
+ VK_MULTIALLOC(ma);
+ VK_MULTIALLOC_DECL(&ma, struct raw_data_object, data_obj, 1);
+ VK_MULTIALLOC_DECL_SIZE(&ma, void, obj_key_data, key_size);
+ VK_MULTIALLOC_DECL_SIZE(&ma, void, obj_data, data_size);
+
+ if (!vk_multialloc_alloc(&ma, &device->alloc,
+ VK_SYSTEM_ALLOCATION_SCOPE_DEVICE))
+ return NULL;
+
+ vk_pipeline_cache_object_init(device, &data_obj->base,
+ &raw_data_object_ops,
+ obj_key_data, key_size);
+ data_obj->data = obj_data;
+ data_obj->data_size = data_size;
+
+ memcpy(obj_key_data, key_data, key_size);
+ memcpy(obj_data, data, data_size);
+
+ return data_obj;
+}
+
+static bool
+object_keys_equal(const void *void_a, const void *void_b)
+{
+ const struct vk_pipeline_cache_object *a = void_a, *b = void_b;
+ if (a->key_size != b->key_size)
+ return false;
+
+ return memcmp(a->key_data, b->key_data, a->key_size) == 0;
+}
+
+static uint32_t
+object_key_hash(const void *void_object)
+{
+ const struct vk_pipeline_cache_object *object = void_object;
+ return _mesa_hash_data(object->key_data, object->key_size);
+}
+
+static void
+vk_pipeline_cache_lock(struct vk_pipeline_cache *cache)
+{
+
+ if (!(cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT))
+ simple_mtx_lock(&cache->lock);
+}
+
+static void
+vk_pipeline_cache_unlock(struct vk_pipeline_cache *cache)
+{
+ if (!(cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT))
+ simple_mtx_unlock(&cache->lock);
+}
+
+static void
+vk_pipeline_cache_remove_object(struct vk_pipeline_cache *cache,
+ uint32_t hash,
+ struct vk_pipeline_cache_object *object)
+{
+ vk_pipeline_cache_lock(cache);
+ struct set_entry *entry =
+ _mesa_set_search_pre_hashed(cache->object_cache, hash, object);
+ if (entry && entry->key == (const void *)object) {
+ /* Drop the reference owned by the cache */
+ vk_pipeline_cache_object_unref(object);
+
+ _mesa_set_remove(cache->object_cache, entry);
+ }
+ vk_pipeline_cache_unlock(cache);
+
+ /* Drop our reference */
+ vk_pipeline_cache_object_unref(object);
+}
+
+/* Consumes references to both search and replace and produces a reference */
+static struct vk_pipeline_cache_object *
+vk_pipeline_cache_replace_object(struct vk_pipeline_cache *cache,
+ uint32_t hash,
+ struct vk_pipeline_cache_object *search,
+ struct vk_pipeline_cache_object *replace)
+{
+ assert(object_keys_equal(search, replace));
+
+ vk_pipeline_cache_lock(cache);
+ struct set_entry *entry =
+ _mesa_set_search_pre_hashed(cache->object_cache, hash, search);
+
+ struct vk_pipeline_cache_object *found = NULL;
+ if (entry) {
+ if (entry->key == (const void *)search) {
+ /* Drop the reference owned by the cache */
+ vk_pipeline_cache_object_unref(search);
+
+ entry->key = vk_pipeline_cache_object_ref(replace);
+ } else {
+ found = vk_pipeline_cache_object_ref((void *)entry->key);
+ }
+ } else {
+ /* I guess the object was purged? Re-add it to the cache */
+ vk_pipeline_cache_object_ref(replace);
+ _mesa_set_add_pre_hashed(cache->object_cache, hash, replace);
+ }
+ vk_pipeline_cache_unlock(cache);
+
+ vk_pipeline_cache_object_unref(search);
+
+ if (found) {
+ vk_pipeline_cache_object_unref(replace);
+ return found;
+ } else {
+ return replace;
+ }
+}
+
+static bool
+vk_pipeline_cache_object_serialize(struct vk_pipeline_cache *cache,
+ struct vk_pipeline_cache_object *object,
+ struct blob *blob, uint32_t *data_size)
+{
+ if (object->ops->serialize == NULL)
+ return false;
+
+ assert(blob->size == align64(blob->size, VK_PIPELINE_CACHE_BLOB_ALIGN));
+ size_t start = blob->size;
+
+ /* Special case for if we're writing to a NULL blob (just to get the size)
+ * and we already know the data size of the allocation. This should make
+ * the first GetPipelineCacheData() call to get the data size faster in the
+ * common case where a bunch of our objects were loaded from a previous
+ * cache or where we've already serialized the cache once.
+ */
+ if (blob->data == NULL && blob->fixed_allocation) {
+ *data_size = p_atomic_read(&object->data_size);
+ if (*data_size > 0) {
+ blob_write_bytes(blob, NULL, *data_size);
+ return true;
+ }
+ }
+
+ if (!object->ops->serialize(object, blob)) {
+ vk_logw(VK_LOG_OBJS(cache),
+ "Failed to serialize pipeline cache object");
+ return false;
+ }
+
+ size_t size = blob->size - start;
+ if (size > UINT32_MAX) {
+ vk_logw(VK_LOG_OBJS(cache),
+ "Skipping giant (4 GiB or larger) object");
+ return false;
+ }
+
+ if (blob->out_of_memory) {
+ vk_logw(VK_LOG_OBJS(cache),
+ "Insufficient memory for pipeline cache data");
+ return false;
+ }
+
+ *data_size = (uint32_t)size;
+ p_atomic_set(&object->data_size, *data_size);
+
+ return true;
+}
+
+static struct vk_pipeline_cache_object *
+vk_pipeline_cache_object_deserialize(struct vk_pipeline_cache *cache,
+ const void *key_data, uint32_t key_size,
+ const void *data, size_t data_size,
+ const struct vk_pipeline_cache_object_ops *ops)
+{
+ if (unlikely(ops->deserialize == NULL)) {
+ vk_logw(VK_LOG_OBJS(cache),
+ "Pipeline cache object cannot be deserialized");
+ return NULL;
+ }
+
+ struct blob_reader reader;
+ blob_reader_init(&reader, data, data_size);
+
+ struct vk_pipeline_cache_object *object =
+ ops->deserialize(cache->base.device, key_data, key_size, &reader);
+
+ if (object == NULL) {
+ vk_logw(VK_LOG_OBJS(cache),
+ "Deserializing pipeline cache object failed");
+ return NULL;
+ }
+
+ assert(reader.current == reader.end && !reader.overrun);
+ assert(object->device == cache->base.device);
+ assert(object->ops == ops);
+ assert(object->ref_cnt == 1);
+ assert(object->key_size == key_size);
+ assert(memcmp(object->key_data, key_data, key_size) == 0);
+
+ return object;
+}
+
+struct vk_pipeline_cache_object *
+vk_pipeline_cache_lookup_object(struct vk_pipeline_cache *cache,
+ const void *key_data, size_t key_size,
+ const struct vk_pipeline_cache_object_ops *ops,
+ bool *cache_hit)
+{
+ assert(key_size <= UINT32_MAX);
+ assert(ops != NULL);
+
+ if (cache_hit != NULL)
+ *cache_hit = false;
+
+ struct vk_pipeline_cache_object key = {
+ .key_data = key_data,
+ .key_size = key_size,
+ };
+ uint32_t hash = object_key_hash(&key);
+
+ struct vk_pipeline_cache_object *object = NULL;
+
+ if (cache != NULL && cache->object_cache != NULL) {
+ vk_pipeline_cache_lock(cache);
+ struct set_entry *entry =
+ _mesa_set_search_pre_hashed(cache->object_cache, hash, &key);
+ if (entry) {
+ object = vk_pipeline_cache_object_ref((void *)entry->key);
+ if (cache_hit != NULL)
+ *cache_hit = true;
+ }
+ vk_pipeline_cache_unlock(cache);
+ }
+
+ if (object == NULL) {
+#ifdef ENABLE_SHADER_CACHE
+ struct disk_cache *disk_cache = cache->base.device->physical->disk_cache;
+ if (disk_cache != NULL) {
+ cache_key cache_key;
+ disk_cache_compute_key(disk_cache, key_data, key_size, cache_key);
+
+ size_t data_size;
+ uint8_t *data = disk_cache_get(disk_cache, cache_key, &data_size);
+ if (data) {
+ object = vk_pipeline_cache_object_deserialize(cache,
+ key_data, key_size,
+ data, data_size,
+ ops);
+ free(data);
+ if (object != NULL)
+ return vk_pipeline_cache_add_object(cache, object);
+ }
+ }
+#endif
+
+ /* No disk cache or not found in the disk cache */
+ return NULL;
+ }
+
+ if (object->ops == &raw_data_object_ops) {
+ /* The object isn't fully formed yet and we need to deserialize it into
+ * a real object before it can be used.
+ */
+ struct raw_data_object *data_obj =
+ container_of(object, struct raw_data_object, base);
+
+ assert(ops != &raw_data_object_ops);
+
+ struct vk_pipeline_cache_object *real_object =
+ vk_pipeline_cache_object_deserialize(cache,
+ data_obj->base.key_data,
+ data_obj->base.key_size,
+ data_obj->data,
+ data_obj->data_size, ops);
+ if (real_object == NULL) {
+ vk_pipeline_cache_remove_object(cache, hash, object);
+ return NULL;
+ }
+
+ object = vk_pipeline_cache_replace_object(cache, hash, object,
+ real_object);
+ }
+
+ assert(object->ops == ops);
+
+ return object;
+}
+
+struct vk_pipeline_cache_object *
+vk_pipeline_cache_add_object(struct vk_pipeline_cache *cache,
+ struct vk_pipeline_cache_object *object)
+{
+ assert(object->ops != NULL);
+
+ if (cache->object_cache == NULL)
+ return object;
+
+ uint32_t hash = object_key_hash(object);
+
+ vk_pipeline_cache_lock(cache);
+ bool found = false;
+ struct set_entry *entry =
+ _mesa_set_search_or_add_pre_hashed(cache->object_cache,
+ hash, object, &found);
+
+ struct vk_pipeline_cache_object *found_object = NULL;
+ if (found) {
+ found_object = vk_pipeline_cache_object_ref((void *)entry->key);
+ } else {
+ /* The cache now owns a reference */
+ vk_pipeline_cache_object_ref(object);
+ }
+ vk_pipeline_cache_unlock(cache);
+
+ if (found) {
+ vk_pipeline_cache_object_unref(object);
+ return found_object;
+ } else {
+ /* If it wasn't in the object cache, it might not be in the disk cache
+ * either. Better try and add it.
+ */
+
+#ifdef ENABLE_SHADER_CACHE
+ struct disk_cache *disk_cache = cache->base.device->physical->disk_cache;
+ if (object->ops->serialize != NULL && disk_cache) {
+ struct blob blob;
+ blob_init(&blob);
+
+ if (object->ops->serialize(object, &blob) && !blob.out_of_memory) {
+ cache_key cache_key;
+ disk_cache_compute_key(disk_cache, object->key_data,
+ object->key_size, cache_key);
+
+ disk_cache_put(disk_cache, cache_key, blob.data, blob.size, NULL);
+ }
+
+ blob_finish(&blob);
+ }
+#endif
+
+ return object;
+ }
+}
+
+static int32_t
+find_type_for_ops(const struct vk_physical_device *pdevice,
+ const struct vk_pipeline_cache_object_ops *ops)
+{
+ const struct vk_pipeline_cache_object_ops *const *import_ops =
+ pdevice->pipeline_cache_import_ops;
+
+ if (import_ops == NULL)
+ return -1;
+
+ for (int32_t i = 0; import_ops[i]; i++) {
+ if (import_ops[i] == ops)
+ return i;
+ }
+
+ return -1;
+}
+
+static const struct vk_pipeline_cache_object_ops *
+find_ops_for_type(const struct vk_physical_device *pdevice,
+ int32_t type)
+{
+ const struct vk_pipeline_cache_object_ops *const *import_ops =
+ pdevice->pipeline_cache_import_ops;
+
+ if (import_ops == NULL || type < 0)
+ return NULL;
+
+ return import_ops[type];
+}
+
+static void
+vk_pipeline_cache_load(struct vk_pipeline_cache *cache,
+ const void *data, size_t size)
+{
+ struct blob_reader blob;
+ blob_reader_init(&blob, data, size);
+
+ struct vk_pipeline_cache_header header;
+ blob_copy_bytes(&blob, &header, sizeof(header));
+ uint32_t count = blob_read_uint32(&blob);
+ if (blob.overrun)
+ return;
+
+ if (memcmp(&header, &cache->header, sizeof(header)) != 0)
+ return;
+
+ for (uint32_t i = 0; i < count; i++) {
+ int32_t type = blob_read_uint32(&blob);
+ uint32_t key_size = blob_read_uint32(&blob);
+ uint32_t data_size = blob_read_uint32(&blob);
+ const void *key_data = blob_read_bytes(&blob, key_size);
+ blob_reader_align(&blob, VK_PIPELINE_CACHE_BLOB_ALIGN);
+ const void *data = blob_read_bytes(&blob, data_size);
+ if (blob.overrun)
+ break;
+
+ const struct vk_pipeline_cache_object_ops *ops =
+ find_ops_for_type(cache->base.device->physical, type);
+
+ struct vk_pipeline_cache_object *object;
+ if (ops != NULL) {
+ object = vk_pipeline_cache_object_deserialize(cache,
+ key_data, key_size,
+ data, data_size, ops);
+ } else {
+ struct raw_data_object *data_obj =
+ raw_data_object_create(cache->base.device,
+ key_data, key_size,
+ data, data_size);
+ object = data_obj != NULL ? &data_obj->base : NULL;
+ }
+ if (object == NULL)
+ continue;
+
+ object = vk_pipeline_cache_add_object(cache, object);
+ vk_pipeline_cache_object_unref(object);
+ }
+}
+
+struct vk_pipeline_cache *
+vk_pipeline_cache_create(struct vk_device *device,
+ const struct vk_pipeline_cache_create_info *info,
+ const VkAllocationCallbacks *pAllocator)
+{
+ static const struct VkPipelineCacheCreateInfo default_create_info = {
+ .sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO,
+ };
+ struct vk_pipeline_cache *cache;
+
+ const struct VkPipelineCacheCreateInfo *pCreateInfo =
+ info->pCreateInfo != NULL ? info->pCreateInfo : &default_create_info;
+
+ assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO);
+ assert(pCreateInfo->flags == 0);
+
+ cache = vk_object_alloc(device, pAllocator, sizeof(*cache),
+ VK_OBJECT_TYPE_PIPELINE_CACHE);
+ if (cache == NULL)
+ return NULL;
+
+ cache->flags = pCreateInfo->flags;
+
+ struct VkPhysicalDeviceProperties pdevice_props;
+ device->physical->dispatch_table.GetPhysicalDeviceProperties(
+ vk_physical_device_to_handle(device->physical), &pdevice_props);
+
+ cache->header = (struct vk_pipeline_cache_header) {
+ .header_size = sizeof(struct vk_pipeline_cache_header),
+ .header_version = VK_PIPELINE_CACHE_HEADER_VERSION_ONE,
+ .vendor_id = pdevice_props.vendorID,
+ .device_id = pdevice_props.deviceID,
+ };
+ memcpy(cache->header.uuid, pdevice_props.pipelineCacheUUID, VK_UUID_SIZE);
+
+ simple_mtx_init(&cache->lock, mtx_plain);
+
+ if (info->force_enable ||
+ env_var_as_boolean("VK_ENABLE_PIPELINE_CACHE", true)) {
+ cache->object_cache = _mesa_set_create(NULL, object_key_hash,
+ object_keys_equal);
+ }
+
+ if (cache->object_cache && pCreateInfo->initialDataSize > 0) {
+ vk_pipeline_cache_load(cache, pCreateInfo->pInitialData,
+ pCreateInfo->initialDataSize);
+ }
+
+ return cache;
+}
+
+static void
+object_unref_cb(struct set_entry *entry)
+{
+ vk_pipeline_cache_object_unref((void *)entry->key);
+}
+
+void
+vk_pipeline_cache_destroy(struct vk_pipeline_cache *cache,
+ const VkAllocationCallbacks *pAllocator)
+{
+ if (cache->object_cache)
+ _mesa_set_destroy(cache->object_cache, object_unref_cb);
+ simple_mtx_destroy(&cache->lock);
+ vk_object_free(cache->base.device, pAllocator, cache);
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+vk_common_CreatePipelineCache(VkDevice _device,
+ const VkPipelineCacheCreateInfo *pCreateInfo,
+ const VkAllocationCallbacks *pAllocator,
+ VkPipelineCache *pPipelineCache)
+{
+ VK_FROM_HANDLE(vk_device, device, _device);
+ struct vk_pipeline_cache *cache;
+
+ struct vk_pipeline_cache_create_info info = {
+ .pCreateInfo = pCreateInfo,
+ };
+ cache = vk_pipeline_cache_create(device, &info, pAllocator);
+ if (cache == NULL)
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+
+ *pPipelineCache = vk_pipeline_cache_to_handle(cache);
+
+ return VK_SUCCESS;
+}
+
+VKAPI_ATTR void VKAPI_CALL
+vk_common_DestroyPipelineCache(VkDevice device,
+ VkPipelineCache pipelineCache,
+ const VkAllocationCallbacks *pAllocator)
+{
+ VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache);
+
+ if (cache == NULL)
+ return;
+
+ assert(cache->base.device == vk_device_from_handle(device));
+ vk_pipeline_cache_destroy(cache, pAllocator);
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+vk_common_GetPipelineCacheData(VkDevice _device,
+ VkPipelineCache pipelineCache,
+ size_t *pDataSize,
+ void *pData)
+{
+ VK_FROM_HANDLE(vk_device, device, _device);
+ VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache);
+
+ struct blob blob;
+ if (pData) {
+ blob_init_fixed(&blob, pData, *pDataSize);
+ } else {
+ blob_init_fixed(&blob, NULL, SIZE_MAX);
+ }
+
+ blob_write_bytes(&blob, &cache->header, sizeof(cache->header));
+
+ uint32_t count = 0;
+ intptr_t count_offset = blob_reserve_uint32(&blob);
+ if (count_offset < 0) {
+ *pDataSize = 0;
+ blob_finish(&blob);
+ return VK_INCOMPLETE;
+ }
+
+ vk_pipeline_cache_lock(cache);
+
+ VkResult result = VK_SUCCESS;
+ if (cache->object_cache != NULL) {
+ set_foreach(cache->object_cache, entry) {
+ struct vk_pipeline_cache_object *object = (void *)entry->key;
+
+ if (object->ops->serialize == NULL)
+ continue;
+
+ size_t blob_size_save = blob.size;
+
+ int32_t type = find_type_for_ops(device->physical, object->ops);
+ blob_write_uint32(&blob, type);
+ blob_write_uint32(&blob, object->key_size);
+ intptr_t data_size_resv = blob_reserve_uint32(&blob);
+ blob_write_bytes(&blob, object->key_data, object->key_size);
+
+ blob_align(&blob, VK_PIPELINE_CACHE_BLOB_ALIGN);
+
+ uint32_t data_size;
+ if (!vk_pipeline_cache_object_serialize(cache, object,
+ &blob, &data_size)) {
+ blob.size = blob_size_save;
+ if (blob.out_of_memory) {
+ result = VK_INCOMPLETE;
+ break;
+ }
+
+ /* Failed for some other reason; keep going */
+ continue;
+ }
+
+ /* vk_pipeline_cache_object_serialize should have failed */
+ assert(!blob.out_of_memory);
+
+ assert(data_size_resv >= 0);
+ blob_overwrite_uint32(&blob, data_size_resv, data_size);
+ }
+ }
+
+ vk_pipeline_cache_unlock(cache);
+
+ blob_overwrite_uint32(&blob, count_offset, count);
+
+ *pDataSize = blob.size;
+
+ blob_finish(&blob);
+
+ return result;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+vk_common_MergePipelineCaches(VkDevice device,
+ VkPipelineCache dstCache,
+ uint32_t srcCacheCount,
+ const VkPipelineCache *pSrcCaches)
+{
+ VK_FROM_HANDLE(vk_pipeline_cache, dst, dstCache);
+
+ if (!dst->object_cache)
+ return VK_SUCCESS;
+
+ vk_pipeline_cache_lock(dst);
+
+ for (uint32_t i = 0; i < srcCacheCount; i++) {
+ VK_FROM_HANDLE(vk_pipeline_cache, src, pSrcCaches[i]);
+
+ if (!src->object_cache)
+ continue;
+
+ assert(src != dst);
+ if (src == dst)
+ continue;
+
+ vk_pipeline_cache_lock(src);
+
+ set_foreach(src->object_cache, src_entry) {
+ struct vk_pipeline_cache_object *src_object = (void *)src_entry->key;
+
+ bool found_in_dst = false;
+ struct set_entry *dst_entry =
+ _mesa_set_search_or_add_pre_hashed(dst->object_cache,
+ src_entry->hash,
+ src_object, &found_in_dst);
+ if (found_in_dst) {
+ struct vk_pipeline_cache_object *dst_object = (void *)dst_entry->key;
+ if (dst_object->ops == &raw_data_object_ops &&
+ src_object->ops != &raw_data_object_ops) {
+ /* Even though dst has the object, it only has the blob version
+ * which isn't as useful. Replace it with the real object.
+ */
+ vk_pipeline_cache_object_unref(dst_object);
+ dst_entry->key = vk_pipeline_cache_object_ref(src_object);
+ }
+ } else {
+ /* We inserted src_object in dst so it needs a reference */
+ assert(dst_entry->key == (const void *)src_object);
+ vk_pipeline_cache_object_ref(src_object);
+ }
+ }
+
+ vk_pipeline_cache_unlock(src);
+ }
+
+ vk_pipeline_cache_unlock(dst);
+
+ return VK_SUCCESS;
+}
diff --git a/src/vulkan/runtime/vk_pipeline_cache.h b/src/vulkan/runtime/vk_pipeline_cache.h
new file mode 100644
index 00000000000..2340a2c12d1
--- /dev/null
+++ b/src/vulkan/runtime/vk_pipeline_cache.h
@@ -0,0 +1,243 @@
+/*
+ * Copyright © 2021 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifndef VK_PIPELINE_CACHE_H
+#define VK_PIPELINE_CACHE_H
+
+#include "vk_object.h"
+#include "vk_util.h"
+
+#include "util/simple_mtx.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* #include "util/blob.h" */
+struct blob;
+struct blob_reader;
+
+/* #include "util/set.h" */
+struct set;
+
+struct vk_pipeline_cache;
+struct vk_pipeline_cache_object;
+
+#define VK_PIPELINE_CACHE_BLOB_ALIGN 8
+
+struct vk_pipeline_cache_object_ops {
+ /** Writes this cache object to the given blob
+ *
+ * Because the cache works with both raw blob data and driver object data
+ * and can't always tell the difference between the two, we have to be very
+ * careful about alignments when [de]serializing. When serialize() is
+ * called, the blob will be aligned to VK_PIPELINE_CACHE_BLOB_ALIGN. The
+ * driver must be careful to not [de]serialize any data types which require
+ * a higher alignment. When deserialize() is called, the blob_reader is
+ * also guaranteed to be aligned to VK_PIPELINE_CACHE_BLOB_ALIGN.
+ *
+ * Returns true on success
+ *
+ * This function is optional. Objects without [de]serialization support
+ * will still be cached in memory but will not be placed in the disk cache
+ * and will not exported to the client when vkGetPipelineCacheData() is
+ * called.
+ */
+ bool (*serialize)(struct vk_pipeline_cache_object *object,
+ struct blob *blob);
+
+ /** Constructs an object from cached data
+ *
+ * See serialize() for details about data alignment.
+ *
+ * returns the created object
+ *
+ * This function is optional.
+ */
+ struct vk_pipeline_cache_object *(*deserialize)(struct vk_device *device,
+ const void *key_data,
+ size_t key_size,
+ struct blob_reader *blob);
+
+ /** Destroys the object
+ *
+ * Called when vk_pipeline_cache_object.ref_cnt hits 0.
+ */
+ void (*destroy)(struct vk_pipeline_cache_object *object);
+};
+
+/** Base struct for cached objects
+ *
+ * A vk_pipeline_cache stores any number of vk_pipeline_cache_object's, each
+ * of which has an associated key of arbitrary size. Cached objects are
+ * reference counted so that they can exist in multiple caches (for example,
+ * when vkMergePipelineCaches() is called) and so that they can persist after
+ * the pipeline cache is destroyed. Each object also has a pointer to a
+ * vk_pipeline_cache_object_ops table which the pipeline cache uses to
+ * [de]serialize the object and clean it up when the reference count hits 0.
+ *
+ * The rest of the details of any given object are entirely up to the driver.
+ * The driver may even have multiple types of objects (distinguished by their
+ * vk_pipeline_cache_object_ops table) in the cache so long as it guarantees
+ * it never has two objects of different types with the same key.
+ */
+struct vk_pipeline_cache_object {
+ struct vk_device *device;
+ const struct vk_pipeline_cache_object_ops *ops;
+ uint32_t ref_cnt;
+
+ uint32_t data_size;
+ const void *key_data;
+ uint32_t key_size;
+};
+
+static inline void
+vk_pipeline_cache_object_init(struct vk_device *device,
+ struct vk_pipeline_cache_object *object,
+ const struct vk_pipeline_cache_object_ops *ops,
+ const void *key_data, uint32_t key_size)
+{
+ object->device = device;
+ object->ops = ops;
+ p_atomic_set(&object->ref_cnt, 1);
+ object->data_size = 0; /* Unknown */
+ object->key_data = key_data;
+ object->key_size = key_size;
+}
+
+static inline void
+vk_pipeline_cache_object_finish(struct vk_pipeline_cache_object *object)
+{
+ assert(p_atomic_read(&object->ref_cnt) <= 1);
+}
+
+static inline struct vk_pipeline_cache_object *
+vk_pipeline_cache_object_ref(struct vk_pipeline_cache_object *object)
+{
+ assert(object && p_atomic_read(&object->ref_cnt) >= 1);
+ p_atomic_inc(&object->ref_cnt);
+ return object;
+}
+
+static inline void
+vk_pipeline_cache_object_unref(struct vk_pipeline_cache_object *object)
+{
+ assert(object && p_atomic_read(&object->ref_cnt) >= 1);
+ if (p_atomic_dec_zero(&object->ref_cnt))
+ object->ops->destroy(object);
+}
+
+/** A generic implementation of VkPipelineCache */
+struct vk_pipeline_cache {
+ struct vk_object_base base;
+
+ /* pCreateInfo::flags */
+ VkPipelineCacheCreateFlags flags;
+
+ struct vk_pipeline_cache_header header;
+
+ /** Protects object_cache */
+ simple_mtx_t lock;
+
+ struct set *object_cache;
+};
+
+VK_DEFINE_NONDISP_HANDLE_CASTS(vk_pipeline_cache, base, VkPipelineCache,
+ VK_OBJECT_TYPE_PIPELINE_CACHE)
+
+struct vk_pipeline_cache_create_info {
+ /* The pCreateInfo for this pipeline cache, if any.
+ *
+ * For driver-internal caches, this is allowed to be NULL.
+ */
+ const VkPipelineCacheCreateInfo *pCreateInfo;
+
+ /** If true, ignore VK_ENABLE_PIPELINE_CACHE and enable anyway */
+ bool force_enable;
+};
+
+struct vk_pipeline_cache *
+vk_pipeline_cache_create(struct vk_device *device,
+ const struct vk_pipeline_cache_create_info *info,
+ const VkAllocationCallbacks *pAllocator);
+void
+vk_pipeline_cache_destroy(struct vk_pipeline_cache *cache,
+ const VkAllocationCallbacks *pAllocator);
+
+/** Attempts to look up an object in the cache by key
+ *
+ * If an object is found in the cache matching the given key, *cache_hit is
+ * set to true and a reference to that object is returned.
+ *
+ * If the driver sets vk_device.disk_cache, we attempt to look up any missing
+ * objects in the disk cache before declaring failure. If an object is found
+ * in the disk cache but not the in-memory cache, *cache_hit is set to false.
+ *
+ * The deserialization of pipeline cache objects found in the cache data
+ * provided via VkPipelineCacheCreateInfo::pInitialData happens during
+ * vk_pipeline_cache_lookup() rather than during vkCreatePipelineCache().
+ * Prior to the first vk_pipeline_cache_lookup() of a given object, it is
+ * stored as an internal raw data object with the same hash. This allows us
+ * to avoid any complex object type tagging in the serialized cache. It does,
+ * however, mean that drivers need to be careful to ensure that objects with
+ * different types (ops) have different keys.
+ *
+ * Returns a reference to the object, if found
+ */
+struct vk_pipeline_cache_object * MUST_CHECK
+vk_pipeline_cache_lookup_object(struct vk_pipeline_cache *cache,
+ const void *key_data, size_t key_size,
+ const struct vk_pipeline_cache_object_ops *ops,
+ bool *cache_hit);
+
+/** Adds an object to the pipeline cache
+ *
+ * This function adds the given object to the pipeline cache. We do not
+ * specify a key here because the key is part of the object. See also
+ * vk_pipeline_cache_object_init().
+ *
+ * This function consumes a reference to the object and returns a reference to
+ * the (possibly different) object in the cache. The intended usage pattern
+ * is as follows:
+ *
+ * key = compute_key();
+ * struct vk_pipeline_cache_object *object =
+ * vk_pipeline_cache_lookup_object(cache, &key, sizeof(key),
+ * &driver_type_ops, &cache_hit);
+ * if (object != NULL)
+ * return container_of(object, driver_type, base);
+ *
+ * object = do_compile();
+ * assert(object != NULL);
+ *
+ * object = vk_pipeline_cache_add_object(cache, object);
+ * return container_of(object, driver_type, base);
+ */
+struct vk_pipeline_cache_object * MUST_CHECK
+vk_pipeline_cache_add_object(struct vk_pipeline_cache *cache,
+ struct vk_pipeline_cache_object *object);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* VK_PIPELINE_CACHE_H */
More information about the mesa-commit
mailing list