Merge "Fix ColorBufferVk layout on snapshot load again" into main

9 files changed, 134 insertions, 34 deletions

diff --git a/common/end2end/GfxstreamEnd2EndTests.cpp b/common/end2end/GfxstreamEnd2EndTests.cpp
index abee747d..2481034b 100644
--- a/common/end2end/GfxstreamEnd2EndTests.cpp
+++ b/common/end2end/GfxstreamEnd2EndTests.cpp
@@ -78,6 +78,7 @@ std::string TestParams::ToString() const {
     ret += "Gl";
     ret += (with_vk ? "With" : "Without");
     ret += "Vk";
+    ret += "SampleCount" + std::to_string(samples);
     if (!with_features.empty()) {
         ret += "WithFeatures_";
         ret += Join(with_features, "_");
diff --git a/common/end2end/GfxstreamEnd2EndTests.h b/common/end2end/GfxstreamEnd2EndTests.h
index 37051f23..f06082f2 100644
--- a/common/end2end/GfxstreamEnd2EndTests.h
+++ b/common/end2end/GfxstreamEnd2EndTests.h
@@ -450,6 +450,7 @@ enum class GfxstreamTransport {
 struct TestParams {
     bool with_gl;
     bool with_vk;
+    int samples = 1;
     std::unordered_set<std::string> with_features;
     GfxstreamTransport with_transport = GfxstreamTransport::kVirtioGpuAsg;
 
diff --git a/common/end2end/GfxstreamEnd2EndVkSnapshotPipelineTests.cpp b/common/end2end/GfxstreamEnd2EndVkSnapshotPipelineTests.cpp
index ac44708d..ec22f3c8 100644
--- a/common/end2end/GfxstreamEnd2EndVkSnapshotPipelineTests.cpp
+++ b/common/end2end/GfxstreamEnd2EndVkSnapshotPipelineTests.cpp
@@ -55,6 +55,9 @@ class GfxstreamEnd2EndVkSnapshotPipelineTest : public GfxstreamEnd2EndTest {
     static const uint32_t kFbHeight = 32;
 };
 
+class GfxstreamEnd2EndVkSnapshotPipelineWithMultiSamplingTest
+    : public GfxstreamEnd2EndVkSnapshotPipelineTest {};
+
 template <typename DurationType>
 constexpr uint64_t AsVkTimeout(DurationType duration) {
     return static_cast<uint64_t>(
@@ -71,6 +74,7 @@ vkhpp::UniqueRenderPass GfxstreamEnd2EndVkSnapshotPipelineTest::createRenderPass
     vkhpp::Device device) {
     vkhpp::AttachmentDescription colorAttachmentDescription = {
         .format = vkhpp::Format::eR8G8B8A8Unorm,
+        .samples = static_cast<vkhpp::SampleCountFlagBits>(GetParam().samples),
         .loadOp = vkhpp::AttachmentLoadOp::eLoad,
         .storeOp = vkhpp::AttachmentStoreOp::eStore,
         .initialLayout = vkhpp::ImageLayout::eColorAttachmentOptimal,
@@ -166,7 +170,7 @@ std::unique_ptr<PipelineInfo> GfxstreamEnd2EndVkSnapshotPipelineTest::createPipe
     };
 
     const vkhpp::PipelineMultisampleStateCreateInfo pipelineMultisampleStateCreateInfo = {
-        .rasterizationSamples = vkhpp::SampleCountFlagBits::e1,
+        .rasterizationSamples = static_cast<vkhpp::SampleCountFlagBits>(GetParam().samples),
     };
     const vkhpp::PipelineDepthStencilStateCreateInfo pipelineDepthStencilStateCreateInfo = {};
     const vkhpp::PipelineColorBlendAttachmentState pipelineColorBlendAttachmentState = {
@@ -225,7 +229,7 @@ std::unique_ptr<ImageInfo> GfxstreamEnd2EndVkSnapshotPipelineTest::createColorAt
         .usage = vkhpp::ImageUsageFlagBits::eColorAttachment | vkhpp::ImageUsageFlagBits::eSampled |
                  vkhpp::ImageUsageFlagBits::eTransferDst | vkhpp::ImageUsageFlagBits::eTransferSrc,
         .sharingMode = vkhpp::SharingMode::eExclusive,
-        .samples = vkhpp::SampleCountFlagBits::e1,
+        .samples = static_cast<vkhpp::SampleCountFlagBits>(GetParam().samples),
     };
     res->image = device.createImageUnique(imageCreateInfo).value;
 
@@ -354,7 +358,7 @@ TEST_P(GfxstreamEnd2EndVkSnapshotPipelineTest, CanSnapshotFramebuffer) {
     SnapshotSaveAndLoad();
 }
 
-TEST_P(GfxstreamEnd2EndVkSnapshotPipelineTest, CanSubmitQueue) {
+TEST_P(GfxstreamEnd2EndVkSnapshotPipelineWithMultiSamplingTest, CanSubmitQueue) {
     auto [instance, physicalDevice, device, queue, queueFamilyIndex] =
         VK_ASSERT(SetUpTypicalVkTestEnvironment());
     auto pipelineInfo = createPipeline(device.get());
@@ -504,6 +508,25 @@ INSTANTIATE_TEST_CASE_P(GfxstreamEnd2EndTests, GfxstreamEnd2EndVkSnapshotPipelin
                             },
                         }),
                         &GetTestName);
+
+INSTANTIATE_TEST_CASE_P(GfxstreamEnd2EndTests,
+                        GfxstreamEnd2EndVkSnapshotPipelineWithMultiSamplingTest,
+                        ::testing::ValuesIn({
+                            TestParams{
+                                .with_gl = false,
+                                .with_vk = true,
+                                .samples = 1,
+                                .with_features = {"VulkanSnapshots"},
+                            },
+                            TestParams{
+                                .with_gl = false,
+                                .with_vk = true,
+                                .samples = 4,
+                                .with_features = {"VulkanSnapshots"},
+                            },
+                        }),
+                        &GetTestName);
+
 }  // namespace
 }  // namespace tests
 }  // namespace gfxstream
\ No newline at end of file
diff --git a/host/vulkan/ColorBufferVk.cpp b/host/vulkan/ColorBufferVk.cpp
index d8b66c6d..40fac75a 100644
--- a/host/vulkan/ColorBufferVk.cpp
+++ b/host/vulkan/ColorBufferVk.cpp
@@ -25,16 +25,15 @@ std::unique_ptr<ColorBufferVk> ColorBufferVk::create(uint32_t handle, uint32_t w
                                                      FrameworkFormat frameworkFormat,
                                                      bool vulkanOnly, uint32_t memoryProperty,
                                                      android::base::Stream* stream) {
-    VkImageLayout initLayout = VK_IMAGE_LAYOUT_UNDEFINED;
-    if (getGlobalVkEmulation()->features.VulkanSnapshots.enabled && stream) {
-        initLayout = static_cast<VkImageLayout>(stream->getBe32());
-    }
     if (!createVkColorBuffer(width, height, format, frameworkFormat, handle, vulkanOnly,
                              memoryProperty)) {
         GL_LOG("Failed to create ColorBufferVk:%d", handle);
         return nullptr;
     }
-
+    if (getGlobalVkEmulation()->features.VulkanSnapshots.enabled && stream) {
+        VkImageLayout currentLayout = static_cast<VkImageLayout>(stream->getBe32());
+        setColorBufferCurrentLayout(handle, currentLayout);
+    }
     return std::unique_ptr<ColorBufferVk>(new ColorBufferVk(handle));
 }
 
diff --git a/host/vulkan/VkCommonOperations.cpp b/host/vulkan/VkCommonOperations.cpp
index c451296e..bd7db479 100644
--- a/host/vulkan/VkCommonOperations.cpp
+++ b/host/vulkan/VkCommonOperations.cpp
@@ -1924,13 +1924,9 @@ static bool updateExternalMemoryInfo(VK_EXT_MEMORY_HANDLE extMemHandle,
 // We should make it so the guest can only allocate external images/
 // buffers of one type index for image and one type index for buffer
 // to begin with, via filtering from the host.
-//
-// initLayout was used for snapshot purpose, to recover image layout
-// after snapshot load.
 
 bool initializeVkColorBufferLocked(
-    uint32_t colorBufferHandle, VkImageLayout initLayout = VK_IMAGE_LAYOUT_UNDEFINED,
-    VK_EXT_MEMORY_HANDLE extMemHandle = VK_EXT_MEMORY_HANDLE_INVALID) {
+    uint32_t colorBufferHandle, VK_EXT_MEMORY_HANDLE extMemHandle = VK_EXT_MEMORY_HANDLE_INVALID) {
     auto infoPtr = android::base::find(sVkEmulation->colorBuffers, colorBufferHandle);
     // Not initialized
     if (!infoPtr) {
@@ -1990,7 +1986,7 @@ bool initializeVkColorBufferLocked(
     imageCi->sharingMode = VK_SHARING_MODE_EXCLUSIVE;
     imageCi->queueFamilyIndexCount = 0;
     imageCi->pQueueFamilyIndices = nullptr;
-    imageCi->initialLayout = initLayout;
+    imageCi->initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
 
     // Create the image. If external memory is supported, make it external.
     VkExternalMemoryImageCreateInfo extImageCi = {
@@ -2021,7 +2017,6 @@ bool initializeVkColorBufferLocked(
     bool useDedicated = sVkEmulation->useDedicatedAllocations;
 
     infoPtr->imageCreateInfoShallow = vk_make_orphan_copy(*imageCi);
-    infoPtr->currentLayout = infoPtr->imageCreateInfoShallow.initialLayout;
     infoPtr->currentQueueFamilyIndex = sVkEmulation->queueFamilyIndex;
 
     if (!useDedicated && vk->vkGetImageMemoryRequirements2KHR) {
@@ -2196,7 +2191,7 @@ static bool createVkColorBufferLocked(uint32_t width, uint32_t height, GLenum in
 
 bool createVkColorBuffer(uint32_t width, uint32_t height, GLenum internalFormat,
                          FrameworkFormat frameworkFormat, uint32_t colorBufferHandle,
-                         bool vulkanOnly, uint32_t memoryProperty, VkImageLayout initLayout) {
+                         bool vulkanOnly, uint32_t memoryProperty) {
     if (!sVkEmulation || !sVkEmulation->live) {
         GFXSTREAM_ABORT(FatalError(ABORT_REASON_OTHER)) << "VkEmulation not available.";
     }
@@ -2217,7 +2212,7 @@ bool createVkColorBuffer(uint32_t width, uint32_t height, GLenum internalFormat,
         return true;
     }
 
-    return initializeVkColorBufferLocked(colorBufferHandle, initLayout);
+    return initializeVkColorBufferLocked(colorBufferHandle);
 }
 
 std::optional<VkColorBufferMemoryExport> exportColorBufferMemory(uint32_t colorBufferHandle) {
@@ -2308,8 +2303,7 @@ bool importExtMemoryHandleToVkColorBuffer(uint32_t colorBufferHandle, uint32_t t
     AutoLock lock(sVkEmulationLock);
     // Initialize the colorBuffer with the external memory handle
     // Note that this will fail if the colorBuffer memory was previously initialized.
-    return initializeVkColorBufferLocked(colorBufferHandle, VK_IMAGE_LAYOUT_UNDEFINED,
-                                         extMemHandle);
+    return initializeVkColorBufferLocked(colorBufferHandle, extMemHandle);
 }
 
 VkEmulation::ColorBufferInfo getColorBufferInfo(uint32_t colorBufferHandle) {
@@ -2683,12 +2677,18 @@ static bool updateColorBufferFromBytesLocked(uint32_t colorBufferHandle, uint32_
     sVkEmulation->debugUtilsHelper.cmdBeginDebugLabel(
         commandBuffer, "updateColorBufferFromBytes(ColorBuffer:%d)", colorBufferHandle);
 
+    bool isSnapshotLoad =
+        VkDecoderGlobalState::get()->getSnapshotState() == VkDecoderGlobalState::Loading;
+    VkImageLayout currentLayout = colorBufferInfo->currentLayout;
+    if (isSnapshotLoad) {
+        currentLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+    }
     const VkImageMemoryBarrier toTransferDstImageBarrier = {
         .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
         .pNext = nullptr,
         .srcAccessMask = 0,
         .dstAccessMask = VK_ACCESS_HOST_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT,
-        .oldLayout = colorBufferInfo->currentLayout,
+        .oldLayout = currentLayout,
         .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
         .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
         .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
@@ -2703,17 +2703,42 @@ static bool updateColorBufferFromBytesLocked(uint32_t colorBufferHandle, uint32_
             },
     };
 
-    vk->vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
-                             VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1,
+    vk->vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
+                             VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, 0, nullptr, 1,
                              &toTransferDstImageBarrier);
 
-    colorBufferInfo->currentLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
-
     // Copy to staging buffer
     vk->vkCmdCopyBufferToImage(commandBuffer, sVkEmulation->staging.buffer, colorBufferInfo->image,
                                VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, bufferImageCopies.size(),
                                bufferImageCopies.data());
 
+    if (isSnapshotLoad && colorBufferInfo->currentLayout != VK_IMAGE_LAYOUT_UNDEFINED) {
+        const VkImageMemoryBarrier toCurrentLayoutImageBarrier = {
+            .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+            .pNext = nullptr,
+            .srcAccessMask = VK_ACCESS_HOST_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT,
+            .dstAccessMask = VK_ACCESS_NONE_KHR,
+            .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
+            .newLayout = colorBufferInfo->currentLayout,
+            .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+            .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+            .image = colorBufferInfo->image,
+            .subresourceRange =
+                {
+                    .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+                    .baseMipLevel = 0,
+                    .levelCount = 1,
+                    .baseArrayLayer = 0,
+                    .layerCount = 1,
+                },
+        };
+        vk->vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_HOST_BIT,
+                                 VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, nullptr, 0, nullptr, 1,
+                                 &toCurrentLayoutImageBarrier);
+    } else {
+        colorBufferInfo->currentLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+    }
+
     sVkEmulation->debugUtilsHelper.cmdEndDebugLabel(commandBuffer);
 
     VK_CHECK(vk->vkEndCommandBuffer(commandBuffer));
diff --git a/host/vulkan/VkCommonOperations.h b/host/vulkan/VkCommonOperations.h
index 9007ffb4..04035882 100644
--- a/host/vulkan/VkCommonOperations.h
+++ b/host/vulkan/VkCommonOperations.h
@@ -463,8 +463,7 @@ std::unique_ptr<VkImageCreateInfo> generateColorBufferVkImageCreateInfo(VkFormat
 
 bool createVkColorBuffer(uint32_t width, uint32_t height, GLenum format,
                          FrameworkFormat frameworkFormat, uint32_t colorBufferHandle,
-                         bool vulkanOnly, uint32_t memoryProperty,
-                         VkImageLayout initLayout = VK_IMAGE_LAYOUT_UNDEFINED);
+                         bool vulkanOnly, uint32_t memoryProperty);
 
 bool teardownVkColorBuffer(uint32_t colorBufferHandle);
 
diff --git a/host/vulkan/VkDecoderGlobalState.cpp b/host/vulkan/VkDecoderGlobalState.cpp
index f00750e9..cbf70374 100644
--- a/host/vulkan/VkDecoderGlobalState.cpp
+++ b/host/vulkan/VkDecoderGlobalState.cpp
@@ -6262,6 +6262,7 @@ class VkDecoderGlobalState::Impl {
     GOLDFISH_VK_LIST_NON_DISPATCHABLE_HANDLE_TYPES(DEFINE_BOXED_NON_DISPATCHABLE_HANDLE_API_IMPL)
 
     VkDecoderSnapshot* snapshot() { return &mSnapshot; }
+    SnapshotState getSnapshotState() { return mSnapshotState; }
 
    private:
     bool isEmulatedInstanceExtension(const char* name) const {
@@ -7161,11 +7162,6 @@ class VkDecoderGlobalState::Impl {
     std::unordered_map<LinearImageCreateInfo, LinearImageProperties, LinearImageCreateInfo::Hash>
         mLinearImageProperties;
 
-    enum SnapshotState {
-        Normal,
-        Saving,
-        Loading,
-    };
     SnapshotState mSnapshotState = SnapshotState::Normal;
 };
 
@@ -7191,6 +7187,10 @@ void VkDecoderGlobalState::reset() {
 // Snapshots
 bool VkDecoderGlobalState::snapshotsEnabled() const { return mImpl->snapshotsEnabled(); }
 
+VkDecoderGlobalState::SnapshotState VkDecoderGlobalState::getSnapshotState() const {
+    return mImpl->getSnapshotState();
+}
+
 const gfxstream::host::FeatureSet& VkDecoderGlobalState::getFeatures() const { return mImpl->getFeatures(); }
 
 bool VkDecoderGlobalState::vkCleanupEnabled() const { return mImpl->vkCleanupEnabled(); }
diff --git a/host/vulkan/VkDecoderGlobalState.h b/host/vulkan/VkDecoderGlobalState.h
index 61cd1d0f..0655a192 100644
--- a/host/vulkan/VkDecoderGlobalState.h
+++ b/host/vulkan/VkDecoderGlobalState.h
@@ -74,9 +74,17 @@ class VkDecoderGlobalState {
     // For testing only - destroys the global instance of VkDecoderGlobalState.
     static void reset();
 
+    enum SnapshotState {
+        Normal,
+        Saving,
+        Loading,
+    };
+
     // Snapshot save/load
     bool snapshotsEnabled() const;
 
+    SnapshotState getSnapshotState() const;
+
     const gfxstream::host::FeatureSet& getFeatures() const;
 
     // Whether to clean up VK instance.
diff --git a/host/vulkan/VkDecoderSnapshotUtils.cpp b/host/vulkan/VkDecoderSnapshotUtils.cpp
index 59105c46..e7fe8c43 100644
--- a/host/vulkan/VkDecoderSnapshotUtils.cpp
+++ b/host/vulkan/VkDecoderSnapshotUtils.cpp
@@ -272,6 +272,7 @@ void saveImageContent(android::base::Stream* stream, StateBlock* stateBlock, VkI
     dispatch->vkUnmapMemory(stateBlock->device, readbackMemory);
     dispatch->vkDestroyBuffer(stateBlock->device, readbackBuffer, nullptr);
     dispatch->vkFreeMemory(stateBlock->device, readbackMemory, nullptr);
+    dispatch->vkFreeCommandBuffers(stateBlock->device, stateBlock->commandPool, 1, &commandBuffer);
 }
 
 void loadImageContent(android::base::Stream* stream, StateBlock* stateBlock, VkImage image,
@@ -279,9 +280,6 @@ void loadImageContent(android::base::Stream* stream, StateBlock* stateBlock, VkI
     if (imageInfo->layout == VK_IMAGE_LAYOUT_UNDEFINED) {
         return;
     }
-    if (imageInfo->imageCreateInfoShallow.samples != VK_SAMPLE_COUNT_1_BIT) {
-        return;
-    }
     VkEmulation* vkEmulation = getGlobalVkEmulation();
     VulkanDispatch* dispatch = vkEmulation->dvk;
     const VkImageCreateInfo& imageCreateInfo = imageInfo->imageCreateInfoShallow;
@@ -299,6 +297,51 @@ void loadImageContent(android::base::Stream* stream, StateBlock* stateBlock, VkI
     };
     VkFence fence;
     _RUN_AND_CHECK(dispatch->vkCreateFence(stateBlock->device, &fenceCreateInfo, nullptr, &fence));
+    if (imageInfo->imageCreateInfoShallow.samples != VK_SAMPLE_COUNT_1_BIT) {
+        // Set the layout and quit
+        // TODO: resolve and save image content
+        // TODO: get the right aspect
+        VkImageAspectFlags aspects = VK_IMAGE_ASPECT_COLOR_BIT;
+        VkImageMemoryBarrier imgMemoryBarrier = {
+            .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+            .pNext = nullptr,
+            .srcAccessMask = static_cast<VkAccessFlags>(~VK_ACCESS_NONE_KHR),
+            .dstAccessMask = static_cast<VkAccessFlags>(~VK_ACCESS_NONE_KHR),
+            .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+            .newLayout = imageInfo->layout,
+            .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+            .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+            .image = image,
+            .subresourceRange = VkImageSubresourceRange{.aspectMask = aspects,
+                                                        .baseMipLevel = 0,
+                                                        .levelCount = VK_REMAINING_MIP_LEVELS,
+                                                        .baseArrayLayer = 0,
+                                                        .layerCount = VK_REMAINING_ARRAY_LAYERS}};
+        VkCommandBufferBeginInfo beginInfo{
+            .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
+        };
+        _RUN_AND_CHECK(dispatch->vkBeginCommandBuffer(commandBuffer, &beginInfo) != VK_SUCCESS);
+
+        dispatch->vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
+                                       VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0,
+                                       nullptr, 1, &imgMemoryBarrier);
+
+        _RUN_AND_CHECK(dispatch->vkEndCommandBuffer(commandBuffer));
+
+        // Execute the command to copy image
+        VkSubmitInfo submitInfo = {
+            .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
+            .commandBufferCount = 1,
+            .pCommandBuffers = &commandBuffer,
+        };
+        _RUN_AND_CHECK(dispatch->vkQueueSubmit(stateBlock->queue, 1, &submitInfo, fence));
+        _RUN_AND_CHECK(
+            dispatch->vkWaitForFences(stateBlock->device, 1, &fence, VK_TRUE, 3000000000L));
+        dispatch->vkDestroyFence(stateBlock->device, fence, nullptr);
+        dispatch->vkFreeCommandBuffers(stateBlock->device, stateBlock->commandPool, 1,
+                                       &commandBuffer);
+        return;
+    }
     VkBufferCreateInfo bufferCreateInfo = {
         .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
         .size = static_cast<VkDeviceSize>(
@@ -422,6 +465,7 @@ void loadImageContent(android::base::Stream* stream, StateBlock* stateBlock, VkI
     dispatch->vkUnmapMemory(stateBlock->device, stagingMemory);
     dispatch->vkDestroyBuffer(stateBlock->device, stagingBuffer, nullptr);
     dispatch->vkFreeMemory(stateBlock->device, stagingMemory, nullptr);
+    dispatch->vkFreeCommandBuffers(stateBlock->device, stateBlock->commandPool, 1, &commandBuffer);
 }
 
 }  // namespace vk