Generate debug info for DEBUG'ed shaders

If we're debugging an HLSL shader, tell DXC to emit debug info with the `-fspv-debug=rich` flag.

Add tests - including for existing `DEBUG` parser logic that wasn't tested before.

10 files changed, 347 insertions, 9 deletions

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index b8dce3d..00e3e54 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -140,6 +140,7 @@ if (${AMBER_ENABLE_TESTS})
     amberscript/parser_clear_test.cc
     amberscript/parser_compile_options_test.cc
     amberscript/parser_copy_test.cc
+    amberscript/parser_debug_test.cc
     amberscript/parser_depth_test.cc
     amberscript/parser_device_feature_test.cc
     amberscript/parser_expect_test.cc
diff --git a/src/amberscript/parser.cc b/src/amberscript/parser.cc
index 0936bcd..1a5a197 100644
--- a/src/amberscript/parser.cc
+++ b/src/amberscript/parser.cc
@@ -2606,6 +2606,15 @@ Result Parser::ParseDebug() {
     return res;
   }
 
+  // As ParseRun() succeeded, we know it emplaced a run command at the back of
+  // the command_list_.
+  auto cmd = command_list_.back().get();
+
+  // We also know this command must derive from PipelineCommand, as it is
+  // runnable.
+  auto pipeline_cmd = static_cast<PipelineCommand*>(cmd);
+  auto pipeline = pipeline_cmd->GetPipeline();
+
   auto dbg = debug::Script::Create();
   for (auto token = tokenizer_->NextToken();; token = tokenizer_->NextToken()) {
     if (token->IsEOL())
@@ -2616,7 +2625,7 @@ Result Parser::ParseDebug() {
       break;
 
     if (token->AsString() == "THREAD") {
-      res = ParseDebugThread(dbg.get());
+      res = ParseDebugThread(dbg.get(), pipeline);
       if (!res.IsSuccess()) {
         return res;
       }
@@ -2625,14 +2634,18 @@ Result Parser::ParseDebug() {
     }
   }
 
-  command_list_.back()->SetDebugScript(std::move(dbg));
+  cmd->SetDebugScript(std::move(dbg));
 
   return Result();
 }
 
-Result Parser::ParseDebugThread(debug::Events* dbg) {
+Result Parser::ParseDebugThread(debug::Events* dbg, Pipeline* pipeline) {
   auto token = tokenizer_->NextToken();
   if (token->AsString() == "GLOBAL_INVOCATION_ID") {
+    for (auto& shader : pipeline->GetShaders()) {
+      shader.SetEmitDebugInfo(true);
+    }
+
     uint32_t invocation[3] = {};
     for (int i = 0; i < 3; i++) {
       token = tokenizer_->NextToken();
@@ -2650,6 +2663,12 @@ Result Parser::ParseDebugThread(debug::Events* dbg) {
     dbg->BreakOnComputeGlobalInvocation(invocation[0], invocation[1],
                                         invocation[2], thread);
   } else if (token->AsString() == "VERTEX_INDEX") {
+    for (auto& shader : pipeline->GetShaders()) {
+      if (shader.GetShaderType() == kShaderTypeVertex) {
+        shader.SetEmitDebugInfo(true);
+      }
+    }
+
     token = tokenizer_->NextToken();
     if (!token->IsInteger())
       return Result("expected vertex index");
@@ -2663,6 +2682,12 @@ Result Parser::ParseDebugThread(debug::Events* dbg) {
 
     dbg->BreakOnVertexIndex(vertex_index, thread);
   } else if (token->AsString() == "FRAGMENT_WINDOW_SPACE_POSITION") {
+    for (auto& shader : pipeline->GetShaders()) {
+      if (shader.GetShaderType() == kShaderTypeFragment) {
+        shader.SetEmitDebugInfo(true);
+      }
+    }
+
     token = tokenizer_->NextToken();
     if (!token->IsInteger())
       return Result("expected x unsigned integer coordinate");
diff --git a/src/amberscript/parser.h b/src/amberscript/parser.h
index f181f13..e8cb835 100644
--- a/src/amberscript/parser.h
+++ b/src/amberscript/parser.h
@@ -74,7 +74,7 @@ class Parser : public amber::Parser {
   Result ParsePipelineStencil(Pipeline* pipeline);
   Result ParseRun();
   Result ParseDebug();
-  Result ParseDebugThread(debug::Events*);
+  Result ParseDebugThread(debug::Events*, Pipeline* pipeline);
   Result ParseDebugThreadBody(debug::Thread* thread);
   Result ParseClear();
   Result ParseClearColor();
diff --git a/src/amberscript/parser_debug_test.cc b/src/amberscript/parser_debug_test.cc
new file mode 100644
index 0000000..8ce8577
--- /dev/null
+++ b/src/amberscript/parser_debug_test.cc
@@ -0,0 +1,296 @@
+// Copyright 2020 The Amber Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or parseried.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+#include "src/amberscript/parser.h"
+#include "src/shader_data.h"
+
+namespace amber {
+namespace amberscript {
+
+namespace {
+class ThreadEventRecorder : public debug::Thread {
+  std::stringstream& events;
+  std::string indent = "  ";
+
+ public:
+  explicit ThreadEventRecorder(std::stringstream& ev) : events(ev) {}
+
+  void StepOver() override { events << indent << "STEP_OVER" << std::endl; }
+  void StepIn() override { events << indent << "STEP_IN" << std::endl; }
+  void StepOut() override { events << indent << "STEP_OUT" << std::endl; }
+  void Continue() override { events << indent << "CONTINUE" << std::endl; }
+  void ExpectLocation(const debug::Location& location,
+                      const std::string& line) override {
+    events << indent << "EXPECT LOCATION \"" << location.file << "\" "
+           << location.line;
+    if (!line.empty()) {
+      events << " \"" << line << "\"";
+    }
+    events << std::endl;
+  }
+  void ExpectCallstack(
+      const std::vector<debug::StackFrame>& callstack) override {
+    events << indent << "EXPECT CALLSTACK";
+    for (auto& frame : callstack) {
+      events << indent << "  " << frame.name << " " << frame.location.file
+             << ":" << frame.location.line << " " << std::endl;
+    }
+    events << std::endl;
+  }
+  void ExpectLocal(const std::string& name, int64_t value) override {
+    events << indent << "EXPECT LOCAL \"" << name << "\" EQ " << value
+           << std::endl;
+  }
+  void ExpectLocal(const std::string& name, double value) override {
+    events << indent << "EXPECT LOCAL \"" << name << "\" EQ " << value
+           << std::endl;
+  }
+  void ExpectLocal(const std::string& name, const std::string& value) override {
+    events << indent << "EXPECT LOCAL \"" << name << "\" EQ \"" << value << "\""
+           << std::endl;
+  }
+};
+
+class EventRecorder : public debug::Events {
+ public:
+  std::stringstream events;
+
+  void record(const std::shared_ptr<const debug::ThreadScript>& script) {
+    ThreadEventRecorder thread{events};
+    script->Run(&thread);
+  }
+  void BreakOnComputeGlobalInvocation(
+      uint32_t x,
+      uint32_t y,
+      uint32_t z,
+      const std::shared_ptr<const debug::ThreadScript>& script) override {
+    events << "THREAD GLOBAL_INVOCATION_ID " << x << " " << y << " " << z
+           << std::endl;
+    record(script);
+    events << "END" << std::endl;
+  }
+  void BreakOnVertexIndex(
+      uint32_t index,
+      const std::shared_ptr<const debug::ThreadScript>& script) override {
+    events << "THREAD VERTEX_INDEX " << index << std::endl;
+    record(script);
+    events << "END" << std::endl;
+  }
+  void BreakOnFragmentWindowSpacePosition(
+      uint32_t x,
+      uint32_t y,
+      const std::shared_ptr<const debug::ThreadScript>& script) override {
+    events << "THREAD FRAGMENT_WINDOW_SPACE_POSITION " << x << " " << y
+           << std::endl;
+    record(script);
+    events << "END" << std::endl;
+  }
+};
+}  // namespace
+
+using AmberScriptParserTest = testing::Test;
+
+TEST_F(AmberScriptParserTest, DebugEventsScript) {
+  std::string dbg = R"(THREAD GLOBAL_INVOCATION_ID 1 2 3
+  EXPECT LOCATION "compute.hlsl" 2
+  STEP_IN
+  EXPECT LOCAL "one" EQ 1
+  STEP_OUT
+  EXPECT LOCAL "pi" EQ 3.14
+  STEP_OVER
+  EXPECT LOCAL "cat" EQ "meow"
+  CONTINUE
+END
+THREAD VERTEX_INDEX 2
+  EXPECT LOCATION "vertex.hlsl" 2 "  dog:woof cat:meow duck:quack"
+END
+THREAD FRAGMENT_WINDOW_SPACE_POSITION 4 5
+  EXPECT LOCATION "fragment.hlsl" 42
+  CONTINUE
+END
+)";
+
+  std::string in = R"(
+SHADER compute dbg_compute GLSL
+void main() {}
+END
+
+PIPELINE compute my_pipeline
+  ATTACH dbg_compute
+END
+
+DEBUG my_pipeline 2 4 5
+)" + dbg + "END";
+
+  Parser parser;
+  Result r = parser.Parse(in);
+  ASSERT_TRUE(r.IsSuccess()) << r.Error();
+
+  auto script = parser.GetScript();
+  const auto& commands = script->GetCommands();
+  ASSERT_EQ(1U, commands.size());
+
+  auto* cmd = commands[0].get();
+  ASSERT_TRUE(cmd->IsCompute());
+  auto* compute = cmd->AsCompute();
+  EXPECT_EQ(2U, compute->GetX());
+  EXPECT_EQ(4U, compute->GetY());
+  EXPECT_EQ(5U, compute->GetZ());
+
+  EventRecorder event_recorder;
+  compute->GetDebugScript()->Run(&event_recorder);
+  EXPECT_EQ(dbg, event_recorder.events.str());
+
+  auto& shaders = compute->GetPipeline()->GetShaders();
+  ASSERT_EQ(1U, shaders.size());
+
+  EXPECT_EQ(true, shaders[0].GetEmitDebugInfo());
+}
+
+TEST_F(AmberScriptParserTest, DebugEmitDebugInfoVertex) {
+  std::string dbg = R"()";
+
+  std::string in = R"(
+SHADER vertex dbg_vertex GLSL
+void main() {}
+END
+
+SHADER fragment dbg_fragment GLSL
+void main() {}
+END
+
+BUFFER position_buf DATA_TYPE R8G8_SNORM DATA
+ 1 1 2 2 3 3
+END
+
+PIPELINE graphics my_pipeline
+  ATTACH dbg_vertex
+  ATTACH dbg_fragment
+  VERTEX_DATA position_buf LOCATION 0
+END
+
+DEBUG my_pipeline DRAW_ARRAY AS TRIANGLE_LIST START_IDX 0 COUNT 1
+  THREAD VERTEX_INDEX 100
+  END
+END)";
+
+  Parser parser;
+  Result r = parser.Parse(in);
+  ASSERT_TRUE(r.IsSuccess()) << r.Error();
+
+  auto script = parser.GetScript();
+  const auto& commands = script->GetCommands();
+  ASSERT_EQ(1U, commands.size());
+  auto* cmd = commands[0].get();
+  ASSERT_TRUE(cmd->IsDrawArrays());
+  auto* draw = cmd->AsDrawArrays();
+
+  for (auto& shader : draw->GetPipeline()->GetShaders()) {
+    bool expect_debug_info = shader.GetShaderType() == kShaderTypeVertex;
+    EXPECT_EQ(expect_debug_info, shader.GetEmitDebugInfo())
+        << "Emit debug info for shader type " << shader.GetShaderType();
+  }
+}
+
+TEST_F(AmberScriptParserTest, DebugEmitDebugInfoFragment) {
+  std::string dbg = R"()";
+
+  std::string in = R"(
+SHADER vertex dbg_vertex GLSL
+void main() {}
+END
+
+SHADER fragment dbg_fragment GLSL
+void main() {}
+END
+
+BUFFER position_buf DATA_TYPE R8G8_SNORM DATA
+ 1 1 2 2 3 3
+END
+
+PIPELINE graphics my_pipeline
+  ATTACH dbg_vertex
+  ATTACH dbg_fragment
+  VERTEX_DATA position_buf LOCATION 0
+END
+
+DEBUG my_pipeline DRAW_ARRAY AS TRIANGLE_LIST START_IDX 0 COUNT 1
+  THREAD FRAGMENT_WINDOW_SPACE_POSITION 1 2
+  END
+END)";
+
+  Parser parser;
+  Result r = parser.Parse(in);
+  ASSERT_TRUE(r.IsSuccess()) << r.Error();
+
+  auto script = parser.GetScript();
+  const auto& commands = script->GetCommands();
+  ASSERT_EQ(1U, commands.size());
+  auto* cmd = commands[0].get();
+  ASSERT_TRUE(cmd->IsDrawArrays());
+  auto* draw = cmd->AsDrawArrays();
+
+  for (auto& shader : draw->GetPipeline()->GetShaders()) {
+    bool expect_debug_info = shader.GetShaderType() == kShaderTypeFragment;
+    EXPECT_EQ(expect_debug_info, shader.GetEmitDebugInfo())
+        << "Emit debug info for shader type " << shader.GetShaderType();
+  }
+}
+
+TEST_F(AmberScriptParserTest, DebugEmitNoDebugInfo) {
+  std::string dbg = R"()";
+
+  std::string in = R"(
+SHADER vertex dbg_vertex GLSL
+void main() {}
+END
+
+SHADER fragment dbg_fragment GLSL
+void main() {}
+END
+
+BUFFER position_buf DATA_TYPE R8G8_SNORM DATA
+ 1 1 2 2 3 3
+END
+
+PIPELINE graphics my_pipeline
+  ATTACH dbg_vertex
+  ATTACH dbg_fragment
+  VERTEX_DATA position_buf LOCATION 0
+END
+
+RUN my_pipeline DRAW_ARRAY AS TRIANGLE_LIST START_IDX 0 COUNT 1
+)";
+
+  Parser parser;
+  Result r = parser.Parse(in);
+  ASSERT_TRUE(r.IsSuccess()) << r.Error();
+
+  auto script = parser.GetScript();
+  const auto& commands = script->GetCommands();
+  ASSERT_EQ(1U, commands.size());
+  auto* cmd = commands[0].get();
+  ASSERT_TRUE(cmd->IsDrawArrays());
+  auto* draw = cmd->AsDrawArrays();
+
+  for (auto& shader : draw->GetPipeline()->GetShaders()) {
+    EXPECT_EQ(false, shader.GetEmitDebugInfo());
+  }
+}
+
+}  // namespace amberscript
+}  // namespace amber
diff --git a/src/dxc_helper.cc b/src/dxc_helper.cc
index 8fb0f8a..8fa0482 100644
--- a/src/dxc_helper.cc
+++ b/src/dxc_helper.cc
@@ -137,6 +137,7 @@ Result Compile(const std::string& src,
                const std::string& spv_env,
                const std::string& filename,
                const VirtualFileStore* virtual_files,
+               bool emit_debug_info,
                std::vector<uint32_t>* generated_binary) {
   if (hlsl::options::initHlslOptTable()) {
     DxcCleanupThreadMalloc();
@@ -177,6 +178,10 @@ Result Compile(const std::string& src,
   std::string filepath = filename.empty() ? ("amber." + profile) : filename;
 
   std::vector<const wchar_t*> dxc_flags(kDxcFlags, &kDxcFlags[kDxcFlagsCount]);
+  if (emit_debug_info) {  // Enable debug info generation
+    dxc_flags.emplace_back(L"-fspv-debug=rich");
+  }
+
   const wchar_t* target_env = nullptr;
   if (!spv_env.compare("spv1.3") || !spv_env.compare("vulkan1.1")) {
     target_env = L"-fspv-target-env=vulkan1.1";
diff --git a/src/dxc_helper.h b/src/dxc_helper.h
index eb83216..2cd4212 100644
--- a/src/dxc_helper.h
+++ b/src/dxc_helper.h
@@ -34,6 +34,7 @@ Result Compile(const std::string& src,
                const std::string& profile_str,
                const std::string& spv_env,
                const VirtualFileStore* virtual_files,
+               bool emit_debug_info,
                std::vector<uint32_t>* generated_binary);
 
 }  // namespace dxchelper
diff --git a/src/pipeline.cc b/src/pipeline.cc
index e61f74c..7df2401 100644
--- a/src/pipeline.cc
+++ b/src/pipeline.cc
@@ -54,8 +54,9 @@ Pipeline::ShaderInfo::ShaderInfo(Shader* shader, ShaderType type)
       entry_point_("main"),
       required_subgroup_size_setting_(RequiredSubgroupSizeSetting::kNotSet),
       required_subgroup_size_(0),
-      varying_subgroup_size_(0),
-      require_full_subgroups_(0) {}
+      varying_subgroup_size_(false),
+      require_full_subgroups_(false),
+      emit_debug_info_(false) {}
 
 Pipeline::ShaderInfo::ShaderInfo(const ShaderInfo&) = default;
 
diff --git a/src/pipeline.h b/src/pipeline.h
index 6d2a2cb..212255b 100644
--- a/src/pipeline.h
+++ b/src/pipeline.h
@@ -90,6 +90,9 @@ class Pipeline {
     }
     bool GetRequireFullSubgroups() const { return require_full_subgroups_; }
 
+    void SetEmitDebugInfo(const bool isSet) { emit_debug_info_ = isSet; }
+    bool GetEmitDebugInfo() const { return emit_debug_info_; }
+
     void SetShader(Shader* shader) { shader_ = shader; }
     const Shader* GetShader() const { return shader_; }
 
@@ -174,6 +177,7 @@ class Pipeline {
     uint32_t required_subgroup_size_;
     bool varying_subgroup_size_;
     bool require_full_subgroups_;
+    bool emit_debug_info_;
   };
 
   /// Information on a buffer attached to the pipeline.
diff --git a/src/shader_compiler.cc b/src/shader_compiler.cc
index 285dd97..c0084d8 100644
--- a/src/shader_compiler.cc
+++ b/src/shader_compiler.cc
@@ -133,7 +133,7 @@ std::pair<Result, std::vector<uint32_t>> ShaderCompiler::Compile(
 
 #if AMBER_ENABLE_DXC
   } else if (shader->GetFormat() == kShaderFormatHlsl) {
-    Result r = CompileHlsl(shader, &results);
+    Result r = CompileHlsl(shader, shader_info->GetEmitDebugInfo(), &results);
     if (!r.IsSuccess())
       return {r, {}};
 #endif  // AMBER_ENABLE_DXC
@@ -262,6 +262,7 @@ Result ShaderCompiler::CompileGlsl(const Shader*,
 
 #if AMBER_ENABLE_DXC
 Result ShaderCompiler::CompileHlsl(const Shader* shader,
+                                   bool emit_debug_info,
                                    std::vector<uint32_t>* result) const {
   std::string target;
   if (shader->GetType() == kShaderTypeCompute)
@@ -276,10 +277,12 @@ Result ShaderCompiler::CompileHlsl(const Shader* shader,
     return Result("Unknown shader type");
 
   return dxchelper::Compile(shader->GetData(), "main", target, spv_env_,
-                            shader->GetFilePath(), virtual_files_, result);
+                            shader->GetFilePath(), virtual_files_,
+                            emit_debug_info, result);
 }
 #else
 Result ShaderCompiler::CompileHlsl(const Shader*,
+                                   bool,
                                    std::vector<uint32_t>*) const {
   return {};
 }
diff --git a/src/shader_compiler.h b/src/shader_compiler.h
index 849f731..5470069 100644
--- a/src/shader_compiler.h
+++ b/src/shader_compiler.h
@@ -60,7 +60,9 @@ class ShaderCompiler {
  private:
   Result ParseHex(const std::string& data, std::vector<uint32_t>* result) const;
   Result CompileGlsl(const Shader* shader, std::vector<uint32_t>* result) const;
-  Result CompileHlsl(const Shader* shader, std::vector<uint32_t>* result) const;
+  Result CompileHlsl(const Shader* shader,
+                     bool emit_debug_info,
+                     std::vector<uint32_t>* result) const;
 #if AMBER_ENABLE_CLSPV
   Result CompileOpenCLC(Pipeline::ShaderInfo* shader,
                         Pipeline* pipeline,