[AmberScript] Add support for runtime arrays (#909)

This CL adds the ability to specify the data_type for an AmberScript buffer to be an array.

7 files changed, 263 insertions, 65 deletions

diff --git a/docs/amber_script.md b/docs/amber_script.md
index 54a4e24..6dd2443 100644
--- a/docs/amber_script.md
+++ b/docs/amber_script.md
@@ -209,6 +209,8 @@ either image buffers or, what the target API would refer to as a buffer.
  * vec[2,3,4]{type}
  * mat[2,3,4]x[2,3,4]{type}  (mat<columns>x<rows>)
  * Any of the `Image Formats` listed below.
+ * For any of the non-Image Formats types above appending '[]' will treat the
+    data as an array. e.g. int8[], vec2<float>[]
 
 Sized arrays and structures are not currently representable.
 
diff --git a/src/amberscript/parser.cc b/src/amberscript/parser.cc
index 525a60f..3b75348 100644
--- a/src/amberscript/parser.cc
+++ b/src/amberscript/parser.cc
@@ -54,32 +54,41 @@ ProbeSSBOCommand::Comparator ToComparator(const std::string& in) {
   return ProbeSSBOCommand::Comparator::kLessOrEqual;
 }
 
-std::unique_ptr<type::Type> ToType(const std::string& str) {
+std::unique_ptr<type::Type> ToType(const std::string& str_in) {
+  std::string str = str_in;
+
+  bool is_array = false;
+  if (str.length() > 2 && str[str.length() - 2] == '[' &&
+      str[str.length() - 1] == ']') {
+    is_array = true;
+    str = str.substr(0, str.length() - 2);
+  }
+
   TypeParser parser;
-  if (str == "int8")
-    return parser.Parse("R8_SINT");
-  if (str == "int16")
-    return parser.Parse("R16_SINT");
-  if (str == "int32")
-    return parser.Parse("R32_SINT");
-  if (str == "int64")
-    return parser.Parse("R64_SINT");
-  if (str == "uint8")
-    return parser.Parse("R8_UINT");
-  if (str == "uint16")
-    return parser.Parse("R16_UINT");
-  if (str == "uint32")
-    return parser.Parse("R32_UINT");
-  if (str == "uint64")
-    return parser.Parse("R64_UINT");
-  if (str == "float16")
-    return parser.Parse("R16_SFLOAT");
-  if (str == "float")
-    return parser.Parse("R32_SFLOAT");
-  if (str == "double")
-    return parser.Parse("R64_SFLOAT");
-
-  if (str.length() > 7 && str.substr(0, 3) == "vec") {
+  std::unique_ptr<type::Type> type;
+  if (str == "int8") {
+    type = parser.Parse("R8_SINT");
+  } else if (str == "int16") {
+    type = parser.Parse("R16_SINT");
+  } else if (str == "int32") {
+    type = parser.Parse("R32_SINT");
+  } else if (str == "int64") {
+    type = parser.Parse("R64_SINT");
+  } else if (str == "uint8") {
+    type = parser.Parse("R8_UINT");
+  } else if (str == "uint16") {
+    type = parser.Parse("R16_UINT");
+  } else if (str == "uint32") {
+    type = parser.Parse("R32_UINT");
+  } else if (str == "uint64") {
+    type = parser.Parse("R64_UINT");
+  } else if (str == "float16") {
+    type = parser.Parse("R16_SFLOAT");
+  } else if (str == "float") {
+    type = parser.Parse("R32_SFLOAT");
+  } else if (str == "double") {
+    type = parser.Parse("R64_SFLOAT");
+  } else if (str.length() > 7 && str.substr(0, 3) == "vec") {
     if (str[4] != '<' || str[str.length() - 1] != '>')
       return nullptr;
 
@@ -87,7 +96,7 @@ std::unique_ptr<type::Type> ToType(const std::string& str) {
     if (component_count < 2 || component_count > 4)
       return nullptr;
 
-    auto type = ToType(str.substr(5, str.length() - 6));
+    type = ToType(str.substr(5, str.length() - 6));
     if (!type)
       return nullptr;
 
@@ -97,10 +106,7 @@ std::unique_ptr<type::Type> ToType(const std::string& str) {
     }
 
     type->SetRowCount(static_cast<uint32_t>(component_count));
-    return type;
-  }
-
-  if (str.length() > 9 && str.substr(0, 3) == "mat") {
+  } else if (str.length() > 9 && str.substr(0, 3) == "mat") {
     if (str[4] != 'x' || str[6] != '<' || str[str.length() - 1] != '>')
       return nullptr;
 
@@ -112,7 +118,7 @@ std::unique_ptr<type::Type> ToType(const std::string& str) {
     if (row_count < 2 || row_count > 4)
       return nullptr;
 
-    auto type = ToType(str.substr(7, str.length() - 8));
+    type = ToType(str.substr(7, str.length() - 8));
     if (!type)
       return nullptr;
     if (!type->IsNumber() || type->IsArray() || type->IsVec() ||
@@ -122,9 +128,14 @@ std::unique_ptr<type::Type> ToType(const std::string& str) {
 
     type->SetRowCount(static_cast<uint32_t>(row_count));
     type->SetColumnCount(static_cast<uint32_t>(column_count));
-    return type;
   }
-  return nullptr;
+
+  if (!type)
+    return nullptr;
+  if (is_array)
+    type->SetIsRuntimeArray();
+
+  return type;
 }
 
 AddressMode StrToAddressMode(std::string str) {
diff --git a/src/amberscript/parser_buffer_test.cc b/src/amberscript/parser_buffer_test.cc
index 9286154..b6cbf43 100644
--- a/src/amberscript/parser_buffer_test.cc
+++ b/src/amberscript/parser_buffer_test.cc
@@ -212,6 +212,72 @@ END)";
   }
 }
 
+TEST_F(AmberScriptParserTest, BufferDataArrayStd140) {
+  std::string in = R"(
+BUFFER my_buffer DATA_TYPE uint32[] STD140 DATA
+1 2 3 4
+55 99 1234
+END)";
+
+  Parser parser;
+  Result r = parser.Parse(in);
+  ASSERT_TRUE(r.IsSuccess()) << r.Error();
+
+  auto script = parser.GetScript();
+  const auto& buffers = script->GetBuffers();
+  ASSERT_EQ(1U, buffers.size());
+
+  ASSERT_TRUE(buffers[0] != nullptr);
+  EXPECT_EQ("my_buffer", buffers[0]->GetName());
+
+  auto* buffer = buffers[0].get();
+  EXPECT_TRUE(buffer->GetFormat()->IsUint32());
+  EXPECT_EQ(Format::Layout::kStd140, buffer->GetFormat()->GetLayout());
+  EXPECT_EQ(7U, buffer->ElementCount());
+  EXPECT_EQ(28U * sizeof(uint32_t), buffer->GetSizeInBytes());
+
+  std::vector<uint32_t> results = {1,  0, 0, 0, 2,    0, 0,  0, 3, 0,
+                                   0,  0, 4, 0, 0,    0, 55, 0, 0, 0,
+                                   99, 0, 0, 0, 1234, 0, 0,  0};
+  const auto* data = buffer->GetValues<uint32_t>();
+  for (size_t i = 0; i < results.size(); ++i) {
+    EXPECT_EQ(results[i], data[i]);
+  }
+}
+
+TEST_F(AmberScriptParserTest, BufferDataArrayStd430) {
+  std::string in = R"(
+BUFFER my_buffer DATA_TYPE uint32[] STD430 DATA
+1 2 3 4
+55 99 1234
+END)";
+
+  Parser parser;
+  Result r = parser.Parse(in);
+  ASSERT_TRUE(r.IsSuccess()) << r.Error();
+
+  auto script = parser.GetScript();
+  const auto& buffers = script->GetBuffers();
+  ASSERT_EQ(1U, buffers.size());
+
+  ASSERT_TRUE(buffers[0] != nullptr);
+  EXPECT_EQ("my_buffer", buffers[0]->GetName());
+
+  auto* buffer = buffers[0].get();
+  EXPECT_TRUE(buffer->GetFormat()->IsUint32());
+  EXPECT_EQ(Format::Layout::kStd430, buffer->GetFormat()->GetLayout());
+  EXPECT_EQ(7U, buffer->ElementCount());
+  EXPECT_EQ(7U, buffer->ValueCount());
+  EXPECT_EQ(7U * sizeof(uint32_t), buffer->GetSizeInBytes());
+
+  std::vector<uint32_t> results = {1, 2, 3, 4, 55, 99, 1234};
+  const auto* data = buffer->GetValues<uint32_t>();
+  ASSERT_EQ(results.size(), buffer->ValueCount());
+  for (size_t i = 0; i < results.size(); ++i) {
+    EXPECT_EQ(results[i], data[i]);
+  }
+}
+
 TEST_F(AmberScriptParserTest, BufferDataOneLine) {
   std::string in = "BUFFER my_buffer DATA_TYPE uint32 DATA 1 2 3 4 END";
 
@@ -719,6 +785,7 @@ struct BufferData {
   size_t num_bits;
   size_t row_count;
   size_t column_count;
+  bool is_array;
 };
 
 using AmberScriptParserBufferDataTypeTest = testing::TestWithParam<BufferData>;
@@ -743,30 +810,55 @@ TEST_P(AmberScriptParserBufferDataTypeTest, BufferTypes) {
   EXPECT_EQ(test_data.row_count, fmt->GetType()->RowCount());
   EXPECT_EQ(test_data.column_count, fmt->GetType()->ColumnCount());
 
-  EXPECT_EQ(test_data.type, fmt->GetSegments()[0].GetFormatMode());
-  EXPECT_EQ(test_data.num_bits, fmt->GetSegments()[0].GetNumBits());
+  auto& seg = fmt->GetSegments()[0];
+  EXPECT_EQ(test_data.type, seg.GetFormatMode());
+  EXPECT_EQ(test_data.num_bits, seg.GetNumBits());
+  EXPECT_EQ(test_data.is_array, fmt->GetType()->IsArray());
 }
 INSTANTIATE_TEST_SUITE_P(
     AmberScriptParserTestsDataType,
     AmberScriptParserBufferDataTypeTest,
-    testing::Values(BufferData{"int8", FormatMode::kSInt, 8, 1, 1},
-                    BufferData{"int16", FormatMode::kSInt, 16, 1, 1},
-                    BufferData{"int32", FormatMode::kSInt, 32, 1, 1},
-                    BufferData{"int64", FormatMode::kSInt, 64, 1, 1},
-                    BufferData{"uint8", FormatMode::kUInt, 8, 1, 1},
-                    BufferData{"uint16", FormatMode::kUInt, 16, 1, 1},
-                    BufferData{"uint32", FormatMode::kUInt, 32, 1, 1},
-                    BufferData{"uint64", FormatMode::kUInt, 64, 1, 1},
-                    BufferData{"vec2<int8>", FormatMode::kSInt, 8, 2, 1},
-                    BufferData{"vec4<uint32>", FormatMode::kUInt, 32, 4, 1},
-                    BufferData{"mat2x4<int32>", FormatMode::kSInt, 32, 4, 2},
-                    BufferData{"mat4x2<uint16>", FormatMode::kUInt, 16, 2, 4},
-                    BufferData{"B8G8R8_UNORM", FormatMode::kUNorm, 8, 3, 1},
-                    BufferData{"float", FormatMode::kSFloat, 32, 1, 1},
-                    BufferData{"double", FormatMode::kSFloat, 64, 1, 1},
-                    BufferData{"vec3<float>", FormatMode::kSFloat, 32, 3, 1},
-                    BufferData{"mat3x3<float>", FormatMode::kSFloat, 32, 3,
-                               3}));  // NOLINT(whitespace/parens)
+    testing::Values(
+        BufferData{"int8", FormatMode::kSInt, 8, 1, 1, false},
+        BufferData{"int16", FormatMode::kSInt, 16, 1, 1, false},
+        BufferData{"int32", FormatMode::kSInt, 32, 1, 1, false},
+        BufferData{"int64", FormatMode::kSInt, 64, 1, 1, false},
+        BufferData{"uint8", FormatMode::kUInt, 8, 1, 1, false},
+        BufferData{"uint16", FormatMode::kUInt, 16, 1, 1, false},
+        BufferData{"uint32", FormatMode::kUInt, 32, 1, 1, false},
+        BufferData{"uint64", FormatMode::kUInt, 64, 1, 1, false},
+        BufferData{"float", FormatMode::kSFloat, 32, 1, 1, false},
+        BufferData{"double", FormatMode::kSFloat, 64, 1, 1, false},
+        BufferData{"vec2<int8>", FormatMode::kSInt, 8, 2, 1, false},
+        BufferData{"vec3<float>", FormatMode::kSFloat, 32, 3, 1, false},
+        BufferData{"vec4<uint32>", FormatMode::kUInt, 32, 4, 1, false},
+        BufferData{"mat2x4<int32>", FormatMode::kSInt, 32, 4, 2, false},
+        BufferData{"mat3x3<float>", FormatMode::kSFloat, 32, 3, 3, false},
+        BufferData{"mat4x2<uint16>", FormatMode::kUInt, 16, 2, 4, false},
+        BufferData{"B8G8R8_UNORM", FormatMode::kUNorm, 8, 3, 1,
+                   false}));  // NOLINT(whitespace/parens)
+
+INSTANTIATE_TEST_SUITE_P(
+    AmberScriptParserTestsDataType2,
+    AmberScriptParserBufferDataTypeTest,
+    testing::Values(
+        BufferData{"int8[]", FormatMode::kSInt, 8, 1, 1, true},
+        BufferData{"int16[]", FormatMode::kSInt, 16, 1, 1, true},
+        BufferData{"int32[]", FormatMode::kSInt, 32, 1, 1, true},
+        BufferData{"int64[]", FormatMode::kSInt, 64, 1, 1, true},
+        BufferData{"uint8[]", FormatMode::kUInt, 8, 1, 1, true},
+        BufferData{"uint16[]", FormatMode::kUInt, 16, 1, 1, true},
+        BufferData{"uint32[]", FormatMode::kUInt, 32, 1, 1, true},
+        BufferData{"uint64[]", FormatMode::kUInt, 64, 1, 1, true},
+        BufferData{"float[]", FormatMode::kSFloat, 32, 1, 1, true},
+        BufferData{"double[]", FormatMode::kSFloat, 64, 1, 1, true},
+        BufferData{"vec2<int8>[]", FormatMode::kSInt, 8, 2, 1, true},
+        BufferData{"vec3<float>[]", FormatMode::kSFloat, 32, 3, 1, true},
+        BufferData{"vec4<uint32>[]", FormatMode::kUInt, 32, 4, 1, true},
+        BufferData{"mat2x4<int32>[]", FormatMode::kSInt, 32, 4, 2, true},
+        BufferData{"mat3x3<float>[]", FormatMode::kSFloat, 32, 3, 3, true},
+        BufferData{"mat4x2<uint16>[]", FormatMode::kUInt, 16, 2, 4,
+                   true}));  // NOLINT(whitespace/parens)
 
 struct NameData {
   const char* name;
@@ -792,6 +884,9 @@ INSTANTIATE_TEST_SUITE_P(
     AmberScriptParserBufferDataTypeInvalidTestSamples,
     AmberScriptParserBufferDataTypeInvalidTest,
     testing::Values(NameData{"int17"},
+                    NameData{"int["},
+                    NameData{"int]"},
+                    NameData{"B8G8R8_UNORM[]"},
                     NameData{"uintt0"},
                     NameData{"vec7<uint8>"},
                     NameData{"vec27<uint8>"},
diff --git a/src/type_parser.cc b/src/type_parser.cc
index a6779f0..7df8be5 100644
--- a/src/type_parser.cc
+++ b/src/type_parser.cc
@@ -40,12 +40,16 @@ std::unique_ptr<type::Type> TypeParser::Parse(const std::string& data) {
   for (;;) {
     size_t next_pos = data.rfind('_', cur_pos);
     if (next_pos == std::string::npos) {
-      if (cur_pos != std::string::npos)
-        ProcessChunk(data.substr(0, cur_pos + 1));
+      if (cur_pos != std::string::npos) {
+        if (!ProcessChunk(data.substr(0, cur_pos + 1)))
+          return nullptr;
+      }
       break;
     }
 
-    ProcessChunk(data.substr(next_pos + 1, cur_pos - next_pos));
+    if (!ProcessChunk(data.substr(next_pos + 1, cur_pos - next_pos)))
+      return nullptr;
+
     cur_pos = next_pos - 1;
   }
 
@@ -78,8 +82,9 @@ void TypeParser::AddPiece(FormatComponentType type,
   pieces_.insert(pieces_.begin(), Pieces{type, mode, bits});
 }
 
-void TypeParser::ProcessChunk(const std::string& data) {
-  assert(data.size() > 0);
+bool TypeParser::ProcessChunk(const std::string& data) {
+  if (data.size() == 0)
+    return false;
 
   if (data[0] == 'P') {
     if (data == "PACK8")
@@ -89,9 +94,9 @@ void TypeParser::ProcessChunk(const std::string& data) {
     else if (data == "PACK32")
       pack_size_ = 32;
     else
-      assert(false);
+      return false;
 
-    return;
+    return true;
   }
 
   if (data[0] == 'U') {
@@ -104,9 +109,9 @@ void TypeParser::ProcessChunk(const std::string& data) {
     else if (data == "USCALED")
       mode_ = FormatMode::kUScaled;
     else
-      assert(false);
+      return false;
 
-    return;
+    return true;
   }
 
   if (data[0] == 'S') {
@@ -123,9 +128,9 @@ void TypeParser::ProcessChunk(const std::string& data) {
     else if (data == "S8")
       AddPiece(FormatComponentType::kS, mode_, 8);
     else
-      assert(false);
+      return false;
 
-    return;
+    return true;
   }
 
   int32_t cur_pos = static_cast<int32_t>(data.size()) - 1;
@@ -167,6 +172,7 @@ void TypeParser::ProcessChunk(const std::string& data) {
 
     --cur_pos;
   }
+  return true;
 }
 
 // static
diff --git a/src/type_parser.h b/src/type_parser.h
index d737e86..3c3581d 100644
--- a/src/type_parser.h
+++ b/src/type_parser.h
@@ -41,7 +41,7 @@ class TypeParser {
 
  private:
   std::unique_ptr<type::Type> ParseGlslFormat(const std::string& fmt);
-  void ProcessChunk(const std::string&);
+  bool ProcessChunk(const std::string&);
   void AddPiece(FormatComponentType type, FormatMode mode, uint8_t bits);
   void FlushPieces(type::Type* type, FormatMode mode);
 
diff --git a/tests/cases/runtime_array_std140.amber b/tests/cases/runtime_array_std140.amber
new file mode 100644
index 0000000..009a700
--- /dev/null
+++ b/tests/cases/runtime_array_std140.amber
@@ -0,0 +1,42 @@
+#!amber
+# Copyright 2020 The Amber Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SHADER vertex vert_shader PASSTHROUGH
+SHADER fragment frag_shader GLSL
+#version 430
+layout(location = 0) out vec4 color_out;
+layout(std140, binding = 0) readonly buffer Data {
+  float d[];
+} data;
+
+void main() {
+  color_out = vec4(data.d[0]/255, data.d[1]/255, data.d[2]/255, data.d[3]/255);
+}
+END
+
+BUFFER framebuffer FORMAT B8G8R8A8_UNORM
+
+BUFFER data DATA_TYPE float[] STD140 DATA 1.0 64.0 128.0 220.0 END
+
+PIPELINE graphics my_pipeline
+  ATTACH vert_shader
+  ATTACH frag_shader
+
+  BIND BUFFER framebuffer AS color LOCATION 0
+  BIND BUFFER data AS storage DESCRIPTOR_SET 0 BINDING 0
+END
+
+RUN my_pipeline DRAW_RECT POS 0 0 SIZE 250 250
+EXPECT framebuffer IDX 0 0 SIZE 250 250 EQ_RGBA 1 64 128 220
diff --git a/tests/cases/runtime_array_std430.amber b/tests/cases/runtime_array_std430.amber
new file mode 100644
index 0000000..a30658f
--- /dev/null
+++ b/tests/cases/runtime_array_std430.amber
@@ -0,0 +1,42 @@
+#!amber
+# Copyright 2020 The Amber Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SHADER vertex vert_shader PASSTHROUGH
+SHADER fragment frag_shader GLSL
+#version 430
+layout(location = 0) out vec4 color_out;
+layout(std430, binding = 0) readonly buffer Data {
+  float d[];
+} data;
+
+void main() {
+  color_out = vec4(data.d[0]/255, data.d[1]/255, data.d[2]/255, data.d[3]/255);
+}
+END
+
+BUFFER framebuffer FORMAT B8G8R8A8_UNORM
+
+BUFFER data DATA_TYPE float[] STD430 DATA 1.0 64.0 128.0 220.0 END
+
+PIPELINE graphics my_pipeline
+  ATTACH vert_shader
+  ATTACH frag_shader
+
+  BIND BUFFER framebuffer AS color LOCATION 0
+  BIND BUFFER data AS storage DESCRIPTOR_SET 0 BINDING 0
+END
+
+RUN my_pipeline DRAW_RECT POS 0 0 SIZE 250 250
+EXPECT framebuffer IDX 0 0 SIZE 250 250 EQ_RGBA 1 64 128 220