Add support for "NaN" as a double value (#595)

Add support for "NaN" as a double when tokenizing

2 files changed, 45 insertions, 8 deletions

diff --git a/src/tokenizer.cc b/src/tokenizer.cc
index f24655c..54bf88c 100644
--- a/src/tokenizer.cc
+++ b/src/tokenizer.cc
@@ -99,10 +99,15 @@ std::unique_ptr<Token> Tokenizer::NextToken() {
       data_.substr(current_position_, end_pos - current_position_);
   current_position_ = end_pos;
 
+  // Check for "NaN" explicitly.
+  bool is_nan =
+      (tok_str.size() == 3 && std::tolower(tok_str[0]) == 'n' &&
+       std::tolower(tok_str[1]) == 'a' && std::tolower(tok_str[2]) == 'n');
+
   // Starts with an alpha is a string.
-  if (!std::isdigit(tok_str[0]) &&
-      !(tok_str[0] == '-' && std::isdigit(tok_str[1])) &&
-      !(tok_str[0] == '.' && std::isdigit(tok_str[1]))) {
+  if (!is_nan && !std::isdigit(tok_str[0]) &&
+      !(tok_str[0] == '-' && tok_str.size() >= 2 && std::isdigit(tok_str[1])) &&
+      !(tok_str[0] == '.' && tok_str.size() >= 2 && std::isdigit(tok_str[1]))) {
     // If we've got a continuation, skip over the end of line and get the next
     // token.
     if (tok_str == "\\") {
@@ -126,17 +131,21 @@ std::unique_ptr<Token> Tokenizer::NextToken() {
   }
 
   // Handle hex strings
-  if (tok_str.size() > 2 && tok_str[0] == '0' && tok_str[1] == 'x') {
+  if (!is_nan && tok_str.size() > 2 && tok_str[0] == '0' && tok_str[1] == 'x') {
     auto tok = MakeUnique<Token>(TokenType::kHex);
     tok->SetStringValue(tok_str);
     return tok;
   }
 
   bool is_double = false;
-  for (const char ch : tok_str) {
-    if (ch == '.') {
-      is_double = true;
-      break;
+  if (is_nan) {
+    is_double = true;
+  } else {
+    for (const char ch : tok_str) {
+      if (ch == '.') {
+        is_double = true;
+        break;
+      }
     }
   }
 
diff --git a/src/tokenizer_test.cc b/src/tokenizer_test.cc
index d50ec46..fd7dba4 100644
--- a/src/tokenizer_test.cc
+++ b/src/tokenizer_test.cc
@@ -14,6 +14,7 @@
 
 #include "src/tokenizer.h"
 
+#include <cmath>
 #include <limits>
 
 #include "gtest/gtest.h"
@@ -77,6 +78,33 @@ TEST_F(TokenizerTest, ProcessDouble) {
   EXPECT_TRUE(next->IsEOS());
 }
 
+namespace {
+
+void TestNaN(const std::string& nan_str) {
+  Tokenizer t(nan_str);
+  auto next = t.NextToken();
+  ASSERT_TRUE(next != nullptr);
+  EXPECT_TRUE(next->IsDouble());
+  EXPECT_TRUE(std::isnan(next->AsDouble()));
+
+  next = t.NextToken();
+  ASSERT_TRUE(next != nullptr);
+  EXPECT_TRUE(next->IsEOS());
+}
+
+}  // namespace
+
+TEST_F(TokenizerTest, ProcessNaN) {
+  TestNaN("nan");
+  TestNaN("naN");
+  TestNaN("nAn");
+  TestNaN("nAN");
+  TestNaN("Nan");
+  TestNaN("NaN");
+  TestNaN("NAn");
+  TestNaN("NAN");
+}
+
 TEST_F(TokenizerTest, ProcessNegativeDouble) {
   Tokenizer t("-123.456");
   auto next = t.NextToken();