[quickfix][Refs #28] resolved bug on merimee-MH.json, but still failing on euro symbol

[add][wip] unicode test suite

.gitignore

@@ -40,6 +40,7 @@ vgcore.*
 /test/wrapped_test
 /test/optional_test
 /test/levenshtein_test
+/test/unicode_test
 /cmake_install.cmake
 /Makefile
 .fuse_hidden*

CMakeLists.txt

@@ -21,6 +21,7 @@ add_executable(jsonstroll
     src/linearHistory.cpp
     src/outputFlag.cpp
     src/streamConsumer.cpp
+    src/unicode.cpp
     src/searchPattern.cpp
     src/inputSequence.cpp
     src/levenshtein.cpp
@@ -51,6 +52,7 @@ add_executable(json_test
     src/searchPattern.cpp
     src/levenshtein.cpp
     src/streamConsumer.cpp
+    src/unicode.cpp
     src/jsonArray.cpp
     src/jsonObjectEntry.cpp
     src/jsonObject.cpp
@@ -82,6 +84,28 @@ set_property(
     PROPERTY RUNTIME_OUTPUT_DIRECTORY test
     )
 
+add_executable(unicode_test
+    src/unicode.cpp
+    src/jsonException.cpp
+    src/streamConsumer.cpp
+    src/jsonArray.cpp
+    src/jsonContainer.cpp
+    src/jsonObject.cpp
+    src/jsonObjectEntry.cpp
+    src/jsonElement.cpp
+    src/jsonPrimitive.cpp
+    src/linearHistory.cpp
+    src/searchPattern.cpp
+    src/warning.cpp
+
+    test/src/unicode.cpp
+    )
+add_test(unicode_test test/unicode_test)
+set_property(
+    TARGET unicode_test
+    PROPERTY RUNTIME_OUTPUT_DIRECTORY test
+    )
+
 add_executable(levenshtein_test
     src/levenshtein.cpp
     src/jsonContainer.cpp

include/streamConsumer.hh

@@ -52,7 +52,7 @@ class StreamConsumer
         static std::string extractUnicode(const char *);
         static std::string extractUnicode(const std::string &);
 
-    private:
+    protected:
         /**
          * @return non-null on successfully read JSonElement, or null if token (',', '[', ...)
         **/

include/unicode.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+unsigned char hexbyte(const char c);
+
+template<typename T>
+static T hexbyte(const char str[], unsigned int len)
+{
+    T result = 0;
+    for (unsigned int i =0; i < len; ++i)
+        result = (result << 4) + hexbyte(str[i]);
+    return result;
+}
+
+

src/main.cpp

@@ -40,8 +40,6 @@ void runDiff(const Params &params)
     std::deque<Warning> warns;
     std::deque<std::string> inputNames;
 
-    if (!params.isIgnoringUnicode())
-        setlocale(LC_ALL, "");
     for (std::pair<std::string, std::basic_istream<char>*> input : inputs)
     {
         StreamConsumer *stream;
@@ -92,8 +90,6 @@ void run(const Params &params)
     CurseSimpleOutput *out = new CurseSimpleOutput(params);
     std::list<Warning> warns;
 
-    if (!params.isIgnoringUnicode())
-        setlocale(LC_ALL, "");
     for (std::pair<std::string, std::basic_istream<char>*> input : inputs)
     {
         StreamConsumer *stream;
@@ -152,6 +148,8 @@ int main(int ac, char **av)
 
     if (_run)
     {
+        if (!params->isIgnoringUnicode())
+            setlocale(LC_ALL, "");
         if (params->isDiff())
             runDiff(*params);
         else

src/streamConsumer.cpp

@@ -8,6 +8,7 @@
 #include <sstream>
 #include "jsonElement.hh"
 #include "streamConsumer.hh"
+#include "unicode.hpp"
 
 StreamConsumer::StreamConsumer(std::istream &s): stream(s), root(nullptr)
 { }
@@ -335,33 +336,14 @@ JSonElement *StreamConsumer::consumeToken(JSonContainer *parent, std::stringstre
     return nullptr;
 }
 
-static unsigned char hexbyte(const char c)
-{
-    if (c >= '0' && c <= '9')
-        return c - '0';
-    if (c >= 'A' && c <= 'F')
-        return c - 'A' + 10;
-    if (c >= 'a' && c <= 'f')
-        return c - 'a' + 10;
-    throw std::invalid_argument(JsonHexvalueException::msg(c));
-}
-
-template<typename T>
-static T hexbyte(const char str[], unsigned int len)
-{
-    T result = 0;
-    for (unsigned int i =0; i < len; ++i)
-        result = (result << 4) + hexbyte(str[i]);
-    return result;
-}
-
 void StreamConsumer::appendUnicode(const char unicode[4], std::stringstream &buf)
 {
     unsigned short uni = hexbyte<unsigned short>(unicode, 4);
     char test[5];
+
     bzero(test, sizeof(*test) *5);
     snprintf(test, 4, "%lc", uni);
-    buf.write(test, 3);
+    buf.write(test, 2);
 }
 
 std::string StreamConsumer::extractUnicode(const char *buf)

src/unicode.cpp

@@ -0,0 +1,15 @@
+#include <stdexcept>
+#include "unicode.hpp"
+#include "jsonException.hh"
+
+unsigned char hexbyte(const char c)
+{
+    if (c >= '0' && c <= '9')
+        return c - '0';
+    if (c >= 'A' && c <= 'F')
+        return c - 'A' + 10;
+    if (c >= 'a' && c <= 'f')
+        return c - 'a' + 10;
+    throw std::invalid_argument(JsonHexvalueException::msg(c));
+}
+

test/src/unicode.cpp

@@ -0,0 +1,61 @@
+#include <sstream>
+#include <iostream>
+#include "unicode.hpp"
+#include "streamConsumer.hh"
+
+#define FAILED(got, op, expt) {std::cout << __FILE__ << ":" << __LINE__ << ": failed asserting " << got << " " << op << " expected " << expt << std::endl; return false; }
+
+class StreamConsumerTester: public StreamConsumer
+{
+    public:
+        static const std::string getStringFromUnicode(const char unicode[4])
+        {
+            std::stringstream ss;
+            appendUnicode(unicode, ss);
+            return ss.str();
+        };
+
+        static bool test()
+        {
+            std::string s = getStringFromUnicode("00e8");
+            if (s != "è")
+                FAILED(s, "!=", "è");
+            return true;
+        };
+};
+
+bool simpleTest()
+{
+    if (hexbyte<unsigned short>("0020", 4) != 32)
+        FAILED(hexbyte<unsigned short>("0020", 4), "!=", 32);
+
+    if (hexbyte<unsigned short>("20", 2) != 32)
+        FAILED(hexbyte<unsigned short>("2020", 4), "!=", 32);
+
+    if (hexbyte<unsigned short>("2020", 4) != 8224)
+        FAILED(hexbyte<unsigned short>("2020", 4), "!=", 8224);
+
+    if (hexbyte<unsigned short>("FFFF", 4) != 65535)
+        FAILED(hexbyte<unsigned short>("FFFF", 4), "!=", 65535);
+
+    if (hexbyte<unsigned short>("0000", 4) != 0)
+        FAILED(hexbyte<unsigned short>("0000", 4), "!=", 0);
+
+    if (hexbyte<unsigned short>("", 0) != 0)
+        FAILED(hexbyte<unsigned short>("", 0), "!=", 0);
+
+    if (hexbyte<unsigned short>("002020", 6) != 8224)
+        FAILED(hexbyte<unsigned short>("2020", 6), "!=", 8224);
+
+    return true;
+}
+
+int main()
+{
+    if (!simpleTest())
+        exit(EXIT_FAILURE);
+    if (!StreamConsumerTester::test())
+        exit(EXIT_FAILURE);
+    exit(EXIT_SUCCESS);
+}
+