Merge branch 'jsonstroller' of isundil/jsonStroller into master

CMakeLists.txt

@@ -14,6 +14,7 @@ add_executable(jsonstroll
     src/linearHistory.cpp
     src/outputFlag.cpp
     src/streamConsumer.cpp
+    src/searchPattern.cpp
 
     src/jsonElement.cpp
     src/jsonArray.cpp
@@ -42,6 +43,7 @@ add_executable(json_test
     src/jsonContainer.cpp
     src/warning.cpp
     src/linearHistory.cpp
+    src/searchPattern.cpp
     src/streamConsumer.cpp
     src/jsonArray.cpp
     src/jsonObjectEntry.cpp

doc/jsonstroll.1

@@ -1,7 +1,7 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.3.
-.TH JSONSTROLL "1" "August 2016" "jsonstroll (jsonstroller suite) 1.0RC1 generated on Aug 12 2016" "User Commands"
+.TH JSONSTROLL "1" "August 2016" "jsonstroll (jsonstroller suite) 1.0RC1 generated on Aug 15 2016" "User Commands"
 .SH NAME
-jsonstroll \- manual page for jsonstroll (jsonstroller suite) 1.0RC1 generated on Aug 12 2016
+jsonstroll \- manual page for jsonstroll (jsonstroller suite) 1.0RC1 generated on Aug 15 2016
 .SH SYNOPSIS
 .B jsonstroll
 [\fI\,OPTIONS\/\fR]

include/curseOutput.hh

@@ -18,7 +18,7 @@ class JSonElement;
 class JSonContainer;
 class JSonArray;
 class JSonObject;
-template<class T> class Optional;
+class SearchPattern;
 
 class CurseOutput
 {
@@ -47,10 +47,12 @@ class CurseOutput
          * Initialize ncurses
         **/
         void init();
+
         /**
          * Release ncurses
         **/
         void shutdown();
+
         /**
          * return false if bottom of screen is touched
          * redraw all data
@@ -64,19 +66,23 @@ class CurseOutput
          * redraw item and children
         **/
         bool redraw(std::pair<int, int> &, const std::pair<unsigned int, unsigned int> &maxWidth, const JSonElement *item);
+
         /**
          * Wait for input
          * @return false if ncurses should stop
         **/
         bool readInput();
+
         /**
          * get the screen size
         **/
         const std::pair<unsigned int, unsigned int> getScreenSize() const;
+
         /**
          * set the select_up and select_down pointers, scroll to selection if it is above view port
         **/
         void checkSelection(const JSonElement *item, const std::pair<int, int>&);
+
         /**
          * Return the number of lines written while writting nbChar bytes
          * @param nbChar col written
@@ -84,6 +90,7 @@ class CurseOutput
          * @return the number of line written
         **/
         static unsigned int getNbLines(float nbChar, unsigned int maxWidth);
+
         /**
          * get flags to be passed to write.
          * Contains indications on who to write item
@@ -116,16 +123,18 @@ class CurseOutput
          * prompt for user input, and return it
          * @throws noInputException if user use a kill-key (to exit buffer)
         **/
-        const std::string inputSearch();
+        const SearchPattern *inputSearch();
+
         /**
          * find occurences of search result and fill this#search_result
         **/
-        unsigned int search(const std::string &, const JSonElement *);
+        unsigned int search(const SearchPattern &, const JSonElement *);
 
         /**
          * Write a message on the last line, using color
         **/
         void writeBottomLine(const std::string &currentBuffer, short color) const;
+        void writeBottomLine(const std::wstring &currentBuffer, short color) const;
 
         /**
          * unfold all item's parents
@@ -143,10 +152,12 @@ class CurseOutput
          * Root item
         **/
         const JSonElement *data;
+
         /**
          * selected item
         **/
         const JSonElement *selection;
+
         /**
          * currently searching pattern and its results
         **/
@@ -156,27 +167,33 @@ class CurseOutput
          * prev/next items to be selected on up/down keys
         **/
         const JSonElement *select_up, *select_down;
+
         /**
          * Program params (ac/av)
         **/
         const Params &params;
+
         /**
          * ncurses stuff
         **/
         SCREEN *screen;
         FILE *screen_fd;
+
         /**
          * Used by signals to stop reading input and shutdown ncurses
         **/
         bool breakLoop;
+
         /**
          * Viewport start
         **/
         int scrollTop;
+
         /**
          * initialized colors
         **/
         std::set<char /* OutputFlag::TYPE_SOMETHING */> colors;
+
         /**
          * Selection helpers
          * Used for moving viewport

include/jsonElement.hh

@@ -9,6 +9,7 @@
 #include <string>
 
 class JSonContainer;
+class SearchPattern;
 
 class JSonElement
 {
@@ -46,7 +47,7 @@ class JSonElement
         /**
          * check if this element match SearchQuery
         **/
-        virtual bool match(const std::string &) const;
+        virtual bool match(const SearchPattern &) const;
 
     private:
         JSonElement();

include/jsonPrimitive.hh

@@ -8,6 +8,8 @@
 
 #include "jsonElement.hh"
 
+class Null {}; //Null primitive
+
 class AJSonPrimitive
 {
     public:

include/outputFlag.hh

@@ -51,6 +51,7 @@ class OutputFlag
         static const char TYPE_OBJ;
         static const char TYPE_OBJKEY;
         static const char TYPE_ARR;
+        static const char TYPE_NULL;
 
         static const char SPECIAL_NONE;
         static const char SPECIAL_SEARCH;

include/params.hh

@@ -43,6 +43,7 @@ class Params: public AParams
          * can be file stream (-f), stringstream ( -- INPUT), or std::cin (none)
         **/
         std::basic_istream<char> &getInput() const;
+
         /**
          * false if invalid argument is passed
         **/
@@ -53,11 +54,13 @@ class Params: public AParams
          * @param program name
         **/
         virtual void usage() const noexcept;
+
         /**
          * print version number
          * @param program name
         **/
         virtual void version() const noexcept;
+
         /**
          * get argv[0]
         **/

include/searchPattern.hh

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <string>
+
+class JSonElement;
+
+class SearchPattern
+{
+    public:
+        SearchPattern(const std::string &);
+        ~SearchPattern();
+
+        bool isEmpty() const;
+        bool match(const std::string &other, const JSonElement *) const;
+
+        /**
+         * Comparison function, for std::search use
+        **/
+        bool operator()(char a, char b);
+
+    private:
+        void evalFlags(const char *);
+
+        std::string pattern;
+        short flags;
+        short typeFlag;
+
+        static const short FLAG_CASE;
+        static const short FLAG_WHOLEWORD;
+        static const short FLAG_WHOLESTR;
+
+        static const short TYPE_BOOL;
+        static const short TYPE_NUMBER;
+        static const short TYPE_STRING;
+        static const short TYPE_OBJKEY;
+};
+

include/streamConsumer.hh

@@ -45,16 +45,26 @@ class StreamConsumer
         **/
         StreamConsumer *withConfig(const AParams *);
 
+        /**
+         * find \uXXXX in buffer and replace them
+        **/
+        static std::string extractUnicode(const char *);
+        static std::string extractUnicode(const std::string &);
+
     private:
         /**
          * @return non-null on successfully read JSonElement, or null if token (',', '[', ...)
         **/
-        JSonElement *consumeToken(JSonContainer *parent, std::string &buf);
+        JSonElement *consumeToken(JSonContainer *parent, std::stringstream &buf);
+        JSonElement *consumeString(JSonContainer *parent, std::stringstream &buf);
+        JSonElement *consumeBool(JSonContainer *parent, std::stringstream &buf, char c);
+        JSonElement *consumeNumber(JSonContainer *parent, std::stringstream &buf, char c);
+        JSonElement *consumeNull(JSonContainer *parent, std::stringstream &buf);
+
         /**
          * read next item, fill object or array if found
         **/
         JSonElement *readNext(JSonContainer *parent);
-
         /**
          * fill object
         **/
@@ -63,14 +73,16 @@ class StreamConsumer
          * fill array
         **/
         JSonArray *readArray(JSonContainer *parent);
+
         /**
          * out of token, should we ignore that char ?
         **/
         bool ignoreChar(char c) const noexcept;
+
         /**
          * compute unicode value and append it to buffer
         **/
-        static void appendUnicode(const char [4], std::string &);
+        static void appendUnicode(const char [4], std::stringstream &);
 
         /**
          * input stream

src/curseOutput.cpp

@@ -12,11 +12,13 @@
 #include <signal.h>
 #include <string.h>
 
+#include "searchPattern.hh"
 #include "curseOutput.hh"
 #include "jsonPrimitive.hh"
 #include "jsonObject.hh"
 #include "jsonArray.hh"
 #include "except.hh"
+#include "streamConsumer.hh"
 
 static CurseOutput *runningInst = nullptr;
 class SelectionOutOfRange { };
@@ -360,6 +362,8 @@ const OutputFlag CurseOutput::getFlag(const JSonElement *item) const
         res.type(OutputFlag::TYPE_STRING);
     else if (dynamic_cast<const JSonPrimitive<bool> *>(i))
         res.type(OutputFlag::TYPE_BOOL);
+    else if (dynamic_cast<const JSonPrimitive<Null> *>(i))
+        res.type(OutputFlag::TYPE_NULL);
     else if (dynamic_cast<const AJSonPrimitive *>(i))
         res.type(OutputFlag::TYPE_NUMBER);
     else if (dynamic_cast<const JSonObject*>(i))
@@ -504,18 +508,14 @@ bool CurseOutput::readInput()
 
             case '/':
             {
-                std::string search_pattern;
-                try {
-                    search_pattern = inputSearch();
-                }
-                catch (interruptedException &e)
-                {
+                const SearchPattern *search_pattern = inputSearch();
+                if (!search_pattern)
                     return true;
-                }
                 search_result.clear();
-                if (search_pattern.empty())
+                if (search_pattern->isEmpty())
                     return true;
-                search(search_pattern, data);
+                search(*search_pattern, data);
+                delete search_pattern;
             }
 
             case 'n':
@@ -530,7 +530,7 @@ bool CurseOutput::readInput()
     return false;
 }
 
-unsigned int CurseOutput::search(const std::string &search_pattern, const JSonElement *current)
+unsigned int CurseOutput::search(const SearchPattern &search_pattern, const JSonElement *current)
 {
     const JSonContainer *container = dynamic_cast<const JSonContainer *> (current);
     const JSonObjectEntry *objEntry = dynamic_cast<const JSonObjectEntry *> (current);
@@ -615,24 +615,23 @@ void CurseOutput::unfold(const JSonElement *item)
     }
 }
 
-const std::string CurseOutput::inputSearch()
+const SearchPattern *CurseOutput::inputSearch()
 {
-    std::string buffer;
+    std::wstring buffer;
     bool abort = false;
 
     curs_set(true);
-    keypad(stdscr, false);
     wtimeout(stdscr, -1);
     while (!abort)
     {
         int c;
 
-        writeBottomLine('/' +buffer, OutputFlag::SPECIAL_SEARCH);
+        writeBottomLine(L'/' +buffer, OutputFlag::SPECIAL_SEARCH);
         refresh();
-        c = getch();
-        if (c == '\n')
+        c = getwchar();
+        if (c == L'\r')
             break;
-        else if (c == '\b' || c == 127)
+        else if (c == L'\b' || c == 127)
         {
             if (!buffer.empty())
                 buffer.pop_back();
@@ -643,18 +642,26 @@ const std::string CurseOutput::inputSearch()
             buffer += c;
     }
     wtimeout(stdscr, 150);
-    keypad(stdscr, true);
     curs_set(false);
 
-    if (abort)
-        throw interruptedException();
-    return buffer;
+    {
+        const size_t size = buffer.size();
+        char bytesString[size * sizeof(wchar_t)];
+        wcstombs(&bytesString[0], buffer.c_str(), sizeof(bytesString));
+        std::string str;
+        if (params.isIgnoringUnicode())
+            str = bytesString;
+        else
+            str = StreamConsumer::extractUnicode(bytesString);
+        return abort ? nullptr : new SearchPattern(str);
+    }
 }
 
 void CurseOutput::writeBottomLine(const std::string &buffer, short color) const
 {
     const std::pair<unsigned int, unsigned int> screenSize = getScreenSize();
-    size_t bufsize = buffer.size();
+    const size_t bufsize = buffer.size();
+
     if (params.colorEnabled())
         attron(COLOR_PAIR(color));
     mvprintw(screenSize.second -1, 0, "%s%*c", buffer.c_str(), screenSize.first - bufsize, ' ');
@@ -663,6 +670,19 @@ void CurseOutput::writeBottomLine(const std::string &buffer, short color) const
         attroff(COLOR_PAIR(color));
 }
 
+void CurseOutput::writeBottomLine(const std::wstring &buffer, short color) const
+{
+    const std::pair<unsigned int, unsigned int> screenSize = getScreenSize();
+    const size_t bufsize = buffer.size();
+
+    if (params.colorEnabled())
+        attron(COLOR_PAIR(color));
+    mvprintw(screenSize.second -1, 0, "%S%*c", buffer.c_str(), screenSize.first - bufsize, ' ');
+    move(screenSize.second -1, bufsize);
+    if (params.colorEnabled())
+        attroff(COLOR_PAIR(color));
+}
+
 void CurseOutput::init()
 {
     if (!isatty(fileno(stdin)) || !isatty(fileno(stdout)))
@@ -688,6 +708,7 @@ void CurseOutput::init()
         start_color();
         init_pair(OutputFlag::TYPE_NUMBER, COLOR_GREEN, COLOR_BLACK);
         init_pair(OutputFlag::TYPE_BOOL, COLOR_RED, COLOR_BLACK);
+        init_pair(OutputFlag::TYPE_NULL, COLOR_RED, COLOR_BLACK);
         init_pair(OutputFlag::TYPE_STRING, COLOR_CYAN, COLOR_BLACK);
         init_pair(OutputFlag::TYPE_OBJKEY, COLOR_CYAN, COLOR_BLACK);
         init_pair(OutputFlag::SPECIAL_SEARCH, COLOR_WHITE, COLOR_BLUE);
@@ -696,6 +717,7 @@ void CurseOutput::init()
         colors.insert(OutputFlag::TYPE_BOOL);
         colors.insert(OutputFlag::TYPE_STRING);
         colors.insert(OutputFlag::TYPE_OBJKEY);
+        colors.insert(OutputFlag::TYPE_NULL);
     }
 
     signal(SIGWINCH, _resizeFnc);

src/jsonElement.cpp

@@ -7,6 +7,7 @@
 #include "jsonElement.hh"
 #include "jsonContainer.hh"
 #include "jsonObjectEntry.hh"
+#include "searchPattern.hh"
 
 JSonElement::JSonElement(JSonElement *p): parent(p)
 { }
@@ -77,9 +78,8 @@ const JSonElement* JSonElement::findNext() const
     return parent->findNext();
 }
 
-bool JSonElement::match(const std::string &search_pattern) const
+bool JSonElement::match(const SearchPattern &searchPattern) const
 {
-    const std::string str = stringify();
-    return str.find(search_pattern) != str.npos;
+    return searchPattern.match(stringify(), this);
 }
 

src/jsonObjectEntry.cpp

@@ -37,45 +37,3 @@ std::string JSonObjectEntry::stringify() const
     return key;
 }
 
-/*
-const JSonElement *JSonObjectEntry::findPrev() const
-{
-    const JSonObject *parent = (JSonObject*) getParent();
-    if (parent == nullptr)
-        return nullptr; // Root node, can't have brothers
-    std::list<JSonElement *>::const_iterator it = parent->cbegin();
-    const JSonObjectEntry *ent = (const JSonObjectEntry *)(*it);
-    const JSonObjectEntry *prevElem = ent;
-    if (prevElem == this)
-        return nullptr; // First item
-    while ((++it) != parent->cend())
-    {
-        ent = (const JSonObjectEntry *)(*it);
-        if (*it == this)
-            return prevElem;
-        prevElem = ent;
-    }
-    return nullptr;
-}
-
-const JSonElement* JSonObjectEntry::findNext() const
-{
-    const JSonObject *parent = (const JSonObject*) this->getParent();
-    if (parent == nullptr)
-        return nullptr; // Root node, can't have brothers
-    JSonContainer::const_iterator it = parent->cbegin();
-    while (it != parent->cend())
-    {
-        if (*it == this)
-        {
-            it++;
-            if (it == parent->cend())
-                return parent->findNext(); // Last item
-            return *it;
-        }
-        it++;
-    }
-    return parent->findNext();
-}
-*/
-

src/jsonPrimitive.cpp

@@ -9,6 +9,7 @@
 AJSonPrimitive::~AJSonPrimitive()
 {}
 
+template<> JSonPrimitive<Null>::~JSonPrimitive() {}
 template<> JSonPrimitive<double>::~JSonPrimitive() {}
 template<> JSonPrimitive<bool>::~JSonPrimitive() {}
 template<> JSonPrimitive<int>::~JSonPrimitive() {}
@@ -18,6 +19,9 @@ template<> JSonPrimitive<std::string>::~JSonPrimitive() {}
 template<> std::string JSonPrimitive<std::string>::toString() const
 { return value; }
 
+template<> std::string JSonPrimitive<Null>::toString() const
+{ return "null"; }
+
 template<> std::string JSonPrimitive<double>::toString() const
 {
     return std::to_string(value);

src/linearHistory.cpp

@@ -23,7 +23,7 @@ void LinearHistory::put(char item)
         line++;
         willReset = false;
     }
-    if (item == '\n')
+    if (item == L'\n' || item == L'\r')
         willReset = true;
     else
         WrappedBuffer<char, ERROR_HISTORY_LEN>::put(item);

src/main.cpp

@@ -20,7 +20,6 @@ void displayException(const Params *params, const std::string &type, const JsonE
 
 void run(Params *params)
 {
-
     StreamConsumer stream(StreamConsumer(params->getInput()));
     stream.withConfig(params);
     CurseOutput *out;

src/outputFlag.cpp

@@ -13,6 +13,7 @@ const char OutputFlag::TYPE_BOOL    =3;
 const char OutputFlag::TYPE_OBJ     =4;
 const char OutputFlag::TYPE_OBJKEY  =5;
 const char OutputFlag::TYPE_ARR     =6;
+const char OutputFlag::TYPE_NULL    =7;
 
 const char OutputFlag::SPECIAL_NONE     =50;
 const char OutputFlag::SPECIAL_SEARCH   =51;

src/params.cpp

@@ -13,7 +13,7 @@
 
 #include "config.h"
 
-Params::Params(char **av) :progName(*av), strict(true)
+Params::Params(char **av): input(nullptr), progName(*av), strict(true)
 {
     av++;
     while (*av)
@@ -36,9 +36,7 @@ bool Params::read()
         if (!input)
         {
             if (tmp == "--")
-            {
                 input = new std::stringstream();
-            }
             else if (tmp == "-W")
                 strict = false;
             else if (tmp == "--ascii")
@@ -115,7 +113,7 @@ void Params::usage() const noexcept
     << "if not INPUT nor FILENAME, use standard input" << std::endl << std::endl
 
     << "  FILENAME\t\tread input from filename instead of stdin" << std::endl
-    << "  INPUT\t\tuse this as input instead of stdin" << std::endl
+    << "  INPUT\t\t\tuse this as input instead of stdin" << std::endl
     << "  -W \t\t\tconsider continuing on non-blocking errors" << std::endl
     << "  --ascii\t\tignore unicode values" << std::endl
     << "  --color[=MODE]\tcolorize output, MODE can be never or always (default when ommited)" << std::endl

src/searchPattern.cpp

@@ -0,0 +1,114 @@
+#include <algorithm>
+#include <sstream>
+#include "searchPattern.hh"
+
+#include "jsonObjectEntry.hh"
+#include "jsonPrimitive.hh"
+
+SearchPattern::SearchPattern(const std::string &input): flags(0)
+{
+    size_t pos = 0;
+    bool escaped = false;
+    std::stringstream ss;
+
+    for (pos =0; input[pos] && (input[pos] == ' ' || input[pos] == '\t'); ++pos); //trim leading characters
+    for (; input[pos]; ++pos)
+    {
+        if (escaped)
+        {
+            if (input[pos] == '/')
+                ss.put(input[pos]);
+            else
+                ss.put('\\').put(input[pos]);
+            escaped = false;
+        }
+        else if (input[pos] == '\\')
+            escaped = true;
+        else if (input[pos] == '/')
+        {
+            pattern = ss.str();
+            evalFlags(&input[pos +1]);
+            return;
+        }
+        else
+            ss.put(input[pos]);
+    }
+    pattern = ss.str();
+}
+
+SearchPattern::~SearchPattern()
+{ }
+
+void SearchPattern::evalFlags(const char *s)
+{
+    while (*s)
+    {
+        if (*s == 'i')
+        {
+            flags |= SearchPattern::FLAG_CASE;
+            std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower);
+        }
+        else if (*s == 'b')
+            typeFlag = SearchPattern::TYPE_BOOL;
+        else if (*s == 'n')
+            typeFlag = SearchPattern::TYPE_NUMBER;
+        else if (*s == 's')
+            typeFlag = SearchPattern::TYPE_STRING;
+        else if (*s == 'o')
+            typeFlag = SearchPattern::TYPE_OBJKEY;
+        else if (*s == 'w')
+            flags |= SearchPattern::FLAG_WHOLEWORD;
+        else if (*s == 'f')
+            flags |= SearchPattern::FLAG_WHOLESTR;
+        s++;
+    }
+}
+
+bool SearchPattern::isEmpty() const
+{ return pattern.empty(); }
+
+bool SearchPattern::operator()(char a, char b)
+{
+    if (a == '\t')
+        a = ' ';
+    if (flags & SearchPattern::FLAG_CASE)
+        return std::tolower(a) == b;
+    return a == b;
+}
+
+bool SearchPattern::match(const std::string &str, const JSonElement *type) const
+{
+    if (typeFlag)
+    {
+        if (typeFlag == SearchPattern::TYPE_BOOL && !(dynamic_cast<const JSonPrimitive<bool> *>(type)))
+            return false;
+        else if (typeFlag == SearchPattern::TYPE_STRING && !(dynamic_cast<const JSonPrimitive<std::string> *>(type)))
+            return false;
+        else if (typeFlag == SearchPattern::TYPE_OBJKEY && !(dynamic_cast<const JSonObjectEntry *>(type)))
+            return false;
+        else if (typeFlag == SearchPattern::TYPE_NUMBER &&
+                !(dynamic_cast<const JSonPrimitive<int> *>(type)) &&
+                !(dynamic_cast<const JSonPrimitive<double> *>(type)) &&
+                !(dynamic_cast<const JSonPrimitive<long long> *>(type)))
+            return false;
+    }
+    if ((flags & FLAG_WHOLESTR && str.length() != pattern.length())
+            || pattern.length() > str.length())
+        return false;
+    if (flags & FLAG_WHOLEWORD && str.length() > pattern.length())
+    {
+        std::string pattern = " " +this->pattern +' ';
+        return std::search(str.begin(), str.end(), pattern.begin(), pattern.end(), *this) != str.end();
+    }
+    return std::search(str.begin(), str.end(), pattern.begin(), pattern.end(), *this) != str.end();
+}
+
+const short SearchPattern::FLAG_CASE        = 1;
+const short SearchPattern::FLAG_WHOLEWORD   = 2;
+const short SearchPattern::FLAG_WHOLESTR    = 4;
+
+const short SearchPattern::TYPE_BOOL        = 1;
+const short SearchPattern::TYPE_NUMBER      = 2;
+const short SearchPattern::TYPE_STRING      = 3;
+const short SearchPattern::TYPE_OBJKEY      = 4;
+

src/streamConsumer.cpp

@@ -34,8 +34,9 @@ StreamConsumer *StreamConsumer::read()
 
 JSonElement *StreamConsumer::readNext(JSonContainer *parent)
 {
-    std::string buf;
-    JSonElement *root = consumeToken(parent, buf);
+    std::stringstream sbuf;
+    JSonElement *root = consumeToken(parent, sbuf);
+    const std::string buf = sbuf.str();
 
     if (root == nullptr)
     {
@@ -63,17 +64,17 @@ JSonObject *StreamConsumer::readObject(JSonContainer *parent)
 {
     JSonElement *keyObj;
     JSonObject *result = nullptr;
-    std::string buf;
+    std::stringstream buf;
 
     do
     {
         keyObj = consumeToken(result, buf);
-        if (result == nullptr && keyObj == nullptr && buf == "}")
+        if (result == nullptr && keyObj == nullptr && buf.str() == "}")
             return new JSonObject(parent);
         JSonPrimitive<std::string> *key = dynamic_cast<JSonPrimitive<std::string> *>(keyObj);
         if (key == nullptr)
             throw JSonObject::NotAKeyException(stream.tellg(), history);
-        if (consumeToken(parent, buf) != nullptr || buf != ":")
+        if (consumeToken(parent, buf) != nullptr || buf.str() != ":")
             throw JsonUnexpectedException(':', stream.tellg(), history);
         if (result == nullptr)
             result = new JSonObject(parent);
@@ -91,15 +92,16 @@ JSonObject *StreamConsumer::readObject(JSonContainer *parent)
         result->push(key->getValue(), child);
         delete keyObj;
         keyObj = consumeToken(result, buf);
-    } while (!keyObj && buf != "}");
+    } while (!keyObj && buf.str() != "}");
     return result;
 }
 
 JSonArray *StreamConsumer::readArray(JSonContainer *parent)
 {
     JSonArray *result = nullptr;
-    std::string buf;
-    JSonElement *child = consumeToken(parent, buf);
+    std::stringstream sbuf;
+    JSonElement *child = consumeToken(parent, sbuf);
+    std::string buf = sbuf.str();
 
     if (child == nullptr && buf == "]")
         return new JSonArray(parent); //Empty object
@@ -115,159 +117,221 @@ JSonArray *StreamConsumer::readArray(JSonContainer *parent)
             result = new JSonArray(parent);
         child->setParent(result);
         result->push_back(child);
-        child = consumeToken(result, buf);
+        child = consumeToken(result, sbuf);
+        buf = sbuf.str();
         if (child != nullptr)
             throw JsonUnexpectedException(']', stream.tellg(), history);
         else if (buf == "]")
             break;
         else if (buf != ",")
             throw JsonUnexpectedException(']', stream.tellg(), history);
-        child = consumeToken(result, buf);
+        child = consumeToken(result, sbuf);
+        buf = sbuf.str();
     } while (true);
     return result;
 }
 
-JSonElement *StreamConsumer::consumeToken(JSonContainer *parent, std::string &buf)
+JSonElement *StreamConsumer::consumeString(JSonContainer *parent, std::stringstream &buf)
 {
     bool escaped = false;
-    bool inString = false;
-    bool inBool = false;
-    bool inNumber = false;
-    bool numberIsDouble = false;
+
+    buf.str("");
+    buf.clear();
 
     while (stream.good())
     {
         char c = stream.get();
         history.put(c);
 
-        if (inString)
+        if (!escaped)
         {
-            if (!escaped)
-            {
-                if (c == '"')
-                    return new JSonPrimitive<std::string>(parent, buf);
-                else if (c == '\\')
-                    escaped = true;
-                else
-                    buf += c;
-            }
+            if (c == '"')
+                return new JSonPrimitive<std::string>(parent, buf.str());
+            else if (c == '\\')
+                escaped = true;
             else
+                buf.write(&c, 1);
+        }
+        else
+        {
+            if (c == '\\' || c == '"')
+                buf.write("\"", 1);
+            else if (c == 'u')
             {
-                if (c == '\\' || c == '"')
-                {
-                    buf += c;
-                    escaped = false;
-                }
-                else if (c == 'u')
+                if (params && params->isIgnoringUnicode())
+                    buf.write("\\u", 2);
+                else
                 {
-                    if (params && params->isIgnoringUnicode())
-                    {
-                        buf += "\\u";
-                        escaped = false;
+                    char unicodeBuf[4];
+                    stream.read(unicodeBuf, 4);
+                    std::streamsize gcount = stream.gcount();
+                    history.put(unicodeBuf, gcount);
+                    if (gcount != 4)
+                        break;
+                    try {
+                        appendUnicode(unicodeBuf, buf);
                     }
-                    else
+                    catch (std::invalid_argument &e)
                     {
-                        char unicodeBuf[4];
-                        stream.read(unicodeBuf, 4);
-                        std::streamsize gcount = stream.gcount();
-                        history.put(unicodeBuf, gcount);
-                        if (gcount != 4)
-                            break;
-                        try {
-                            appendUnicode(unicodeBuf, buf);
-                        }
-                        catch (std::invalid_argument &e)
-                        {
-                            throw JsonHexvalueException(e.what(), stream.tellg(), history);
-                        }
-                        escaped = false;
+                        throw JsonHexvalueException(e.what(), stream.tellg(), history);
                     }
                 }
-                else
-                    throw JsonEscapedException(c, stream.tellg(), history);
             }
-        }
-        else if (inBool)
-        {
-            if (c == 'a' || c == 'e' || c == 'l' || c == 'r' || c == 's' || c == 'u')
-                buf += c;
-            else if (buf == "true")
-            {
-                history.pop_back();
-                stream.unget();
-                return new JSonPrimitive<bool>(parent, true);
-            }
-            else if (buf == "false")
+            else if (params->isStrict())
+                throw JsonEscapedException(c, stream.tellg(), history);
+            else
             {
-                history.pop_back();
-                stream.unget();
-                return new JSonPrimitive<bool>(parent, false);
+                buf.write("\\", 1).write(&c, 1);
+                warnings.push_back(Warning(JsonEscapedException(c, stream.tellg(), history)));
             }
-            else if (ignoreChar(c))
-                ;
-            else
+            escaped = false;
+        }
+    }
+    buf.str("");
+    buf.clear();
+    return nullptr;
+}
+
+JSonElement *StreamConsumer::consumeBool(JSonContainer *parent, std::stringstream &buf, char firstChar)
+{
+    size_t read =1;
+
+    buf.str("");
+    buf.clear();
+    buf.write(&firstChar, 1);
+
+    //TODO batch-get 3 char, then do that
+    while (stream.good())
+    {
+        char c = stream.get();
+        history.put(c);
+
+        if (c == 'a' || c == 'e' || c == 'l' || c == 'r' || c == 's' || c == 'u')
+        {
+            if ((read >= 5 && firstChar == 'f') || (read >= 4 && firstChar == 't'))
                 throw JsonFormatException(stream.tellg(), history);
+            buf.write(&c, 1);
+            read++;
         }
-        else if (inNumber)
+        else if (buf.str() == "true")
         {
-            if (c >= '0' && c <= '9')
-                buf += c;
-            else if (c == '.' && !numberIsDouble)
-            {
-                numberIsDouble = true;
-                buf += c;
-            }
-            else
-            {
-                history.pop_back();
-                stream.unget();
-                if (numberIsDouble)
-                {
-                    try {
-                        return new JSonPrimitive<double>(parent, atof(buf.c_str()));
-                    } catch (std::runtime_error &e)
-                    {
-                        throw JsonFormatException(stream.tellg(), history);
-                    }
-                }
-                try
-                {
-                    return new JSonPrimitive<int>(parent, std::stoi(buf));
-                }
-                catch(std::out_of_range e)
-                {
-                    return new JSonPrimitive<long long>(parent, std::stol(buf));
-                }
-            }
+            history.pop_back();
+            stream.unget();
+            return new JSonPrimitive<bool>(parent, true);
+        }
+        else if (buf.str() == "false")
+        {
+            history.pop_back();
+            stream.unget();
+            return new JSonPrimitive<bool>(parent, false);
         }
+        else if (ignoreChar(c))
+            ;
         else
+            throw JsonFormatException(stream.tellg(), history);
+    }
+    buf.str("");
+    buf.clear();
+    return nullptr;
+}
+
+JSonElement *StreamConsumer::consumeNumber(JSonContainer *parent, std::stringstream &buf, char firstChar)
+{
+    bool numberIsDouble = false;
+
+    buf.str("");
+    buf.clear();
+    buf.write(&firstChar, 1);
+
+    while (stream.good())
+    {
+        char c = stream.get();
+        history.put(c);
+
+        if (c >= '0' && c <= '9')
+            buf.write(&c, 1);
+        else if (c == '.' && !numberIsDouble)
         {
-            //!InString, !inbool
-            if (c == '"')
-            {
-                buf = "";
-                inString = true;
-            }
-            else if (c == 't' || c == 'f')
+            numberIsDouble = true;
+            buf.write(&c, 1);
+        }
+        else
+        {
+            history.pop_back();
+            stream.unget();
+            if (numberIsDouble)
             {
-                buf = c;
-                inBool = true;
+                try {
+                    return new JSonPrimitive<double>(parent, atof(buf.str().c_str()));
+                } catch (std::runtime_error &e)
+                {
+                    throw JsonFormatException(stream.tellg(), history);
+                }
             }
-            else if (c == '{' || c == '[' || c == '}' || c == ']' || c == ':' || c == ',')
+            try
             {
-                buf = c;
-                return nullptr;
+                return new JSonPrimitive<int>(parent, std::stoi(buf.str()));
             }
-            else if ((c >= '0' && c <= '9') || c == '.' || c == '-')
+            catch(std::out_of_range e)
             {
-                buf = c;
-                inNumber = true;
+                return new JSonPrimitive<long long>(parent, std::stol(buf.str()));
             }
-            else if (!ignoreChar(c))
-                throw JsonFormatException(stream.tellg(), history);
         }
     }
-    buf = "";
+    buf.str("");
+    buf.clear();
+    return nullptr;
+}
+
+JSonElement *StreamConsumer::consumeNull(JSonContainer *parent, std::stringstream &buf)
+{
+    char _buf[5] = { 'n', '\0', '\0', '\0', '\0' };
+
+    buf.str("");
+    buf.clear();
+
+    stream.read(&_buf[1], 3);
+    buf.write(_buf, 4);
+    history.put(&_buf[1], 3);
+    if (!stream.good())
+    {
+        buf.str("");
+        buf.clear();
+        return nullptr;
+    }
+    if (std::string("null") == _buf)
+        return new JSonPrimitive<Null>(parent, Null());
+    throw JsonFormatException(stream.tellg(), history);
+}
+
+JSonElement *StreamConsumer::consumeToken(JSonContainer *parent, std::stringstream &buf)
+{
+    while (stream.good())
+    {
+        char c = stream.get();
+        history.put(c);
+
+        //!InString, !inbool
+        if (c == '"')
+            return consumeString(parent, buf);
+        else if (c == 't' || c == 'f')
+            return consumeBool(parent, buf, c);
+        else if (c == 'n')
+            return consumeNull(parent, buf);
+        else if ((c >= '0' && c <= '9') || c == '.' || c == '-')
+            return consumeNumber(parent, buf, c);
+        else if (c == '{' || c == '[' || c == '}' || c == ']' || c == ':' || c == ',')
+        {
+            buf.str("");
+            buf.clear();
+            buf.write(&c, 1);
+            return nullptr;
+        }
+        else if (!ignoreChar(c))
+            throw JsonFormatException(stream.tellg(), history);
+    }
+    buf.str("");
+    buf.clear();
     return nullptr;
 }
 
@@ -291,13 +355,35 @@ static T hexbyte(const char str[], unsigned int len)
     return result;
 }
 
-void StreamConsumer::appendUnicode(const char unicode[4], std::string &buf)
+void StreamConsumer::appendUnicode(const char unicode[4], std::stringstream &buf)
 {
     unsigned short uni = hexbyte<unsigned short>(unicode, 4);
     char test[5];
     bzero(test, sizeof(*test) *5);
     snprintf(test, 4, "%lc", uni);
-    buf += test;
+    buf.write(test, 4);
+}
+
+std::string StreamConsumer::extractUnicode(const char *buf)
+{
+    std::stringstream result;
+
+    for (; *buf; buf++)
+    {
+        if (*buf == '\\' && buf[1] == 'u' && buf[2] && buf[3] && buf[4] && buf[5])
+        {
+            appendUnicode(buf +2, result);
+            buf += 6;
+        }
+        else
+            result.write(buf, 1);
+    }
+    return result.str();
+}
+
+std::string StreamConsumer::extractUnicode(const std::string &buf)
+{
+    return extractUnicode(buf.c_str());
 }
 
 bool StreamConsumer::ignoreChar(char c) const noexcept

test/longline.json

@@ -1 +1 @@
-[" Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a tortor sed lectus rhoncus porttitor. Duis condimentum enim ac consequat vulputate. Sed feugiat sapien id hendrerit congue. Nunc in nulla nec mauris sollicitudin porta. Quisque sapien velit, interdum nec sodales vitae, tristique quis tortor. Nam quam libero, placerat sed erat quis, ultrices semper augue. Aenean vitae leo eu massa porta fringilla sed et arcu.  Etiam ex tellus, laoreet at ante non, auctor tempus nisi. Vestibulum blandit ipsum ligula, in ultrices neque malesuada id. Donec non convallis sem. Nunc ac orci eu dui finibus pharetra. Mauris quis purus turpis. In porta felis lectus, in bibendum nisl molestie vitae. Donec luctus quam nec tristique luctus. Donec ultrices ante id cursus scelerisque. Vestibulum tempus massa et finibus sollicitudin. ", "EOF"]
+["Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a tortor sed lectus rhoncus porttitor. Duis condimentum enim ac consequat vulputate. Sed feugiat sapien id hendrerit congue. Nunc in nulla nec mauris sollicitudin porta. Quisque sapien velit, interdum nec sodales vitae, tristique quis tortor. Nam quam libero, placerat sed erat quis, ultrices semper augue. Aenean vitae leo eu massa porta fringilla sed et arcu.  Etiam ex tellus, laoreet at ante non, auctor tempus nisi. Vestibulum blandit ipsum ligula, in ultrices neque malesuada id. Donec non convallis sem. Nunc ac orci eu dui finibus pharetra. Mauris quis purus turpis. In porta felis lectus, in bibendum nisl molestie vitae. Donec luctus quam nec tristique luctus. Donec ultrices ante id cursus scelerisque. Vestibulum tempus massa et finibus sollicitudin.", "EOF"]

test/test.json

@@ -4,7 +4,7 @@
         "0":"897316929176464ebc9ad085f31e7284",
         "1":"b026324c6904b2a9cb4b88d6d61c81d1",
         "2":"26ab0db90d72e28ad0ba1e22ee510510",
-        "3":"6d7fce9fee471194aa8b5b6e47267f03",
+        "3":"6d7fc\\e9/fee471194aa8b5b6e47267f03",
         "4":"48a24b70a0b376535542b996af517398",
         "5":"1dcca23355272056f04fe8bf20edfce0",
         "6":"9ae0ea9e3c9c6e1b9b6252c8395efdc1",
@@ -18,7 +18,7 @@
         "14":"367764329430db34be92fd14a7a770ee",
         "15":"8c9eb686bf3eb5bd83d9373eadf6504b",
         "16":"5b6b41ed9b343fed9cd05a66d36650f0",
-        "17":"4d095eeac8ed659b1ce69dcef32ed0dc",
+        "17":"4d095eeAC8ed659b1ce69dcef32ed0dc",
         "18":"cf4278314ef8e4b996e1b798d8eb92cf",
         "19":"3bb50ff8eeb7ad116724b56a820139fa",
         "20":"dbbf8220893d497d403bb9cdf49db7a4",