o Replacing regex based parsing with simpler and more correct tokenizing.

o Flushing output after each sample.
o Adding back tests.

5 files changed, 134 insertions, 79 deletions

diff --git a/sensor/CMakeLists.txt b/sensor/CMakeLists.txt
index 14804f1..d571e38 100644
--- a/sensor/CMakeLists.txt
+++ b/sensor/CMakeLists.txt
@@ -10,3 +10,5 @@ include_directories(include)
 
 # Boost
 find_package(Boost COMPONENTS regex system REQUIRED)
+
+add_subdirectory(test)
diff --git a/sensor/main/io.cpp b/sensor/main/io.cpp
index 57c0b18..cc713b9 100644
--- a/sensor/main/io.cpp
+++ b/sensor/main/io.cpp
@@ -1,17 +1,17 @@
 #include "trygvis/sensor/io.h"
 
-#include <ostream>
-#include <vector>
 #include <map>
-#include <mutex>
 #include "json.hpp"
-#include "boost/regex.hpp"
+#include "boost/tokenizer.hpp"
+#include <boost/algorithm/string.hpp>
 
 namespace trygvis {
 namespace sensor {
 namespace io {
 
 using namespace std;
+using boost::tokenizer;
+using boost::escaped_list_separator;
 using json = nlohmann::json;
 
 ThreadSafeSampleOutputStream::ThreadSafeSampleOutputStream(unique_ptr<SampleOutputStream> underlying)
@@ -37,12 +37,12 @@ CsvSampleOutputStream::CsvSampleOutputStream(shared_ptr<ostream> stream, KeyDict
 }
 
 void CsvSampleOutputStream::write(SampleRecord const &sample) {
-// Skip empty records
+    // Skip empty records
     if (sample.empty()) {
         return;
     }
 
-// Build the dict with the keys from the first sample.
+    // Build the dict with the keys from the first sample.
     if (dict.empty()) {
         SampleKeyIndex index = 0;
         auto ptr = sample.cbegin();
@@ -81,7 +81,7 @@ void CsvSampleOutputStream::write(SampleRecord const &sample) {
         }
     }
 
-    s << endl;
+    s << endl << flush;
 }
 
 void CsvSampleOutputStream::writeHeader() {
@@ -98,7 +98,7 @@ void CsvSampleOutputStream::writeHeader() {
         }
     }
 
-    s << endl;
+    s << endl << flush;
 }
 
 JsonSampleOutputStream::JsonSampleOutputStream(shared_ptr<ostream> stream, KeyDictionary &dict) :
@@ -106,7 +106,7 @@ JsonSampleOutputStream::JsonSampleOutputStream(shared_ptr<ostream> stream, KeyDi
 }
 
 void JsonSampleOutputStream::write(SampleRecord const &sample) {
-// Skip empty records
+    // Skip empty records
     if (sample.empty()) {
         return;
     }
@@ -128,14 +128,14 @@ void JsonSampleOutputStream::write(SampleRecord const &sample) {
             auto o = sample.at(sampleKey);
 
             if (o) {
-// Make sure that the key is registered in the dictionary
+                // Make sure that the key is registered in the dictionary
                 dict.indexOf(sampleKey->name);
                 doc[sampleKey->name] = o.get();
             }
         }
     }
 
-    *stream.get() << doc << endl;
+    *stream.get() << doc << endl << flush;
 }
 
 KeyValueSampleOutputStream::KeyValueSampleOutputStream(shared_ptr<ostream> stream, KeyDictionary &dict) :
@@ -143,7 +143,7 @@ KeyValueSampleOutputStream::KeyValueSampleOutputStream(shared_ptr<ostream> strea
 }
 
 void KeyValueSampleOutputStream::write(SampleRecord const &sample) {
-// Skip empty records
+    // Skip empty records
     if (sample.empty()) {
         return;
     }
@@ -176,14 +176,14 @@ void KeyValueSampleOutputStream::write(SampleRecord const &sample) {
                 } else {
                     s << ", ";
                 }
-// Make sure that the key is registered in the dictionary
+                // Make sure that the key is registered in the dictionary
                 dict.indexOf(sampleKey->name);
                 s << sampleKey->name << "=" << o.get();
             }
         }
     }
 
-    *stream.get() << endl;
+    s << endl << flush;
 }
 
 RrdSampleOutputStream::RrdSampleOutputStream(shared_ptr<ostream> stream,
@@ -204,7 +204,7 @@ RrdSampleOutputStream::RrdSampleOutputStream(shared_ptr<ostream> stream,
 }
 
 void RrdSampleOutputStream::write(SampleRecord const &sample) {
-// Skip empty records
+    // Skip empty records
     if (sample.empty()) {
         return;
     }
@@ -239,7 +239,7 @@ void RrdSampleOutputStream::write(SampleRecord const &sample) {
         s << (value ? value.get() : "U");
     }
 
-    *stream.get() << endl;
+    s << endl << flush;
 }
 
 SqlSampleOutputStream::SqlSampleOutputStream(shared_ptr<ostream> stream, KeyDictionary &dict, string table_name) :
@@ -292,7 +292,7 @@ void SqlSampleOutputStream::write(SampleRecord const &values) {
 //        }
 //    }
 //
-//    (*stream.get()) << "INSERT INTO " << table_name << "(" << fs << ") VALUES(" << vs << ");" << endl;
+//    (*stream.get()) << "INSERT INTO " << table_name << "(" << fs << ") VALUES(" << vs << ");" << endl << flush;
 }
 
 void KeyValueSampleStreamParser::process(mutable_buffers_1 buffer) {
@@ -321,28 +321,27 @@ void KeyValueSampleStreamParser::process(mutable_buffers_1 buffer) {
 }
 
 void KeyValueSampleStreamParser::process_line(shared_ptr<vector<uint8_t>> packet) {
-    auto timestamp = std::chrono::system_clock::now().time_since_epoch().count();
     auto s = std::string((char *) packet->data(), packet->size());
 
-    static const boost::regex e("([#_a-zA-Z0-9]+) *= *([0-9]+)");
-
-    auto start = s.cbegin();
-    auto end = s.cend();
-    boost::match_results <std::string::const_iterator> what;
-    boost::match_flag_type flags = boost::match_default;
+    typedef tokenizer<escaped_list_separator<char>> Tokenizer;
+    Tokenizer tokens(s);
 
     SampleRecord sample(dict);
 
-    while (regex_search(start, end, what, e, flags)) {
-        auto name = static_cast<string>(what[1]);
-        auto value = static_cast<string>(what[2]);
-        start = what[0].second;
+    for (auto token : tokens) {
+        auto index = token.find('=');
+
+        if (index == string::npos) {
+            continue;
+        }
+
+        auto name = token.substr(0, index);
+        boost::algorithm::trim(name);
+        auto value = token.substr(index + 1);
+        boost::algorithm::trim(value);
 
         auto key = dict.indexOf(name);
         sample.set(key, value);
-
-        flags |= boost::match_prev_avail;
-        flags |= boost::match_not_bob;
     }
 
     output->write(sample);
diff --git a/sensor/test/CMakeLists.txt b/sensor/test/CMakeLists.txt
new file mode 100644
index 0000000..2e68532
--- /dev/null
+++ b/sensor/test/CMakeLists.txt
@@ -0,0 +1,26 @@
+find_package(Boost COMPONENTS log regex unit_test_framework REQUIRED)
+
+# If we can change directory here add_definition and test-specific stuff could be moved to the test directory
+file(GLOB TEST_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *Test.cpp)
+add_definitions(-DBOOST_TEST_DYN_LINK)
+
+foreach(testSrc ${TEST_SRCS})
+    get_filename_component(testName ${testSrc} NAME_WE)
+
+    #Add compile target
+    add_executable(${testName} ${testSrc})
+
+    include_directories(../include)
+
+    target_link_libraries(${testName} trygvis-sensor)
+    target_link_libraries(${testName} ${Boost_LIBRARIES})
+
+#    set_target_properties(${testName} PROPERTIES
+#        RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test)
+
+    #Finally add it to test execution -
+    #Notice the WORKING_DIRECTORY and COMMAND
+    add_test(NAME ${testName}
+             WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/testBin
+             COMMAND ${CMAKE_BINARY_DIR}/testBin/${testName})
+endforeach(testSrc)
diff --git a/sensor/test/SampleTest.cpp b/sensor/test/SampleTest.cpp
new file mode 100644
index 0000000..2404500
--- /dev/null
+++ b/sensor/test/SampleTest.cpp
@@ -0,0 +1,76 @@
+#include "trygvis/sensor.h"
+#include "trygvis/sensor/io.h"
+
+#define BOOST_TEST_MODULE "SampleTest"
+
+#include <boost/test/unit_test.hpp>
+
+using namespace trygvis::sensor;
+using namespace trygvis::sensor::io;
+using namespace std;
+using namespace boost;
+
+BOOST_AUTO_TEST_CASE(key_value_parser) {
+    KeyDictionary dict;
+
+    auto buffer = make_shared<VectorSampleOutputStream>();
+
+    auto parser = make_shared<KeyValueSampleStreamParser>(buffer, dict);
+
+    char data[] = "a=1,   b=2, c=3\n";
+    parser->process(boost::asio::buffer(data, sizeof(data)));
+    BOOST_CHECK_EQUAL(buffer->samples.size(), 1);
+    BOOST_CHECK_EQUAL(dict.size(), 3);
+    auto it = dict.begin();
+    BOOST_CHECK_EQUAL((*it)->name, "a");
+    BOOST_CHECK_EQUAL((*it++)->index, 0);
+    BOOST_CHECK_EQUAL((*it)->name, "b");
+    BOOST_CHECK_EQUAL((*it++)->index, 1);
+    BOOST_CHECK_EQUAL((*it)->name, "c");
+    BOOST_CHECK_EQUAL((*it++)->index, 2);
+}
+
+BOOST_AUTO_TEST_CASE(key_value_parser2) {
+    KeyDictionary dict;
+
+    auto buffer = make_shared<VectorSampleOutputStream>();
+
+    auto parser = make_shared<KeyValueSampleStreamParser>(buffer, dict);
+
+    char data[] = "now=1,sensor 1=0.000999999833333\n";
+    parser->process(boost::asio::buffer(data, sizeof(data)));
+    BOOST_CHECK_EQUAL(buffer->samples.size(), 1);
+    SampleRecord& sample = buffer->samples[0];
+    BOOST_CHECK_EQUAL(dict.size(), 2);
+    auto it = dict.begin();
+    BOOST_CHECK_EQUAL((*it)->name, "now");
+    BOOST_CHECK_EQUAL(!sample.at(*it).operator!(), true);
+    BOOST_CHECK_EQUAL(sample.at(*it).get(), "1");
+    BOOST_CHECK_EQUAL((*it++)->index, 0);
+
+    BOOST_CHECK_EQUAL((*it)->name, "sensor 1");
+    BOOST_CHECK_EQUAL(!sample.at(*it).operator!(), true);
+    BOOST_CHECK_EQUAL(sample.at(*it).get(), "0.000999999833333");
+    BOOST_CHECK_EQUAL((*it++)->index, 1);
+}
+
+BOOST_AUTO_TEST_CASE(key_value_parser_with_custom_dict) {
+    KeyDictionary dict;
+
+    dict.indexOf("c");
+    dict.indexOf("b");
+
+    auto buffer = make_shared<VectorSampleOutputStream>();
+
+    auto parser = make_shared<KeyValueSampleStreamParser>(buffer, dict);
+
+    char data[] = "a=1,   b=2, c=3\n";
+    parser->process(boost::asio::buffer(data, sizeof(data)));
+    BOOST_CHECK_EQUAL(buffer->samples.size(), 1);
+    BOOST_CHECK_EQUAL(dict.size(), 3);
+    auto it = dict.begin();
+    BOOST_CHECK_EQUAL((*it)->name, "c");
+    BOOST_CHECK_EQUAL((*it++)->index, 0);
+    BOOST_CHECK_EQUAL((*it)->name, "b");
+    BOOST_CHECK_EQUAL((*it++)->index, 1);
+}
diff --git a/sensor/test/SoilMoistureIoTest.cpp b/sensor/test/SoilMoistureIoTest.cpp
deleted file mode 100644
index 574885c..0000000
--- a/sensor/test/SoilMoistureIoTest.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-#include "SensorSample.h"
-
-#define BOOST_TEST_MODULE "SoilMoistureIoTest"
-
-#include <boost/test/unit_test.hpp>
-
-using namespace trygvis::soil_moisture;
-
-BOOST_AUTO_TEST_CASE(key_value_parser) {
-    KeyDictionary dict;
-
-    auto buffer = make_shared<VectorSampleOutputStream>();
-
-    auto parser = new KeyValueSampleStreamParser(buffer, dict);
-
-    char data[] = "a=1,   b=2, c=3\n";
-    parser->process(boost::asio::buffer(data, sizeof(data)));
-    BOOST_CHECK_EQUAL(buffer->samples.size(), 1);
-    BOOST_CHECK_EQUAL(dict.size(), 3);
-    auto it = dict.begin();
-    BOOST_CHECK_EQUAL((*it)->name, "a");
-    BOOST_CHECK_EQUAL((*it++)->index, 0);
-    BOOST_CHECK_EQUAL((*it)->name, "b");
-    BOOST_CHECK_EQUAL((*it++)->index, 1);
-    BOOST_CHECK_EQUAL((*it)->name, "c");
-    BOOST_CHECK_EQUAL((*it++)->index, 2);
-}
-
-BOOST_AUTO_TEST_CASE(key_value_parser_with_custom_dict) {
-    KeyDictionary dict;
-
-    dict.indexOf("c");
-    dict.indexOf("b");
-
-    auto buffer = make_shared<VectorSampleOutputStream>();
-
-    auto parser = new KeyValueSampleStreamParser(buffer, dict);
-
-    char data[] = "a=1,   b=2, c=3\n";
-    parser->process(boost::asio::buffer(data, sizeof(data)));
-    BOOST_CHECK_EQUAL(buffer->samples.size(), 1);
-    BOOST_CHECK_EQUAL(dict.size(), 3);
-    auto it = dict.begin();
-    BOOST_CHECK_EQUAL((*it)->name, "c");
-    BOOST_CHECK_EQUAL((*it++)->index, 0);
-    BOOST_CHECK_EQUAL((*it)->name, "b");
-    BOOST_CHECK_EQUAL((*it++)->index, 1);
-}