From f110a0912efb7245d6d548aa8a5ac0c89bcd9dc0 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Mon, 23 Mar 2015 00:14:56 +0100 Subject: o Replacing regex based parsing with simpler and more correct tokenizing. o Flushing output after each sample. o Adding back tests. --- sensor/CMakeLists.txt | 2 + sensor/main/io.cpp | 61 +++++++++++++++--------------- sensor/test/CMakeLists.txt | 26 +++++++++++++ sensor/test/SampleTest.cpp | 76 ++++++++++++++++++++++++++++++++++++++ sensor/test/SoilMoistureIoTest.cpp | 48 ------------------------ 5 files changed, 134 insertions(+), 79 deletions(-) create mode 100644 sensor/test/CMakeLists.txt create mode 100644 sensor/test/SampleTest.cpp delete mode 100644 sensor/test/SoilMoistureIoTest.cpp diff --git a/sensor/CMakeLists.txt b/sensor/CMakeLists.txt index 14804f1..d571e38 100644 --- a/sensor/CMakeLists.txt +++ b/sensor/CMakeLists.txt @@ -10,3 +10,5 @@ include_directories(include) # Boost find_package(Boost COMPONENTS regex system REQUIRED) + +add_subdirectory(test) diff --git a/sensor/main/io.cpp b/sensor/main/io.cpp index 57c0b18..cc713b9 100644 --- a/sensor/main/io.cpp +++ b/sensor/main/io.cpp @@ -1,17 +1,17 @@ #include "trygvis/sensor/io.h" -#include -#include #include -#include #include "json.hpp" -#include "boost/regex.hpp" +#include "boost/tokenizer.hpp" +#include namespace trygvis { namespace sensor { namespace io { using namespace std; +using boost::tokenizer; +using boost::escaped_list_separator; using json = nlohmann::json; ThreadSafeSampleOutputStream::ThreadSafeSampleOutputStream(unique_ptr underlying) @@ -37,12 +37,12 @@ CsvSampleOutputStream::CsvSampleOutputStream(shared_ptr stream, KeyDict } void CsvSampleOutputStream::write(SampleRecord const &sample) { -// Skip empty records + // Skip empty records if (sample.empty()) { return; } -// Build the dict with the keys from the first sample. + // Build the dict with the keys from the first sample. if (dict.empty()) { SampleKeyIndex index = 0; auto ptr = sample.cbegin(); @@ -81,7 +81,7 @@ void CsvSampleOutputStream::write(SampleRecord const &sample) { } } - s << endl; + s << endl << flush; } void CsvSampleOutputStream::writeHeader() { @@ -98,7 +98,7 @@ void CsvSampleOutputStream::writeHeader() { } } - s << endl; + s << endl << flush; } JsonSampleOutputStream::JsonSampleOutputStream(shared_ptr stream, KeyDictionary &dict) : @@ -106,7 +106,7 @@ JsonSampleOutputStream::JsonSampleOutputStream(shared_ptr stream, KeyDi } void JsonSampleOutputStream::write(SampleRecord const &sample) { -// Skip empty records + // Skip empty records if (sample.empty()) { return; } @@ -128,14 +128,14 @@ void JsonSampleOutputStream::write(SampleRecord const &sample) { auto o = sample.at(sampleKey); if (o) { -// Make sure that the key is registered in the dictionary + // Make sure that the key is registered in the dictionary dict.indexOf(sampleKey->name); doc[sampleKey->name] = o.get(); } } } - *stream.get() << doc << endl; + *stream.get() << doc << endl << flush; } KeyValueSampleOutputStream::KeyValueSampleOutputStream(shared_ptr stream, KeyDictionary &dict) : @@ -143,7 +143,7 @@ KeyValueSampleOutputStream::KeyValueSampleOutputStream(shared_ptr strea } void KeyValueSampleOutputStream::write(SampleRecord const &sample) { -// Skip empty records + // Skip empty records if (sample.empty()) { return; } @@ -176,14 +176,14 @@ void KeyValueSampleOutputStream::write(SampleRecord const &sample) { } else { s << ", "; } -// Make sure that the key is registered in the dictionary + // Make sure that the key is registered in the dictionary dict.indexOf(sampleKey->name); s << sampleKey->name << "=" << o.get(); } } } - *stream.get() << endl; + s << endl << flush; } RrdSampleOutputStream::RrdSampleOutputStream(shared_ptr stream, @@ -204,7 +204,7 @@ RrdSampleOutputStream::RrdSampleOutputStream(shared_ptr stream, } void RrdSampleOutputStream::write(SampleRecord const &sample) { -// Skip empty records + // Skip empty records if (sample.empty()) { return; } @@ -239,7 +239,7 @@ void RrdSampleOutputStream::write(SampleRecord const &sample) { s << (value ? value.get() : "U"); } - *stream.get() << endl; + s << endl << flush; } SqlSampleOutputStream::SqlSampleOutputStream(shared_ptr stream, KeyDictionary &dict, string table_name) : @@ -292,7 +292,7 @@ void SqlSampleOutputStream::write(SampleRecord const &values) { // } // } // -// (*stream.get()) << "INSERT INTO " << table_name << "(" << fs << ") VALUES(" << vs << ");" << endl; +// (*stream.get()) << "INSERT INTO " << table_name << "(" << fs << ") VALUES(" << vs << ");" << endl << flush; } void KeyValueSampleStreamParser::process(mutable_buffers_1 buffer) { @@ -321,28 +321,27 @@ void KeyValueSampleStreamParser::process(mutable_buffers_1 buffer) { } void KeyValueSampleStreamParser::process_line(shared_ptr> packet) { - auto timestamp = std::chrono::system_clock::now().time_since_epoch().count(); auto s = std::string((char *) packet->data(), packet->size()); - static const boost::regex e("([#_a-zA-Z0-9]+) *= *([0-9]+)"); - - auto start = s.cbegin(); - auto end = s.cend(); - boost::match_results what; - boost::match_flag_type flags = boost::match_default; + typedef tokenizer> Tokenizer; + Tokenizer tokens(s); SampleRecord sample(dict); - while (regex_search(start, end, what, e, flags)) { - auto name = static_cast(what[1]); - auto value = static_cast(what[2]); - start = what[0].second; + for (auto token : tokens) { + auto index = token.find('='); + + if (index == string::npos) { + continue; + } + + auto name = token.substr(0, index); + boost::algorithm::trim(name); + auto value = token.substr(index + 1); + boost::algorithm::trim(value); auto key = dict.indexOf(name); sample.set(key, value); - - flags |= boost::match_prev_avail; - flags |= boost::match_not_bob; } output->write(sample); diff --git a/sensor/test/CMakeLists.txt b/sensor/test/CMakeLists.txt new file mode 100644 index 0000000..2e68532 --- /dev/null +++ b/sensor/test/CMakeLists.txt @@ -0,0 +1,26 @@ +find_package(Boost COMPONENTS log regex unit_test_framework REQUIRED) + +# If we can change directory here add_definition and test-specific stuff could be moved to the test directory +file(GLOB TEST_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *Test.cpp) +add_definitions(-DBOOST_TEST_DYN_LINK) + +foreach(testSrc ${TEST_SRCS}) + get_filename_component(testName ${testSrc} NAME_WE) + + #Add compile target + add_executable(${testName} ${testSrc}) + + include_directories(../include) + + target_link_libraries(${testName} trygvis-sensor) + target_link_libraries(${testName} ${Boost_LIBRARIES}) + +# set_target_properties(${testName} PROPERTIES +# RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test) + + #Finally add it to test execution - + #Notice the WORKING_DIRECTORY and COMMAND + add_test(NAME ${testName} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/testBin + COMMAND ${CMAKE_BINARY_DIR}/testBin/${testName}) +endforeach(testSrc) diff --git a/sensor/test/SampleTest.cpp b/sensor/test/SampleTest.cpp new file mode 100644 index 0000000..2404500 --- /dev/null +++ b/sensor/test/SampleTest.cpp @@ -0,0 +1,76 @@ +#include "trygvis/sensor.h" +#include "trygvis/sensor/io.h" + +#define BOOST_TEST_MODULE "SampleTest" + +#include + +using namespace trygvis::sensor; +using namespace trygvis::sensor::io; +using namespace std; +using namespace boost; + +BOOST_AUTO_TEST_CASE(key_value_parser) { + KeyDictionary dict; + + auto buffer = make_shared(); + + auto parser = make_shared(buffer, dict); + + char data[] = "a=1, b=2, c=3\n"; + parser->process(boost::asio::buffer(data, sizeof(data))); + BOOST_CHECK_EQUAL(buffer->samples.size(), 1); + BOOST_CHECK_EQUAL(dict.size(), 3); + auto it = dict.begin(); + BOOST_CHECK_EQUAL((*it)->name, "a"); + BOOST_CHECK_EQUAL((*it++)->index, 0); + BOOST_CHECK_EQUAL((*it)->name, "b"); + BOOST_CHECK_EQUAL((*it++)->index, 1); + BOOST_CHECK_EQUAL((*it)->name, "c"); + BOOST_CHECK_EQUAL((*it++)->index, 2); +} + +BOOST_AUTO_TEST_CASE(key_value_parser2) { + KeyDictionary dict; + + auto buffer = make_shared(); + + auto parser = make_shared(buffer, dict); + + char data[] = "now=1,sensor 1=0.000999999833333\n"; + parser->process(boost::asio::buffer(data, sizeof(data))); + BOOST_CHECK_EQUAL(buffer->samples.size(), 1); + SampleRecord& sample = buffer->samples[0]; + BOOST_CHECK_EQUAL(dict.size(), 2); + auto it = dict.begin(); + BOOST_CHECK_EQUAL((*it)->name, "now"); + BOOST_CHECK_EQUAL(!sample.at(*it).operator!(), true); + BOOST_CHECK_EQUAL(sample.at(*it).get(), "1"); + BOOST_CHECK_EQUAL((*it++)->index, 0); + + BOOST_CHECK_EQUAL((*it)->name, "sensor 1"); + BOOST_CHECK_EQUAL(!sample.at(*it).operator!(), true); + BOOST_CHECK_EQUAL(sample.at(*it).get(), "0.000999999833333"); + BOOST_CHECK_EQUAL((*it++)->index, 1); +} + +BOOST_AUTO_TEST_CASE(key_value_parser_with_custom_dict) { + KeyDictionary dict; + + dict.indexOf("c"); + dict.indexOf("b"); + + auto buffer = make_shared(); + + auto parser = make_shared(buffer, dict); + + char data[] = "a=1, b=2, c=3\n"; + parser->process(boost::asio::buffer(data, sizeof(data))); + BOOST_CHECK_EQUAL(buffer->samples.size(), 1); + BOOST_CHECK_EQUAL(dict.size(), 3); + auto it = dict.begin(); + BOOST_CHECK_EQUAL((*it)->name, "c"); + BOOST_CHECK_EQUAL((*it++)->index, 0); + BOOST_CHECK_EQUAL((*it)->name, "b"); + BOOST_CHECK_EQUAL((*it++)->index, 1); +} diff --git a/sensor/test/SoilMoistureIoTest.cpp b/sensor/test/SoilMoistureIoTest.cpp deleted file mode 100644 index 574885c..0000000 --- a/sensor/test/SoilMoistureIoTest.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include "SensorSample.h" - -#define BOOST_TEST_MODULE "SoilMoistureIoTest" - -#include - -using namespace trygvis::soil_moisture; - -BOOST_AUTO_TEST_CASE(key_value_parser) { - KeyDictionary dict; - - auto buffer = make_shared(); - - auto parser = new KeyValueSampleStreamParser(buffer, dict); - - char data[] = "a=1, b=2, c=3\n"; - parser->process(boost::asio::buffer(data, sizeof(data))); - BOOST_CHECK_EQUAL(buffer->samples.size(), 1); - BOOST_CHECK_EQUAL(dict.size(), 3); - auto it = dict.begin(); - BOOST_CHECK_EQUAL((*it)->name, "a"); - BOOST_CHECK_EQUAL((*it++)->index, 0); - BOOST_CHECK_EQUAL((*it)->name, "b"); - BOOST_CHECK_EQUAL((*it++)->index, 1); - BOOST_CHECK_EQUAL((*it)->name, "c"); - BOOST_CHECK_EQUAL((*it++)->index, 2); -} - -BOOST_AUTO_TEST_CASE(key_value_parser_with_custom_dict) { - KeyDictionary dict; - - dict.indexOf("c"); - dict.indexOf("b"); - - auto buffer = make_shared(); - - auto parser = new KeyValueSampleStreamParser(buffer, dict); - - char data[] = "a=1, b=2, c=3\n"; - parser->process(boost::asio::buffer(data, sizeof(data))); - BOOST_CHECK_EQUAL(buffer->samples.size(), 1); - BOOST_CHECK_EQUAL(dict.size(), 3); - auto it = dict.begin(); - BOOST_CHECK_EQUAL((*it)->name, "c"); - BOOST_CHECK_EQUAL((*it++)->index, 0); - BOOST_CHECK_EQUAL((*it)->name, "b"); - BOOST_CHECK_EQUAL((*it++)->index, 1); -} -- cgit v1.2.3