From 5d6e4c971857b5e05d7c277e7eafcb9d64dbc6a7 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Tue, 17 Mar 2015 21:29:40 +0100 Subject: o An improved KeyDictionary. --- apps/SoilMoistureIo.cpp | 157 ++++++++++++++++++-------------------------- apps/SoilMoistureIo.h | 141 ++++++++++++++++++++++++--------------- apps/sample-convert.cpp | 8 +-- apps/sample-timestamp.cpp | 20 +++--- apps/sm-serial-read.cpp | 8 +-- test/CMakeLists.txt | 5 +- test/SoilMoistureIoTest.cpp | 48 ++++++++++++++ 7 files changed, 222 insertions(+), 165 deletions(-) create mode 100644 test/SoilMoistureIoTest.cpp diff --git a/apps/SoilMoistureIo.cpp b/apps/SoilMoistureIo.cpp index b8a8b64..ad8a3bb 100644 --- a/apps/SoilMoistureIo.cpp +++ b/apps/SoilMoistureIo.cpp @@ -15,40 +15,26 @@ void VectorSampleOutputStream::write(SampleRecord sample) { samples.emplace_back(sample); } -vector KeyDictionary::findIndexes(vector keys) { - vector indexes; - - for (auto &key: keys) { - auto index = indexOf(key); - indexes.push_back(index); - } - - return move(indexes); -} - -CsvSampleOutputStream::CsvSampleOutputStream(KeyDictionary &dict, unique_ptr stream) : - dict(dict), stream(move(stream)), headerWritten(false) { +CsvSampleOutputStream::CsvSampleOutputStream(unique_ptr stream, KeyDictionary &dict) + : stream(move(stream)), headerWritten(false), dict(dict) { } -CsvSampleOutputStream::CsvSampleOutputStream(KeyDictionary &dict, unique_ptr stream, vector fieldKeys) - : - dict(dict), stream(move(stream)), headerWritten(false), fields(dict.findIndexes(fieldKeys)) { -} - -void CsvSampleOutputStream::write(SampleRecord values) { +void CsvSampleOutputStream::write(SampleRecord sample) { // Skip empty records - if (values.empty()) { + if (sample.empty()) { return; } - if (fields.empty()) { - KeyDictionary::index_t index = 0; - auto ptr = values.begin(); - while (ptr != values.end()) { + // Build the dict with the keys from the first sample. + if (dict.empty()) { + SampleKeyIndex index = 0; + auto ptr = sample.begin(); + while (ptr != sample.end()) { auto o = *ptr; if (o) { - fields.push_back(index); + auto name = sample.dict.at(index)->name; + dict.indexOf(name); } ptr++; @@ -63,14 +49,15 @@ void CsvSampleOutputStream::write(SampleRecord values) { auto &s = *stream.get(); - auto i = fields.begin(); - while (i != fields.end()) { - if (i != fields.begin()) { + auto it = dict.begin(); + while (it != dict.end()) { + if (it != dict.begin()) { s << ","; } - auto index = *i++; - auto o = values.at(index); + auto key = *it++; + auto sampleKey = sample.dict.indexOf(key->name); + auto o = sample.at(sampleKey); if (o) { s << o.get(); @@ -83,13 +70,13 @@ void CsvSampleOutputStream::write(SampleRecord values) { void CsvSampleOutputStream::writeHeader() { auto &s = *stream.get(); - auto i = fields.begin(); - while (i != fields.end()) { - s << dict.nameOf(*i); + auto i = dict.begin(); + while (i != dict.end()) { + s << (*i)->name; i++; - if (i != fields.end()) { + if (i != dict.end()) { s << ","; } } @@ -97,48 +84,45 @@ void CsvSampleOutputStream::writeHeader() { s << endl; } -JsonSampleOutputStream::JsonSampleOutputStream(KeyDictionary &dict, unique_ptr stream) : - dict(dict), stream(move(stream)), filterFields(false) { +JsonSampleOutputStream::JsonSampleOutputStream(unique_ptr stream, KeyDictionary &dict) : + dict(dict), stream(move(stream)) { } -JsonSampleOutputStream::JsonSampleOutputStream(KeyDictionary &dict, unique_ptr stream, vector fields) - : - dict(dict), stream(move(stream)), fields(dict.findIndexes(fields)), filterFields(true) { -} - -void JsonSampleOutputStream::write(SampleRecord values) { - throw sample_exception("deimplemented"); +void JsonSampleOutputStream::write(SampleRecord sample) { + // Skip empty records + if (sample.empty()) { + return; + } json doc({}); -// if (filterFields) { -// for (auto &f: fields) { -// auto value = values.find(f); -// -// if (value != values.end()) { -// doc[f] = value->second; -// } -// } -// } else { -// for (auto &v: values) { -// doc[v.first] = v.second; -// } -// } + if (!dict.empty()) { + for (auto &key: dict) { + auto sampleKey = sample.dict.indexOf(key->name); - *stream.get() << doc << endl; -} + auto value = sample.at(sampleKey); + + if (value) { + doc[key->name] = value.get(); + } + } + } else { + for (auto &sampleKey: sample.dict) { + auto o = sample.at(sampleKey); + + if (o) { + // Make sure that the key is registered in the dictionary + dict.indexOf(sampleKey->name); + doc[sampleKey->name] = o.get(); + } + } + } -SqlSampleOutputStream::SqlSampleOutputStream(KeyDictionary &dict, unique_ptr stream, string table_name) : - dict(dict), stream(move(stream)), table_name(table_name), filter_fields(false) { + *stream.get() << doc << endl; } -SqlSampleOutputStream::SqlSampleOutputStream(KeyDictionary &dict, unique_ptr stream, string table_name, vector fields) - : - dict(dict), - stream(move(stream)), - table_name(table_name), - fields(dict.findIndexes(fields)), - filter_fields(true) { +SqlSampleOutputStream::SqlSampleOutputStream(unique_ptr stream, KeyDictionary &dict, string table_name) : + dict(dict), stream(move(stream)), table_name(table_name) { } void SqlSampleOutputStream::write(SampleRecord values) { @@ -229,15 +213,15 @@ void CsvSampleParser::process_line(shared_ptr> packet) { SampleRecord sample(dict); while (regex_search(start, end, what, e, flags)) { - auto key = static_cast(what[1]); + auto name = static_cast(what[1]); auto value = static_cast(what[2]); start = what[0].second; map values; - values[key] = value; + values[name] = value; - auto index = dict.indexOf(key); - sample.set(index, value); + auto key = dict.indexOf(name); + sample.set(key, value); flags |= boost::match_prev_avail; flags |= boost::match_not_bob; @@ -246,8 +230,8 @@ void CsvSampleParser::process_line(shared_ptr> packet) { output->write(sample); } -AutoSampleParser::AutoSampleParser(KeyDictionary &dict, shared_ptr output) : - SampleStreamParser(sample_format_type::AUTO), csvParser(new CsvSampleParser(dict, output)) { +AutoSampleParser::AutoSampleParser(shared_ptr output, KeyDictionary &dict) : + SampleStreamParser(sample_format_type::AUTO), csvParser(new CsvSampleParser(output, dict)) { // Directly select the parser now until we have more than one parser parser = std::move(csvParser); type_ = sample_format_type::CSV; @@ -279,38 +263,25 @@ unique_ptr open_sample_input_stream( shared_ptr output, sample_format_type type) { if (type == sample_format_type::CSV) { - return make_unique(dict, output); + return make_unique(output, dict); } else if (type == sample_format_type::AUTO) { - return make_unique(dict, output); + return make_unique(output, dict); } else { throw sample_exception("Unsupported format type: " + to_string(type)); } } unique_ptr open_sample_output_stream( - KeyDictionary &dict, - sample_format_type type, unique_ptr output, - o> fields) { + KeyDictionary &dict, + sample_format_type type) { if (type == sample_format_type::CSV) { - if (fields) { - return make_unique(dict, move(output), fields.get()); - } else { - return make_unique(dict, move(output)); - } + return make_unique(move(output), dict); } else if (type == sample_format_type::JSON) { - if (fields) { - return make_unique(dict, move(output), fields.get()); - } else { - return make_unique(dict, move(output)); - } + return make_unique(move(output), dict); // } else if (type == sample_format_type::SQL) { -// if (fields) { -// return make_unique(dict, move(output), table_name, fields.get()); -// } else { -// return make_unique(dict, move(output), table_name); -// } +// return make_unique(dict, move(output), table_name); } else { throw sample_exception("Unsupported format type: " + to_string(type)); } diff --git a/apps/SoilMoistureIo.h b/apps/SoilMoistureIo.h index b8f0b52..473c098 100644 --- a/apps/SoilMoistureIo.h +++ b/apps/SoilMoistureIo.h @@ -11,6 +11,7 @@ #include #include +// TODO: rename to trygvis::sample namespace trygvis { namespace soil_moisture { @@ -43,10 +44,9 @@ unique_ptr open_sample_input_stream( sample_format_type type = sample_format_type::AUTO); unique_ptr open_sample_output_stream( - KeyDictionary &dict, - sample_format_type type, unique_ptr output, - o> fields = o>()); + KeyDictionary &dict, + sample_format_type type); class sample_exception : public runtime_error { public: @@ -54,61 +54,102 @@ public: } }; +class KeyDictionary; + +using SampleKeyVector = vector; +using SampleKeyIndex = SampleKeyVector::size_type; + struct SampleKey { - // TODO: only the dictionary should be able to create keys - SampleKey(string &name) : name(name) { +private: + SampleKey(const SampleKey& that) = delete; + SampleKey(SampleKeyIndex index, const string &name) : index(index), name(name) { if (name.length() == 0) { throw sample_exception("Bad sample key."); } } +public: + friend class KeyDictionary; + inline bool operator==(const SampleKey &that) const { return name == that.name; } - string name; + const SampleKeyIndex index; + const string name; }; class KeyDictionary { public: - typedef vector v; - typedef v::size_type index_t; - KeyDictionary() { } - index_t indexOf(const SampleKey key) { - index_t i = 0; - for (auto ptr = keys.begin(); ptr != keys.end(); ptr++, i++) { - if (*ptr == key) { - return i; + ~KeyDictionary() { + std::for_each(keys.begin(), keys.end(), std::default_delete()); + } + KeyDictionary(KeyDictionary& that) = delete; + + SampleKey *indexOf(const string key) { + SampleKeyIndex i = 0; + for (auto ptr = keys.cbegin(); ptr != keys.cend(); ptr++, i++) { + if ((*ptr)->name == key) { + return *ptr; } } - keys.push_back(key); + i = keys.size(); + auto sample_key = new SampleKey(i, key); + keys.push_back(sample_key); + + return sample_key; + } + + SampleKey *at(SampleKeyIndex i) const { + if (i >= keys.size()) { + throw sample_exception("Out of bounds"); + } + + return keys.at(i); + } + + vector findIndexes(SampleKeyVector &keys) { + vector indexes; + + for (auto &key: keys) { + auto index = indexOf(key->name); + indexes.push_back(index); + } - return keys.size() - 1; + return move(indexes); } - vector findIndexes(v keys); + inline + SampleKeyVector::const_iterator end() const { + return keys.cend(); + } inline - v::const_iterator begin() { - return keys.begin(); + SampleKeyVector::const_iterator begin() const { + return keys.cbegin(); } +// string nameOf(SampleKeyIndex index) { +// return keys.at(index).name; +// } + inline - v::const_iterator end() { - return keys.end(); + SampleKeyVector::size_type size() const { + return keys.size(); } - string nameOf(index_t index) { - return keys.at(index).name; + inline + bool empty() const { + return keys.empty(); } private: - v keys; + SampleKeyVector keys; }; class SampleRecord { @@ -137,7 +178,8 @@ public: return values.empty(); } - o at(size_t index) { + o at(const SampleKey *key) const { + SampleKeyIndex index = key->index; if (index >= values.size()) { return o(); } @@ -145,15 +187,15 @@ public: return values.at(index); } - void set(const KeyDictionary::index_t index, const std::string &value) { - values.resize(max(values.size(), index + 1)); + void set(const SampleKey *key, const std::string &value) { + values.resize(max(values.size(), key->index + 1)); - values[index] = o(value); + values.assign(key->index, o(value)); } template - const o lexical_at(KeyDictionary::index_t index) { - auto value = at(index); + const o lexical_at(const SampleKey *key) { + auto value = at(key); if (!value) { return o(); @@ -163,7 +205,7 @@ public: } string to_string() { - KeyDictionary::index_t i = 0; + SampleKeyIndex i = 0; string s; for (auto ptr = values.begin(); ptr != values.end(); ptr++, i++) { auto o = *ptr; @@ -174,13 +216,13 @@ public: auto value = o.get(); - s += dict.nameOf(i) + " = " + value + ", "; + s += dict.at(i)->name + " = " + value + ", "; } return s; } -private: KeyDictionary &dict; +private: vec values; }; @@ -200,49 +242,42 @@ public: class CsvSampleOutputStream : public SampleOutputStream { public: - CsvSampleOutputStream(KeyDictionary &dict, unique_ptr stream); - - CsvSampleOutputStream(KeyDictionary &dict, unique_ptr stream, vector fields); + CsvSampleOutputStream(unique_ptr stream, KeyDictionary &dict); - void write(SampleRecord values); + void write(SampleRecord sample); + const KeyDictionary &getDict() { + return dict; + } + private: void writeHeader(); KeyDictionary &dict; unique_ptr stream; bool headerWritten; - vector fields; }; class JsonSampleOutputStream : public SampleOutputStream { public: - JsonSampleOutputStream(KeyDictionary &dict, unique_ptr stream); - - JsonSampleOutputStream(KeyDictionary &dict, unique_ptr stream, vector fields); + JsonSampleOutputStream(unique_ptr stream, KeyDictionary &dict); - void write(SampleRecord values); + void write(SampleRecord sample) override; private: KeyDictionary &dict; unique_ptr stream; - bool filterFields; - vector fields; }; class SqlSampleOutputStream : public SampleOutputStream { public: - SqlSampleOutputStream(KeyDictionary &dict, unique_ptr stream, string table_name); - - SqlSampleOutputStream(KeyDictionary &dict, unique_ptr stream, string table_name, vector fields); + SqlSampleOutputStream(unique_ptr stream, KeyDictionary &dict, string table_name); - void write(SampleRecord values); + void write(SampleRecord sample) override; private: KeyDictionary &dict; unique_ptr stream; - bool filter_fields; - vector fields; const string table_name; }; @@ -264,8 +299,8 @@ protected: class CsvSampleParser : public SampleStreamParser { public: - CsvSampleParser(KeyDictionary &dict, shared_ptr output) : - SampleStreamParser(sample_format_type::CSV), dict(dict), output(output), + CsvSampleParser(shared_ptr output, KeyDictionary &dict) : + SampleStreamParser(sample_format_type::CSV), output(output), dict(dict), line(make_shared>()) { } @@ -282,7 +317,7 @@ private: class AutoSampleParser : public SampleStreamParser { public: - AutoSampleParser(KeyDictionary &dict, shared_ptr output); + AutoSampleParser(shared_ptr output, KeyDictionary &dict); private: unique_ptr parser; diff --git a/apps/sample-convert.cpp b/apps/sample-convert.cpp index b3e5c02..5e87a15 100644 --- a/apps/sample-convert.cpp +++ b/apps/sample-convert.cpp @@ -61,22 +61,22 @@ public: } if (output_format == "plain") { - output = make_shared(dict, move(outputStream)); + output = make_shared(move(outputStream), dict); } else if (output_format == "json") { - output = make_shared(dict, move(outputStream)); + output = make_shared(move(outputStream), dict); } else if (output_format == "sql") { if (table_name.size() == 0) { cerr << "Missing option: table-name" << endl; return EXIT_FAILURE; } - output = make_shared(dict, move(outputStream), table_name); + output = make_shared(move(outputStream), dict, table_name); } else { cerr << "Unsupported output format: " << output_format << endl; return EXIT_FAILURE; } - auto input = make_shared(dict, output); + auto input = make_shared(output, dict); char data[100]; while (!inputStream->eof()) { diff --git a/apps/sample-timestamp.cpp b/apps/sample-timestamp.cpp index 6ac2f86..dd9ab3c 100644 --- a/apps/sample-timestamp.cpp +++ b/apps/sample-timestamp.cpp @@ -14,12 +14,12 @@ namespace po = boost::program_options; class TimestampFixingSampleOutputStream : public SampleOutputStream { public: - TimestampFixingSampleOutputStream(KeyDictionary dict, string timestamp_name, string now_name, time_t start_time, shared_ptr output) : - timestamp_index(dict.indexOf(timestamp_name)), now_index(dict.indexOf(now_name)), start_time_(start_time), output_(output) { + TimestampFixingSampleOutputStream(shared_ptr output, KeyDictionary &dict, string timestamp_name, string now_name, time_t start_time) : + timestamp_key(dict.indexOf(timestamp_name)), now_key(dict.indexOf(now_name)), start_time_(start_time), output_(output) { } virtual void write(SampleRecord sample) override { - o relative_time_o = sample.lexical_at(now_index); + o relative_time_o = sample.lexical_at(now_key); if (!relative_time_o) { return; @@ -28,13 +28,13 @@ public: long relative_time = relative_time_o.get(); string new_value = std::to_string(start_time_ + relative_time); - sample.set(timestamp_index, new_value); + sample.set(timestamp_key, new_value); output_->write(sample); }; private: - KeyDictionary::index_t now_index, timestamp_index; + const SampleKey* now_key, *timestamp_key; time_t start_time_; shared_ptr output_; }; @@ -43,7 +43,7 @@ class sample_timestamp : public app { private: string input_file, timestamp_name, now_name; - KeyDictionary::index_t now_index; + SampleKey* now_key; public: sample_timestamp() : input_file("") { @@ -78,7 +78,7 @@ public: KeyDictionary dict; - now_index = dict.indexOf(now_name); + now_key = dict.indexOf(now_name); auto sample_buffer = make_shared(); unique_ptr parser = open_sample_input_stream(dict, sample_buffer); @@ -101,7 +101,7 @@ public: SampleRecord sample = *--sample_buffer->samples.end(); - o s = sample.at(now_index); + o s = sample.at(now_key); if (!s) { cerr << "Missing key '" + now_name + "'." << endl; cerr << "keys: " << sample.to_string() << endl; @@ -129,8 +129,8 @@ public: return EXIT_FAILURE; } - auto output_stream = open_sample_output_stream(dict, parser->type(), unique_ptr(&cout)); - auto p = make_shared(dict, "timestamp", now_name, start_time, move(output_stream)); + auto output_stream = open_sample_output_stream(unique_ptr(&cout), dict, parser->type()); + auto p = make_shared(move(output_stream), dict, "timestamp", now_name, start_time); parser = open_sample_input_stream(dict, p, parser->type()); int recordCount = 0; diff --git a/apps/sm-serial-read.cpp b/apps/sm-serial-read.cpp index c7fb695..c52e7f9 100644 --- a/apps/sm-serial-read.cpp +++ b/apps/sm-serial-read.cpp @@ -126,17 +126,17 @@ public: unique_ptr outputStream = unique_ptr(&cout); if (format == Format::JSON) { - output = make_shared(dict, std::move(outputStream)); + output = make_shared(std::move(outputStream), dict); } else if (format == Format::SQL) { - output = make_shared(dict, std::move(outputStream), "raw"); + output = make_shared(std::move(outputStream), dict, "raw"); } else if (format == Format::PLAIN) { - output = make_shared(dict, std::move(outputStream)); + output = make_shared(std::move(outputStream), dict); } else { cerr << "Unsupported format: " << boost::lexical_cast(format) << endl; return EXIT_FAILURE; } - shared_ptr input = make_shared(dict, output); + shared_ptr input = make_shared(output, dict); port_handler(port_name, port, input).run(); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e9f217f..e8dc219 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,4 +1,4 @@ -find_package(Boost COMPONENTS log unit_test_framework REQUIRED) +find_package(Boost COMPONENTS log regex unit_test_framework REQUIRED) # If we can change directory here add_definition and test-specific stuff could be moved to the test directory file(GLOB TEST_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *Test.cpp) @@ -18,6 +18,9 @@ foreach(testSrc ${TEST_SRCS}) target_link_libraries(${testName} pthread) target_link_libraries(${testName} ${Boost_LIBRARIES}) + include_directories("${PROJECT_SOURCE_DIR}/apps") + target_link_libraries(${testName} trygvis-apps) + #I like to move testing binaries into a testBin directory set_target_properties(${testName} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test) diff --git a/test/SoilMoistureIoTest.cpp b/test/SoilMoistureIoTest.cpp new file mode 100644 index 0000000..4a508b9 --- /dev/null +++ b/test/SoilMoistureIoTest.cpp @@ -0,0 +1,48 @@ +#include "SoilMoistureIo.h" + +#define BOOST_TEST_MODULE "SoilMoistureIoTest" + +#include + +using namespace trygvis::soil_moisture; + +BOOST_AUTO_TEST_CASE(csv_parser) { + KeyDictionary dict; + + auto buffer = make_shared(); + + auto parser = new CsvSampleParser(buffer, dict); + + char data[] = "a=1, b=2, c=3\n"; + parser->process(boost::asio::buffer(data, sizeof(data))); + BOOST_CHECK_EQUAL(buffer->samples.size(), 1); + BOOST_CHECK_EQUAL(dict.size(), 3); + auto it = dict.begin(); + BOOST_CHECK_EQUAL((*it)->name, "a"); + BOOST_CHECK_EQUAL((*it++)->index, 0); + BOOST_CHECK_EQUAL((*it)->name, "b"); + BOOST_CHECK_EQUAL((*it++)->index, 1); + BOOST_CHECK_EQUAL((*it)->name, "c"); + BOOST_CHECK_EQUAL((*it++)->index, 2); +} + +BOOST_AUTO_TEST_CASE(csv_parser_with_custom_dict) { + KeyDictionary dict; + + dict.indexOf("c"); + dict.indexOf("b"); + + auto buffer = make_shared(); + + auto parser = new CsvSampleParser(buffer, dict); + + char data[] = "a=1, b=2, c=3\n"; + parser->process(boost::asio::buffer(data, sizeof(data))); + BOOST_CHECK_EQUAL(buffer->samples.size(), 1); + BOOST_CHECK_EQUAL(dict.size(), 3); + auto it = dict.begin(); + BOOST_CHECK_EQUAL((*it)->name, "c"); + BOOST_CHECK_EQUAL((*it++)->index, 0); + BOOST_CHECK_EQUAL((*it)->name, "b"); + BOOST_CHECK_EQUAL((*it++)->index, 1); +} -- cgit v1.2.3