From 3c09000edc080046569e61c0595475e36688e3e8 Mon Sep 17 00:00:00 2001 From: Aleksei Nikiforov Date: Wed, 11 Dec 2019 17:16:28 +0300 Subject: [PATCH] Implement string_array type Update AbstractRecordField interface to match new use-case --- auditd-clickhouse-datatypes.json | 2 +- auditd-datatypes.cpp | 5 +- auditd-datatypes.hpp | 2 +- auditd-plugin-clickhouse.cpp | 9 +-- auditd-record.cpp | 108 +++++++++++++++++++++++++------ auditd-record.hpp | 39 ++++++++--- 6 files changed, 130 insertions(+), 35 deletions(-) diff --git a/auditd-clickhouse-datatypes.json b/auditd-clickhouse-datatypes.json index 9fe9d9d..42b94cb 100644 --- a/auditd-clickhouse-datatypes.json +++ b/auditd-clickhouse-datatypes.json @@ -229,6 +229,6 @@ "watch": "string" }, "datatypes_arrays": { - "^a\\d+$": { "type": "string", "dbname": "arg_a" } + "^a\\d+$": { "type": "string_array", "dbname": "arg_a" } } } diff --git a/auditd-datatypes.cpp b/auditd-datatypes.cpp index d4124f3..3f493c9 100644 --- a/auditd-datatypes.cpp +++ b/auditd-datatypes.cpp @@ -31,7 +31,7 @@ static std::map s_datatypes_map; static std::list > s_datatype_regexps_map; -static std::map(const std::string &name, auparse_state_t *record)> > s_type_creation_map; +static std::map(const std::string &name)> > s_type_creation_map; void read_datatypes_map(const std::string &config_filename) { @@ -41,6 +41,7 @@ void read_datatypes_map(const std::string &config_filename) s_type_creation_map["integer"] = &IntegerRecordField::createRecord; s_type_creation_map["string"] = &InterpretedStringRecordField::createRecord; + s_type_creation_map["string_array"] = &InterpretedStringArrayRecordField::createRecord; boost::property_tree::ptree clickhouse_config_tree; @@ -84,7 +85,7 @@ std::list > get_datatype_regex return s_datatype_regexps_map; } -std::map(const std::string &name, auparse_state_t *record)> > get_type_creation_map() +std::map(const std::string &name)> > get_type_creation_map() { return s_type_creation_map; } diff --git a/auditd-datatypes.hpp b/auditd-datatypes.hpp index c323294..d95e253 100644 --- a/auditd-datatypes.hpp +++ b/auditd-datatypes.hpp @@ -32,6 +32,6 @@ void read_datatypes_map(const std::string &config_filename); std::map get_datatypes_map(); std::list > get_datatype_regexps_map(); -std::map(const std::string &name, auparse_state_t *record)> > get_type_creation_map(); +std::map(const std::string &name)> > get_type_creation_map(); #endif /* AUDITD_PLUGIN_CLICKHOUSE_DATATYPES_HPP */ diff --git a/auditd-plugin-clickhouse.cpp b/auditd-plugin-clickhouse.cpp index ca3267b..ddbe037 100644 --- a/auditd-plugin-clickhouse.cpp +++ b/auditd-plugin-clickhouse.cpp @@ -134,8 +134,9 @@ void auparse_callback(auparse_state_t *au, auparse_cb_event_t cb_event_type, voi std::string construct_clickhouse_datatype_string(const std::string &name, const std::string &audit_type) { static const std::map audit_table_map = { - { "string", "String" }, - { "integer", "Nested( IntValue UInt64, InterpretedValue LowCardinality(String) )" } + { "string", "Nullable(String)" }, + { "integer", "Nested( IntValue Nullable(UInt64), InterpretedValue LowCardinality(Nullable(String)) )" }, + { "string_array", "Array(String)" } }; std::string clickhouse_type; @@ -148,10 +149,10 @@ std::string construct_clickhouse_datatype_string(const std::string &name, const { // Fallback to string fprintf(stderr, "Warning: unknown database type for record name \"%s\"\n", name.c_str()); - clickhouse_type = "String"; + clickhouse_type = "Nullable(String)"; } - return name + " Nullable(" + clickhouse_type + ")"; + return name + " " + clickhouse_type; } int main(int argc, char **argv) diff --git a/auditd-record.cpp b/auditd-record.cpp index 6b5b271..690e0f4 100644 --- a/auditd-record.cpp +++ b/auditd-record.cpp @@ -20,6 +20,7 @@ #include "auditd-record.hpp" +#include #include #include #include @@ -77,22 +78,20 @@ AbstractRecordFieldFactory& AbstractRecordFieldFactory::instance() return instance; } -std::shared_ptr AbstractRecordFieldFactory::createFromAuditRecord(const std::string &name, auparse_state_t *record) +std::shared_ptr AbstractRecordFieldFactory::createFromAuditRecord(const std::string &field_name, auparse_type_t field_type) { AbstractRecordFieldFactory &inst = instance(); - auparse_type_t record_type = static_cast(auparse_get_field_type(record)); - auto record_type_factory = inst.m_factoryMap.find(record_type); - + auto record_type_factory = inst.m_factoryMap.find(field_type); if ((record_type_factory != inst.m_factoryMap.end()) && (record_type_factory->second)) { - return record_type_factory->second(name, record); + return record_type_factory->second(field_name); } else { // fallback to string - fprintf(stderr, "Warning: unknown record type for record name \"%s\"\n", name.c_str()); - return InterpretedStringRecordField::createRecord(name, record); + fprintf(stderr, "Warning: unknown record type for record name \"%s\"\n", field_name.c_str()); + return InterpretedStringRecordField::createRecord(field_name); } } @@ -142,20 +141,28 @@ void CommonStringRecordField::addToColumn(const std::vectorAppend(std::make_shared(data_column, null_column)); } -std::shared_ptr StringRecordField::createRecord(const std::string &name, auparse_state_t *record) +std::shared_ptr StringRecordField::createRecord(const std::string &name) { std::shared_ptr result; - result.reset(new StringRecordField(name, record)); + result.reset(new StringRecordField(name)); return result; } -StringRecordField::StringRecordField(const std::string &name, auparse_state_t *record) - : CommonStringRecordField(name) +void StringRecordField::addOrUpdateValue(auparse_state_t *record) { if (record) { m_value = auparse_get_field_str(record); } + else + { + m_value = boost::none; + } +} + +StringRecordField::StringRecordField(const std::string &name) + : CommonStringRecordField(name) +{ } AbstractRecordField::Type StringRecordField::getType() const @@ -163,20 +170,28 @@ AbstractRecordField::Type StringRecordField::getType() const return AbstractRecordField::Type::String; } -std::shared_ptr InterpretedStringRecordField::createRecord(const std::string &name, auparse_state_t *record) +std::shared_ptr InterpretedStringRecordField::createRecord(const std::string &name) { std::shared_ptr result; - result.reset(new InterpretedStringRecordField(name, record)); + result.reset(new InterpretedStringRecordField(name)); return result; } -InterpretedStringRecordField::InterpretedStringRecordField(const std::string &name, auparse_state_t *record) +InterpretedStringRecordField::InterpretedStringRecordField(const std::string &name) : CommonStringRecordField(name) +{ +} + +void InterpretedStringRecordField::addOrUpdateValue(auparse_state_t *record) { if (record) { m_value = auparse_interpret_field(record); } + else + { + m_value = boost::none; + } } AbstractRecordField::Type InterpretedStringRecordField::getType() const @@ -184,20 +199,30 @@ AbstractRecordField::Type InterpretedStringRecordField::getType() const return AbstractRecordField::Type::InterpretedString; } -std::shared_ptr IntegerRecordField::createRecord(const std::string &name, auparse_state_t *record) +std::shared_ptr IntegerRecordField::createRecord(const std::string &name) { std::shared_ptr result; - result.reset(new IntegerRecordField(name, record)); + result.reset(new IntegerRecordField(name)); return result; } -IntegerRecordField::IntegerRecordField(const std::string &name, auparse_state_t *record) - : InterpretedStringRecordField(name, record) +IntegerRecordField::IntegerRecordField(const std::string &name) + : InterpretedStringRecordField(name) { +} + +void IntegerRecordField::addOrUpdateValue(auparse_state_t *record) +{ + InterpretedStringRecordField::addOrUpdateValue(record); + if (record) { m_int_value = auparse_get_field_int(record); } + else + { + m_int_value = boost::none; + } } void IntegerRecordField::addToColumn(const std::vector &columns) const @@ -246,3 +271,50 @@ AbstractRecordField::Type IntegerRecordField::getType() const { return AbstractRecordField::Type::Int; } + +std::shared_ptr InterpretedStringArrayRecordField::createRecord(const std::string &name) +{ + std::shared_ptr result; + result.reset(new InterpretedStringArrayRecordField(name)); + return result; +} + +void InterpretedStringArrayRecordField::addOrUpdateValue(auparse_state_t *record) +{ + if (record) + { + m_value.push_back(auparse_interpret_field(record)); + } +} + +void InterpretedStringArrayRecordField::addToColumn(const std::vector &columns) const +{ + if ((columns.size() != 1) || (!columns[0])) + { + throw std::runtime_error("InterpretedStringArrayRecordField::addToColumn: invalid columns argument"); + } + + auto array = columns[0]->As(); + if (!array) + { + throw std::runtime_error("Invalid column type: not ColumnArray"); + } + + auto data_column = std::make_shared(); + for (auto iter = m_value.begin(); iter != m_value.end(); ++iter) + { + data_column->Append(*iter); + } + + array->AppendAsColumn(data_column); +} + +InterpretedStringArrayRecordField::InterpretedStringArrayRecordField(const std::string &name) + : AbstractRecordField(name) +{ +} + +AbstractRecordField::Type InterpretedStringArrayRecordField::getType() const +{ + return AbstractRecordField::Type::InterpretedStringArray; +} diff --git a/auditd-record.hpp b/auditd-record.hpp index e7ab833..815ff65 100644 --- a/auditd-record.hpp +++ b/auditd-record.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -44,11 +45,13 @@ public: { Int, String, - InterpretedString + InterpretedString, + InterpretedStringArray }; virtual ~AbstractRecordField() = default; + virtual void addOrUpdateValue(auparse_state_t *record) = 0; virtual void addToColumn(const std::vector &columns) const = 0; virtual Type getType() const = 0; @@ -80,7 +83,7 @@ private: class AbstractRecordFieldFactory { public: - std::shared_ptr createFromAuditRecord(const std::string &name, auparse_state_t *record); + std::shared_ptr createFromAuditRecord(const std::string &field_name, auparse_type_t field_type); static AbstractRecordFieldFactory& instance(); @@ -100,7 +103,7 @@ protected: AbstractRecordFieldFactory(const AbstractRecordFieldFactory &other) = delete; AbstractRecordFieldFactory& operator=(const AbstractRecordFieldFactory &other) = delete; - std::map(const std::string &name, auparse_state_t *record)> > m_factoryMap; + std::map(const std::string &name)> > m_factoryMap; }; class CommonStringRecordField: public AbstractRecordField @@ -117,37 +120,55 @@ protected: class StringRecordField: public CommonStringRecordField { public: - static std::shared_ptr createRecord(const std::string &name, auparse_state_t *record); + static std::shared_ptr createRecord(const std::string &name); + virtual void addOrUpdateValue(auparse_state_t *record) override; virtual Type getType() const override; protected: - StringRecordField(const std::string &name, auparse_state_t *record); + explicit StringRecordField(const std::string &name); }; class InterpretedStringRecordField: public CommonStringRecordField { public: - static std::shared_ptr createRecord(const std::string &name, auparse_state_t *record); + static std::shared_ptr createRecord(const std::string &name); + virtual void addOrUpdateValue(auparse_state_t *record) override; virtual Type getType() const override; protected: - InterpretedStringRecordField(const std::string &name, auparse_state_t *record); + explicit InterpretedStringRecordField(const std::string &name); }; class IntegerRecordField: public InterpretedStringRecordField { public: - static std::shared_ptr createRecord(const std::string &name, auparse_state_t *record); + static std::shared_ptr createRecord(const std::string &name); + virtual void addOrUpdateValue(auparse_state_t *record) override; virtual void addToColumn(const std::vector &columns) const override; virtual Type getType() const override; protected: - IntegerRecordField(const std::string &name, auparse_state_t *record); + explicit IntegerRecordField(const std::string &name); boost::optional m_int_value; }; +class InterpretedStringArrayRecordField: public AbstractRecordField +{ +public: + static std::shared_ptr createRecord(const std::string &name); + + virtual void addOrUpdateValue(auparse_state_t *record) override; + virtual void addToColumn(const std::vector &columns) const override; + virtual Type getType() const override; + +protected: + explicit InterpretedStringArrayRecordField(const std::string &name); + + std::list m_value; +}; + #endif /* AUDITD_PLUGIN_CLICKHOUSE_RECORD_HPP */