darktemplar/auditd-plugin-clickhouse
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implement string_array type

Browse Source 
Update AbstractRecordField interface to match new use-case
This commit is contained in:
Aleksei Nikiforov 2019-12-11 17:16:28 +03:00
parent 3ba3561664
commit 3c09000edc
6 changed files with 130 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
auditd-clickhouse-datatypes.json
View File

@ -229,6 +229,6 @@
"watch": "string"
},
"datatypes_arrays": {
"^a\\d+$": { "type": "string", "dbname": "arg_a" }
"^a\\d+$": { "type": "string_array", "dbname": "arg_a" }
}
}

5
auditd-datatypes.cpp
View File

@ -31,7 +31,7 @@
static std::map<std::string, std::string> s_datatypes_map;
static std::list<std::tuple<std::string, std::string, std::string> > s_datatype_regexps_map;
static std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name, auparse_state_t *record)> > s_type_creation_map;
static std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > s_type_creation_map;
void read_datatypes_map(const std::string &config_filename)
{
@ -41,6 +41,7 @@ void read_datatypes_map(const std::string &config_filename)
s_type_creation_map["integer"] = &IntegerRecordField::createRecord;
s_type_creation_map["string"] = &InterpretedStringRecordField::createRecord;
s_type_creation_map["string_array"] = &InterpretedStringArrayRecordField::createRecord;
boost::property_tree::ptree clickhouse_config_tree;
@ -84,7 +85,7 @@ std::list<std::tuple<std::string, std::string, std::string> > get_datatype_regex
return s_datatype_regexps_map;
}
std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name, auparse_state_t *record)> > get_type_creation_map()
std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > get_type_creation_map()
{
return s_type_creation_map;
}

2
auditd-datatypes.hpp
View File

@ -32,6 +32,6 @@
void read_datatypes_map(const std::string &config_filename);
std::map<std::string, std::string> get_datatypes_map();
std::list<std::tuple<std::string, std::string, std::string> > get_datatype_regexps_map();
std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name, auparse_state_t *record)> > get_type_creation_map();
std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > get_type_creation_map();
#endif /* AUDITD_PLUGIN_CLICKHOUSE_DATATYPES_HPP */

9
auditd-plugin-clickhouse.cpp
View File

@ -134,8 +134,9 @@ void auparse_callback(auparse_state_t *au, auparse_cb_event_t cb_event_type, voi
std::string construct_clickhouse_datatype_string(const std::string &name, const std::string &audit_type)
{
static const std::map<std::string, std::string> audit_table_map = {
{ "string", "String" },
{ "integer", "Nested( IntValue UInt64, InterpretedValue LowCardinality(String) )" }
{ "string", "Nullable(String)" },
{ "integer", "Nested( IntValue Nullable(UInt64), InterpretedValue LowCardinality(Nullable(String)) )" },
{ "string_array", "Array(String)" }
};
std::string clickhouse_type;
@ -148,10 +149,10 @@ std::string construct_clickhouse_datatype_string(const std::string &name, const
{
// Fallback to string
fprintf(stderr, "Warning: unknown database type for record name \"%s\"\n", name.c_str());
clickhouse_type = "String";
clickhouse_type = "Nullable(String)";
}
return name + " Nullable(" + clickhouse_type + ")";
return name + " " + clickhouse_type;
}
int main(int argc, char **argv)

108
auditd-record.cpp
View File

@ -20,6 +20,7 @@
#include "auditd-record.hpp"
#include <clickhouse-cpp/columns/array.h>
#include <clickhouse-cpp/columns/nullable.h>
#include <clickhouse-cpp/columns/numeric.h>
#include <clickhouse-cpp/columns/string.h>
@ -77,22 +78,20 @@ AbstractRecordFieldFactory& AbstractRecordFieldFactory::instance()
return instance;
}
std::shared_ptr<AbstractRecordField> AbstractRecordFieldFactory::createFromAuditRecord(const std::string &name, auparse_state_t *record)
std::shared_ptr<AbstractRecordField> AbstractRecordFieldFactory::createFromAuditRecord(const std::string &field_name, auparse_type_t field_type)
{
AbstractRecordFieldFactory &inst = instance();
auparse_type_t record_type = static_cast<auparse_type_t>(auparse_get_field_type(record));
auto record_type_factory = inst.m_factoryMap.find(record_type);
auto record_type_factory = inst.m_factoryMap.find(field_type);
if ((record_type_factory != inst.m_factoryMap.end()) && (record_type_factory->second))
{
return record_type_factory->second(name, record);
return record_type_factory->second(field_name);
}
else
{
// fallback to string
fprintf(stderr, "Warning: unknown record type for record name \"%s\"\n", name.c_str());
return InterpretedStringRecordField::createRecord(name, record);
fprintf(stderr, "Warning: unknown record type for record name \"%s\"\n", field_name.c_str());
return InterpretedStringRecordField::createRecord(field_name);
}
}
@ -142,20 +141,28 @@ void CommonStringRecordField::addToColumn(const std::vector<clickhouse::ColumnRe
columns[0]->Append(std::make_shared<clickhouse::ColumnNullable>(data_column, null_column));
}
std::shared_ptr<AbstractRecordField> StringRecordField::createRecord(const std::string &name, auparse_state_t *record)
std::shared_ptr<AbstractRecordField> StringRecordField::createRecord(const std::string &name)
{
std::shared_ptr<AbstractRecordField> result;
result.reset(new StringRecordField(name, record));
result.reset(new StringRecordField(name));
return result;
}
StringRecordField::StringRecordField(const std::string &name, auparse_state_t *record)
: CommonStringRecordField(name)
void StringRecordField::addOrUpdateValue(auparse_state_t *record)
{
if (record)
{
m_value = auparse_get_field_str(record);
}
else
{
m_value = boost::none;
}
}
StringRecordField::StringRecordField(const std::string &name)
: CommonStringRecordField(name)
{
}
AbstractRecordField::Type StringRecordField::getType() const
@ -163,20 +170,28 @@ AbstractRecordField::Type StringRecordField::getType() const
return AbstractRecordField::Type::String;
}
std::shared_ptr<AbstractRecordField> InterpretedStringRecordField::createRecord(const std::string &name, auparse_state_t *record)
std::shared_ptr<AbstractRecordField> InterpretedStringRecordField::createRecord(const std::string &name)
{
std::shared_ptr<AbstractRecordField> result;
result.reset(new InterpretedStringRecordField(name, record));
result.reset(new InterpretedStringRecordField(name));
return result;
}
InterpretedStringRecordField::InterpretedStringRecordField(const std::string &name, auparse_state_t *record)
InterpretedStringRecordField::InterpretedStringRecordField(const std::string &name)
: CommonStringRecordField(name)
{
}
void InterpretedStringRecordField::addOrUpdateValue(auparse_state_t *record)
{
if (record)
{
m_value = auparse_interpret_field(record);
}
else
{
m_value = boost::none;
}
}
AbstractRecordField::Type InterpretedStringRecordField::getType() const
@ -184,20 +199,30 @@ AbstractRecordField::Type InterpretedStringRecordField::getType() const
return AbstractRecordField::Type::InterpretedString;
}
std::shared_ptr<AbstractRecordField> IntegerRecordField::createRecord(const std::string &name, auparse_state_t *record)
std::shared_ptr<AbstractRecordField> IntegerRecordField::createRecord(const std::string &name)
{
std::shared_ptr<AbstractRecordField> result;
result.reset(new IntegerRecordField(name, record));
result.reset(new IntegerRecordField(name));
return result;
}
IntegerRecordField::IntegerRecordField(const std::string &name, auparse_state_t *record)
: InterpretedStringRecordField(name, record)
IntegerRecordField::IntegerRecordField(const std::string &name)
: InterpretedStringRecordField(name)
{
}
void IntegerRecordField::addOrUpdateValue(auparse_state_t *record)
{
InterpretedStringRecordField::addOrUpdateValue(record);
if (record)
{
m_int_value = auparse_get_field_int(record);
}
else
{
m_int_value = boost::none;
}
}
void IntegerRecordField::addToColumn(const std::vector<clickhouse::ColumnRef> &columns) const
@ -246,3 +271,50 @@ AbstractRecordField::Type IntegerRecordField::getType() const
{
return AbstractRecordField::Type::Int;
}
std::shared_ptr<AbstractRecordField> InterpretedStringArrayRecordField::createRecord(const std::string &name)
{
std::shared_ptr<AbstractRecordField> result;
result.reset(new InterpretedStringArrayRecordField(name));
return result;
}
void InterpretedStringArrayRecordField::addOrUpdateValue(auparse_state_t *record)
{
if (record)
{
m_value.push_back(auparse_interpret_field(record));
}
}
void InterpretedStringArrayRecordField::addToColumn(const std::vector<clickhouse::ColumnRef> &columns) const
{
if ((columns.size() != 1) || (!columns[0]))
{
throw std::runtime_error("InterpretedStringArrayRecordField::addToColumn: invalid columns argument");
}
auto array = columns[0]->As<clickhouse::ColumnArray>();
if (!array)
{
throw std::runtime_error("Invalid column type: not ColumnArray");
}
auto data_column = std::make_shared<clickhouse::ColumnString>();
for (auto iter = m_value.begin(); iter != m_value.end(); ++iter)
{
data_column->Append(*iter);
}
array->AppendAsColumn(data_column);
}
InterpretedStringArrayRecordField::InterpretedStringArrayRecordField(const std::string &name)
: AbstractRecordField(name)
{
}
AbstractRecordField::Type InterpretedStringArrayRecordField::getType() const
{
return AbstractRecordField::Type::InterpretedStringArray;
}

39
auditd-record.hpp
View File

@ -25,6 +25,7 @@
#include <map>
#include <functional>
#include <string>
#include <list>
#include <stdint.h>
@ -44,11 +45,13 @@ public:
{
Int,
String,
InterpretedString
InterpretedString,
InterpretedStringArray
};
virtual ~AbstractRecordField() = default;
virtual void addOrUpdateValue(auparse_state_t *record) = 0;
virtual void addToColumn(const std::vector<clickhouse::ColumnRef> &columns) const = 0;
virtual Type getType() const = 0;
@ -80,7 +83,7 @@ private:
class AbstractRecordFieldFactory
{
public:
std::shared_ptr<AbstractRecordField> createFromAuditRecord(const std::string &name, auparse_state_t *record);
std::shared_ptr<AbstractRecordField> createFromAuditRecord(const std::string &field_name, auparse_type_t field_type);
static AbstractRecordFieldFactory& instance();
@ -100,7 +103,7 @@ protected:
AbstractRecordFieldFactory(const AbstractRecordFieldFactory &other) = delete;
AbstractRecordFieldFactory& operator=(const AbstractRecordFieldFactory &other) = delete;
std::map<auparse_type_t, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name, auparse_state_t *record)> > m_factoryMap;
std::map<auparse_type_t, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > m_factoryMap;
};
class CommonStringRecordField: public AbstractRecordField
@ -117,37 +120,55 @@ protected:
class StringRecordField: public CommonStringRecordField
{
public:
static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name, auparse_state_t *record);
static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name);
virtual void addOrUpdateValue(auparse_state_t *record) override;
virtual Type getType() const override;
protected:
StringRecordField(const std::string &name, auparse_state_t *record);
explicit StringRecordField(const std::string &name);
};
class InterpretedStringRecordField: public CommonStringRecordField
{
public:
static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name, auparse_state_t *record);
static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name);
virtual void addOrUpdateValue(auparse_state_t *record) override;
virtual Type getType() const override;
protected:
InterpretedStringRecordField(const std::string &name, auparse_state_t *record);
explicit InterpretedStringRecordField(const std::string &name);
};
class IntegerRecordField: public InterpretedStringRecordField
{
public:
static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name, auparse_state_t *record);
static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name);
virtual void addOrUpdateValue(auparse_state_t *record) override;
virtual void addToColumn(const std::vector<clickhouse::ColumnRef> &columns) const override;
virtual Type getType() const override;
protected:
IntegerRecordField(const std::string &name, auparse_state_t *record);
explicit IntegerRecordField(const std::string &name);
boost::optional<int> m_int_value;
};
class InterpretedStringArrayRecordField: public AbstractRecordField
{
public:
static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name);
virtual void addOrUpdateValue(auparse_state_t *record) override;
virtual void addToColumn(const std::vector<clickhouse::ColumnRef> &columns) const override;
virtual Type getType() const override;
protected:
explicit InterpretedStringArrayRecordField(const std::string &name);
std::list<std::string> m_value;
};
#endif /* AUDITD_PLUGIN_CLICKHOUSE_RECORD_HPP */