darktemplar/auditd-plugin-clickhouse
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Don't store record names in the record itself

Browse Source
This commit is contained in:
Aleksei Nikiforov 2020-01-29 14:47:27 +03:00
parent 69df9988ad
commit 17d6a6489c
6 changed files with 56 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
auditd-datatypes.cpp
View File

@ -31,7 +31,7 @@
static std::map<std::string, std::string> s_datatypes_map;
static std::list<std::tuple<std::string, std::string, std::string> > s_datatype_regexps_map;
static std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > s_type_creation_map;
static std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>()> > s_type_creation_map;
void read_datatypes_map(const std::string &config_filename)
{
@ -85,7 +85,7 @@ std::list<std::tuple<std::string, std::string, std::string> > get_datatype_regex
return s_datatype_regexps_map;
}
std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > get_type_creation_map()
std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>()> > get_type_creation_map()
{
return s_type_creation_map;
}

2
auditd-datatypes.hpp
View File

@ -32,6 +32,6 @@
void read_datatypes_map(const std::string &config_filename);
std::map<std::string, std::string> get_datatypes_map();
std::list<std::tuple<std::string, std::string, std::string> > get_datatype_regexps_map();
std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > get_type_creation_map();
std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>()> > get_type_creation_map();
#endif /* AUDITD_PLUGIN_CLICKHOUSE_DATATYPES_HPP */

18
auditd-plugin-clickhouse.cpp
View File

@ -77,7 +77,7 @@ struct CallbackData
{
std::map<std::string, std::string> datatypes_map;
std::list<std::tuple<std::string, std::string, std::string> > datatype_regexps_map;
std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > type_creation_map;
std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>()> > type_creation_map;
std::set<std::string> all_fields_set;
clickhouse::Client *clickhouse_client;
@ -115,7 +115,7 @@ void initialize_data_block(
std::map<std::string, clickhouse::ColumnRef > &data,
const std::map<std::string, std::string> &datatypes_map,
const std::list<std::tuple<std::string, std::string, std::string> > &datatype_regexps_map,
const std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > &generators)
const std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>()> > &generators)
{
data["record_time"] = std::make_shared<clickhouse::ColumnDateTime>();
data["record_milli"] = std::make_shared<clickhouse::ColumnUInt64>();
@ -127,7 +127,7 @@ void initialize_data_block(
auto factory_iter = generators.find(iter->second);
if (factory_iter != generators.end())
{
auto columns = factory_iter->second(sanitize_column_name(iter->first))->generateColumnsAndNames();
auto columns = factory_iter->second()->generateColumnsAndNames(sanitize_column_name(iter->first));
for (auto column_iter = columns.begin(); column_iter != columns.end(); ++column_iter)
{
@ -145,7 +145,7 @@ void initialize_data_block(
auto factory_iter = generators.find(std::get<1>(*iter));
if (factory_iter != generators.end())
{
auto columns = factory_iter->second(sanitize_column_name(std::get<2>(*iter)))->generateColumnsAndNames();
auto columns = factory_iter->second()->generateColumnsAndNames(sanitize_column_name(std::get<2>(*iter)));
for (auto column_iter = columns.begin(); column_iter != columns.end(); ++column_iter)
{
@ -160,7 +160,7 @@ void initialize_data_block(
// also add "unknown_field"
{
auto columns = InterpretedStringArrayRecordField::createRecord(sanitize_column_name("unknown_field"))->generateColumnsAndNames();
auto columns = InterpretedStringArrayRecordField::createRecord()->generateColumnsAndNames(sanitize_column_name("unknown_field"));
for (auto column_iter = columns.begin(); column_iter != columns.end(); ++column_iter)
{
@ -180,7 +180,7 @@ void generate_clickhouse_columns_from_audit_records(
for (auto iter = record.fields.begin(); iter != record.fields.end(); ++iter)
{
auto columns = iter->second->generateColumnsAndNames();
auto columns = iter->second->generateColumnsAndNames(iter->first);
iter->second->addToColumn(columns);
for (auto column_iter = columns.begin(); column_iter != columns.end(); ++column_iter)
@ -307,12 +307,12 @@ void auparse_callback(auparse_state_t *au, auparse_cb_event_t cb_event_type, voi
auto iter = callback_data->type_creation_map.find(database_type);
if (iter != callback_data->type_creation_map.end())
{
data_ptr = iter->second(database_name);
data_ptr = iter->second();
}
else
{
Logger::write("Warning: no creator function found for data type \"%s\", using \"string\" as fallback", database_type.c_str());
data_ptr = InterpretedStringRecordField::createRecord(database_name);
data_ptr = InterpretedStringRecordField::createRecord();
}
audit_record->fields[database_name] = data_ptr;
@ -364,7 +364,7 @@ void auparse_callback(auparse_state_t *au, auparse_cb_event_t cb_event_type, voi
auto factory_iter = callback_data->type_creation_map.find(type_name);
if (factory_iter != callback_data->type_creation_map.end())
{
audit_record->fields[*iter] = factory_iter->second(*iter);
audit_record->fields[*iter] = factory_iter->second();
}
}
else

90
auditd-record.cpp
View File

@ -215,20 +215,10 @@ std::string generate_name_for_audit_record(const AuditRecord &record)
+ ".json";
}
AbstractRecordField::AbstractRecordField(const std::string &name)
: m_name(name)
{
}
CommonStringRecordField::CommonStringRecordField(const std::string &name)
: AbstractRecordField(name)
{
}
std::vector<AbstractRecordField::Column> CommonStringRecordField::generateColumnsAndNames() const
std::vector<AbstractRecordField::Column> CommonStringRecordField::generateColumnsAndNames(const std::string &name) const
{
return std::vector<AbstractRecordField::Column> {
Column { m_name, std::make_shared<clickhouse::ColumnNullable>(std::make_shared<clickhouse::ColumnString>(), std::make_shared<clickhouse::ColumnUInt8>()) }
Column { name, std::make_shared<clickhouse::ColumnNullable>(std::make_shared<clickhouse::ColumnString>(), std::make_shared<clickhouse::ColumnUInt8>()) }
};
}
@ -278,18 +268,15 @@ void CommonStringRecordField::setStringValue(const boost::optional<std::string>
m_value = value;
}
std::shared_ptr<StringRecordField> StringRecordField::createRecord(const std::string &name)
std::shared_ptr<StringRecordField> StringRecordField::createRecord()
{
class StringRecordFieldPublic: public StringRecordField
{
public:
explicit StringRecordFieldPublic(const std::string &l_name)
: StringRecordField(l_name)
{
}
StringRecordFieldPublic() = default;
};
return std::make_shared<StringRecordFieldPublic>(name);
return std::make_shared<StringRecordFieldPublic>();
}
void StringRecordField::addOrUpdateValue(auparse_state_t *record)
@ -304,33 +291,20 @@ void StringRecordField::addOrUpdateValue(auparse_state_t *record)
}
}
StringRecordField::StringRecordField(const std::string &name)
: CommonStringRecordField(name)
{
}
AbstractRecordField::Type StringRecordField::getType() const
{
return AbstractRecordField::Type::String;
}
std::shared_ptr<InterpretedStringRecordField> InterpretedStringRecordField::createRecord(const std::string &name)
std::shared_ptr<InterpretedStringRecordField> InterpretedStringRecordField::createRecord()
{
class InterpretedStringRecordFieldPublic: public InterpretedStringRecordField
{
public:
explicit InterpretedStringRecordFieldPublic(const std::string &l_name)
: InterpretedStringRecordField(l_name)
{
}
InterpretedStringRecordFieldPublic() = default;
};
return std::make_shared<InterpretedStringRecordFieldPublic>(name);
}
InterpretedStringRecordField::InterpretedStringRecordField(const std::string &name)
: CommonStringRecordField(name)
{
return std::make_shared<InterpretedStringRecordFieldPublic>();
}
void InterpretedStringRecordField::addOrUpdateValue(auparse_state_t *record)
@ -350,23 +324,15 @@ AbstractRecordField::Type InterpretedStringRecordField::getType() const
return AbstractRecordField::Type::InterpretedString;
}
std::shared_ptr<IntegerRecordField> IntegerRecordField::createRecord(const std::string &name)
std::shared_ptr<IntegerRecordField> IntegerRecordField::createRecord()
{
class IntegerRecordFieldPublic: public IntegerRecordField
{
public:
explicit IntegerRecordFieldPublic(const std::string &l_name)
: IntegerRecordField(l_name)
{
}
IntegerRecordFieldPublic() = default;
};
return std::make_shared<IntegerRecordFieldPublic>(name);
}
IntegerRecordField::IntegerRecordField(const std::string &name)
: InterpretedStringRecordField(name)
{
return std::make_shared<IntegerRecordFieldPublic>();
}
void IntegerRecordField::addOrUpdateValue(auparse_state_t *record)
@ -383,11 +349,11 @@ void IntegerRecordField::addOrUpdateValue(auparse_state_t *record)
}
}
std::vector<AbstractRecordField::Column> IntegerRecordField::generateColumnsAndNames() const
std::vector<AbstractRecordField::Column> IntegerRecordField::generateColumnsAndNames(const std::string &name) const
{
return std::vector<AbstractRecordField::Column> {
Column { m_name + "_IntValue", std::make_shared<clickhouse::ColumnNullable>(std::make_shared<clickhouse::ColumnInt64>(), std::make_shared<clickhouse::ColumnUInt8>()) },
Column { m_name + "_InterpretedValue", std::make_shared<clickhouse::ColumnNullable>(std::make_shared<clickhouse::ColumnString>(), std::make_shared<clickhouse::ColumnUInt8>()) }
Column { name + "_IntValue", std::make_shared<clickhouse::ColumnNullable>(std::make_shared<clickhouse::ColumnInt64>(), std::make_shared<clickhouse::ColumnUInt8>()) },
Column { name + "_InterpretedValue", std::make_shared<clickhouse::ColumnNullable>(std::make_shared<clickhouse::ColumnString>(), std::make_shared<clickhouse::ColumnUInt8>()) }
};
}
@ -448,18 +414,15 @@ void IntegerRecordField::setIntValue(const boost::optional<int> &value)
m_int_value = value;
}
std::shared_ptr<InterpretedStringArrayRecordField> InterpretedStringArrayRecordField::createRecord(const std::string &name)
std::shared_ptr<InterpretedStringArrayRecordField> InterpretedStringArrayRecordField::createRecord()
{
class InterpretedStringArrayRecordFieldPublic: public InterpretedStringArrayRecordField
{
public:
explicit InterpretedStringArrayRecordFieldPublic(const std::string &l_name)
: InterpretedStringArrayRecordField(l_name)
{
}
InterpretedStringArrayRecordFieldPublic() = default;
};
return std::make_shared<InterpretedStringArrayRecordFieldPublic>(name);
return std::make_shared<InterpretedStringArrayRecordFieldPublic>();
}
void InterpretedStringArrayRecordField::addOrUpdateValue(auparse_state_t *record)
@ -471,11 +434,11 @@ void InterpretedStringArrayRecordField::addOrUpdateValue(auparse_state_t *record
}
}
std::vector<AbstractRecordField::Column> InterpretedStringArrayRecordField::generateColumnsAndNames() const
std::vector<AbstractRecordField::Column> InterpretedStringArrayRecordField::generateColumnsAndNames(const std::string &name) const
{
return std::vector<AbstractRecordField::Column> {
Column { m_name + "_Name", std::make_shared<clickhouse::ColumnArray>(std::make_shared<clickhouse::ColumnString>()) },
Column { m_name + "_Value", std::make_shared<clickhouse::ColumnArray>(std::make_shared<clickhouse::ColumnString>()) }
Column { name + "_Name", std::make_shared<clickhouse::ColumnArray>(std::make_shared<clickhouse::ColumnString>()) },
Column { name + "_Value", std::make_shared<clickhouse::ColumnArray>(std::make_shared<clickhouse::ColumnString>()) }
};
}
@ -514,11 +477,6 @@ void InterpretedStringArrayRecordField::addToColumn(const std::vector<Column> &c
array_values->AppendAsColumn(value_column);
}
InterpretedStringArrayRecordField::InterpretedStringArrayRecordField(const std::string &name)
: AbstractRecordField(name)
{
}
AbstractRecordField::Type InterpretedStringArrayRecordField::getType() const
{
return AbstractRecordField::Type::InterpretedStringArray;
@ -685,7 +643,7 @@ std::shared_ptr<AuditRecord> AuditRecord::fromPtree(const boost::property_tree::
{
case AbstractRecordField::Type::Int:
{
auto record_field = IntegerRecordField::createRecord(iter->first);
auto record_field = IntegerRecordField::createRecord();
if (record_field)
{
auto int_value = iter->second.get_child_optional("value_int");
@ -707,7 +665,7 @@ std::shared_ptr<AuditRecord> AuditRecord::fromPtree(const boost::property_tree::
case AbstractRecordField::Type::String:
{
auto record_field = StringRecordField::createRecord(iter->first);
auto record_field = StringRecordField::createRecord();
if (record_field)
{
auto str_value = iter->second.get_child_optional("value_str");
@ -723,7 +681,7 @@ std::shared_ptr<AuditRecord> AuditRecord::fromPtree(const boost::property_tree::
case AbstractRecordField::Type::InterpretedString:
{
auto record_field = InterpretedStringRecordField::createRecord(iter->first);
auto record_field = InterpretedStringRecordField::createRecord();
if (record_field)
{
auto str_value = iter->second.get_child_optional("value_str");
@ -739,7 +697,7 @@ std::shared_ptr<AuditRecord> AuditRecord::fromPtree(const boost::property_tree::
case AbstractRecordField::Type::InterpretedStringArray:
{
auto record_field = InterpretedStringArrayRecordField::createRecord(iter->first);
auto record_field = InterpretedStringArrayRecordField::createRecord();
if (record_field)
{
std::list<std::string> names, values;

32
auditd-record.hpp
View File

@ -69,18 +69,14 @@ public:
virtual ~AbstractRecordField() = default;
virtual void addOrUpdateValue(auparse_state_t *record) = 0;
virtual std::vector<Column> generateColumnsAndNames() const = 0;
virtual std::vector<Column> generateColumnsAndNames(const std::string &name) const = 0;
virtual void addToColumn(const std::vector<Column> &columns) const = 0;
virtual Type getType() const = 0;
std::string getName() const { return m_name; }
protected:
explicit AbstractRecordField(const std::string &name);
AbstractRecordField() = default;
AbstractRecordField(const AbstractRecordField &other) = default;
AbstractRecordField& operator=(const AbstractRecordField &other) = default;
std::string m_name;
};
struct AuditRecord
@ -103,14 +99,14 @@ struct AuditRecord
class CommonStringRecordField: public AbstractRecordField
{
public:
virtual std::vector<Column> generateColumnsAndNames() const override;
virtual std::vector<Column> generateColumnsAndNames(const std::string &name) const override;
virtual void addToColumn(const std::vector<Column> &columns) const override;
const boost::optional<std::string>& getStringValue() const;
void setStringValue(const boost::optional<std::string> &value);
protected:
explicit CommonStringRecordField(const std::string &name);
CommonStringRecordField() = default;
boost::optional<std::string> m_value;
};
@ -118,34 +114,34 @@ protected:
class StringRecordField: public CommonStringRecordField
{
public:
static std::shared_ptr<StringRecordField> createRecord(const std::string &name);
static std::shared_ptr<StringRecordField> createRecord();
virtual void addOrUpdateValue(auparse_state_t *record) override;
virtual Type getType() const override;
protected:
explicit StringRecordField(const std::string &name);
StringRecordField() = default;
};
class InterpretedStringRecordField: public CommonStringRecordField
{
public:
static std::shared_ptr<InterpretedStringRecordField> createRecord(const std::string &name);
static std::shared_ptr<InterpretedStringRecordField> createRecord();
virtual void addOrUpdateValue(auparse_state_t *record) override;
virtual Type getType() const override;
protected:
explicit InterpretedStringRecordField(const std::string &name);
InterpretedStringRecordField() = default;
};
class IntegerRecordField: public InterpretedStringRecordField
{
public:
static std::shared_ptr<IntegerRecordField> createRecord(const std::string &name);
static std::shared_ptr<IntegerRecordField> createRecord();
virtual void addOrUpdateValue(auparse_state_t *record) override;
virtual std::vector<Column> generateColumnsAndNames() const override;
virtual std::vector<Column> generateColumnsAndNames(const std::string &name) const override;
virtual void addToColumn(const std::vector<Column> &columns) const override;
virtual Type getType() const override;
@ -153,7 +149,7 @@ public:
void setIntValue(const boost::optional<int> &value);
protected:
explicit IntegerRecordField(const std::string &name);
IntegerRecordField() = default;
boost::optional<int> m_int_value;
};
@ -161,10 +157,10 @@ protected:
class InterpretedStringArrayRecordField: public AbstractRecordField
{
public:
static std::shared_ptr<InterpretedStringArrayRecordField> createRecord(const std::string &name);
static std::shared_ptr<InterpretedStringArrayRecordField> createRecord();
virtual void addOrUpdateValue(auparse_state_t *record) override;
virtual std::vector<Column> generateColumnsAndNames() const override;
virtual std::vector<Column> generateColumnsAndNames(const std::string &name) const override;
virtual void addToColumn(const std::vector<Column> &columns) const override;
virtual Type getType() const override;
@ -173,7 +169,7 @@ public:
void setArrays(std::list<std::string> names_array, std::list<std::string> values_array);
protected:
explicit InterpretedStringArrayRecordField(const std::string &name);
InterpretedStringArrayRecordField() = default;
std::list<std::string> m_names_array;
std::list<std::string> m_values_array;

14
test_audit_record.cpp
View File

@ -42,18 +42,18 @@ static AuditRecord generateTestRecord()
original_record.filename = "some random file name";
auto string_field = InterpretedStringRecordField::createRecord("string_record");
auto string_field = InterpretedStringRecordField::createRecord();
string_field->setStringValue(std::string("string_record_value"));
original_record.fields[string_field->getName()] = string_field;
original_record.fields["string_record"] = string_field;
auto integer_field = IntegerRecordField::createRecord("integer_record");
auto integer_field = IntegerRecordField::createRecord();
integer_field->setIntValue(500);
integer_field->setStringValue(std::string("five hundred"));
original_record.fields[integer_field->getName()] = integer_field;
original_record.fields["integer_record"] = integer_field;
auto array_field = InterpretedStringArrayRecordField::createRecord("array_record");
auto array_field = InterpretedStringArrayRecordField::createRecord();
array_field->setArrays({"first name", "second name"}, {"first value", "second value"});
original_record.fields[array_field->getName()] = array_field;
original_record.fields["array_record"] = array_field;
return original_record;
}
@ -102,8 +102,6 @@ TEST(AuditRecord, Serialization)
continue;
}
EXPECT_EQ(iter->second->getName(), second_iter->second->getName());
switch (iter->second->getType())
{
case AbstractRecordField::Type::Int: