auditd-plugin-clickhouse/auditd-record.cpp
2020-01-27 13:07:59 +03:00

757 lines
24 KiB
C++

/*
* auditd-plugin-clickhouse is an auditd plugin for sending auditd data
* to clickhouse DB.
* Copyright (C) 2019-2020 Aleksei Nikiforov <darktemplar@basealt.ru>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
#include "auditd-record.hpp"
#include "logging.hpp"
#include "utils.hpp"
#include <functional>
#include <set>
#include <boost/lexical_cast.hpp>
#include <clickhouse-cpp/columns/array.h>
#include <clickhouse-cpp/columns/nullable.h>
#include <clickhouse-cpp/columns/numeric.h>
#include <clickhouse-cpp/columns/string.h>
namespace {
std::map<auparse_type_t, std::function<bool(const std::string &)> >& get_audit_type_check_instance()
{
static std::map<auparse_type_t, std::function<bool(const std::string &)> > instance;
return instance;
}
class AuditTypeCheckRegister
{
public:
AuditTypeCheckRegister(auparse_type_t type, const std::function<bool(const std::string &)> &func)
{
get_audit_type_check_instance()[type] = func;
}
};
std::function<bool(const std::string &)> integer_record_type_check_function()
{
return [](const std::string &value) -> bool {
std::set<std::string> allowed_values = {
"integer"
};
return (allowed_values.find(value) != allowed_values.end());
};
}
std::function<bool(const std::string &)> interpreted_string_record_type_check_function()
{
return [](const std::string &value) -> bool {
std::set<std::string> allowed_values = {
"string",
"string_array"
};
return (allowed_values.find(value) != allowed_values.end());
};
}
std::function<bool(const std::string &)> any_record_type_check_function()
{
return [](const std::string &value) -> bool {
return true;
};
}
} // unnamed namespace
#define register_record_type(audit_type, check_function) \
static const AuditTypeCheckRegister audit_type_check_register_##audit_type(audit_type, check_function)
register_record_type(AUPARSE_TYPE_UNCLASSIFIED, any_record_type_check_function());
register_record_type(AUPARSE_TYPE_UID, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_GID, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_SYSCALL, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_ARCH, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_EXIT, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_ESCAPED, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_PERM, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_MODE, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_SOCKADDR, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_FLAGS, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_PROMISC, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_CAPABILITY, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_SUCCESS, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_A0, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_A1, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_A2, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_A3, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_SIGNAL, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_LIST, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_TTY_DATA, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_SESSION, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_CAP_BITMAP, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_NFPROTO, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_ICMPTYPE, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_PROTOCOL, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_ADDR, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_PERSONALITY, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_SECCOMP, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_OFLAG, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_MMAP, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_MODE_SHORT, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_MAC_LABEL, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_PROCTITLE, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_HOOK, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_NETACTION, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_MACPROTO, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_IOCTL_REQ, integer_record_type_check_function());
register_record_type(AUPARSE_TYPE_ESCAPED_KEY, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_ESCAPED_FILE, interpreted_string_record_type_check_function());
register_record_type(AUPARSE_TYPE_FANOTIFY, interpreted_string_record_type_check_function());
#undef register_record_type
bool check_field_type(auparse_type_t field_type, const std::string &database_type, const std::string &database_field_name)
{
const auto &check_instance = get_audit_type_check_instance();
auto check_type_function = check_instance.find(field_type);
if (check_type_function != check_instance.end())
{
return check_type_function->second(database_type);
}
else
{
Logger::write("Warning: unknown record type for record name \"%s\"", database_field_name.c_str());
return interpreted_string_record_type_check_function()(database_type);
}
}
std::string field_type_to_string(auparse_type_t field_type)
{
#define field_type_macro(T) { T, #T }
static const std::map<auparse_type_t, std::string> s_field_type_to_string_map = {
field_type_macro(AUPARSE_TYPE_UNCLASSIFIED),
field_type_macro(AUPARSE_TYPE_UID),
field_type_macro(AUPARSE_TYPE_GID),
field_type_macro(AUPARSE_TYPE_SYSCALL),
field_type_macro(AUPARSE_TYPE_ARCH),
field_type_macro(AUPARSE_TYPE_EXIT),
field_type_macro(AUPARSE_TYPE_ESCAPED),
field_type_macro(AUPARSE_TYPE_PERM),
field_type_macro(AUPARSE_TYPE_MODE),
field_type_macro(AUPARSE_TYPE_SOCKADDR),
field_type_macro(AUPARSE_TYPE_FLAGS),
field_type_macro(AUPARSE_TYPE_PROMISC),
field_type_macro(AUPARSE_TYPE_CAPABILITY),
field_type_macro(AUPARSE_TYPE_SUCCESS),
field_type_macro(AUPARSE_TYPE_A0),
field_type_macro(AUPARSE_TYPE_A1),
field_type_macro(AUPARSE_TYPE_A2),
field_type_macro(AUPARSE_TYPE_A3),
field_type_macro(AUPARSE_TYPE_SIGNAL),
field_type_macro(AUPARSE_TYPE_LIST),
field_type_macro(AUPARSE_TYPE_TTY_DATA),
field_type_macro(AUPARSE_TYPE_SESSION),
field_type_macro(AUPARSE_TYPE_CAP_BITMAP),
field_type_macro(AUPARSE_TYPE_NFPROTO),
field_type_macro(AUPARSE_TYPE_ICMPTYPE),
field_type_macro(AUPARSE_TYPE_PROTOCOL),
field_type_macro(AUPARSE_TYPE_ADDR),
field_type_macro(AUPARSE_TYPE_PERSONALITY),
field_type_macro(AUPARSE_TYPE_SECCOMP),
field_type_macro(AUPARSE_TYPE_OFLAG),
field_type_macro(AUPARSE_TYPE_MMAP),
field_type_macro(AUPARSE_TYPE_MODE_SHORT),
field_type_macro(AUPARSE_TYPE_MAC_LABEL),
field_type_macro(AUPARSE_TYPE_PROCTITLE),
field_type_macro(AUPARSE_TYPE_HOOK),
field_type_macro(AUPARSE_TYPE_NETACTION),
field_type_macro(AUPARSE_TYPE_MACPROTO),
field_type_macro(AUPARSE_TYPE_IOCTL_REQ),
field_type_macro(AUPARSE_TYPE_ESCAPED_KEY),
field_type_macro(AUPARSE_TYPE_ESCAPED_FILE),
field_type_macro(AUPARSE_TYPE_FANOTIFY)
};
#undef field_type_macro
auto iter = s_field_type_to_string_map.find(field_type);
if (iter != s_field_type_to_string_map.end())
{
return iter->second;
}
else
{
Logger::write("Warning: unknown field type for field type id \"%d\"", field_type);
return "unknown";
}
}
std::string generate_name_for_audit_record(const AuditRecord &record)
{
return boost::lexical_cast<std::string>(record.time)
+ "_" + boost::lexical_cast<std::string>(record.milliseconds)
+ "_" + boost::lexical_cast<std::string>(record.serial)
+ "_" + record.node
+ ".json";
}
AbstractRecordField::AbstractRecordField(const std::string &name)
: m_name(name)
{
}
CommonStringRecordField::CommonStringRecordField(const std::string &name)
: AbstractRecordField(name)
{
}
std::vector<AbstractRecordField::Column> CommonStringRecordField::generateColumnsAndNames() const
{
return std::vector<AbstractRecordField::Column> {
Column { m_name, std::make_shared<clickhouse::ColumnNullable>(std::make_shared<clickhouse::ColumnString>(), std::make_shared<clickhouse::ColumnUInt8>()) }
};
}
void CommonStringRecordField::addToColumn(const std::vector<Column> &columns) const
{
if ((columns.size() != 1) || (!columns[0].value))
{
throw std::runtime_error("CommonStringRecordField::addToColumn: invalid columns argument");
}
auto nullable = columns[0].value->As<clickhouse::ColumnNullable>();
if (!nullable)
{
throw std::runtime_error("Invalid column type: not ColumnNullable");
}
auto nested = nullable->Nested();
if ((!nested) || (nested->Type()->GetCode() != clickhouse::Type::String))
{
throw std::runtime_error("Invalid nested column type: not String");
}
auto data_column = std::make_shared<clickhouse::ColumnString>();
auto null_column = std::make_shared<clickhouse::ColumnUInt8>();
if (m_value)
{
data_column->Append(*m_value);
null_column->Append(0);
}
else
{
data_column->Append(std::string());
null_column->Append(1);
}
columns[0].value->Append(std::make_shared<clickhouse::ColumnNullable>(data_column, null_column));
}
const boost::optional<std::string>& CommonStringRecordField::getStringValue() const
{
return m_value;
}
void CommonStringRecordField::setStringValue(const boost::optional<std::string> &value)
{
m_value = value;
}
std::shared_ptr<StringRecordField> StringRecordField::createRecord(const std::string &name)
{
std::shared_ptr<StringRecordField> result;
result.reset(new StringRecordField(name));
return result;
}
void StringRecordField::addOrUpdateValue(auparse_state_t *record)
{
if (record)
{
m_value = string_or_null(auparse_get_field_str(record));
}
else
{
m_value = boost::none;
}
}
StringRecordField::StringRecordField(const std::string &name)
: CommonStringRecordField(name)
{
}
AbstractRecordField::Type StringRecordField::getType() const
{
return AbstractRecordField::Type::String;
}
std::shared_ptr<InterpretedStringRecordField> InterpretedStringRecordField::createRecord(const std::string &name)
{
std::shared_ptr<InterpretedStringRecordField> result;
result.reset(new InterpretedStringRecordField(name));
return result;
}
InterpretedStringRecordField::InterpretedStringRecordField(const std::string &name)
: CommonStringRecordField(name)
{
}
void InterpretedStringRecordField::addOrUpdateValue(auparse_state_t *record)
{
if (record)
{
m_value = string_or_null(auparse_interpret_field(record));
}
else
{
m_value = boost::none;
}
}
AbstractRecordField::Type InterpretedStringRecordField::getType() const
{
return AbstractRecordField::Type::InterpretedString;
}
std::shared_ptr<IntegerRecordField> IntegerRecordField::createRecord(const std::string &name)
{
std::shared_ptr<IntegerRecordField> result;
result.reset(new IntegerRecordField(name));
return result;
}
IntegerRecordField::IntegerRecordField(const std::string &name)
: InterpretedStringRecordField(name)
{
}
void IntegerRecordField::addOrUpdateValue(auparse_state_t *record)
{
InterpretedStringRecordField::addOrUpdateValue(record);
if (record)
{
m_int_value = auparse_get_field_int(record);
}
else
{
m_int_value = boost::none;
}
}
std::vector<AbstractRecordField::Column> IntegerRecordField::generateColumnsAndNames() const
{
return std::vector<AbstractRecordField::Column> {
Column { m_name + "_IntValue", std::make_shared<clickhouse::ColumnNullable>(std::make_shared<clickhouse::ColumnInt64>(), std::make_shared<clickhouse::ColumnUInt8>()) },
Column { m_name + "_InterpretedValue", std::make_shared<clickhouse::ColumnNullable>(std::make_shared<clickhouse::ColumnString>(), std::make_shared<clickhouse::ColumnUInt8>()) }
};
}
void IntegerRecordField::addToColumn(const std::vector<Column> &columns) const
{
if ((columns.size() != 2) || (!columns[0].value) || (!columns[1].value))
{
throw std::runtime_error("IntegerRecordField::addToColumn: invalid columns argument");
}
auto nullable = columns[0].value->As<clickhouse::ColumnNullable>();
if (!nullable)
{
throw std::runtime_error("Invalid column type: not ColumnNullable");
}
auto nested = nullable->Nested();
if ((!nested) || (nested->Type()->GetCode() != clickhouse::Type::Int64))
{
throw std::runtime_error("Invalid nested column type: not Int64");
}
auto data_column = std::make_shared<clickhouse::ColumnInt64>();
auto null_column = std::make_shared<clickhouse::ColumnUInt8>();
if (m_int_value)
{
data_column->Append(*m_int_value);
null_column->Append(0);
}
else
{
data_column->Append(0);
null_column->Append(1);
}
std::vector<Column> string_columns;
string_columns.push_back(columns[1]);
columns[0].value->Append(std::make_shared<clickhouse::ColumnNullable>(data_column, null_column));
// now also add string value
InterpretedStringRecordField::addToColumn(string_columns);
}
AbstractRecordField::Type IntegerRecordField::getType() const
{
return AbstractRecordField::Type::Int;
}
const boost::optional<int>& IntegerRecordField::getIntValue() const
{
return m_int_value;
}
void IntegerRecordField::setIntValue(const boost::optional<int> &value)
{
m_int_value = value;
}
std::shared_ptr<InterpretedStringArrayRecordField> InterpretedStringArrayRecordField::createRecord(const std::string &name)
{
std::shared_ptr<InterpretedStringArrayRecordField> result;
result.reset(new InterpretedStringArrayRecordField(name));
return result;
}
void InterpretedStringArrayRecordField::addOrUpdateValue(auparse_state_t *record)
{
if (record)
{
m_names_array.push_back(string_or_null(auparse_get_field_name(record)));
m_values_array.push_back(string_or_null(auparse_interpret_field(record)));
}
}
std::vector<AbstractRecordField::Column> InterpretedStringArrayRecordField::generateColumnsAndNames() const
{
return std::vector<AbstractRecordField::Column> {
Column { m_name + "_Name", std::make_shared<clickhouse::ColumnArray>(std::make_shared<clickhouse::ColumnString>()) },
Column { m_name + "_Value", std::make_shared<clickhouse::ColumnArray>(std::make_shared<clickhouse::ColumnString>()) }
};
}
void InterpretedStringArrayRecordField::addToColumn(const std::vector<Column> &columns) const
{
if ((columns.size() != 2) || (!columns[0].value) || (!columns[1].value))
{
throw std::runtime_error("InterpretedStringArrayRecordField::addToColumn: invalid columns argument");
}
auto array_names = columns[0].value->As<clickhouse::ColumnArray>();
if (!array_names)
{
throw std::runtime_error("Invalid column 0 type: not ColumnArray");
}
auto array_values = columns[1].value->As<clickhouse::ColumnArray>();
if (!array_values)
{
throw std::runtime_error("Invalid column 1 type: not ColumnArray");
}
auto name_column = std::make_shared<clickhouse::ColumnString>();
for (auto iter = m_names_array.begin(); iter != m_names_array.end(); ++iter)
{
name_column->Append(*iter);
}
auto value_column = std::make_shared<clickhouse::ColumnString>();
for (auto iter = m_values_array.begin(); iter != m_values_array.end(); ++iter)
{
value_column->Append(*iter);
}
array_names->AppendAsColumn(name_column);
array_values->AppendAsColumn(value_column);
}
InterpretedStringArrayRecordField::InterpretedStringArrayRecordField(const std::string &name)
: AbstractRecordField(name)
{
}
AbstractRecordField::Type InterpretedStringArrayRecordField::getType() const
{
return AbstractRecordField::Type::InterpretedStringArray;
}
const std::list<std::string>& InterpretedStringArrayRecordField::getNamesArray() const
{
return m_names_array;
}
const std::list<std::string>& InterpretedStringArrayRecordField::getValuesArray() const
{
return m_values_array;
}
void InterpretedStringArrayRecordField::setArrays(std::list<std::string> names_array, std::list<std::string> values_array)
{
if (names_array.size() != values_array.size())
{
throw std::runtime_error("InterpretedStringArrayRecordField::setArrays: array sizes mismatch");
}
m_names_array.swap(names_array);
m_values_array.swap(values_array);
}
boost::property_tree::ptree AuditRecord::toPtree() const
{
boost::property_tree::ptree data;
data.put_child("time", boost::property_tree::ptree(boost::lexical_cast<std::string>(this->time)));
data.put_child("milliseconds", boost::property_tree::ptree(boost::lexical_cast<std::string>(this->milliseconds)));
data.put_child("serial", boost::property_tree::ptree(boost::lexical_cast<std::string>(this->serial)));
data.put_child("node", boost::property_tree::ptree(this->node));
boost::property_tree::ptree fields_data;
for (auto iter = this->fields.begin(); iter != this->fields.end(); ++iter)
{
boost::property_tree::ptree item;
item.put_child("type", boost::property_tree::ptree(boost::lexical_cast<std::string>(static_cast<int>(iter->second->getType()))));
switch (iter->second->getType())
{
case AbstractRecordField::Type::Int:
{
auto ptr = std::dynamic_pointer_cast<IntegerRecordField>(iter->second);
if (ptr)
{
auto int_value = ptr->getIntValue();
if (int_value)
{
item.put_child("value_int", boost::property_tree::ptree(boost::lexical_cast<std::string>(*int_value)));
}
auto str_value = ptr->getStringValue();
if (str_value)
{
item.put_child("value_str", boost::property_tree::ptree(*str_value));
}
}
}
break;
case AbstractRecordField::Type::String:
case AbstractRecordField::Type::InterpretedString:
{
auto ptr = std::dynamic_pointer_cast<CommonStringRecordField>(iter->second);
if (ptr)
{
auto str_value = ptr->getStringValue();
if (str_value)
{
item.put_child("value_str", boost::property_tree::ptree(*str_value));
}
}
}
break;
case AbstractRecordField::Type::InterpretedStringArray:
{
auto ptr = std::dynamic_pointer_cast<InterpretedStringArrayRecordField>(iter->second);
if (ptr)
{
const auto &names_list = ptr->getNamesArray();
const auto &values_list = ptr->getValuesArray();
boost::property_tree::ptree names_tree, values_tree;
for (auto array_iter = names_list.begin(); array_iter != names_list.end(); ++array_iter)
{
names_tree.push_back(std::make_pair(std::string(), boost::property_tree::ptree(*array_iter)));
}
for (auto array_iter = values_list.begin(); array_iter != values_list.end(); ++array_iter)
{
values_tree.push_back(std::make_pair(std::string(), boost::property_tree::ptree(*array_iter)));
}
item.put_child("names", names_tree);
item.put_child("values", values_tree);
}
}
break;
}
fields_data.put_child(iter->first, item);
}
data.put_child("fields", fields_data);
return data;
}
std::shared_ptr<AuditRecord> AuditRecord::fromPtree(const boost::property_tree::ptree &data)
{
std::shared_ptr<AuditRecord> result = std::make_shared<AuditRecord>();
{
auto child = data.get_child_optional("time");
if (child)
{
result->time = boost::lexical_cast<decltype(AuditRecord::time)>(child->get_value<std::string>());
}
}
{
auto child = data.get_child_optional("milliseconds");
if (child)
{
result->milliseconds = boost::lexical_cast<decltype(AuditRecord::milliseconds)>(child->get_value<std::string>());
}
}
{
auto child = data.get_child_optional("serial");
if (child)
{
result->serial = boost::lexical_cast<decltype(AuditRecord::serial)>(child->get_value<std::string>());
}
}
{
auto child = data.get_child_optional("node");
if (child)
{
result->node = boost::lexical_cast<decltype(AuditRecord::node)>(child->get_value<std::string>());
}
}
{
auto fields = data.get_child_optional("fields");
if (fields)
{
for (auto iter = fields->begin(); iter != fields->end(); ++iter)
{
auto field_type = iter->second.get_child_optional("type");
if (field_type)
{
auto field_type_value = static_cast<AbstractRecordField::Type>(boost::lexical_cast<int>(field_type->get_value<std::string>()));
switch (field_type_value)
{
case AbstractRecordField::Type::Int:
{
auto record_field = IntegerRecordField::createRecord(iter->first);
if (record_field)
{
auto int_value = iter->second.get_child_optional("value_int");
if (int_value)
{
record_field->setIntValue(boost::lexical_cast<int>(int_value->get_value<std::string>()));
}
auto str_value = iter->second.get_child_optional("value_str");
if (str_value)
{
record_field->setStringValue(str_value->get_value<std::string>());
}
result->fields[iter->first] = record_field;
}
}
break;
case AbstractRecordField::Type::String:
{
auto record_field = StringRecordField::createRecord(iter->first);
if (record_field)
{
auto str_value = iter->second.get_child_optional("value_str");
if (str_value)
{
record_field->setStringValue(str_value->get_value<std::string>());
}
result->fields[iter->first] = record_field;
}
}
break;
case AbstractRecordField::Type::InterpretedString:
{
auto record_field = InterpretedStringRecordField::createRecord(iter->first);
if (record_field)
{
auto str_value = iter->second.get_child_optional("value_str");
if (str_value)
{
record_field->setStringValue(str_value->get_value<std::string>());
}
result->fields[iter->first] = record_field;
}
}
break;
case AbstractRecordField::Type::InterpretedStringArray:
{
auto record_field = InterpretedStringArrayRecordField::createRecord(iter->first);
if (record_field)
{
std::list<std::string> names, values;
auto name_child = iter->second.get_child_optional("names");
if (name_child)
{
for (auto child_iter = name_child->begin(); child_iter != name_child->end(); ++child_iter)
{
names.push_back(child_iter->second.get_value<std::string>());
}
}
auto value_child = iter->second.get_child_optional("values");
if (value_child)
{
for (auto child_iter = value_child->begin(); child_iter != value_child->end(); ++child_iter)
{
values.push_back(child_iter->second.get_value<std::string>());
}
}
record_field->setArrays(names, values);
result->fields[iter->first] = record_field;
}
}
break;
}
}
}
}
}
return result;
}
bool AuditRecord::operator<(const AuditRecord &other) const
{
return std::make_tuple(this->time, this->milliseconds, this->serial, this->node)
< std::make_tuple(other.time, other.milliseconds, other.serial, other.node);
}