Implement string_array type

Update AbstractRecordField interface to match new use-case

auditd-clickhouse-datatypes.json

@@ -229,6 +229,6 @@
 		"watch": "string"
 	},
 	"datatypes_arrays": {
-		"^a\\d+$": { "type": "string", "dbname": "arg_a" }
+		"^a\\d+$": { "type": "string_array", "dbname": "arg_a" }
 	}
 }

auditd-datatypes.cpp

@@ -31,7 +31,7 @@
 
 static std::map<std::string, std::string> s_datatypes_map;
 static std::list<std::tuple<std::string, std::string, std::string> > s_datatype_regexps_map;
-static std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name, auparse_state_t *record)> > s_type_creation_map;
+static std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > s_type_creation_map;
 
 void read_datatypes_map(const std::string &config_filename)
 {
@@ -41,6 +41,7 @@ void read_datatypes_map(const std::string &config_filename)
 
 	s_type_creation_map["integer"] = &IntegerRecordField::createRecord;
 	s_type_creation_map["string"] = &InterpretedStringRecordField::createRecord;
+	s_type_creation_map["string_array"] = &InterpretedStringArrayRecordField::createRecord;
 
 	boost::property_tree::ptree clickhouse_config_tree;
 
@@ -84,7 +85,7 @@ std::list<std::tuple<std::string, std::string, std::string> > get_datatype_regex
 	return s_datatype_regexps_map;
 }
 
-std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name, auparse_state_t *record)> > get_type_creation_map()
+std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > get_type_creation_map()
 {
 	return s_type_creation_map;
 }

auditd-datatypes.hpp

@@ -32,6 +32,6 @@
 void read_datatypes_map(const std::string &config_filename);
 std::map<std::string, std::string> get_datatypes_map();
 std::list<std::tuple<std::string, std::string, std::string> > get_datatype_regexps_map();
-std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name, auparse_state_t *record)> > get_type_creation_map();
+std::map<std::string, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > get_type_creation_map();
 
 #endif /* AUDITD_PLUGIN_CLICKHOUSE_DATATYPES_HPP */

auditd-plugin-clickhouse.cpp

@@ -134,8 +134,9 @@ void auparse_callback(auparse_state_t *au, auparse_cb_event_t cb_event_type, voi
 std::string construct_clickhouse_datatype_string(const std::string &name, const std::string &audit_type)
 {
 	static const std::map<std::string, std::string> audit_table_map = {
-		{ "string", "String" },
+		{ "string", "Nullable(String)" },
-		{ "integer", "Nested( IntValue UInt64, InterpretedValue LowCardinality(String) )" }
+		{ "integer", "Nested( IntValue Nullable(UInt64), InterpretedValue LowCardinality(Nullable(String)) )" },
+		{ "string_array", "Array(String)" }
 	};
 
 	std::string clickhouse_type;
@@ -148,10 +149,10 @@ std::string construct_clickhouse_datatype_string(const std::string &name, const
 	{
 		// Fallback to string
 		fprintf(stderr, "Warning: unknown database type for record name \"%s\"\n", name.c_str());
-		clickhouse_type = "String";
+		clickhouse_type = "Nullable(String)";
 	}
 
-	return name + " Nullable(" + clickhouse_type + ")";
+	return name + " " + clickhouse_type;
 }
 
 int main(int argc, char **argv)

auditd-record.cpp

@@ -20,6 +20,7 @@
 
 #include "auditd-record.hpp"
 
+#include <clickhouse-cpp/columns/array.h>
 #include <clickhouse-cpp/columns/nullable.h>
 #include <clickhouse-cpp/columns/numeric.h>
 #include <clickhouse-cpp/columns/string.h>
@@ -77,22 +78,20 @@ AbstractRecordFieldFactory& AbstractRecordFieldFactory::instance()
 	return instance;
 }
 
-std::shared_ptr<AbstractRecordField> AbstractRecordFieldFactory::createFromAuditRecord(const std::string &name, auparse_state_t *record)
+std::shared_ptr<AbstractRecordField> AbstractRecordFieldFactory::createFromAuditRecord(const std::string &field_name, auparse_type_t field_type)
 {
 	AbstractRecordFieldFactory &inst = instance();
 
-	auparse_type_t record_type = static_cast<auparse_type_t>(auparse_get_field_type(record));
+	auto record_type_factory = inst.m_factoryMap.find(field_type);
-	auto record_type_factory = inst.m_factoryMap.find(record_type);
-
 	if ((record_type_factory != inst.m_factoryMap.end()) && (record_type_factory->second))
 	{
-		return record_type_factory->second(name, record);
+		return record_type_factory->second(field_name);
 	}
 	else
 	{
 		// fallback to string
-		fprintf(stderr, "Warning: unknown record type for record name \"%s\"\n", name.c_str());
+		fprintf(stderr, "Warning: unknown record type for record name \"%s\"\n", field_name.c_str());
-		return InterpretedStringRecordField::createRecord(name, record);
+		return InterpretedStringRecordField::createRecord(field_name);
 	}
 }
 
@@ -142,20 +141,28 @@ void CommonStringRecordField::addToColumn(const std::vector<clickhouse::ColumnRe
 	columns[0]->Append(std::make_shared<clickhouse::ColumnNullable>(data_column, null_column));
 }
 
-std::shared_ptr<AbstractRecordField> StringRecordField::createRecord(const std::string &name, auparse_state_t *record)
+std::shared_ptr<AbstractRecordField> StringRecordField::createRecord(const std::string &name)
 {
 	std::shared_ptr<AbstractRecordField> result;
-	result.reset(new StringRecordField(name, record));
+	result.reset(new StringRecordField(name));
 	return result;
 }
 
-StringRecordField::StringRecordField(const std::string &name, auparse_state_t *record)
+void StringRecordField::addOrUpdateValue(auparse_state_t *record)
-	: CommonStringRecordField(name)
 {
 	if (record)
 	{
 		m_value = auparse_get_field_str(record);
 	}
+	else
+	{
+		m_value = boost::none;
+	}
+}
+
+StringRecordField::StringRecordField(const std::string &name)
+	: CommonStringRecordField(name)
+{
 }
 
 AbstractRecordField::Type StringRecordField::getType() const
@@ -163,20 +170,28 @@ AbstractRecordField::Type StringRecordField::getType() const
 	return AbstractRecordField::Type::String;
 }
 
-std::shared_ptr<AbstractRecordField> InterpretedStringRecordField::createRecord(const std::string &name, auparse_state_t *record)
+std::shared_ptr<AbstractRecordField> InterpretedStringRecordField::createRecord(const std::string &name)
 {
 	std::shared_ptr<AbstractRecordField> result;
-	result.reset(new InterpretedStringRecordField(name, record));
+	result.reset(new InterpretedStringRecordField(name));
 	return result;
 }
 
-InterpretedStringRecordField::InterpretedStringRecordField(const std::string &name, auparse_state_t *record)
+InterpretedStringRecordField::InterpretedStringRecordField(const std::string &name)
 	: CommonStringRecordField(name)
+{
+}
+
+void InterpretedStringRecordField::addOrUpdateValue(auparse_state_t *record)
 {
 	if (record)
 	{
 		m_value = auparse_interpret_field(record);
 	}
+	else
+	{
+		m_value = boost::none;
+	}
 }
 
 AbstractRecordField::Type InterpretedStringRecordField::getType() const
@@ -184,20 +199,30 @@ AbstractRecordField::Type InterpretedStringRecordField::getType() const
 	return AbstractRecordField::Type::InterpretedString;
 }
 
-std::shared_ptr<AbstractRecordField> IntegerRecordField::createRecord(const std::string &name, auparse_state_t *record)
+std::shared_ptr<AbstractRecordField> IntegerRecordField::createRecord(const std::string &name)
 {
 	std::shared_ptr<AbstractRecordField> result;
-	result.reset(new IntegerRecordField(name, record));
+	result.reset(new IntegerRecordField(name));
 	return result;
 }
 
-IntegerRecordField::IntegerRecordField(const std::string &name, auparse_state_t *record)
+IntegerRecordField::IntegerRecordField(const std::string &name)
-	: InterpretedStringRecordField(name, record)
+	: InterpretedStringRecordField(name)
 {
+}
+
+void IntegerRecordField::addOrUpdateValue(auparse_state_t *record)
+{
+	InterpretedStringRecordField::addOrUpdateValue(record);
+	
 	if (record)
 	{
 		m_int_value = auparse_get_field_int(record);
 	}
+	else
+	{
+		m_int_value = boost::none;
+	}
 }
 
 void IntegerRecordField::addToColumn(const std::vector<clickhouse::ColumnRef> &columns) const
@@ -246,3 +271,50 @@ AbstractRecordField::Type IntegerRecordField::getType() const
 {
 	return AbstractRecordField::Type::Int;
 }
+
+std::shared_ptr<AbstractRecordField> InterpretedStringArrayRecordField::createRecord(const std::string &name)
+{
+	std::shared_ptr<AbstractRecordField> result;
+	result.reset(new InterpretedStringArrayRecordField(name));
+	return result;
+}
+
+void InterpretedStringArrayRecordField::addOrUpdateValue(auparse_state_t *record)
+{
+	if (record)
+	{
+		m_value.push_back(auparse_interpret_field(record));
+	}
+}
+
+void InterpretedStringArrayRecordField::addToColumn(const std::vector<clickhouse::ColumnRef> &columns) const
+{
+	if ((columns.size() != 1) || (!columns[0]))
+	{
+		throw std::runtime_error("InterpretedStringArrayRecordField::addToColumn: invalid columns argument");
+	}
+
+	auto array = columns[0]->As<clickhouse::ColumnArray>();
+	if (!array)
+	{
+		throw std::runtime_error("Invalid column type: not ColumnArray");
+	}
+
+	auto data_column = std::make_shared<clickhouse::ColumnString>();
+	for (auto iter = m_value.begin(); iter != m_value.end(); ++iter)
+	{
+		data_column->Append(*iter);
+	}
+
+	array->AppendAsColumn(data_column);
+}
+
+InterpretedStringArrayRecordField::InterpretedStringArrayRecordField(const std::string &name)
+	: AbstractRecordField(name)
+{
+}
+
+AbstractRecordField::Type InterpretedStringArrayRecordField::getType() const
+{
+	return AbstractRecordField::Type::InterpretedStringArray;
+}

auditd-record.hpp

@@ -25,6 +25,7 @@
 #include <map>
 #include <functional>
 #include <string>
+#include <list>
 
 #include <stdint.h>
 
@@ -44,11 +45,13 @@ public:
 	{
 		Int,
 		String,
-		InterpretedString
+		InterpretedString,
+		InterpretedStringArray
 	};
 
 	virtual ~AbstractRecordField() = default;
 
+	virtual void addOrUpdateValue(auparse_state_t *record) = 0;
 	virtual void addToColumn(const std::vector<clickhouse::ColumnRef> &columns) const = 0;
 	virtual Type getType() const = 0;
 
@@ -80,7 +83,7 @@ private:
 class AbstractRecordFieldFactory
 {
 public:
-	std::shared_ptr<AbstractRecordField> createFromAuditRecord(const std::string &name, auparse_state_t *record);
+	std::shared_ptr<AbstractRecordField> createFromAuditRecord(const std::string &field_name, auparse_type_t field_type);
 
 	static AbstractRecordFieldFactory& instance();
 
@@ -100,7 +103,7 @@ protected:
 	AbstractRecordFieldFactory(const AbstractRecordFieldFactory &other) = delete;
 	AbstractRecordFieldFactory& operator=(const AbstractRecordFieldFactory &other) = delete;
 
-	std::map<auparse_type_t, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name, auparse_state_t *record)> > m_factoryMap;
+	std::map<auparse_type_t, std::function<std::shared_ptr<AbstractRecordField>(const std::string &name)> > m_factoryMap;
 };
 
 class CommonStringRecordField: public AbstractRecordField
@@ -117,37 +120,55 @@ protected:
 class StringRecordField: public CommonStringRecordField
 {
 public:
-	static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name, auparse_state_t *record);
+	static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name);
 
+	virtual void addOrUpdateValue(auparse_state_t *record) override;
 	virtual Type getType() const override;
 
 protected:
-	StringRecordField(const std::string &name, auparse_state_t *record);
+	explicit StringRecordField(const std::string &name);
 };
 
 class InterpretedStringRecordField: public CommonStringRecordField
 {
 public:
-	static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name, auparse_state_t *record);
+	static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name);
 
+	virtual void addOrUpdateValue(auparse_state_t *record) override;
 	virtual Type getType() const override;
 
 protected:
-	InterpretedStringRecordField(const std::string &name, auparse_state_t *record);
+	explicit InterpretedStringRecordField(const std::string &name);
 };
 
 class IntegerRecordField: public InterpretedStringRecordField
 {
 public:
-	static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name, auparse_state_t *record);
+	static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name);
 
+	virtual void addOrUpdateValue(auparse_state_t *record) override;
 	virtual void addToColumn(const std::vector<clickhouse::ColumnRef> &columns) const override;
 	virtual Type getType() const override;
 
 protected:
-	IntegerRecordField(const std::string &name, auparse_state_t *record);
+	explicit IntegerRecordField(const std::string &name);
 
 	boost::optional<int> m_int_value;
 };
 
+class InterpretedStringArrayRecordField: public AbstractRecordField
+{
+public:
+	static std::shared_ptr<AbstractRecordField> createRecord(const std::string &name);
+
+	virtual void addOrUpdateValue(auparse_state_t *record) override;
+	virtual void addToColumn(const std::vector<clickhouse::ColumnRef> &columns) const override;
+	virtual Type getType() const override;
+
+protected:
+	explicit InterpretedStringArrayRecordField(const std::string &name);
+
+	std::list<std::string> m_value;
+};
+
 #endif /* AUDITD_PLUGIN_CLICKHOUSE_RECORD_HPP */