MEDIUM: 51d: add support for 51Degrees V4 with Hash algorithm

This patch also adds a set of new global options:

- 51degrees-use-performance-graph { on | off }
- 51degrees-use-predictive-graph { on | off }
- 51degrees-drift <number>
- 51degrees-difference <number>
- 51degrees-allow-unmatched { on | off }

To build using the latest 51Degrees V4 engine with Hash algorithm, set
USE_51DEGREES_V4=1.

Other supported build options are 51DEGREES_INC, 51DEGREES_LIB and
51DEGREES_SRC which needs to be set to the directory that contains
headers and C files. For example:

make TARGET=<target> USE_51DEGREES_V4=1 51DEGREES_SRC='51D_REPO_PATH'/src
This commit is contained in:
Dragan Dosen 2022-02-14 13:05:45 +01:00 committed by Willy Tarreau
parent eaded987ee
commit a9800a0f58
7 changed files with 937 additions and 19 deletions

View File

@ -50,6 +50,7 @@
# USE_PROMEX : enable the Prometheus exporter
# USE_DEVICEATLAS : enable DeviceAtlas api.
# USE_51DEGREES : enable third party device detection library from 51Degrees
# USE_51DEGREES_V4 : enable use of 51Degrees V4 engine with Hash algorithm
# USE_WURFL : enable WURFL detection library from Scientiamobile
# USE_SYSTEMD : enable sd_notify() support.
# USE_OBSOLETE_LINKER : use when the linker fails to emit __start_init/__stop_init
@ -300,7 +301,8 @@ use_opts = USE_EPOLL USE_KQUEUE USE_NETFILTER \
USE_LINUX_SPLICE USE_LIBCRYPT USE_CRYPT_H USE_ENGINE \
USE_GETADDRINFO USE_OPENSSL USE_OPENSSL_WOLFSSL USE_LUA \
USE_ACCEPT4 USE_CLOSEFROM USE_ZLIB USE_SLZ USE_CPU_AFFINITY \
USE_TFO USE_NS USE_DL USE_RT USE_DEVICEATLAS USE_51DEGREES \
USE_TFO USE_NS USE_DL USE_RT \
USE_DEVICEATLAS USE_51DEGREES USE_51DEGREES_V4 \
USE_WURFL USE_SYSTEMD USE_OBSOLETE_LINKER USE_PRCTL USE_PROCCTL \
USE_THREAD_DUMP USE_EVPORTS USE_OT USE_QUIC USE_PROMEX \
USE_MEMORY_PROFILING USE_SHM_OPEN
@ -688,22 +690,44 @@ OPTIONS_CFLAGS += $(if $(DEVICEATLAS_INC),-I$(DEVICEATLAS_INC))
endif
ifneq ($(USE_51DEGREES),)
ifneq ($(USE_51DEGREES_V4),)
$(error cannot compile both 51Degrees V3 and V4 engine support)
endif
endif
ifneq ($(USE_51DEGREES)$(USE_51DEGREES_V4),)
# Use 51DEGREES_SRC and possibly 51DEGREES_INC and 51DEGREES_LIB to force path
# to 51degrees headers and libraries if needed.
51DEGREES_SRC =
51DEGREES_INC = $(51DEGREES_SRC)
51DEGREES_LIB = $(51DEGREES_SRC)
ifneq ($(USE_51DEGREES_V4),)
_51DEGREES_SRC = $(shell find $(51DEGREES_LIB) -maxdepth 2 -name '*.c')
OPTIONS_OBJS += $(_51DEGREES_SRC:%.c=%.o)
else
OPTIONS_OBJS += $(51DEGREES_LIB)/../cityhash/city.o
OPTIONS_OBJS += $(51DEGREES_LIB)/51Degrees.o
endif
OPTIONS_OBJS += addons/51degrees/51d.o
OPTIONS_CFLAGS += $(if $(51DEGREES_INC),-I$(51DEGREES_INC))
ifneq ($(USE_51DEGREES_V4),)
OPTIONS_CFLAGS += -DUSE_51DEGREES_V4
endif
ifeq ($(USE_THREAD),)
OPTIONS_CFLAGS += -DFIFTYONEDEGREES_NO_THREADING
ifneq ($(USE_51DEGREES_V4),)
OPTIONS_CFLAGS += -DFIFTYONE_DEGREES_NO_THREADING
endif
else
ifeq ($(USE_51DEGREES_V4),)
OPTIONS_OBJS += $(51DEGREES_LIB)/../threading.o
endif
endif
OPTIONS_LDFLAGS += $(if $(51DEGREES_LIB),-L$(51DEGREES_LIB)) -lm
ifneq ($(USE_51DEGREES_V4),)
OPTIONS_LDFLAGS += -latomic
endif
endif
ifneq ($(USE_WURFL),)

View File

@ -11,11 +11,19 @@
#include <haproxy/http_ana.h>
#include <haproxy/http_fetch.h>
#include <haproxy/http_htx.h>
#include <haproxy/htx.h>
#include <haproxy/sample.h>
#include <haproxy/thread.h>
#include <haproxy/tools.h>
#include <haproxy/xxhash.h>
#ifdef USE_51DEGREES_V4
#include <hash/hash.h>
#undef MAP_TYPE
#include <hash/fiftyone.h>
#else
#include <51Degrees.h>
#endif
struct _51d_property_names {
struct list list;
@ -29,10 +37,23 @@ static unsigned long long _51d_lru_seed;
__decl_spinlock(_51d_lru_lock);
#endif
#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
#define _51D_HEADERS_BUFFER_SIZE BUFSIZE
static THREAD_LOCAL struct {
char **buf;
int max;
int count;
} _51d_headers;
static THREAD_LOCAL fiftyoneDegreesResultsHash *_51d_results = NULL;
#endif
static struct {
char property_separator; /* the separator to use in the response for the values. this is taken from 51degrees-property-separator from config. */
struct list property_names; /* list of properties to load into the data set. this is taken from 51degrees-property-name-list from config. */
char *data_file_path;
#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
int header_count; /* number of HTTP headers related to device detection. */
struct buffer *header_names; /* array of HTTP header names. */
fiftyoneDegreesDataSet data_set; /* data set used with the pattern and trie detection methods. */
@ -44,6 +65,14 @@ static struct {
#ifdef FIFTYONEDEGREES_NO_THREADING
fiftyoneDegreesDeviceOffsets device_offsets; /* Memory used for device offsets. */
#endif
#endif
#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
fiftyoneDegreesResourceManager manager;
int use_perf_graph;
int use_pred_graph;
int drift;
int difference;
int allow_unmatched;
#endif
int cache_size;
} global_51degrees = {
@ -52,6 +81,14 @@ static struct {
.data_file_path = NULL,
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
.data_set = { },
#endif
#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
.manager = { },
.use_perf_graph = -1,
.use_pred_graph = -1,
.drift = -1,
.difference = -1,
.allow_unmatched = -1,
#endif
.cache_size = 0,
};
@ -313,6 +350,185 @@ unsigned long long _51d_req_hash(const struct arg *args, fiftyoneDegreesWorkset*
}
#endif
#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
static int _51d_use_perf_graph(char **args, int section_type, struct proxy *curpx,
const struct proxy *defpx, const char *file, int line,
char **err)
{
if (too_many_args(1, args, err, NULL))
return -1;
if (strcmp(args[1], "on") == 0)
global_51degrees.use_perf_graph = 1;
else if (strcmp(args[1], "off") == 0)
global_51degrees.use_perf_graph = 0;
else {
memprintf(err, "'%s' expects either 'on' or 'off' but got '%s'.", args[0], args[1]);
return -1;
}
return 0;
}
static int _51d_use_pred_graph(char **args, int section_type, struct proxy *curpx,
const struct proxy *defpx, const char *file, int line,
char **err)
{
if (too_many_args(1, args, err, NULL))
return -1;
if (strcmp(args[1], "on") == 0)
global_51degrees.use_pred_graph = 1;
else if (strcmp(args[1], "off") == 0)
global_51degrees.use_pred_graph = 0;
else {
memprintf(err, "'%s' expects either 'on' or 'off' but got '%s'.", args[0], args[1]);
return -1;
}
return 0;
}
static int _51d_drift(char **args, int section_type, struct proxy *curpx,
const struct proxy *defpx, const char *file, int line,
char **err)
{
if (*(args[1]) == 0) {
memprintf(err, "'%s' expects a positive numeric value.", args[0]);
return -1;
}
global_51degrees.drift = atoi(args[1]);
if (global_51degrees.drift < 0) {
memprintf(err, "'%s' expects a positive numeric value, got '%s'.",
args[0], args[1]);
return -1;
}
return 0;
}
static int _51d_difference(char **args, int section_type, struct proxy *curpx,
const struct proxy *defpx, const char *file, int line,
char **err)
{
if (*(args[1]) == 0) {
memprintf(err, "'%s' expects a positive numeric value.", args[0]);
return -1;
}
global_51degrees.difference = atoi(args[1]);
if (global_51degrees.difference < 0) {
memprintf(err, "'%s' expects a positive numeric value, got '%s'.",
args[0], args[1]);
return -1;
}
return 0;
}
static int _51d_allow_unmatched(char **args, int section_type, struct proxy *curpx,
const struct proxy *defpx, const char *file, int line,
char **err)
{
if (too_many_args(1, args, err, NULL))
return -1;
if (strcmp(args[1], "on") == 0)
global_51degrees.allow_unmatched = 1;
else if (strcmp(args[1], "off") == 0)
global_51degrees.allow_unmatched = 0;
else {
memprintf(err, "'%s' expects either 'on' or 'off' but got '%s'.", args[0], args[1]);
return -1;
}
return 0;
}
static int _51d_init_internal()
{
fiftyoneDegreesDataSetHash *ds;
int hdr_count;
int i, ret = 0;
ds = (fiftyoneDegreesDataSetHash *)fiftyoneDegreesDataSetGet(&global_51degrees.manager);
hdr_count = ds->b.b.uniqueHeaders->count;
if (hdr_count > _51d_headers.max)
hdr_count = _51d_headers.max;
_51d_results = fiftyoneDegreesResultsHashCreate(&global_51degrees.manager, hdr_count, 0);
if (!_51d_results)
goto out;
for (i = 0; i < hdr_count; i++) {
_51d_headers.buf[i] = malloc(_51D_HEADERS_BUFFER_SIZE);
if (!_51d_headers.buf[i])
goto out;
_51d_headers.count++;
}
/* success */
ret = 1;
out:
fiftyoneDegreesDataSetRelease((fiftyoneDegreesDataSetBase *)ds);
return ret;
}
static fiftyoneDegreesEvidenceKeyValuePairArray * _51d_get_evidence(struct sample *smp)
{
fiftyoneDegreesEvidenceKeyValuePairArray *evidence;
fiftyoneDegreesDataSetHash *ds;
size_t size;
struct channel *chn;
struct htx *htx;
struct http_hdr_ctx ctx;
struct ist name;
int i;
chn = (smp->strm ? &smp->strm->req : NULL);
// No need to null check as this has already been carried out in the
// calling method
htx = smp_prefetch_htx(smp, chn, NULL, 1);
ALREADY_CHECKED(htx);
ds = (fiftyoneDegreesDataSetHash *)_51d_results->b.b.dataSet;
size = _51d_headers.count * 2;
evidence = fiftyoneDegreesEvidenceCreate(size);
if (!evidence)
return NULL;
for (i = 0; i < _51d_headers.count; i++) {
fiftyoneDegreesHeader *hdr = &ds->b.b.uniqueHeaders->items[i];
name = ist2(hdr->name, hdr->nameLength);
ctx.blk = NULL;
if (http_find_header(htx, name, &ctx, 1)) {
size_t len = ctx.value.len;
if (unlikely(len >= _51D_HEADERS_BUFFER_SIZE))
len = _51D_HEADERS_BUFFER_SIZE - 1;
memcpy(_51d_headers.buf[i], ctx.value.ptr, len);
_51d_headers.buf[i][len] = '\0';
fiftyoneDegreesEvidenceAddString(
evidence,
FIFTYONE_DEGREES_EVIDENCE_HTTP_HEADER_STRING,
name.ptr,
_51d_headers.buf[i]);
}
}
return evidence;
}
#endif
#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
static void _51d_process_match(const struct arg *args, struct sample *smp, fiftyoneDegreesWorkset* ws)
{
@ -325,11 +541,22 @@ static void _51d_process_match(const struct arg *args, struct sample *smp, fifty
const char **requiredProperties = fiftyoneDegreesGetRequiredPropertiesNames(&global_51degrees.data_set);
int requiredPropertiesCount = fiftyoneDegreesGetRequiredPropertiesCount(&global_51degrees.data_set);
#endif
const char* property_name;
int j;
#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
static void _51d_process_match(const struct arg *args, struct sample *smp)
{
char valuesBuffer[1024];
#endif
char no_data[] = "NoData"; /* response when no data could be found */
struct buffer *temp = get_trash_chunk();
int j, i = 0, found;
const char* property_name;
int i = 0, found;
#if defined(FIFTYONE_DEGREES_HASH_INCLUDED)
FIFTYONE_DEGREES_EXCEPTION_CREATE;
#endif
/* Loop through property names passed to the filter and fetch them from the dataset. */
while (args[i].data.str.area) {
@ -379,6 +606,19 @@ static void _51d_process_match(const struct arg *args, struct sample *smp, fifty
break;
}
}
#endif
#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
FIFTYONE_DEGREES_EXCEPTION_CLEAR;
found = fiftyoneDegreesResultsHashGetValuesString(
_51d_results, args[i].data.str.area,
valuesBuffer, 1024, "|",
exception);
if (FIFTYONE_DEGREES_EXCEPTION_FAILED || found <= 0)
found = 0;
else
chunk_appendf(temp, "%s", valuesBuffer);
#endif
if (!found)
chunk_appendf(temp, "%s", no_data);
@ -414,16 +654,19 @@ static void _51d_set_smp(struct sample *smp)
static int _51d_fetch(const struct arg *args, struct sample *smp, const char *kw, void *private)
{
struct channel *chn;
struct htx *htx;
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
fiftyoneDegreesWorkset* ws; /* workset for detection */
struct lru64 *lru = NULL;
#endif
#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED
fiftyoneDegreesDeviceOffsets *offsets; /* Offsets for detection */
#endif
struct channel *chn;
struct htx *htx;
#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
fiftyoneDegreesEvidenceKeyValuePairArray *evidence = NULL;
FIFTYONE_DEGREES_EXCEPTION_CREATE;
#endif
chn = (smp->strm ? &smp->strm->req : NULL);
htx = smp_prefetch_htx(smp, chn, NULL, 1);
@ -496,6 +739,21 @@ static int _51d_fetch(const struct arg *args, struct sample *smp, const char *kw
_51d_insert_cache_entry(smp, lru, (void*)args);
#endif
#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
evidence = _51d_get_evidence(smp);
if (!evidence)
return 0;
fiftyoneDegreesResultsHashFromEvidence(
_51d_results, evidence, exception);
fiftyoneDegreesEvidenceFree(evidence);
if (FIFTYONE_DEGREES_EXCEPTION_FAILED)
return 0;
_51d_process_match(args, smp);
#endif
_51d_set_smp(smp);
return 1;
}
@ -509,6 +767,9 @@ static int _51d_conv(const struct arg *args, struct sample *smp, void *private)
#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED
fiftyoneDegreesDeviceOffsets *offsets; /* Offsets for detection */
#endif
#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
FIFTYONE_DEGREES_EXCEPTION_CREATE;
#endif
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
@ -540,6 +801,7 @@ static int _51d_conv(const struct arg *args, struct sample *smp, void *private)
smp->data.u.str.area[smp->data.u.str.data] = '\0';
/* Perform detection. */
#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
fiftyoneDegreesMatch(ws, smp->data.u.str.area);
_51d_process_match(args, smp, ws);
@ -570,6 +832,15 @@ static int _51d_conv(const struct arg *args, struct sample *smp, void *private)
#endif
#endif
#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
fiftyoneDegreesResultsHashFromUserAgent(_51d_results, smp->data.u.str.area,
smp->data.u.str.data, exception);
if (FIFTYONE_DEGREES_EXCEPTION_FAILED)
return 0;
_51d_process_match(args, smp);
#endif
_51d_set_smp(smp);
return 1;
}
@ -619,10 +890,18 @@ void _51d_init_http_headers()
static int init_51degrees(void)
{
int i = 0;
struct buffer *temp;
struct _51d_property_names *name;
char **_51d_property_list = NULL;
#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
struct buffer *temp;
fiftyoneDegreesDataSetInitStatus _51d_dataset_status = DATA_SET_INIT_STATUS_NOT_SET;
#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
fiftyoneDegreesConfigHash config = fiftyoneDegreesHashInMemoryConfig;
fiftyoneDegreesPropertiesRequired properties = fiftyoneDegreesPropertiesDefault;
fiftyoneDegreesMemoryReader reader;
fiftyoneDegreesStatusCode status;
FIFTYONE_DEGREES_EXCEPTION_CREATE;
#endif
if (!global_51degrees.data_file_path)
return ERR_NONE;
@ -643,6 +922,7 @@ static int init_51degrees(void)
_51d_property_list[i++] = name->name;
}
#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
_51d_dataset_status = fiftyoneDegreesInitWithPropertyArray(global_51degrees.data_file_path, &global_51degrees.data_set, (const char**)_51d_property_list, i);
temp = get_trash_chunk();
@ -708,6 +988,63 @@ static int init_51degrees(void)
}
#endif
#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
config.b.b.freeData = true;
if (global_51degrees.use_perf_graph != -1)
config.usePerformanceGraph = global_51degrees.use_perf_graph;
if (global_51degrees.use_pred_graph != -1)
config.usePredictiveGraph = global_51degrees.use_pred_graph;
if (global_51degrees.drift > 0)
config.drift = global_51degrees.drift;
if (global_51degrees.difference > 0)
config.difference = global_51degrees.difference;
if (global_51degrees.allow_unmatched != -1)
config.b.allowUnmatched = global_51degrees.allow_unmatched;
config.strings.concurrency =
config.properties.concurrency =
config.values.concurrency =
config.profiles.concurrency =
config.nodes.concurrency =
config.profileOffsets.concurrency =
config.maps.concurrency =
config.components.concurrency =
config.rootNodes.concurrency = global.nbthread;
properties.array = (const char **)_51d_property_list;
properties.count = i;
status = fiftyoneDegreesFileReadToByteArray(global_51degrees.data_file_path, &reader);
if (status == FIFTYONE_DEGREES_STATUS_SUCCESS && !FIFTYONE_DEGREES_EXCEPTION_FAILED) {
FIFTYONE_DEGREES_EXCEPTION_CLEAR;
status = fiftyoneDegreesHashInitManagerFromMemory(
&global_51degrees.manager,
&config,
&properties,
reader.startByte,
reader.length,
exception);
}
free(_51d_property_list);
_51d_property_list = NULL;
i = 0;
if (status != FIFTYONE_DEGREES_STATUS_SUCCESS || FIFTYONE_DEGREES_EXCEPTION_FAILED) {
const char *message = fiftyoneDegreesStatusGetMessage(status, global_51degrees.data_file_path);
if (message)
ha_alert("51Degrees Setup - Error reading 51Degrees data file. %s\n",
message);
else
ha_alert("51Degrees Setup - Error reading 51Degrees data file.\n");
return ERR_ALERT | ERR_FATAL;
}
#endif
return ERR_NONE;
}
@ -715,6 +1052,7 @@ static void deinit_51degrees(void)
{
struct _51d_property_names *_51d_prop_name, *_51d_prop_nameb;
#if defined(FIFTYONEDEGREES_H_PATTERN_INCLUDED) || defined(FIFTYONEDEGREES_H_TRIE_INCLUDED)
free(global_51degrees.header_names);
#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED
if (global_51degrees.pool)
@ -727,6 +1065,7 @@ static void deinit_51degrees(void)
free(global_51degrees.header_offsets);
#endif
fiftyoneDegreesDataSetFree(&global_51degrees.data_set);
#endif
ha_free(&global_51degrees.data_file_path);
list_for_each_entry_safe(_51d_prop_name, _51d_prop_nameb, &global_51degrees.property_names, list) {
@ -739,11 +1078,60 @@ static void deinit_51degrees(void)
#endif
}
#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
static int init_51degrees_per_thread()
{
if (!global_51degrees.data_file_path) {
/* noop */
return 1;
}
_51d_headers.max = global.tune.max_http_hdr;
_51d_headers.buf = calloc(_51d_headers.max, sizeof(*_51d_headers.buf));
_51d_headers.count = 0;
if (!_51d_headers.buf)
return 0;
if (!_51d_init_internal())
return 0;
return 1;
}
static void deinit_51degrees_per_thread()
{
int i;
if (_51d_results) {
fiftyoneDegreesResultsHashFree(_51d_results);
_51d_results = NULL;
}
if (_51d_headers.buf) {
for (i = 0; i < _51d_headers.max; i++)
free(_51d_headers.buf[i]);
free(_51d_headers.buf);
_51d_headers.buf = NULL;
}
_51d_headers.max = 0;
_51d_headers.count = 0;
}
#endif
static struct cfg_kw_list _51dcfg_kws = {{ }, {
{ CFG_GLOBAL, "51degrees-data-file", _51d_data_file },
{ CFG_GLOBAL, "51degrees-property-name-list", _51d_property_name_list },
{ CFG_GLOBAL, "51degrees-property-separator", _51d_property_separator },
{ CFG_GLOBAL, "51degrees-cache-size", _51d_cache_size },
#ifdef FIFTYONE_DEGREES_HASH_INCLUDED
{ CFG_GLOBAL, "51degrees-use-performance-graph", _51d_use_perf_graph },
{ CFG_GLOBAL, "51degrees-use-predictive-graph", _51d_use_pred_graph },
{ CFG_GLOBAL, "51degrees-drift", _51d_drift },
{ CFG_GLOBAL, "51degrees-difference", _51d_difference },
{ CFG_GLOBAL, "51degrees-allow-unmatched", _51d_allow_unmatched },
#endif
{ 0, NULL, NULL },
}};
@ -780,4 +1168,12 @@ REGISTER_POST_DEINIT(deinit_51degrees);
#else
REGISTER_BUILD_OPTS("Built with 51Degrees Trie support (dummy library).");
#endif
#elif defined(FIFTYONE_DEGREES_HASH_INCLUDED)
REGISTER_PER_THREAD_INIT(init_51degrees_per_thread);
REGISTER_PER_THREAD_DEINIT(deinit_51degrees_per_thread);
#ifndef FIFTYONEDEGREES_DUMMY_LIB
REGISTER_BUILD_OPTS("Built with 51Degrees V4 Hash support.");
#else
REGISTER_BUILD_OPTS("Built with 51Degrees V4 Hash support (dummy library).");
#endif
#endif

View File

@ -0,0 +1,34 @@
/* *********************************************************************
* This Original Work is copyright of 51 Degrees Mobile Experts Limited.
* Copyright 2022 51 Degrees Mobile Experts Limited, Davidson House,
* Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
*
* This Original Work is licensed under the European Union Public Licence
* (EUPL) v.1.2 and is subject to its terms as set out below.
*
* If a copy of the EUPL was not distributed with this file, You can obtain
* one at https://opensource.org/licenses/EUPL-1.2.
*
* The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
* amended by the European Commission) shall be deemed incompatible for
* the purposes of the Work and the provisions of the compatibility
* clause in Article 5 of the EUPL shall not apply.
*
* If using the Work as, or as part of, a network application, by
* including the attribution notice(s) required under Article 5 of the EUPL
* in the end user terms of the application under an appropriate heading,
* such notice(s) shall fulfill the requirements of that article.
* ********************************************************************* */
/* *********************************************************************
* Dummy library for HAProxy. This does not function, and is designed
* solely for HAProxy testing purposes.
* *********************************************************************/
#ifndef FIFTYONE_DEGREES_SYNONYM_HASH_INCLUDED
#define FIFTYONE_DEGREES_SYNONYM_HASH_INCLUDED
#ifndef FIFTYONEDEGREES_DUMMY_LIB
#define FIFTYONEDEGREES_DUMMY_LIB
#endif
#endif

View File

@ -0,0 +1,127 @@
/* *********************************************************************
* This Original Work is copyright of 51 Degrees Mobile Experts Limited.
* Copyright 2022 51 Degrees Mobile Experts Limited, Davidson House,
* Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
*
* This Original Work is the subject of the following patents and patent
* applications, owned by 51 Degrees Mobile Experts Limited of 5 Charlotte
* Close, Caversham, Reading, Berkshire, United Kingdom RG4 7BY:
* European Patent No. 3438848; and
* United States Patent No. 10,482,175.
*
* This Original Work is licensed under the European Union Public Licence
* (EUPL) v.1.2 and is subject to its terms as set out below.
*
* If a copy of the EUPL was not distributed with this file, You can obtain
* one at https://opensource.org/licenses/EUPL-1.2.
*
* The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
* amended by the European Commission) shall be deemed incompatible for
* the purposes of the Work and the provisions of the compatibility
* clause in Article 5 of the EUPL shall not apply.
*
* If using the Work as, or as part of, a network application, by
* including the attribution notice(s) required under Article 5 of the EUPL
* in the end user terms of the application under an appropriate heading,
* such notice(s) shall fulfill the requirements of that article.
* ********************************************************************* */
/* *********************************************************************
* Dummy library for HAProxy. This does not function, and is designed
* solely for HAProxy testing purposes.
* *********************************************************************/
#include "hash.h"
#include "fiftyone.h"
static fiftyoneDegreesHeaders dummyHeaders = { };
static fiftyoneDegreesDataSetBase dummyDataSet = { &dummyHeaders };
fiftyoneDegreesDataSetBase* fiftyoneDegreesDataSetGet(
fiftyoneDegreesResourceManager *manager) {
return &dummyDataSet;
}
void fiftyoneDegreesResultsHashFree(
fiftyoneDegreesResultsHash* results) {
return;
}
static fiftyoneDegreesResultsHash dummyResults = { };
fiftyoneDegreesResultsHash* fiftyoneDegreesResultsHashCreate(
fiftyoneDegreesResourceManager *manager,
uint32_t userAgentCapacity,
uint32_t overridesCapacity) {
return &dummyResults;
}
void fiftyoneDegreesDataSetRelease(fiftyoneDegreesDataSetBase *dataSet) {
return;
}
static fiftyoneDegreesEvidenceKeyValuePairArray dummyEvidence = { };
fiftyoneDegreesEvidenceKeyValuePairArray*
fiftyoneDegreesEvidenceCreate(uint32_t capacity) {
return &dummyEvidence;
}
fiftyoneDegreesEvidenceKeyValuePair* fiftyoneDegreesEvidenceAddString(
fiftyoneDegreesEvidenceKeyValuePairArray *evidence,
fiftyoneDegreesEvidencePrefix prefix,
const char *field,
const char *originalValue) {
return NULL;
}
size_t fiftyoneDegreesResultsHashGetValuesString(
fiftyoneDegreesResultsHash* results,
const char *propertyName,
char *buffer,
size_t bufferLength,
const char *separator,
fiftyoneDegreesException *exception) {
return 0;
}
void fiftyoneDegreesResultsHashFromEvidence(
fiftyoneDegreesResultsHash *results,
fiftyoneDegreesEvidenceKeyValuePairArray *evidence,
fiftyoneDegreesException *exception) {
return;
}
void fiftyoneDegreesEvidenceFree(fiftyoneDegreesEvidenceKeyValuePairArray *evidence) {
return;
}
void fiftyoneDegreesResultsHashFromUserAgent(
fiftyoneDegreesResultsHash *results,
const char* userAgent,
size_t userAgentLength,
fiftyoneDegreesException *exception) {
return;
}
fiftyoneDegreesStatusCode fiftyoneDegreesFileReadToByteArray(
const char *fileName,
fiftyoneDegreesMemoryReader *reader) {
return FIFTYONE_DEGREES_STATUS_SUCCESS;
}
fiftyoneDegreesStatusCode
fiftyoneDegreesHashInitManagerFromMemory(
fiftyoneDegreesResourceManager *manager,
fiftyoneDegreesConfigHash *config,
fiftyoneDegreesPropertiesRequired *properties,
void *memory,
long size,
fiftyoneDegreesException *exception) {
return FIFTYONE_DEGREES_STATUS_SUCCESS;
}
const char* fiftyoneDegreesStatusGetMessage(
fiftyoneDegreesStatusCode status,
const char *fileName) {
return NULL;
}

View File

@ -0,0 +1,277 @@
/* *********************************************************************
* This Original Work is copyright of 51 Degrees Mobile Experts Limited.
* Copyright 2022 51 Degrees Mobile Experts Limited, Davidson House,
* Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
*
* This Original Work is the subject of the following patents and patent
* applications, owned by 51 Degrees Mobile Experts Limited of 5 Charlotte
* Close, Caversham, Reading, Berkshire, United Kingdom RG4 7BY:
* European Patent No. 3438848; and
* United States Patent No. 10,482,175.
*
* This Original Work is licensed under the European Union Public Licence
* (EUPL) v.1.2 and is subject to its terms as set out below.
*
* If a copy of the EUPL was not distributed with this file, You can obtain
* one at https://opensource.org/licenses/EUPL-1.2.
*
* The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
* amended by the European Commission) shall be deemed incompatible for
* the purposes of the Work and the provisions of the compatibility
* clause in Article 5 of the EUPL shall not apply.
*
* If using the Work as, or as part of, a network application, by
* including the attribution notice(s) required under Article 5 of the EUPL
* in the end user terms of the application under an appropriate heading,
* such notice(s) shall fulfill the requirements of that article.
* ********************************************************************* */
/* *********************************************************************
* Dummy library for HAProxy. This does not function, and is designed
* solely for HAProxy testing purposes.
* *********************************************************************/
#ifndef FIFTYONE_DEGREES_HASH_INCLUDED
#define FIFTYONE_DEGREES_HASH_INCLUDED
#ifndef FIFTYONEDEGREES_DUMMY_LIB
#define FIFTYONEDEGREES_DUMMY_LIB
#endif
#include <stdlib.h>
#include <inttypes.h>
typedef int bool;
enum { false, true };
typedef unsigned char byte;
typedef enum e_fiftyone_degrees_status_code {
FIFTYONE_DEGREES_STATUS_SUCCESS,
FIFTYONE_DEGREES_STATUS_NOT_SET,
} fiftyoneDegreesStatusCode;
typedef struct fiftyone_degrees_exception_t {
unsigned int status;
} fiftyoneDegreesException;
#define FIFTYONE_DEGREES_EXCEPTION_CLEAR \
exception->status = FIFTYONE_DEGREES_STATUS_NOT_SET;
#define FIFTYONE_DEGREES_EXCEPTION_OKAY \
(exception == NULL || exception->status == FIFTYONE_DEGREES_STATUS_NOT_SET)
#define FIFTYONE_DEGREES_EXCEPTION_FAILED \
(!FIFTYONE_DEGREES_EXCEPTION_OKAY)
#define FIFTYONE_DEGREES_EXCEPTION_CREATE \
fiftyoneDegreesException exceptionValue; \
fiftyoneDegreesException *exception = &exceptionValue; \
FIFTYONE_DEGREES_EXCEPTION_CLEAR
#define FIFTYONE_DEGREES_ARRAY_TYPE(t, m) \
typedef struct fiftyone_degrees_array_##t##_t { \
uint32_t count; \
uint32_t capacity; \
t *items; \
m \
} t##Array;
typedef struct fiftyone_degrees_results_base_t {
void *dataSet;
} fiftyoneDegreesResultsBase;
typedef struct fiftyone_degrees_results_device_detection_t {
fiftyoneDegreesResultsBase b;
} fiftyoneDegreesResultsDeviceDetection;
typedef struct fiftyone_degrees_collection_item_t {
} fiftyoneDegreesCollectionItem;
typedef struct fiftyone_degrees_list_t {
} fiftyoneDegreesList;
typedef struct fiftyone_degrees_evidence_key_value_pair_t {
} fiftyoneDegreesEvidenceKeyValuePair;
#define EVIDENCE_KEY_VALUE_MEMBERS \
struct fiftyone_degrees_array_fiftyoneDegreesEvidenceKeyValuePair_t* pseudoEvidence;
FIFTYONE_DEGREES_ARRAY_TYPE(
fiftyoneDegreesEvidenceKeyValuePair,
EVIDENCE_KEY_VALUE_MEMBERS)
#define FIFTYONE_DEGREES_RESULTS_HASH_MEMBERS \
fiftyoneDegreesResultsDeviceDetection b; \
fiftyoneDegreesCollectionItem propertyItem; \
fiftyoneDegreesList values; \
fiftyoneDegreesEvidenceKeyValuePairArray* pseudoEvidence;
typedef struct fiftyone_degrees_result_hash_t {
} fiftyoneDegreesResultHash;
FIFTYONE_DEGREES_ARRAY_TYPE(
fiftyoneDegreesResultHash,
FIFTYONE_DEGREES_RESULTS_HASH_MEMBERS)
typedef fiftyoneDegreesResultHashArray fiftyoneDegreesResultsHash;
typedef struct fiftyone_degrees_resource_manager_t {
} fiftyoneDegreesResourceManager;
typedef struct fiftyone_degrees_header_t {
const char* name;
size_t nameLength;
} fiftyoneDegreesHeader;
#define FIFTYONE_DEGREES_HEADERS_MEMBERS \
bool expectUpperPrefixedHeaders; \
uint32_t pseudoHeadersCount;
FIFTYONE_DEGREES_ARRAY_TYPE(
fiftyoneDegreesHeader,
FIFTYONE_DEGREES_HEADERS_MEMBERS);
typedef fiftyoneDegreesHeaderArray fiftyoneDegreesHeaders;
typedef struct fiftyone_degrees_dataset_base_t {
fiftyoneDegreesHeaders *uniqueHeaders;
} fiftyoneDegreesDataSetBase;
typedef struct fiftyone_degrees_dataset_device_detection_t {
fiftyoneDegreesDataSetBase b;
} fiftyoneDegreesDataSetDeviceDetection;
typedef struct fiftyone_degrees_dataset_hash_t {
fiftyoneDegreesDataSetDeviceDetection b;
} fiftyoneDegreesDataSetHash;
typedef enum e_fiftyone_degrees_evidence_prefix {
FIFTYONE_DEGREES_EVIDENCE_HTTP_HEADER_STRING = 1 << 0,
FIFTYONE_DEGREES_EVIDENCE_HTTP_HEADER_IP_ADDRESSES = 1 << 1,
FIFTYONE_DEGREES_EVIDENCE_SERVER = 1 << 2,
FIFTYONE_DEGREES_EVIDENCE_QUERY = 1 << 3,
FIFTYONE_DEGREES_EVIDENCE_COOKIE = 1 << 4,
FIFTYONE_DEGREES_EVIDENCE_IGNORE = 1 << 7,
} fiftyoneDegreesEvidencePrefix;
typedef struct fiftyone_degrees_config_base_t {
bool freeData;
} fiftyoneDegreesConfigBase;
typedef struct fiftyone_degrees_config_device_detecton_t {
fiftyoneDegreesConfigBase b;
bool allowUnmatched;
} fiftyoneDegreesConfigDeviceDetection;
typedef struct fiftyone_degrees_collection_config_t {
uint16_t concurrency;
} fiftyoneDegreesCollectionConfig;
typedef struct fiftyone_degrees_config_hash_t {
fiftyoneDegreesConfigDeviceDetection b;
fiftyoneDegreesCollectionConfig strings;
fiftyoneDegreesCollectionConfig components;
fiftyoneDegreesCollectionConfig maps;
fiftyoneDegreesCollectionConfig properties;
fiftyoneDegreesCollectionConfig values;
fiftyoneDegreesCollectionConfig profiles;
fiftyoneDegreesCollectionConfig rootNodes;
fiftyoneDegreesCollectionConfig nodes;
fiftyoneDegreesCollectionConfig profileOffsets;
int32_t difference;
int32_t drift;
bool usePerformanceGraph;
bool usePredictiveGraph;
} fiftyoneDegreesConfigHash;
fiftyoneDegreesConfigHash fiftyoneDegreesHashInMemoryConfig;
typedef struct fiftyone_degrees_property_available_t {
} fiftyoneDegreesPropertyAvailable;
FIFTYONE_DEGREES_ARRAY_TYPE(fiftyoneDegreesPropertyAvailable,)
typedef fiftyoneDegreesPropertyAvailableArray fiftyoneDegreesPropertiesAvailable;
typedef struct fiftyone_degrees_properties_required_t {
const char **array;
int count;
const char *string;
fiftyoneDegreesPropertiesAvailable *existing;
} fiftyoneDegreesPropertiesRequired;
fiftyoneDegreesPropertiesRequired fiftyoneDegreesPropertiesDefault;
typedef struct fiftyone_degrees_memory_reader_t {
byte *startByte;
byte *current;
byte *lastByte;
long length;
} fiftyoneDegreesMemoryReader;
fiftyoneDegreesDataSetBase* fiftyoneDegreesDataSetGet(
fiftyoneDegreesResourceManager *manager);
void fiftyoneDegreesResultsHashFree(
fiftyoneDegreesResultsHash* results);
fiftyoneDegreesResultsHash* fiftyoneDegreesResultsHashCreate(
fiftyoneDegreesResourceManager *manager,
uint32_t userAgentCapacity,
uint32_t overridesCapacity);
void fiftyoneDegreesDataSetRelease(fiftyoneDegreesDataSetBase *dataSet);
fiftyoneDegreesEvidenceKeyValuePairArray* fiftyoneDegreesEvidenceCreate(uint32_t capacity);
fiftyoneDegreesEvidenceKeyValuePair* fiftyoneDegreesEvidenceAddString(
fiftyoneDegreesEvidenceKeyValuePairArray *evidence,
fiftyoneDegreesEvidencePrefix prefix,
const char *field,
const char *originalValue);
size_t fiftyoneDegreesResultsHashGetValuesString(
fiftyoneDegreesResultsHash* results,
const char *propertyName,
char *buffer,
size_t bufferLength,
const char *separator,
fiftyoneDegreesException *exception);
void fiftyoneDegreesResultsHashFromEvidence(
fiftyoneDegreesResultsHash *results,
fiftyoneDegreesEvidenceKeyValuePairArray *evidence,
fiftyoneDegreesException *exception);
void fiftyoneDegreesEvidenceFree(fiftyoneDegreesEvidenceKeyValuePairArray *evidence);
void fiftyoneDegreesResultsHashFromUserAgent(
fiftyoneDegreesResultsHash *results,
const char* userAgent,
size_t userAgentLength,
fiftyoneDegreesException *exception);
fiftyoneDegreesStatusCode fiftyoneDegreesFileReadToByteArray(
const char *fileName,
fiftyoneDegreesMemoryReader *reader);
fiftyoneDegreesStatusCode
fiftyoneDegreesHashInitManagerFromMemory(
fiftyoneDegreesResourceManager *manager,
fiftyoneDegreesConfigHash *config,
fiftyoneDegreesPropertiesRequired *properties,
void *memory,
long size,
fiftyoneDegreesException *exception);
const char* fiftyoneDegreesStatusGetMessage(
fiftyoneDegreesStatusCode status,
const char *fileName);
#endif

View File

@ -19,24 +19,31 @@ official git repository :
git clone https://git.51Degrees.com/Device-Detection.git -b v3.2.10
- or use the new 3.2.12.12 version which continues to receive database
- use newer 3.2.12.12 version which continues to receive database
updates and supports a new Hash Trie algorithm, but which is not
compatible with older Trie databases :
git clone https://github.com/51Degrees/Device-Detection.git -b v3.2.12
then run 'make' with USE_51DEGREES and 51DEGREES_SRC set. Both 51DEGREES_INC
and 51DEGREES_LIB may additionally be used to force specific different paths
for .o and .h, but will default to 51DEGREES_SRC. Make sure to replace
'51D_REPO_PATH' with the path to the 51Degrees repository.
- or use the latest 51Degrees version 4 with 51Degrees Hash algorithm,
not compatible with older databases :
51Degrees provide 3 different detection algorithms:
git clone --recurse-submodules https://github.com/51Degrees/device-detection-cxx.git
then run 'make' with USE_51DEGREES or USE_51DEGREES_V4 (if using 51Degrees
version 4), and 51DEGREES_SRC set. Both 51DEGREES_INC and 51DEGREES_LIB may
additionally be used to force specific different paths for .o and .h, but
will default to 51DEGREES_SRC. Make sure to replace '51D_REPO_PATH' with
the path to the 51Degrees repository.
51Degrees provide 4 different detection algorithms:
1. Pattern - balances main memory usage and CPU.
2. Trie - a very high performance detection solution which uses more main
memory than Pattern.
3. Hash Trie - replaces Trie, 3x faster, 80% lower memory consumption and
tuning options.
4. 51Degrees V4 Hash - only with 51Degrees Device Detection V4.
To make with 51Degrees Pattern algorithm use the following command line.
@ -46,29 +53,38 @@ To use the 51Degrees Trie algorithm use the following command line.
$ make TARGET=<target> USE_51DEGREES=1 51DEGREES_SRC='51D_REPO_PATH'/src/trie
To build with the 51Degrees Device Detection V4 use the following command line.
$ make TARGET=<target> USE_51DEGREES_V4=1 51DEGREES_SRC='51D_REPO_PATH'/src
A data file containing information about devices, browsers, operating systems
and their associated signatures is then needed. 51Degrees provide a free
database with Github repo for this purpose. These free data files are located
in '51D_REPO_PATH'/data with the extensions .dat for Pattern data and .trie for
Trie data. Free Hash Trie data file can be obtained by signing up for a licence
key at https://51degrees.com/products/store/on-premise-device-detection.
If using the 51degrees version 4, the free hash data file is located in
'51D_REPO_PATH'/device-detection-data with the .hash extension.
For HAProxy developers who need to verify that their changes didn't affect the
51Degrees implementation, a dummy library is provided in the
"addons/51degrees/dummy" directory. This does not function, but implements the
API such that the 51Degrees module can be used (but not return any meaningful
information). To test either Pattern or Hash Trie, build with:
information). To test either Pattern or Hash Trie, or the 51Degrees version 4
Hash algorithm, build with:
$ make TARGET=<target> USE_51DEGREES=1 51DEGREES_SRC=addons/51degrees/dummy/pattern
or
$ make TARGET=<target> USE_51DEGREES=1 51DEGREES_SRC=addons/51degrees/dummy/trie
or
$ make TARGET=<target> USE_51DEGREES_V4=1 51DEGREES_SRC=addons/51degrees/dummy/v4hash
respectively.
The configuration file needs to set the following parameters:
global
51degrees-data-file path to the Pattern or Trie data file
51degrees-data-file path to the Pattern, Trie or V4 Hash data file
51degrees-property-name-list list of 51Degrees properties to detect
51degrees-property-separator separator to use between values
51degrees-cache-size LRU-based cache size (disabled by default)
@ -137,6 +153,12 @@ use the correct data.
When used with Trie the Method, Difference and Rank properties are not
available.
When using the 51Degrees V4 Hash algorithm, the hash format data file needs
to be provided as in the following example.
global
51degrees-data-file '51D_REPO_PATH'/device-detection-data/51Degrees-LiteV4.1.hash
The free Lite data file contains information about screen size in pixels and
whether the device is a mobile. A full list of available properties is located
on the 51Degrees web site at:

View File

@ -997,10 +997,15 @@ of them have command-line equivalents.
The following keywords are supported in the "global" section :
* Process management and security
- 51degrees-allow-unmatched
- 51degrees-cache-size
- 51degrees-data-file
- 51degrees-difference
- 51degrees-drift
- 51degrees-property-name-list
- 51degrees-property-separator
- 51degrees-use-performance-graph
- 51degrees-use-predictive-graph
- ca-base
- chroot
- cluster-secret
@ -1169,7 +1174,7 @@ The following keywords are supported in the "global" section :
file should be unzipped and accessible by HAProxy with relevant permissions.
Please note that this option is only available when HAProxy has been
compiled with USE_51DEGREES.
compiled with USE_51DEGREES or USE_51DEGREES_V4.
51degrees-property-name-list [<string> ...]
A list of 51Degrees property names to be load from the dataset. A full list
@ -1177,14 +1182,14 @@ The following keywords are supported in the "global" section :
https://51degrees.com/resources/property-dictionary
Please note that this option is only available when HAProxy has been
compiled with USE_51DEGREES.
compiled with USE_51DEGREES or USE_51DEGREES_V4.
51degrees-property-separator <char>
A char that will be appended to every property value in a response header
containing 51Degrees results. If not set that will be set as ','.
Please note that this option is only available when HAProxy has been
compiled with USE_51DEGREES.
compiled with USE_51DEGREES or USE_51DEGREES_V4.
51degrees-cache-size <number>
Sets the size of the 51Degrees converter cache to <number> entries. This
@ -1192,7 +1197,40 @@ The following keywords are supported in the "global" section :
By default, this cache is disabled.
Please note that this option is only available when HAProxy has been
compiled with USE_51DEGREES.
compiled with USE_51DEGREES or USE_51DEGREES_V4.
51degrees-use-performance-graph { on | off }
Enables ('on') or disables ('off') the use of the performance graph in
the detection process. The default value depends on 51Degrees library.
Please note that this option is only available when HAProxy has been
compiled with USE_51DEGREES_V4.
51degrees-use-predictive-graph { on | off }
Enables ('on') or disables ('off') the use of the predictive graph in
the detection process. The default value depends on 51Degrees library.
Please note that this option is only available when HAProxy has been
compiled with USE_51DEGREES_V4.
51degrees-drift <number>
Sets the drift value that a detection can allow.
Please note that this option is only available when HAProxy has been
compiled with USE_51DEGREES_V4.
51degrees-difference <number>
Sets the difference value that a detection can allow.
Please note that this option is only available when HAProxy has been
compiled with USE_51DEGREES_V4.
51degrees-allow-unmatched { on | off }
Enables ('on') or disables ('off') the use of unmatched nodes in the
detection process. The default value depends on 51Degrees library.
Please note that this option is only available when HAProxy has been
compiled with USE_51DEGREES_V4.
ca-base <dir>
Assigns a default directory to fetch SSL CA certificates and CRLs from when a