From fc6c032d8d3269a7a46c20019039f10c8f38cabf Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 16 Nov 2012 16:12:27 +0100 Subject: [PATCH] MEDIUM: global: add support for CPU binding on Linux ("cpu-map") The new "cpu-map" directive allows one to assign the CPU sets that a process is allowed to bind to. This is useful in combination with the "nbproc" and "bind-process" directives. The support is implicit on Linux 2.6.28 and above. --- Makefile | 7 +++++ doc/configuration.txt | 15 +++++++++ include/types/global.h | 3 ++ src/cfgparse.c | 69 ++++++++++++++++++++++++++++++++++++++++++ src/haproxy.c | 14 ++++++++- 5 files changed, 107 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f13848c47..9c7a75f23 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ # USE_ACCEPT4 : enable use of accept4() on linux. Automatic. # USE_MY_ACCEPT4 : use own implemention of accept4() if glibc < 2.10. # USE_ZLIB : enable zlib library support. +# USE_CPU_AFFINITY : enable pinning processes to CPU on Linux. Automatic. # # Options can be forced by specifying "USE_xxx=1" or can be disabled by using # "USE_xxx=" (empty string). @@ -241,6 +242,7 @@ ifeq ($(TARGET),linux2628) USE_LINUX_TPROXY= implicit USE_ACCEPT4 = implicit USE_FUTEX = implicit + USE_CPU_AFFINITY= implicit else ifeq ($(TARGET),solaris) # This is for Solaris 8 @@ -437,6 +439,11 @@ OPTIONS_CFLAGS += -DCONFIG_HAP_LINUX_VSYSCALL BUILD_OPTIONS += $(call ignore_implicit,USE_VSYSCALL) endif +ifneq ($(USE_CPU_AFFINITY),) +OPTIONS_CFLAGS += -DUSE_CPU_AFFINITY +BUILD_OPTIONS += $(call ignore_implicit,USE_CPU_AFFINITY) +endif + ifneq ($(USE_MY_SPLICE),) OPTIONS_CFLAGS += -DUSE_MY_SPLICE BUILD_OPTIONS += $(call ignore_implicit,USE_MY_SPLICE) diff --git a/doc/configuration.txt b/doc/configuration.txt index 4d250155d..84a3f9a48 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -499,6 +499,21 @@ chroot with superuser privileges. It is important to ensure that is both empty and unwritable to anyone. +cpu-map <"all"|"odd"|"even"|process_num> ... + On Linux 2.6 and above, it is possible to bind a process to a specific CPU + set. This means that the process will never run on other CPUs. The "cpu-map" + directive specifies CPU sets for process sets. The first argument is the + process number to bind. This process must have a number between 1 and 32, + and any process IDs above nbproc are ignored. It is possible to specify all + processes at once using "all", only odd numbers using "odd" or even numbers + using "even", just like with the "bind-process" directive. The second and + forthcoming arguments are CPU sets. Each CPU set is either a unique number + between 0 and 31 or a range with two such numbers delimited by a dash ('-'). + Multiple CPU numbers or ranges may be specified, and the processes will be + allowed to bind to all of them. Obviously, multiple "cpu-map" directives may + be specified. Each "cpu-map" directive will replace the previous ones when + they overlap. + crt-base Assigns a default directory to fetch SSL certificates from when a relative path is used with "crtfile" directives. Absolute locations specified after diff --git a/include/types/global.h b/include/types/global.h index 9d2eedf4d..3cd077245 100644 --- a/include/types/global.h +++ b/include/types/global.h @@ -130,6 +130,9 @@ struct global { int level; /* access level (ACCESS_LVL_*) */ } ux; } unix_bind; +#ifdef USE_CPU_AFFINITY + unsigned long cpu_map[32]; /* list of CPU masks for the 32 first processes */ +#endif struct proxy *stats_fe; /* the frontend holding the stats settings */ }; diff --git a/src/cfgparse.c b/src/cfgparse.c index 13363dbcc..569fcd354 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -1160,6 +1160,75 @@ int cfg_parse_global(const char *file, int linenum, char **args, int kwm) err_code |= ERR_ALERT | ERR_FATAL; } } + else if (strcmp(args[0], "cpu-map") == 0) { /* map a process list to a CPU set */ +#ifdef USE_CPU_AFFINITY + int cur_arg, i; + unsigned int proc = 0; + unsigned long cpus = 0; + + if (strcmp(args[1], "all") == 0) + proc = 0xFFFFFFFF; + else if (strcmp(args[1], "odd") == 0) + proc = 0x55555555; + else if (strcmp(args[1], "even") == 0) + proc = 0xAAAAAAAA; + else { + proc = atoi(args[1]); + if (proc >= 1 && proc <= 32) + proc = 1 << (proc - 1); + } + + if (!proc || !*args[2]) { + Alert("parsing [%s:%d]: %s expects a process number including 'all', 'odd', 'even', or a number from 1 to 32, followed by a list of CPU ranges with numbers from 0 to 31.\n", + file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + + cur_arg = 2; + while (*args[cur_arg]) { + unsigned int low, high; + + if (isdigit(*args[cur_arg])) { + char *dash = strchr(args[cur_arg], '-'); + + low = high = str2uic(args[cur_arg]); + if (dash) + high = str2uic(dash + 1); + + if (high < low) { + unsigned int swap = low; + low = high; + high = swap; + } + + if (low < 0 || high >= sizeof(long) * 8) { + Alert("parsing [%s:%d]: %s supports CPU numbers from 0 to %d.\n", + file, linenum, args[0], (int)(sizeof(long) * 8 - 1)); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + + while (low <= high) + cpus |= 1UL << low++; + } + else { + Alert("parsing [%s:%d]: %s : '%s' is not a CPU range.\n", + file, linenum, args[0], args[cur_arg]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + cur_arg++; + } + for (i = 0; i < 32; i++) + if (proc & (1 << i)) + global.cpu_map[i] = cpus; +#else + Alert("parsing [%s:%d] : '%s' is not enabled, please check build options for USE_CPU_AFFINITY.\n", file, linenum, args[0]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; +#endif + } else { struct cfg_kw_list *kwl; int index; diff --git a/src/haproxy.c b/src/haproxy.c index c4122e2d7..20fb56d75 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -1,6 +1,6 @@ /* * HA-Proxy : High Availability-enabled HTTP/TCP proxy - * Copyright 2000-2011 Willy Tarreau . + * Copyright 2000-2012 Willy Tarreau . * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -44,6 +44,11 @@ #include #include #include +#ifdef USE_CPU_AFFINITY +#define __USE_GNU +#include +#undef __USE_GNU +#endif #ifdef DEBUG_FULL #include @@ -1467,6 +1472,13 @@ int main(int argc, char **argv) } relative_pid++; /* each child will get a different one */ } + +#ifdef USE_CPU_AFFINITY + if (proc < global.nbproc && /* child */ + proc < 32 && /* only the first 32 processes may be pinned */ + global.cpu_map[proc]) /* only do this if the process has a CPU map */ + sched_setaffinity(0, sizeof(unsigned long), (void *)&global.cpu_map[proc]); +#endif /* close the pidfile both in children and father */ if (pidfd >= 0) { //lseek(pidfd, 0, SEEK_SET); /* debug: emulate eglibc bug */