diff --git a/WHATS_NEW b/WHATS_NEW index b55292dad..7eafd4158 100644 --- a/WHATS_NEW +++ b/WHATS_NEW @@ -1,5 +1,6 @@ Version 2.02.59 - =================================== + Make cluster log communication structures architecture independant. Fix cluster log issues with in-memory bitmaps. Improve target type compatibility checking in _percent_run(). Add 'target_status_compatible' method to 'struct segtype_handler'. diff --git a/daemons/cmirrord/Makefile.in b/daemons/cmirrord/Makefile.in index debab2746..b33aa6a84 100644 --- a/daemons/cmirrord/Makefile.in +++ b/daemons/cmirrord/Makefile.in @@ -21,7 +21,7 @@ CPG_CFLAGS = @CPG_CFLAGS@ SACKPT_LIBS = @SACKPT_LIBS@ SACKPT_CFLAGS = @SACKPT_CFLAGS@ -SOURCES = clogd.c cluster.c functions.c link_mon.c local.c logging.c +SOURCES = clogd.c cluster.c compat.c functions.c link_mon.c local.c logging.c TARGETS = cmirrord diff --git a/daemons/cmirrord/cluster.c b/daemons/cmirrord/cluster.c index 27d626270..bd2fd2e90 100644 --- a/daemons/cmirrord/cluster.c +++ b/daemons/cmirrord/cluster.c @@ -34,6 +34,8 @@ #include "logging.h" #include "link_mon.h" #include "cluster.h" +#include "compat.h" +#include "xlate.h" /* Open AIS error codes */ #define str_ais_error(x) \ @@ -66,10 +68,6 @@ ((x) == SA_AIS_ERR_NO_SECTIONS) ? "SA_AIS_ERR_NO_SECTIONS" : \ "ais_error_unknown" -#define DM_ULOG_RESPONSE 0x1000 /* in last byte of 32-bit value */ -#define DM_ULOG_CHECKPOINT_READY 21 -#define DM_ULOG_MEMBER_JOIN 22 - #define _RQ_TYPE(x) \ ((x) == DM_ULOG_CHECKPOINT_READY) ? "DM_ULOG_CHECKPOINT_READY": \ ((x) == DM_ULOG_MEMBER_JOIN) ? "DM_ULOG_MEMBER_JOIN": \ @@ -168,6 +166,14 @@ int cluster_send(struct clog_request *rq) iov.iov_base = rq; iov.iov_len = sizeof(struct clog_request) + rq->u_rq.data_size; + rq->u.version[0] = xlate64(CLOG_TFR_VERSION); + rq->u.version[1] = CLOG_TFR_VERSION; + + r = clog_request_to_network(rq); + if (r < 0) + /* FIXME: Better error code for byteswap failure? */ + return -EINVAL; + if (entry->cpg_state != VALID) return -EINVAL; @@ -211,9 +217,9 @@ static struct clog_request *get_matching_rq(struct clog_request *rq, { struct clog_request *match, *n; - dm_list_iterate_items_safe(match, n, l) + dm_list_iterate_items_gen_safe(match, n, l, u.list) if (match->u_rq.seq == rq->u_rq.seq) { - dm_list_del(&match->list); + dm_list_del(&match->u.list); return match; } @@ -298,7 +304,7 @@ static int handle_cluster_response(struct clog_cpg *entry, if (dm_list_empty(&entry->working_list)) LOG_ERROR(" [none]"); - dm_list_iterate_items(orig_rq, &entry->working_list) + dm_list_iterate_items_gen(orig_rq, &entry->working_list, u.list) LOG_ERROR(" [%s] %s:%u", SHORT_UUID(orig_rq->u_rq.uuid), _RQ_TYPE(orig_rq->u_rq.request_type), @@ -578,7 +584,7 @@ rr_create_retry: } memset(rq, 0, sizeof(*rq)); - dm_list_init(&rq->list); + dm_list_init(&rq->u.list); rq->u_rq.request_type = DM_ULOG_CHECKPOINT_READY; rq->originator = cp->requester; /* FIXME: hack to overload meaning of originator */ strncpy(rq->u_rq.uuid, cp->uuid, CPG_MAX_NAME_LENGTH); @@ -803,8 +809,8 @@ static int resend_requests(struct clog_cpg *entry) entry->resend_requests = 0; - dm_list_iterate_items_safe(rq, n, &entry->working_list) { - dm_list_del(&rq->list); + dm_list_iterate_items_gen_safe(rq, n, &entry->working_list, u.list) { + dm_list_del(&rq->u.list); if (strcmp(entry->name.value, rq->u_rq.uuid)) { LOG_ERROR("[%s] Stray request from another log (%s)", @@ -890,8 +896,8 @@ static int flush_startup_list(struct clog_cpg *entry) struct clog_request *rq, *n; struct checkpoint_data *new; - dm_list_iterate_items_safe(rq, n, &entry->startup_list) { - dm_list_del(&rq->list); + dm_list_iterate_items_gen_safe(rq, n, &entry->startup_list, u.list) { + dm_list_del(&rq->u.list); if (rq->u_rq.request_type == DM_ULOG_MEMBER_JOIN) { new = prepare_checkpoint(entry, rq->originator); @@ -945,6 +951,10 @@ static void cpg_message_callback(cpg_handle_t handle, const struct cpg_name *gna struct clog_request *tmp_rq; struct clog_cpg *match; + if (clog_request_from_network(rq, msg_len) < 0) + /* Error message comes from 'clog_request_from_network' */ + return; + match = find_clog_cpg(handle); if (!match) { LOG_ERROR("Unable to find clog_cpg for cluster message"); @@ -968,8 +978,8 @@ static void cpg_message_callback(cpg_handle_t handle, const struct cpg_name *gna return; } memcpy(tmp_rq, rq, sizeof(*rq) + rq->u_rq.data_size); - dm_list_init(&tmp_rq->list); - dm_list_add( &match->working_list, &tmp_rq->list); + dm_list_init(&tmp_rq->u.list); + dm_list_add( &match->working_list, &tmp_rq->u.list); } if (rq->u_rq.request_type == DM_ULOG_POSTSUSPEND) { @@ -990,7 +1000,7 @@ static void cpg_message_callback(cpg_handle_t handle, const struct cpg_name *gna SHORT_UUID(rq->u_rq.uuid), nodeid, (dm_list_empty(&match->working_list)) ? " -- working_list empty": ""); - dm_list_iterate_items(tmp_rq, &match->working_list) + dm_list_iterate_items_gen(tmp_rq, &match->working_list, u.list) LOG_COND(log_resend_requests, "[%s] %s/%u", SHORT_UUID(tmp_rq->u_rq.uuid), @@ -1075,8 +1085,8 @@ static void cpg_message_callback(cpg_handle_t handle, const struct cpg_name *gna memcpy(tmp_rq, rq, sizeof(*rq) + rq->u_rq.data_size); tmp_rq->pit_server = match->lowest_id; - dm_list_init(&tmp_rq->list); - dm_list_add(&match->startup_list, &tmp_rq->list); + dm_list_init(&tmp_rq->u.list); + dm_list_add(&match->startup_list, &tmp_rq->u.list); goto out; } @@ -1206,8 +1216,8 @@ static void cpg_join_callback(struct clog_cpg *match, } rq->u_rq.request_type = DM_ULOG_MEMBER_JOIN; rq->originator = joined->nodeid; - dm_list_init(&rq->list); - dm_list_add(&match->startup_list, &rq->list); + dm_list_init(&rq->u.list); + dm_list_add(&match->startup_list, &rq->u.list); out: /* Find the lowest_id, i.e. the server */ @@ -1256,8 +1266,8 @@ static void cpg_leave_callback(struct clog_cpg *match, cluster_postsuspend(match->name.value, match->luid); - dm_list_iterate_items_safe(rq, n, &match->working_list) { - dm_list_del(&rq->list); + dm_list_iterate_items_gen_safe(rq, n, &match->working_list, u.list) { + dm_list_del(&rq->u.list); if (rq->u_rq.request_type == DM_ULOG_POSTSUSPEND) kernel_send(&rq->u_rq); @@ -1286,13 +1296,13 @@ static void cpg_leave_callback(struct clog_cpg *match, SHORT_UUID(match->name.value), left->nodeid); free_checkpoint(c_cp); } - dm_list_iterate_items_safe(rq, n, &match->startup_list) { + dm_list_iterate_items_gen_safe(rq, n, &match->startup_list, u.list) { if ((rq->u_rq.request_type == DM_ULOG_MEMBER_JOIN) && (rq->originator == left->nodeid)) { LOG_COND(log_checkpoint, "[%s] Removing pending ckpt from startup list (%u is leaving)", SHORT_UUID(match->name.value), left->nodeid); - dm_list_del(&rq->list); + dm_list_del(&rq->u.list); free(rq); } } @@ -1352,7 +1362,7 @@ static void cpg_leave_callback(struct clog_cpg *match, */ i = 1; /* We do not have a DM_ULOG_MEMBER_JOIN entry of our own */ - dm_list_iterate_items(rq, &match->startup_list) + dm_list_iterate_items_gen(rq, &match->startup_list, u.list) if (rq->u_rq.request_type == DM_ULOG_MEMBER_JOIN) i++; @@ -1526,8 +1536,8 @@ static void abort_startup(struct clog_cpg *del) LOG_DBG("[%s] CPG teardown before checkpoint received", SHORT_UUID(del->name.value)); - dm_list_iterate_items_safe(rq, n, &del->startup_list) { - dm_list_del(&rq->list); + dm_list_iterate_items_gen_safe(rq, n, &del->startup_list, u.list) { + dm_list_del(&rq->u.list); LOG_DBG("[%s] Ignoring request from %u: %s", SHORT_UUID(del->name.value), rq->originator, @@ -1640,12 +1650,12 @@ void cluster_debug(void) break; LOG_ERROR(" CKPTs waiting : %d", i); LOG_ERROR(" Working list:"); - dm_list_iterate_items(rq, &entry->working_list) + dm_list_iterate_items_gen(rq, &entry->working_list, u.list) LOG_ERROR(" %s/%u", _RQ_TYPE(rq->u_rq.request_type), rq->u_rq.seq); LOG_ERROR(" Startup list:"); - dm_list_iterate_items(rq, &entry->startup_list) + dm_list_iterate_items_gen(rq, &entry->startup_list, u.list) LOG_ERROR(" %s/%u", _RQ_TYPE(rq->u_rq.request_type), rq->u_rq.seq); diff --git a/daemons/cmirrord/cluster.h b/daemons/cmirrord/cluster.h index 36c3bf955..7e053740d 100644 --- a/daemons/cmirrord/cluster.h +++ b/daemons/cmirrord/cluster.h @@ -15,6 +15,10 @@ #include "libdevmapper.h" #include "dm-log-userspace.h" +#define DM_ULOG_RESPONSE 0x1000 /* in last byte of 32-bit value */ +#define DM_ULOG_CHECKPOINT_READY 21 +#define DM_ULOG_MEMBER_JOIN 22 + /* * There is other information in addition to what can * be found in the dm_ulog_request structure that we @@ -23,7 +27,22 @@ * available. */ struct clog_request { - struct dm_list list; + /* + * If we don't use a union, the structure size will + * vary between 32-bit and 64-bit machines. So, we + * pack two 64-bit version numbers in there to force + * the size of the structure to be the same. + * + * The two version numbers also help us with endian + * issues. The first is always little endian, while + * the second is in native format of the sending + * machine. If the two are equal, there is no need + * to do endian conversions. + */ + union { + uint64_t version[2]; /* LE version and native version */ + struct dm_list list; + } u; /* * 'originator' is the machine from which the requests diff --git a/daemons/cmirrord/common.h b/daemons/cmirrord/common.h index 5498a9ca5..cb70bf96e 100644 --- a/daemons/cmirrord/common.h +++ b/daemons/cmirrord/common.h @@ -13,21 +13,21 @@ #define __CLUSTER_LOG_COMMON_DOT_H__ /* -#define EXIT_SUCCESS 0 -#define EXIT_FAILURE 1 -*/ - + * If there are problems when forking off to become a daemon, + * the child will exist with one of these codes. This allows + * the parent to know the reason for the failure and print it + * to the launching terminal. + * + * #define EXIT_SUCCESS 0 (from stdlib.h) + * #define EXIT_FAILURE 1 (from stdlib.h) + */ #define EXIT_LOCKFILE 2 - #define EXIT_KERNEL_SOCKET 3 /* Failed netlink socket create */ #define EXIT_KERNEL_BIND 4 #define EXIT_KERNEL_SETSOCKOPT 5 - #define EXIT_CLUSTER_CKPT_INIT 6 /* Failed to init checkpoint */ - #define EXIT_QUEUE_NOMEM 7 - #define DM_ULOG_REQUEST_SIZE 1024 #endif /* __CLUSTER_LOG_COMMON_DOT_H__ */ diff --git a/daemons/cmirrord/compat.c b/daemons/cmirrord/compat.c new file mode 100644 index 000000000..0653e38ba --- /dev/null +++ b/daemons/cmirrord/compat.c @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2010 Red Hat, Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the GNU Lesser General Public License v.2.1. + */ +#include +#include +#include +#include +#include "dm-log-userspace.h" +#include "logging.h" +#include "cluster.h" +#include "xlate.h" + +#include "compat.h" + +/* + * Older versions of the log daemon communicate with different + * versions of the inter-machine communication structure, which + * varies in size and fields. The older versions append the + * standard upstream version of the structure to every request. + * COMPAT_OFFSET is where the upstream structure starts. + */ +#define COMPAT_OFFSET 256 + +static void v5_data_endian_switch(struct clog_request *rq, int to_network) +{ + int i, end; + int64_t *pi64; + uint64_t *pu64; + uint32_t rq_type = rq->u_rq.request_type & ~DM_ULOG_RESPONSE; + + if (rq->u_rq.request_type & DM_ULOG_RESPONSE) { + switch (rq_type) { + case DM_ULOG_CTR: + case DM_ULOG_DTR: + LOG_ERROR("Invalid response type in endian switch"); + exit(EXIT_FAILURE); + + case DM_ULOG_PRESUSPEND: + case DM_ULOG_POSTSUSPEND: + case DM_ULOG_RESUME: + case DM_ULOG_FLUSH: + case DM_ULOG_MARK_REGION: + case DM_ULOG_CLEAR_REGION: + case DM_ULOG_SET_REGION_SYNC: + case DM_ULOG_CHECKPOINT_READY: + case DM_ULOG_MEMBER_JOIN: + case DM_ULOG_STATUS_INFO: + case DM_ULOG_STATUS_TABLE: + /* No outbound data */ + break; + + case DM_ULOG_GET_REGION_SIZE: + case DM_ULOG_GET_SYNC_COUNT: + pu64 = (uint64_t *)rq->u_rq.data; + *pu64 = xlate64(*pu64); + break; + case DM_ULOG_IS_CLEAN: + case DM_ULOG_IN_SYNC: + pi64 = (int64_t *)rq->u_rq.data; + *pi64 = xlate64(*pi64); + break; + case DM_ULOG_GET_RESYNC_WORK: + case DM_ULOG_IS_REMOTE_RECOVERING: + pi64 = (int64_t *)rq->u_rq.data; + pu64 = ((uint64_t *)rq->u_rq.data) + 1; + *pi64 = xlate64(*pi64); + *pu64 = xlate64(*pu64); + break; + default: + LOG_ERROR("Unknown request type, %u", rq_type); + return; + } + } else { + switch (rq_type) { + case DM_ULOG_CTR: + case DM_ULOG_DTR: + LOG_ERROR("Invalid request type in endian switch"); + exit(EXIT_FAILURE); + + case DM_ULOG_PRESUSPEND: + case DM_ULOG_POSTSUSPEND: + case DM_ULOG_RESUME: + case DM_ULOG_GET_REGION_SIZE: + case DM_ULOG_FLUSH: + case DM_ULOG_GET_RESYNC_WORK: + case DM_ULOG_GET_SYNC_COUNT: + case DM_ULOG_STATUS_INFO: + case DM_ULOG_STATUS_TABLE: + case DM_ULOG_CHECKPOINT_READY: + case DM_ULOG_MEMBER_JOIN: + /* No incoming data */ + break; + case DM_ULOG_IS_CLEAN: + case DM_ULOG_IN_SYNC: + case DM_ULOG_IS_REMOTE_RECOVERING: + pu64 = (uint64_t *)rq->u_rq.data; + *pu64 = xlate64(*pu64); + break; + case DM_ULOG_MARK_REGION: + case DM_ULOG_CLEAR_REGION: + end = rq->u_rq.data_size/sizeof(uint64_t); + + pu64 = (uint64_t *)rq->u_rq.data; + for (i = 0; i < end; i++) + pu64[i] = xlate64(pu64[i]); + break; + case DM_ULOG_SET_REGION_SYNC: + pu64 = (uint64_t *)rq->u_rq.data; + pi64 = ((int64_t *)rq->u_rq.data) + 1; + *pu64 = xlate64(*pu64); + *pi64 = xlate64(*pi64); + break; + default: + LOG_ERROR("Unknown request type, %u", rq_type); + exit(EXIT_FAILURE); + } + } +} + +static int v5_endian_to_network(struct clog_request *rq) +{ + int size; + struct dm_ulog_request *u_rq = &rq->u_rq; + + size = sizeof(*rq) + u_rq->data_size; + + u_rq->error = xlate32(u_rq->error); + u_rq->seq = xlate32(u_rq->seq); + u_rq->request_type = xlate32(u_rq->request_type); + u_rq->data_size = xlate64(u_rq->data_size); + + rq->originator = xlate32(rq->originator); + + v5_data_endian_switch(rq, 1); + + return size; +} + +int clog_request_to_network(struct clog_request *rq) +{ + int r; + + /* FIXME: Remove this safety check */ + if (rq->u.version[0] != xlate64(rq->u.version[1])) { + LOG_ERROR("Programmer error: version[0] must be LE"); + exit(EXIT_FAILURE); + } + + /* + * Are we already running in the endian mode we send + * over the wire? + */ + if (rq->u.version[0] == rq->u.version[1]) + return 0; + + r = v5_endian_to_network(rq); + if (r < 0) + return r; + return 0; +} + +static int v5_endian_from_network(struct clog_request *rq) +{ + int size; + struct dm_ulog_request *u_rq = &rq->u_rq; + + u_rq->error = xlate32(u_rq->error); + u_rq->seq = xlate32(u_rq->seq); + u_rq->request_type = xlate32(u_rq->request_type); + u_rq->data_size = xlate64(u_rq->data_size); + + rq->originator = xlate32(rq->originator); + + size = sizeof(*rq) + u_rq->data_size; + + v5_data_endian_switch(rq, 0); + + return size; +} + +int clog_request_from_network(void *data, size_t data_len) +{ + uint64_t *vp = data; + uint64_t version = xlate64(vp[0]); + uint64_t unconverted_version = vp[1]; + struct clog_request *rq = data; + + switch (version) { + case 5: /* Upstream */ + if (version == unconverted_version) + return 0; + break; + case 4: /* RHEL 5.[45] */ + case 3: /* RHEL 5.3 */ + case 2: /* RHEL 5.2 */ + /* FIXME: still need to account for payload */ + if (data_len < (COMPAT_OFFSET + sizeof(*rq))) + return -ENOSPC; + + rq = data + COMPAT_OFFSET; + break; + default: + LOG_ERROR("Unable to process cluster message: " + "Incompatible version"); + return -EINVAL; + } + + v5_endian_from_network(rq); + return 0; +} diff --git a/daemons/cmirrord/compat.h b/daemons/cmirrord/compat.h new file mode 100644 index 000000000..fc100ecd0 --- /dev/null +++ b/daemons/cmirrord/compat.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2010 Red Hat, Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the GNU Lesser General Public License v.2.1. + */ +#ifndef __COMPAT_DOT_H__ +#define __COMPAT_DOT_H__ + +/* + * The intermachine communication structure version are: + * 0: Unused + * 1: Never in the wild + * 2: RHEL 5.2 + * 3: RHEL 5.3 + * 4: RHEL 5.4, RHEL 5.5 + * 5: RHEL 6, Current Upstream Format + */ +#define CLOG_TFR_VERSION 5 + +int clog_request_to_network(struct clog_request *rq); +int clog_request_from_network(void *data, size_t data_len); + +#endif /* __COMPAT_DOT_H__ */