andy/fence-virt
1
0
Fork 0
forked from altcloud/fence-virt
Code Issues Pull Requests Releases Wiki Activity

Make cluster mode plugin work

Browse Source 
* Need to fix cpg_get_ids() - it is segfaulting
* Need to reconcile "in-cluster" vs "fenced"
* Message routing works.  It doesn't matter which node
  gets the fence request - fence requests are handled
  after being distributed by cpg.
* This makes fence_virtd 99% behavior-like compared to
  fence_xvmd

Signed-off-by: Lon Hohberger <lhh@redhat.com>
This commit is contained in:
Lon Hohberger 2009-09-16 20:07:46 -04:00
parent 01a1b16d7e
commit 76331b0c3c
6 changed files with 916 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/xvm.h
View File

@ -55,7 +55,8 @@ typedef enum {
FENCE_REBOOT = 0x2, /* Hit the reset button */ FENCE_REBOOT = 0x2, /* Hit the reset button */
FENCE_ON = 0x3, /* Turn the VM on */ FENCE_ON = 0x3, /* Turn the VM on */
FENCE_STATUS = 0x4, /* virtual machine status (off/on) */ FENCE_STATUS = 0x4, /* virtual machine status (off/on) */
FENCE_DEVSTATUS = 0x5 /* List virtual machines */ FENCE_DEVSTATUS = 0x5, /* Status of the fencing device */
FENCE_HOSTLIST = 0x6 /* List VMs controllable */
} fence_cmd_t; } fence_cmd_t;
#define DEFAULT_TTL 4 #define DEFAULT_TTL 4

650
server/checkpoint.c
View File

@ -28,8 +28,11 @@
#include <server_plugin.h> #include <server_plugin.h>
#include <string.h> #include <string.h>
#include <malloc.h> #include <malloc.h>
#include <syslog.h>
#include <errno.h> #include <errno.h>
#include <unistd.h>
#include <libvirt/libvirt.h> #include <libvirt/libvirt.h>
#include <pthread.h>
#ifdef HAVE_OPENAIS_CPG_H #ifdef HAVE_OPENAIS_CPG_H
#include <openais/cpg.h> #include <openais/cpg.h>
#else #else
@ -38,6 +41,13 @@
#endif #endif
#endif #endif
#include <libcman.h>
#include <debug.h>
#include "virt.h"
#include "xvm.h"
#include "checkpoint.h"
#define NAME "checkpoint" #define NAME "checkpoint"
#define VERSION "0.8" #define VERSION "0.8"
@ -47,8 +57,6 @@
struct check_info { struct check_info {
int magic; int magic;
int pad; int pad;
cpg_handle_t h;
virConnectPtr vp;
}; };
#define VALIDATE(arg) \ #define VALIDATE(arg) \
@ -60,8 +68,531 @@ do {\
} while(0) } while(0)
static void *checkpoint_handle = NULL;
static virt_list_t *local_vms = NULL;
static char *uri = NULL;
static int use_uuid = 0;
static int static int
ckpt_null(const char *vm_name, void *priv) virt_list_update(virConnectPtr vp, virt_list_t **vl, int my_id)
{
virt_list_t *list = NULL;
list = vl_get(vp, my_id);
if (!list)
return -1;
if (*vl)
vl_free(*vl);
*vl = list;
return 0;
}
static int
get_cman_ids(cman_handle_t ch, uint32_t *my_id, uint32_t *high_id)
{
int max_nodes;
int actual;
cman_node_t *nodes = NULL;
cman_node_t me;
uint32_t high = 0;
int ret = -1, x, _local = 0;
if (!my_id && !high_id)
return 0;
if (!ch) {
_local = 1;
ch = cman_init(NULL);
}
if (!ch)
return -1;
max_nodes = cman_get_node_count(ch);
if (max_nodes <= 0)
goto out;
if (my_id) {
memset(&me, 0, sizeof(me));
if (cman_get_node(ch, CMAN_NODEID_US, &me) < 0)
goto out;
*my_id = me.cn_nodeid;
}
if (!high_id) {
ret = 0;
goto out;
}
nodes = malloc(sizeof(cman_node_t) * max_nodes);
if (!nodes)
goto out;
memset(nodes, 0, sizeof(cman_node_t) * max_nodes);
if (cman_get_nodes(ch, max_nodes, &actual, nodes) < 0)
goto out;
for (x = 0; x < actual; x++)
if (nodes[x].cn_nodeid > high && nodes[x].cn_member)
high = nodes[x].cn_nodeid;
*high_id = high;
ret = 0;
out:
if (nodes)
free(nodes);
if (ch && _local)
cman_finish(ch);
return ret;
}
static int
node_operational(uint32_t nodeid)
{
cman_handle_t ch;
cman_node_t node;
ch = cman_init(NULL);
if (!ch)
return -1;
memset(&node, 0, sizeof(node));
if (cman_get_node(ch, nodeid, &node) == 0) {
cman_finish(ch);
return !!node.cn_member;
}
cman_finish(ch);
return 0;
}
static int
get_domain_state_ckpt(void *hp, const char *domain, vm_state_t *state)
{
errno = EINVAL;
if (!hp || !domain || !state || !strlen((char *)domain))
return -1;
if (!strcmp(DOMAIN0NAME, (char *)domain))
return -1;
return ckpt_read(hp, domain, state, sizeof(*state));
}
static inline int
wait_domain(const char *vm_name, virConnectPtr vp, int timeout)
{
int tries = 0;
int response = 1;
virDomainPtr vdp;
virDomainInfo vdi;
if (use_uuid) {
vdp = virDomainLookupByUUID(vp, (const unsigned char *)vm_name);
} else {
vdp = virDomainLookupByName(vp, vm_name);
}
if (!vdp)
return 0;
/* Check domain liveliness. If the domain is still here,
we return failure, and the client must then retry */
/* XXX On the xen 3.0.4 API, we will be able to guarantee
synchronous virDomainDestroy, so this check will not
be necessary */
do {
sleep(1);
if (use_uuid) {
vdp = virDomainLookupByUUID(vp,
(const unsigned char *)vm_name);
} else {
vdp = virDomainLookupByName(vp, vm_name);
}
if (!vdp) {
dbg_printf(2, "Domain no longer exists\n");
response = 0;
break;
}
memset(&vdi, 0, sizeof(vdi));
virDomainGetInfo(vdp, &vdi);
virDomainFree(vdp);
if (vdi.state == VIR_DOMAIN_SHUTOFF) {
dbg_printf(2, "Domain has been shut off\n");
response = 0;
break;
}
dbg_printf(4, "Domain still exists (state %d) "
"after %d seconds\n",
vdi.state, tries);
if (++tries >= timeout)
break;
} while (1);
return response;
}
/*
Returns: 0 - operational
1 - dead or presumed so
2 - VM not local and I am not the right node to deal with it
3 - VM status unknown; cannot operate on it
*/
static int
cluster_virt_status(const char *vm_name, uint32_t *owner)
{
vm_state_t chk_state;
virt_state_t *vs;
uint32_t me, high_id;
int ret = 0;
dbg_printf(80, "%s %s\n", __FUNCTION__, vm_name);
/* if we can't find the high ID, we can't do anything useful */
/* This should be cpg_get_ids() but it's segfaulting for some
reason :( */
if (get_cman_ids(NULL, &me, &high_id) != 0)
return 2;
if (use_uuid) {
vs = vl_find_uuid(local_vms, vm_name);
} else {
vs = vl_find_name(local_vms, vm_name);
}
if (!vs) {
ret = 2; /* not found locally */
if (me != high_id)
goto out;
if (get_domain_state_ckpt(checkpoint_handle,
vm_name, &chk_state)) {
dbg_printf(2, "High ID: Unknown VM\n");
ret = 3;
goto out;
}
if (node_operational(chk_state.s_owner)) {
*owner = chk_state.s_owner;
dbg_printf(2, "High ID: Owner is operational\n");
ret = 2;
} else {
dbg_printf(2, "High ID: Owner is dead; returning 'off'\n");
ret = 1;
}
} else if (vs->v_state.s_state == VIR_DOMAIN_SHUTOFF) {
ret = 1; /* local and off */
}
out:
dbg_printf(80, "%s %s\n", __FUNCTION__, vm_name);
return ret;
}
static void
store_domains_by_name(void *hp, virt_list_t *vl)
{
int x;
if (!vl)
return;
for (x = 0; x < vl->vm_count; x++) {
if (!strcmp(DOMAIN0NAME, vl->vm_states[x].v_name))
continue;
dbg_printf(2, "Storing %s\n", vl->vm_states[x].v_name);
ckpt_write(hp, vl->vm_states[x].v_name,
&vl->vm_states[x].v_state,
sizeof(vm_state_t));
}
}
static void
store_domains_by_uuid(void *hp, virt_list_t *vl)
{
int x;
if (!vl)
return;
for (x = 0; x < vl->vm_count; x++) {
if (!strcmp(DOMAIN0UUID, vl->vm_states[x].v_uuid))
continue;
dbg_printf(2, "Storing %s\n", vl->vm_states[x].v_uuid);
ckpt_write(hp, vl->vm_states[x].v_uuid,
&vl->vm_states[x].v_state,
sizeof(vm_state_t));
}
}
static void
update_local_vms(void)
{
virConnectPtr vp = NULL;
uint32_t my_id = 0;
cpg_get_ids(&my_id, NULL);
vp = virConnectOpen(uri);
if (!vp) {
syslog(LOG_ERR, "Failed to connect to hypervisor\n");
}
virt_list_update(vp, &local_vms, my_id);
vl_print(local_vms);
if (use_uuid)
store_domains_by_uuid(checkpoint_handle, local_vms);
else
store_domains_by_name(checkpoint_handle, local_vms);
virConnectClose(vp);
}
static int
do_off(const char *vm_name)
{
virConnectPtr vp;
virDomainPtr vdp;
virDomainInfo vdi;
int ret = -1;
dbg_printf(5, "%s %s\n", __FUNCTION__, vm_name);
vp = virConnectOpen(uri);
if (!vp)
return 1;
if (use_uuid) {
vdp = virDomainLookupByUUID(vp,
(const unsigned char *)vm_name);
} else {
vdp = virDomainLookupByName(vp, vm_name);
}
if (!vdp ||
((virDomainGetInfo(vdp, &vdi) == 0) &&
(vdi.state == VIR_DOMAIN_SHUTOFF))) {
dbg_printf(2, "Nothing to do - domain does not exist\n");
if (vdp)
virDomainFree(vdp);
return 0;
}
syslog(LOG_NOTICE, "Destroying domain %s\n", vm_name);
dbg_printf(2, "[OFF] Calling virDomainDestroy\n");
ret = virDomainDestroy(vdp);
if (ret < 0) {
syslog(LOG_NOTICE, "Failed to destroy domain: %d\n", ret);
printf("virDomainDestroy() failed: %d\n", ret);
ret = 1;
goto out;
}
if (ret) {
syslog(LOG_NOTICE,
"Domain %s still exists; fencing failed\n",
vm_name);
printf("Domain %s still exists; fencing failed\n", vm_name);
ret = 1;
goto out;
}
ret = 0;
out:
virConnectClose(vp);
return ret;
}
static int
do_reboot(const char *vm_name)
{
virConnectPtr vp;
virDomainPtr vdp, nvdp;
virDomainInfo vdi;
char *domain_desc;
int ret;
//uuid_unparse(vm_uuid, uu_string);
dbg_printf(5, "%s %s\n", __FUNCTION__, vm_name);
vp = virConnectOpen(uri);
if (!vp)
return 1;
if (use_uuid) {
vdp = virDomainLookupByUUID(vp,
(const unsigned char *)vm_name);
} else {
vdp = virDomainLookupByName(vp, vm_name);
}
if (!vdp || ((virDomainGetInfo(vdp, &vdi) == 0) &&
(vdi.state == VIR_DOMAIN_SHUTOFF))) {
dbg_printf(2, "[libvirt:REBOOT] Nothing to "
"do - domain does not exist\n");
if (vdp)
virDomainFree(vdp);
return 0;
}
syslog(LOG_NOTICE, "Rebooting domain %s\n", vm_name);
printf("Rebooting domain %s...\n", vm_name);
domain_desc = virDomainGetXMLDesc(vdp, 0);
if (!domain_desc) {
printf("Failed getting domain description from "
"libvirt\n");
}
dbg_printf(2, "[REBOOT] Calling virDomainDestroy(%p)\n", vdp);
ret = virDomainDestroy(vdp);
if (ret < 0) {
printf("virDomainDestroy() failed: %d/%d\n", ret, errno);
free(domain_desc);
virDomainFree(vdp);
ret = 1;
goto out;
}
ret = wait_domain(vm_name, vp, 15);
if (ret) {
syslog(LOG_NOTICE, "Domain %s still exists; fencing failed\n",
vm_name);
printf("Domain %s still exists; fencing failed\n", vm_name);
if (domain_desc)
free(domain_desc);
ret = 1;
goto out;
}
if (!domain_desc) {
ret = 0;
goto out;
}
/* 'on' is not a failure */
ret = 0;
dbg_printf(3, "[[ XML Domain Info ]]\n");
dbg_printf(3, "%s\n[[ XML END ]]\n", domain_desc);
dbg_printf(2, "Calling virDomainCreateLinux()...\n");
nvdp = virDomainCreateLinux(vp, domain_desc, 0);
if (nvdp == NULL) {
/* More recent versions of libvirt or perhaps the
* KVM back-end do not let you create a domain from
* XML if there is already a defined domain description
* with the same name that it knows about. You must
* then call virDomainCreate() */
dbg_printf(2, "Failed; Trying virDomainCreate()...\n");
if (virDomainCreate(vdp) < 0) {
syslog(LOG_NOTICE,
"Could not restart %s\n",
vm_name);
dbg_printf(1, "Failed to recreate guest"
" %s!\n", vm_name);
}
}
free(domain_desc);
out:
virConnectClose(vp);
return ret;
}
static void
do_real_work(void *data, size_t len, uint32_t nodeid, uint32_t seqno)
{
struct ckpt_fence_req *req = data;
struct ckpt_fence_req reply;
uint32_t owner;
int ret;
memcpy(&reply, req, sizeof(reply));
update_local_vms();
switch(req->request) {
case FENCE_STATUS:
ret = cluster_virt_status(req->vm_name, &owner);
if (ret == 2) {
return;
}
break;
case FENCE_OFF:
ret = cluster_virt_status(req->vm_name, &owner);
if (ret != 0) {
return;
}
/* Must be running locally to perform 'off' */
ret = do_off(req->vm_name);
break;
case FENCE_REBOOT:
ret = cluster_virt_status(req->vm_name, &owner);
if (ret != 0) {
return;
}
/* Must be running locally to perform 'reboot' */
ret = do_reboot(req->vm_name);
break;
}
cpg_send_reply(&reply, sizeof(reply), nodeid, seqno);
}
static int
do_request(const char *vm_name, int request, uint32_t seqno)
{
struct ckpt_fence_req freq, *frp;
size_t retlen;
uint32_t seq;
int ret;
memset(&freq, 0, sizeof(freq));
snprintf(freq.vm_name, sizeof(freq.vm_name), vm_name);
freq.request = request;
freq.seqno = seqno;
if (cpg_send_req(&freq, sizeof(freq), &seq) != 0) {
printf("Failed to send\n");
return 1;
}
if (cpg_wait_reply((void *)&frp, &retlen, seq) != 0) {
printf("Failed to receive\n");
return 1;
}
ret = frp->response;
free(frp);
return ret;
}
static int
checkpoint_null(const char *vm_name, void *priv)
{ {
VALIDATE(priv); VALIDATE(priv);
printf("[CKPT] Null operation on %s\n", vm_name); printf("[CKPT] Null operation on %s\n", vm_name);
@ -71,17 +602,17 @@ ckpt_null(const char *vm_name, void *priv)
static int static int
ckpt_off(const char *vm_name, uint32_t seqno, void *priv) checkpoint_off(const char *vm_name, uint32_t seqno, void *priv)
{ {
VALIDATE(priv); VALIDATE(priv);
printf("[CKPT] OFF operation on %s seq %d\n", vm_name, seqno); printf("[CKPT] OFF operation on %s seq %d\n", vm_name, seqno);
return 1; return do_request(vm_name, FENCE_OFF, seqno);
} }
static int static int
ckpt_on(const char *vm_name, uint32_t seqno, void *priv) checkpoint_on(const char *vm_name, uint32_t seqno, void *priv)
{ {
VALIDATE(priv); VALIDATE(priv);
printf("[CKPT] ON operation on %s seq %d\n", vm_name, seqno); printf("[CKPT] ON operation on %s seq %d\n", vm_name, seqno);
@ -91,7 +622,7 @@ ckpt_on(const char *vm_name, uint32_t seqno, void *priv)
static int static int
ckpt_devstatus(void *priv) checkpoint_devstatus(void *priv)
{ {
printf("[CKPT] Device status\n"); printf("[CKPT] Device status\n");
VALIDATE(priv); VALIDATE(priv);
@ -101,31 +632,76 @@ ckpt_devstatus(void *priv)
static int static int
ckpt_status(const char *vm_name, void *priv) checkpoint_status(const char *vm_name, void *priv)
{ {
VALIDATE(priv); VALIDATE(priv);
printf("[CKPT] STATUS operation on %s\n", vm_name); printf("[CKPT] STATUS operation on %s\n", vm_name);
return 1; return do_request(vm_name, FENCE_STATUS, 0);
} }
static int static int
ckpt_reboot(const char *vm_name, uint32_t seqno, void *priv) checkpoint_reboot(const char *vm_name, uint32_t seqno, void *priv)
{ {
VALIDATE(priv); VALIDATE(priv);
printf("[CKPT] REBOOT operation on %s seq %d\n", vm_name, seqno); printf("[CKPT] REBOOT operation on %s seq %d\n", vm_name, seqno);
return 1; return do_request(vm_name, FENCE_REBOOT, 0);
} }
static int static int
ckpt_init(backend_context_t *c, config_object_t *config) checkpoint_init(backend_context_t *c, config_object_t *config)
{ {
//char value[256]; char value[1024];
struct check_info *info = NULL; struct check_info *info = NULL;
int x;
#ifdef _MODULE
if (sc_get(config, "fence_virtd/@debug", value, sizeof(value))==0)
dset(atoi(value));
#endif
if (sc_get(config, "backends/libvirt/@uri",
value, sizeof(value)) == 0) {
uri = strdup(value);
if (!uri) {
free(info);
return -1;
}
dbg_printf(1, "Using %s\n", uri);
}
if (sc_get(config, "backends/checkpoint/@uri",
value, sizeof(value)) == 0) {
if (uri)
free(uri);
uri = strdup(value);
if (!uri) {
free(info);
return -1;
}
dbg_printf(1, "Using %s\n", uri);
}
/* Naming scheme is a top-level configuration option */
if (sc_get(config, "fence_virtd/@name_mode",
value, sizeof(value)-1) == 0) {
dbg_printf(1, "Got %s for name_mode\n", value);
if (!strcasecmp(value, "uuid")) {
use_uuid = 1;
} else if (!strcasecmp(value, "name")) {
use_uuid = 0;
} else {
dbg_printf(1, "Unsupported name_mode: %s\n", value);
}
}
if (cpg_start(PACKAGE_NAME, do_real_work) < 0) {
return -1;
}
info = malloc(sizeof(*info)); info = malloc(sizeof(*info));
if (!info) if (!info)
return -1; return -1;
@ -134,13 +710,29 @@ ckpt_init(backend_context_t *c, config_object_t *config)
info->magic = MAGIC; info->magic = MAGIC;
x = 0;
while ((checkpoint_handle = ckpt_init(
"vm_states", 262144, 4096, 64, 10
)) == NULL) {
if (!x) {
dbg_printf(1, "Could not initialize "
"saCkPt; retrying...\n");
x = 1;
}
sleep(3);
}
if (x)
dbg_printf(1, "Checkpoint initialized\n");
update_local_vms();
*c = (void *)info; *c = (void *)info;
return 0; return 0;
} }
static int static int
ckpt_shutdown(backend_context_t c) checkpoint_shutdown(backend_context_t c)
{ {
struct check_info *info = (struct check_info *)c; struct check_info *info = (struct check_info *)c;
@ -148,25 +740,27 @@ ckpt_shutdown(backend_context_t c)
info->magic = 0; info->magic = 0;
free(info); free(info);
cpg_stop();
return 0; return 0;
} }
static fence_callbacks_t ckpt_callbacks = { static fence_callbacks_t checkpoint_callbacks = {
.null = ckpt_null, .null = checkpoint_null,
.off = ckpt_off, .off = checkpoint_off,
.on = ckpt_on, .on = checkpoint_on,
.reboot = ckpt_reboot, .reboot = checkpoint_reboot,
.status = ckpt_status, .status = checkpoint_status,
.devstatus = ckpt_devstatus .devstatus = checkpoint_devstatus
}; };
static backend_plugin_t ckpt_plugin = { static backend_plugin_t checkpoint_plugin = {
.name = NAME, .name = NAME,
.version = VERSION, .version = VERSION,
.callbacks = &ckpt_callbacks, .callbacks = &checkpoint_callbacks,
.init = ckpt_init, .init = checkpoint_init,
.cleanup = ckpt_shutdown, .cleanup = checkpoint_shutdown,
}; };
@ -180,12 +774,12 @@ BACKEND_VER_SYM(void)
const backend_plugin_t * const backend_plugin_t *
BACKEND_INFO_SYM(void) BACKEND_INFO_SYM(void)
{ {
return &ckpt_plugin; return &checkpoint_plugin;
} }
#else #else
static void __attribute__((constructor)) static void __attribute__((constructor))
ckpt_register_plugin(void) checkpoint_register_plugin(void)
{ {
plugin_reg_backend(&ckpt_plugin); plugin_reg_backend(&checkpoint_plugin);
} }
#endif #endif

316
server/cpg.c
View File

@ -11,6 +11,7 @@
#include <time.h> #include <time.h>
#include <sys/uio.h> #include <sys/uio.h>
#include <list.h> #include <list.h>
#include <pthread.h>
#ifdef HAVE_OPENAIS_CPG_H #ifdef HAVE_OPENAIS_CPG_H
#include <openais/cpg.h> #include <openais/cpg.h>
#else #else
@ -19,6 +20,81 @@
#endif #endif
#endif #endif
#include "checkpoint.h"
#define NODE_ID_NONE ((uint32_t)-1)
struct msg_queue_node {
list_head();
uint32_t seqno;
#define STATE_CLEAR 0
#define STATE_MESSAGE 1
uint32_t state;
void *msg;
size_t msglen;
};
struct wire_msg {
#define TYPE_REQUEST 0
#define TYPE_REPLY 1
uint32_t type;
uint32_t seqno;
uint32_t target;
uint32_t pad;
char data[0];
};
static uint32_t seqnum = 0, my_node_id = NODE_ID_NONE;
static struct msg_queue_node *pending= NULL;
static cpg_handle_t cpg_handle;
static struct cpg_name gname;
static pthread_mutex_t cpg_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t cpg_cond = PTHREAD_COND_INITIALIZER;
static pthread_t cpg_thread = 0;
static request_callback_fn req_callback_fn;
int
cpg_get_ids(uint32_t *my_id, uint32_t *high_id)
{
#if 0
/* This is segfaulting for some reason */
struct cpg_address cpg_nodes[CPG_MEMBERS_MAX];
uint32_t high = my_node_id;
int count = CPG_MEMBERS_MAX, x;
if (!my_id && !high_id)
return 0;
if (my_id)
*my_id = my_node_id;
if (!high_id)
return 0;
memset(&cpg_nodes, 0, sizeof(cpg_nodes));
if (cpg_membership_get(cpg_handle, &gname,
cpg_nodes, &count) != CPG_OK)
return -1;
for (x = 0; x < count; x++) {
if (cpg_nodes[x].nodeid > high) {
high = cpg_nodes[x].nodeid;
}
}
*high_id = high;
return 0;
#endif
return -ENOSYS;
}
void void
#ifdef HAVE_OPENAIS_CPG_H #ifdef HAVE_OPENAIS_CPG_H
@ -37,8 +113,62 @@ cpg_deliver_func(cpg_handle_t h,
size_t msglen) size_t msglen)
#endif #endif
{ {
printf("%s (len = %d) from node %d pid %d\n)\n", struct msg_queue_node *n;
(char *)msg, (int)msglen, nodeid, pid); struct wire_msg *m = msg;
int x, found;
pthread_mutex_lock(&cpg_mutex);
if (m->type == TYPE_REPLY) {
/* Reply to a request we sent */
found = 0;
list_for(&pending, n, x) {
if (m->seqno != n->seqno)
continue;
if (m->target != my_node_id)
continue;
found = 1;
break;
}
if (!found)
goto out_unlock;
/* Copy our message in to a buffer */
n->msglen = msglen - sizeof(*m);
if (!n->msglen) {
/* XXX do what? */
}
n->msg = malloc(n->msglen);
if (!n->msg) {
goto out_unlock;
}
n->state = STATE_MESSAGE;
memcpy(n->msg, (char *)msg + sizeof(*m), n->msglen);
list_remove(&pending, n);
list_insert(&pending, n);
#if 0
printf("Seqnum %d replied; removing from list",
n->seqno);
#endif
pthread_cond_broadcast(&cpg_cond);
goto out_unlock;
}
pthread_mutex_unlock(&cpg_mutex);
if (m->type == TYPE_REQUEST) {
req_callback_fn(&m->data, msglen - sizeof(*m),
nodeid, m->seqno);
}
return;
out_unlock:
pthread_mutex_unlock(&cpg_mutex);
} }
void void
@ -68,25 +198,128 @@ static cpg_callbacks_t my_callbacks = {
int int
cpg_send(cpg_handle_t h, void *data, size_t len) cpg_send_req(void *data, size_t len, uint32_t *seqno)
{ {
struct iovec iov; struct iovec iov;
struct msg_queue_node *n;
struct wire_msg *m;
size_t msgsz = sizeof(*m) + len;
int ret;
iov.iov_base = data; n = malloc(sizeof(*n));
iov.iov_len = len; if (!n)
return cpg_mcast_joined(h, CPG_TYPE_AGREED, &iov, 1); return -1;
m = malloc(msgsz);
if (!m)
return -1;
/* only incremented on send */
n->state = STATE_CLEAR;
n->msg = NULL;
n->msglen = 0;
pthread_mutex_lock(&cpg_mutex);
list_insert(&pending, n);
n->seqno = ++seqnum;
m->seqno = seqnum;
*seqno = seqnum;
pthread_mutex_unlock(&cpg_mutex);
m->type = TYPE_REQUEST; /* XXX swab? */
m->target = NODE_ID_NONE;
memcpy(&m->data, data, len);
iov.iov_base = m;
iov.iov_len = msgsz;
ret = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, &iov, 1);
free(m);
if (ret == CPG_OK)
return 0;
return -1;
} }
int int
cpg_start(const char *name, cpg_handle_t *handle, struct cpg_name *grpname) cpg_send_reply(void *data, size_t len, uint32_t nodeid,
uint32_t seqno)
{
struct iovec iov;
struct wire_msg *m;
size_t msgsz = sizeof(*m) + len;
int ret;
m = malloc(msgsz);
if (!m)
return -1;
/* only incremented on send */
m->seqno = seqno;
m->type = TYPE_REPLY; /* XXX swab? */
m->target = nodeid;
memcpy(&m->data, data, len);
iov.iov_base = m;
iov.iov_len = msgsz;
ret = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, &iov, 1);
free(m);
if (ret == CPG_OK)
return 0;
return -1;
}
int
cpg_wait_reply(void **data, size_t *len, uint32_t seqno)
{
struct msg_queue_node *n;
int x, found = 0;
while (!found) {
found = 0;
pthread_mutex_lock(&cpg_mutex);
pthread_cond_wait(&cpg_cond, &cpg_mutex);
list_for(&pending, n, x) {
if (n->seqno != seqno)
continue;
if (n->state != STATE_MESSAGE)
continue;
found = 1;
break;
}
pthread_mutex_unlock(&cpg_mutex);
}
list_remove(&pending, n);
pthread_mutex_unlock(&cpg_mutex);
*data = n->msg;
*len = n->msglen;
free(n);
return 0;
}
static void *
cpg_dispatch_thread(void *arg)
{
cpg_dispatch(cpg_handle, CPG_DISPATCH_BLOCKING);
return NULL;
}
int
cpg_start(const char *name, request_callback_fn func)
{ {
cpg_handle_t h; cpg_handle_t h;
struct cpg_name gname;
errno = EINVAL; errno = EINVAL;
if (!name || !handle || !grpname) if (!name)
return -1; return -1;
gname.length = snprintf(gname.value, gname.length = snprintf(gname.value,
@ -99,6 +332,7 @@ cpg_start(const char *name, cpg_handle_t *handle, struct cpg_name *grpname)
if (gname.length <= 0) if (gname.length <= 0)
return -1; return -1;
memset(&h, 0, sizeof(h)); memset(&h, 0, sizeof(h));
if (cpg_initialize(&h, &my_callbacks) != CPG_OK) { if (cpg_initialize(&h, &my_callbacks) != CPG_OK) {
perror("cpg_initialize"); perror("cpg_initialize");
@ -110,18 +344,32 @@ cpg_start(const char *name, cpg_handle_t *handle, struct cpg_name *grpname)
return -1; return -1;
} }
memcpy(handle, &h, sizeof(h));
memcpy(grpname, &gname, sizeof(gname)); pthread_mutex_lock(&cpg_mutex);
cpg_local_get(h, &my_node_id);
pthread_create(&cpg_thread, NULL, cpg_dispatch_thread, NULL);
memcpy(&cpg_handle, &h, sizeof(h));
req_callback_fn = func;
pthread_mutex_unlock(&cpg_mutex);
return 0; return 0;
} }
int int
cpg_end(cpg_handle_t h, struct cpg_name *gname) cpg_stop(void)
{ {
cpg_leave(h, gname); pthread_cancel(cpg_thread);
cpg_finalize(h); pthread_join(cpg_thread, NULL);
cpg_leave(cpg_handle, &gname);
cpg_finalize(cpg_handle);
return 0; return 0;
} }
@ -136,42 +384,40 @@ go_away(int sig)
} }
void
request_callback(void *data, size_t len, uint32_t nodeid, uint32_t seqno)
{
char *msg = data;
printf("msg = %s\n", msg);
cpg_send_reply("fail.", 7, nodeid, seqno);
}
int int
main(int argc, char **argv) main(int argc, char **argv)
{ {
cpg_handle_t h; uint32_t seqno = 0;
struct cpg_name gname;
fd_set rfds;
int fd; int fd;
char *data;
size_t len;
signal(SIGINT, go_away); signal(SIGINT, go_away);
if (cpg_start("lhh1", &h, &gname) < 0) { if (cpg_start("lhh1", request_callback) < 0) {
perror("cpg_start"); perror("cpg_start");
return 1; return 1;
} }
if (cpg_fd_get(h, &fd) != CPG_OK) { cpg_send_req("hi", 2, &seqno);
perror("cpg_fd_get"); cpg_wait_reply(&data, &len, seqno);
return -1;
}
cpg_send(h, "hi", 2); printf("%s\n", data);
while (please_quit != 1) {
FD_ZERO(&rfds);
FD_SET(fd, &rfds);
if (select(fd+1, &rfds, NULL, NULL, NULL) < 0)
continue;
cpg_dispatch(h, CPG_DISPATCH_ALL);
}
printf("going bye\n"); printf("going bye\n");
cpg_leave(h, &gname); cpg_stop();
cpg_finalize(h);
return 0; return 0;
} }

4
server/virt.c
View File

@ -177,7 +177,7 @@ vl_print(virt_list_t *vl)
virt_state_t * virt_state_t *
vl_find_name(virt_list_t *vl, char *name) vl_find_name(virt_list_t *vl, const char *name)
{ {
int x; int x;
@ -194,7 +194,7 @@ vl_find_name(virt_list_t *vl, char *name)
virt_state_t * virt_state_t *
vl_find_uuid(virt_list_t *vl, char *uuid) vl_find_uuid(virt_list_t *vl, const char *uuid)
{ {
int x; int x;

10
server/virt.h
View File

@ -65,15 +65,15 @@ int vl_cmp(virt_list_t *left, virt_list_t *right);
void vl_print(virt_list_t *vl); void vl_print(virt_list_t *vl);
void vl_free(virt_list_t *old); void vl_free(virt_list_t *old);
virt_state_t * vl_find_uuid(virt_list_t *vl, char *name); virt_state_t * vl_find_uuid(virt_list_t *vl, const char *name);
virt_state_t * vl_find_name(virt_list_t *vl, char *name); virt_state_t * vl_find_name(virt_list_t *vl, const char *name);
typedef void ckpt_handle; typedef void ckpt_handle;
int ckpt_read(void *hp, char *secid, void *buf, size_t maxlen); int ckpt_read(void *hp, const char *secid, void *buf, size_t maxlen);
int ckpt_finish(void *hp); int ckpt_finish(void *hp);
int ckpt_write(void *hp, char *secid, void *buf, size_t maxlen); int ckpt_write(void *hp, const char *secid, void *buf, size_t maxlen);
void *ckpt_init(char *ckpt_name, int maxlen, int maxsec, int maxseclen, void *ckpt_init(const char *ckpt_name, int maxlen, int maxsec, int maxseclen,
int timeout); int timeout);

7
server/vm_states.c
View File

@ -245,8 +245,9 @@ ckpt_init(char *ckpt_name, int maxlen, int maxsec,
if (err != SA_AIS_OK) { if (err != SA_AIS_OK) {
free(h); free(h);
return NULL; return NULL;
} else } else {
h->ck_ready = READY_MAGIC; h->ck_ready = READY_MAGIC;
}
if (ckpt_open(h, ckpt_name, maxlen, maxsec, maxseclen, if (ckpt_open(h, ckpt_name, maxlen, maxsec, maxseclen,
timeout) < 0) { timeout) < 0) {
@ -262,7 +263,7 @@ ckpt_init(char *ckpt_name, int maxlen, int maxsec,
int int
ckpt_write(void *hp, char *secid, void *buf, size_t maxlen) ckpt_write(void *hp, const char *secid, void *buf, size_t maxlen)
{ {
ckpt_handle *h = (ckpt_handle *)hp; ckpt_handle *h = (ckpt_handle *)hp;
SaCkptIOVectorElementT iov = {SA_CKPT_DEFAULT_SECTION_ID, SaCkptIOVectorElementT iov = {SA_CKPT_DEFAULT_SECTION_ID,
@ -302,7 +303,7 @@ ckpt_write(void *hp, char *secid, void *buf, size_t maxlen)
int int
ckpt_read(void *hp, char *secid, void *buf, size_t maxlen) ckpt_read(void *hp, const char *secid, void *buf, size_t maxlen)
{ {
ckpt_handle *h = (ckpt_handle *)hp; ckpt_handle *h = (ckpt_handle *)hp;
SaCkptIOVectorElementT iov = {SA_CKPT_DEFAULT_SECTION_ID, SaCkptIOVectorElementT iov = {SA_CKPT_DEFAULT_SECTION_ID,