diff --git a/ctdb/Makefile.in b/ctdb/Makefile.in index 940f9794a65..e085303c901 100644 --- a/ctdb/Makefile.in +++ b/ctdb/Makefile.in @@ -30,7 +30,7 @@ CTDB_OBJ = $(CTDB_COMMON_OBJ) $(CTDB_TCP_OBJ) OBJS = @TDBOBJ@ @TALLOCOBJ@ @LIBREPLACEOBJ@ @INFINIBAND_WRAPPER_OBJ@ $(EXTRA_OBJ) $(EVENTS_OBJ) $(CTDB_OBJ) -BINS = bin/ctdb_test bin/ctdb_bench bin/ctdb_fetch @INFINIBAND_BINS@ +BINS = bin/ctdb_test bin/ctdb_bench bin/ctdb_messaging bin/ctdb_fetch @INFINIBAND_BINS@ DIRS = lib bin @@ -61,6 +61,10 @@ bin/ctdb_fetch: $(OBJS) tests/ctdb_fetch.o @echo Linking $@ @$(CC) $(CFLAGS) -o $@ tests/ctdb_fetch.o $(OBJS) $(LIB_FLAGS) +bin/ctdb_messaging: $(OBJS) tests/ctdb_messaging.o + @echo Linking $@ + @$(CC) $(CFLAGS) -o $@ tests/ctdb_messaging.o $(OBJS) $(LIB_FLAGS) + bin/ibwrapper_test: $(OBJS) ib/ibwrapper_test.o @echo Linking $@ @$(CC) $(CFLAGS) -o $@ ib/ibwrapper_test.o $(OBJS) $(LIB_FLAGS) diff --git a/ctdb/common/ctdb_client.c b/ctdb/common/ctdb_client.c index e837a1af879..f2e551091ce 100644 --- a/ctdb/common/ctdb_client.c +++ b/ctdb/common/ctdb_client.c @@ -275,9 +275,14 @@ int ctdb_client_set_message_handler(struct ctdb_context *ctdb, uint32_t srvid, void *private) { - struct ctdb_register_call c; + struct ctdb_req_register c; int res; + /* if the domain socket is not yet open, open it */ + if (ctdb->daemon.sd==-1) { + ux_socket_connect(ctdb); + } + ZERO_STRUCT(c); c.hdr.length = sizeof(c); @@ -311,3 +316,34 @@ int ctdb_set_message_handler(struct ctdb_context *ctdb, return ctdb_daemon_set_message_handler(ctdb, srvid, handler, private); } + +int ctdb_client_send_message(struct ctdb_context *ctdb, uint32_t vnn, + uint32_t srvid, TDB_DATA data) +{ + struct ctdb_req_message *r; + int len, res; + + len = offsetof(struct ctdb_req_message, data) + data.dsize; + r = ctdb->methods->allocate_pkt(ctdb, len); + CTDB_NO_MEMORY(ctdb, r); + talloc_set_name_const(r, "req_message packet"); + + r->hdr.length = len; + r->hdr.ctdb_magic = CTDB_MAGIC; + r->hdr.ctdb_version = CTDB_VERSION; + r->hdr.operation = CTDB_REQ_MESSAGE; + r->hdr.destnode = vnn; + r->hdr.srcnode = ctdb->vnn; + r->hdr.reqid = 0; + r->srvid = srvid; + r->datalen = data.dsize; + memcpy(&r->data[0], data.dptr, data.dsize); + + res = ctdb_client_queue_pkt(ctdb, &r->hdr); + if (res != 0) { + return res; + } + + talloc_free(r); + return 0; +} diff --git a/ctdb/common/ctdb_daemon.c b/ctdb/common/ctdb_daemon.c index b520dc7f0d8..fd656cc0fb5 100644 --- a/ctdb/common/ctdb_daemon.c +++ b/ctdb/common/ctdb_daemon.c @@ -72,7 +72,8 @@ static void daemon_message_handler(struct ctdb_context *ctdb, uint32_t srvid, /* construct a message to send to the client containing the data */ len = offsetof(struct ctdb_req_message, data) + data.dsize; r = ctdbd_allocate_pkt(ctdb, len); - CTDB_NO_MEMORY(ctdb, r); + +/*XXX cant use this since it returns an int CTDB_NO_MEMORY(ctdb, r);*/ talloc_set_name_const(r, "req_message packet"); r->hdr.length = len; @@ -86,7 +87,7 @@ static void daemon_message_handler(struct ctdb_context *ctdb, uint32_t srvid, ctdb_queue_send(client->queue, (uint8_t *)&r->hdr, len); talloc_free(r); - return 0; + return; } @@ -98,7 +99,6 @@ static void daemon_request_register_message_handler(struct ctdb_client *client, struct ctdb_req_register *c) { int res; - printf("XXX registering messaging handler %u in daemon\n", c->srvid); res = ctdb_register_message_handler(client->ctdb, client, c->srvid, daemon_message_handler, client); @@ -119,6 +119,33 @@ static int ctdb_client_destructor(struct ctdb_client *client) } +/* + this is called when the ctdb daemon received a ctdb request message + from a local client over the unix domain socket + */ +static void daemon_request_message_from_client(struct ctdb_client *client, + struct ctdb_req_message *c) +{ + TDB_DATA data; + int res; + + /* maybe the message is for another client on this node */ + if (ctdb_get_vnn(client->ctdb)==c->hdr.destnode) { + ctdb_request_message(client->ctdb, (struct ctdb_req_header *)c); + return; + } + + /* its for a remote node */ + data.dptr = &c->data[0]; + data.dsize = c->datalen; + res = ctdb_daemon_send_message(client->ctdb, c->hdr.destnode, + c->srvid, data); + if (res != 0) { + printf("Failed to send message to remote node %u\n", + c->hdr.destnode); + } +} + /* this is called when the ctdb daemon received a ctdb request call from a local client over the unix domain socket @@ -189,12 +216,12 @@ static void client_incoming_packet(struct ctdb_client *client, void *data, size_ if (hdr->ctdb_magic != CTDB_MAGIC) { ctdb_set_error(client->ctdb, "Non CTDB packet rejected\n"); - return; + goto done; } if (hdr->ctdb_version != CTDB_VERSION) { ctdb_set_error(client->ctdb, "Bad CTDB version 0x%x rejected\n", hdr->ctdb_version); - return; + goto done; } switch (hdr->operation) { @@ -206,8 +233,12 @@ static void client_incoming_packet(struct ctdb_client *client, void *data, size_ daemon_request_register_message_handler(client, (struct ctdb_req_register *)hdr); break; + case CTDB_REQ_MESSAGE: + daemon_request_message_from_client(client, (struct ctdb_req_message *)hdr); + break; } +done: talloc_free(data); } @@ -396,3 +427,10 @@ void *ctdbd_allocate_pkt(struct ctdb_context *ctdb, size_t len) return talloc_size(ctdb, size); } +int ctdb_daemon_set_message_handler(struct ctdb_context *ctdb, uint32_t srvid, + ctdb_message_fn_t handler, + void *private) +{ + return ctdb_register_message_handler(ctdb, ctdb, srvid, handler, private); +} + diff --git a/ctdb/common/ctdb_message.c b/ctdb/common/ctdb_message.c index 27c5c64bc0e..088f882dbd5 100644 --- a/ctdb/common/ctdb_message.c +++ b/ctdb/common/ctdb_message.c @@ -37,22 +37,30 @@ void ctdb_request_message(struct ctdb_context *ctdb, struct ctdb_req_header *hdr) { struct ctdb_req_message *c = (struct ctdb_req_message *)hdr; + struct ctdb_message_list *ml; TDB_DATA data; - if (ctdb->message_handler == NULL) { + + /* XXX we need a must faster way of finding the matching srvid + - maybe a tree? */ + for (ml=ctdb->message_list;ml;ml=ml->next) { + if (ml->srvid == c->srvid) break; + } + if (ml == NULL) { printf("no msg handler\n"); /* no registered message handler */ return; } + data.dptr = &c->data[0]; data.dsize = c->datalen; - ctdb->message_handler(ctdb, c->srvid, data, ctdb->message_private); + ml->message_handler(ctdb, c->srvid, data, ml->message_private); } /* send a ctdb message */ -int ctdb_send_message(struct ctdb_context *ctdb, uint32_t vnn, +int ctdb_daemon_send_message(struct ctdb_context *ctdb, uint32_t vnn, uint32_t srvid, TDB_DATA data) { struct ctdb_req_message *r; @@ -80,12 +88,26 @@ int ctdb_send_message(struct ctdb_context *ctdb, uint32_t vnn, return 0; } +/* + send a ctdb message +*/ +int ctdb_send_message(struct ctdb_context *ctdb, uint32_t vnn, + uint32_t srvid, TDB_DATA data) +{ + if (ctdb->flags & CTDB_FLAG_DAEMON_MODE) { + return ctdb_client_send_message(ctdb, vnn, srvid, data); + } + return ctdb_daemon_send_message(ctdb, vnn, srvid, data); +} + + /* when a client goes away, we need to remove its srvid handler from the list */ static int message_handler_destructor(struct ctdb_message_list *m) { DLIST_REMOVE(m->ctdb->message_list, m); + return 0; } /* diff --git a/ctdb/include/ctdb.h b/ctdb/include/ctdb.h index f77b34abbda..becdea7cd77 100644 --- a/ctdb/include/ctdb.h +++ b/ctdb/include/ctdb.h @@ -152,8 +152,9 @@ uint32_t ctdb_get_num_nodes(struct ctdb_context *ctdb); /* setup a handler for ctdb messages */ typedef void (*ctdb_message_fn_t)(struct ctdb_context *, uint32_t srvid, TDB_DATA data, void *); -int ctdb_set_message_handler(struct ctdb_context *ctdb, ctdb_message_fn_t handler, - uint32_t srvid, void *private); +int ctdb_set_message_handler(struct ctdb_context *ctdb, uint32_t srvid, + ctdb_message_fn_t handler, + void *private); int ctdb_call(struct ctdb_db_context *ctdb_db, struct ctdb_call *call); @@ -180,4 +181,10 @@ struct ctdb_record_handle *ctdb_fetch_lock(struct ctdb_db_context *ctdb_db, TALL */ int ctdb_record_store(struct ctdb_record_handle *rec, TDB_DATA data); +int ctdb_register_message_handler(struct ctdb_context *ctdb, + TALLOC_CTX *mem_ctx, + uint32_t srvid, + ctdb_message_fn_t handler, + void *private); + #endif diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h index 416156482e6..af183168ddd 100644 --- a/ctdb/include/ctdb_private.h +++ b/ctdb/include/ctdb_private.h @@ -269,7 +269,7 @@ struct ctdb_reply_dmaster { uint8_t data[1]; }; -struct ctdb_register_call { +struct ctdb_req_register { struct ctdb_req_header hdr; uint32_t srvid; }; @@ -368,4 +368,17 @@ struct ctdb_call_state *ctdb_client_call_send(struct ctdb_db_context *ctdb_db, */ int ctdb_client_call_recv(struct ctdb_call_state *state, struct ctdb_call *call); +int ctdb_daemon_set_message_handler(struct ctdb_context *ctdb, uint32_t srvid, + ctdb_message_fn_t handler, + void *private); + +int ctdb_client_send_message(struct ctdb_context *ctdb, uint32_t vnn, + uint32_t srvid, TDB_DATA data); + +/* + send a ctdb message +*/ +int ctdb_daemon_send_message(struct ctdb_context *ctdb, uint32_t vnn, + uint32_t srvid, TDB_DATA data); + #endif diff --git a/ctdb/tests/bench.sh b/ctdb/tests/bench.sh index b9b93b01b68..50e9e08f995 100755 --- a/ctdb/tests/bench.sh +++ b/ctdb/tests/bench.sh @@ -3,7 +3,7 @@ killall -q ctdb_bench echo "Trying 2 nodes" -bin/ctdb_bench --nlist tests/nodes.txt --listen 127.0.0.2:9001 --pid=55 $* & -bin/ctdb_bench --nlist tests/nodes.txt --listen 127.0.0.1:9001 --pid=66 $* +bin/ctdb_bench --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* & +bin/ctdb_bench --nlist tests/nodes.txt --listen 127.0.0.1:9001 $* killall -q ctdb_bench diff --git a/ctdb/tests/ctdb_bench.c b/ctdb/tests/ctdb_bench.c index 0d56d2ec289..658a2e4dadd 100644 --- a/ctdb/tests/ctdb_bench.c +++ b/ctdb/tests/ctdb_bench.c @@ -294,7 +294,7 @@ int main(int argc, const char *argv[]) /* start the protocol running */ ret = ctdb_start(ctdb); - ctdb_set_message_handler(ctdb, ring_message_handler, 0, &msg_count); + ctdb_set_message_handler(ctdb, 0, ring_message_handler,&msg_count); /* wait until all nodes are connected (should not be needed outside of test code) */ diff --git a/ctdb/tests/ctdb_fetch.c b/ctdb/tests/ctdb_fetch.c index b7317f7de1f..da7eeda7ced 100644 --- a/ctdb/tests/ctdb_fetch.c +++ b/ctdb/tests/ctdb_fetch.c @@ -260,7 +260,7 @@ int main(int argc, const char *argv[]) /* start the protocol running */ ret = ctdb_start(ctdb); - ctdb_set_message_handler(ctdb, message_handler, 0, &msg_count); + ctdb_set_message_handler(ctdb, 0, message_handler, &msg_count); /* wait until all nodes are connected (should not be needed outside of test code) */ diff --git a/ctdb/tests/ctdb_messaging.c b/ctdb/tests/ctdb_messaging.c new file mode 100644 index 00000000000..5ee02c5e400 --- /dev/null +++ b/ctdb/tests/ctdb_messaging.c @@ -0,0 +1,171 @@ +/* + test of messaging + + Copyright (C) Andrew Tridgell 2006 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include "includes.h" +#include "lib/events/events.h" +#include "system/filesys.h" +#include "popt.h" + +static int timelimit = 10; +static int num_records = 10; +static int num_msgs = 1; +static int num_repeats = 100; + + +/* + handler for messages in bench_ring() +*/ +static void message_handler(struct ctdb_context *ctdb, uint32_t srvid, + TDB_DATA data, void *private) +{ +printf("client vnn:%d received a message to srvid:%d\n",ctdb_get_vnn(ctdb),srvid); +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + const char *nlist = NULL; + const char *transport = "tcp"; + const char *myaddress = NULL; + int self_connect=0; + int daemon_mode=0; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + { "nlist", 0, POPT_ARG_STRING, &nlist, 0, "node list file", "filename" }, + { "listen", 0, POPT_ARG_STRING, &myaddress, 0, "address to listen on", "address" }, + { "transport", 0, POPT_ARG_STRING, &transport, 0, "protocol transport", NULL }, + { "self-connect", 0, POPT_ARG_NONE, &self_connect, 0, "enable self connect", "boolean" }, + { "daemon", 0, POPT_ARG_NONE, &daemon_mode, 0, "spawn a ctdb daemon", "boolean" }, + { "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" }, + { "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" }, + { "num-msgs", 'n', POPT_ARG_INT, &num_msgs, 0, "num_msgs", "integer" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + int ret; + poptContext pc; + struct event_context *ev; + pid_t pid; + uint32_t srvid; + TDB_DATA data; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + if (nlist == NULL || myaddress == NULL) { + printf("You must provide a node list with --nlist and an address with --listen\n"); + exit(1); + } + + ev = event_context_init(NULL); + + /* initialise ctdb */ + ctdb = ctdb_init(ev); + if (ctdb == NULL) { + printf("Failed to init ctdb\n"); + exit(1); + } + + if (self_connect) { + ctdb_set_flags(ctdb, CTDB_FLAG_SELF_CONNECT); + } + if (daemon_mode) { + ctdb_set_flags(ctdb, CTDB_FLAG_DAEMON_MODE); + } + + ret = ctdb_set_transport(ctdb, transport); + if (ret == -1) { + printf("ctdb_set_transport failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + /* tell ctdb what address to listen on */ + ret = ctdb_set_address(ctdb, myaddress); + if (ret == -1) { + printf("ctdb_set_address failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + /* tell ctdb what nodes are available */ + ret = ctdb_set_nlist(ctdb, nlist); + if (ret == -1) { + printf("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666); + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + /* start the protocol running */ + ret = ctdb_start(ctdb); + +/*XXX why does this block forever? ctdb_connect_wait(ctdb);*/ + + pid=fork(); + if (pid) { + srvid=0; + } else { + srvid=1; + } + + /* wait until all nodes are connected (should not be needed + outside of test code) */ + data.dptr=NULL; + data.dsize=0; + ctdb_set_message_handler(ctdb, srvid, message_handler, NULL); + +sleep(3); +printf("sending message from vnn:%d to vnn:%d/srvid:%d\n",ctdb_get_vnn(ctdb),ctdb_get_vnn(ctdb), 1-srvid); + ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), 1-srvid, data); + + while(1){ + event_loop_once(ev); + } + + /* shut it down */ + talloc_free(ctdb); + return 0; +} diff --git a/ctdb/tests/messaging.sh b/ctdb/tests/messaging.sh new file mode 100755 index 00000000000..179a2bef88d --- /dev/null +++ b/ctdb/tests/messaging.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +killall -q ctdb_messaging + +echo "Trying 2 nodes" +bin/ctdb_messaging --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* & +bin/ctdb_messaging --nlist tests/nodes.txt --listen 127.0.0.1:9001 $* + +killall -q ctdb_messaging