From 0240fe9d58c1b1b5fb4d8d8f5a316220b4efc167 Mon Sep 17 00:00:00 2001 From: Dan Smith Date: Thu, 26 Jun 2008 16:09:48 +0000 Subject: [PATCH] [LXC] Add setup/cleanup of container network interfaces --- src/lxc_conf.h | 8 ++ src/lxc_container.c | 77 ++++++++++++++ src/lxc_driver.c | 244 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 323 insertions(+), 6 deletions(-) diff --git a/src/lxc_conf.h b/src/lxc_conf.h index 0434e0c779..5d26b350a2 100644 --- a/src/lxc_conf.h +++ b/src/lxc_conf.h @@ -35,6 +35,12 @@ #define LXC_MAX_XML_LENGTH 16384 #define LXC_MAX_ERROR_LEN 1024 #define LXC_DOMAIN_TYPE "lxc" +#define LXC_PARENT_SOCKET 0 +#define LXC_CONTAINER_SOCKET 1 + +/* messages between parent and container */ +typedef char lxc_message_t; +#define LXC_CONTINUE_MSG 'c' /* types of networks for containers */ enum lxc_net_type { @@ -97,6 +103,8 @@ struct __lxc_vm { int containerTtyFd; char *containerTty; + int sockpair[2]; + lxc_vm_def_t *def; lxc_vm_t *next; diff --git a/src/lxc_container.c b/src/lxc_container.c index e2c74f363d..23698baf22 100644 --- a/src/lxc_container.c +++ b/src/lxc_container.c @@ -36,6 +36,7 @@ #include "lxc_conf.h" #include "util.h" #include "memory.h" +#include "veth.h" #define DEBUG(fmt,...) VIR_DEBUG(__FILE__, fmt, __VA_ARGS__) #define DEBUG0(msg) VIR_DEBUG(__FILE__, "%s", msg) @@ -158,6 +159,72 @@ exit_with_error: exit(rc); } +/** + * lxcWaitForContinue: + * @vm: Pointer to vm structure + * + * This function will wait for the container continue message from the + * parent process. It will send this message on the socket pair stored in + * the vm structure once it has completed the post clone container setup. + * + * Returns 0 on success or -1 in case of error + */ +static int lxcWaitForContinue(lxc_vm_t *vm) +{ + int rc = -1; + lxc_message_t msg; + int readLen; + + readLen = saferead(vm->sockpair[LXC_CONTAINER_SOCKET], &msg, sizeof(msg)); + if (readLen != sizeof(msg)) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("Failed to read the container continue message: %s"), + strerror(errno)); + goto error_out; + } + + DEBUG0("Received container continue message"); + + close(vm->sockpair[LXC_PARENT_SOCKET]); + vm->sockpair[LXC_PARENT_SOCKET] = -1; + close(vm->sockpair[LXC_CONTAINER_SOCKET]); + vm->sockpair[LXC_CONTAINER_SOCKET] = -1; + + rc = 0; + +error_out: + return rc; +} + +/** + * lxcEnableInterfaces: + * @vm: Pointer to vm structure + * + * This function will enable the interfaces for this container. + * + * Returns 0 on success or nonzero in case of error + */ +static int lxcEnableInterfaces(const lxc_vm_t *vm) +{ + int rc = 0; + const lxc_net_def_t *net; + + for (net = vm->def->nets; net; net = net->next) { + DEBUG("Enabling %s", net->containerVeth); + rc = vethInterfaceUpOrDown(net->containerVeth, 1); + if (0 != rc) { + goto error_out; + } + } + + /* enable lo device only if there were other net devices */ + if (vm->def->nets) + rc = vethInterfaceUpOrDown("lo", 1); + +error_out: + return rc; +} + /** * lxcChild: * @argv: Pointer to container arguments @@ -210,6 +277,16 @@ int lxcChild( void *argv ) goto cleanup; } + /* Wait for interface devices to show up */ + if (0 != (rc = lxcWaitForContinue(vm))) { + goto cleanup; + } + + /* enable interfaces */ + if (0 != (rc = lxcEnableInterfaces(vm))) { + goto cleanup; + } + rc = lxcExecWithTty(vm); /* this function will only return if an error occured */ diff --git a/src/lxc_driver.c b/src/lxc_driver.c index 03044e2664..f58cf58b7d 100644 --- a/src/lxc_driver.c +++ b/src/lxc_driver.c @@ -44,6 +44,9 @@ #include "memory.h" #include "util.h" #include "memory.h" +#include "bridge.h" +#include "qemu_conf.h" +#include "veth.h" /* debug macros */ #define DEBUG(fmt,...) VIR_DEBUG(__FILE__, fmt, __VA_ARGS__) @@ -394,6 +397,202 @@ static char *lxcDomainDumpXML(virDomainPtr dom, return lxcGenerateXML(dom->conn, driver, vm, vm->def); } +/** + * lxcSetupInterfaces: + * @conn: pointer to connection + * @vm: pointer to virtual machine structure + * + * Sets up the container interfaces by creating the veth device pairs and + * attaching the parent end to the appropriate bridge. The container end + * will moved into the container namespace later after clone has been called. + * + * Returns 0 on success or -1 in case of error + */ +static int lxcSetupInterfaces(virConnectPtr conn, + lxc_vm_t *vm) +{ + int rc = -1; + lxc_driver_t *driver = conn->privateData; + struct qemud_driver *networkDriver = + (struct qemud_driver *)(conn->networkPrivateData); + lxc_net_def_t *net = vm->def->nets; + char* bridge; + char parentVeth[PATH_MAX] = ""; + char containerVeth[PATH_MAX] = ""; + + if ((vm->def->nets != NULL) && (driver->have_netns == 0)) { + lxcError(conn, NULL, VIR_ERR_NO_SUPPORT, + _("System lacks NETNS support")); + return -1; + } + + for (net = vm->def->nets; net; net = net->next) { + if (LXC_NET_NETWORK == net->type) { + virNetworkPtr network = virNetworkLookupByName(conn, net->txName); + if (!network) { + goto error_exit; + } + + bridge = virNetworkGetBridgeName(network); + + virNetworkFree(network); + + } else { + bridge = net->txName; + } + + DEBUG("bridge: %s", bridge); + if (NULL == bridge) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to get bridge for interface")); + goto error_exit; + } + + DEBUG0("calling vethCreate()"); + if (NULL != net->parentVeth) { + strcpy(parentVeth, net->parentVeth); + } + if (NULL != net->containerVeth) { + strcpy(containerVeth, net->containerVeth); + } + DEBUG("parentVeth: %s, containerVeth: %s", parentVeth, containerVeth); + if (0 != (rc = vethCreate(parentVeth, PATH_MAX, containerVeth, PATH_MAX))) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to create veth device pair: %d"), rc); + goto error_exit; + } + if (NULL == net->parentVeth) { + net->parentVeth = strdup(parentVeth); + } + if (NULL == net->containerVeth) { + net->containerVeth = strdup(containerVeth); + } + + if ((NULL == net->parentVeth) || (NULL == net->containerVeth)) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to allocate veth names")); + goto error_exit; + } + + if (!(networkDriver->brctl) && (rc = brInit(&(networkDriver->brctl)))) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("cannot initialize bridge support: %s"), + strerror(rc)); + goto error_exit; + } + + if (0 != (rc = brAddInterface(networkDriver->brctl, bridge, parentVeth))) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to add %s device to %s: %s"), + parentVeth, + bridge, + strerror(rc)); + goto error_exit; + } + + if (0 != (rc = vethInterfaceUpOrDown(parentVeth, 1))) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to enable parent ns veth device: %d"), rc); + goto error_exit; + } + + } + + rc = 0; + +error_exit: + return rc; +} + +/** + * lxcMoveInterfacesToNetNs: + * @conn: pointer to connection + * @vm: pointer to virtual machine structure + * + * Starts a container process by calling clone() with the namespace flags + * + * Returns 0 on success or -1 in case of error + */ +static int lxcMoveInterfacesToNetNs(virConnectPtr conn, + const lxc_vm_t *vm) +{ + int rc = -1; + lxc_net_def_t *net; + + for (net = vm->def->nets; net; net = net->next) { + if (0 != moveInterfaceToNetNs(net->containerVeth, vm->def->id)) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to move interface %s to ns %d"), + net->containerVeth, vm->def->id); + goto error_exit; + } + } + + rc = 0; + +error_exit: + return rc; +} + +/** + * lxcCleanupInterfaces: + * @conn: pointer to connection + * @vm: pointer to virtual machine structure + * + * Cleans up the container interfaces by deleting the veth device pairs. + * + * Returns 0 on success or -1 in case of error + */ +static int lxcCleanupInterfaces(const lxc_vm_t *vm) +{ + int rc = -1; + lxc_net_def_t *net; + + for (net = vm->def->nets; net; net = net->next) { + if (0 != (rc = vethDelete(net->parentVeth))) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to delete veth: %s"), net->parentVeth); + /* will continue to try to cleanup any other interfaces */ + } + } + + return 0; +} + +/** + * lxcSendContainerContinue: + * @vm: pointer to virtual machine structure + * + * Sends the continue message via the socket pair stored in the vm + * structure. + * + * Returns 0 on success or -1 in case of error + */ +static int lxcSendContainerContinue(const lxc_vm_t *vm) +{ + int rc = -1; + lxc_message_t msg = LXC_CONTINUE_MSG; + int writeCount = 0; + + if (NULL == vm) { + goto error_out; + } + + writeCount = safewrite(vm->sockpair[LXC_PARENT_SOCKET], &msg, + sizeof(msg)); + if (writeCount != sizeof(msg)) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("unable to send container continue message: %s"), + strerror(errno)); + goto error_out; + } + + rc = 0; + +error_out: + return rc; +} + /** * lxcStartContainer: * @conn: pointer to connection @@ -423,6 +622,9 @@ static int lxcStartContainer(virConnectPtr conn, flags = CLONE_NEWPID|CLONE_NEWNS|CLONE_NEWUTS|CLONE_NEWUSER|CLONE_NEWIPC|SIGCHLD; + if (vm->def->nets != NULL) + flags |= CLONE_NEWNET; + vm->def->id = clone(lxcChild, stacktop, flags, (void *)vm); DEBUG("clone() returned, %d", vm->def->id); @@ -819,15 +1021,42 @@ static int lxcVmStart(virConnectPtr conn, close(vm->parentTty); close(vm->containerTtyFd); - rc = lxcStartContainer(conn, driver, vm); - - if (rc == 0) { - vm->state = VIR_DOMAIN_RUNNING; - driver->ninactivevms--; - driver->nactivevms++; + if (0 != (rc = lxcSetupInterfaces(conn, vm))) { + goto cleanup; } + /* create a socket pair to send continue message to the container once */ + /* we've completed the post clone configuration */ + if (0 != socketpair(PF_UNIX, SOCK_STREAM, 0, vm->sockpair)) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("sockpair failed: %s"), strerror(errno)); + goto cleanup; + } + + /* check this rc */ + + rc = lxcStartContainer(conn, driver, vm); + if (rc != 0) + goto cleanup; + + rc = lxcMoveInterfacesToNetNs(conn, vm); + if (rc != 0) + goto cleanup; + + rc = lxcSendContainerContinue(vm); + if (rc != 0) + goto cleanup; + + vm->state = VIR_DOMAIN_RUNNING; + driver->ninactivevms--; + driver->nactivevms++; + cleanup: + close(vm->sockpair[LXC_PARENT_SOCKET]); + vm->sockpair[LXC_PARENT_SOCKET] = -1; + close(vm->sockpair[LXC_CONTAINER_SOCKET]); + vm->sockpair[LXC_CONTAINER_SOCKET] = -1; + return rc; } @@ -958,6 +1187,9 @@ static int lxcVMCleanup(lxc_driver_t *driver, lxc_vm_t * vm) int waitRc; int childStatus = -1; + /* if this fails, we'll continue. it will report any errors */ + lxcCleanupInterfaces(vm); + while (((waitRc = waitpid(vm->def->id, &childStatus, 0)) == -1) && errno == EINTR);