diff --git a/src/libsystemd-bus/bus-internal.h b/src/libsystemd-bus/bus-internal.h index 05184fd5607..504dac7f09a 100644 --- a/src/libsystemd-bus/bus-internal.h +++ b/src/libsystemd-bus/bus-internal.h @@ -33,6 +33,7 @@ #include "sd-bus.h" #include "bus-error.h" #include "bus-match.h" +#include "bus-kernel.h" struct reply_callback { sd_bus_message_handler_t callback; @@ -152,6 +153,9 @@ struct sd_bus { unsigned iteration_counter; void *kdbus_buffer; + + struct memfd_cache memfd_cache[MEMFD_CACHE_MAX]; + unsigned n_memfd_cache; }; static inline void bus_unrefp(sd_bus **b) { diff --git a/src/libsystemd-bus/bus-kernel.c b/src/libsystemd-bus/bus-kernel.c index 071f2b9708f..b11df300b60 100644 --- a/src/libsystemd-bus/bus-kernel.c +++ b/src/libsystemd-bus/bus-kernel.c @@ -189,6 +189,7 @@ static int bus_message_setup_kmsg(sd_bus *b, sd_bus_message *m) { uint64_t unique; size_t sz, dl; int r; + struct bus_body_part *part; assert(b); assert(m); @@ -209,7 +210,8 @@ static int bus_message_setup_kmsg(sd_bus *b, sd_bus_message *m) { sz = offsetof(struct kdbus_msg, items); /* Add in fixed header, fields header and payload */ - sz += 3 * ALIGN8(offsetof(struct kdbus_item, vec) + sizeof(struct kdbus_vec)); + sz += (1 + !!m->fields + m->n_body_parts) * + ALIGN8(offsetof(struct kdbus_item, vec) + sizeof(struct kdbus_vec)); /* Add space for bloom filter */ sz += ALIGN8(offsetof(struct kdbus_item, data) + BLOOM_SIZE); @@ -251,8 +253,8 @@ static int bus_message_setup_kmsg(sd_bus *b, sd_bus_message *m) { if (m->fields) append_payload_vec(&d, m->fields, ALIGN8(m->header->fields_size)); - if (m->body) - append_payload_vec(&d, m->body, m->header->body_size); + for (part = &m->body; part && part->size > 0; part = part->next) + append_payload_vec(&d, part->data, part->size); if (m->kdbus->dst_id == KDBUS_DST_ID_BROADCAST) { void *p; @@ -395,7 +397,10 @@ static void close_kdbus_msg(sd_bus *bus, struct kdbus_msg *k) { } } -static bool range_contains(size_t astart, size_t asize, size_t bstart, size_t bsize, void *a, void **b) { +static bool range_contains( + size_t astart, size_t asize, + size_t bstart, size_t bsize, + void *a, void **b) { if (bstart < astart) return false; @@ -484,11 +489,35 @@ static int bus_kernel_make_message(sd_bus *bus, struct kdbus_msg *k, sd_bus_mess l = d->size - offsetof(struct kdbus_item, data); if (d->type == KDBUS_MSG_PAYLOAD_VEC) { + size_t begin_body; + /* Fill in fields material */ range_contains(idx, d->vec.size, ALIGN8(sizeof(struct bus_header)), BUS_MESSAGE_FIELDS_SIZE(m), UINT64_TO_PTR(d->vec.address), &m->fields); - range_contains(idx, d->vec.size, ALIGN8(sizeof(struct bus_header)) + ALIGN8(BUS_MESSAGE_FIELDS_SIZE(m)), - BUS_MESSAGE_BODY_SIZE(m), UINT64_TO_PTR(d->vec.address), &m->body); + + begin_body = ALIGN8(sizeof(struct bus_header)) + ALIGN8(BUS_MESSAGE_FIELDS_SIZE(m)); + + if (idx + d->vec.size > begin_body) { + struct bus_body_part *part; + + /* Contains body material */ + + part = message_append_part(m); + if (!part) { + sd_bus_message_unref(m); + return -ENOMEM; + } + + if (idx >= begin_body) { + part->data = (void*) d->vec.address; + part->size = d->vec.size; + } else { + part->data = (uint8_t*) (uintptr_t) d->vec.address + (begin_body - idx); + part->size = d->vec.size - (begin_body - idx); + } + + part->sealed = true; + } idx += d->vec.size; @@ -524,8 +553,7 @@ static int bus_kernel_make_message(sd_bus *bus, struct kdbus_msg *k, sd_bus_mess log_debug("Got unknown field from kernel %llu", d->type); } - if ((BUS_MESSAGE_FIELDS_SIZE(m) > 0 && !m->fields) || - (BUS_MESSAGE_BODY_SIZE(m) > 0 && !m->body)) { + if ((BUS_MESSAGE_FIELDS_SIZE(m) > 0 && !m->fields) || BUS_MESSAGE_SIZE(m) != idx) { sd_bus_message_unref(m); return -EBADMSG; } @@ -646,3 +674,79 @@ int bus_kernel_create(const char *name, char **s) { return fd; } + +int bus_kernel_pop_memfd(sd_bus *bus, void **address, size_t *size) { + struct memfd_cache *c; + + assert(address); + assert(size); + + if (!bus || !bus->is_kernel) + return -ENOTSUP; + + if (bus->n_memfd_cache <= 0) { + int fd; + + fd = ioctl(bus->input_fd, KDBUS_CMD_MEMFD_NEW, &fd); + if (fd < 0) + return -errno; + + *address = NULL; + *size = 0; + return fd; + } + + c = &bus->memfd_cache[-- bus->n_memfd_cache]; + + assert(c->fd >= 0); + assert(c->size == 0 || c->address); + + *address = c->address; + *size = c->size; + + return c->fd; +} + +void bus_kernel_push_memfd(sd_bus *bus, int fd, void *address, size_t size) { + struct memfd_cache *c; + + assert(fd >= 0); + assert(size == 0 || address); + + if (!bus || !bus->is_kernel || + bus->n_memfd_cache >= ELEMENTSOF(bus->memfd_cache)) { + + if (size > 0) + assert_se(munmap(address, PAGE_ALIGN(size)) == 0); + + close_nointr_nofail(fd); + return; + } + + c = &bus->memfd_cache[bus->n_memfd_cache++]; + c->fd = fd; + c->address = address; + + /* If overly long, let's return a bit to the OS */ + if (size > MEMFD_CACHE_ITEM_SIZE_MAX) { + uint64_t sz = MEMFD_CACHE_ITEM_SIZE_MAX; + + ioctl(bus->input_fd, KDBUS_CMD_MEMFD_SIZE_SET, &sz); + + c->size = MEMFD_CACHE_ITEM_SIZE_MAX; + } else + c->size = size; +} + +void bus_kernel_flush_memfd(sd_bus *b) { + unsigned i; + + assert(b); + + for (i = 0; i < b->n_memfd_cache; i++) { + if (b->memfd_cache[i].size > 0) + assert_se(munmap(b->memfd_cache[i].address, PAGE_ALIGN(b->memfd_cache[i].size)) == 0); + + close_nointr_nofail(b->memfd_cache[i].fd); + } +} diff --git a/src/libsystemd-bus/bus-kernel.h b/src/libsystemd-bus/bus-kernel.h index ac746afe037..ed3f987ccda 100644 --- a/src/libsystemd-bus/bus-kernel.h +++ b/src/libsystemd-bus/bus-kernel.h @@ -23,6 +23,15 @@ #include "sd-bus.h" +#define MEMFD_CACHE_MAX 32 +#define MEMFD_CACHE_ITEM_SIZE_MAX (128*1024) + +struct memfd_cache { + int fd; + void *address; + size_t size; +}; + int bus_kernel_connect(sd_bus *b); int bus_kernel_take_fd(sd_bus *b); @@ -30,3 +39,8 @@ int bus_kernel_write_message(sd_bus *bus, sd_bus_message *m); int bus_kernel_read_message(sd_bus *bus, sd_bus_message **m); int bus_kernel_create(const char *name, char **s); + +int bus_kernel_pop_memfd(sd_bus *bus, void **address, size_t *size); +void bus_kernel_push_memfd(sd_bus *bus, int fd, void *address, size_t size); + +void bus_kernel_flush_memfd(sd_bus *bus); diff --git a/src/libsystemd-bus/bus-message.c b/src/libsystemd-bus/bus-message.c index cc2b78a8c5c..7444a30a51f 100644 --- a/src/libsystemd-bus/bus-message.c +++ b/src/libsystemd-bus/bus-message.c @@ -21,6 +21,7 @@ #include #include +#include #include "util.h" #include "utf8.h" @@ -36,7 +37,65 @@ static int message_append_basic(sd_bus_message *m, char type, const void *p, const void **stored); -static void reset_containers(sd_bus_message *m) { +static void *adjust_pointer(const void *p, void *old_base, size_t sz, void *new_base) { + + if (p == NULL) + return NULL; + + if (old_base == new_base) + return (void*) p; + + if ((uint8_t*) p < (uint8_t*) old_base) + return (void*) p; + + if ((uint8_t*) p >= (uint8_t*) old_base + sz) + return (void*) p; + + return (uint8_t*) new_base + ((uint8_t*) p - (uint8_t*) old_base); +} + +static void message_free_part(sd_bus_message *m, struct bus_body_part *part) { + assert(m); + assert(part); + + if (part->memfd >= 0) { + + if (!part->sealed) + bus_kernel_push_memfd(m->bus, part->memfd, part->data, part->mapped); + else { + if (part->size > 0) + assert_se(munmap(part->data, PAGE_ALIGN(part->size)) == 0); + + close_nointr_nofail(part->memfd); + } + + } else if (part->free_this) + free(part->data); + + if (part != &m->body) + free(part); +} + +static void message_reset_parts(sd_bus_message *m) { + struct bus_body_part *part; + + assert(m); + + part = &m->body; + while (m->n_body_parts > 0) { + struct bus_body_part *next = part->next; + message_free_part(m, part); + part = next; + m->n_body_parts--; + } + + m->body_end = NULL; + + m->cached_rindex_part = NULL; + m->cached_rindex_part_begin = 0; +} + +static void message_reset_containers(sd_bus_message *m) { unsigned i; assert(m); @@ -60,8 +119,7 @@ static void message_free(sd_bus_message *m) { if (m->free_fields) free(m->fields); - if (m->free_body) - free(m->body); + message_reset_parts(m); if (m->free_kdbus) free(m->kdbus); @@ -77,9 +135,12 @@ static void message_free(sd_bus_message *m) { if (m->bus) sd_bus_unref(m->bus); + if (m->iovec != m->iovec_fixed) + free(m->iovec); + free(m->cmdline_array); - reset_containers(m); + message_reset_containers(m); free(m->root_container.signature); free(m->peeked_signature); @@ -91,7 +152,7 @@ static void message_free(sd_bus_message *m) { } static void* buffer_extend(void **p, uint32_t *sz, size_t align, size_t extend) { - size_t start, n; + size_t start, end; void *k; assert(p); @@ -99,15 +160,15 @@ static void* buffer_extend(void **p, uint32_t *sz, size_t align, size_t extend) assert(align > 0); start = ALIGN_TO((size_t) *sz, align); - n = start + extend; + end = start + extend; - if (n == *sz) + if (end == *sz) return (uint8_t*) *p + start; - if (n > (size_t) ((uint32_t) -1)) + if (end > (size_t) ((uint32_t) -1)) return NULL; - k = realloc(*p, n); + k = realloc(*p, end); if (!k) return NULL; @@ -116,38 +177,37 @@ static void* buffer_extend(void **p, uint32_t *sz, size_t align, size_t extend) memset((uint8_t*) k + *sz, 0, start - *sz); *p = k; - *sz = n; + *sz = end; return (uint8_t*) k + start; } static void *message_extend_fields(sd_bus_message *m, size_t align, size_t sz) { - void *p, *o; + void *p, *op; + size_t os; assert(m); - o = m->fields; - p = buffer_extend(&m->fields, &m->header->fields_size, align, sz); - if (!p) + if (m->poisoned) return NULL; - if (o != m->fields) { - /* Adjust quick access pointers */ + op = m->fields; + os = m->header->fields_size; - if (m->path) - m->path = (const char*) m->fields + (m->path - (const char*) o); - if (m->interface) - m->interface = (const char*) m->fields + (m->interface - (const char*) o); - if (m->member) - m->member = (const char*) m->fields + (m->member - (const char*) o); - if (m->destination) - m->destination = (const char*) m->fields + (m->destination - (const char*) o); - if (m->sender) - m->sender = (const char*) m->fields + (m->sender - (const char*) o); - if (m->error.name) - m->error.name = (const char*) m->fields + (m->error.name - (const char*) o); + p = buffer_extend(&m->fields, &m->header->fields_size, align, sz); + if (!p) { + m->poisoned = true; + return NULL; } + /* Adjust quick access pointers */ + m->path = adjust_pointer(m->path, op, os, m->fields); + m->interface = adjust_pointer(m->interface, op, os, m->fields); + m->member = adjust_pointer(m->member, op, os, m->fields); + m->destination = adjust_pointer(m->destination, op, os, m->fields); + m->sender = adjust_pointer(m->sender, op, os, m->fields); + m->error.name = adjust_pointer(m->error.name, op, os, m->fields); + m->free_fields = true; return p; @@ -183,7 +243,7 @@ static int message_append_field_string( memcpy(p + 8, s, l + 1); if (ret) - *ret = (const char*) p + 8; + *ret = (char*) p + 8; return 0; } @@ -331,9 +391,15 @@ int bus_message_from_malloc( } m->fields = (uint8_t*) buffer + sizeof(struct bus_header); - m->body = (uint8_t*) buffer + sizeof(struct bus_header) + ALIGN8(BUS_MESSAGE_FIELDS_SIZE(m)); + + m->n_body_parts = 1; + m->body.data = (uint8_t*) buffer + sizeof(struct bus_header) + ALIGN8(BUS_MESSAGE_FIELDS_SIZE(m)); + m->body.size = length - sizeof(struct bus_header) - ALIGN8(BUS_MESSAGE_FIELDS_SIZE(m)); + m->body.sealed = true; + m->body.memfd = -1; m->n_iovec = 1; + m->iovec = m->iovec_fixed; m->iovec[0].iov_base = buffer; m->iovec[0].iov_len = length; @@ -1022,35 +1088,188 @@ static struct bus_container *message_get_container(sd_bus_message *m) { return m->containers + m->n_containers - 1; } +struct bus_body_part *message_append_part(sd_bus_message *m) { + struct bus_body_part *part; + + assert(m); + + if (m->poisoned) + return NULL; + + if (m->n_body_parts <= 0) { + part = &m->body; + zero(*part); + } else { + assert(m->body_end); + + part = new0(struct bus_body_part, 1); + if (!part) { + m->poisoned = true; + return NULL; + } + + m->body_end->next = part; + } + + part->memfd = -1; + m->body_end = part; + m->n_body_parts ++; + + return part; +} + +static void part_zero(struct bus_body_part *part, size_t sz) { + assert(part); + assert(sz > 0); + assert(sz < 8); + + part->data = NULL; + part->size = sz; +} + +static int part_make_space( + struct sd_bus_message *m, + struct bus_body_part *part, + size_t sz, + void **q) { + + void *n; + int r; + + assert(m); + assert(part); + assert(!part->sealed); + + if (m->poisoned) + return -ENOMEM; + + if (!part->data && part->memfd < 0) + part->memfd = bus_kernel_pop_memfd(m->bus, &part->data, &part->mapped); + + if (part->memfd >= 0) { + uint64_t u = sz; + + r = ioctl(part->memfd, KDBUS_CMD_MEMFD_SIZE_SET, &u); + if (r < 0) { + m->poisoned = true; + return -errno; + } + + if (sz > part->mapped) { + size_t psz = PAGE_ALIGN(sz); + + if (part->mapped <= 0) + n = mmap(NULL, psz, PROT_READ|PROT_WRITE, MAP_SHARED, part->memfd, 0); + else + n = mremap(part->data, part->mapped, psz, MREMAP_MAYMOVE); + + if (n == MAP_FAILED) { + m->poisoned = true; + return -errno; + } + + part->mapped = psz; + part->data = n; + } + } else { + n = realloc(part->data, sz); + if (!n) { + m->poisoned = true; + return -ENOMEM; + } + + part->data = n; + part->free_this = true; + } + + if (q) + *q = part->data ? (uint8_t*) part->data + part->size : NULL; + + part->size = sz; + return 0; +} + static void *message_extend_body(sd_bus_message *m, size_t align, size_t sz) { - void *p, *o; - size_t added; struct bus_container *c; + struct bus_body_part *part = NULL; + size_t start_body, end_body, padding, start_part, end_part, added; + bool add_new_part; + void *p; + int r; assert(m); assert(align > 0); + assert(!m->sealed); - o = m->body; - added = m->header->body_size; - - p = buffer_extend(&m->body, &m->header->body_size, align, sz); - if (!p) + if (m->poisoned) return NULL; - added = m->header->body_size - added; + start_body = ALIGN_TO((size_t) m->header->body_size, align); + end_body = start_body + sz; - for (c = m->containers; c < m->containers + m->n_containers; c++) - if (c->array_size) { - c->array_size = (uint32_t*) ((uint8_t*) m->body + ((uint8_t*) c->array_size - (uint8_t*) o)); - *c->array_size += added; - } + padding = start_body - m->header->body_size; + added = padding + sz; - if (o != m->body) { - if (m->error.message) - m->error.message = (const char*) m->body + (m->error.message - (const char*) o); + /* Check for 32bit overflows */ + if (end_body > (size_t) ((uint32_t) -1)) { + m->poisoned = true; + return NULL; } - m->free_body = true; + add_new_part = + m->n_body_parts <= 0 || + m->body_end->sealed || + padding != ALIGN_TO(m->body_end->size, align) - m->body_end->size; + + if (add_new_part) { + if (padding > 0) { + part = message_append_part(m); + if (!part) + return NULL; + + part_zero(part, padding); + } + + part = message_append_part(m); + if (!part) + return NULL; + + r = part_make_space(m, part, sz, &p); + if (r < 0) + return NULL; + } else { + void *op; + size_t os; + + part = m->body_end; + op = part->data; + os = part->size; + + start_part = ALIGN_TO(part->size, align); + end_part = start_part + sz; + + r = part_make_space(m, part, end_part, &p); + if (r < 0) + return NULL; + + if (padding > 0) { + memset(p, 0, padding); + p = (uint8_t*) p + padding; + } + + /* Readjust pointers */ + for (c = m->containers; c < m->containers + m->n_containers; c++) + c->array_size = adjust_pointer(c->array_size, op, os, part->data); + + m->error.message = (const char*) adjust_pointer(m->error.message, op, os, part->data); + } + + /* Update counters */ + for (c = m->containers; c < m->containers + m->n_containers; c++) + if (c->array_size) + *c->array_size += added; + + m->header->body_size = end_body; return p; } @@ -1073,6 +1292,8 @@ int message_append_basic(sd_bus_message *m, char type, const void *p, const void return -EPERM; if (!bus_type_is_basic(type)) return -EINVAL; + if (m->poisoned) + return -ESTALE; c = message_get_container(m); @@ -1087,8 +1308,10 @@ int message_append_basic(sd_bus_message *m, char type, const void *p, const void return -ENXIO; e = strextend(&c->signature, CHAR_TO_STR(type), NULL); - if (!e) + if (!e) { + m->poisoned = true; return -ENOMEM; + } } switch (type) { @@ -1139,6 +1362,7 @@ int message_append_basic(sd_bus_message *m, char type, const void *p, const void f = realloc(m->fds, sizeof(int) * (m->n_fds + 1)); if (!f) { + m->poisoned = true; r = -ENOMEM; goto fail; } @@ -1199,10 +1423,6 @@ int message_append_basic(sd_bus_message *m, char type, const void *p, const void return 0; fail: - /* Truncate extended signature again */ - if (e) - c->signature[c->index] = 0; - if (fd >= 0) close_nointr_nofail(fd); @@ -1217,7 +1437,6 @@ int sd_bus_message_append_string_space(sd_bus_message *m, size_t size, char **s) struct bus_container *c; char *e; void *a; - int r; if (!m) return -EINVAL; @@ -1225,6 +1444,8 @@ int sd_bus_message_append_string_space(sd_bus_message *m, size_t size, char **s) return -EINVAL; if (m->sealed) return -EPERM; + if (m->poisoned) + return -ESTALE; c = message_get_container(m); @@ -1239,16 +1460,16 @@ int sd_bus_message_append_string_space(sd_bus_message *m, size_t size, char **s) return -ENXIO; e = strextend(&c->signature, CHAR_TO_STR(SD_BUS_TYPE_STRING), NULL); - if (!e) + if (!e) { + m->poisoned = true; return -ENOMEM; + } } a = message_extend_body(m, 4, 4 + size + 1); - if (!a) { - r = -ENOMEM; - goto fail; - } + if (!a) + return -ENOMEM; *(uint32_t*) a = size; *s = (char*) a + 4; @@ -1259,12 +1480,6 @@ int sd_bus_message_append_string_space(sd_bus_message *m, size_t size, char **s) c->index++; return 0; - -fail: - if (e) - c->signature[c->index] = 0; - - return r; } static int bus_message_open_array( @@ -1275,9 +1490,10 @@ static int bus_message_open_array( unsigned nindex; char *e = NULL; - void *a, *b; + void *a, *op; int alignment; - size_t saved; + size_t os; + struct bus_body_part *o; assert(m); assert(c); @@ -1309,39 +1525,34 @@ static int bus_message_open_array( /* Extend the existing signature */ e = strextend(&c->signature, CHAR_TO_STR(SD_BUS_TYPE_ARRAY), contents, NULL); - if (!e) + if (!e) { + m->poisoned = true; return -ENOMEM; + } nindex = e - c->signature; } - saved = m->header->body_size; a = message_extend_body(m, 4, 4); - if (!a) { - /* Truncate extended signature again */ - if (e) - c->signature[c->index] = 0; - + if (!a) return -ENOMEM; - } - b = m->body; - if (!message_extend_body(m, alignment, 0)) { - /* Add alignment between size and first element */ - if (e) - c->signature[c->index] = 0; + o = m->body_end; + op = m->body_end->data; + os = m->body_end->size; - m->header->body_size = saved; + /* Add alignment between size and first element */ + if (!message_extend_body(m, alignment, 0)) return -ENOMEM; - } if (c->enclosing != SD_BUS_TYPE_ARRAY) c->index = nindex; - /* m->body might have changed so let's readjust a */ - a = (uint8_t*) m->body + ((uint8_t*) a - (uint8_t*) b); - *(uint32_t*) a = 0; + /* location of array size might have changed so let's readjust a */ + if (o == m->body_end) + a = adjust_pointer(a, op, os, m->body_end->data); + *(uint32_t*) a = 0; *array_size = a; return 0; } @@ -1375,19 +1586,16 @@ static int bus_message_open_variant( return -ENXIO; e = strextend(&c->signature, CHAR_TO_STR(SD_BUS_TYPE_VARIANT), NULL); - if (!e) + if (!e) { + m->poisoned = true; return -ENOMEM; + } } l = strlen(contents); a = message_extend_body(m, 1, 1 + l + 1); - if (!a) { - /* Truncate extended signature again */ - if (e) - c->signature[c->index] = 0; - + if (!a) return -ENOMEM; - } *(uint8_t*) a = l; memcpy((uint8_t*) a + 1, contents, l + 1); @@ -1429,19 +1637,17 @@ static int bus_message_open_struct( return -ENXIO; e = strextend(&c->signature, CHAR_TO_STR(SD_BUS_TYPE_STRUCT_BEGIN), contents, CHAR_TO_STR(SD_BUS_TYPE_STRUCT_END), NULL); - if (!e) + if (!e) { + m->poisoned = true; return -ENOMEM; + } nindex = e - c->signature; } /* Align contents to 8 byte boundary */ - if (!message_extend_body(m, 8, 0)) { - if (e) - c->signature[c->index] = 0; - + if (!message_extend_body(m, 8, 0)) return -ENOMEM; - } if (c->enclosing != SD_BUS_TYPE_ARRAY) c->index = nindex; @@ -1507,18 +1713,25 @@ int sd_bus_message_open_container( return -EPERM; if (!contents) return -EINVAL; + if (m->poisoned) + return -ESTALE; /* Make sure we have space for one more container */ w = realloc(m->containers, sizeof(struct bus_container) * (m->n_containers + 1)); - if (!w) + if (!w) { + m->poisoned = true; return -ENOMEM; + } + m->containers = w; c = message_get_container(m); signature = strdup(contents); - if (!signature) + if (!signature) { + m->poisoned = true; return -ENOMEM; + } /* Save old index in the parent container, in case we have to * abort this container */ @@ -1562,6 +1775,8 @@ int sd_bus_message_close_container(sd_bus_message *m) { return -EPERM; if (m->n_containers <= 0) return -EINVAL; + if (m->poisoned) + return -ESTALE; c = message_get_container(m); if (c->enclosing != SD_BUS_TYPE_ARRAY) @@ -1574,37 +1789,6 @@ int sd_bus_message_close_container(sd_bus_message *m) { return 0; } -static void message_abort_container(sd_bus_message *m) { - struct bus_container *c; - size_t delta; - - assert(m); - assert(!m->sealed); - assert(m->n_containers > 0); - - c = message_get_container(m); - - /* Undo appends */ - assert(m->header->body_size >= c->before); - delta = m->header->body_size - c->before; - m->header->body_size = c->before; - - /* Free container */ - free(c->signature); - m->n_containers--; - - /* Correct index of new top-level container */ - c = message_get_container(m); - c->index = c->saved_index; - - /* Correct array sizes all the way up */ - for (c = m->containers; c < m->containers + m->n_containers; c++) - if (c->array_size) { - assert(*c->array_size >= delta); - *c->array_size -= delta; - } -} - typedef struct { const char *types; unsigned n_struct; @@ -1849,6 +2033,8 @@ int sd_bus_message_append(sd_bus_message *m, const char *types, ...) { return -EINVAL; if (m->sealed) return -EPERM; + if (m->poisoned) + return -ESTALE; if (!types) return 0; @@ -1872,6 +2058,8 @@ int sd_bus_message_append_array_space(sd_bus_message *m, char type, size_t size, return -EINVAL; if (!ptr && size > 0) return -EINVAL; + if (m->poisoned) + return -ESTALE; align = bus_type_get_alignment(type); sz = bus_type_get_size(type); @@ -1887,21 +2075,15 @@ int sd_bus_message_append_array_space(sd_bus_message *m, char type, size_t size, return r; a = message_extend_body(m, align, size); - if (!a) { - r = -ENOMEM; - goto fail; - } + if (!a) + return -ENOMEM; r = sd_bus_message_close_container(m); if (r < 0) - goto fail; + return r; *ptr = a; return 0; - -fail: - message_abort_container(m); - return r; } int sd_bus_message_append_array(sd_bus_message *m, char type, const void *ptr, size_t size) { @@ -1922,15 +2104,15 @@ int sd_bus_message_append_array(sd_bus_message *m, char type, const void *ptr, s } static int buffer_peek(const void *p, uint32_t sz, size_t *rindex, size_t align, size_t nbytes, void **r) { - size_t k, start, n; + size_t k, start, end; assert(rindex); assert(align > 0); start = ALIGN_TO((size_t) *rindex, align); - n = start + nbytes; + end = start + nbytes; - if (n > sz) + if (end > sz) return -EBADMSG; /* Verify that padding is 0 */ @@ -1941,7 +2123,7 @@ static int buffer_peek(const void *p, uint32_t sz, size_t *rindex, size_t align, if (r) *r = (uint8_t*) p + start; - *rindex = n; + *rindex = end; return 1; } @@ -1958,7 +2140,50 @@ static bool message_end_of_array(sd_bus_message *m, size_t index) { return index >= c->begin + BUS_MESSAGE_BSWAP32(m, *c->array_size); } -static int message_peek_body(sd_bus_message *m, size_t *rindex, size_t align, size_t nbytes, void **ret) { +static struct bus_body_part* find_part(sd_bus_message *m, size_t index, size_t sz, void **p) { + struct bus_body_part *part; + size_t begin; + assert(m); + + if (m->cached_rindex_part && index >= m->cached_rindex_part_begin) { + part = m->cached_rindex_part; + begin = m->cached_rindex_part_begin; + } else { + part = &m->body; + begin = 0; + } + + while (part) { + if (index < begin) + return NULL; + + if (index + sz <= begin + part->size) { + if (p) + *p = part->data ? (uint8_t*) part->data + index - begin : NULL; + + m->cached_rindex_part = part; + m->cached_rindex_part_begin = begin; + + return part; + } + + part = part->next; + } + + return NULL; +} + +static int message_peek_body( + sd_bus_message *m, + size_t *rindex, + size_t align, + size_t nbytes, + void **ret) { + + size_t k, start, end, padding; + struct bus_body_part *part; + uint8_t *q; + assert(m); assert(rindex); assert(align > 0); @@ -1966,7 +2191,34 @@ static int message_peek_body(sd_bus_message *m, size_t *rindex, size_t align, si if (message_end_of_array(m, *rindex)) return 0; - return buffer_peek(m->body, BUS_MESSAGE_BODY_SIZE(m), rindex, align, nbytes, ret); + start = ALIGN_TO((size_t) *rindex, align); + padding = start - *rindex; + end = start + nbytes; + + if (end > BUS_MESSAGE_BODY_SIZE(m)) + return -EBADMSG; + + part = find_part(m, *rindex, padding, (void**) &q); + if (!part) + return -EBADMSG; + + if (q) { + /* Verify padding */ + for (k = 0; k < padding; k++) + if (q[k] != 0) + return -EBADMSG; + } + + part = find_part(m, start, nbytes, (void**) &q); + if (!part || !q) + return -EBADMSG; + + *rindex = end; + + if (ret) + *ret = q; + + return 1; } static bool validate_nul(const char *s, size_t l) { @@ -2601,7 +2853,7 @@ int sd_bus_message_rewind(sd_bus_message *m, int complete) { return -EPERM; if (complete) { - reset_containers(m); + message_reset_containers(m); m->rindex = 0; m->root_container.index = 0; @@ -3620,6 +3872,8 @@ int bus_message_dump(sd_bus_message *m) { int bus_message_get_blob(sd_bus_message *m, void **buffer, size_t *sz) { size_t total; void *p, *e; + unsigned i; + struct bus_body_part *part; assert(m); assert(buffer); @@ -3640,8 +3894,8 @@ int bus_message_get_blob(sd_bus_message *m, void **buffer, size_t *sz) { e = mempset(e, 0, 8 - (m->header->fields_size % 8)); } - if (m->body) - e = mempcpy(e, m->body, m->header->body_size); + for (i = 0, part = &m->body; i < m->n_body_parts; i++, part = part->next) + e = mempcpy(e, part->data, part->size); assert(total == (size_t) ((uint8_t*) e - (uint8_t*) p)); diff --git a/src/libsystemd-bus/bus-message.h b/src/libsystemd-bus/bus-message.h index 86a41a73b68..1c50c7357a1 100644 --- a/src/libsystemd-bus/bus-message.h +++ b/src/libsystemd-bus/bus-message.h @@ -50,6 +50,16 @@ struct bus_header { uint32_t fields_size; } _packed_; +struct bus_body_part { + void *data; + size_t size; + size_t mapped; + int memfd; + bool free_this:1; + bool sealed:1; + struct bus_body_part *next; +}; + struct sd_bus_message { unsigned n_ref; @@ -80,19 +90,22 @@ struct sd_bus_message { bool gid_valid:1; bool free_header:1; bool free_fields:1; - bool free_body:1; bool free_kdbus:1; bool free_fds:1; bool release_kdbus:1; + bool poisoned:1; struct bus_header *header; void *fields; - void *body; - struct kdbus_msg *kdbus; + struct bus_body_part body; + struct bus_body_part *body_end; + unsigned n_body_parts; char *label; size_t rindex; + struct bus_body_part *cached_rindex_part; + size_t cached_rindex_part_begin; uint32_t n_fds; int *fds; @@ -100,9 +113,12 @@ struct sd_bus_message { struct bus_container root_container, *containers; unsigned n_containers; - struct iovec iovec[3]; + struct iovec *iovec; + struct iovec iovec_fixed[3]; unsigned n_iovec; + struct kdbus_msg *kdbus; + char *peeked_signature; usec_t timeout; @@ -199,3 +215,5 @@ int bus_message_append_ap(sd_bus_message *m, const char *types, va_list ap); int bus_message_parse_fields(sd_bus_message *m); int bus_header_size(struct bus_header *h, size_t *sum); + +struct bus_body_part *message_append_part(sd_bus_message *m); diff --git a/src/libsystemd-bus/bus-socket.c b/src/libsystemd-bus/bus-socket.c index 8a86b02c681..4dafe1fdb7f 100644 --- a/src/libsystemd-bus/bus-socket.c +++ b/src/libsystemd-bus/bus-socket.c @@ -58,7 +58,7 @@ static void iovec_advance(struct iovec iov[], unsigned *idx, size_t size) { } } -static void append_iovec(sd_bus_message *m, const void *p, size_t sz) { +static int append_iovec(sd_bus_message *m, const void *p, size_t sz) { assert(m); assert(p); assert(sz > 0); @@ -66,22 +66,51 @@ static void append_iovec(sd_bus_message *m, const void *p, size_t sz) { m->iovec[m->n_iovec].iov_base = (void*) p; m->iovec[m->n_iovec].iov_len = sz; m->n_iovec++; + + return 0; } -static void bus_message_setup_iovec(sd_bus_message *m) { +static int bus_message_setup_iovec(sd_bus_message *m) { + struct bus_body_part *part; + unsigned n; + int r; + assert(m); assert(m->sealed); if (m->n_iovec > 0) - return; + return 0; - append_iovec(m, m->header, sizeof(*m->header)); + assert(!m->iovec); - if (m->fields) - append_iovec(m, m->fields, ALIGN8(m->header->fields_size)); + n = 1 + !!m->fields + m->n_body_parts; + if (n < ELEMENTSOF(m->iovec_fixed)) + m->iovec = m->iovec_fixed; + else { + m->iovec = new(struct iovec, n); + if (!m->iovec) + return -ENOMEM; + } - if (m->body) - append_iovec(m, m->body, m->header->body_size); + r = append_iovec(m, m->header, sizeof(*m->header)); + if (r < 0) + return r; + + if (m->fields) { + r = append_iovec(m, m->fields, ALIGN8(m->header->fields_size)); + if (r < 0) + return r; + } + + for (part = &m->body; part && part->size > 0; part = part->next) { + r = append_iovec(m, part->data, part->size); + if (r < 0) + return r; + } + + assert(n == m->n_iovec); + + return 0; } bool bus_socket_auth_needs_write(sd_bus *b) { @@ -749,6 +778,7 @@ int bus_socket_write_message(sd_bus *bus, sd_bus_message *m, size_t *idx) { ssize_t k; size_t n; unsigned j; + int r; assert(bus); assert(m); @@ -758,7 +788,9 @@ int bus_socket_write_message(sd_bus *bus, sd_bus_message *m, size_t *idx) { if (*idx >= BUS_MESSAGE_SIZE(m)) return 0; - bus_message_setup_iovec(m); + r = bus_message_setup_iovec(m); + if (r < 0) + return r; n = m->n_iovec * sizeof(struct iovec); iov = alloca(n); diff --git a/src/libsystemd-bus/sd-bus.c b/src/libsystemd-bus/sd-bus.c index 7d6d848ec57..2537ba52d59 100644 --- a/src/libsystemd-bus/sd-bus.c +++ b/src/libsystemd-bus/sd-bus.c @@ -86,9 +86,10 @@ static void bus_free(sd_bus *b) { } hashmap_free(b->object_callbacks); - bus_match_free(&b->match_callbacks); + bus_kernel_flush_memfd(b); + free(b); }