basic: util - fix errorhandling in unhexmem()

We were ignoring failures from unhexchar, which meant that invalid
hex characters were being turned into garbage rather than the string
rejected.

Fix this by making unhexmem return an error code, also change the API
slightly, to return the size of the returned memory, reflecting the
fact that the memory is a binary blob,and not a string.

For convenience, still append a trailing NULL byte to the returned
memory (not included in the returned size), allowing callers to
treat it as a string without doing a second copy.

src/basic/util.c

@@ -916,30 +916,42 @@ char *hexmem(const void *p, size_t l) {
         return r;
 }
 
-void *unhexmem(const char *p, size_t l) {
-        uint8_t *r, *z;
+int unhexmem(const char *p, size_t l, void **mem, size_t *len) {
+        _cleanup_free_ uint8_t *r = NULL;
+        uint8_t *z;
         const char *x;
 
+        assert(mem);
+        assert(len);
         assert(p);
 
         z = r = malloc((l + 1) / 2 + 1);
         if (!r)
-                return NULL;
+                return -ENOMEM;
 
         for (x = p; x < p + l; x += 2) {
                 int a, b;
 
                 a = unhexchar(x[0]);
-                if (x+1 < p + l)
+                if (a < 0)
+                        return a;
+                else if (x+1 < p + l) {
                         b = unhexchar(x[1]);
-                else
+                        if (b < 0)
+                                return b;
+                } else
                         b = 0;
 
                 *(z++) = (uint8_t) a << 4 | (uint8_t) b;
         }
 
         *z = 0;
-        return r;
+
+        *mem = r;
+        r = NULL;
+        *len = (l + 1) / 2;
+
+        return 0;
 }
 
 char octchar(int x) {

src/basic/util.h

@@ -614,7 +614,7 @@ static inline void *mempset(void *s, int c, size_t n) {
 }
 
 char *hexmem(const void *p, size_t l);
-void *unhexmem(const char *p, size_t l);
+int unhexmem(const char *p, size_t l, void **mem, size_t *len);
 
 char *strextend(char **x, ...) _sentinel_;
 char *strrep(const char *s, unsigned n);

src/libsystemd-network/sd-dhcp-lease.c

@@ -811,13 +811,12 @@ int sd_dhcp_lease_load(sd_dhcp_lease **ret, const char *lease_file) {
         }
 
         if (client_id_hex) {
-                if (strlen (client_id_hex) % 2)
+                if (strlen(client_id_hex) % 2)
                         return -EINVAL;
 
-                lease->client_id = unhexmem (client_id_hex, strlen (client_id_hex));
-                if (!lease->client_id)
-                        return -ENOMEM;
-                lease->client_id_len = strlen (client_id_hex) / 2;
+                r = unhexmem(client_id_hex, strlen(client_id_hex), (void**) &lease->client_id, &lease->client_id_len);
+                if (r < 0)
+                        return r;
         }
 
         *ret = lease;

src/libsystemd/sd-bus/bus-socket.c

@@ -264,6 +264,8 @@ static bool line_begins(const char *s, size_t m, const char *word) {
 
 static int verify_anonymous_token(sd_bus *b, const char *p, size_t l) {
         _cleanup_free_ char *token = NULL;
+        size_t len;
+        int r;
 
         if (!b->anonymous_auth)
                 return 0;
@@ -276,11 +278,12 @@ static int verify_anonymous_token(sd_bus *b, const char *p, size_t l) {
 
         if (l % 2 != 0)
                 return 0;
-        token = unhexmem(p, l);
-        if (!token)
-                return -ENOMEM;
 
-        if (memchr(token, 0, l/2))
+        r = unhexmem(p, l, (void **) &token, &len);
+        if (r < 0)
+                return 0;
+
+        if (memchr(token, 0, len))
                 return 0;
 
         return !!utf8_is_valid(token);
@@ -288,6 +291,7 @@ static int verify_anonymous_token(sd_bus *b, const char *p, size_t l) {
 
 static int verify_external_token(sd_bus *b, const char *p, size_t l) {
         _cleanup_free_ char *token = NULL;
+        size_t len;
         uid_t u;
         int r;
 
@@ -307,11 +311,11 @@ static int verify_external_token(sd_bus *b, const char *p, size_t l) {
         if (l % 2 != 0)
                 return 0;
 
-        token = unhexmem(p, l);
-        if (!token)
-                return -ENOMEM;
+        r = unhexmem(p, l, (void**) &token, &len);
+        if (r < 0)
+                return 0;
 
-        if (memchr(token, 0, l/2))
+        if (memchr(token, 0, len))
                 return 0;
 
         r = parse_uid(token, &u);

src/test/test-util.c

@@ -410,6 +410,30 @@ static void test_undecchar(void) {
         assert_se(undecchar('9') == 9);
 }
 
+static void test_unhexmem(void) {
+        const char *hex = "efa214921";
+        const char *hex_invalid = "efa214921o";
+        _cleanup_free_ char *hex2 = NULL;
+        _cleanup_free_ void *mem = NULL;
+        size_t len;
+
+        assert_se(unhexmem(hex, strlen(hex), &mem, &len) == 0);
+        assert_se(unhexmem(hex, strlen(hex) + 1, &mem, &len) == -EINVAL);
+        assert_se(unhexmem(hex_invalid, strlen(hex_invalid), &mem, &len) == -EINVAL);
+
+        assert_se((hex2 = hexmem(mem, len)));
+
+        free(mem);
+
+        assert_se(memcmp(hex, hex2, strlen(hex)) == 0);
+
+        free(hex2);
+
+        assert_se(unhexmem(hex, strlen(hex) - 1, &mem, &len) == 0);
+        assert_se((hex2 = hexmem(mem, len)));
+        assert_se(memcmp(hex, hex2, strlen(hex) - 1) == 0);
+}
+
 static void test_cescape(void) {
         _cleanup_free_ char *escaped;
 
@@ -1808,6 +1832,7 @@ int main(int argc, char *argv[]) {
         test_unoctchar();
         test_decchar();
         test_undecchar();
+        test_unhexmem();
         test_cescape();
         test_cunescape();
         test_foreach_word();