hexdecoct: several cleanups for base64_append()

- add missing assertions,
- use size_t for buffser size or memory index,
- handle empty input more gracefully,
- return the length or the result string,
- fix off-by-one issue when the prefix is already long enough.

(cherry picked from commit c21b316964)
(cherry picked from commit acb0414a1f)

src/basic/hexdecoct.c

@@ -649,58 +649,74 @@ ssize_t base64mem_full(
         return z - r;
 }
 
-static int base64_append_width(
-                char **prefix, int plen,
-                char sep, int indent,
-                const void *p, size_t l,
-                int width) {
+static ssize_t base64_append_width(
+                char **prefix,
+                size_t plen,
+                char sep,
+                size_t indent,
+                const void *p,
+                size_t l,
+                size_t width) {
 
         _cleanup_free_ char *x = NULL;
         char *t, *s;
-        ssize_t len, avail, line, lines;
+        size_t lines;
+        ssize_t len;
+
+        assert(prefix);
+        assert(*prefix || plen == 0);
+        assert(p || l == 0);
 
         len = base64mem(p, l, &x);
-        if (len <= 0)
+        if (len < 0)
                 return len;
+        if (len == 0)
+                return plen;
 
         lines = DIV_ROUND_UP(len, width);
 
-        if ((size_t) plen >= SSIZE_MAX - 1 - 1 ||
+        if (plen >= SSIZE_MAX - 1 - 1 ||
             lines > (SSIZE_MAX - plen - 1 - 1) / (indent + width + 1))
                 return -ENOMEM;
 
-        t = realloc(*prefix, (ssize_t) plen + 1 + 1 + (indent + width + 1) * lines);
+        t = realloc(*prefix, plen + 1 + 1 + (indent + width + 1) * lines);
         if (!t)
                 return -ENOMEM;
 
-        t[plen] = sep;
+        s = t + plen;
+        for (size_t line = 0; line < lines; line++) {
+                size_t act = MIN(width, (size_t) len);
 
-        for (line = 0, s = t + plen + 1, avail = len; line < lines; line++) {
-                int act = MIN(width, avail);
+                if (line > 0)
+                        sep = '\n';
 
-                if (line > 0 || sep == '\n') {
-                        memset(s, ' ', indent);
-                        s += indent;
+                if (s > t) {
+                        *s++ = sep;
+                        if (sep == '\n')
+                                s = mempset(s, ' ', indent);
                 }
 
                 s = mempcpy(s, x + width * line, act);
-                *(s++) = line < lines - 1 ? '\n' : '\0';
-                avail -= act;
+                len -= act;
         }
-        assert(avail == 0);
+        assert(len == 0);
 
+        *s = '\0';
         *prefix = t;
-        return 0;
+        return s - t;
 }
 
-int base64_append(
-                char **prefix, int plen,
-                const void *p, size_t l,
-                int indent, int width) {
+ssize_t base64_append(
+                char **prefix,
+                size_t plen,
+                const void *p,
+                size_t l,
+                size_t indent,
+                size_t width) {
 
         if (plen > width / 2 || plen + indent > width)
                 /* leave indent on the left, keep last column free */
-                return base64_append_width(prefix, plen, '\n', indent, p, l, width - indent - 1);
+                return base64_append_width(prefix, plen, '\n', indent, p, l, width - indent);
         else
                 /* leave plen on the left, keep last column free */
                 return base64_append_width(prefix, plen, ' ', plen + 1, p, l, width - plen - 1);

src/basic/hexdecoct.h

@@ -38,9 +38,13 @@ static inline ssize_t base64mem(const void *p, size_t l, char **ret) {
         return base64mem_full(p, l, SIZE_MAX, ret);
 }
 
-int base64_append(char **prefix, int plen,
-                  const void *p, size_t l,
-                  int margin, int width);
+ssize_t base64_append(
+                char **prefix,
+                size_t plen,
+                const void *p,
+                size_t l,
+                size_t margin,
+                size_t width);
 int unbase64mem_full(const char *p, size_t l, bool secure, void **mem, size_t *len);
 static inline int unbase64mem(const char *p, size_t l, void **mem, size_t *len) {
         return unbase64mem_full(p, l, false, mem, len);