backported LZMA payload compression from SuSE

build/pack.c

@@ -441,6 +441,11 @@ int writeRPM(Header *hdrp, const char *fileName, int type,
 	    /* Add prereq on rpm version that understands bzip2 payloads */
 	    (void) rpmlibNeedsFeature(h, "PayloadIsBzip2", "3.0.5-1");
 	}
+	if (s[1] == 'l' && s[2] == 'z') {
+	    (void) headerAddEntry(h, RPMTAG_PAYLOADCOMPRESSOR, RPM_STRING_TYPE,
+		"lzma", 1);
+	    (void) rpmlibNeedsFeature(h, "PayloadIsLzma", "4.4.2-1");
+	}
 	strcpy(buf, rpmio_flags);
 	buf[s - rpmio_flags] = '\0';
 	(void) headerAddEntry(h, RPMTAG_PAYLOADFLAGS, RPM_STRING_TYPE, buf+1, 1);

lib/psm.c

@@ -2066,6 +2066,8 @@ assert(psm->mi == NULL);
 	    t = stpcpy(t, ".gzdio");
 	if (!strcmp(payload_compressor, "bzip2"))
 	    t = stpcpy(t, ".bzdio");
+	if (!strcmp(payload_compressor, "lzma"))
+	    t = stpcpy(t, ".lzdio");
 	rc = RPMRC_OK;
     }	break;
 

lib/rpmlibprov.c

@@ -22,6 +22,9 @@ static struct rpmlibProvides_s {
     { "rpmlib(PayloadIsBzip2)",		"3.0.5-1",
 	(RPMSENSE_RPMLIB|RPMSENSE_EQUAL),
     N_("package payload is compressed using bzip2.") },
+    { "rpmlib(PayloadIsLzma)",         "4.4.2-1",
+	(RPMSENSE_RPMLIB|RPMSENSE_EQUAL),
+    N_("package payload can be compressed using lzma.") },
     { "rpmlib(PayloadFilesHavePrefix)",	"4.0-1",
 	(RPMSENSE_RPMLIB|RPMSENSE_EQUAL),
     N_("package payload file(s) have \"./\" prefix.") },

rpm-4_0.spec

@@ -51,8 +51,8 @@ Requires: glibc-core
 
 BuildPreReq: automake >= 1.7.1, autoconf >= 2.53, rpm >= 3.0.6-ipl24mdk, %_bindir/subst
 
-# Automatically added by buildreq on Tue Mar 25 2008 and edited manually.
+# Automatically added by buildreq on Sat May 24 2008 and edited manually.
-BuildRequires: bzlib-devel-static glibc-devel-static libbeecrypt-devel-static libdb4.4-devel-static libpopt-devel-static zlib-devel-static
+BuildRequires: bzlib-devel-static libbeecrypt-devel-static libdb4.4-devel-static libelf-devel-static liblzma-devel-static libpopt-devel-static python-devel zlib-devel-static
 
 %package -n lib%name
 Summary: Shared libraries required for applications which will manipulate RPM packages

rpm2cpio.c

@@ -53,6 +53,8 @@ int main(int argc, char **argv)
 	    t = stpcpy(t, ".gzdio");
 	if (!strcmp(payload_compressor, "bzip2"))
 	    t = stpcpy(t, ".bzdio");
+	if (!strcmp(payload_compressor, "lzma"))
+	    t = stpcpy(t, ".lzdio");
     }
 
     gzdi = Fdopen(fdi, rpmio_flags);	/* XXX gzdi == fdi */

rpmio/Makefile.am

@@ -18,7 +18,7 @@ lib_LTLIBRARIES = librpmio.la
 librpmio_la_SOURCES = digest.c macro.c rpmio.c rpmlog.c rpmmalloc.c \
 	rpmpgp.c rpmrpc.c strcasecmp.c stubs.c url.c ugid.c
 librpmio_la_LDFLAGS = -release @VERSION@
-librpmio_la_LIBADD = @LIBBEECRYPT@ @LIBBZ2@ @LIBZ@
+librpmio_la_LIBADD = @LIBBEECRYPT@ @LIBBZ2@ @LIBZ@ -llzma
 
 clean-local:
 	rm -f *.o

rpmio/rpmio.c

@@ -86,6 +86,7 @@ static int inet_aton(const char *cp, struct in_addr *inp)
 #define	FDONLY(fd)	assert(fdGetIo(fd) == fdio)
 #define	GZDONLY(fd)	assert(fdGetIo(fd) == gzdio)
 #define	BZDONLY(fd)	assert(fdGetIo(fd) == bzdio)
+#define	LZDONLY(fd)	assert(fdGetIo(fd) == lzdio)
 
 #define	UFDONLY(fd)	/* assert(fdGetIo(fd) == ufdio) */
 
@@ -183,6 +184,8 @@ static /*@observer@*/ const char * fdbg(/*@null@*/ FD_t fd)
 	} else if (fps->io == bzdio) {
 	    sprintf(be, "BZD %p fdno %d", fps->fp, fps->fdno);
 #endif
+	} else if (fps->io == lzdio) {
+	    sprintf(be, "LZD %p fdno %d", fps->fp, fps->fdno);
 	} else if (fps->io == fpio) {
 	    /*@+voidabstract@*/
 	    sprintf(be, "%s %p(%d) fdno %d",
@@ -2550,6 +2553,348 @@ FDIO_t bzdio = /*@-compmempass@*/ &bzdio_s /*@=compmempass@*/ ;
 /*@=moduncon@*/
 #endif	/* HAVE_BZLIB_H */
 
+#include <sys/types.h>
+#include <inttypes.h>
+#include <lzma.h>
+
+#define kBufferSize (1 << 15)
+
+typedef struct lzfile {
+  /* IO buffer */
+    uint8_t buf[kBufferSize];
+
+    lzma_stream strm;
+
+    FILE *file;
+
+    int encoding;
+    int eof;
+
+} LZFILE;
+
+static LZFILE *lzopen_internal(const char *path, const char *mode, int fd)
+{
+    int level = 5;
+    int encoding = 0;
+    FILE *fp;
+    LZFILE *lzfile;
+    lzma_ret ret;
+
+    for (; *mode; mode++) {
+	if (*mode == 'w')
+	    encoding = 1;
+	else if (*mode == 'r')
+	    encoding = 0;
+	else if (*mode >= '1' && *mode <= '9')
+	    level = *mode - '0';
+    }
+    if (fd != -1)
+	fp = fdopen(fd, encoding ? "w" : "r");
+    else
+	fp = fopen(path, encoding ? "w" : "r");
+    if (!fp)
+	return 0;
+    lzfile = calloc(1, sizeof(*lzfile));
+    if (!lzfile) {
+	fclose(fp);
+	return 0;
+    }
+    lzfile->file = fp;
+    lzfile->encoding = encoding;
+    lzfile->eof = 0;
+    lzfile->strm = LZMA_STREAM_INIT_VAR;
+    if (encoding) {
+	lzma_options_alone alone;
+	alone.uncompressed_size = LZMA_VLI_VALUE_UNKNOWN;
+	memcpy(&alone.lzma, &lzma_preset_lzma[level - 1], sizeof(alone.lzma));
+	ret = lzma_alone_encoder(&lzfile->strm, &alone);
+    } else {
+	ret = lzma_auto_decoder(&lzfile->strm, 0, 0);
+    }
+    if (ret != LZMA_OK) {
+	fclose(fp);
+	free(lzfile);
+	return 0;
+    }
+    return lzfile;
+}
+
+static LZFILE *lzopen(const char *path, const char *mode)
+{
+    return lzopen_internal(path, mode, -1);
+}
+
+static LZFILE *lzdopen(int fd, const char *mode)
+{
+    if (fd < 0)
+	return 0;
+    return lzopen_internal(0, mode, fd);
+}
+
+static int lzflush(LZFILE *lzfile)
+{
+    return fflush(lzfile->file);
+}
+
+static int lzclose(LZFILE *lzfile)
+{
+    lzma_ret ret;
+    int n;
+
+    if (!lzfile)
+	return -1;
+    if (lzfile->encoding) {
+	for (;;) {
+	    lzfile->strm.avail_out = kBufferSize;
+	    lzfile->strm.next_out = lzfile->buf;
+	    ret = lzma_code(&lzfile->strm, LZMA_FINISH);
+	    if (ret != LZMA_OK && ret != LZMA_STREAM_END)
+		return -1;
+	    n = kBufferSize - lzfile->strm.avail_out;
+	    if (n && fwrite(lzfile->buf, 1, n, lzfile->file) != n)
+		return -1;
+	    if (ret == LZMA_STREAM_END)
+		break;
+	}
+    }
+    lzma_end(&lzfile->strm);
+    return fclose(lzfile->file);
+    free(lzfile);
+}
+
+static ssize_t lzread(LZFILE *lzfile, void *buf, size_t len)
+{
+    lzma_ret ret;
+    int eof = 0;
+
+    if (!lzfile || lzfile->encoding)
+      return -1;
+    if (lzfile->eof)
+      return 0;
+    lzfile->strm.next_out = buf;
+    lzfile->strm.avail_out = len;
+    for (;;) {
+	if (!lzfile->strm.avail_in) {
+	    lzfile->strm.next_in = lzfile->buf;
+	    lzfile->strm.avail_in = fread(lzfile->buf, 1, kBufferSize, lzfile->file);
+	    if (!lzfile->strm.avail_in)
+		eof = 1;
+	}
+	ret = lzma_code(&lzfile->strm, LZMA_RUN);
+	if (ret == LZMA_STREAM_END) {
+	    lzfile->eof = 1;
+	    return len - lzfile->strm.avail_out;
+	}
+	if (ret != LZMA_OK)
+	    return -1;
+	if (!lzfile->strm.avail_out)
+	    return len;
+	if (eof)
+	    return -1;
+      }
+}
+
+static ssize_t lzwrite(LZFILE *lzfile, void *buf, size_t len)
+{
+    lzma_ret ret;
+    int n;
+    if (!lzfile || !lzfile->encoding)
+	return -1;
+    if (!len)
+	return 0;
+    lzfile->strm.next_in = buf;
+    lzfile->strm.avail_in = len;
+    for (;;) {
+	lzfile->strm.next_out = lzfile->buf;
+	lzfile->strm.avail_out = kBufferSize;
+	ret = lzma_code(&lzfile->strm, LZMA_RUN);
+	if (ret != LZMA_OK)
+	    return -1;
+	n = kBufferSize - lzfile->strm.avail_out;
+	if (n && fwrite(lzfile->buf, 1, n, lzfile->file) != n)
+	    return -1;
+	if (!lzfile->strm.avail_in)
+	    return len;
+    }
+}
+
+/* =============================================================== */
+
+static inline /*@dependent@*/ void * lzdFileno(FD_t fd)
+	/*@*/
+{
+    void * rc = NULL;
+    int i;
+
+    FDSANE(fd);
+    for (i = fd->nfps; i >= 0; i--) {
+/*@-boundsread@*/
+	    FDSTACK_t * fps = &fd->fps[i];
+/*@=boundsread@*/
+	    if (fps->io != lzdio)
+		continue;
+	    rc = fps->fp;
+	break;
+    }
+    
+    return rc;
+}
+
+/*@-globuse@*/
+static /*@null@*/ FD_t lzdOpen(const char * path, const char * mode)
+	/*@globals fileSystem @*/
+	/*@modifies fileSystem @*/
+{
+    FD_t fd;
+    LZFILE *lzfile;
+    if ((lzfile = lzopen(path, mode)) == NULL)
+	return NULL;
+    fd = fdNew("open (lzdOpen)");
+    fdPop(fd); fdPush(fd, lzdio, lzfile, -1);
+    return fdLink(fd, "lzdOpen");
+}
+/*@=globuse@*/
+
+
+/*@-globuse@*/
+static /*@null@*/ FD_t lzdFdopen(void * cookie, const char * fmode)
+	/*@globals fileSystem, internalState @*/
+	/*@modifies fileSystem, internalState @*/
+{
+    FD_t fd = c2f(cookie);
+    int fdno;
+    LZFILE *lzfile;
+
+    if (fmode == NULL) return NULL;
+    fdno = fdFileno(fd);
+    fdSetFdno(fd, -1);          /* XXX skip the fdio close */
+    if (fdno < 0) return NULL;
+    lzfile = lzdopen(fdno, fmode);
+    if (lzfile == NULL) return NULL;
+    fdPush(fd, lzdio, lzfile, fdno);
+    return fdLink(fd, "lzdFdopen");
+}
+/*@=globuse@*/
+
+/*@-globuse@*/
+static int lzdFlush(FD_t fd)
+	/*@globals fileSystem @*/
+	/*@modifies fileSystem @*/
+{
+    return lzflush(lzdFileno(fd));
+}
+/*@=globuse@*/
+
+/* =============================================================== */
+/*@-globuse@*/
+/*@-mustmod@*/          /* LCL: *buf is modified */
+static ssize_t lzdRead(void * cookie, /*@out@*/ char * buf, size_t count)
+	/*@globals fileSystem, internalState @*/
+	/*@modifies *buf, fileSystem, internalState @*/
+{
+    FD_t fd = c2f(cookie);
+    LZFILE *lzfile;
+    ssize_t rc = 0;
+
+    if (fd->bytesRemain == 0) return 0; /* XXX simulate EOF */
+    lzfile = lzdFileno(fd);
+    fdstat_enter(fd, FDSTAT_READ);
+    if (lzfile)
+	/*@-compdef@*/
+	rc = lzread(lzfile, buf, count);
+	/*@=compdef@*/
+    if (rc == -1) {
+	fd->errcookie = "Lzma: decoding error";
+    } else if (rc >= 0) {
+	fdstat_exit(fd, FDSTAT_READ, rc);
+	/*@-compdef@*/
+	if (fd->ndigests && rc > 0) fdUpdateDigests(fd, (void *)buf, rc);
+	/*@=compdef@*/
+    }
+    return rc;
+}
+/*@=mustmod@*/
+/*@=globuse@*/
+
+/*@-globuse@*/
+static ssize_t lzdWrite(void * cookie, const char * buf, size_t count)
+/*@globals fileSystem, internalState @*/
+/*@modifies fileSystem, internalState @*/
+{
+    FD_t fd = c2f(cookie);
+    LZFILE *lzfile;
+    ssize_t rc = 0;
+
+    if (fd == NULL || fd->bytesRemain == 0) return 0;   /* XXX simulate EOF */
+
+    if (fd->ndigests && count > 0) fdUpdateDigests(fd, (void *)buf, count);
+
+    lzfile = lzdFileno(fd);
+
+    fdstat_enter(fd, FDSTAT_WRITE);
+    rc = lzwrite(lzfile, (void *)buf, count);
+    if (rc < 0) {
+	fd->errcookie = "Lzma: encoding error";
+    } else if (rc > 0) {
+	fdstat_exit(fd, FDSTAT_WRITE, rc);
+    }
+    return rc;
+}
+
+static inline int lzdSeek(void * cookie, /*@unused@*/ _libio_pos_t pos,
+			/*@unused@*/ int whence)
+	/*@*/
+{
+    FD_t fd = c2f(cookie);
+
+    LZDONLY(fd);
+    return -2;
+}
+
+static int lzdClose( /*@only@*/ void * cookie)
+	/*@globals fileSystem, internalState @*/
+	/*@modifies fileSystem, internalState @*/
+{
+    FD_t fd = c2f(cookie);
+    LZFILE *lzfile;
+    int rc;
+
+    lzfile = lzdFileno(fd);
+
+    if (lzfile == NULL) return -2;
+    fdstat_enter(fd, FDSTAT_CLOSE);
+    /*@-dependenttrans@*/
+    rc = lzclose(lzfile);
+    /*@=dependenttrans@*/
+
+    /* XXX TODO: preserve fd if errors */
+
+    if (fd) {
+	if (rc == -1) {
+	    fd->errcookie = strerror(ferror(lzfile->file));
+	} else if (rc >= 0) {
+	    fdstat_exit(fd, FDSTAT_CLOSE, rc);
+	}
+    }
+
+DBGIO(fd, (stderr, "==>\tlzdClose(%p) rc %lx %s\n", cookie, (unsigned long)rc, fdbg(fd)));
+
+    if (_rpmio_debug || rpmIsDebug()) fdstat_print(fd, "LZDIO", stderr);
+    /*@-branchstate@*/
+    if (rc == 0)
+	fd = fdFree(fd, "open (lzdClose)");
+    /*@=branchstate@*/
+    return rc;
+}
+
+/*@-type@*/ /* LCL: function typedefs */
+static struct FDIO_s lzdio_s = {
+  lzdRead, lzdWrite, lzdSeek, lzdClose, XfdLink, XfdFree, XfdNew, fdFileno,
+  NULL, lzdOpen, lzdFileno, lzdFlush,	NULL, NULL, NULL, NULL, NULL
+};
+/*@=type@*/
+FDIO_t lzdio = /*@-compmempass@*/ &lzdio_s /*@=compmempass@*/ ;
+
 /* =============================================================== */
 /*@observer@*/
 static const char * getFdErrstr (FD_t fd)
@@ -2568,7 +2913,9 @@ static const char * getFdErrstr (FD_t fd)
 	errstr = fd->errcookie;
     } else
 #endif	/* HAVE_BZLIB_H */
-
+    if (fdGetIo(fd) == lzdio) {
+    errstr = fd->errcookie;
+    } else 
     {
 	errstr = (fd->syserrno ? strerror(fd->syserrno) : "");
     }
@@ -2866,6 +3213,9 @@ fprintf(stderr, "*** Fdopen(%p,%s) %s\n", fd, fmode, fdbg(fd));
 	    fd = bzdFdopen(fd, zstdio);
 	    /*@=internalglobs@*/
 #endif
+    } else if (!strcmp(end, "lzdio")) {
+        iof = lzdio;
+        fd = lzdFdopen(fd, zstdio);
 	} else if (!strcmp(end, "ufdio")) {
 	    iof = ufdio;
 	} else if (!strcmp(end, "fadio")) {
@@ -3056,6 +3406,9 @@ int Ferror(FD_t fd)
 	    ec = (fd->syserrno  || fd->errcookie != NULL) ? -1 : 0;
 	    i--;	/* XXX fdio under bzdio always has fdno == -1 */
 #endif
+    } else if (fps->io == lzdio) {
+	    ec = (fd->syserrno  || fd->errcookie != NULL) ? -1 : 0;
+	    i--;	/* XXX fdio under lzdio always has fdno == -1 */
 	} else {
 	/* XXX need to check ufdio/gzdio/bzdio/fdio errors correctly. */
 	    ec = (fdFileno(fd) < 0 ? -1 : 0);

rpmio/rpmio.h

@@ -618,6 +618,10 @@ int ufdGetFile( /*@killref@*/ FD_t sfd, FD_t tfd)
  */
 /*@observer@*/ /*@unchecked@*/ extern FDIO_t bzdio;
 
+/**
+ */
+/*@observer@*/ /*@unchecked@*/ extern FDIO_t lzdio;
+
 /**
  */
 /*@observer@*/ /*@unchecked@*/ extern FDIO_t fadio;