4.0.4-alt113

- Implemented limited support for large files: a 2GB+ file can now be packaged,
  but the total size of uncompressed cpio payload is capped at 4 GB.
- Automatically downgrade LZMA compression levels 7-9 -> 6 on small payloads.
This commit is contained in:
Alexey Tourbin
2018-06-26 13:03:42 +03:00
10 changed files with 163 additions and 59 deletions

View File

@ -1469,7 +1469,7 @@ static rpmRC addFile(FileList fl, const char * diskPath)
statp->st_mtime = now;
statp->st_ctime = now;
} else if (lstat(diskPath, statp)) {
rpmlog(RPMLOG_ERR, _("File not found: %s\n"), diskPath);
rpmlog(RPMLOG_ERR, "%m: %s\n", diskPath);
fl->processingFailed = 1;
return RPMRC_FAIL;
}
@ -1485,7 +1485,7 @@ static rpmRC addFile(FileList fl, const char * diskPath)
while ((p = strchr(p + 1, '/'))) {
*p = '\0';
if (lstat(dp, &st)) {
rpmlog(RPMLOG_ERR, _("File not found: %s\n"), diskPath);
rpmlog(RPMLOG_ERR, "%m: %s\n", diskPath);
fl->processingFailed = 1;
return RPMRC_FAIL;
}
@ -2048,11 +2048,10 @@ static int finalizeSize(TFI_t fi)
{
if (fi == NULL)
return 0;
int totalFileSize = 0;
uint32_t totalFileSize = 0;
int partialHardlinkSets = 0;
int i, j;
for (i = 0; i < fi->fc; i++) {
if (fi->actions[i] == FA_SKIP) // GHOST
for (int i = 0; i < fi->fc; i++) {
if (fi->actions[i] == FA_SKIP) // %ghost
continue;
if (!S_ISREG(fi->fsts[i].st_mode))
continue;
@ -2062,7 +2061,8 @@ static int finalizeSize(TFI_t fi)
}
assert(fi->fsts[i].st_nlink > 1);
int found = 0;
for (j = 0; j < i; j++) {
// Look backwards, more likely in the same dir.
for (int j = i - 1; j >= 0; j--) {
if (fi->actions[j] == FA_SKIP)
continue;
if (fi->fsts[i].st_dev != fi->fsts[j].st_dev)
@ -2077,7 +2077,7 @@ static int finalizeSize(TFI_t fi)
// first hardlink occurrence
totalFileSize += fi->fsts[i].st_size;
int nlink = 1;
for (j = i + 1; j < fi->fc; j++) {
for (int j = i + 1; j < fi->fc; j++) {
if (fi->actions[j] == FA_SKIP)
continue;
if (fi->fsts[i].st_dev != fi->fsts[j].st_dev)

View File

@ -42,7 +42,7 @@ static inline int genSourceRpmName(Spec spec)
* @todo Create transaction set *much* earlier.
*/
static int cpio_doio(FD_t fdo, /*@unused@*/ Header h, CSA_t csa,
const char * fmodeMacro)
const char * fmode)
/*@globals rpmGlobalMacroContext,
fileSystem@*/
/*@modifies fdo, csa, rpmGlobalMacroContext, fileSystem @*/
@ -55,14 +55,11 @@ static int cpio_doio(FD_t fdo, /*@unused@*/ Header h, CSA_t csa,
FD_t cfd;
int rc, ec;
{ const char *fmode = rpmExpand(fmodeMacro, NULL);
if (!(fmode && fmode[0] == 'w'))
fmode = xstrdup("w9.gzdio");
{
/*@-nullpass@*/
(void) Fflush(fdo);
cfd = Fdopen(fdDup(Fileno(fdo)), fmode);
/*@=nullpass@*/
fmode = _free(fmode);
}
if (cfd == NULL)
return 1;
@ -380,6 +377,71 @@ static int rpmLeadVersion(void)
return rpmlead_version;
}
// Estimate the uncompressed size of cpio archive before it is actually written.
static uint64_t calcArchiveSize(TFI_t fi)
{
uint64_t size = 124; // cpio trailer
for (int i = 0; i < fi->fc; i++) {
if (fi->actions[i] == FA_SKIP) // %ghost
continue;
size += 110; // cpio header
size += strlen(fi->apath[i]) + 1;
size = (size + 3) & ~(uint64_t)3;
if (!S_ISREG(fi->fsts[i].st_mode) && !S_ISLNK(fi->fsts[i].st_mode))
continue;
if (fi->fsts[i].st_nlink > 1) {
// Only the first hardlink occurrence counts.
int found = 0;
for (int j = i - 1; j >= 0; j--) {
if (fi->actions[j] == FA_SKIP)
continue;
if (fi->fsts[i].st_dev != fi->fsts[j].st_dev)
continue;
if (fi->fsts[i].st_ino != fi->fsts[j].st_ino)
continue;
found = 1;
break;
}
if (found)
continue;
}
// Valid for symlinks, target not null-terminated.
size += fi->fsts[i].st_size;
size = (size + 3) & ~(uint64_t)3;
}
return size;
}
// LZMA compressions levels 6-9 are equivalent, except for the dictionary size,
// which is 8-64M. For smaller inputs, levels 7-9 are downgraded automatically.
static void downgradeLzmaLevel(char *mode, uint64_t archiveSize)
{
#define C(c) if (!(c)) return
C(mode[1] == '7' || mode[1] == '8' || mode[1] == '9');
C(mode[2] == '.');
C(mode[3] == 'l' || mode[3] == 'x');
C(mode[4] == 'z');
#define S(m) ((m << 20) + (m << 10))
switch (mode[1]) {
case '9':
if (archiveSize > S(32))
break;
mode[1] = '8';
/*@fallthrough@*/
case '8':
if (archiveSize > S(16))
break;
mode[1] = '7';
/*@fallthrough@*/
case '7':
if (archiveSize > S(8))
break;
mode[1] = '6';
}
#undef C
#undef S
}
int writeRPM(Header *hdrp, const char *fileName, int type,
CSA_t csa, char *passPhrase, const char **cookie)
{
@ -394,6 +456,13 @@ int writeRPM(Header *hdrp, const char *fileName, int type,
Header sig = NULL;
int rc = 0;
uint64_t archiveSize = calcArchiveSize(csa->cpioList);
if (archiveSize >= UINT32_MAX) { // sic, UINT32_MAX proper is kind of special
rpmError(RPMERR_FWRITE, "cpio archive too big - %uM\n",
(unsigned)(archiveSize>>20));
return RPMERR_FWRITE;
}
/* Transfer header reference form *hdrp to h. */
h = headerLink(*hdrp);
*hdrp = headerFree(*hdrp);
@ -409,7 +478,7 @@ int writeRPM(Header *hdrp, const char *fileName, int type,
if (type == RPMLEAD_BINARY)
providePackageNVR(h);
const char *rpmio_flags = NULL;
char *rpmio_flags = NULL;
const char *N, *dash;
/* Save payload information */
@ -428,10 +497,11 @@ int writeRPM(Header *hdrp, const char *fileName, int type,
break;
}
/*@=branchstate@*/
if (!(rpmio_flags && *rpmio_flags)) {
if (!(rpmio_flags && *rpmio_flags == 'w')) {
rpmio_flags = _free(rpmio_flags);
rpmio_flags = xstrdup("w9.gzdio");
}
downgradeLzmaLevel(rpmio_flags, archiveSize);
s = strchr(rpmio_flags, '.');
if (s) {
(void) headerAddEntry(h, RPMTAG_PAYLOADFORMAT, RPM_STRING_TYPE, "cpio", 1);
@ -518,10 +588,21 @@ int writeRPM(Header *hdrp, const char *fileName, int type,
* archive size within the header region.
*/
if (Fileno(csa->cpioFdIn) < 0) {
if (archiveSize != csa->cpioArchiveSize) {
rpmError(RPMERR_FWRITE, "wrong archive size: %" PRIu64 " -> %u\n",
archiveSize, csa->cpioArchiveSize);
rc = RPMERR_FWRITE;
goto exit;
}
HGE_t hge = (HGE_t)headerGetEntryMinMemory;
int_32 * archiveSize;
if (hge(h, RPMTAG_ARCHIVESIZE, NULL, (void *)&archiveSize, NULL))
*archiveSize = csa->cpioArchiveSize;
uint32_t *sizep;
if (hge(h, RPMTAG_ARCHIVESIZE, NULL, (void *)&sizep, NULL))
*sizep = csa->cpioArchiveSize;
else {
rpmError(RPMERR_FWRITE, "cannot add RPMTAG_ARCHIVESIZE\n");
rc = RPMERR_FWRITE;
goto exit;
}
}
(void) Fflush(fd);

View File

@ -26,6 +26,8 @@ export CFLAGS
AC_PROG_CPP
AC_PROG_GCC_TRADITIONAL
AC_SYS_LARGEFILE
dnl Does this platform require array notation to assign to a va_list?
dnl If cross-compiling, we assume va_list is "normal". If this breaks
dnl you, set ac_cv_valistisarray=true and maybe define HAVE_VA_LIST_AS_ARRAY

View File

@ -691,7 +691,7 @@ static int expandRegular(/*@special@*/ FSM_t fsm)
{
const char * fmd5sum;
const struct stat * st = &fsm->sb;
int left = st->st_size;
size_t left = st->st_size;
int rc = 0;
rc = fsmStage(fsm, FSM_WOPEN);
@ -758,7 +758,7 @@ static int writeFile(/*@special@*/ FSM_t fsm, int writeData)
struct stat * ost = &fsm->osb;
size_t pos = fdGetCpioPos(fsm->cfd);
char * symbuf = NULL;
int left;
size_t left;
int xx;
int rc;

View File

@ -137,11 +137,11 @@ static inline rpmRC checkSize(FD_t fd, int siglen, int pad, int datalen)
}
rpmMessage((rc == RPMRC_OK ? RPMMESS_DEBUG : RPMMESS_WARNING),
_("Expected size: %12d = lead(%d)+sigs(%d)+pad(%d)+data(%d)\n"),
(int)sizeof(struct rpmlead)+siglen+pad+datalen,
(int)sizeof(struct rpmlead), siglen, pad, datalen);
_("Expected size: %12zu = lead(%zu)+sigs(%d)+pad(%d)+data(%u)\n"),
sizeof(struct rpmlead)+siglen+pad+datalen,
sizeof(struct rpmlead), siglen, pad, (unsigned)datalen);
rpmMessage((rc == RPMRC_OK ? RPMMESS_DEBUG : RPMMESS_WARNING),
_(" Actual size: %12d\n"), (int)st.st_size);
_(" Actual size: %12zu\n"), (size_t)st.st_size);
return rc;
}

View File

@ -173,7 +173,7 @@ int rpmVerifyFile(const char * root, Header h, int filenum,
}
if (flags & RPMVERIFY_FILESIZE) {
int_32 * sizeList;
uint32_t *sizeList;
if (!hge(h, RPMTAG_FILESIZES, NULL, (void **) &sizeList, NULL)
|| sizeList[filenum] != sb.st_size)

View File

@ -5,7 +5,7 @@
Name: rpm-build
Version: 4.0.4
Release: alt112
Release: alt113
%define ifdef() %if %{expand:%%{?%{1}:1}%%{!?%{1}:0}}
%define get_dep() %(rpm -q --qf '%%{NAME} >= %%|SERIAL?{%%{SERIAL}:}|%%{VERSION}-%%{RELEASE}' %1 2>/dev/null || echo '%1 >= unknown')
@ -512,6 +512,11 @@ mv -T %buildroot%_rpmlibdir/{,build}macros
%endif #with python
%changelog
* Tue Jun 26 2018 Alexey Tourbin <at@altlinux.ru> 4.0.4-alt113
- Implemented limited support for large files: a 2GB+ file can now be packaged,
but the total size of uncompressed cpio payload is capped at 4 GB.
- Automatically downgrade LZMA compression levels 7-9 -> 6 on small payloads.
* Fri Jun 08 2018 Gleb F-Malinovskiy <glebfm@altlinux.org> 4.0.4-alt112
- imz@:
+ Turned on running %%__find_{conflicts,obsoletes} if they are defined.

View File

@ -2803,7 +2803,7 @@ static char * formatValue(sprintfTag tag, Header h,
} else {
need = 10 + tag->pad + 20;
val = xmalloc(need+1);
strcat(buf, "d");
strcat(buf, "u");
/*@-formatconst@*/
sprintf(val, buf, intVal);
/*@=formatconst@*/
@ -3228,7 +3228,7 @@ static char * shescapeFormat(int_32 type, hPTR_t data,
if (type == RPM_INT32_TYPE) {
result = xmalloc(padding + 20);
strcat(formatPrefix, "d");
strcat(formatPrefix, "u");
/*@-formatconst@*/
sprintf(result, formatPrefix, *((int_32 *) data));
/*@=formatconst@*/

View File

@ -161,6 +161,11 @@ int domd5(const char * fn, unsigned char * digest, int asAscii)
/*@-globs -internalglobs -mods @*/
fdno = open_dso(path, &pid, &fsize);
/*@=globs =internalglobs =mods @*/
/* Traditionally rpm-4.0 endows zero-sized files with empty md5sum. */
if (fdno >= 0 && fsize == 0) {
close(fdno);
fdno = -1;
}
if (fdno < 0) {
rc = 1;
goto exit;
@ -174,26 +179,27 @@ int domd5(const char * fn, unsigned char * digest, int asAscii)
DIGEST_CTX ctx;
void * mapped;
/* Preallocate ctx, no malloc calls between mmap and munmap. */
ctx = rpmDigestInit(PGPHASHALGO_MD5, RPMDIGEST_NONE);
mapped = mmap(NULL, fsize, PROT_READ, MAP_SHARED, fdno, 0);
if (mapped == (void *)-1) {
xx = close(fdno);
rc = 1;
break;
if (mapped == (void *)-1) { // probably EOVERFLOW
rpmDigestFinal(ctx, NULL, NULL, asAscii);
goto readloop;
}
#ifdef MADV_SEQUENTIAL
xx = madvise(mapped, fsize, MADV_SEQUENTIAL);
#endif
ctx = rpmDigestInit(PGPHASHALGO_MD5, RPMDIGEST_NONE);
xx = rpmDigestUpdate(ctx, mapped, fsize);
xx = rpmDigestFinal(ctx, (void **)&md5sum, &md5len, asAscii);
xx = munmap(mapped, fsize);
xx = rpmDigestFinal(ctx, (void **)&md5sum, &md5len, asAscii);
xx = close(fdno);
break;
} /*@fallthrough@*/
#endif
default:
readloop:
/* Either use the pipe to prelink -y or open the URL. */
fd = (pid != 0) ? fdDup(fdno) : Fopen(fn, "r.ufdio");
(void) close(fdno);

View File

@ -2787,7 +2787,7 @@ static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int x
encoding = 1;
else if (*mode == 'r')
encoding = 0;
else if (*mode >= '1' && *mode <= '9')
else if (*mode >= '0' && *mode <= '9')
level = *mode - '0';
}
if (fd != -1)
@ -2795,12 +2795,8 @@ static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int x
else
fp = fopen(path, encoding ? "w" : "r");
if (!fp)
return 0;
lzfile = calloc(1, sizeof(*lzfile));
if (!lzfile) {
fclose(fp);
return 0;
}
return NULL;
lzfile = xcalloc(1, sizeof(*lzfile));
lzfile->file = fp;
lzfile->encoding = encoding;
lzfile->eof = 0;
@ -2808,14 +2804,20 @@ static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int x
if (encoding) {
lzma_lzma_preset(&lzfile->options, level);
#if 1 /* tweak options for better compression */
if (level >= 2) {
unsigned int dict_size = 1<<20;
if (lzfile->options.dict_size < dict_size)
lzfile->options.dict_size = dict_size;
unsigned int nice_len = (level < LZMA_PRESET_DEFAULT) ? 64 : 128;
if (lzfile->options.nice_len < nice_len)
lzfile->options.nice_len = nice_len;
}
if (level == 5)
/* This level is still used by default in girar-builder, as of
* early 2018, due to a historical accident: sometime around 10
* years ago it was using a smaller dictionary and was deemed more
* appropriate for the minimum system requirements at the time.
* Since xz v4.999.9beta-161-gb4b1cbc, level 5 and level 6,
* the default one, both use 8M dictionary; the only difference
* is that level 5 should compress slightly faster. */
lzfile->options.nice_len = 64; /* default 32 */
else if (level > 5)
/* The constant 112 was found to be a local optimum on some cpio
* inputs. Both size and "saved bytes per second" metrics were
* taken into account. */
lzfile->options.nice_len = 112; /* default 64 */
#endif
if (xz) {
lzfile->filters[0].id = LZMA_FILTER_LZMA2;
@ -2835,7 +2837,7 @@ static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int x
if (ret != LZMA_OK) {
fclose(fp);
free(lzfile);
return 0;
return NULL;
}
return lzfile;
}
@ -3104,7 +3106,7 @@ static ssize_t lzdWrite(void * cookie, const char * buf, size_t count)
fdstat_enter(fd, FDSTAT_WRITE);
rc = lzwrite(lzfile, (void *)buf, count);
if (rc < 0) {
if (rc == -1) {
fd->errcookie = "Lzma: encoding error";
} else if (rc > 0) {
fdstat_exit(fd, FDSTAT_WRITE, rc);
@ -3244,9 +3246,6 @@ DBGIO(fd, (stderr, "==> Fread(%p,%u,%u,%p) %s\n", buf, (unsigned)size, (unsigned
size_t Fwrite(const void *buf, size_t size, size_t nmemb, FD_t fd)
{
fdio_write_function_t _write;
int rc;
FDSANE(fd);
/*@-modfilesys@*/
DBGIO(fd, (stderr, "==> Fwrite(%p,%u,%u,%p) %s\n", buf, (unsigned)size, (unsigned)nmemb, (fd ? fd : NULL), fdbg(fd)));
@ -3254,17 +3253,28 @@ DBGIO(fd, (stderr, "==> Fwrite(%p,%u,%u,%p) %s\n", buf, (unsigned)size, (unsigne
if (fdGetIo(fd) == fpio) {
/*@+voidabstract -nullpass@*/
rc = fwrite(buf, size, nmemb, fdGetFILE(fd));
size_t ret = fwrite(buf, size, nmemb, fdGetFILE(fd));
/*@=voidabstract =nullpass@*/
return rc;
return ret * size;
}
/*@-nullderef@*/
_write = FDIOVEC(fd, write);
fdio_write_function_t _write = FDIOVEC(fd, write);
/*@=nullderef@*/
assert(_write);
rc = (_write ? _write(fd, buf, size * nmemb) : -2);
return rc;
// XXX check for overflow instead of assuming that size=1.
size_t n = size * nmemb;
// XXX sloppy mixing of size_t and ssize_t is going on here.
ssize_t ret = _write(fd, buf, n);
if (ret == -1)
return 0;
if (ret == n)
return n;
if (ret < 0)
return 0;
return ret;
}
int Fseek(FD_t fd, _libio_off_t offset, int whence) {