4.0.4-alt113
- Implemented limited support for large files: a 2GB+ file can now be packaged, but the total size of uncompressed cpio payload is capped at 4 GB. - Automatically downgrade LZMA compression levels 7-9 -> 6 on small payloads.
This commit is contained in:
@ -1469,7 +1469,7 @@ static rpmRC addFile(FileList fl, const char * diskPath)
|
||||
statp->st_mtime = now;
|
||||
statp->st_ctime = now;
|
||||
} else if (lstat(diskPath, statp)) {
|
||||
rpmlog(RPMLOG_ERR, _("File not found: %s\n"), diskPath);
|
||||
rpmlog(RPMLOG_ERR, "%m: %s\n", diskPath);
|
||||
fl->processingFailed = 1;
|
||||
return RPMRC_FAIL;
|
||||
}
|
||||
@ -1485,7 +1485,7 @@ static rpmRC addFile(FileList fl, const char * diskPath)
|
||||
while ((p = strchr(p + 1, '/'))) {
|
||||
*p = '\0';
|
||||
if (lstat(dp, &st)) {
|
||||
rpmlog(RPMLOG_ERR, _("File not found: %s\n"), diskPath);
|
||||
rpmlog(RPMLOG_ERR, "%m: %s\n", diskPath);
|
||||
fl->processingFailed = 1;
|
||||
return RPMRC_FAIL;
|
||||
}
|
||||
@ -2048,11 +2048,10 @@ static int finalizeSize(TFI_t fi)
|
||||
{
|
||||
if (fi == NULL)
|
||||
return 0;
|
||||
int totalFileSize = 0;
|
||||
uint32_t totalFileSize = 0;
|
||||
int partialHardlinkSets = 0;
|
||||
int i, j;
|
||||
for (i = 0; i < fi->fc; i++) {
|
||||
if (fi->actions[i] == FA_SKIP) // GHOST
|
||||
for (int i = 0; i < fi->fc; i++) {
|
||||
if (fi->actions[i] == FA_SKIP) // %ghost
|
||||
continue;
|
||||
if (!S_ISREG(fi->fsts[i].st_mode))
|
||||
continue;
|
||||
@ -2062,7 +2061,8 @@ static int finalizeSize(TFI_t fi)
|
||||
}
|
||||
assert(fi->fsts[i].st_nlink > 1);
|
||||
int found = 0;
|
||||
for (j = 0; j < i; j++) {
|
||||
// Look backwards, more likely in the same dir.
|
||||
for (int j = i - 1; j >= 0; j--) {
|
||||
if (fi->actions[j] == FA_SKIP)
|
||||
continue;
|
||||
if (fi->fsts[i].st_dev != fi->fsts[j].st_dev)
|
||||
@ -2077,7 +2077,7 @@ static int finalizeSize(TFI_t fi)
|
||||
// first hardlink occurrence
|
||||
totalFileSize += fi->fsts[i].st_size;
|
||||
int nlink = 1;
|
||||
for (j = i + 1; j < fi->fc; j++) {
|
||||
for (int j = i + 1; j < fi->fc; j++) {
|
||||
if (fi->actions[j] == FA_SKIP)
|
||||
continue;
|
||||
if (fi->fsts[i].st_dev != fi->fsts[j].st_dev)
|
||||
|
101
build/pack.c
101
build/pack.c
@ -42,7 +42,7 @@ static inline int genSourceRpmName(Spec spec)
|
||||
* @todo Create transaction set *much* earlier.
|
||||
*/
|
||||
static int cpio_doio(FD_t fdo, /*@unused@*/ Header h, CSA_t csa,
|
||||
const char * fmodeMacro)
|
||||
const char * fmode)
|
||||
/*@globals rpmGlobalMacroContext,
|
||||
fileSystem@*/
|
||||
/*@modifies fdo, csa, rpmGlobalMacroContext, fileSystem @*/
|
||||
@ -55,14 +55,11 @@ static int cpio_doio(FD_t fdo, /*@unused@*/ Header h, CSA_t csa,
|
||||
FD_t cfd;
|
||||
int rc, ec;
|
||||
|
||||
{ const char *fmode = rpmExpand(fmodeMacro, NULL);
|
||||
if (!(fmode && fmode[0] == 'w'))
|
||||
fmode = xstrdup("w9.gzdio");
|
||||
{
|
||||
/*@-nullpass@*/
|
||||
(void) Fflush(fdo);
|
||||
cfd = Fdopen(fdDup(Fileno(fdo)), fmode);
|
||||
/*@=nullpass@*/
|
||||
fmode = _free(fmode);
|
||||
}
|
||||
if (cfd == NULL)
|
||||
return 1;
|
||||
@ -380,6 +377,71 @@ static int rpmLeadVersion(void)
|
||||
return rpmlead_version;
|
||||
}
|
||||
|
||||
// Estimate the uncompressed size of cpio archive before it is actually written.
|
||||
static uint64_t calcArchiveSize(TFI_t fi)
|
||||
{
|
||||
uint64_t size = 124; // cpio trailer
|
||||
for (int i = 0; i < fi->fc; i++) {
|
||||
if (fi->actions[i] == FA_SKIP) // %ghost
|
||||
continue;
|
||||
size += 110; // cpio header
|
||||
size += strlen(fi->apath[i]) + 1;
|
||||
size = (size + 3) & ~(uint64_t)3;
|
||||
if (!S_ISREG(fi->fsts[i].st_mode) && !S_ISLNK(fi->fsts[i].st_mode))
|
||||
continue;
|
||||
if (fi->fsts[i].st_nlink > 1) {
|
||||
// Only the first hardlink occurrence counts.
|
||||
int found = 0;
|
||||
for (int j = i - 1; j >= 0; j--) {
|
||||
if (fi->actions[j] == FA_SKIP)
|
||||
continue;
|
||||
if (fi->fsts[i].st_dev != fi->fsts[j].st_dev)
|
||||
continue;
|
||||
if (fi->fsts[i].st_ino != fi->fsts[j].st_ino)
|
||||
continue;
|
||||
found = 1;
|
||||
break;
|
||||
}
|
||||
if (found)
|
||||
continue;
|
||||
}
|
||||
// Valid for symlinks, target not null-terminated.
|
||||
size += fi->fsts[i].st_size;
|
||||
size = (size + 3) & ~(uint64_t)3;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
// LZMA compressions levels 6-9 are equivalent, except for the dictionary size,
|
||||
// which is 8-64M. For smaller inputs, levels 7-9 are downgraded automatically.
|
||||
static void downgradeLzmaLevel(char *mode, uint64_t archiveSize)
|
||||
{
|
||||
#define C(c) if (!(c)) return
|
||||
C(mode[1] == '7' || mode[1] == '8' || mode[1] == '9');
|
||||
C(mode[2] == '.');
|
||||
C(mode[3] == 'l' || mode[3] == 'x');
|
||||
C(mode[4] == 'z');
|
||||
#define S(m) ((m << 20) + (m << 10))
|
||||
switch (mode[1]) {
|
||||
case '9':
|
||||
if (archiveSize > S(32))
|
||||
break;
|
||||
mode[1] = '8';
|
||||
/*@fallthrough@*/
|
||||
case '8':
|
||||
if (archiveSize > S(16))
|
||||
break;
|
||||
mode[1] = '7';
|
||||
/*@fallthrough@*/
|
||||
case '7':
|
||||
if (archiveSize > S(8))
|
||||
break;
|
||||
mode[1] = '6';
|
||||
}
|
||||
#undef C
|
||||
#undef S
|
||||
}
|
||||
|
||||
int writeRPM(Header *hdrp, const char *fileName, int type,
|
||||
CSA_t csa, char *passPhrase, const char **cookie)
|
||||
{
|
||||
@ -394,6 +456,13 @@ int writeRPM(Header *hdrp, const char *fileName, int type,
|
||||
Header sig = NULL;
|
||||
int rc = 0;
|
||||
|
||||
uint64_t archiveSize = calcArchiveSize(csa->cpioList);
|
||||
if (archiveSize >= UINT32_MAX) { // sic, UINT32_MAX proper is kind of special
|
||||
rpmError(RPMERR_FWRITE, "cpio archive too big - %uM\n",
|
||||
(unsigned)(archiveSize>>20));
|
||||
return RPMERR_FWRITE;
|
||||
}
|
||||
|
||||
/* Transfer header reference form *hdrp to h. */
|
||||
h = headerLink(*hdrp);
|
||||
*hdrp = headerFree(*hdrp);
|
||||
@ -409,7 +478,7 @@ int writeRPM(Header *hdrp, const char *fileName, int type,
|
||||
if (type == RPMLEAD_BINARY)
|
||||
providePackageNVR(h);
|
||||
|
||||
const char *rpmio_flags = NULL;
|
||||
char *rpmio_flags = NULL;
|
||||
const char *N, *dash;
|
||||
|
||||
/* Save payload information */
|
||||
@ -428,10 +497,11 @@ int writeRPM(Header *hdrp, const char *fileName, int type,
|
||||
break;
|
||||
}
|
||||
/*@=branchstate@*/
|
||||
if (!(rpmio_flags && *rpmio_flags)) {
|
||||
if (!(rpmio_flags && *rpmio_flags == 'w')) {
|
||||
rpmio_flags = _free(rpmio_flags);
|
||||
rpmio_flags = xstrdup("w9.gzdio");
|
||||
}
|
||||
downgradeLzmaLevel(rpmio_flags, archiveSize);
|
||||
s = strchr(rpmio_flags, '.');
|
||||
if (s) {
|
||||
(void) headerAddEntry(h, RPMTAG_PAYLOADFORMAT, RPM_STRING_TYPE, "cpio", 1);
|
||||
@ -518,10 +588,21 @@ int writeRPM(Header *hdrp, const char *fileName, int type,
|
||||
* archive size within the header region.
|
||||
*/
|
||||
if (Fileno(csa->cpioFdIn) < 0) {
|
||||
if (archiveSize != csa->cpioArchiveSize) {
|
||||
rpmError(RPMERR_FWRITE, "wrong archive size: %" PRIu64 " -> %u\n",
|
||||
archiveSize, csa->cpioArchiveSize);
|
||||
rc = RPMERR_FWRITE;
|
||||
goto exit;
|
||||
}
|
||||
HGE_t hge = (HGE_t)headerGetEntryMinMemory;
|
||||
int_32 * archiveSize;
|
||||
if (hge(h, RPMTAG_ARCHIVESIZE, NULL, (void *)&archiveSize, NULL))
|
||||
*archiveSize = csa->cpioArchiveSize;
|
||||
uint32_t *sizep;
|
||||
if (hge(h, RPMTAG_ARCHIVESIZE, NULL, (void *)&sizep, NULL))
|
||||
*sizep = csa->cpioArchiveSize;
|
||||
else {
|
||||
rpmError(RPMERR_FWRITE, "cannot add RPMTAG_ARCHIVESIZE\n");
|
||||
rc = RPMERR_FWRITE;
|
||||
goto exit;
|
||||
}
|
||||
}
|
||||
|
||||
(void) Fflush(fd);
|
||||
|
@ -26,6 +26,8 @@ export CFLAGS
|
||||
AC_PROG_CPP
|
||||
AC_PROG_GCC_TRADITIONAL
|
||||
|
||||
AC_SYS_LARGEFILE
|
||||
|
||||
dnl Does this platform require array notation to assign to a va_list?
|
||||
dnl If cross-compiling, we assume va_list is "normal". If this breaks
|
||||
dnl you, set ac_cv_valistisarray=true and maybe define HAVE_VA_LIST_AS_ARRAY
|
||||
|
@ -691,7 +691,7 @@ static int expandRegular(/*@special@*/ FSM_t fsm)
|
||||
{
|
||||
const char * fmd5sum;
|
||||
const struct stat * st = &fsm->sb;
|
||||
int left = st->st_size;
|
||||
size_t left = st->st_size;
|
||||
int rc = 0;
|
||||
|
||||
rc = fsmStage(fsm, FSM_WOPEN);
|
||||
@ -758,7 +758,7 @@ static int writeFile(/*@special@*/ FSM_t fsm, int writeData)
|
||||
struct stat * ost = &fsm->osb;
|
||||
size_t pos = fdGetCpioPos(fsm->cfd);
|
||||
char * symbuf = NULL;
|
||||
int left;
|
||||
size_t left;
|
||||
int xx;
|
||||
int rc;
|
||||
|
||||
|
@ -137,11 +137,11 @@ static inline rpmRC checkSize(FD_t fd, int siglen, int pad, int datalen)
|
||||
}
|
||||
|
||||
rpmMessage((rc == RPMRC_OK ? RPMMESS_DEBUG : RPMMESS_WARNING),
|
||||
_("Expected size: %12d = lead(%d)+sigs(%d)+pad(%d)+data(%d)\n"),
|
||||
(int)sizeof(struct rpmlead)+siglen+pad+datalen,
|
||||
(int)sizeof(struct rpmlead), siglen, pad, datalen);
|
||||
_("Expected size: %12zu = lead(%zu)+sigs(%d)+pad(%d)+data(%u)\n"),
|
||||
sizeof(struct rpmlead)+siglen+pad+datalen,
|
||||
sizeof(struct rpmlead), siglen, pad, (unsigned)datalen);
|
||||
rpmMessage((rc == RPMRC_OK ? RPMMESS_DEBUG : RPMMESS_WARNING),
|
||||
_(" Actual size: %12d\n"), (int)st.st_size);
|
||||
_(" Actual size: %12zu\n"), (size_t)st.st_size);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
@ -173,7 +173,7 @@ int rpmVerifyFile(const char * root, Header h, int filenum,
|
||||
}
|
||||
|
||||
if (flags & RPMVERIFY_FILESIZE) {
|
||||
int_32 * sizeList;
|
||||
uint32_t *sizeList;
|
||||
|
||||
if (!hge(h, RPMTAG_FILESIZES, NULL, (void **) &sizeList, NULL)
|
||||
|| sizeList[filenum] != sb.st_size)
|
||||
|
@ -5,7 +5,7 @@
|
||||
|
||||
Name: rpm-build
|
||||
Version: 4.0.4
|
||||
Release: alt112
|
||||
Release: alt113
|
||||
|
||||
%define ifdef() %if %{expand:%%{?%{1}:1}%%{!?%{1}:0}}
|
||||
%define get_dep() %(rpm -q --qf '%%{NAME} >= %%|SERIAL?{%%{SERIAL}:}|%%{VERSION}-%%{RELEASE}' %1 2>/dev/null || echo '%1 >= unknown')
|
||||
@ -512,6 +512,11 @@ mv -T %buildroot%_rpmlibdir/{,build}macros
|
||||
%endif #with python
|
||||
|
||||
%changelog
|
||||
* Tue Jun 26 2018 Alexey Tourbin <at@altlinux.ru> 4.0.4-alt113
|
||||
- Implemented limited support for large files: a 2GB+ file can now be packaged,
|
||||
but the total size of uncompressed cpio payload is capped at 4 GB.
|
||||
- Automatically downgrade LZMA compression levels 7-9 -> 6 on small payloads.
|
||||
|
||||
* Fri Jun 08 2018 Gleb F-Malinovskiy <glebfm@altlinux.org> 4.0.4-alt112
|
||||
- imz@:
|
||||
+ Turned on running %%__find_{conflicts,obsoletes} if they are defined.
|
||||
|
@ -2803,7 +2803,7 @@ static char * formatValue(sprintfTag tag, Header h,
|
||||
} else {
|
||||
need = 10 + tag->pad + 20;
|
||||
val = xmalloc(need+1);
|
||||
strcat(buf, "d");
|
||||
strcat(buf, "u");
|
||||
/*@-formatconst@*/
|
||||
sprintf(val, buf, intVal);
|
||||
/*@=formatconst@*/
|
||||
@ -3228,7 +3228,7 @@ static char * shescapeFormat(int_32 type, hPTR_t data,
|
||||
|
||||
if (type == RPM_INT32_TYPE) {
|
||||
result = xmalloc(padding + 20);
|
||||
strcat(formatPrefix, "d");
|
||||
strcat(formatPrefix, "u");
|
||||
/*@-formatconst@*/
|
||||
sprintf(result, formatPrefix, *((int_32 *) data));
|
||||
/*@=formatconst@*/
|
||||
|
@ -161,6 +161,11 @@ int domd5(const char * fn, unsigned char * digest, int asAscii)
|
||||
/*@-globs -internalglobs -mods @*/
|
||||
fdno = open_dso(path, &pid, &fsize);
|
||||
/*@=globs =internalglobs =mods @*/
|
||||
/* Traditionally rpm-4.0 endows zero-sized files with empty md5sum. */
|
||||
if (fdno >= 0 && fsize == 0) {
|
||||
close(fdno);
|
||||
fdno = -1;
|
||||
}
|
||||
if (fdno < 0) {
|
||||
rc = 1;
|
||||
goto exit;
|
||||
@ -174,26 +179,27 @@ int domd5(const char * fn, unsigned char * digest, int asAscii)
|
||||
DIGEST_CTX ctx;
|
||||
void * mapped;
|
||||
|
||||
/* Preallocate ctx, no malloc calls between mmap and munmap. */
|
||||
ctx = rpmDigestInit(PGPHASHALGO_MD5, RPMDIGEST_NONE);
|
||||
mapped = mmap(NULL, fsize, PROT_READ, MAP_SHARED, fdno, 0);
|
||||
if (mapped == (void *)-1) {
|
||||
xx = close(fdno);
|
||||
rc = 1;
|
||||
break;
|
||||
if (mapped == (void *)-1) { // probably EOVERFLOW
|
||||
rpmDigestFinal(ctx, NULL, NULL, asAscii);
|
||||
goto readloop;
|
||||
}
|
||||
|
||||
#ifdef MADV_SEQUENTIAL
|
||||
xx = madvise(mapped, fsize, MADV_SEQUENTIAL);
|
||||
#endif
|
||||
|
||||
ctx = rpmDigestInit(PGPHASHALGO_MD5, RPMDIGEST_NONE);
|
||||
xx = rpmDigestUpdate(ctx, mapped, fsize);
|
||||
xx = rpmDigestFinal(ctx, (void **)&md5sum, &md5len, asAscii);
|
||||
xx = munmap(mapped, fsize);
|
||||
xx = rpmDigestFinal(ctx, (void **)&md5sum, &md5len, asAscii);
|
||||
xx = close(fdno);
|
||||
break;
|
||||
} /*@fallthrough@*/
|
||||
#endif
|
||||
default:
|
||||
readloop:
|
||||
/* Either use the pipe to prelink -y or open the URL. */
|
||||
fd = (pid != 0) ? fdDup(fdno) : Fopen(fn, "r.ufdio");
|
||||
(void) close(fdno);
|
||||
|
@ -2787,7 +2787,7 @@ static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int x
|
||||
encoding = 1;
|
||||
else if (*mode == 'r')
|
||||
encoding = 0;
|
||||
else if (*mode >= '1' && *mode <= '9')
|
||||
else if (*mode >= '0' && *mode <= '9')
|
||||
level = *mode - '0';
|
||||
}
|
||||
if (fd != -1)
|
||||
@ -2795,12 +2795,8 @@ static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int x
|
||||
else
|
||||
fp = fopen(path, encoding ? "w" : "r");
|
||||
if (!fp)
|
||||
return 0;
|
||||
lzfile = calloc(1, sizeof(*lzfile));
|
||||
if (!lzfile) {
|
||||
fclose(fp);
|
||||
return 0;
|
||||
}
|
||||
return NULL;
|
||||
lzfile = xcalloc(1, sizeof(*lzfile));
|
||||
lzfile->file = fp;
|
||||
lzfile->encoding = encoding;
|
||||
lzfile->eof = 0;
|
||||
@ -2808,14 +2804,20 @@ static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int x
|
||||
if (encoding) {
|
||||
lzma_lzma_preset(&lzfile->options, level);
|
||||
#if 1 /* tweak options for better compression */
|
||||
if (level >= 2) {
|
||||
unsigned int dict_size = 1<<20;
|
||||
if (lzfile->options.dict_size < dict_size)
|
||||
lzfile->options.dict_size = dict_size;
|
||||
unsigned int nice_len = (level < LZMA_PRESET_DEFAULT) ? 64 : 128;
|
||||
if (lzfile->options.nice_len < nice_len)
|
||||
lzfile->options.nice_len = nice_len;
|
||||
}
|
||||
if (level == 5)
|
||||
/* This level is still used by default in girar-builder, as of
|
||||
* early 2018, due to a historical accident: sometime around 10
|
||||
* years ago it was using a smaller dictionary and was deemed more
|
||||
* appropriate for the minimum system requirements at the time.
|
||||
* Since xz v4.999.9beta-161-gb4b1cbc, level 5 and level 6,
|
||||
* the default one, both use 8M dictionary; the only difference
|
||||
* is that level 5 should compress slightly faster. */
|
||||
lzfile->options.nice_len = 64; /* default 32 */
|
||||
else if (level > 5)
|
||||
/* The constant 112 was found to be a local optimum on some cpio
|
||||
* inputs. Both size and "saved bytes per second" metrics were
|
||||
* taken into account. */
|
||||
lzfile->options.nice_len = 112; /* default 64 */
|
||||
#endif
|
||||
if (xz) {
|
||||
lzfile->filters[0].id = LZMA_FILTER_LZMA2;
|
||||
@ -2835,7 +2837,7 @@ static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int x
|
||||
if (ret != LZMA_OK) {
|
||||
fclose(fp);
|
||||
free(lzfile);
|
||||
return 0;
|
||||
return NULL;
|
||||
}
|
||||
return lzfile;
|
||||
}
|
||||
@ -3104,7 +3106,7 @@ static ssize_t lzdWrite(void * cookie, const char * buf, size_t count)
|
||||
|
||||
fdstat_enter(fd, FDSTAT_WRITE);
|
||||
rc = lzwrite(lzfile, (void *)buf, count);
|
||||
if (rc < 0) {
|
||||
if (rc == -1) {
|
||||
fd->errcookie = "Lzma: encoding error";
|
||||
} else if (rc > 0) {
|
||||
fdstat_exit(fd, FDSTAT_WRITE, rc);
|
||||
@ -3244,9 +3246,6 @@ DBGIO(fd, (stderr, "==> Fread(%p,%u,%u,%p) %s\n", buf, (unsigned)size, (unsigned
|
||||
|
||||
size_t Fwrite(const void *buf, size_t size, size_t nmemb, FD_t fd)
|
||||
{
|
||||
fdio_write_function_t _write;
|
||||
int rc;
|
||||
|
||||
FDSANE(fd);
|
||||
/*@-modfilesys@*/
|
||||
DBGIO(fd, (stderr, "==> Fwrite(%p,%u,%u,%p) %s\n", buf, (unsigned)size, (unsigned)nmemb, (fd ? fd : NULL), fdbg(fd)));
|
||||
@ -3254,17 +3253,28 @@ DBGIO(fd, (stderr, "==> Fwrite(%p,%u,%u,%p) %s\n", buf, (unsigned)size, (unsigne
|
||||
|
||||
if (fdGetIo(fd) == fpio) {
|
||||
/*@+voidabstract -nullpass@*/
|
||||
rc = fwrite(buf, size, nmemb, fdGetFILE(fd));
|
||||
size_t ret = fwrite(buf, size, nmemb, fdGetFILE(fd));
|
||||
/*@=voidabstract =nullpass@*/
|
||||
return rc;
|
||||
return ret * size;
|
||||
}
|
||||
|
||||
/*@-nullderef@*/
|
||||
_write = FDIOVEC(fd, write);
|
||||
fdio_write_function_t _write = FDIOVEC(fd, write);
|
||||
/*@=nullderef@*/
|
||||
assert(_write);
|
||||
|
||||
rc = (_write ? _write(fd, buf, size * nmemb) : -2);
|
||||
return rc;
|
||||
// XXX check for overflow instead of assuming that size=1.
|
||||
size_t n = size * nmemb;
|
||||
|
||||
// XXX sloppy mixing of size_t and ssize_t is going on here.
|
||||
ssize_t ret = _write(fd, buf, n);
|
||||
if (ret == -1)
|
||||
return 0;
|
||||
if (ret == n)
|
||||
return n;
|
||||
if (ret < 0)
|
||||
return 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
int Fseek(FD_t fd, _libio_off_t offset, int whence) {
|
||||
|
Reference in New Issue
Block a user