Qt Creator project files

The resulting binary is only ~ 0.08% (1216 bytes) bigger.

Makefile

@@ -1,15 +1,25 @@
 PACKAGE	= propagator
 
+SHELL := /bin/bash
+
 DESTDIR =
 
 bindir  = /usr/bin
-libdir  = /usr/lib
+multiarch := $(shell $(CROSS_COMPILE)gcc -print-multiarch)
+ifneq (,$(strip $(multiarch)))
+libdir := /usr/lib/$(multiarch)
+else
+libdir := $(patsubst %/,%,/usr/lib/$(patsubst %-,%,$(CROSS_COMPILE)))
+endif
 
 INSTALL = /bin/install
 
 #---------------------------------------------------------------
 L ?= GLIBC
 #L = KLIBC
+ifeq ($(L),GLIBC)
+CC = $(CROSS_COMPILE)gcc
+endif
 
 ifeq ($(L),KLIBC)
 CC = klcc
@@ -23,18 +33,18 @@ F = STDIO
 endif
 endif
 
-CFLAGS += -Os -pipe -Wall
+CFLAGS += -O2 -flto -pipe -Wall
 
 GLIBC_INCLUDES =
 KLIBC_INCLUDES =
 INCLUDES = $($(L)_INCLUDES)
 
-GLIBC_LDFLAGS = -static
+GLIBC_LDFLAGS = -static -flto -O2
 KLIBC_LDFLAGS =
 MUSL_LDFLAGS = -static
 LDFLAGS += $($(L)_LDFLAGS)
 
-STRIPCMD = strip -R .note -R .comment
+STRIPCMD = $(CROSS_COMPILE)strip -R .note -R .comment
 
 #---------------------------------------------------------------
 DEFS = -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64
@@ -65,7 +75,7 @@ COMPILE = $(CC) $(CFLAGS) $(DEFS)
 
 #- frontends
 NEWT_FRONTEND_SRC = newt-frontend.c
-NEWT_FRONTEND_LIBS = $(libdir)/libnewt.a $(libdir)/libslang.a
+NEWT_FRONTEND_LIBS = $(libdir)/libnewt.a $(libdir)/libslang.a $(libdir)/libdl.a
 
 STDIO_FRONTEND_SRC = stdio-frontend.c
 STDIO_FRONTEND_LIBS =
@@ -126,13 +136,23 @@ clean:
 .depend: version.h
 	$(CPP) $(CFLAGS) -M $(ALLSRC) > .depend
 
-ALL_TESTS := test_parse_content_length
+ALL_TESTS := test_parse_content_length \
+	test_basename_dirname \
+	test_parse_ftp_filesize
+
+TESTS_COMMON_SRC := test_common.c
 
 test: $(ALL_TESTS)
 	@set -e; \
 	for tst in $(ALL_TESTS); do echo "$$tst"; ./$$tst; done
 
-test_parse_content_length: test_parse_content_length.c url.c
+test_parse_content_length: test_parse_content_length.c url.c $(TEST_COMMON_SRC)
+	$(CC) $(CFLAGS) -flto $(INIT_DEFS) -o $@ $<
+
+test_basename_dirname: test_basename_dirname.c url.c $(TEST_COMMON_SRC)
+	$(CC) $(CFLAGS) -flto $(INIT_DEFS) -o $@ $<
+
+test_parse_ftp_filesize: test_parse_ftp_filesize.c url.c $(TEST_COMMON_SRC)
 	$(CC) $(CFLAGS) -flto $(INIT_DEFS) -o $@ $<
 
 ifeq (.depend,$(wildcard .depend))

devel/README.md

@@ -0,0 +1,5 @@
+# Propagator - developer scripts
+
+This directory contains scripts which might be useful for propagator
+**development**. These scripts should **NOT** be packaged with propagator.
+

devel/fetchfromtask.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+# Retreive propagator rpm from 'task' repository and extract
+# /usr/sbin/propagator binary from it
+#
+# Dependencies:
+# pythonic:
+# - requests
+#   https://github.com/psf/requests, version 2.25.1 is known to work
+# others:
+# - coreutils
+#   sha1sum utility
+# - cpio
+# - rpm2cpio
+# - xz-utils
+#   xzcat utility
+
+import argparse
+import logging
+import os
+import re
+import requests
+import platform
+import shutil
+import subprocess
+
+NATIVE_ARCH = platform.uname().machine
+# Note: propagator is not built for armhf
+ALT_ARCHES = ['aarch64', 'i586', 'ppc64le', 'x86_64']
+
+
+def guess_url(taskid, pkgname='propagator', arch=NATIVE_ARCH):
+    pkg_regex = re.compile(f'^{pkgname}-(?!debuginfo).*\\.{arch}\\.rpm$')
+    baseurl = f"http://git.altlinux.org/tasks/{taskid}/build/repo/{arch}"
+
+    pkglist_url = f'{baseurl}/base/pkglist.task.xz'
+    logging.debug('guess_url: downloading pkglist from %s', pkglist_url)
+    req = requests.get(pkglist_url)
+    req.raise_for_status()
+    logging.debug('guess_url: uncompressing pkglist with xzcat')
+    xz = subprocess.Popen(['xzcat'],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            stdin=subprocess.PIPE)
+    pkglist, errs = xz.communicate(input=req.content)
+    rc = xz.wait()
+    if rc != 0:
+        logging.error('failed to unpack pkglist, error %d', rc)
+        raise RuntimeError(f"xzcat returned {rc}")
+
+    logging.debug('guess_url: extracting strings from pkglist')
+    proc = subprocess.Popen(['strings'],
+              stdout=subprocess.PIPE,
+              stderr=subprocess.PIPE,
+              stdin=subprocess.PIPE)
+    strings, errs = proc.communicate(input=pkglist)
+    rc = proc.wait()
+    if rc != 0:
+        logging.error('strings failed: %d', rc)
+        raise RuntimeError(f"strings failed: {rc}")
+    rpm_name = None
+    for line in strings.decode('utf-8').split('\n'):
+        logging.debug('guess_url: processing "%s"', line)
+        if pkg_regex.match(line) is not None:
+            logging.debug('OK, found "%s": "%s"', pkgname, line)
+            rpm_name = line
+            break
+    if rpm_name is None:
+        raise RuntimeError(f"No package {pkgname} in task {taskid} found")
+    return rpm_name, f"{baseurl}/RPMS.task/{rpm_name}"
+
+
+def download(url):
+    subprocess.check_call(['wget', '-N', url])
+
+
+def rm_rf(path):
+    try:
+        shutil.rmtree(path)
+    except FileNotFoundError:
+        pass
+
+def extract_rpm(rpm_name, arch=NATIVE_ARCH, pattern='*'):
+    rm_rf(arch)
+    os.makedirs(arch)
+    logging.debug('extract_rpm: running "rpm2cpio %s"', rpm_name)
+    rpm2cpio = subprocess.Popen(['rpm2cpio', rpm_name],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+    logging.debug('extract_rpm: piping to "cpio -id"')
+    cpio = subprocess.Popen(['cpio', '-id', '--no-absolute-filenames', pattern],
+            stdin=rpm2cpio.stdout,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=arch)
+    rpm2cpio.stdout.close() # Allow rpm2cpio to receive a SIGPIPE if cpio exits
+    out, err = cpio.communicate()
+    rc1 = rpm2cpio.wait()
+    rc2 = cpio.wait()
+    if rc1 != 0:
+        raise RuntimeError(f'rpm2cpio failed: {rc1}')
+    if rc2 != 0:
+        logging.error('extract_rpm: cpio failed: %d, "%s"', rc2, err)
+        raise RuntimeError(f'cpio failed: {rc2}')
+
+
+def process(taskid, arch):
+    rpm_name, rpm_url = guess_url(taskid, pkgname='propagator', arch=arch)
+    download(rpm_url)
+    extract_rpm(rpm_name, arch=arch, pattern='./usr/sbin/propagator')
+
+
+def main():
+    parser = argparse.ArgumentParser(description='extract propagator binary from a "task" repo')
+    parser.add_argument('taskid', type=int)
+    parser.add_argument('-v', '--verbose', action='count', default=0,
+                        help='Be more verbose')
+    parser.add_argument('-a', '--arch',
+                        choices=ALT_ARCHES + ['all'],
+                        default=NATIVE_ARCH,
+                        help='architecture')
+    args = parser.parse_args()
+    loglevel = logging.INFO
+    if args.verbose >= 1:
+        loglevel = logging.DEBUG
+    logging.basicConfig(format='fetchfromtask:%(levelname)s:%(message)s',
+                        level=loglevel)
+    if args.arch == 'all':
+        for arch in ALT_ARCHES:
+            process(args.taskid, arch)
+    else:
+        process(args.taskid, args.arch)
+
+
+if __name__ == '__main__':
+
+    main()

devel/implant-propagator.py

@@ -0,0 +1,345 @@
+#!/usr/bin/env python3
+#
+# Injects a custom /sbin/init-bin into ALT Linux ISO, or rather
+# into the initramfs the ISO contains.
+#
+# 1. Extract initramfs from the image
+# 2. Unpack initramfs into staging directory
+# 3. Replace staging_dir/sbin/init-bin with the given binary
+# 4. Pack staging_dir as a new initramfs
+# 5. Make an ISO based on the given one and the newly built initramfs.
+#
+# Dependencies
+# pythonic: standard library only
+# others:
+# - coreutils
+#   sha1sum utility
+# - cpio
+# - fakeroot
+# - findutils
+#   find utility
+# - gzip
+#   zcat utility
+# - pigz
+# - xorriso
+
+import argparse
+import logging
+import os
+import shutil
+import sys
+import tempfile
+
+from subprocess import (
+        check_call,
+        check_output,
+        Popen,
+        PIPE,
+)
+
+END_OF_ARCHIVE = 'premature end of archive'
+SHA1SUM_BINARY = '/usr/bin/sha1sum'
+
+
+def xorriso_find(iso, name):
+    cmd = [
+            'xorriso', '-indev', f'stdio:{iso}',
+            '-find', '/', '-type', 'f', '-name', name
+    ]
+    logging.debug('xorriso_find: running: %s', ' '.join(cmd))
+    out = check_output(cmd, encoding='utf-8')
+    logging.debug('xorriso_find: got: %s', out.strip())
+
+    files = [l.strip("'") for l in out.strip().split('\n')]
+    count = len(files)
+    if count == 0:
+        logging.error('xorriso_find: no file %s found in %s', name, iso)
+        raise RuntimeError(f'no file {name} in {iso}')
+    elif count > 1:
+        logging.error('xorriso_find: %d files named %s found in %s',
+                      count, name, iso)
+        for f in files:
+            logging.error('xorriso_find: %s', f)
+        raise RuntimeError(f"multiple files {name} in {iso}")
+    else:
+        return files[0]
+
+
+def xorriso_cat(iso, path):
+    cmd = [
+            'xorriso', '-indev', f'stdio:{iso}', '-osirrox', 'on',
+            '-concat', 'append', '-', path
+    ]
+    logging.debug('xorriso_cat: running "%s"', str(cmd))
+    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    return proc
+
+
+def xorriso_extract(iso, path, dest):
+    cmd = [
+            'xorriso', '-indev', f'stdio:{iso}', '-osirrox', 'on',
+            '-extract', path, dest
+    ]
+    check_call(cmd)
+
+
+def start_zcat(fileobj):
+    return Popen(['zcat'], stdin=fileobj, stdout=PIPE, stderr=PIPE)
+
+
+def fakeroot_cpio(instream, targetdir, fakedb):
+    fakeroot_cmd = ['fakeroot', '-i', fakedb, '-s', fakedb]
+    cpio_cmd = ['cpio', '-id', '-u', '--no-absolute-filenames']
+    logging.debug('fakeroot_cpio: running %s', ' '.join(fakeroot_cmd + cpio_cmd))
+    proc = Popen(fakeroot_cmd + cpio_cmd,
+                 cwd=targetdir,
+                 stdin=instream,
+                 stdout=PIPE,
+                 stderr=PIPE)
+    out, err = proc.communicate()
+    rc = proc.wait()
+    out = out.decode('utf-8').strip()
+    err = err.decode('utf-8').strip()
+    if rc != 0 and not err.endswith(END_OF_ARCHIVE):
+        logging.info('fakeroot_cpio: cpio: %s (exit code: %d)', err, rc)
+    elif out:
+        logging.debug('fakeroot_cpio: cpio: %s', out + '\n' + err)
+    return rc, out, err
+
+
+def rm_rf(path):
+    try:
+        shutil.rmtree(path)
+    except FileNotFoundError:
+        pass
+
+
+def ln_sf(src, dest):
+    try:
+        os.symlink(src, dest)
+    except FileExistsError:
+        os.remove(dest)
+        os.symlink(src, dest)
+
+
+def extract_full_cz(iso, path, targetdir, fakedb=None):
+    rm_rf(targetdir)
+    os.makedirs(targetdir)
+
+    xorriso = xorriso_cat(iso, path)
+    zcat = start_zcat(xorriso.stdout)
+    xorriso.stdout.close()
+
+    cpio_count = 0
+    if fakedb is None:
+        fakedb = 'fake.db'
+    while True:
+        rc, out, err = fakeroot_cpio(zcat.stdout, targetdir, fakedb)
+        if rc != 0:
+            break
+        cpio_count += 1
+
+    rc = xorriso.wait()
+    if rc != 0:
+        logging.error('extract_full_cz: xorriso "%s" failed: %d', xorriso.args, rc)
+        raise RuntimeError(f"xorriso failed: {rc}")
+    rc = zcat.wait()
+    if rc != 0:
+        logging.error('extract_full_cz: zcat failed: %d', rc)
+        raise RuntimeError(f"zcat failed: {rc}")
+    if cpio_count == 0:
+        logging.error('extract_full_cz: no cpio archives have been '\
+                      'extracted from %s:%s', iso, path)
+        raise RuntimeError('nothing has been extracted')
+
+    logging.debug('extract_full_cz: %s:%s has been successfully extracted '\
+                  'in %s', iso, path, targetdir)
+
+
+def cpio_pack(path, dest, fakedb):
+    fakeroot_cmd = ['fakeroot', '-i', fakedb]
+    find_cmd = ['find', '.']
+    logging.debug('cpio_pack: running "%s" in "%s"',
+                  ' '.join(fakeroot_cmd + find_cmd), path)
+    find = Popen(fakeroot_cmd + find_cmd,
+                 cwd=path, stdout=PIPE, stderr=PIPE)
+    cpio_cmd = 'cpio -Hnewc --create'.split()
+    logging.debug('cpio_pack: running "%s" in "%s"',
+                  ' '.join(fakeroot_cmd + cpio_cmd), path)
+    cpio = Popen(fakeroot_cmd + cpio_cmd,
+                 cwd=path,
+                 stdin=find.stdout,
+                 stdout=PIPE,
+                 stderr=PIPE)
+    find.stdout.close()
+    with open(dest, 'w') as outf:
+        pigz = Popen(['pigz'], stdin=cpio.stdout, stdout=outf)
+        cpio.stdout.close()
+    rc = pigz.wait()
+    if rc != 0:
+        logging.error("cpio_pack: pigz failed: %d", rc)
+        raise RuntimeError(f"pigz failed: {rc}")
+    rc = cpio.wait()
+    if rc != 0:
+        logging.error("cpio_pack: cpio failed: %d", rc)
+        raise RuntimeError(f"cpio failed: {rc}")
+    rc = find.wait()
+    if rc != 0:
+        logging.error("cpio_pack: find failed: %d", rc)
+        raise RuntimeError(f"find failed: {rc}")
+
+
+def fakeroot_copy(path, dest, fakedb):
+    cmd = ['fakeroot', '-i', fakedb, '-s', fakedb, 'cp', '-af', path, dest]
+    logging.debug('fakeroot_copy: running %s', cmd)
+    check_call(cmd)
+
+
+def replace_propagator(iso, propagator, new_initramfs, initramfs=None):
+    if initramfs is None:
+        full_cz = xorriso_find(iso, 'full.cz')
+        logging.debug('replace_propagator: found initramfs: %s', full_cz)
+    else:
+        full_cz = initramfs
+        logging.debug('replace_propagator: using %s as initramfs', full_cz)
+    with tempfile.TemporaryDirectory(suffix='.initramfs') as tmpdir:
+        with tempfile.NamedTemporaryFile(suffix='.fake.db') as fakedb_file:
+            fakedb = fakedb_file.name
+            logging.debug('replace_propagator: unpacking initramfs %s:%s into %s',
+                           iso, full_cz, tmpdir)
+            extract_full_cz(iso, full_cz, tmpdir, fakedb=fakedb)
+            sbin_init = os.path.join(tmpdir, 'sbin/init-bin')
+            logging.debug('replace_propagator: installing new propagator at %s', sbin_init)
+            fakeroot_copy(propagator, sbin_init, fakedb)
+            cpio_pack(tmpdir, new_initramfs, fakedb)
+    return full_cz
+
+
+def sha1(path):
+    out = check_output(['sha1sum', path], encoding='utf-8')
+    return out.strip().split()[0]
+
+
+def sha1oniso(iso, path):
+    cmd = [
+            'xorriso', '-indev', f'stdio:{iso}', '-osirrox', 'on',
+            '-concat', 'pipe', '+', SHA1SUM_BINARY, '+', path, '--'
+    ]
+    logging.debug('sha1oniso: running %s', ' '.join(cmd))
+    out = check_output(cmd, encoding='utf-8')
+    return out.strip().split()[0]
+
+
+def manifest(iso, path, propagator):
+    lines = [
+        sha1oniso(iso, path),
+        sha1(propagator)
+    ]
+    proc = Popen(['sha1sum', '-'], encoding='utf-8',
+                 stdin=PIPE, stdout=PIPE, stderr=PIPE)
+    out, err = proc.communicate(input='\n'.join(lines))
+    rc = proc.wait()
+    if rc != 0:
+        logging.error('sha1sum failed: %s (%d)', err, rc)
+        raise RuntimeError(f'manifest: sha1sum failed: {rc}')
+    return out.strip().split()[0]
+
+
+def replace_initramfs(iso, path, new_initramfs, new_iso=None):
+    if new_iso is None:
+        logging.debug('replace_initramfs: guessing new ISO name')
+        iso_name = os.path.basename(iso)
+        new_iso = iso_name.rsplit('.iso', maxsplit=1)[0]
+        logging.debug('replace_initramfs: computing SHA1 of %s', new_initramfs)
+        new_sha1 = sha1(new_initramfs)
+        new_iso = f"{new_iso}-{new_sha1}.iso"
+        logging.debug('replace_initramfs: new ISO name: %s', new_iso)
+
+    cmd = [
+            'xorriso', 
+            '-indev', f'stdio:{iso}',
+            '-outdev', f'stdio:{new_iso}.tmp',
+            '-update', new_initramfs, path,
+            '-boot_image', 'any', 'replay'
+    ]
+    logging.debug('replace_initramfs: running %s', ' '.join(cmd))
+    check_call(cmd)
+    os.rename(f'{new_iso}.tmp', new_iso)
+    logging.debug('replace_initramfs: successfully generated %s', new_iso)
+    return new_iso
+
+
+def implant_propagator(iso, propagator, new_iso=None, initramfs=None):
+    """Inject propagator into initramfs on ISO"""
+    if initramfs is None:
+        full_cz = xorriso_find(iso, 'full.cz')
+        logging.debug('implant_propagator: found initramfs: %s', full_cz)
+    else:
+        full_cz = initramfs
+        logging.debug('implant_propagator: using %s as initramfs', full_cz)
+    new_manifest = manifest(iso, full_cz, propagator)
+    logging.debug('implant_propagator: new initramfs manifest: %s', new_manifest)
+    new_initramfs = f'{new_manifest}.full.cz'
+    in_iso_path = replace_propagator(iso, propagator, new_initramfs,
+                                     initramfs=full_cz)
+    logging.info('implant_propagator: generated new initramfs %s', new_initramfs)
+    new_iso_name = replace_initramfs(iso, in_iso_path, new_initramfs)
+    return new_iso_name, new_initramfs
+
+
+def direct_boot(iso, propagator,
+                kernel=None, initramfs=None, symlinks=False):
+    """Prepare kernel and initramfs for direct boot with QEMU"""
+    if initramfs is None:
+        initramfs = xorriso_find(iso, 'full.cz')
+    if kernel is None:
+        kernel =  xorriso_find(iso, 'vmlinuz')
+    new_manifest = manifest(iso, initramfs, propagator)
+    new_initramfs = f'{new_manifest}.full.cz'
+    kernel_sha1 = sha1oniso(iso, kernel)
+    kernel_filename = f'{kernel_sha1}.vmlinuz'
+    replace_propagator(iso, propagator, f'{new_initramfs}.tmp',
+                       initramfs=initramfs)
+    xorriso_extract(iso, kernel, f'{kernel_filename}.tmp')
+    os.rename(f'{kernel_filename}.tmp', kernel_filename)
+    os.rename(f'{new_initramfs}.tmp', new_initramfs)
+    if symlinks:
+        ln_sf(kernel_filename, 'vmlinuz.latest')
+        ln_sf(new_initramfs, 'full.cz.latest')
+    print(kernel_filename)
+    print(new_initramfs)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Inject propagator into ISO',
+                                     prog='implant-propagator')
+    parser.add_argument('-v', '--verbose', action='count', default=0,
+                        help='Be more verbose')
+    parser.add_argument('-d', '--direct-boot', action='store_true',
+                        default=False,
+                        help='Extract kernel and initramfs for direct boot')
+    parser.add_argument('-i', '--initramfs',
+                        help='path to initramfs (in ISO) to operate on')
+    parser.add_argument('-k', '--kernel',
+                        help='extract this kernel image for direct boot')
+    parser.add_argument('iso', help='ISO to operate on')
+    parser.add_argument('propagator', help='Binary to inject')
+
+    args = parser.parse_args()
+    level = logging.INFO
+    if args.verbose >= 1:
+        level = logging.DEBUG
+
+    logging.basicConfig(format='implant-propagator:%(levelname)s:%(message)s',
+                        level=level)
+    if args.direct_boot:
+        direct_boot(args.iso, args.propagator,
+                    kernel=args.kernel, initramfs=args.initramfs)
+    else:
+        iso, initramfs = implant_propagator(args.iso, args.propagator,
+                                            initramfs=args.initramfs)
+        print(iso)
+
+
+if __name__ == '__main__':
+    main()

network.c

@@ -900,8 +900,10 @@ enum return_type ftp_prepare(void)
 	do {
 		char location_full[500];
 		int ftp_serv_response;
-		int fd, size;
+		int fd;
+		unsigned long size;
 		char *tmp;
+		int is_iso = 0;
 
 		snprintf(location_full, sizeof(location_full),
 				 "Please enter the name or IP address of the FTP server, "
@@ -927,7 +929,15 @@ enum return_type ftp_prepare(void)
 			results = RETURN_BACK;
 			continue;
 		}
+
 		strcpy(location_full, answers[1]);
+		log_message("FTP: trying to retrive %s", location_full);
+		fd = ftp_start_download(ftp_serv_response, location_full, &size);
+		if (fd >= 0) {
+			is_iso = 1;
+			goto download;
+		}
+
 		tmp = get_ramdisk_realname();
 		strcat(location_full, tmp);
 		free(tmp);
@@ -949,6 +959,7 @@ enum return_type ftp_prepare(void)
 			continue;
 		}
 
+	download:
 		if (!ramdisk_possible(size)) {
 			close(fd);
 			close(ftp_serv_response);
@@ -956,9 +967,9 @@ enum return_type ftp_prepare(void)
 					   BYTES2MB(size)*2, BYTES2MB(total_memory()));
 			return RETURN_ERROR;
 		}
-		log_message("FTP: size of download %d bytes", size);
+		log_message("FTP: size of download %lu bytes", size);
 
-		results = load_ramdisk_fd(fd, size);
+		results = load_ramdisk_or_iso(fd, size, is_iso);
 		if (results == RETURN_OK)
 			ftp_end_data_command(ftp_serv_response);
 		else

propagator.cflags

@@ -0,0 +1 @@
+-std=c17

propagator.config

@@ -0,0 +1,6 @@
+#define _GNU_SOURCE
+#define _FILE_OFFSET_BITS 64
+#define ENABLE_CIFS
+#define SPAWN_SHELL
+#define SPAWN_SPLASH
+#define DISABLE_ADSL

propagator.creator

@@ -0,0 +1 @@
+[General]

propagator.cxxflags

@@ -0,0 +1 @@
+-std=c++17

propagator.files

@@ -0,0 +1,47 @@
+adsl.c
+adsl.h
+automatic.c
+automatic.h
+cdrom.c
+cdrom.h
+common.c
+common.h
+config-stage1.h
+dhcp.c
+dhcp.h
+disk.c
+disk.h
+dns.c
+dns.h
+doc/documented..frontend.h
+frontend-common.c
+frontend.h
+gen_init_cpio.c
+init.c
+init.h
+insmod.h
+log.c
+log.h
+lomount.c
+lomount.h
+modules.c
+modules.h
+mount.c
+mount.h
+network.c
+network.h
+newt-frontend.c
+probing.c
+probing.h
+sha256.c
+sha256.h
+stage1.c
+stage1.h
+stdio-frontend.c
+test_url.c
+tools.c
+tools.h
+udev.c
+udev.h
+url.c
+url.h

propagator.includes

@@ -0,0 +1 @@
+.

propagator.spec

@@ -48,6 +48,9 @@ including init and various helpers for hw probing and bootstrapping.
 %_sbindir/propagator
 
 %changelog
+* `env LC_ALL=C date +'%a %b %d %Y'` Alexey Sheplyakov <asheplyakov@altlinux.org> `date +%Y%m%d`-alt1
+- Support booting complete ISOs via FTP (closes: #NNNNN)
+
 * Thu Sep 22 2021 Alexey Sheplyakov <asheplyakov@altlinux.org> 20210922-alt1
 - Support booting complete ISOs via HTTP (closes: #40710)
 

test_basename_dirname.c

@@ -0,0 +1,60 @@
+#include <stdio.h>
+#include "url.c" /* yes, include the **source** file */
+#include "test_common.c" /* yes, include the **source** file */
+
+static int
+test_basename_dirname(const char *path,
+		      const char *dir_expected,
+		      const char *name_expected,
+		      int shouldfail)
+{
+	int ret = 0, err = 0;
+	char *dir = NULL, *name = NULL;
+
+	err = basename_dirname(path, &dir, &name);
+	if (!shouldfail) {
+		if (err < 0) {
+			log_message("%s: FAIL: path='%s'", __func__, path);
+			return 1;
+		}
+		if (strcmp(dir, dir_expected)) {
+			log_message("%s: FAIL: expected dir '%s' != actual '%s'",
+				    __func__, dir_expected, dir);
+			ret += 1;
+		}
+		if (strcmp(name, name_expected)) {
+			log_message("%s: FAIL: expected name '%s' != actual '%s'",
+				    __func__, name_expected, name);
+			ret += 1;
+		}
+	} else {
+		if (err == 0) {
+			log_message("%s: XPASS: path='%s'", __func__, path);
+			ret = 1;
+		}
+	}
+	if (!shouldfail && ret == 0) {
+		log_message("%s: PASS: path='%s', dir='%s', name='%s'",
+			    __func__, path, dir, name);
+	}
+	if (shouldfail && err != 0) {
+		log_message("%s: XFAIL: path='%s'", __func__, path);
+	}
+	if (dir) {
+		free(dir);
+	}
+	if (name) {
+		free(name);
+	}
+	return ret;
+}
+
+int main(int argc, char **argv) {
+	int errc = 0;
+	errc += test_basename_dirname("/abs/name", "/abs", "name", 0);
+	errc += test_basename_dirname("rel/name", "rel", "name", 0);
+	errc += test_basename_dirname("aaa", "", "", 1);
+
+	return errc;
+}
+

test_common.c

@@ -0,0 +1,14 @@
+#ifndef PROPAGATOR_TEST_COMMON_H
+#define PROPAGATOR_TEST_COMMON_H
+#include <stdio.h>
+#include <stdarg.h>
+
+void log_message(const char *msg, ...) {
+	va_list args;
+	va_start(args, msg);
+	vfprintf(stderr, msg, args);
+	fprintf(stderr, "\n");
+	fflush(stderr);
+}
+
+#endif /* PROPAGATOR_TEST_COMMON_H */

test_parse_content_length.c

@@ -1,14 +1,7 @@
 #include <stdio.h>
 #include <stdarg.h>
 
-void log_message(const char *msg, ...) {
-	va_list args;
-	va_start(args, msg);
-	vfprintf(stderr, msg, args);
-	fprintf(stderr, "\n");
-	fflush(stderr);
-}
-
+#include "test_common.c" /* yes, include the source file */
 #include "url.c" /* yes, include the source file */
 
 static int test_parse_content_length(const char *headers, unsigned long expected, int shouldfail) {

test_parse_ftp_filesize.c

@@ -0,0 +1,74 @@
+#include <stdio.h>
+#include "url.c" /* yes, include the **source** file */
+#include "test_common.c" /* yes, include the **source** file */
+
+static int
+test_parse_ftp_filesize(const char *line, const char *name,
+		unsigned long size_expected, int shouldfail)
+{
+	unsigned long size = 0;
+	int err = 0, ret = 0;
+	err = parse_ftp_filesize(line, name, &size);
+	if (shouldfail) {
+		if (err == 0) {
+			log_message("%s: XPASS: name='%s', line='%s'",
+				__func__, name, line);
+			ret += 1;
+		}
+	} else {
+		if (err != 0) {
+			log_message("%s: FAIL: name='%s', line='%s'", __func__, line, name);
+			ret += 1;
+		} else if (size != size_expected) {
+			log_message("%s: FAIL: size=%lu, expected=%lu, line='%s'",
+				    __func__, size, size_expected, line);
+			ret += 1;
+		}
+	}
+	if (shouldfail && err != 0) {
+		log_message("%s: XFAIL: line='%s'", __func__, line);
+	}
+	if (!shouldfail && ret == 0) {
+	       log_message("%s: PASS: size=%lu, line='%s'",
+			   __func__, size_expected, line);
+	}
+	return ret;
+}
+
+int main(int argc, char **argv) {
+	int err = 0;
+	err += test_parse_ftp_filesize(
+		"-rw-r--r--    1 ftp      ftp      3000000000 Apr 28 15:43 slinux-9.1-x86_64.iso",
+		"slinux-9.1-x86_64.iso",
+		3000000000UL,
+		0
+	);
+	err += test_parse_ftp_filesize(
+		"-rw-r--r--    1 ftp      ftp      123456789012345678901234567890 Apr 28 15:43 slinux-9.1-x86_64.iso",
+		"slinux-9.1-x86_64.iso",
+		ULONG_MAX, /* too big */
+		0);
+
+	/* invalid/unknown size is not an error with FTP */
+	err += test_parse_ftp_filesize("-rw-r--r--    1 ftp      ftp      xyz Apr 28 15:43 slinux-9.1-x86_64.iso",
+		"slinux-9.1-x86_64.iso",
+		0,
+		0); 
+
+	/* invalid/unknown size is not an error with FTP */
+	err += test_parse_ftp_filesize(
+		"??? slinux-9.1-x86_64.iso",
+		"slinux-9.1-x86_64.iso",
+		0,
+		0); 
+
+	/* file name not in the reply */
+	err += test_parse_ftp_filesize(
+		"NOTFOUND",
+		"slinux-9.1-x86_64.iso",
+		0,
+		1
+	);
+
+	return err;
+}

test_url.c

@@ -0,0 +1,52 @@
+#include <stdio.h>
+#include "url.c" /* yes, include the **source** file */
+
+static int test_basename_dirname()
+{
+	int ret = 0;
+	char *dir = NULL, *file = NULL;
+	const char *path = "/dir/name";
+
+	if (basename_dirname(path, &dir, &file) < 0) {
+		fprintf(stderr, "%s: unexpected failure", __func__);
+		return 1;
+	}
+	if (strcmp(dir, "/dir")) {
+		fprintf(stderr, "expected '/dir', got: '%s'\n", dir);
+		ret += 1;
+	}
+	if (strcmp(file, "name")) {
+		fprintf(stderr, "expected 'name', got: '%s'\n", file);
+		ret += 1;
+	}
+	if (dir) {
+		free(dir);
+	}
+	if (file) {
+		free(file);
+	}
+	return ret;
+}
+
+static int test_parse_ftp_filesize_ok()
+{
+	const char *sample = "-rw-r--r--    1 ftp      ftp      3000000000 Apr 28 15:43 slinux-9.1-x86_64.iso";
+	const char *name = "slinux-9.1-x86_64.iso";
+	unsigned long size = 0, expected = 3000000000UL;
+	if (parse_ftp_filesize(sample, name, &size) != 0) {
+		fprintf(stderr, "%s: unexpected failure", __func__);
+		return 1;
+	}
+	if (size != expected) {
+		fprintf(stderr, "%s: expected size=%lu, got %lu\n", __func__, expected, size);
+		return 1;
+	}
+	return 0;
+}
+
+int main(int argc, char **argv) {
+	int err = 0;
+	err += test_basename_dirname();
+	err += test_parse_ftp_filesize_ok();
+	return err;
+}

tools.c

@@ -35,6 +35,7 @@
 #include <sys/types.h>
 #include <sys/mount.h>
 #include <sys/poll.h>
+#include <sys/sendfile.h>
 #include <sys/sysinfo.h>
 #include <sys/mman.h> /* memfd_create */
 
@@ -362,7 +363,7 @@ static int write_exactly(int fd, const char *buf, size_t len)
  * @return 0 on success, -1 on error
  * @note if size == 0 keep copying until EOF on src_fd
  */
-static int copy_loop(int dst_fd, int src_fd, unsigned long size)
+static int copy_loop_dumb(int dst_fd, int src_fd, unsigned long size)
 {
 	char buf[32768];
 	unsigned long bytes_written = 0;
@@ -390,6 +391,45 @@ static int copy_loop(int dst_fd, int src_fd, unsigned long size)
 	return 0;
 }
 
+static int copy_loop_sendfile(int dst_fd, int src_fd, unsigned long size)
+{
+	ssize_t dl;
+	size_t chunk = 1024*1024;
+	unsigned long bytes_written = 0;
+	while (size > 0) {
+		dl = sendfile(dst_fd, src_fd, NULL, chunk < size ? chunk : size);
+		if (dl < 0) {
+			log_message("%s: sendfile: error %s", __func__, strerror(errno));
+			return -1;
+		}
+		size -= (unsigned long)dl;
+		bytes_written += (unsigned long)dl;
+		update_progression((int)BYTES2MB(bytes_written));
+	}
+	return 0;
+}
+
+static int is_mmapable(int fd)
+{
+	void *dummy;
+	size_t len = 65536; /* don't bother to figure out page size */
+	dummy = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
+	if (dummy != MAP_FAILED) {
+		munmap(dummy, len);
+		return 1;
+	} else {
+		return 0;
+	}
+}
+
+static int copy_loop(int dst_fd, int src_fd, unsigned long size)
+{
+	if (!size || !is_mmapable(src_fd))
+		return copy_loop_dumb(dst_fd, src_fd, size);
+	else
+		return copy_loop_sendfile(dst_fd, src_fd, size);
+}
+
 int make_ramfd(unsigned long size)
 {
 	int ramfd;
@@ -513,6 +553,8 @@ int splash_verbose()
 	char * av[] = 	{ "/bin/plymouth" , "plymouth" , "quit", NULL };
 	log_message( "%s: %s\n", av[0], av[2] );
 	return 	spawn(av);
+#else
+	return 0;
 #endif
 }
 
@@ -527,6 +569,8 @@ int update_splash( char * state )
 	char * av[] = 	{ "/bin/plymouth" , "plymouth" , "--update" , state, NULL };
 	log_message( "%s: %s\n", av[0], av[3] );
 	return 	spawn(av);
+#else
+	return 0;
 #endif
 }
 

url.c

@@ -314,76 +314,232 @@ int ftp_data_command(int sock, char * command, char * param)
 }
 
 
-static int ftp_get_filesize(int sock, char * remotename)
-{
-	int size = 0;
-	char buf[2000];
-	char file[500];
-	char * ptr;
-	int fd, rc, tot;
+enum SEARCH_RESULT {
+	ERROR = -1,
+	FOUND = 0,
+	CONTINUE = 1,
+};
+
+/**
+ * @brief guess the file size from FTP LIST command output line
+ * @param line a line from FTP LIST command output
+ * @param name name of file being searched
+ * @param size pointer to the file size, set by this function
+ * @return CONTINUE, if `line` constains no `name`
+ *         ERROR on error (size being NULL)
+ *         0 otherwise, **including** unknown or too big file size
+ */
+static int
+parse_ftp_filesize(const char *line, const char *name, unsigned long *size) {
+	/* line:
+	 * -rw-r--r--    1 ftp      ftp      5570224128 Apr 28 15:43 slinux-9.1-x86_64.iso
+	 * name:
+	 * slinux-9.1-x86_64.iso
+	 */
 	int i;
-
-	strcpy(buf, remotename);
-	ptr = strrchr(buf, '/');
-	if (!*ptr)
-		return -1;
-	*ptr = '\0';
-
-	strcpy(file, ptr+1);
-
-	if ((rc = ftp_command(sock, "CWD", buf))) {
-		return -1;
-	}
-
-	fd = ftp_data_command(sock, "LIST", NULL);
-	if (fd <= 0) {
-		close(sock);
-		return -1;
-	}
-
-	ptr = buf;
-	while ((tot = read(fd, ptr, sizeof(buf) - (ptr - buf))) != 0)
-		ptr += tot;
-	*ptr = '\0';
-	close(fd);
-
-	if (!(ptr = strstr(buf, file))) {
-		log_message("FTP/get_filesize: Bad mood, directory does not contain searched file (%s)", file);
-		if (ftp_end_data_command(sock))
-			close(sock);
-		return -1;
+	const char *ptr = strstr(line, name);
+	if (!ptr) {
+		return CONTINUE;
+	}
+	if (!size) {
+		return ERROR;
 	}
 
+	/*
+	 * -rw-r--r--    1 ftp      ftp      5570224128 Apr 28 15:43 slinux-9.1-x86_64.iso
+	 *                                                           ^ ptr
+	 * The word before timestamp string should be a file size.
+	 * There are two types of timestamp strings:
+	 * 1) recent: Apr 28 15:43
+	 * 2) > 1 year: Jul 28 2020
+	 * Both of them consist of 3 words.
+	 * Skip 4 words backwars, so ptr is at the file size:
+	 *
+	 * -rw-r--r--    1 ftp      ftp      5570224128 Apr 28 15:43 slinux-9.1-x86_64.iso
+	 *                                   ^ptr
+	 *
+	 * XXX: the FTP protocol does not standardize the output of
+	 * the LIST command, hence the above is just a guess
+	 */
 	for (i=0; i<4; i++) {
-		while (*ptr && *ptr != ' ')
+		while (ptr > line && *ptr != ' ')
 			ptr--;
-		while (*ptr && *ptr == ' ')
+		while (ptr > line && *ptr == ' ')
 			ptr--;
 	}
-	while (*ptr && *ptr != ' ')
+	/*
+	 * -rw-r--r--    1 ftp      ftp      5570224128 Apr 28 15:43 slinux-9.1-x86_64.iso
+	 *                                            ^ptr
+	 * move ptr to the beginning of the word (size)
+	 */
+	while (ptr > line && *ptr != ' ')
 		ptr--;
 
-	if (ptr)
-		size = charstar_to_int(ptr+1);
-	else
-		size = 0;
-
-	if (ftp_end_data_command(sock)) {
-		close(sock);
-		return -1;
+	errno = 0;
+	*size = strtoul(ptr + 1, NULL, 10);
+	if (*size == 0) {
+		 /* perhaps server sends outputs LIST info in a different format */
+		log_message("%s: failed to parse server reply", __func__);
+	} else if (*size == ULONG_MAX) {
+		/* Keep ULONG_MAX as is so the caller knows the file is too big */
+		log_message("%s: file %s is too big (>= %lu bytes)", __func__, name, ULONG_MAX);
+	} else if (errno != 0) {
+		log_message("%s: strtoul failed: %s", __func__, strerror(errno));
+		*size = 0; /* jist in a case */
 	}
-
-	return size;
+	if (!*size) {
+		/* XXX: strtoul might return 0 *without* setting errno */
+		log_message("%s: failed to find out size of %s", __func__, name);
+	}
+	return 0;
 }
 
-
-int ftp_start_download(int sock, char * remotename, int * size)
+int basename_dirname(const char *abspath, char **dirp, char **namep)
 {
-	if ((*size = ftp_get_filesize(sock, remotename)) == -1) {
+	int ret = 0;
+	const char *ptr, *dir = NULL, *name = NULL;
+	ssize_t dirlen;
+	if (!dirp || !namep) {
+		ret = -1;
+		goto out;
+	}
+
+	ptr = strrchr(abspath, '/');
+	if (!ptr) {
+		ret = -1;
+		log_message("%s: abspath contains no '/'", __func__);
+		goto out;
+	}
+	/* /dir/name'
+	 *     ^ ptr = abspath + 4
+	 * strlen('/dir') == 4 == ptr - abspath
+	 */
+	dirlen =  ptr - abspath;
+	if (dirlen <= 0) {
+		ret = -1;
+		log_message("%s: dirlen <= 0", __func__);
+		goto out;
+	}
+	dir = strndup(abspath, dirlen);
+	if (!dir) {
+		ret = -1;
+		log_message("%s: strndup failed", __func__);
+		goto out;
+	}
+	name = strdup(ptr + 1);
+	if (!name) {
+		ret = -1;
+		log_message("%s: strdup failed", __func__);
+		goto out;
+	}
+	if (*dirp) {
+		free(*dirp);
+	}
+	*dirp = (char *)dir;
+	if (*namep) {
+		free(*namep);
+	}
+	*namep = (char *)name;
+out:
+	if (ret != 0 && dir) {
+		free((char *)dir);
+	}
+	if (ret != 0 && name) {
+		free((char *)name);
+	}
+	return ret;
+}
+
+static int
+ftp_get_filesize(int sock, const char *remotename, unsigned long *size)
+{
+	char *file = NULL, *dir = NULL;
+	int ret = 0, fd = -1;
+	FILE *listf = NULL;
+	char *line = NULL;
+	size_t line_len = 0;
+
+	ret = basename_dirname(remotename, &dir, &file);
+	if (ret) {
+		log_message("%s: failed to split '%s'", __func__, remotename);
+		goto out;
+	}
+
+	if (ftp_command(sock, "CWD", dir)) {
+		ret = ERROR;
+		goto out;
+	}
+	fd = ftp_data_command(sock, "LIST", file);
+	if (fd < 0) {
+		ret = ERROR;
+		log_message("%s: LIST ftp command failed", __func__);
+		goto out;
+	}
+
+	listf = fdopen(fd, "r");
+	if (!listf) {
+		ret = ERROR;
+		log_message("%s: fdopen error: %s", __func__, strerror(errno));
+		goto out;
+	}
+
+	ret = CONTINUE;
+
+	while (!feof(listf) && !ferror(listf)) {
+		if (getline(&line, &line_len, listf) < 0) {
+			log_message("%s: getline error: %s", __func__, strerror(errno));
+			ret = ERROR;
+			goto out;
+		}
+		ret = parse_ftp_filesize(line, file, size);
+		if (ret != CONTINUE) {
+			break;
+		}
+	}
+	if (CONTINUE == ret) {
+		log_message("%s: file '%s' not found in LIST output", __func__, file);
+	} else if (ERROR == ret) {
+		log_message("%s: failed to parse file '%s' size", __func__, file);
+	}
+out:
+	if (line) {
+		free(line);
+	}
+	if (file) {
+		free(file);
+	}
+	if (dir) {
+		free(dir);
+	}
+	if (listf) {
+		fclose(listf); /* also closes fd */
+	}
+	if (fd >= 0 && !listf) {
+		close(fd);
+	}
+	if (fd >= 0) {
+		if (!ftp_end_data_command(sock)) {
+			log_message("%s: ftp_end_data_command failed", __func__);
+			if (!ret) {
+				ret = ERROR;
+			}
+		}
+	}
+	return ret;
+}
+
+int ftp_start_download(int sock, char * remotename, unsigned long * size)
+{
+	if (ftp_get_filesize(sock, remotename, size) != 0) {
 		log_message("FTP: could not get filesize (trying to continue)");
 		*size = 0;
 	}
-	return ftp_data_command(sock, "RETR", remotename);
+	if (*size == ULONG_MAX) {
+		log_message("FTP: file '%s' is too big (>= %lu)", remotename, *size);
+		return FTPERR_FILE2BIG;
+	} else {
+		return ftp_data_command(sock, "RETR", remotename);
+	}
 }
 
 

url.h

@@ -23,11 +23,28 @@
 #define _URL_H_
 
 int ftp_open_connection(char * host, char * name, char * password, char * proxy);
-int ftp_start_download(int sock, char * remotename, int * size);
+int ftp_start_download(int sock, char * remotename, unsigned long * size);
 int ftp_end_data_command(int sock);
 
 int http_download_file(char * hostname, char * remotename, unsigned long * size);
 
+/**
+ * @brief split the absolute path into directory and file name
+ * @param abspath the path to operate on
+ * @param dirp, pointer to directory name, will be set by this function
+ * @param namep, pointer to file name, will be set by this function
+ * @returns -1 on error, otherwise 0
+ *
+ * @note Memory for directory and file name is allocated by this
+ *       function. Freeing it is the responsibility of the caller.
+ * @note dirp and namep are NOT altered on error
+ * @note On error the caller don't have to cleanup dirp, namep
+ *
+ * XXX: dirname and basename in libc are too tricky (can change
+ *      the argument, return pointers that can't be freed, etc),
+ *      hence this function
+ */
+int basename_dirname(const char *abspath, char **dirp, char **namep);
 
 #define FTPERR_BAD_SERVER_RESPONSE   -1
 #define FTPERR_SERVER_IO_ERROR       -2
@@ -39,6 +56,7 @@ int http_download_file(char * hostname, char * remotename, unsigned long * size)
 #define FTPERR_PASSIVE_ERROR         -8
 #define FTPERR_FAILED_DATA_CONNECT   -9
 #define FTPERR_FILE_NOT_FOUND        -10
+#define FTPERR_FILE2BIG              -11
 #define FTPERR_UNKNOWN               -100
 
 #endif