2022-06-11 01:19:22 +03:00
#!/usr/bin/python3
import argparse
2022-07-30 02:45:16 +03:00
import json
2022-06-11 01:19:22 +03:00
import os
import re
import subprocess
2022-12-08 03:47:41 +03:00
import textwrap
2022-06-11 01:19:22 +03:00
from graphlib import TopologicalSorter
from pathlib import Path
from jinja2 import Template
2022-12-06 04:00:08 +03:00
ORG_DIR = Path ( " org " )
2022-06-11 01:19:22 +03:00
2022-12-12 02:34:20 +03:00
class Tasks :
def __init__ ( self , tasks ) :
if tasks is None :
self . _tasks = None
else :
self . _tasks = json . loads ( Path ( tasks ) . read_text ( ) )
def get ( self , branch , image ) :
if self . _tasks is None :
return [ ]
else :
if branch_tasks := self . _tasks . get ( branch ) :
return [ n for n , i in branch_tasks . items ( ) if image in i or len ( i ) == 0 ]
2022-06-11 01:19:22 +03:00
class DockerBuilder :
def make_from_re ( self ) :
registry = r " (?P<registry>[ \ w.:]+) "
organization = r " (?P<organization> \ w+) "
name = r " (?P<name> \ w+) "
tag = r " (?P<tag>[ \ w.]+) "
return f " ^FROM (:? { registry } /)?(:? { organization } /)? { name } (:?: { tag } )?$ "
2022-12-06 04:00:08 +03:00
def __init__ (
self ,
registry ,
organization ,
overwrite_organization ,
latest ,
dry_run ,
images_info ,
2022-12-12 02:34:20 +03:00
tasks : Tasks ,
2022-12-06 04:00:08 +03:00
) :
2022-06-11 01:19:22 +03:00
self . from_re = re . compile ( self . make_from_re ( ) )
2022-12-06 04:00:08 +03:00
self . org_dir = ORG_DIR
self . images_dir = ORG_DIR / organization
2022-06-11 01:19:22 +03:00
self . registry = registry
self . organization = organization
2022-12-06 04:00:08 +03:00
if overwrite_organization :
self . overwrite_organization = overwrite_organization
else :
self . overwrite_organization = organization
2022-06-11 01:19:22 +03:00
self . latest = latest
self . dry_run = dry_run
2022-07-30 02:45:16 +03:00
self . images_info = images_info
2022-12-12 02:34:20 +03:00
self . tasks = tasks
2022-06-11 01:19:22 +03:00
2022-12-06 04:00:08 +03:00
def full_image ( self , image ) :
return f " { self . organization } / { image } "
2022-06-11 01:19:22 +03:00
def forall_images ( consume_result ) :
def forall_images_decorator ( f ) :
def wrapped ( self , * args , * * kwargs ) :
for image in self . images_dir . iterdir ( ) :
2022-12-12 02:34:20 +03:00
image_name = " / " . join ( image . parts [ 1 : ] )
2022-06-11 01:19:22 +03:00
local_kwargs = {
2022-12-12 02:34:20 +03:00
" image_name " : image_name ,
2022-06-11 01:19:22 +03:00
" image " : image ,
" dockerfile " : image / " Dockerfile " ,
" dockerfile_template " : image / " Dockerfile.template " ,
}
new_kwargs = kwargs | local_kwargs
yield f ( self , * args , * * new_kwargs )
def consumer ( * args , * * kwargs ) :
for _ in wrapped ( * args , * * kwargs ) :
pass
if consume_result :
return consumer
else :
return wrapped
return forall_images_decorator
@forall_images ( consume_result = True )
def remove_dockerfiles ( self , * * kwargs ) :
if kwargs [ " dockerfile " ] . exists ( ) :
kwargs [ " dockerfile " ] . unlink ( )
@forall_images ( consume_result = True )
def render_dockerfiles ( self , branch , * * kwargs ) :
2022-12-08 03:47:41 +03:00
def install_pakages ( * names ) :
2022-12-12 02:34:20 +03:00
tasks = self . tasks . get ( branch , kwargs [ " image_name " ] )
if tasks :
apt_repo = " \\ \n apt-get install apt-repo -y && \\ "
linux32 = ' $([ " $(rpm --eval % _host_cpu) " = i586 ] && echo linux32) '
for task in tasks :
apt_repo + = f " \n { linux32 } apt-repo add { task } && \\ "
apt_repo + = f " \n apt-get update && \\ "
else :
apt_repo = " \\ "
update_command = f """ RUN apt-get update && { apt_repo } """
install_command = f """
apt - get install - y { ' ' . join ( names ) } & & \\
rm - f / var / cache / apt / archives / * . rpm \\
/ var / cache / apt / * . bin \\
/ var / lib / apt / lists / * . *
"""
install_command = textwrap . dedent ( install_command ) . rstrip ( " \n " )
install_command = textwrap . indent ( install_command , " " * 4 )
return update_command + install_command
2022-12-08 03:47:41 +03:00
2022-06-11 01:19:22 +03:00
if kwargs [ " dockerfile_template " ] . exists ( ) :
if self . registry :
registry = self . registry . rstrip ( " / " ) + " / "
alt_image = " alt/alt "
else :
registry = " "
alt_image = " alt "
rendered = Template ( kwargs [ " dockerfile_template " ] . read_text ( ) ) . render (
alt_image = alt_image ,
branch = branch ,
2022-12-08 03:47:41 +03:00
install_pakages = install_pakages ,
2022-06-11 01:19:22 +03:00
organization = self . organization ,
registry = registry ,
)
kwargs [ " dockerfile " ] . write_text ( rendered + " \n " )
@forall_images ( consume_result = False )
def get_requires ( self , * * kwargs ) :
requires = set ( )
for line in kwargs [ " dockerfile " ] . read_text ( ) . splitlines ( ) :
if match := re . match ( self . from_re , line ) :
from_image = match . groupdict ( )
if from_image [ " organization " ] == self . organization :
requires . add ( from_image [ " name " ] )
return ( kwargs [ " image " ] . name , requires )
def get_build_order ( self ) :
requires = { }
for image , image_requires in self . get_requires ( ) :
requires [ image ] = image_requires
ts = TopologicalSorter ( requires )
return ts . static_order ( )
def render_full_tag ( self , image , tag ) :
if self . registry :
registry = self . registry . rstrip ( " / " ) + " / "
else :
registry = " "
if tag :
tag = f " : { tag } "
2022-12-06 04:00:08 +03:00
return f " { registry } { self . overwrite_organization } / { image } { tag } "
2022-06-11 01:19:22 +03:00
def run ( self , cmd , * args , * * kwargs ) :
if self . dry_run :
pre_cmd = [ " echo " ]
else :
pre_cmd = [ ]
subprocess . run ( pre_cmd + cmd , * args , * * kwargs )
def build ( self , image , arches , tag ) :
new_env = os . environ | { " DOCKER_BUILDKIT " : " 1 " }
2022-12-06 04:00:08 +03:00
info = self . images_info . get ( self . full_image ( image ) , { } )
2022-07-30 02:45:16 +03:00
if tag in info . get ( " skip-branches " , [ ] ) :
return
2022-12-06 04:00:08 +03:00
msg = " Building image {} for branch {} and {} arches " . format (
self . full_image ( image ) ,
tag ,
arches ,
)
print ( msg )
2022-07-30 02:45:16 +03:00
build_arches = set ( arches ) - set ( info . get ( " skip-arches " , [ ] ) )
platforms = " , " . join ( [ f " linux/ { a } " for a in build_arches ] )
2022-06-11 01:19:22 +03:00
full_name = self . render_full_tag ( image , tag )
if tag == self . latest :
lates_name = self . render_full_tag ( image , " latest " )
names = f " { full_name } , { lates_name } "
else :
names = full_name
cmd = [
" buildctl " ,
" build " ,
2022-12-09 02:14:46 +03:00
" --progress=plain " ,
2022-06-11 01:19:22 +03:00
" --frontend=dockerfile.v0 " ,
" --local " ,
" context=. " ,
" --local " ,
" dockerfile=. " ,
" --opt " ,
f " platform= { platforms } " ,
" --output " ,
f ' type=image, " name= { names } " ,push=true ' ,
]
self . run (
cmd ,
cwd = self . images_dir / image ,
env = new_env ,
)
def parse_args ( ) :
2022-06-14 16:13:19 +03:00
stages = [ " build " , " remove_dockerfiles " , " render_dockerfiles " ]
2022-06-11 01:19:22 +03:00
arches = [ " amd64 " , " 386 " , " arm64 " , " arm " , " ppc64le " ]
branches = [ " p9 " , " p10 " , " sisyphus " ]
2022-12-06 04:00:08 +03:00
organizations = list ( ORG_DIR . iterdir ( ) )
images = [ f " { o . name } / { i . name } " for o in organizations for i in o . iterdir ( ) ]
organizations = [ o . name for o in organizations ]
2022-06-11 01:19:22 +03:00
parser = argparse . ArgumentParser (
formatter_class = argparse . ArgumentDefaultsHelpFormatter ,
)
2022-12-06 04:00:08 +03:00
images_group = parser . add_mutually_exclusive_group ( required = True )
images_group . add_argument (
" -i " ,
" --images " ,
nargs = " + " ,
default = images ,
choices = images ,
help = " list of branches " ,
)
images_group . add_argument (
" -o " ,
" --organizations " ,
nargs = " + " ,
default = organizations ,
choices = organizations ,
help = " build all images from these organizations " ,
)
2022-06-11 01:19:22 +03:00
parser . add_argument (
" -r " ,
" --registry " ,
default = " registry.altlinux.org " ,
)
parser . add_argument (
2022-12-06 04:00:08 +03:00
" --overwrite-organization " ,
2022-06-11 01:19:22 +03:00
)
parser . add_argument (
" -l " ,
" --latest " ,
default = " p10 " ,
)
2022-12-12 02:34:20 +03:00
parser . add_argument (
" --tasks " ,
type = Tasks ,
default = Tasks ( None ) ,
)
2022-06-11 01:19:22 +03:00
parser . add_argument (
" --dry-run " ,
action = " store_true " ,
help = " print instead of running docker commands " ,
)
parser . add_argument (
" --skip-images " ,
nargs = " + " ,
default = [ ] ,
choices = images ,
help = " list of skipping images " ,
)
parser . add_argument (
" -a " ,
" --arches " ,
nargs = " + " ,
default = arches ,
choices = arches ,
help = " list of arches " ,
)
parser . add_argument (
" --skip-arches " ,
nargs = " + " ,
default = [ ] ,
choices = arches ,
help = " list of skipping arches " ,
)
parser . add_argument (
" -b " ,
" --branches " ,
nargs = " + " ,
default = branches ,
choices = branches ,
help = " list of branches " ,
)
parser . add_argument (
" --skip-branches " ,
nargs = " + " ,
default = [ ] ,
choices = branches ,
help = " list of skipping branches " ,
)
parser . add_argument (
" --stages " ,
nargs = " + " ,
default = stages ,
choices = stages ,
help = " list of stages " ,
)
parser . add_argument (
" --skip-stages " ,
nargs = " + " ,
default = [ ] ,
choices = stages ,
help = " list of skipping stages " ,
)
args = parser . parse_args ( )
args . stages = set ( args . stages ) - set ( args . skip_stages )
args . branches = set ( args . branches ) - set ( args . skip_branches )
args . images = set ( args . images ) - set ( args . skip_images )
return args
2022-07-30 02:45:16 +03:00
def get_images_info ( ) :
result = { }
images_info = Path ( " images-info.json " )
if images_info . exists ( ) :
result = json . loads ( images_info . read_text ( ) )
return result
2022-06-11 01:19:22 +03:00
def main ( ) :
args = parse_args ( )
2022-07-30 02:45:16 +03:00
images_info = get_images_info ( )
2022-12-06 04:00:08 +03:00
for organization in args . organizations :
for branch in args . branches :
db = DockerBuilder (
args . registry ,
organization ,
args . overwrite_organization ,
args . latest ,
args . dry_run ,
images_info ,
2022-12-12 02:34:20 +03:00
args . tasks ,
2022-12-06 04:00:08 +03:00
)
if " remove_dockerfiles " in args . stages :
db . remove_dockerfiles ( )
if " render_dockerfiles " in args . stages :
db . render_dockerfiles ( branch )
if " build " in args . stages :
for image in db . get_build_order ( ) :
if f " { organization } / { image } " not in args . images :
continue
if " build " in args . stages :
db . build ( image , args . arches , branch )
2022-06-11 01:19:22 +03:00
if __name__ == " __main__ " :
main ( )
# vim: colorcolumn=89