2019-08-10 21:29:25 +00:00
// This file contains the logic for building our CI for Drone. The idea here is
// that we create a pipeline for all of the major tasks we need to perform
// (e.g. builds, E2E testing, conformance testing, releases). Each pipeline
// after the default builds on a previous pipeline.
2019-09-06 22:37:37 -05:00
// Generate with `drone jsonnet --source ./hack/drone.jsonnet --stream --format`
2020-04-07 22:49:12 +03:00
// Sign with `drone sign talos-systems/talos --save`
2019-08-10 21:29:25 +00:00
2019-08-02 16:08:24 -05:00
local build_container = ' a u t o n o m y / b u i l d - c o n t a i n e r : l a t e s t ' ;
2020-09-26 00:00:43 +03:00
local local_registry = ' r e g i s t r y . d e v . t a l o s - s y s t e m s . i o ' ;
2019-12-23 10:36:14 -08:00
2019-08-10 21:29:25 +00:00
local volumes = {
dockersock : {
pipeline : {
2019-08-02 16:08:24 -05:00
name : ' d o c k e r s o c k ' ,
2019-08-10 21:29:25 +00:00
temp : { } ,
} ,
step : {
name : $ . dockersock . pipeline . name ,
2019-08-02 16:08:24 -05:00
path : ' / v a r / r u n ' ,
2019-08-10 21:29:25 +00:00
} ,
2019-08-09 03:45:13 +00:00
} ,
2019-08-10 21:29:25 +00:00
2020-07-27 23:24:07 +03:00
outerdockersock : {
pipeline : {
name : ' o u t e r d o c k e r s o c k ' ,
host : {
path : ' / v a r / c i - d o c k e r '
} ,
} ,
step : {
name : $ . outerdockersock . pipeline . name ,
path : ' / v a r / o u t e r - r u n ' ,
} ,
} ,
2019-12-23 10:36:14 -08:00
docker : {
pipeline : {
name : ' d o c k e r ' ,
temp : { } ,
} ,
step : {
name : $ . docker . pipeline . name ,
path : ' / r o o t / . d o c k e r / b u i l d x ' ,
} ,
} ,
kube : {
pipeline : {
name : ' k u b e ' ,
temp : { } ,
} ,
step : {
name : $ . kube . pipeline . name ,
path : ' / r o o t / . k u b e ' ,
} ,
} ,
2019-08-10 21:29:25 +00:00
dev : {
pipeline : {
2019-08-02 16:08:24 -05:00
name : ' d e v ' ,
2019-08-10 21:29:25 +00:00
host : {
2019-08-02 16:08:24 -05:00
path : ' / d e v ' ,
2019-08-10 21:29:25 +00:00
} ,
} ,
step : {
name : $ . dev . pipeline . name ,
2019-08-02 16:08:24 -05:00
path : ' / d e v ' ,
2019-08-10 21:29:25 +00:00
} ,
2019-08-09 03:45:13 +00:00
} ,
2019-08-10 21:29:25 +00:00
2020-06-29 21:36:51 +03:00
tmp : {
pipeline : {
name : ' t m p ' ,
2020-07-30 16:21:38 +03:00
temp : {
' m e d i u m ' : ' m e m o r y ' ,
} ,
2020-06-29 21:36:51 +03:00
} ,
step : {
name : $ . tmp . pipeline . name ,
path : ' / t m p ' ,
} ,
} ,
2019-08-10 21:29:25 +00:00
ForStep ( ) : [
self . dockersock . step ,
2020-07-27 23:24:07 +03:00
self . outerdockersock . step ,
2019-12-23 10:36:14 -08:00
self . docker . step ,
self . kube . step ,
2019-08-10 21:29:25 +00:00
self . dev . step ,
2020-07-30 00:36:58 +03:00
self . tmp . step ,
2019-08-10 21:29:25 +00:00
] ,
ForPipeline ( ) : [
self . dockersock . pipeline ,
2020-07-27 23:24:07 +03:00
self . outerdockersock . pipeline ,
2019-12-23 10:36:14 -08:00
self . docker . pipeline ,
self . kube . pipeline ,
2019-08-10 21:29:25 +00:00
self . dev . pipeline ,
2020-06-29 21:36:51 +03:00
self . tmp . pipeline ,
2019-08-10 21:29:25 +00:00
] ,
} ;
2019-08-09 03:45:13 +00:00
2019-08-10 21:29:25 +00:00
// This provides the docker service.
2019-08-09 03:45:13 +00:00
local docker = {
2019-08-02 16:08:24 -05:00
name : ' d o c k e r ' ,
image : ' d o c k e r : 1 9 . 0 3 - d i n d ' ,
entrypoint : [ ' d o c k e r d ' ] ,
2019-08-09 03:45:13 +00:00
privileged : true ,
command : [
2019-08-02 16:08:24 -05:00
' - - d n s = 8 . 8 . 8 . 8 ' ,
' - - d n s = 8 . 8 . 4 . 4 ' ,
2020-12-17 15:47:58 +03:00
' - - m t u = 1 4 5 0 ' ,
2019-08-02 16:08:24 -05:00
' - - l o g - l e v e l = e r r o r ' ,
2019-12-27 18:12:04 +00:00
] ,
2021-02-16 22:30:26 +03:00
// Set resource requests to ensure that only three builds can be performed at a
2020-07-23 11:21:42 -07:00
// time. We set it on the service so that we get the scheduling restricitions
// while still allowing parallel steps.
resources : {
requests : {
2021-02-16 22:30:26 +03:00
cpu : 12000 ,
memory : ' 1 8 G i B ' ,
2020-07-23 11:21:42 -07:00
} ,
} ,
2019-08-10 21:29:25 +00:00
volumes : volumes . ForStep ( ) ,
2019-08-09 03:45:13 +00:00
} ;
2019-12-24 09:28:58 -08:00
// Sets up the CI environment
local setup_ci = {
name : ' s e t u p - c i ' ,
2019-12-23 10:36:14 -08:00
image : ' a u t o n o m y / b u i l d - c o n t a i n e r : l a t e s t ' ,
2020-11-10 09:12:26 -08:00
pull : " a l w a y s " ,
2019-08-09 03:45:13 +00:00
privileged : true ,
2020-09-26 00:00:43 +03:00
2019-12-23 10:36:14 -08:00
commands : [
2020-11-10 09:12:26 -08:00
' s e t u p - c i ' ,
2020-01-23 20:52:02 -08:00
' m a k e . / _ o u t / s o n o b u o y ' ,
' m a k e . / _ o u t / k u b e c t l ' ,
2019-12-23 10:36:14 -08:00
] ,
volumes : volumes . ForStep ( ) ,
2019-08-09 03:45:13 +00:00
} ;
2019-08-10 21:29:25 +00:00
// Step standardizes the creation of build steps. The name of the step is used
// as the target when building the make command. For example, if name equals
// "test", the resulting step command will be "make test". This is done to
// encourage alignment between this file and the Makefile, and gives us a
// standardized structure that should make things easier to reason about if we
// know that each step is essentially a Makefile target.
2020-07-01 21:20:24 +03:00
local Step ( name , image = ' ' , target = ' ' , privileged = false , depends_on = [ ] , environment = { } , extra_volumes = [ ] , when = { } ) = {
2019-08-02 16:08:24 -05:00
local make = if target = = ' ' then std.format ( ' m a k e % s ' , name ) else std.format ( ' m a k e % s ' , target ) ,
2019-12-04 10:22:36 -05:00
2020-07-23 11:21:42 -07:00
local common_env_vars = {
2020-09-26 00:00:43 +03:00
" P L A T F O R M " : " l i n u x / a m d 6 4 , l i n u x / a r m 6 4 " ,
2020-07-23 11:21:42 -07:00
} ,
2019-08-09 03:45:13 +00:00
2019-08-10 21:29:25 +00:00
name : name ,
2019-12-04 10:22:36 -05:00
image : if image = = ' ' then build_container else image ,
2019-09-23 21:45:21 -07:00
pull : " a l w a y s " ,
2019-08-10 21:29:25 +00:00
commands : [ make ] ,
2020-01-21 19:37:12 -08:00
privileged : privileged ,
2019-08-10 21:29:25 +00:00
environment : common_env_vars + environment ,
2020-06-29 21:36:51 +03:00
volumes : volumes . ForStep ( ) + extra_volumes ,
2019-08-09 03:45:13 +00:00
depends_on : [ x . name for x in depends_on ] ,
2020-07-01 21:20:24 +03:00
when : when ,
2019-08-09 03:45:13 +00:00
} ;
2019-08-10 21:29:25 +00:00
// Pipeline is a way to standardize the creation of pipelines. It supports
// using and existing pipeline as a base.
2020-07-23 11:21:42 -07:00
local Pipeline ( name , steps = [ ] , depends_on = [ ] , with_docker = true , disable_clone = false , type = ' k u b e r n e t e s ' ) = {
2019-08-02 16:08:24 -05:00
kind : ' p i p e l i n e ' ,
2020-07-23 11:21:42 -07:00
type : type ,
2019-08-09 03:45:13 +00:00
name : name ,
2020-07-23 11:21:42 -07:00
[ if type = = ' d i g i t a l o c e a n ' then ' t o k e n ' ] : {
from_secret : ' d i g i t a l o c e a n _ t o k e n '
} ,
// See https://slugs.do-api.dev/.
[ if type = = ' d i g i t a l o c e a n ' then ' s e r v e r ' ] : {
image : ' u b u n t u - 2 0 - 0 4 - x 6 4 ' ,
size : ' c - 3 2 ' ,
region : ' n y c 3 ' ,
} ,
[ if with_docker then ' s e r v i c e s ' ] : [ docker ] ,
2019-12-10 17:44:07 +00:00
[ if disable_clone then ' c l o n e ' ] : {
disable : true ,
} ,
2019-09-23 21:45:21 -07:00
steps : steps ,
2019-08-10 21:29:25 +00:00
volumes : volumes . ForPipeline ( ) ,
depends_on : [ x . name for x in depends_on ] ,
2019-08-09 03:45:13 +00:00
} ;
2019-08-10 21:29:25 +00:00
// Default pipeline.
2020-10-21 16:59:52 +03:00
local generate = Step ( " g e n e r a t e " , target = " g e n e r a t e d o c s " , depends_on = [ setup_ci ] ) ;
local check_dirty = Step ( " c h e c k - d i r t y " , depends_on = [ generate ] ) ;
2021-02-25 20:44:01 +03:00
local build = Step ( " b u i l d " , target = " t a l o s c t l - l i n u x t a l o s c t l - d a r w i n k e r n e l i n i t r a m f s i n s t a l l e r t a l o s " , depends_on = [ check_dirty ] , environment = { " IMAGE_REGISTRY" : local_registry , " PUSH" : true } ) ;
2021-02-16 22:30:26 +03:00
local lint = Step ( " l i n t " , depends_on = [ build ] ) ;
local talosctl_cni_bundle = Step ( ' t a l o s c t l - c n i - b u n d l e ' , depends_on = [ build , lint ] ) ;
2021-02-24 15:43:27 +03:00
local iso_amd64 = Step ( " i s o - a m d 6 4 " , target = " i s o " , depends_on = [ build ] , environment = { " IMAGE_REGISTRY" : local_registry } ) ;
local iso_arm64 = Step ( " i s o - a r m 6 4 " , target = " i s o " , depends_on = [ build ] , environment = { " IMAGE_REGISTRY" : local_registry , " DOCKER_HOST" : " t c p : / / d o c k e r - a r m 6 4 . c i . s v c : 2 3 7 6 " } ) ;
local images_amd64 = Step ( " i m a g e s - a m d 6 4 " , target = " i m a g e s " , depends_on = [ iso_amd64 ] , environment = { " IMAGE_REGISTRY" : local_registry } ) ;
local images_arm64 = Step ( " i m a g e s - a r m 6 4 " , target = " i m a g e s " , depends_on = [ iso_arm64 ] , environment = { " IMAGE_REGISTRY" : local_registry , " DOCKER_HOST" : " t c p : / / d o c k e r - a r m 6 4 . c i . s v c : 2 3 7 6 " } ) ;
local sbcs_arm64 = Step ( " s b c s - a r m 6 4 " , target = " s b c s " , depends_on = [ images_amd64 , images_arm64 ] , environment = { " IMAGE_REGISTRY" : local_registry , " DOCKER_HOST" : " t c p : / / d o c k e r - a r m 6 4 . c i . s v c : 2 3 7 6 " } ) ;
2021-02-16 22:30:26 +03:00
local unit_tests = Step ( " u n i t - t e s t s " , target = " u n i t - t e s t s u n i t - t e s t s - r a c e " , depends_on = [ build , lint ] ) ;
2021-02-24 15:43:27 +03:00
local e2e_docker = Step ( " e 2 e - d o c k e r - s h o r t " , depends_on = [ build , unit_tests ] , target = " e 2 e - d o c k e r " , environment = { " SHORT_INTEGRATION_TEST" : " y e s " , " IMAGE_REGISTRY" : local_registry } ) ;
local e2e_qemu = Step ( " e 2 e - q e m u - s h o r t " , privileged = true , target = " e 2 e - q e m u " , depends_on = [ build , unit_tests , talosctl_cni_bundle ] , environment = { " IMAGE_REGISTRY" : local_registry , " SHORT_INTEGRATION_TEST" : " y e s " } , when = { event : [ ' p u l l _ r e q u e s t ' ] } ) ;
local e2e_iso = Step ( " e 2 e - i s o " , privileged = true , target = " e 2 e - i s o " , depends_on = [ build , unit_tests , iso_amd64 , talosctl_cni_bundle ] , when = { event : [ ' p u l l _ r e q u e s t ' ] } , environment = { " IMAGE_REGISTRY" : local_registry } ) ;
2019-08-09 03:45:13 +00:00
local coverage = {
2019-08-02 16:08:24 -05:00
name : ' c o v e r a g e ' ,
2021-04-19 17:17:48 +00:00
image : ' a u t o n o m y / b u i l d - c o n t a i n e r : l a t e s t ' ,
pull : ' a l w a y s ' ,
2020-01-23 20:52:02 -08:00
environment : {
CODECOV_TOKEN : { from_secret : ' c o d e c o v _ t o k e n ' } ,
2019-08-09 03:45:13 +00:00
} ,
2020-01-23 20:52:02 -08:00
commands : [
2021-04-19 17:17:48 +00:00
' / u s r / l o c a l / b i n / c o d e c o v - f _ o u t / c o v e r a g e . t x t - X f i x '
2020-01-23 20:52:02 -08:00
] ,
2019-08-10 21:29:25 +00:00
when : {
2019-08-02 16:08:24 -05:00
event : [ ' p u l l _ r e q u e s t ' ] ,
2019-08-09 03:45:13 +00:00
} ,
2021-02-16 22:30:26 +03:00
depends_on : [ unit_tests . name ] ,
2019-08-09 03:45:13 +00:00
} ;
2020-01-01 10:28:44 -08:00
local push = {
name : ' p u s h ' ,
image : ' a u t o n o m y / b u i l d - c o n t a i n e r : l a t e s t ' ,
pull : ' a l w a y s ' ,
environment : {
2020-09-23 16:21:43 -07:00
GHCR_USERNAME : { from_secret : ' g h c r _ u s e r n a m e ' } ,
GHCR_PASSWORD : { from_secret : ' g h c r _ t o k e n ' } ,
2020-09-28 16:10:49 +03:00
PLATFORM : " l i n u x / a m d 6 4 , l i n u x / a r m 6 4 " ,
2020-01-01 10:28:44 -08:00
} ,
commands : [ ' m a k e p u s h ' ] ,
volumes : volumes . ForStep ( ) ,
when : {
event : {
exclude : [
' p u l l _ r e q u e s t ' ,
2020-01-01 11:51:48 -08:00
' p r o m o t e ' ,
' c r o n ' ,
2020-01-01 10:28:44 -08:00
] ,
} ,
} ,
2020-07-30 21:44:10 +03:00
depends_on : [ e2e_docker . name , e2e_qemu . name ] ,
2020-01-01 10:28:44 -08:00
} ;
2019-11-27 15:39:53 +00:00
local push_latest = {
name : ' p u s h - l a t e s t ' ,
2019-08-02 16:08:24 -05:00
image : ' a u t o n o m y / b u i l d - c o n t a i n e r : l a t e s t ' ,
pull : ' a l w a y s ' ,
environment : {
2020-09-23 16:21:43 -07:00
GHCR_USERNAME : { from_secret : ' g h c r _ u s e r n a m e ' } ,
GHCR_PASSWORD : { from_secret : ' g h c r _ t o k e n ' } ,
2020-09-28 16:10:49 +03:00
PLATFORM : " l i n u x / a m d 6 4 , l i n u x / a r m 6 4 " ,
2019-08-09 03:45:13 +00:00
} ,
2020-01-01 10:28:44 -08:00
commands : [ ' m a k e p u s h - l a t e s t ' ] ,
2019-08-10 21:29:25 +00:00
volumes : volumes . ForStep ( ) ,
2019-08-09 03:45:13 +00:00
when : {
2020-01-01 11:51:48 -08:00
branch : [
' m a s t e r ' ,
] ,
event : [
' p u s h ' ,
2020-01-01 09:43:24 -08:00
] ,
2019-08-09 03:45:13 +00:00
} ,
2020-11-27 00:03:18 +03:00
depends_on : [ push . name ] ,
2019-08-09 03:45:13 +00:00
} ;
2021-02-16 22:30:26 +03:00
local save_artifacts = {
name : ' s a v e - a r t i f a c t s ' ,
image : ' d o c k e r . i o / d 3 f k / s 3 c m d : l a t e s t ' ,
pull : ' a l w a y s ' ,
environment : {
AWS_ACCESS_KEY_ID : { from_secret : ' r o o k _ a c c e s s _ k e y _ i d ' } ,
AWS_SECRET_ACCESS_KEY : { from_secret : ' r o o k _ s e c r e t _ a c c e s s _ k e y ' } ,
} ,
commands : [
2021-02-17 17:55:45 +03:00
' s 3 c m d - - h o s t = r o o k - c e p h - r g w - c i - s t o r e . r o o k - c e p h . s v c - - h o s t - b u c k e t = r o o k - c e p h - r g w - c i - s t o r e . r o o k - c e p h . s v c - - n o - s s l m b s 3 : / / $ { C I _ C O M M I T _ S H A } $ { D R O N E _ T A G / / . / - } ' ,
2021-03-04 16:30:32 +03:00
' s 3 c m d - - h o s t = r o o k - c e p h - r g w - c i - s t o r e . r o o k - c e p h . s v c - - h o s t - b u c k e t = r o o k - c e p h - r g w - c i - s t o r e . r o o k - c e p h . s v c - - n o - s s l e x p i r e s 3 : / / $ { C I _ C O M M I T _ S H A } $ { D R O N E _ T A G / / . / - } - - e x p i r y - d a y s = 3 ' ,
2021-02-17 17:55:45 +03:00
' s 3 c m d - - h o s t = r o o k - c e p h - r g w - c i - s t o r e . r o o k - c e p h . s v c - - h o s t - b u c k e t = r o o k - c e p h - r g w - c i - s t o r e . r o o k - c e p h . s v c - - n o - s s l - - s t a t s s y n c _ o u t s 3 : / / $ { C I _ C O M M I T _ S H A } $ { D R O N E _ T A G / / . / - } ' ,
2021-02-16 22:30:26 +03:00
] ,
volumes : volumes . ForStep ( ) ,
depends_on : [ build . name , images_amd64 . name , images_arm64 . name , iso_amd64 . name , iso_arm64 . name , sbcs_arm64 . name , talosctl_cni_bundle . name ] ,
} ;
local load_artifacts = {
name : ' l o a d - a r t i f a c t s ' ,
image : ' d o c k e r . i o / d 3 f k / s 3 c m d : l a t e s t ' ,
pull : ' a l w a y s ' ,
environment : {
AWS_ACCESS_KEY_ID : { from_secret : ' r o o k _ a c c e s s _ k e y _ i d ' } ,
AWS_SECRET_ACCESS_KEY : { from_secret : ' r o o k _ s e c r e t _ a c c e s s _ k e y ' } ,
} ,
commands : [
2021-02-17 17:55:45 +03:00
' s 3 c m d - - h o s t = r o o k - c e p h - r g w - c i - s t o r e . r o o k - c e p h . s v c - - h o s t - b u c k e t = r o o k - c e p h - r g w - c i - s t o r e . r o o k - c e p h . s v c - - n o - s s l - - s t a t s s y n c s 3 : / / $ { C I _ C O M M I T _ S H A } $ { D R O N E _ T A G / / . / - } . ' ,
2021-02-16 22:30:26 +03:00
] ,
volumes : volumes . ForStep ( ) ,
depends_on : [ setup_ci . name ] ,
} ;
2019-08-10 21:29:25 +00:00
local default_steps = [
2019-12-24 09:28:58 -08:00
setup_ci ,
2020-04-07 22:49:12 +03:00
generate ,
check_dirty ,
2021-02-16 22:30:26 +03:00
build ,
2020-10-21 16:59:52 +03:00
lint ,
2020-10-30 01:08:05 +03:00
talosctl_cni_bundle ,
2020-12-17 16:00:38 +03:00
iso_amd64 ,
iso_arm64 ,
2020-11-13 17:17:07 +03:00
images_amd64 ,
images_arm64 ,
2020-11-30 18:52:23 -08:00
sbcs_arm64 ,
2019-08-09 03:45:13 +00:00
unit_tests ,
2021-02-16 22:30:26 +03:00
save_artifacts ,
2019-08-09 03:45:13 +00:00
coverage ,
2020-12-07 20:41:07 -08:00
e2e_iso ,
2020-07-30 21:44:10 +03:00
e2e_qemu ,
2020-12-07 20:41:07 -08:00
e2e_docker ,
2020-01-01 10:28:44 -08:00
push ,
2019-11-27 15:39:53 +00:00
push_latest ,
2019-08-09 03:45:13 +00:00
] ;
local default_trigger = {
trigger : {
2019-08-12 18:28:42 +00:00
event : {
2019-08-13 00:40:00 +00:00
exclude : [
2019-08-02 16:08:24 -05:00
' t a g ' ,
' p r o m o t e ' ,
2020-12-04 10:15:31 -05:00
' c r o n ' ,
2019-08-02 16:08:24 -05:00
] ,
2019-08-09 03:45:13 +00:00
} ,
} ,
} ;
2021-02-16 22:30:26 +03:00
local cron_trigger ( schedules ) = {
trigger : {
cron : {
2021-02-17 20:35:12 +03:00
include : schedules ,
2021-02-16 22:30:26 +03:00
} ,
} ,
} ;
2021-02-18 19:07:13 +03:00
local default_pipeline = Pipeline ( ' d e f a u l t ' , default_steps ) + default_trigger ;
2021-02-18 20:44:24 +03:00
local default_cron_pipeline = Pipeline ( ' c r o n - d e f a u l t ' , default_steps ) + cron_trigger ( [ ' t h r i c e - d a i l y ' , ' n i g h t l y ' ] ) ;
2021-02-18 19:07:13 +03:00
// Full integration pipeline.
2021-02-16 22:30:26 +03:00
local default_pipeline_steps = [
setup_ci ,
load_artifacts ,
] ;
2021-02-24 15:43:27 +03:00
local integration_qemu = Step ( " e 2 e - q e m u " , privileged = true , depends_on = [ load_artifacts ] , environment = { " IMAGE_REGISTRY" : local_registry } ) ;
2021-02-16 22:30:26 +03:00
local integration_provision_tests_prepare = Step ( " p r o v i s i o n - t e s t s - p r e p a r e " , privileged = true , depends_on = [ load_artifacts ] ) ;
2021-02-24 15:43:27 +03:00
local integration_provision_tests_track_0 = Step ( " p r o v i s i o n - t e s t s - t r a c k - 0 " , privileged = true , depends_on = [ integration_provision_tests_prepare ] , environment = { " IMAGE_REGISTRY" : local_registry } ) ;
local integration_provision_tests_track_1 = Step ( " p r o v i s i o n - t e s t s - t r a c k - 1 " , privileged = true , depends_on = [ integration_provision_tests_prepare ] , environment = { " IMAGE_REGISTRY" : local_registry } ) ;
2021-02-20 17:07:38 +03:00
local integration_provision_tests_track_2 = Step ( " p r o v i s i o n - t e s t s - t r a c k - 2 " , privileged = true , depends_on = [ integration_provision_tests_prepare ] , environment = { " IMAGE_REGISTRY" : local_registry } ) ;
2021-02-16 22:30:26 +03:00
2021-02-09 22:48:18 +03:00
local integration_cilium = Step ( " e 2 e - c i l i u m - 1 . 9 . 4 " , target = " e 2 e - q e m u " , privileged = true , depends_on = [ load_artifacts ] , environment = {
2020-07-01 18:34:20 +03:00
" S H O R T _ I N T E G R A T I O N _ T E S T " : " y e s " ,
2021-02-09 22:48:18 +03:00
" C U S T O M _ C N I _ U R L " : " h t t p s : / / r a w . g i t h u b u s e r c o n t e n t . c o m / c i l i u m / c i l i u m / v 1 . 9 . 4 / i n s t a l l / k u b e r n e t e s / q u i c k - i n s t a l l . y a m l " ,
2021-02-24 15:43:27 +03:00
" I M A G E _ R E G I S T R Y " : local_registry ,
2020-07-01 18:34:20 +03:00
} ) ;
2020-08-27 23:23:20 +03:00
local integration_uefi = Step ( " e 2 e - u e f i " , target = " e 2 e - q e m u " , privileged = true , depends_on = [ integration_cilium ] , environment = {
" S H O R T _ I N T E G R A T I O N _ T E S T " : " y e s " ,
" W I T H _ U E F I " : " t r u e " ,
2021-02-24 15:43:27 +03:00
" I M A G E _ R E G I S T R Y " : local_registry ,
2020-08-27 23:23:20 +03:00
} ) ;
2020-12-21 21:01:37 +03:00
local integration_disk_image = Step ( " e 2 e - d i s k - i m a g e " , target = " e 2 e - q e m u " , privileged = true , depends_on = [ integration_uefi ] , environment = {
" S H O R T _ I N T E G R A T I O N _ T E S T " : " y e s " ,
" U S E _ D I S K _ I M A G E " : " t r u e " ,
2021-02-24 15:43:27 +03:00
" I M A G E _ R E G I S T R Y " : local_registry ,
2021-01-31 18:23:09 +03:00
} ) ;
2020-12-24 18:06:25 +03:00
local integration_canal_reset = Step ( " e 2 e - c a n a l - r e s e t " , target = " e 2 e - q e m u " , privileged = true , depends_on = [ integration_disk_image ] , environment = {
" I N T E G R A T I O N _ T E S T _ R U N " : " T e s t I n t e g r a t i o n / a p i . R e s e t S u i t e / T e s t R e s e t W i t h S p e c " ,
" C U S T O M _ C N I _ U R L " : " h t t p s : / / d o c s . p r o j e c t c a l i c o . o r g / m a n i f e s t s / c a n a l . y a m l " ,
" R E G I S T R Y " : local_registry ,
} ) ;
2021-02-24 19:46:38 +03:00
local integration_qemu_encrypted_vip = Step ( " e 2 e - e n c r y p t e d - v i p " , target = " e 2 e - q e m u " , privileged = true , depends_on = [ load_artifacts ] , environment = {
2021-01-31 18:23:09 +03:00
" W I T H _ D I S K _ E N C R Y P T I O N " : " t r u e " ,
2021-02-24 19:46:38 +03:00
" W I T H _ V I R T U A L _ I P " : " t r u e " ,
2021-02-24 15:43:27 +03:00
" I M A G E _ R E G I S T R Y " : local_registry ,
2020-12-21 21:01:37 +03:00
} ) ;
2021-02-16 22:30:26 +03:00
2020-09-18 15:56:53 -04:00
local push_edge = {
name : ' p u s h - e d g e ' ,
image : ' a u t o n o m y / b u i l d - c o n t a i n e r : l a t e s t ' ,
pull : ' a l w a y s ' ,
environment : {
2020-09-23 16:21:43 -07:00
GHCR_USERNAME : { from_secret : ' g h c r _ u s e r n a m e ' } ,
2020-09-23 18:03:25 -07:00
GHCR_PASSWORD : { from_secret : ' g h c r _ t o k e n ' } ,
2020-09-18 15:56:53 -04:00
} ,
commands : [ ' m a k e p u s h - e d g e ' ] ,
volumes : volumes . ForStep ( ) ,
when : {
cron : [
' n i g h t l y ' ,
] ,
} ,
depends_on : [
2021-02-16 22:30:26 +03:00
integration_qemu . name ,
2020-09-18 15:56:53 -04:00
] ,
} ;
2020-06-25 22:56:59 +03:00
2021-02-16 22:30:26 +03:00
local integration_trigger ( names ) = {
2020-06-25 22:56:59 +03:00
trigger : {
target : {
2021-02-16 22:30:26 +03:00
include : [ ' i n t e g r a t i o n ' ] + names ,
2020-07-02 00:11:26 +03:00
} ,
} ,
} ;
2021-02-16 22:30:26 +03:00
local integration_pipelines = [
// regular pipelines, triggered on promote events
Pipeline ( ' i n t e g r a t i o n - q e m u ' , default_pipeline_steps + [ integration_qemu , push_edge ] ) + integration_trigger ( [ ' i n t e g r a t i o n - q e m u ' ] ) ,
Pipeline ( ' i n t e g r a t i o n - p r o v i s i o n - 0 ' , default_pipeline_steps + [ integration_provision_tests_prepare , integration_provision_tests_track_0 ] ) + integration_trigger ( [ ' i n t e g r a t i o n - p r o v i s i o n ' , ' i n t e g r a t i o n - p r o v i s i o n - 0 ' ] ) ,
Pipeline ( ' i n t e g r a t i o n - p r o v i s i o n - 1 ' , default_pipeline_steps + [ integration_provision_tests_prepare , integration_provision_tests_track_1 ] ) + integration_trigger ( [ ' i n t e g r a t i o n - p r o v i s i o n ' , ' i n t e g r a t i o n - p r o v i s i o n - 1 ' ] ) ,
2021-02-20 17:07:38 +03:00
Pipeline ( ' i n t e g r a t i o n - p r o v i s i o n - 2 ' , default_pipeline_steps + [ integration_provision_tests_prepare , integration_provision_tests_track_2 ] ) + integration_trigger ( [ ' i n t e g r a t i o n - p r o v i s i o n ' , ' i n t e g r a t i o n - p r o v i s i o n - 2 ' ] ) ,
2020-12-24 18:06:25 +03:00
Pipeline ( ' i n t e g r a t i o n - m i s c ' , default_pipeline_steps + [ integration_cilium , integration_uefi , integration_disk_image , integration_canal_reset ] ) + integration_trigger ( [ ' i n t e g r a t i o n - m i s c ' ] ) ,
2021-02-24 19:46:38 +03:00
Pipeline ( ' i n t e g r a t i o n - q e m u - e n c r y p t e d - v i p ' , default_pipeline_steps + [ integration_qemu_encrypted_vip ] ) + integration_trigger ( [ ' i n t e g r a t i o n - q e m u - e n c r y p t e d - v i p ' ] ) ,
2021-02-16 22:30:26 +03:00
// cron pipelines, triggered on schedule events
2021-02-18 19:07:13 +03:00
Pipeline ( ' c r o n - i n t e g r a t i o n - q e m u ' , default_pipeline_steps + [ integration_qemu , push_edge ] , [ default_cron_pipeline ] ) + cron_trigger ( [ ' t h r i c e - d a i l y ' , ' n i g h t l y ' ] ) ,
Pipeline ( ' c r o n - i n t e g r a t i o n - p r o v i s i o n - 0 ' , default_pipeline_steps + [ integration_provision_tests_prepare , integration_provision_tests_track_0 ] , [ default_cron_pipeline ] ) + cron_trigger ( [ ' t h r i c e - d a i l y ' , ' n i g h t l y ' ] ) ,
Pipeline ( ' c r o n - i n t e g r a t i o n - p r o v i s i o n - 1 ' , default_pipeline_steps + [ integration_provision_tests_prepare , integration_provision_tests_track_1 ] , [ default_cron_pipeline ] ) + cron_trigger ( [ ' t h r i c e - d a i l y ' , ' n i g h t l y ' ] ) ,
2021-02-20 17:07:38 +03:00
Pipeline ( ' c r o n - i n t e g r a t i o n - p r o v i s i o n - 2 ' , default_pipeline_steps + [ integration_provision_tests_prepare , integration_provision_tests_track_2 ] , [ default_cron_pipeline ] ) + cron_trigger ( [ ' t h r i c e - d a i l y ' , ' n i g h t l y ' ] ) ,
2020-12-24 18:06:25 +03:00
Pipeline ( ' c r o n - i n t e g r a t i o n - m i s c ' , default_pipeline_steps + [ integration_cilium , integration_uefi , integration_disk_image , integration_canal_reset ] , [ default_cron_pipeline ] ) + cron_trigger ( [ ' t h r i c e - d a i l y ' , ' n i g h t l y ' ] ) ,
2021-02-24 19:46:38 +03:00
Pipeline ( ' c r o n - i n t e g r a t i o n - q e m u - e n c r y p t e d - v i p ' , default_pipeline_steps + [ integration_qemu_encrypted_vip ] , [ default_cron_pipeline ] ) + cron_trigger ( [ ' t h r i c e - d a i l y ' , ' n i g h t l y ' ] ) ,
2021-02-16 22:30:26 +03:00
] ;
2020-07-02 00:11:26 +03:00
2020-06-25 22:56:59 +03:00
2019-08-10 21:29:25 +00:00
// E2E pipeline.
local creds_env_vars = {
2020-01-23 20:52:02 -08:00
AWS_ACCESS_KEY_ID : { from_secret : ' a w s _ a c c e s s _ k e y _ i d ' } ,
AWS_SECRET_ACCESS_KEY : { from_secret : ' a w s _ s e c r e t _ a c c e s s _ k e y ' } ,
AWS_SVC_ACCT : { from_secret : " a w s _ s v c _ a c c t " } ,
AZURE_SVC_ACCT : { from_secret : " a z u r e _ s v c _ a c c t " } ,
// TODO(andrewrynhard): Rename this to the GCP convention.
GCE_SVC_ACCT : { from_secret : " g c e _ s v c _ a c c t " } ,
PACKET_AUTH_TOKEN : { from_secret : " p a c k e t _ a u t h _ t o k e n " } ,
2019-08-09 03:45:13 +00:00
} ;
2021-02-24 15:43:27 +03:00
local capi_docker = Step ( " e 2 e - d o c k e r " , depends_on = [ load_artifacts ] , target = " e 2 e - d o c k e r " , environment = { " SHORT_INTEGRATION_TEST" : " y e s " , " IMAGE_REGISTRY" : local_registry } ) ;
2021-02-16 22:30:26 +03:00
local e2e_capi = Step ( " e 2 e - c a p i " , depends_on = [ capi_docker ] , environment = creds_env_vars ) ;
2020-07-30 00:36:58 +03:00
local e2e_aws = Step ( " e 2 e - a w s " , depends_on = [ e2e_capi ] , environment = creds_env_vars ) ;
local e2e_azure = Step ( " e 2 e - a z u r e " , depends_on = [ e2e_capi ] , environment = creds_env_vars ) ;
local e2e_gcp = Step ( " e 2 e - g c p " , depends_on = [ e2e_capi ] , environment = creds_env_vars ) ;
2019-08-10 21:29:25 +00:00
2021-02-16 22:30:26 +03:00
local e2e_trigger ( names ) = {
2019-08-09 03:45:13 +00:00
trigger : {
target : {
2021-02-16 22:30:26 +03:00
include : [ ' e 2 e ' ] + names ,
2019-08-09 03:45:13 +00:00
} ,
} ,
} ;
2021-02-16 22:30:26 +03:00
local e2e_pipelines = [
// regular pipelines, triggered on promote events
Pipeline ( ' e 2 e - a w s ' , default_pipeline_steps + [ capi_docker , e2e_capi , e2e_aws ] ) + e2e_trigger ( [ ' e 2 e - a w s ' ] ) ,
Pipeline ( ' e 2 e - g c p ' , default_pipeline_steps + [ capi_docker , e2e_capi , e2e_gcp ] ) + e2e_trigger ( [ ' e 2 e - g c p ' ] ) ,
// cron pipelines, triggered on schedule events
2021-02-18 19:07:13 +03:00
Pipeline ( ' c r o n - e 2 e - a w s ' , default_pipeline_steps + [ capi_docker , e2e_capi , e2e_aws ] , [ default_cron_pipeline ] ) + cron_trigger ( [ ' t h r i c e - d a i l y ' , ' n i g h t l y ' ] ) ,
Pipeline ( ' c r o n - e 2 e - g c p ' , default_pipeline_steps + [ capi_docker , e2e_capi , e2e_gcp ] , [ default_cron_pipeline ] ) + cron_trigger ( [ ' t h r i c e - d a i l y ' , ' n i g h t l y ' ] ) ,
2021-02-16 22:30:26 +03:00
] ;
2019-08-10 21:29:25 +00:00
// Conformance pipeline.
2021-04-14 16:19:15 +03:00
local conformance_k8s_qemu = Step ( " c o n f o r m a n c e - k 8 s - q e m u " , target = " e 2 e - q e m u " , privileged = true , depends_on = [ load_artifacts ] , environment = {
" Q E M U _ W O R K E R S " : " 2 " , // conformance test requires >=2 workers
" Q E M U _ C P U S " : " 4 " , // conformance test in parallel runs with number of CPUs
" T E S T _ M O D E " : " f a s t - c o n f o r m a n c e " ,
" I M A G E _ R E G I S T R Y " : local_registry ,
} ) ;
2019-08-10 21:29:25 +00:00
2021-02-16 22:30:26 +03:00
local conformance_trigger ( names ) = {
2019-08-09 03:45:13 +00:00
trigger : {
target : {
2021-02-16 22:30:26 +03:00
include : [ ' c o n f o r m a n c e ' ] + names ,
2019-08-09 03:45:13 +00:00
} ,
} ,
} ;
2021-02-16 22:30:26 +03:00
local conformance_pipelines = [
2021-04-14 16:19:15 +03:00
// regular pipelines, triggered on promote events
Pipeline ( ' c o n f o r m a n c e - q e m u ' , default_pipeline_steps + [ conformance_k8s_qemu ] ) + conformance_trigger ( [ ' c o n f o r m a n c e - q e m u ' ] ) ,
// cron pipelines, triggered on schedule events
Pipeline ( ' c r o n - c o n f o r m a n c e - q e m u ' , default_pipeline_steps + [ conformance_k8s_qemu ] ) + cron_trigger ( [ ' n i g h t l y ' ] ) ,
2021-02-16 22:30:26 +03:00
] ;
2019-08-10 21:29:25 +00:00
2020-12-11 23:22:56 +03:00
// Cloud images pipeline.
2021-02-16 22:30:26 +03:00
local cloud_images = Step ( " c l o u d - i m a g e s " , depends_on = [ load_artifacts ] , environment = creds_env_vars ) ;
2020-12-11 23:22:56 +03:00
2021-02-16 22:30:26 +03:00
local upload_images_steps = default_pipeline_steps + [
2020-12-11 23:22:56 +03:00
cloud_images ,
] ;
local upload_images_trigger = {
trigger : {
target : {
include : [ ' u p l o a d - i m a g e s ' ] ,
} ,
} ,
} ;
local upload_images_pipeline = Pipeline ( ' u p l o a d - i m a g e s ' , upload_images_steps ) + upload_images_trigger ;
2019-08-10 21:29:25 +00:00
// Release pipeline.
2020-07-30 21:44:10 +03:00
local boot = Step ( ' b o o t ' , depends_on = [ e2e_docker , e2e_qemu ] ) ;
2019-08-11 16:54:33 +00:00
2020-07-30 21:44:10 +03:00
local release_notes = Step ( ' r e l e a s e - n o t e s ' , depends_on = [ e2e_docker , e2e_qemu ] ) ;
2020-07-28 13:55:47 -07:00
2019-08-10 21:29:25 +00:00
// TODO(andrewrynhard): We should run E2E tests on a release.
2019-08-02 16:08:24 -05:00
local release = {
name : ' r e l e a s e ' ,
image : ' p l u g i n s / g i t h u b - r e l e a s e ' ,
settings : {
api_key : { from_secret : ' g i t h u b _ t o k e n ' } ,
2019-08-10 21:29:25 +00:00
draft : true ,
2020-07-28 13:55:47 -07:00
note : ' _ o u t / R E L E A S E _ N O T E S . m d ' ,
2019-12-30 17:16:37 -08:00
files : [
2020-11-13 17:17:07 +03:00
' _ o u t / a w s - a m d 6 4 . t a r . g z ' ,
' _ o u t / a w s - a r m 6 4 . t a r . g z ' ,
' _ o u t / a z u r e - a m d 6 4 . t a r . g z ' ,
' _ o u t / a z u r e - a r m 6 4 . t a r . g z ' ,
2020-09-26 00:00:43 +03:00
' _ o u t / b o o t - a m d 6 4 . t a r . g z ' ,
' _ o u t / b o o t - a r m 6 4 . t a r . g z ' ,
2020-11-13 17:17:07 +03:00
' _ o u t / d i g i t a l - o c e a n - a m d 6 4 . t a r . g z ' ,
' _ o u t / d i g i t a l - o c e a n - a r m 6 4 . t a r . g z ' ,
' _ o u t / g c p - a m d 6 4 . t a r . g z ' ,
' _ o u t / g c p - a r m 6 4 . t a r . g z ' ,
2020-09-26 00:00:43 +03:00
' _ o u t / i n i t r a m f s - a m d 6 4 . x z ' ,
' _ o u t / i n i t r a m f s - a r m 6 4 . x z ' ,
2020-11-25 18:00:02 +03:00
' _ o u t / m e t a l - a m d 6 4 . t a r . g z ' ,
' _ o u t / m e t a l - a r m 6 4 . t a r . g z ' ,
2020-12-01 18:30:02 -08:00
' _ o u t / m e t a l - r p i _ 4 - a r m 6 4 . i m g . x z ' ,
2021-04-03 22:48:31 +03:00
' _ o u t / m e t a l - r o c k p i _ 4 - a r m 6 4 . i m g . x z ' ,
2020-12-04 11:59:08 -08:00
' _ o u t / m e t a l - r o c k 6 4 - a r m 6 4 . i m g . x z ' ,
2020-12-01 18:30:02 -08:00
' _ o u t / m e t a l - b a n a n a p i _ m 6 4 - a r m 6 4 . i m g . x z ' ,
' _ o u t / m e t a l - l i b r e t e c h _ a l l _ h 3 _ c c _ h 5 - a r m 6 4 . i m g . x z ' ,
2020-11-25 09:15:50 -05:00
' _ o u t / o p e n s t a c k - a m d 6 4 . t a r . g z ' ,
' _ o u t / o p e n s t a c k - a r m 6 4 . t a r . g z ' ,
2020-10-31 14:03:56 -07:00
' _ o u t / t a l o s - a m d 6 4 . i s o ' ,
2020-11-13 17:17:07 +03:00
' _ o u t / t a l o s - a r m 6 4 . i s o ' ,
2020-10-30 01:08:05 +03:00
' _ o u t / t a l o s c t l - c n i - b u n d l e - a m d 6 4 . t a r . g z ' ,
' _ o u t / t a l o s c t l - c n i - b u n d l e - a r m 6 4 . t a r . g z ' ,
2020-03-20 17:38:48 -07:00
' _ o u t / t a l o s c t l - d a r w i n - a m d 6 4 ' ,
2021-04-20 15:58:30 +03:00
' _ o u t / t a l o s c t l - d a r w i n - a r m 6 4 ' ,
2020-03-20 17:38:48 -07:00
' _ o u t / t a l o s c t l - l i n u x - a m d 6 4 ' ,
2020-03-21 14:27:03 -07:00
' _ o u t / t a l o s c t l - l i n u x - a r m 6 4 ' ,
2020-03-21 18:08:09 -07:00
' _ o u t / t a l o s c t l - l i n u x - a r m v 7 ' ,
2020-11-13 17:17:07 +03:00
' _ o u t / v m w a r e - a m d 6 4 . o v a ' ,
' _ o u t / v m w a r e - a r m 6 4 . o v a ' ,
2020-09-26 00:00:43 +03:00
' _ o u t / v m l i n u z - a m d 6 4 ' ,
' _ o u t / v m l i n u z - a r m 6 4 ' ,
2019-12-30 17:16:37 -08:00
] ,
2019-08-02 16:08:24 -05:00
checksum : [ ' s h a 2 5 6 ' , ' s h a 5 1 2 ' ] ,
2019-08-10 21:29:25 +00:00
} ,
when : {
2019-08-02 16:08:24 -05:00
event : [ ' t a g ' ] ,
2019-08-10 21:29:25 +00:00
} ,
2021-02-16 22:30:26 +03:00
depends_on : [ build . name , boot . name , talosctl_cni_bundle . name , images_amd64 . name , images_arm64 . name , sbcs_arm64 . name , iso_amd64 . name , iso_arm64 . name , push . name , release_notes . name ]
2019-08-10 21:29:25 +00:00
} ;
local release_steps = default_steps + [
2020-01-09 12:25:25 -08:00
boot ,
2020-07-28 13:55:47 -07:00
release_notes ,
2019-08-11 16:54:33 +00:00
release ,
2019-08-10 21:29:25 +00:00
] ;
2019-08-09 03:45:13 +00:00
local release_trigger = {
trigger : {
2019-08-11 17:52:20 +00:00
event : [
2019-08-02 16:08:24 -05:00
' t a g ' ,
2019-08-11 17:52:20 +00:00
] ,
2019-08-09 03:45:13 +00:00
} ,
} ;
2019-08-02 16:08:24 -05:00
local release_pipeline = Pipeline ( ' r e l e a s e ' , release_steps ) + release_trigger ;
2019-08-10 21:29:25 +00:00
// Notify pipeline.
local notify = {
2019-08-02 16:08:24 -05:00
name : ' s l a c k ' ,
image : ' p l u g i n s / s l a c k ' ,
2020-07-23 11:21:42 -07:00
settings : {
2019-08-02 16:08:24 -05:00
webhook : { from_secret : ' s l a c k _ w e b h o o k ' } ,
2019-09-09 10:51:10 -05:00
channel : ' p r o j - t a l o s - m a i n t a i n e r s ' ,
2019-10-22 16:07:31 +03:00
link_names : true ,
template : ' { { # i f b u i l d . p u l l } }
* { { #success build.status}}✓ Success{{else}}✕ Fail{{/success}}*: {{ repo.owner }}/{{ repo.name }} - <https://github.com/{{ repo.owner }}/{{ repo.name }}/pull/{{ build.pull }}|Pull Request #{{ build.pull }}>
{ { else } }
* { { #success build.status}}✓ Success{{else}}✕ Fail{{/success}}: {{ repo.owner }}/{{ repo.name }} - Build #{{ build.number }}* (type: `{{ build.event }}`)
{ { / if } }
Commit : < https : //github.com/{{ repo.owner }}/{{ repo.name }}/commit/{{ build.commit }}|{{ truncate build.commit 8 }}>
Branch : < https : //github.com/{{ repo.owner }}/{{ repo.name }}/commits/{{ build.branch }}|{{ build.branch }}>
Author : { { build . a u t h o r } }
< { { build . l i n k } } | V i s i t b u i l d p a g e > '
2019-08-10 21:29:25 +00:00
} ,
2020-07-23 11:21:42 -07:00
when : {
status : [
' s u c c e s s ' ,
' f a i l u r e '
] ,
} ,
2019-08-10 21:29:25 +00:00
} ;
local notify_steps = [ notify ] ;
2019-08-09 03:45:13 +00:00
local notify_trigger = {
trigger : {
2019-08-02 16:08:24 -05:00
status : [ ' s u c c e s s ' , ' f a i l u r e ' ] ,
2019-08-09 03:45:13 +00:00
} ,
} ;
2021-02-16 22:30:26 +03:00
local notify_pipeline = Pipeline ( ' n o t i f y ' , notify_steps , [ default_pipeline , upload_images_pipeline , release_pipeline ] + integration_pipelines + e2e_pipelines + conformance_pipelines , false , true ) + notify_trigger ;
2019-08-10 21:29:25 +00:00
// Final configuration file definition.
2019-08-09 03:45:13 +00:00
[
default_pipeline ,
2021-02-18 20:44:24 +03:00
default_cron_pipeline ,
2020-12-11 23:22:56 +03:00
upload_images_pipeline ,
2019-08-09 03:45:13 +00:00
release_pipeline ,
2021-02-16 22:30:26 +03:00
] + integration_pipelines + e2e_pipelines + conformance_pipelines + [
2019-08-09 03:45:13 +00:00
notify_pipeline ,
]
