BREAKING CHANGE: extract Red Hat security advisories to separate repository (#217)

Co-authored-by: DmitriyLewen <dmitriy.lewen@smartforce.io>

.github/workflows/redhat.yml

@@ -0,0 +1,44 @@
+name: Update vuln-list-redhat repo
+on:
+  schedule:
+  - cron: "0 */6 * * *"
+  workflow_dispatch:
+
+jobs:
+  update:
+    name: Update vuln-list-redhat
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+      VULN_LIST_DIR: "vuln-list-redhat"
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version-file: go.mod
+
+    - name: Check out vuln-list-redhat repo
+      uses: actions/checkout@v3
+      with:
+        repository: ${{ github.repository_owner }}/${{ env.VULN_LIST_DIR }}
+        token: ${{ secrets.ACCESS_TOKEN }}
+        path: ${{ env.VULN_LIST_DIR }}
+
+    - name: Setup github user email and name
+      run: |
+        git config --global user.email "action@github.com"
+        git config --global user.name "GitHub Action"
+
+    - name: Compile vuln-list-update
+      run: go build -o vuln-list-update .
+
+    - if: always()
+      name: Red Hat OVALv2
+      run: ./update.sh redhat-oval "Red Hat OVAL v2"
+
+    - if: always()
+      name: Red Hat Security Data API
+      run: ./update.sh redhat "Red Hat Security Data API"

.github/workflows/squash.yml

@@ -6,7 +6,7 @@ on:
     - cron: '0 0 1 * *'
 
 jobs:
-  build:
+  squash:
     runs-on: ubuntu-latest
     steps:
     - name: Install dependencies

.github/workflows/update.yml

@@ -10,18 +10,22 @@ jobs:
     runs-on: ubuntu-latest
     env:
       GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+      VULN_LIST_DIR: "vuln-list"
     steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
-        go-version: 1.18
+        go-version-file: go.mod
 
-    - name: Check out code into the Go module directory
+    - name: Check out vuln-list repo
       uses: actions/checkout@v3
-
-    - name: Set Owner
-      run: echo "VULNLIST_REPOSITORY_OWNER=$(echo ${GITHUB_REPOSITORY} | awk -F / '{print $1}' | sed -e 's/:refs//')" >> $GITHUB_ENV
-      shell: bash
+      with:
+        repository: ${{ github.repository_owner }}/vuln-list
+        token: ${{ secrets.ACCESS_TOKEN }}
+        path: ${{ env.VULN_LIST_DIR }}
 
     - name: Setup github user email and name
       run: |
@@ -33,134 +37,76 @@ jobs:
 
     - if: always()
       name: NVD
-      run: ./vuln-list-update -target nvd
+      run: ./update.sh nvd "NVD"
 
     - if: always()
       name: Alpine Issue Tracker
-      run: ./vuln-list-update -target alpine
+      run: ./update.sh alpine "Alpine Issue Tracker"
 
     - if: always()
       name: Alpine Unfixed Vulnerability Tracker
-      run: ./vuln-list-update -target alpine-unfixed
+      run: ./update.sh alpine-unfixed "Alpine Secshfixes Tracker"
 
     - if: always()
       name: Debian Security Bug Tracker
-      run: ./vuln-list-update -target debian
+      run: ./update.sh debian "Debian Security Bug Tracker"
 
     - if: always()
       name: Ubuntu CVE Tracker
-      run: ./vuln-list-update -target ubuntu
+      run: ./update.sh ubuntu "Ubuntu CVE Tracker"
 
     - if: always()
       name: Amazon Linux AMI Security Advisory
-      run: ./vuln-list-update -target amazon
+      run: ./update.sh amazon "Amazon Linux Security Center"
 
     - if: always()
       name: Oracle Linux OVAL
-      run: ./vuln-list-update -target oracle-oval
-
-    - if: always()
-      name: Red Hat OVALv2
-      run: ./vuln-list-update -target redhat-oval
+      run: ./update.sh oracle-oval "Oracle Linux OVAL"
 
     - if: always()
       name: Photon CVE Advisory
-      run: ./vuln-list-update -target photon
+      run: ./update.sh photon "Photon Security Advisories"
 
     - if: always()
       name: GitHub Security Advisory
-      run: ./vuln-list-update -target ghsa
+      run: ./update.sh ghsa "GitHub Security Advisory"
 
     - if: always()
       name: CWE
-      run: ./vuln-list-update -target cwe
+      run: ./update.sh cwe "CWE"
 
     - if: always()
       name: SUSE CVRF
-      run: ./vuln-list-update -target suse-cvrf
+      run: ./update.sh suse-cvrf "SUSE CVRF"
 
     - if: always()
       name: GitLab Advisory Database
-      run: ./vuln-list-update -target glad
-
-    - if: always()
-      name: Arch Linux Security Advisory
-      run: ./vuln-list-update -target arch-linux
+      run: ./update.sh glad "GitLab Advisory Database"
 
     - if: always()
       name: AlmaLinux Security Advisory
-      run: ./vuln-list-update -target alma
+      run: ./update.sh alma "AlmaLinux Security Advisory"
 
     - if: always()
       name: Rocky Linux Security Advisory
-      run: ./vuln-list-update -target rocky
+      run: ./update.sh rocky "Rocky Linux Security Advisory"
 
     - if: always()
       name: CBL-Mariner Vulnerability Data
-      run: ./vuln-list-update -target mariner
+      run: ./update.sh mariner "CBL-Mariner Vulnerability Data"
 
     - if: always()
       name: OSV Database
-      run: ./vuln-list-update -target osv
+      run: ./update.sh osv "OSV Database"
 
     - if: always()
       name: Go Vulnerability Database
-      run: ./vuln-list-update -target go-vulndb
+      run: ./update.sh go-vulndb "Go Vulnerability Database"
 
     - if: always()
       name: Wolfi Secdb
-      run: ./vuln-list-update -target wolfi
+      run: ./update.sh wolfi "Wolfi Security Data"
 
     - if: always()
       name: Chainguard Secdb
-      run: ./vuln-list-update -target chainguard
-
-    - if: always()
-      name: Known Exploited Vulnerabilities Catalog
-      run: ./vuln-list-update -target kevc
-
-    # Red Hat Security Data API is unstable.
-    # It should be split into small pieces to reduce the impact of failure.
-    - if: always()
-      name: Red Hat Security Data API 1996-2002
-      run: ./vuln-list-update -target redhat -years 1996,1997,1998,1999,2000,2001,2002
-
-    - if: always()
-      name: Red Hat Security Data API 2003-2008
-      run: ./vuln-list-update -target redhat -years 2003,2004,2005,2006,2007,2008
-
-    - if: always()
-      name: Red Hat Security Data API 2009-2011
-      run: ./vuln-list-update -target redhat -years 2009,2010,2011
-
-    - if: always()
-      name: Red Hat Security Data API 2012
-      run: ./vuln-list-update -target redhat -years 2012
-
-    - if: always()
-      name: Red Hat Security Data API 2013
-      run: ./vuln-list-update -target redhat -years 2013
-
-    - if: always()
-      name: Red Hat Security Data API 2014
-      run: ./vuln-list-update -target redhat -years 2014
-
-    - if: always()
-      name: Red Hat Security Data API 2015
-      run: ./vuln-list-update -target redhat -years 2015
-
-    - if: always()
-      name: Red Hat Security Data API 2016
-      run: ./vuln-list-update -target redhat -years 2016
-
-    - if: always()
-      name: Red Hat Security Data API 2017
-      run: ./vuln-list-update -target redhat -years 2017
-
-    - if: always()
-      name: Red Hat Security Data API 2018
-      run: ./vuln-list-update -target redhat -years 2018
-
-    - if: always()
-      name: Red Hat Security Data API 2019-2023
-      run: ./vuln-list-update -target redhat -years 2019,2020,2021,2022,2023
+      run: ./update.sh chainguard "Chainguard Security Data"

git/git.go

@@ -69,7 +69,13 @@ func (gc Config) CloneOrPull(url, repoPath, branch string, debug bool) (map[stri
 }
 
 func clone(url, repoPath, branch string) error {
-	commandAndArgs := []string{"clone", "--depth", "1", url, repoPath}
+	commandAndArgs := []string{
+		"clone",
+		"--depth",
+		"1",
+		url,
+		repoPath,
+	}
 	if branch != "" {
 		commandAndArgs = append(commandAndArgs, "-b", branch)
 	}
@@ -85,7 +91,12 @@ func clone(url, repoPath, branch string) error {
 func pull(url, repoPath, branch string) ([]string, error) {
 	commandArgs := generateGitArgs(repoPath)
 
-	remoteCmd := []string{"remote", "get-url", "--push", "origin"}
+	remoteCmd := []string{
+		"remote",
+		"get-url",
+		"--push",
+		"origin",
+	}
 	output, err := utils.Exec("git", append(commandArgs, remoteCmd...))
 	if err != nil {
 		return nil, xerrors.Errorf("error in git rev-list: %w", err)
@@ -95,7 +106,12 @@ func pull(url, repoPath, branch string) ([]string, error) {
 		return nil, xerrors.Errorf("remote url is %s, target is %s", remoteURL, url)
 	}
 
-	revParseCmd := []string{"rev-list", "-n", "1", "--all"}
+	revParseCmd := []string{
+		"rev-list",
+		"-n",
+		"1",
+		"--all",
+	}
 	output, err = utils.Exec("git", append(commandArgs, revParseCmd...))
 	if err != nil {
 		return nil, xerrors.Errorf("error in git rev-list: %w", err)
@@ -106,7 +122,10 @@ func pull(url, repoPath, branch string) ([]string, error) {
 		return nil, nil
 	}
 
-	pullCmd := []string{"pull", "origin"}
+	pullCmd := []string{
+		"pull",
+		"origin",
+	}
 	if branch != "" {
 		pullCmd = append(pullCmd, branch)
 	}
@@ -114,12 +133,20 @@ func pull(url, repoPath, branch string) ([]string, error) {
 		return nil, xerrors.Errorf("error in git pull: %w", err)
 	}
 
-	fetchCmd := []string{"fetch", "--prune"}
+	fetchCmd := []string{
+		"fetch",
+		"--prune",
+	}
 	if _, err = utils.Exec("git", append(commandArgs, fetchCmd...)); err != nil {
 		return nil, xerrors.Errorf("error in git fetch: %w", err)
 	}
 
-	diffCmd := []string{"diff", commitHash, "HEAD", "--name-only"}
+	diffCmd := []string{
+		"diff",
+		commitHash,
+		"HEAD",
+		"--name-only",
+	}
 	output, err = utils.Exec("git", append(commandArgs, diffCmd...))
 	if err != nil {
 		return nil, err
@@ -130,88 +157,31 @@ func pull(url, repoPath, branch string) ([]string, error) {
 
 func fetchAll(repoPath string) error {
 	commandArgs := generateGitArgs(repoPath)
-	configCmd := []string{"config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"}
+	configCmd := []string{
+		"config",
+		"remote.origin.fetch",
+		"+refs/heads/*:refs/remotes/origin/*",
+	}
 	if _, err := utils.Exec("git", append(commandArgs, configCmd...)); err != nil {
 		return xerrors.Errorf("error in git config: %w", err)
 	}
 
-	fetchCmd := []string{"fetch", "--all"}
+	fetchCmd := []string{
+		"fetch",
+		"--all",
+	}
 	if _, err := utils.Exec("git", append(commandArgs, fetchCmd...)); err != nil {
 		return xerrors.Errorf("error in git fetch: %w", err)
 	}
 	return nil
 }
 
-func (gc Config) Commit(repoPath, targetPath, message string) error {
-	commandArgs := generateGitArgs(repoPath)
-	addCmd := []string{"add", filepath.Join(repoPath, targetPath)}
-	if _, err := utils.Exec("git", append(commandArgs, addCmd...)); err != nil {
-		return xerrors.Errorf("error in git add: %w", err)
-	}
-
-	commitCmd := []string{"commit", "--message", message}
-	if _, err := utils.Exec("git", append(commandArgs, commitCmd...)); err != nil {
-		return xerrors.Errorf("error in git commit: %w", err)
-	}
-
-	return nil
-}
-
-func (gc Config) Push(repoPath, branch string) error {
-	commandArgs := generateGitArgs(repoPath)
-	pushCmd := []string{"push", "origin", branch}
-	if _, err := utils.Exec("git", append(commandArgs, pushCmd...)); err != nil {
-		return xerrors.Errorf("error in git push: %w", err)
-	}
-	return nil
-}
-
-func (gc Config) Clean(repoPath string) error {
-	commandArgs := generateGitArgs(repoPath)
-	resetCmd := []string{"reset", "--hard", "HEAD"}
-	if _, err := utils.Exec("git", append(commandArgs, resetCmd...)); err != nil {
-		return xerrors.Errorf("git reset error: %w", err)
-	}
-
-	cleanCmd := []string{"clean", "-df"}
-	if _, err := utils.Exec("git", append(commandArgs, cleanCmd...)); err != nil {
-		return xerrors.Errorf("git clean error: %w", err)
-	}
-	return nil
-}
-
-func (gc Config) RemoteBranch(repoPath string) ([]string, error) {
-	commandArgs := generateGitArgs(repoPath)
-	branchCmd := []string{"branch", "--remote"}
-	output, err := utils.Exec("git", append(commandArgs, branchCmd...))
-	if err != nil {
-		return nil, xerrors.Errorf("error in git branch: %w", err)
-	}
-	return strings.Split(output, "\n"), nil
-}
-
-func (gc Config) Checkout(repoPath string, branch string) error {
-	commandArgs := generateGitArgs(repoPath)
-	checkoutCmd := []string{"checkout", branch}
-	_, err := utils.Exec("git", append(commandArgs, checkoutCmd...))
-	if err != nil {
-		return xerrors.Errorf("error in git checkout: %w", err)
-	}
-	return nil
-}
-
-func (gc Config) Status(repoPath string) ([]string, error) {
-	commandArgs := generateGitArgs(repoPath)
-
-	statusCmd := []string{"status", "--porcelain"}
-	output, err := utils.Exec("git", append(commandArgs, statusCmd...))
-	if err != nil {
-		return nil, xerrors.Errorf("error in git status: %w", err)
-	}
-	return strings.Split(strings.TrimSpace(output), "\n"), nil
-}
-
 func generateGitArgs(repoPath string) []string {
 	gitDir := filepath.Join(repoPath, ".git")
-	return []string{"--git-dir", gitDir, "--work-tree", repoPath}
+	return []string{
+		"--git-dir",
+		gitDir,
+		"--work-tree",
+		repoPath,
+	}
 }

main.go

@@ -3,17 +3,10 @@ package main
 import (
 	"context"
 	"flag"
-	"fmt"
 	"log"
 	"os"
-	"strconv"
-	"strings"
 	"time"
 
-	"github.com/aquasecurity/vuln-list-update/chainguard"
-	"github.com/aquasecurity/vuln-list-update/kevc"
-	"github.com/aquasecurity/vuln-list-update/wolfi"
-
 	githubql "github.com/shurcooL/githubv4"
 	"golang.org/x/oauth2"
 	"golang.org/x/xerrors"
@@ -23,12 +16,13 @@ import (
 	alpineunfixed "github.com/aquasecurity/vuln-list-update/alpine-unfixed"
 	"github.com/aquasecurity/vuln-list-update/amazon"
 	arch_linux "github.com/aquasecurity/vuln-list-update/arch"
+	"github.com/aquasecurity/vuln-list-update/chainguard"
 	"github.com/aquasecurity/vuln-list-update/cwe"
 	"github.com/aquasecurity/vuln-list-update/debian/tracker"
 	"github.com/aquasecurity/vuln-list-update/ghsa"
-	"github.com/aquasecurity/vuln-list-update/git"
 	"github.com/aquasecurity/vuln-list-update/glad"
 	govulndb "github.com/aquasecurity/vuln-list-update/go-vulndb"
+	"github.com/aquasecurity/vuln-list-update/kevc"
 	"github.com/aquasecurity/vuln-list-update/mariner"
 	"github.com/aquasecurity/vuln-list-update/nvd"
 	oracleoval "github.com/aquasecurity/vuln-list-update/oracle/oval"
@@ -40,18 +34,13 @@ import (
 	susecvrf "github.com/aquasecurity/vuln-list-update/suse/cvrf"
 	"github.com/aquasecurity/vuln-list-update/ubuntu"
 	"github.com/aquasecurity/vuln-list-update/utils"
-)
-
-const (
-	repoURL          = "https://%s@github.com/%s/%s.git"
-	defaultRepoOwner = "aquasecurity"
-	defaultRepoName  = "vuln-list"
+	"github.com/aquasecurity/vuln-list-update/wolfi"
 )
 
 var (
 	target = flag.String("target", "", "update target (nvd, alpine, alpine-unfixed, redhat, redhat-oval, "+
 		"debian, debian-oval, ubuntu, amazon, oracle-oval, suse-cvrf, photon, arch-linux, ghsa, glad, cwe, osv, go-vulndb, mariner, kevc, wolfi, chainguard)")
-	years        = flag.String("years", "", "update years (only redhat)")
+	vulnListDir  = flag.String("vuln-list-dir", "", "vuln-list dir")
 	targetUri    = flag.String("target-uri", "", "alternative repository URI (only glad)")
 	targetBranch = flag.String("target-branch", "", "alternative repository branch (only glad)")
 )
@@ -65,110 +54,67 @@ func main() {
 func run() error {
 	flag.Parse()
 	now := time.Now().UTC()
-	gc := &git.Config{}
-	debug := os.Getenv("VULN_LIST_DEBUG") != ""
 
-	repoOwner := utils.LookupEnv("VULNLIST_REPOSITORY_OWNER", defaultRepoOwner)
-	repoName := utils.LookupEnv("VULNLIST_REPOSITORY_NAME", defaultRepoName)
-
-	// Embed GitHub token to URL
-	githubToken := os.Getenv("GITHUB_TOKEN")
-	url := fmt.Sprintf(repoURL, githubToken, repoOwner, repoName)
-
-	log.Printf("target repository is %s/%s\n", repoOwner, repoName)
-	log.Printf("cloning/pulling into %s", utils.VulnListDir())
-
-	if _, err := gc.CloneOrPull(url, utils.VulnListDir(), "main", debug); err != nil {
-		return xerrors.Errorf("clone or pull error: %w", err)
+	if *vulnListDir != "" {
+		utils.SetVulnListDir(*vulnListDir)
 	}
 
-	defer func() {
-		if debug {
-			return
-		}
-		log.Println("git reset & clean")
-		_ = gc.Clean(utils.VulnListDir())
-	}()
-
-	var commitMsg string
 	switch *target {
 	case "nvd":
 		if err := nvd.Update(now.Year()); err != nil {
 			return xerrors.Errorf("NVD update error: %w", err)
 		}
-		commitMsg = "NVD"
 	case "redhat":
-		var yearList []int
-		for _, y := range strings.Split(*years, ",") {
-			yearInt, err := strconv.Atoi(y)
-			if err != nil {
-				return xerrors.Errorf("invalid years: %w", err)
-			}
-			yearList = append(yearList, yearInt)
-		}
-		if len(yearList) == 0 {
-			return xerrors.New("years must be specified")
-		}
-		if err := securitydataapi.Update(yearList); err != nil {
+		if err := securitydataapi.Update(); err != nil {
 			return xerrors.Errorf("Red Hat Security Data API update error: %w", err)
 		}
-		commitMsg = "RedHat " + *years
 	case "redhat-oval":
 		rc := redhatoval.NewConfig()
 		if err := rc.Update(); err != nil {
 			return xerrors.Errorf("Red Hat OVALv2 update error: %w", err)
 		}
-		commitMsg = "Red Hat OVAL v2"
 	case "debian":
 		dc := tracker.NewClient()
 		if err := dc.Update(); err != nil {
 			return xerrors.Errorf("Debian update error: %w", err)
 		}
-		commitMsg = "Debian Security Bug Tracker"
 	case "ubuntu":
 		if err := ubuntu.Update(); err != nil {
 			return xerrors.Errorf("Ubuntu update error: %w", err)
 		}
-		commitMsg = "Ubuntu CVE Tracker"
 	case "alpine":
 		au := alpine.NewUpdater()
 		if err := au.Update(); err != nil {
 			return xerrors.Errorf("Alpine update error: %w", err)
 		}
-		commitMsg = "Alpine Issue Tracker"
 	case "alpine-unfixed":
 		au := alpineunfixed.NewUpdater()
 		if err := au.Update(); err != nil {
 			return xerrors.Errorf("Alpine Secfixes Tracker update error: %w", err)
 		}
-		commitMsg = "Alpine Secfixes Tracker"
 	case "amazon":
 		ac := amazon.NewConfig()
 		if err := ac.Update(); err != nil {
 			return xerrors.Errorf("Amazon Linux update error: %w", err)
 		}
-		commitMsg = "Amazon Linux Security Center"
 	case "oracle-oval":
 		oc := oracleoval.NewConfig()
 		if err := oc.Update(); err != nil {
 			return xerrors.Errorf("Oracle OVAL update error: %w", err)
 		}
-		commitMsg = "Oracle Linux OVAL"
 	case "suse-cvrf":
 		sc := susecvrf.NewConfig()
 		if err := sc.Update(); err != nil {
 			return xerrors.Errorf("SUSE CVRF update error: %w", err)
 		}
-		commitMsg = "SUSE CVRF"
 	case "photon":
 		pc := photon.NewConfig()
 		if err := pc.Update(); err != nil {
 			return xerrors.Errorf("Photon update error: %w", err)
 		}
-		commitMsg = "Photon Security Advisories"
 	case "ghsa":
 		src := oauth2.StaticTokenSource(
-			&oauth2.Token{AccessToken: githubToken},
+			&oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")},
 		)
 		httpClient := oauth2.NewClient(context.Background(), src)
 
@@ -176,106 +122,64 @@ func run() error {
 		if err := gc.Update(); err != nil {
 			return xerrors.Errorf("GitHub Security Advisory update error: %w", err)
 		}
-		commitMsg = "GitHub Security Advisory"
 	case "glad":
 		gu := glad.NewUpdater(*targetUri, *targetBranch)
 		if err := gu.Update(); err != nil {
 			return xerrors.Errorf("GitLab Advisory Database update error: %w", err)
 		}
-		commitMsg = "GitLab Advisory Database"
 	case "cwe":
 		c := cwe.NewCWEConfig()
 		if err := c.Update(); err != nil {
 			return xerrors.Errorf("CWE update error: %w", err)
 		}
-		commitMsg = "CWE Advisories"
 	case "arch-linux":
 		al := arch_linux.NewArchLinux()
 		if err := al.Update(); err != nil {
 			return xerrors.Errorf("Arch Linux update error: %w", err)
 		}
-		commitMsg = "Arch Linux Security Tracker"
 	case "alma":
 		ac := alma.NewConfig()
 		if err := ac.Update(); err != nil {
 			return xerrors.Errorf("AlmaLinux update error: %w", err)
 		}
-		commitMsg = "AlmaLinux Security Advisory"
 	case "rocky":
 		rc := rocky.NewConfig()
 		if err := rc.Update(); err != nil {
 			return xerrors.Errorf("Rocky Linux update error: %w", err)
 		}
-		commitMsg = "Rocky Linux Security Advisory"
 	case "osv":
 		p := osv.NewOsv()
 		if err := p.Update(); err != nil {
 			return xerrors.Errorf("OSV update error: %w", err)
 		}
-		commitMsg = "OSV Database"
 	case "go-vulndb":
 		src := govulndb.NewVulnDB()
 		if err := src.Update(); err != nil {
 			return xerrors.Errorf("Go Vulnerability Database update error: %w", err)
 		}
-		commitMsg = "Go Vulnerability Database"
 	case "mariner":
 		src := mariner.NewConfig()
 		if err := src.Update(); err != nil {
 			return xerrors.Errorf("CBL-Mariner Vulnerability Data update error: %w", err)
 		}
-		commitMsg = "CBL-Mariner Vulnerability Data"
 	case "kevc":
 		src := kevc.NewConfig()
 		if err := src.Update(); err != nil {
 			return xerrors.Errorf("Known Exploited Vulnerability Catalog update error: %w", err)
 		}
-		commitMsg = "Known Exploited Vulnerability Catalog"
 	case "wolfi":
 		wu := wolfi.NewUpdater()
 		if err := wu.Update(); err != nil {
 			return xerrors.Errorf("Wolfi update error: %w", err)
 		}
-		commitMsg = "Wolfi Security Data"
 	case "chainguard":
 		cu := chainguard.NewUpdater()
 		if err := cu.Update(); err != nil {
 			return xerrors.Errorf("Chainguard update error: %w", err)
 		}
-		commitMsg = "Chainguard Security Data"
 	default:
 		return xerrors.New("unknown target")
 	}
 
-	if debug {
-		return nil
-	}
-
-	if err := utils.SetLastUpdatedDate(*target, now); err != nil {
-		return err
-	}
-
-	log.Println("git status")
-	files, err := gc.Status(utils.VulnListDir())
-	if err != nil {
-		return xerrors.Errorf("git status error: %w", err)
-	}
-
-	// only last_updated.json
-	if len(files) < 2 {
-		log.Println("Skip commit and push")
-		return nil
-	}
-
-	log.Println("git commit")
-	if err = gc.Commit(utils.VulnListDir(), "./", commitMsg); err != nil {
-		return xerrors.Errorf("git commit error: %w", err)
-	}
-
-	log.Println("git push")
-	if err = gc.Push(utils.VulnListDir(), "main"); err != nil {
-		return xerrors.Errorf("git push error: %w", err)
-	}
-
 	return nil
 }

redhat/oval/redhat.go

@@ -22,9 +22,8 @@ import (
 )
 
 const (
-	ovalDir   = "oval"
-	redhatDir = "redhat"
-	cpeDir    = "redhat-cpe"
+	ovalDir = "oval"
+	cpeDir  = "cpe"
 
 	urlFormat    = "https://www.redhat.com/security/data/oval/v2/%s"
 	retry        = 5
@@ -67,7 +66,7 @@ func (c Config) Update() error {
 		return xerrors.Errorf("unable to update repository-to-cpe.json: %w", err)
 	}
 
-	dirPath := filepath.Join(c.VulnListDir, ovalDir, redhatDir)
+	dirPath := filepath.Join(c.VulnListDir, ovalDir)
 	log.Printf("Remove Red Hat OVAL v2 directory %s", dirPath)
 	if err := os.RemoveAll(dirPath); err != nil {
 		return xerrors.Errorf("failed to remove Red Hat OVAL v2 directory: %w", err)
@@ -138,7 +137,7 @@ func (c Config) updateOVAL(ovalFile string) error {
 
 	// e.g. storage-gluster-3-including-unpatched
 	platform := strings.TrimSuffix(file, ".oval.xml.bz2")
-	dirPath := filepath.Join(c.VulnListDir, ovalDir, redhatDir, release, platform)
+	dirPath := filepath.Join(c.VulnListDir, ovalDir, release, platform)
 
 	// write tests/tests.json file
 	if err := utils.WriteJSON(c.AppFs, filepath.Join(dirPath, testsDir), "tests.json", ovalroot.Tests); err != nil {

redhat/oval/testdata/golden/redhat-cpe/repository-to-cpe.json

@@ -1,12 +0,0 @@
-{
-  "3scale-amp-2-for-rhel-8-ppc64le-debug-rpms": [
-    "cpe:/a:redhat:3scale_amp:2.11::el8",
-    "cpe:/a:redhat:3scale_amp:2.12::el8",
-    "cpe:/a:redhat:3scale_amp:2.8::el8"
-  ],
-  "3scale-amp-2-for-rhel-8-ppc64le-rpms": [
-    "cpe:/a:redhat:3scale_amp:2.11::el8",
-    "cpe:/a:redhat:3scale_amp:2.12::el8",
-    "cpe:/a:redhat:3scale_amp:2.8::el8"
-  ]
-}

redhat/securitydataapi/redhat.go

@@ -13,17 +13,18 @@ import (
 )
 
 const (
-	listURL   = "https://access.redhat.com/labs/securitydataapi/cve.json?page=%d&after=%s&per_page=500"
-	cveURL    = "https://access.redhat.com/labs/securitydataapi/cve/%s.json"
-	redhatDir = "redhat"
+	listURL = "https://access.redhat.com/labs/securitydataapi/cve.json?page=%d&after=%s&per_page=500"
+	cveURL  = "https://access.redhat.com/labs/securitydataapi/cve/%s.json"
+	apiDir  = "api"
 
 	concurrency = 10
 	wait        = 1
 	retry       = 20 // Red Hat Security Data API is unstable
 )
 
-func Update(years []int) error {
-	for _, year := range years {
+func Update() error {
+	now := time.Now()
+	for year := 1996; year <= now.Year(); year++ {
 		if err := update(year); err != nil {
 			return xerrors.Errorf("error in RedHat update: %w", err)
 		}
@@ -52,7 +53,7 @@ func update(year int) error {
 	}
 
 	for cveID, cve := range cves {
-		if err = utils.SaveCVEPerYear(filepath.Join(utils.VulnListDir(), redhatDir), cveID, cve); err != nil {
+		if err = utils.SaveCVEPerYear(filepath.Join(utils.VulnListDir(), apiDir), cveID, cve); err != nil {
 			return xerrors.Errorf("failed to save RedHat CVE detail: %w", err)
 		}
 	}

update.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+TARGET=$1
+COMMIT_MSG=$2
+
+if [ -z "$TARGET" ]; then
+  echo "target required"
+  exit 1
+fi
+
+if [ -z "$COMMIT_MSG" ]; then
+  echo "commit message required"
+  exit 1
+fi
+
+./vuln-list-update -vuln-list-dir "$VULN_LIST_DIR" -target "$TARGET"
+
+cd "$VULN_LIST_DIR" || exit 1
+
+if [[ -n $(git status --porcelain) ]]; then
+  git add .
+  git commit -m "${COMMIT_MSG}"
+  git push
+fi

utils/utils.go

@@ -19,6 +19,8 @@ import (
 	pb "gopkg.in/cheggaaa/pb.v1"
 )
 
+var vulnListDir = filepath.Join(CacheDir(), "vuln-list")
+
 func CacheDir() string {
 	cacheDir, err := os.UserCacheDir()
 	if err != nil {
@@ -28,8 +30,12 @@ func CacheDir() string {
 	return dir
 }
 
+func SetVulnListDir(dir string) {
+	vulnListDir = dir
+}
+
 func VulnListDir() string {
-	return filepath.Join(CacheDir(), "vuln-list")
+	return vulnListDir
 }
 
 func SaveCVEPerYear(dirPath string, cveID string, data interface{}) error {