From ecaf1143a96b293bdd8d5164d33197a63d11b244 Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Thu, 22 Jun 2023 10:20:19 +0300 Subject: [PATCH] BREAKING CHANGE: extract Red Hat security advisories to separate repository (#217) Co-authored-by: DmitriyLewen --- .github/workflows/redhat.yml | 44 ++++++ .github/workflows/squash.yml | 2 +- .github/workflows/update.yml | 112 ++++------------ git/git.go | 126 +++++++----------- main.go | 112 ++-------------- redhat/oval/redhat.go | 9 +- .../definitions/2014/CVE-2014-3209.json | 0 .../definitions/2014/RHBA-2014-1396.json | 0 .../definitions/2016/CVE-2016-5361.json | 0 .../2016/CVE-2016-5391.unaffected.json | 0 .../definitions/2018/CVE-2018-5389.json | 0 .../definitions/2020/CVE-2020-28935.json | 0 .../objects/objects.json | 0 .../states/states.json | 0 .../tests/tests.json | 0 .../2020/CVE-2020-0605.unaffected.json | 0 .../2020/CVE-2020-0606.unaffected.json | 0 .../definitions/2020/RHSA-2020-0134.json | 0 .../definitions/2020/RHSA-2020-2249.json | 0 .../objects/objects.json | 0 .../states/states.json | 0 .../tests/tests.json | 0 .../definitions/2019/RHSA-2019-3927.json | 0 .../definitions/2020/CVE-2020-10744.json | 0 .../definitions/2020/CVE-2020-1734.json | 0 .../definitions/2020/CVE-2020-1738.json | 0 .../definitions/2020/RHSA-2020-0215.json | 0 .../objects/objects.json | 0 .../states/states.json | 0 .../tests/tests.json | 0 .../golden/redhat-cpe/repository-to-cpe.json | 12 -- redhat/securitydataapi/redhat.go | 13 +- update.sh | 24 ++++ utils/utils.go | 8 +- 34 files changed, 172 insertions(+), 290 deletions(-) create mode 100644 .github/workflows/redhat.yml rename redhat/oval/testdata/golden/{redhat => }/6/rhel-6-extras-including-unpatched/definitions/2014/CVE-2014-3209.json (100%) rename redhat/oval/testdata/golden/{redhat => }/6/rhel-6-extras-including-unpatched/definitions/2014/RHBA-2014-1396.json (100%) rename redhat/oval/testdata/golden/{redhat => }/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5361.json (100%) rename redhat/oval/testdata/golden/{redhat => }/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5391.unaffected.json (100%) rename redhat/oval/testdata/golden/{redhat => }/6/rhel-6-extras-including-unpatched/definitions/2018/CVE-2018-5389.json (100%) rename redhat/oval/testdata/golden/{redhat => }/6/rhel-6-extras-including-unpatched/definitions/2020/CVE-2020-28935.json (100%) rename redhat/oval/testdata/golden/{redhat => }/6/rhel-6-extras-including-unpatched/objects/objects.json (100%) rename redhat/oval/testdata/golden/{redhat => }/6/rhel-6-extras-including-unpatched/states/states.json (100%) rename redhat/oval/testdata/golden/{redhat => }/6/rhel-6-extras-including-unpatched/tests/tests.json (100%) rename redhat/oval/testdata/golden/{redhat => }/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0605.unaffected.json (100%) rename redhat/oval/testdata/golden/{redhat => }/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0606.unaffected.json (100%) rename redhat/oval/testdata/golden/{redhat => }/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020-0134.json (100%) rename redhat/oval/testdata/golden/{redhat => }/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020-2249.json (100%) rename redhat/oval/testdata/golden/{redhat => }/7/dotnet-3.1-including-unpatched/objects/objects.json (100%) rename redhat/oval/testdata/golden/{redhat => }/7/dotnet-3.1-including-unpatched/states/states.json (100%) rename redhat/oval/testdata/golden/{redhat => }/7/dotnet-3.1-including-unpatched/tests/tests.json (100%) rename redhat/oval/testdata/golden/{redhat => }/8/ansible-2-including-unpatched/definitions/2019/RHSA-2019-3927.json (100%) rename redhat/oval/testdata/golden/{redhat => }/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-10744.json (100%) rename redhat/oval/testdata/golden/{redhat => }/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1734.json (100%) rename redhat/oval/testdata/golden/{redhat => }/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1738.json (100%) rename redhat/oval/testdata/golden/{redhat => }/8/ansible-2-including-unpatched/definitions/2020/RHSA-2020-0215.json (100%) rename redhat/oval/testdata/golden/{redhat => }/8/ansible-2-including-unpatched/objects/objects.json (100%) rename redhat/oval/testdata/golden/{redhat => }/8/ansible-2-including-unpatched/states/states.json (100%) rename redhat/oval/testdata/golden/{redhat => }/8/ansible-2-including-unpatched/tests/tests.json (100%) delete mode 100644 redhat/oval/testdata/golden/redhat-cpe/repository-to-cpe.json create mode 100755 update.sh diff --git a/.github/workflows/redhat.yml b/.github/workflows/redhat.yml new file mode 100644 index 0000000..c725284 --- /dev/null +++ b/.github/workflows/redhat.yml @@ -0,0 +1,44 @@ +name: Update vuln-list-redhat repo +on: + schedule: + - cron: "0 */6 * * *" + workflow_dispatch: + +jobs: + update: + name: Update vuln-list-redhat + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }} + VULN_LIST_DIR: "vuln-list-redhat" + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: go.mod + + - name: Check out vuln-list-redhat repo + uses: actions/checkout@v3 + with: + repository: ${{ github.repository_owner }}/${{ env.VULN_LIST_DIR }} + token: ${{ secrets.ACCESS_TOKEN }} + path: ${{ env.VULN_LIST_DIR }} + + - name: Setup github user email and name + run: | + git config --global user.email "action@github.com" + git config --global user.name "GitHub Action" + + - name: Compile vuln-list-update + run: go build -o vuln-list-update . + + - if: always() + name: Red Hat OVALv2 + run: ./update.sh redhat-oval "Red Hat OVAL v2" + + - if: always() + name: Red Hat Security Data API + run: ./update.sh redhat "Red Hat Security Data API" \ No newline at end of file diff --git a/.github/workflows/squash.yml b/.github/workflows/squash.yml index 21aaa14..c604a08 100644 --- a/.github/workflows/squash.yml +++ b/.github/workflows/squash.yml @@ -6,7 +6,7 @@ on: - cron: '0 0 1 * *' jobs: - build: + squash: runs-on: ubuntu-latest steps: - name: Install dependencies diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 4f4fbe7..2c95e61 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -10,18 +10,22 @@ jobs: runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }} + VULN_LIST_DIR: "vuln-list" steps: + - name: Check out code + uses: actions/checkout@v3 + - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.18 + go-version-file: go.mod - - name: Check out code into the Go module directory + - name: Check out vuln-list repo uses: actions/checkout@v3 - - - name: Set Owner - run: echo "VULNLIST_REPOSITORY_OWNER=$(echo ${GITHUB_REPOSITORY} | awk -F / '{print $1}' | sed -e 's/:refs//')" >> $GITHUB_ENV - shell: bash + with: + repository: ${{ github.repository_owner }}/vuln-list + token: ${{ secrets.ACCESS_TOKEN }} + path: ${{ env.VULN_LIST_DIR }} - name: Setup github user email and name run: | @@ -33,134 +37,76 @@ jobs: - if: always() name: NVD - run: ./vuln-list-update -target nvd + run: ./update.sh nvd "NVD" - if: always() name: Alpine Issue Tracker - run: ./vuln-list-update -target alpine + run: ./update.sh alpine "Alpine Issue Tracker" - if: always() name: Alpine Unfixed Vulnerability Tracker - run: ./vuln-list-update -target alpine-unfixed + run: ./update.sh alpine-unfixed "Alpine Secshfixes Tracker" - if: always() name: Debian Security Bug Tracker - run: ./vuln-list-update -target debian + run: ./update.sh debian "Debian Security Bug Tracker" - if: always() name: Ubuntu CVE Tracker - run: ./vuln-list-update -target ubuntu + run: ./update.sh ubuntu "Ubuntu CVE Tracker" - if: always() name: Amazon Linux AMI Security Advisory - run: ./vuln-list-update -target amazon + run: ./update.sh amazon "Amazon Linux Security Center" - if: always() name: Oracle Linux OVAL - run: ./vuln-list-update -target oracle-oval - - - if: always() - name: Red Hat OVALv2 - run: ./vuln-list-update -target redhat-oval + run: ./update.sh oracle-oval "Oracle Linux OVAL" - if: always() name: Photon CVE Advisory - run: ./vuln-list-update -target photon + run: ./update.sh photon "Photon Security Advisories" - if: always() name: GitHub Security Advisory - run: ./vuln-list-update -target ghsa + run: ./update.sh ghsa "GitHub Security Advisory" - if: always() name: CWE - run: ./vuln-list-update -target cwe + run: ./update.sh cwe "CWE" - if: always() name: SUSE CVRF - run: ./vuln-list-update -target suse-cvrf + run: ./update.sh suse-cvrf "SUSE CVRF" - if: always() name: GitLab Advisory Database - run: ./vuln-list-update -target glad - - - if: always() - name: Arch Linux Security Advisory - run: ./vuln-list-update -target arch-linux + run: ./update.sh glad "GitLab Advisory Database" - if: always() name: AlmaLinux Security Advisory - run: ./vuln-list-update -target alma + run: ./update.sh alma "AlmaLinux Security Advisory" - if: always() name: Rocky Linux Security Advisory - run: ./vuln-list-update -target rocky + run: ./update.sh rocky "Rocky Linux Security Advisory" - if: always() name: CBL-Mariner Vulnerability Data - run: ./vuln-list-update -target mariner + run: ./update.sh mariner "CBL-Mariner Vulnerability Data" - if: always() name: OSV Database - run: ./vuln-list-update -target osv + run: ./update.sh osv "OSV Database" - if: always() name: Go Vulnerability Database - run: ./vuln-list-update -target go-vulndb + run: ./update.sh go-vulndb "Go Vulnerability Database" - if: always() name: Wolfi Secdb - run: ./vuln-list-update -target wolfi + run: ./update.sh wolfi "Wolfi Security Data" - if: always() name: Chainguard Secdb - run: ./vuln-list-update -target chainguard - - - if: always() - name: Known Exploited Vulnerabilities Catalog - run: ./vuln-list-update -target kevc - - # Red Hat Security Data API is unstable. - # It should be split into small pieces to reduce the impact of failure. - - if: always() - name: Red Hat Security Data API 1996-2002 - run: ./vuln-list-update -target redhat -years 1996,1997,1998,1999,2000,2001,2002 - - - if: always() - name: Red Hat Security Data API 2003-2008 - run: ./vuln-list-update -target redhat -years 2003,2004,2005,2006,2007,2008 - - - if: always() - name: Red Hat Security Data API 2009-2011 - run: ./vuln-list-update -target redhat -years 2009,2010,2011 - - - if: always() - name: Red Hat Security Data API 2012 - run: ./vuln-list-update -target redhat -years 2012 - - - if: always() - name: Red Hat Security Data API 2013 - run: ./vuln-list-update -target redhat -years 2013 - - - if: always() - name: Red Hat Security Data API 2014 - run: ./vuln-list-update -target redhat -years 2014 - - - if: always() - name: Red Hat Security Data API 2015 - run: ./vuln-list-update -target redhat -years 2015 - - - if: always() - name: Red Hat Security Data API 2016 - run: ./vuln-list-update -target redhat -years 2016 - - - if: always() - name: Red Hat Security Data API 2017 - run: ./vuln-list-update -target redhat -years 2017 - - - if: always() - name: Red Hat Security Data API 2018 - run: ./vuln-list-update -target redhat -years 2018 - - - if: always() - name: Red Hat Security Data API 2019-2023 - run: ./vuln-list-update -target redhat -years 2019,2020,2021,2022,2023 + run: ./update.sh chainguard "Chainguard Security Data" \ No newline at end of file diff --git a/git/git.go b/git/git.go index c74f9d1..f89a3c5 100644 --- a/git/git.go +++ b/git/git.go @@ -69,7 +69,13 @@ func (gc Config) CloneOrPull(url, repoPath, branch string, debug bool) (map[stri } func clone(url, repoPath, branch string) error { - commandAndArgs := []string{"clone", "--depth", "1", url, repoPath} + commandAndArgs := []string{ + "clone", + "--depth", + "1", + url, + repoPath, + } if branch != "" { commandAndArgs = append(commandAndArgs, "-b", branch) } @@ -85,7 +91,12 @@ func clone(url, repoPath, branch string) error { func pull(url, repoPath, branch string) ([]string, error) { commandArgs := generateGitArgs(repoPath) - remoteCmd := []string{"remote", "get-url", "--push", "origin"} + remoteCmd := []string{ + "remote", + "get-url", + "--push", + "origin", + } output, err := utils.Exec("git", append(commandArgs, remoteCmd...)) if err != nil { return nil, xerrors.Errorf("error in git rev-list: %w", err) @@ -95,7 +106,12 @@ func pull(url, repoPath, branch string) ([]string, error) { return nil, xerrors.Errorf("remote url is %s, target is %s", remoteURL, url) } - revParseCmd := []string{"rev-list", "-n", "1", "--all"} + revParseCmd := []string{ + "rev-list", + "-n", + "1", + "--all", + } output, err = utils.Exec("git", append(commandArgs, revParseCmd...)) if err != nil { return nil, xerrors.Errorf("error in git rev-list: %w", err) @@ -106,7 +122,10 @@ func pull(url, repoPath, branch string) ([]string, error) { return nil, nil } - pullCmd := []string{"pull", "origin"} + pullCmd := []string{ + "pull", + "origin", + } if branch != "" { pullCmd = append(pullCmd, branch) } @@ -114,12 +133,20 @@ func pull(url, repoPath, branch string) ([]string, error) { return nil, xerrors.Errorf("error in git pull: %w", err) } - fetchCmd := []string{"fetch", "--prune"} + fetchCmd := []string{ + "fetch", + "--prune", + } if _, err = utils.Exec("git", append(commandArgs, fetchCmd...)); err != nil { return nil, xerrors.Errorf("error in git fetch: %w", err) } - diffCmd := []string{"diff", commitHash, "HEAD", "--name-only"} + diffCmd := []string{ + "diff", + commitHash, + "HEAD", + "--name-only", + } output, err = utils.Exec("git", append(commandArgs, diffCmd...)) if err != nil { return nil, err @@ -130,88 +157,31 @@ func pull(url, repoPath, branch string) ([]string, error) { func fetchAll(repoPath string) error { commandArgs := generateGitArgs(repoPath) - configCmd := []string{"config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"} + configCmd := []string{ + "config", + "remote.origin.fetch", + "+refs/heads/*:refs/remotes/origin/*", + } if _, err := utils.Exec("git", append(commandArgs, configCmd...)); err != nil { return xerrors.Errorf("error in git config: %w", err) } - fetchCmd := []string{"fetch", "--all"} + fetchCmd := []string{ + "fetch", + "--all", + } if _, err := utils.Exec("git", append(commandArgs, fetchCmd...)); err != nil { return xerrors.Errorf("error in git fetch: %w", err) } return nil } -func (gc Config) Commit(repoPath, targetPath, message string) error { - commandArgs := generateGitArgs(repoPath) - addCmd := []string{"add", filepath.Join(repoPath, targetPath)} - if _, err := utils.Exec("git", append(commandArgs, addCmd...)); err != nil { - return xerrors.Errorf("error in git add: %w", err) - } - - commitCmd := []string{"commit", "--message", message} - if _, err := utils.Exec("git", append(commandArgs, commitCmd...)); err != nil { - return xerrors.Errorf("error in git commit: %w", err) - } - - return nil -} - -func (gc Config) Push(repoPath, branch string) error { - commandArgs := generateGitArgs(repoPath) - pushCmd := []string{"push", "origin", branch} - if _, err := utils.Exec("git", append(commandArgs, pushCmd...)); err != nil { - return xerrors.Errorf("error in git push: %w", err) - } - return nil -} - -func (gc Config) Clean(repoPath string) error { - commandArgs := generateGitArgs(repoPath) - resetCmd := []string{"reset", "--hard", "HEAD"} - if _, err := utils.Exec("git", append(commandArgs, resetCmd...)); err != nil { - return xerrors.Errorf("git reset error: %w", err) - } - - cleanCmd := []string{"clean", "-df"} - if _, err := utils.Exec("git", append(commandArgs, cleanCmd...)); err != nil { - return xerrors.Errorf("git clean error: %w", err) - } - return nil -} - -func (gc Config) RemoteBranch(repoPath string) ([]string, error) { - commandArgs := generateGitArgs(repoPath) - branchCmd := []string{"branch", "--remote"} - output, err := utils.Exec("git", append(commandArgs, branchCmd...)) - if err != nil { - return nil, xerrors.Errorf("error in git branch: %w", err) - } - return strings.Split(output, "\n"), nil -} - -func (gc Config) Checkout(repoPath string, branch string) error { - commandArgs := generateGitArgs(repoPath) - checkoutCmd := []string{"checkout", branch} - _, err := utils.Exec("git", append(commandArgs, checkoutCmd...)) - if err != nil { - return xerrors.Errorf("error in git checkout: %w", err) - } - return nil -} - -func (gc Config) Status(repoPath string) ([]string, error) { - commandArgs := generateGitArgs(repoPath) - - statusCmd := []string{"status", "--porcelain"} - output, err := utils.Exec("git", append(commandArgs, statusCmd...)) - if err != nil { - return nil, xerrors.Errorf("error in git status: %w", err) - } - return strings.Split(strings.TrimSpace(output), "\n"), nil -} - func generateGitArgs(repoPath string) []string { gitDir := filepath.Join(repoPath, ".git") - return []string{"--git-dir", gitDir, "--work-tree", repoPath} + return []string{ + "--git-dir", + gitDir, + "--work-tree", + repoPath, + } } diff --git a/main.go b/main.go index eaba1cb..51b3637 100644 --- a/main.go +++ b/main.go @@ -3,17 +3,10 @@ package main import ( "context" "flag" - "fmt" "log" "os" - "strconv" - "strings" "time" - "github.com/aquasecurity/vuln-list-update/chainguard" - "github.com/aquasecurity/vuln-list-update/kevc" - "github.com/aquasecurity/vuln-list-update/wolfi" - githubql "github.com/shurcooL/githubv4" "golang.org/x/oauth2" "golang.org/x/xerrors" @@ -23,12 +16,13 @@ import ( alpineunfixed "github.com/aquasecurity/vuln-list-update/alpine-unfixed" "github.com/aquasecurity/vuln-list-update/amazon" arch_linux "github.com/aquasecurity/vuln-list-update/arch" + "github.com/aquasecurity/vuln-list-update/chainguard" "github.com/aquasecurity/vuln-list-update/cwe" "github.com/aquasecurity/vuln-list-update/debian/tracker" "github.com/aquasecurity/vuln-list-update/ghsa" - "github.com/aquasecurity/vuln-list-update/git" "github.com/aquasecurity/vuln-list-update/glad" govulndb "github.com/aquasecurity/vuln-list-update/go-vulndb" + "github.com/aquasecurity/vuln-list-update/kevc" "github.com/aquasecurity/vuln-list-update/mariner" "github.com/aquasecurity/vuln-list-update/nvd" oracleoval "github.com/aquasecurity/vuln-list-update/oracle/oval" @@ -40,18 +34,13 @@ import ( susecvrf "github.com/aquasecurity/vuln-list-update/suse/cvrf" "github.com/aquasecurity/vuln-list-update/ubuntu" "github.com/aquasecurity/vuln-list-update/utils" -) - -const ( - repoURL = "https://%s@github.com/%s/%s.git" - defaultRepoOwner = "aquasecurity" - defaultRepoName = "vuln-list" + "github.com/aquasecurity/vuln-list-update/wolfi" ) var ( target = flag.String("target", "", "update target (nvd, alpine, alpine-unfixed, redhat, redhat-oval, "+ "debian, debian-oval, ubuntu, amazon, oracle-oval, suse-cvrf, photon, arch-linux, ghsa, glad, cwe, osv, go-vulndb, mariner, kevc, wolfi, chainguard)") - years = flag.String("years", "", "update years (only redhat)") + vulnListDir = flag.String("vuln-list-dir", "", "vuln-list dir") targetUri = flag.String("target-uri", "", "alternative repository URI (only glad)") targetBranch = flag.String("target-branch", "", "alternative repository branch (only glad)") ) @@ -65,110 +54,67 @@ func main() { func run() error { flag.Parse() now := time.Now().UTC() - gc := &git.Config{} - debug := os.Getenv("VULN_LIST_DEBUG") != "" - repoOwner := utils.LookupEnv("VULNLIST_REPOSITORY_OWNER", defaultRepoOwner) - repoName := utils.LookupEnv("VULNLIST_REPOSITORY_NAME", defaultRepoName) - - // Embed GitHub token to URL - githubToken := os.Getenv("GITHUB_TOKEN") - url := fmt.Sprintf(repoURL, githubToken, repoOwner, repoName) - - log.Printf("target repository is %s/%s\n", repoOwner, repoName) - log.Printf("cloning/pulling into %s", utils.VulnListDir()) - - if _, err := gc.CloneOrPull(url, utils.VulnListDir(), "main", debug); err != nil { - return xerrors.Errorf("clone or pull error: %w", err) + if *vulnListDir != "" { + utils.SetVulnListDir(*vulnListDir) } - defer func() { - if debug { - return - } - log.Println("git reset & clean") - _ = gc.Clean(utils.VulnListDir()) - }() - - var commitMsg string switch *target { case "nvd": if err := nvd.Update(now.Year()); err != nil { return xerrors.Errorf("NVD update error: %w", err) } - commitMsg = "NVD" case "redhat": - var yearList []int - for _, y := range strings.Split(*years, ",") { - yearInt, err := strconv.Atoi(y) - if err != nil { - return xerrors.Errorf("invalid years: %w", err) - } - yearList = append(yearList, yearInt) - } - if len(yearList) == 0 { - return xerrors.New("years must be specified") - } - if err := securitydataapi.Update(yearList); err != nil { + if err := securitydataapi.Update(); err != nil { return xerrors.Errorf("Red Hat Security Data API update error: %w", err) } - commitMsg = "RedHat " + *years case "redhat-oval": rc := redhatoval.NewConfig() if err := rc.Update(); err != nil { return xerrors.Errorf("Red Hat OVALv2 update error: %w", err) } - commitMsg = "Red Hat OVAL v2" case "debian": dc := tracker.NewClient() if err := dc.Update(); err != nil { return xerrors.Errorf("Debian update error: %w", err) } - commitMsg = "Debian Security Bug Tracker" case "ubuntu": if err := ubuntu.Update(); err != nil { return xerrors.Errorf("Ubuntu update error: %w", err) } - commitMsg = "Ubuntu CVE Tracker" case "alpine": au := alpine.NewUpdater() if err := au.Update(); err != nil { return xerrors.Errorf("Alpine update error: %w", err) } - commitMsg = "Alpine Issue Tracker" case "alpine-unfixed": au := alpineunfixed.NewUpdater() if err := au.Update(); err != nil { return xerrors.Errorf("Alpine Secfixes Tracker update error: %w", err) } - commitMsg = "Alpine Secfixes Tracker" case "amazon": ac := amazon.NewConfig() if err := ac.Update(); err != nil { return xerrors.Errorf("Amazon Linux update error: %w", err) } - commitMsg = "Amazon Linux Security Center" case "oracle-oval": oc := oracleoval.NewConfig() if err := oc.Update(); err != nil { return xerrors.Errorf("Oracle OVAL update error: %w", err) } - commitMsg = "Oracle Linux OVAL" case "suse-cvrf": sc := susecvrf.NewConfig() if err := sc.Update(); err != nil { return xerrors.Errorf("SUSE CVRF update error: %w", err) } - commitMsg = "SUSE CVRF" case "photon": pc := photon.NewConfig() if err := pc.Update(); err != nil { return xerrors.Errorf("Photon update error: %w", err) } - commitMsg = "Photon Security Advisories" case "ghsa": src := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: githubToken}, + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, ) httpClient := oauth2.NewClient(context.Background(), src) @@ -176,106 +122,64 @@ func run() error { if err := gc.Update(); err != nil { return xerrors.Errorf("GitHub Security Advisory update error: %w", err) } - commitMsg = "GitHub Security Advisory" case "glad": gu := glad.NewUpdater(*targetUri, *targetBranch) if err := gu.Update(); err != nil { return xerrors.Errorf("GitLab Advisory Database update error: %w", err) } - commitMsg = "GitLab Advisory Database" case "cwe": c := cwe.NewCWEConfig() if err := c.Update(); err != nil { return xerrors.Errorf("CWE update error: %w", err) } - commitMsg = "CWE Advisories" case "arch-linux": al := arch_linux.NewArchLinux() if err := al.Update(); err != nil { return xerrors.Errorf("Arch Linux update error: %w", err) } - commitMsg = "Arch Linux Security Tracker" case "alma": ac := alma.NewConfig() if err := ac.Update(); err != nil { return xerrors.Errorf("AlmaLinux update error: %w", err) } - commitMsg = "AlmaLinux Security Advisory" case "rocky": rc := rocky.NewConfig() if err := rc.Update(); err != nil { return xerrors.Errorf("Rocky Linux update error: %w", err) } - commitMsg = "Rocky Linux Security Advisory" case "osv": p := osv.NewOsv() if err := p.Update(); err != nil { return xerrors.Errorf("OSV update error: %w", err) } - commitMsg = "OSV Database" case "go-vulndb": src := govulndb.NewVulnDB() if err := src.Update(); err != nil { return xerrors.Errorf("Go Vulnerability Database update error: %w", err) } - commitMsg = "Go Vulnerability Database" case "mariner": src := mariner.NewConfig() if err := src.Update(); err != nil { return xerrors.Errorf("CBL-Mariner Vulnerability Data update error: %w", err) } - commitMsg = "CBL-Mariner Vulnerability Data" case "kevc": src := kevc.NewConfig() if err := src.Update(); err != nil { return xerrors.Errorf("Known Exploited Vulnerability Catalog update error: %w", err) } - commitMsg = "Known Exploited Vulnerability Catalog" case "wolfi": wu := wolfi.NewUpdater() if err := wu.Update(); err != nil { return xerrors.Errorf("Wolfi update error: %w", err) } - commitMsg = "Wolfi Security Data" case "chainguard": cu := chainguard.NewUpdater() if err := cu.Update(); err != nil { return xerrors.Errorf("Chainguard update error: %w", err) } - commitMsg = "Chainguard Security Data" default: return xerrors.New("unknown target") } - if debug { - return nil - } - - if err := utils.SetLastUpdatedDate(*target, now); err != nil { - return err - } - - log.Println("git status") - files, err := gc.Status(utils.VulnListDir()) - if err != nil { - return xerrors.Errorf("git status error: %w", err) - } - - // only last_updated.json - if len(files) < 2 { - log.Println("Skip commit and push") - return nil - } - - log.Println("git commit") - if err = gc.Commit(utils.VulnListDir(), "./", commitMsg); err != nil { - return xerrors.Errorf("git commit error: %w", err) - } - - log.Println("git push") - if err = gc.Push(utils.VulnListDir(), "main"); err != nil { - return xerrors.Errorf("git push error: %w", err) - } - return nil } diff --git a/redhat/oval/redhat.go b/redhat/oval/redhat.go index c313d8f..bf8965e 100644 --- a/redhat/oval/redhat.go +++ b/redhat/oval/redhat.go @@ -22,9 +22,8 @@ import ( ) const ( - ovalDir = "oval" - redhatDir = "redhat" - cpeDir = "redhat-cpe" + ovalDir = "oval" + cpeDir = "cpe" urlFormat = "https://www.redhat.com/security/data/oval/v2/%s" retry = 5 @@ -67,7 +66,7 @@ func (c Config) Update() error { return xerrors.Errorf("unable to update repository-to-cpe.json: %w", err) } - dirPath := filepath.Join(c.VulnListDir, ovalDir, redhatDir) + dirPath := filepath.Join(c.VulnListDir, ovalDir) log.Printf("Remove Red Hat OVAL v2 directory %s", dirPath) if err := os.RemoveAll(dirPath); err != nil { return xerrors.Errorf("failed to remove Red Hat OVAL v2 directory: %w", err) @@ -138,7 +137,7 @@ func (c Config) updateOVAL(ovalFile string) error { // e.g. storage-gluster-3-including-unpatched platform := strings.TrimSuffix(file, ".oval.xml.bz2") - dirPath := filepath.Join(c.VulnListDir, ovalDir, redhatDir, release, platform) + dirPath := filepath.Join(c.VulnListDir, ovalDir, release, platform) // write tests/tests.json file if err := utils.WriteJSON(c.AppFs, filepath.Join(dirPath, testsDir), "tests.json", ovalroot.Tests); err != nil { diff --git a/redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2014/CVE-2014-3209.json b/redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2014/CVE-2014-3209.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2014/CVE-2014-3209.json rename to redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2014/CVE-2014-3209.json diff --git a/redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2014/RHBA-2014-1396.json b/redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2014/RHBA-2014-1396.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2014/RHBA-2014-1396.json rename to redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2014/RHBA-2014-1396.json diff --git a/redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5361.json b/redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5361.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5361.json rename to redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5361.json diff --git a/redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5391.unaffected.json b/redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5391.unaffected.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5391.unaffected.json rename to redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5391.unaffected.json diff --git a/redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2018/CVE-2018-5389.json b/redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2018/CVE-2018-5389.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2018/CVE-2018-5389.json rename to redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2018/CVE-2018-5389.json diff --git a/redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2020/CVE-2020-28935.json b/redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2020/CVE-2020-28935.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/definitions/2020/CVE-2020-28935.json rename to redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/definitions/2020/CVE-2020-28935.json diff --git a/redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/objects/objects.json b/redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/objects/objects.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/objects/objects.json rename to redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/objects/objects.json diff --git a/redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/states/states.json b/redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/states/states.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/states/states.json rename to redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/states/states.json diff --git a/redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/tests/tests.json b/redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/tests/tests.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/6/rhel-6-extras-including-unpatched/tests/tests.json rename to redhat/oval/testdata/golden/6/rhel-6-extras-including-unpatched/tests/tests.json diff --git a/redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0605.unaffected.json b/redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0605.unaffected.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0605.unaffected.json rename to redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0605.unaffected.json diff --git a/redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0606.unaffected.json b/redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0606.unaffected.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0606.unaffected.json rename to redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0606.unaffected.json diff --git a/redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020-0134.json b/redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020-0134.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020-0134.json rename to redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020-0134.json diff --git a/redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020-2249.json b/redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020-2249.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020-2249.json rename to redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020-2249.json diff --git a/redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/objects/objects.json b/redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/objects/objects.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/objects/objects.json rename to redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/objects/objects.json diff --git a/redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/states/states.json b/redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/states/states.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/states/states.json rename to redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/states/states.json diff --git a/redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/tests/tests.json b/redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/tests/tests.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/7/dotnet-3.1-including-unpatched/tests/tests.json rename to redhat/oval/testdata/golden/7/dotnet-3.1-including-unpatched/tests/tests.json diff --git a/redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/definitions/2019/RHSA-2019-3927.json b/redhat/oval/testdata/golden/8/ansible-2-including-unpatched/definitions/2019/RHSA-2019-3927.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/definitions/2019/RHSA-2019-3927.json rename to redhat/oval/testdata/golden/8/ansible-2-including-unpatched/definitions/2019/RHSA-2019-3927.json diff --git a/redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-10744.json b/redhat/oval/testdata/golden/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-10744.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-10744.json rename to redhat/oval/testdata/golden/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-10744.json diff --git a/redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1734.json b/redhat/oval/testdata/golden/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1734.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1734.json rename to redhat/oval/testdata/golden/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1734.json diff --git a/redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1738.json b/redhat/oval/testdata/golden/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1738.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1738.json rename to redhat/oval/testdata/golden/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1738.json diff --git a/redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/definitions/2020/RHSA-2020-0215.json b/redhat/oval/testdata/golden/8/ansible-2-including-unpatched/definitions/2020/RHSA-2020-0215.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/definitions/2020/RHSA-2020-0215.json rename to redhat/oval/testdata/golden/8/ansible-2-including-unpatched/definitions/2020/RHSA-2020-0215.json diff --git a/redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/objects/objects.json b/redhat/oval/testdata/golden/8/ansible-2-including-unpatched/objects/objects.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/objects/objects.json rename to redhat/oval/testdata/golden/8/ansible-2-including-unpatched/objects/objects.json diff --git a/redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/states/states.json b/redhat/oval/testdata/golden/8/ansible-2-including-unpatched/states/states.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/states/states.json rename to redhat/oval/testdata/golden/8/ansible-2-including-unpatched/states/states.json diff --git a/redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/tests/tests.json b/redhat/oval/testdata/golden/8/ansible-2-including-unpatched/tests/tests.json similarity index 100% rename from redhat/oval/testdata/golden/redhat/8/ansible-2-including-unpatched/tests/tests.json rename to redhat/oval/testdata/golden/8/ansible-2-including-unpatched/tests/tests.json diff --git a/redhat/oval/testdata/golden/redhat-cpe/repository-to-cpe.json b/redhat/oval/testdata/golden/redhat-cpe/repository-to-cpe.json deleted file mode 100644 index cf66882..0000000 --- a/redhat/oval/testdata/golden/redhat-cpe/repository-to-cpe.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "3scale-amp-2-for-rhel-8-ppc64le-debug-rpms": [ - "cpe:/a:redhat:3scale_amp:2.11::el8", - "cpe:/a:redhat:3scale_amp:2.12::el8", - "cpe:/a:redhat:3scale_amp:2.8::el8" - ], - "3scale-amp-2-for-rhel-8-ppc64le-rpms": [ - "cpe:/a:redhat:3scale_amp:2.11::el8", - "cpe:/a:redhat:3scale_amp:2.12::el8", - "cpe:/a:redhat:3scale_amp:2.8::el8" - ] -} diff --git a/redhat/securitydataapi/redhat.go b/redhat/securitydataapi/redhat.go index 37291f1..042259a 100644 --- a/redhat/securitydataapi/redhat.go +++ b/redhat/securitydataapi/redhat.go @@ -13,17 +13,18 @@ import ( ) const ( - listURL = "https://access.redhat.com/labs/securitydataapi/cve.json?page=%d&after=%s&per_page=500" - cveURL = "https://access.redhat.com/labs/securitydataapi/cve/%s.json" - redhatDir = "redhat" + listURL = "https://access.redhat.com/labs/securitydataapi/cve.json?page=%d&after=%s&per_page=500" + cveURL = "https://access.redhat.com/labs/securitydataapi/cve/%s.json" + apiDir = "api" concurrency = 10 wait = 1 retry = 20 // Red Hat Security Data API is unstable ) -func Update(years []int) error { - for _, year := range years { +func Update() error { + now := time.Now() + for year := 1996; year <= now.Year(); year++ { if err := update(year); err != nil { return xerrors.Errorf("error in RedHat update: %w", err) } @@ -52,7 +53,7 @@ func update(year int) error { } for cveID, cve := range cves { - if err = utils.SaveCVEPerYear(filepath.Join(utils.VulnListDir(), redhatDir), cveID, cve); err != nil { + if err = utils.SaveCVEPerYear(filepath.Join(utils.VulnListDir(), apiDir), cveID, cve); err != nil { return xerrors.Errorf("failed to save RedHat CVE detail: %w", err) } } diff --git a/update.sh b/update.sh new file mode 100755 index 0000000..e3b8349 --- /dev/null +++ b/update.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +TARGET=$1 +COMMIT_MSG=$2 + +if [ -z "$TARGET" ]; then + echo "target required" + exit 1 +fi + +if [ -z "$COMMIT_MSG" ]; then + echo "commit message required" + exit 1 +fi + +./vuln-list-update -vuln-list-dir "$VULN_LIST_DIR" -target "$TARGET" + +cd "$VULN_LIST_DIR" || exit 1 + +if [[ -n $(git status --porcelain) ]]; then + git add . + git commit -m "${COMMIT_MSG}" + git push +fi \ No newline at end of file diff --git a/utils/utils.go b/utils/utils.go index aa9fa17..a64352d 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -19,6 +19,8 @@ import ( pb "gopkg.in/cheggaaa/pb.v1" ) +var vulnListDir = filepath.Join(CacheDir(), "vuln-list") + func CacheDir() string { cacheDir, err := os.UserCacheDir() if err != nil { @@ -28,8 +30,12 @@ func CacheDir() string { return dir } +func SetVulnListDir(dir string) { + vulnListDir = dir +} + func VulnListDir() string { - return filepath.Join(CacheDir(), "vuln-list") + return vulnListDir } func SaveCVEPerYear(dirPath string, cveID string, data interface{}) error {