vuln-list-update/glad/glad.go

package glad

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/afero"
	"golang.org/x/xerrors"
	"gopkg.in/yaml.v2"

	"github.com/aquasecurity/vuln-list-update/git"
	"github.com/aquasecurity/vuln-list-update/utils"
)

const (
	repoURL    = "https://gitlab.com/gitlab-org/advisories-community.git"
	repoBranch = "main"
	gladDir    = "glad" // GitLab Advisory Database
)

var (
	// https://gitlab.com/gitlab-org/advisories-community
	supportedTypes = []string{"gem", "go", "maven", "npm", "nuget", "packagist", "pypi", "nuget", "conan"}
)

type Updater struct {
	alternativeRepoBranch string
	alternativeRepoURL    string
	vulnListDir           string
	cacheDir              string
	appFs                 afero.Fs
}

func NewUpdater(alternativeRepoURL string, alternativeRepoBranch string) Updater {
	return Updater{
		alternativeRepoBranch: alternativeRepoBranch,
		alternativeRepoURL:    alternativeRepoURL,
		vulnListDir:           utils.VulnListDir(),
		cacheDir:              utils.CacheDir(),
		appFs:                 afero.NewOsFs(),
	}
}

func (u Updater) Update() error {
	log.Print("Fetching GitLab Advisory Database (advisories-community)")

	gc := git.Config{}
	dir := filepath.Join(u.cacheDir, gladDir)
	defaultOrAlternativeRepoURL := repoURL
	defaultOrAlternativeRepoBranch := repoBranch

	if len(u.alternativeRepoURL) > 0 {
		defaultOrAlternativeRepoURL = u.alternativeRepoURL
	}

	if len(u.alternativeRepoBranch) > 0 {
		defaultOrAlternativeRepoBranch = u.alternativeRepoBranch
	}

	if _, err := gc.CloneOrPull(defaultOrAlternativeRepoURL, dir, defaultOrAlternativeRepoBranch, false); err != nil {
		return xerrors.Errorf("failed to clone or pull: %w", err)
	}

	log.Println("Walking glad...")
	for _, target := range supportedTypes {
		targetDir := filepath.Join(dir, target)
		if ok, _ := utils.Exists(targetDir); !ok {
			continue
		}
		if err := u.walkDir(targetDir); err != nil {
			return xerrors.Errorf("failed to walk %s: %w", targetDir, err)
		}
	}

	return nil
}

func (u Updater) walkDir(root string) error {
	var advisories []advisory
	err := afero.Walk(u.appFs, root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return xerrors.Errorf("file walk error: %w", err)
		}
		if info.IsDir() {
			return nil
		}

		ext := filepath.Ext(path)
		if ext != ".yaml" && ext != ".yml" {
			return nil
		}

		f, err := u.appFs.Open(path)
		if err != nil {
			return xerrors.Errorf("file open error (%s): %w", path, err)
		}
		defer f.Close()

		var adv advisory
		if err = yaml.NewDecoder(f).Decode(&adv); err != nil {
			return xerrors.Errorf("unable to decode YAML (%s): %w", path, err)
		}
		advisories = append(advisories, adv)

		return nil
	})
	if err != nil {
		return xerrors.Errorf("walk error: %w", err)
	}

	for _, adv := range advisories {
		adv.PackageSlug = strings.TrimSuffix(adv.PackageSlug, "/")

		// Update Identifier to upper case
		// e.g. cvE-2014-3530 => CVE-2014-3530
		// https://gitlab.com/gitlab-org/advisories-community/-/blob/74a18a7968c2bdd2dd901f6c98f06cb1d9684476/maven/org.picketlink/picketlink-common/cvE-2014-3530.yml
		adv.Identifier = strings.ToUpper(adv.Identifier)

		slug := u.searchPrefix(adv, advisories)
		if slug != "" {
			// Update the package_slug to flatten nested packages
			// e.g.  go/k8s.io/kubernetes => go/k8s.io/kubernetes
			//       go/k8s.io/kubernetes/pkg/kubelet/kuberuntime => go/k8s.io/kubernetes
			adv.PackageSlug = slug
		}
		if err = u.save(adv); err != nil {
			return xerrors.Errorf("save error: %w", err)
		}
	}

	return nil
}

func (u Updater) searchPrefix(adv advisory, advisories []advisory) string {
	for _, a := range advisories {
		if a.PackageSlug == adv.PackageSlug {
			continue
		}

		if strings.HasPrefix(adv.PackageSlug, a.PackageSlug) {
			return a.PackageSlug
		}
	}
	return ""
}

func (u Updater) save(adv advisory) error {
	s := strings.Split(adv.PackageSlug, "/")
	dir := filepath.Join(s...)
	dir = filepath.Join(u.vulnListDir, gladDir, dir)

	fileName := fmt.Sprintf("%s.json", adv.Identifier)
	if err := utils.WriteJSON(u.appFs, dir, fileName, adv); err != nil {
		return xerrors.Errorf("unable to write JSON (%s): %w", fileName, err)
	}
	return nil
}