vuln-list-update/glad/glad.go

package glad

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/afero"
	"golang.org/x/xerrors"
	"gopkg.in/yaml.v2"

	"github.com/aquasecurity/vuln-list-update/git"
	"github.com/aquasecurity/vuln-list-update/utils"
)

const (
	repoURL    = "https://gitlab.com/gitlab-org/advisories-community.git"
	repoBranch = "main"
	gladDir    = "glad" // GitLab Advisory Database
)

var (
	// https://gitlab.com/gitlab-org/advisories-community
	supportedTypes = []string{"conan", "gem", "go", "maven", "npm", "nuget", "packagist", "pypi"}
)

type Updater struct {
	alternativeRepoBranch string
	alternativeRepoURL    string
	vulnListDir           string
	cacheDir              string
	appFs                 afero.Fs
}

func NewUpdater(alternativeRepoURL string, alternativeRepoBranch string) Updater {
	return Updater{
		alternativeRepoBranch: alternativeRepoBranch,
		alternativeRepoURL:    alternativeRepoURL,
		vulnListDir:           utils.VulnListDir(),
		cacheDir:              utils.CacheDir(),
		appFs:                 afero.NewOsFs(),
	}
}

func (u Updater) Update() error {
	log.Print("Fetching GitLab Advisory Database (advisories-community)")

	gc := git.Config{}
	dir := filepath.Join(u.cacheDir, gladDir)
	defaultOrAlternativeRepoURL := repoURL
	defaultOrAlternativeRepoBranch := repoBranch

	if len(u.alternativeRepoURL) > 0 {
		defaultOrAlternativeRepoURL = u.alternativeRepoURL
	}

	if len(u.alternativeRepoBranch) > 0 {
		defaultOrAlternativeRepoBranch = u.alternativeRepoBranch
	}

	if _, err := gc.CloneOrPull(defaultOrAlternativeRepoURL, dir, defaultOrAlternativeRepoBranch, false); err != nil {
		return xerrors.Errorf("failed to clone or pull: %w", err)
	}

	log.Println("Removing old glad files...")
	if err := os.RemoveAll(filepath.Join(u.vulnListDir, gladDir)); err != nil {
		return xerrors.Errorf("can't remove a folder with old files %s/%s: %w", u.vulnListDir, gladDir, err)
	}

	log.Println("Walking glad...")
	for _, target := range supportedTypes {
		targetDir := filepath.Join(dir, target)
		if ok, _ := utils.Exists(targetDir); !ok {
			continue
		}
		if err := u.walkDir(targetDir); err != nil {
			return xerrors.Errorf("failed to walk %s: %w", targetDir, err)
		}
	}

	return nil
}

func (u Updater) walkDir(root string) error {
	var advisories []advisory
	err := afero.Walk(u.appFs, root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return xerrors.Errorf("file walk error: %w", err)
		}
		if info.IsDir() {
			return nil
		}

		ext := filepath.Ext(path)
		if ext != ".yaml" && ext != ".yml" {
			return nil
		}

		f, err := u.appFs.Open(path)
		if err != nil {
			return xerrors.Errorf("file open error (%s): %w", path, err)
		}
		defer f.Close()

		var adv advisory
		if err = yaml.NewDecoder(f).Decode(&adv); err != nil {
			return xerrors.Errorf("unable to decode YAML (%s): %w", path, err)
		}
		advisories = append(advisories, adv)

		return nil
	})
	if err != nil {
		return xerrors.Errorf("walk error: %w", err)
	}

	for _, adv := range advisories {
		adv.Identifier = updateIdentifiers(adv.Identifier, adv.Identifiers)
		adv.Identifiers = nil

		// Only 'go' package slugs need to be updated
		if strings.HasPrefix(adv.PackageSlug, "go/") {
			slug := u.searchPrefix(adv.PackageSlug, advisories)
			if slug != "" {
				// Update the package_slug to flatten nested packages
				// e.g.  go/k8s.io/kubernetes => go/k8s.io/kubernetes
				//       go/k8s.io/kubernetes/pkg/kubelet/kuberuntime => go/k8s.io/kubernetes
				adv.PackageSlug = slug
			}
		}

		if err = u.save(adv); err != nil {
			return xerrors.Errorf("save error: %w", err)
		}
	}

	return nil
}

func (u Updater) searchPrefix(pkgSlug string, advisories []advisory) string {
	for _, a := range advisories {
		if pkgSlug == a.PackageSlug {
			continue
		}
		// '/' has been added to skip packages with same prefix
		// e.g.: pkgSlug == go/github.com/apache/thrift-mini
		// a.PackageSlug == go/github.com/apache/thrift
		advSlug := a.PackageSlug
		if !strings.HasSuffix(advSlug, "/") {
			advSlug += "/"
		}

		if strings.HasPrefix(pkgSlug, advSlug) {
			return a.PackageSlug
		}
	}
	return ""
}

func (u Updater) save(adv advisory) error {
	s := strings.Split(adv.PackageSlug, "/")
	dir := filepath.Join(s...)
	dir = filepath.Join(u.vulnListDir, gladDir, dir)

	fileName := fmt.Sprintf("%s.json", adv.Identifier)
	if err := utils.WriteJSON(u.appFs, dir, fileName, adv); err != nil {
		return xerrors.Errorf("unable to write JSON (%s): %w", fileName, err)
	}
	return nil
}

func updateIdentifiers(basicIdentifier string, basicIdentifiers []string) string {
	// Update Identifier to upper case
	// e.g. cvE-2014-3530 => CVE-2014-3530
	// https://gitlab.com/gitlab-org/advisories-community/-/blob/74a18a7968c2bdd2dd901f6c98f06cb1d9684476/maven/org.picketlink/picketlink-common/cvE-2014-3530.yml
	updated := strings.ToUpper(basicIdentifier)

	// If a `basicIdentifier` is not CVE-ID, then try to find CVE-ID or GHSA-ID in `basicIdentifiers`
	if !strings.HasPrefix(updated, "CVE") {
		// Try to find `CVE-ID`
		for i := range basicIdentifiers {
			if ident := strings.ToUpper(basicIdentifiers[i]); strings.HasPrefix(ident, "CVE") {
				return ident
			}
		}
		// If `CVE-ID` is not found, then try to find `GHSA-ID`
		for i := range basicIdentifiers {
			if ident := strings.ToUpper(basicIdentifiers[i]); strings.HasPrefix(ident, "GHSA") {
				// return no uppercase string because GHSA id contains small letters (eg GHSA-qq97-vm5h-rrhg)
				return basicIdentifiers[i]
			}
		}
	}
	return updated
}