vuln-list-update/redhat/oval/redhat.go

package oval

import (
	"bufio"
	"bytes"
	"compress/bzip2"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/cheggaaa/pb"
	"github.com/spf13/afero"
	"golang.org/x/xerrors"

	"github.com/aquasecurity/vuln-list-update/utils"
)

const (
	ovalDir   = "oval"
	redhatDir = "redhat"
	cpeDir    = "redhat-cpe"

	urlFormat    = "https://www.redhat.com/security/data/oval/v2/%s"
	retry        = 5
	pulpManifest = "PULP_MANIFEST"

	repoToCpeURL = "https://www.redhat.com/security/data/metrics/repository-to-cpe.json"

	testsDir       = "tests"
	objectsDir     = "objects"
	statesDir      = "states"
	definitionsDir = "definitions"
)

var (
	ErrInvalidRHSAFormat = errors.New("invalid RHSA-ID format")
	ErrInvalidCVEFormat  = errors.New("invalid CVE-ID format")
)

type Config struct {
	VulnListDir  string
	URLFormat    string
	RepoToCpeURL string
	AppFs        afero.Fs
	Retry        int
}

func NewConfig() Config {
	return Config{
		VulnListDir:  utils.VulnListDir(),
		URLFormat:    urlFormat,
		RepoToCpeURL: repoToCpeURL,
		AppFs:        afero.NewOsFs(),
		Retry:        retry,
	}
}

func (c Config) Update() error {
	log.Println("Updating Red Hat mapping from repositories to CPE names...")
	if err := c.updateRepoToCpe(); err != nil {
		return xerrors.Errorf("unable to update repository-to-cpe.json: %w", err)
	}

	dirPath := filepath.Join(c.VulnListDir, ovalDir, redhatDir)
	log.Printf("Remove Red Hat OVAL v2 directory %s", dirPath)
	if err := os.RemoveAll(dirPath); err != nil {
		return xerrors.Errorf("failed to remove Red Hat OVAL v2 directory: %w", err)
	}

	log.Println("Fetching Red Hat OVAL v2 data...")
	filePaths, err := c.fetchOvalFilePaths()
	if err != nil {
		return xerrors.Errorf("failed to get oval file paths: %w", err)
	}
	for _, ovalFilePath := range filePaths {
		log.Printf("Fetching %s", ovalFilePath)
		if err := c.updateOVAL(ovalFilePath); err != nil {
			return xerrors.Errorf("failed to update Red Hat OVAL v2 json: %w", err)
		}
	}

	return nil
}

func (c Config) updateRepoToCpe() error {
	b, err := utils.FetchURL(c.RepoToCpeURL, "", c.Retry)
	if err != nil {
		return xerrors.Errorf("failed to get %s: %w", c.RepoToCpeURL, err)
	}

	var repoToCPE repositoryToCPE
	if err = json.Unmarshal(b, &repoToCPE); err != nil {
		return xerrors.Errorf("JSON parse error: %w", err)
	}

	mapping := map[string][]string{}
	for repo, cpes := range repoToCPE.Data {
		mapping[repo] = cpes.Cpes
	}

	dir := filepath.Join(c.VulnListDir, cpeDir)
	if err = utils.WriteJSON(c.AppFs, dir, "repository-to-cpe.json", mapping); err != nil {
		return xerrors.Errorf("JSON write error: %w", err)
	}

	return nil
}

func (c Config) updateOVAL(ovalFile string) error {
	// e.g. RHEL8/storage-gluster-3-including-unpatched.oval.xml.bz2
	if !strings.HasPrefix(ovalFile, "RHEL") {
		log.Printf("Skip %s", ovalFile)
		return nil
	}

	// e.g. RHEL8/storage-gluster-3-including-unpatched.oval.xml.bz2
	// => RHEL8/, storage-gluster-3-including-unpatched.oval.xml.bz2
	dir, file := path.Split(ovalFile)
	release := strings.TrimPrefix(path.Clean(dir), "RHEL")

	url := fmt.Sprintf(c.URLFormat, ovalFile)
	res, err := utils.FetchURL(url, "", c.Retry)
	if err != nil {
		return xerrors.Errorf("failed to fetch Red Hat OVAL v2: %w", err)
	}

	bzr := bzip2.NewReader(bytes.NewBuffer(res))
	var ovalroot OvalDefinitions
	if err := xml.NewDecoder(bzr).Decode(&ovalroot); err != nil {
		return xerrors.Errorf("failed to unmarshal Red Hat OVAL v2 XML: %w", err)
	}

	// e.g. storage-gluster-3-including-unpatched
	platform := strings.TrimSuffix(file, ".oval.xml.bz2")
	dirPath := filepath.Join(c.VulnListDir, ovalDir, redhatDir, release, platform)

	// write tests/tests.json file
	if err := utils.WriteJSON(c.AppFs, filepath.Join(dirPath, testsDir), "tests.json", ovalroot.Tests); err != nil {
		return xerrors.Errorf("failed to write tests: %w", err)
	}

	// write objects/objects.json file
	if err := utils.WriteJSON(c.AppFs, filepath.Join(dirPath, objectsDir), "objects.json", ovalroot.Objects); err != nil {
		return xerrors.Errorf("failed to write objects: %w", err)
	}

	// write states/states.json file
	if err := utils.WriteJSON(c.AppFs, filepath.Join(dirPath, statesDir), "states.json", ovalroot.States); err != nil {
		return xerrors.Errorf("failed to write states: %w", err)
	}

	// write definitions
	bar := pb.StartNew(len(ovalroot.Definitions.Definition))
	for _, def := range ovalroot.Definitions.Definition {
		if len(def.Metadata.References) == 0 {
			continue
		}

		// RHSA-ID or CVE-ID
		vulnID := def.Metadata.References[0].RefID
		for _, ref := range def.Metadata.References {
			if strings.HasPrefix(ref.RefID, "RHSA-") {
				vulnID = ref.RefID
			}
		}

		if err := c.saveAdvisoryPerYear(filepath.Join(dirPath, definitionsDir), vulnID, def); err != nil {
			return xerrors.Errorf("failed to save advisory per year: %w", err)
		}

		bar.Increment()
	}
	bar.Finish()

	return nil
}

func (c Config) fetchOvalFilePaths() ([]string, error) {
	res, err := utils.FetchURL(fmt.Sprintf(c.URLFormat, pulpManifest), "", c.Retry)
	if err != nil {
		return nil, xerrors.Errorf("failed to fetch PULP_MANIFEST: %w", err)
	}

	var ovalFilePaths []string
	scanner := bufio.NewScanner(bytes.NewReader(res))
	for scanner.Scan() {
		ss := strings.Split(scanner.Text(), ",")
		if len(ss) < 3 {
			return nil, xerrors.Errorf("failed to parse PULP_MANIFEST: %w", err)
		}
		// skip if size is 0
		if ss[2] == "0" {
			continue
		}

		ovalFilePaths = append(ovalFilePaths, ss[0])
	}
	return ovalFilePaths, nil
}

func (c Config) saveAdvisoryPerYear(dirName string, id string, def Definition) error {
	var year string
	if strings.HasPrefix(id, "CVE") {
		s := strings.Split(id, "-")
		if len(s) != 3 {
			log.Printf("invalid CVE-ID format: %s\n", id)
			return ErrInvalidCVEFormat
		}
		year = s[1]
	} else {
		// e.g. RHSA-2018:0094
		s := strings.Split(id, ":")
		if len(s) != 2 {
			log.Printf("invalid RHSA-ID format: %s\n", id)
			return ErrInvalidRHSAFormat
		}
		s = strings.Split(s[0], "-")
		if len(s) != 2 {
			log.Printf("invalid RHSA-ID format: %s\n", id)
			return ErrInvalidRHSAFormat
		}
		year = s[1]
	}

	fileFmt := "%s.json"
	if strings.HasPrefix(def.ID, "oval:com.redhat.unaffected:def") {
		fileFmt = "%s.unaffected.json"
	}

	yearDir := filepath.Join(dirName, year)
	if err := utils.WriteJSON(c.AppFs, yearDir, fmt.Sprintf(fileFmt, id), def); err != nil {
		return xerrors.Errorf("unable to write a JSON file: %w", err)
	}
	return nil
}