feat(redhat-oval): store repository-to-cpe.json (#121)

* test(redhat): simplify

* feat(redhat): store repository-to-cpe.json

redhat/oval/redhat.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"compress/bzip2"
+	"encoding/json"
 	"encoding/xml"
 	"errors"
 	"fmt"
@@ -21,12 +22,16 @@ import (
 )
 
 const (
-	ovalDir      = "oval"
-	redhatDir    = "redhat"
+	ovalDir   = "oval"
+	redhatDir = "redhat"
+	cpeDir    = "redhat-cpe"
+
 	urlFormat    = "https://www.redhat.com/security/data/oval/v2/%s"
 	retry        = 5
 	pulpManifest = "PULP_MANIFEST"
 
+	repoToCpeURL = "https://www.redhat.com/security/data/metrics/repository-to-cpe.json"
+
 	testsDir       = "tests"
 	objectsDir     = "objects"
 	statesDir      = "states"
@@ -39,22 +44,29 @@ var (
 )
 
 type Config struct {
-	VulnListDir string
-	URLFormat   string
-	AppFs       afero.Fs
-	Retry       int
+	VulnListDir  string
+	URLFormat    string
+	RepoToCpeURL string
+	AppFs        afero.Fs
+	Retry        int
 }
 
 func NewConfig() Config {
 	return Config{
-		VulnListDir: utils.VulnListDir(),
-		URLFormat:   urlFormat,
-		AppFs:       afero.NewOsFs(),
-		Retry:       retry,
+		VulnListDir:  utils.VulnListDir(),
+		URLFormat:    urlFormat,
+		RepoToCpeURL: repoToCpeURL,
+		AppFs:        afero.NewOsFs(),
+		Retry:        retry,
 	}
 }
 
 func (c Config) Update() error {
+	log.Println("Updating Red Hat mapping from repositories to CPE names...")
+	if err := c.updateRepoToCpe(); err != nil {
+		return xerrors.Errorf("unable to update repository-to-cpe.json: %w", err)
+	}
+
 	dirPath := filepath.Join(c.VulnListDir, ovalDir, redhatDir)
 	log.Printf("Remove Red Hat OVAL v2 directory %s", dirPath)
 	if err := os.RemoveAll(dirPath); err != nil {
@@ -68,7 +80,7 @@ func (c Config) Update() error {
 	}
 	for _, ovalFilePath := range filePaths {
 		log.Printf("Fetching %s", ovalFilePath)
-		if err := c.update(ovalFilePath); err != nil {
+		if err := c.updateOVAL(ovalFilePath); err != nil {
 			return xerrors.Errorf("failed to update Red Hat OVAL v2 json: %w", err)
 		}
 	}
@@ -76,7 +88,31 @@ func (c Config) Update() error {
 	return nil
 }
 
-func (c Config) update(ovalFile string) error {
+func (c Config) updateRepoToCpe() error {
+	b, err := utils.FetchURL(c.RepoToCpeURL, "", c.Retry)
+	if err != nil {
+		return xerrors.Errorf("failed to get %s: %w", c.RepoToCpeURL, err)
+	}
+
+	var repoToCPE repositoryToCPE
+	if err = json.Unmarshal(b, &repoToCPE); err != nil {
+		return xerrors.Errorf("JSON parse error: %w", err)
+	}
+
+	mapping := map[string][]string{}
+	for repo, cpes := range repoToCPE.Data {
+		mapping[repo] = cpes.Cpes
+	}
+
+	dir := filepath.Join(c.VulnListDir, cpeDir)
+	if err = utils.WriteJSON(c.AppFs, dir, "repository-to-cpe.json", mapping); err != nil {
+		return xerrors.Errorf("JSON write error: %w", err)
+	}
+
+	return nil
+}
+
+func (c Config) updateOVAL(ovalFile string) error {
 	// e.g. RHEL8/storage-gluster-3-including-unpatched.oval.xml.bz2
 	if !strings.HasPrefix(ovalFile, "RHEL") {
 		log.Printf("Skip %s", ovalFile)

redhat/oval/redhat_test.go

@@ -3,12 +3,11 @@ package oval
 import (
 	"errors"
 	"flag"
-	"strings"
-
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -21,158 +20,105 @@ var update = flag.Bool("update", false, "update golden files")
 
 func TestConfig_Update(t *testing.T) {
 	testCases := []struct {
-		name             string
-		bzip2FileNames   map[string]string
-		goldenFiles      map[string]string
-		expectedErrorMsg string
+		name      string
+		dir       string
+		wantFiles int
+		wantErr   string
 	}{
 		{
-			name: "happy path ",
-
-			bzip2FileNames: map[string]string{
-				"/PULP_MANIFEST": "testdata/PULP_MANIFEST",
-				"/RHEL6/rhel-6-extras-including-unpatched.oval.xml.bz2": "testdata/rhel-6-extras-including-unpatched.oval.xml.bz2",
-				"/RHEL7/dotnet-3.1-including-unpatched.oval.xml.bz2":    "testdata/dotnet-3.1-including-unpatched.oval.xml.bz2",
-				"/RHEL8/ansible-2-including-unpatched.oval.xml.bz2":     "testdata/ansible-2-including-unpatched.oval.xml.bz2",
-			},
-			goldenFiles: map[string]string{
-				"/tmp/oval/redhat/6/rhel-6-extras-including-unpatched/definitions/2014/CVE-2014-3209.json":            "testdata/golden/rhel-6-extras-including-unpatched/CVE-2014-3209.json",
-				"/tmp/oval/redhat/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5361.json":            "testdata/golden/rhel-6-extras-including-unpatched/CVE-2016-5361.json",
-				"/tmp/oval/redhat/6/rhel-6-extras-including-unpatched/definitions/2018/CVE-2018-5389.json":            "testdata/golden/rhel-6-extras-including-unpatched/CVE-2018-5389.json",
-				"/tmp/oval/redhat/6/rhel-6-extras-including-unpatched/definitions/2020/CVE-2020-28935.json":           "testdata/golden/rhel-6-extras-including-unpatched/CVE-2020-28935.json",
-				"/tmp/oval/redhat/6/rhel-6-extras-including-unpatched/definitions/2014/RHBA-2014:1396.json":           "testdata/golden/rhel-6-extras-including-unpatched/RHBA-2014-1396.json",
-				"/tmp/oval/redhat/6/rhel-6-extras-including-unpatched/definitions/2016/CVE-2016-5391.unaffected.json": "testdata/golden/rhel-6-extras-including-unpatched/CVE-2016-5391.unaffected.json",
-				"/tmp/oval/redhat/6/rhel-6-extras-including-unpatched/objects/objects.json":                           "testdata/golden/rhel-6-extras-including-unpatched/objects.json",
-				"/tmp/oval/redhat/6/rhel-6-extras-including-unpatched/states/states.json":                             "testdata/golden/rhel-6-extras-including-unpatched/states.json",
-				"/tmp/oval/redhat/6/rhel-6-extras-including-unpatched/tests/tests.json":                               "testdata/golden/rhel-6-extras-including-unpatched/tests.json",
-
-				"/tmp/oval/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020:0134.json":           "testdata/golden/dotnet-3.1-including-unpatched/RHSA-2020-0134.json",
-				"/tmp/oval/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/RHSA-2020:2249.json":           "testdata/golden/dotnet-3.1-including-unpatched/RHSA-2020-2249.json",
-				"/tmp/oval/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0605.unaffected.json": "testdata/golden/dotnet-3.1-including-unpatched/CVE-2020-0605.unaffected.json",
-				"/tmp/oval/redhat/7/dotnet-3.1-including-unpatched/definitions/2020/CVE-2020-0606.unaffected.json": "testdata/golden/dotnet-3.1-including-unpatched/CVE-2020-0606.unaffected.json",
-				"/tmp/oval/redhat/7/dotnet-3.1-including-unpatched/objects/objects.json":                           "testdata/golden/dotnet-3.1-including-unpatched/objects.json",
-				"/tmp/oval/redhat/7/dotnet-3.1-including-unpatched/states/states.json":                             "testdata/golden/dotnet-3.1-including-unpatched/states.json",
-				"/tmp/oval/redhat/7/dotnet-3.1-including-unpatched/tests/tests.json":                               "testdata/golden/dotnet-3.1-including-unpatched/tests.json",
-
-				"/tmp/oval/redhat/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-10744.json": "testdata/golden/ansible-2-including-unpatched/CVE-2020-10744.json",
-				"/tmp/oval/redhat/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1734.json":  "testdata/golden/ansible-2-including-unpatched/CVE-2020-1734.json",
-				"/tmp/oval/redhat/8/ansible-2-including-unpatched/definitions/2020/CVE-2020-1738.json":  "testdata/golden/ansible-2-including-unpatched/CVE-2020-1738.json",
-				"/tmp/oval/redhat/8/ansible-2-including-unpatched/definitions/2019/RHSA-2019:3927.json": "testdata/golden/ansible-2-including-unpatched/RHSA-2019-3927.json",
-				"/tmp/oval/redhat/8/ansible-2-including-unpatched/definitions/2020/RHSA-2020:0215.json": "testdata/golden/ansible-2-including-unpatched/RHSA-2020-0215.json",
-				"/tmp/oval/redhat/8/ansible-2-including-unpatched/objects/objects.json":                 "testdata/golden/ansible-2-including-unpatched/objects.json",
-				"/tmp/oval/redhat/8/ansible-2-including-unpatched/states/states.json":                   "testdata/golden/ansible-2-including-unpatched/states.json",
-				"/tmp/oval/redhat/8/ansible-2-including-unpatched/tests/tests.json":                     "testdata/golden/ansible-2-including-unpatched/tests.json",
-			},
+			name:      "happy path",
+			dir:       "testdata/happy",
+			wantFiles: 24,
 		},
 		{
-			name: "404",
-			bzip2FileNames: map[string]string{
-				"/PULP_MANIFEST": "testdata/PULP_MANIFEST",
-			},
-			goldenFiles:      map[string]string{},
-			expectedErrorMsg: "failed to fetch Red Hat OVAL v2: failed to fetch URL: HTTP error. status code: 404, url:",
+			name:    "404",
+			dir:     "testdata/missing-oval",
+			wantErr: "failed to fetch Red Hat OVAL v2: failed to fetch URL: HTTP error. status code: 404, url:",
 		},
 		{
-			name: "invalid file format",
-			bzip2FileNames: map[string]string{
-				"/PULP_MANIFEST": "testdata/PULP_MANIFEST",
-				"/RHEL6/rhel-6-extras-including-unpatched.oval.xml.bz2": "testdata/test.txt",
-			},
-			goldenFiles:      map[string]string{},
-			expectedErrorMsg: "failed to unmarshal Red Hat OVAL v2 XML: bzip2 data invalid: bad magic value",
+			name:    "invalid file format",
+			dir:     "testdata/invalid-bzip2",
+			wantErr: "failed to unmarshal Red Hat OVAL v2 XML: bzip2 data invalid: bad magic value",
 		},
 		{
-			name: "broken XML",
-			bzip2FileNames: map[string]string{
-				"/PULP_MANIFEST": "testdata/PULP_MANIFEST",
-				"/RHEL6/rhel-6-extras-including-unpatched.oval.xml.bz2": "testdata/rhel-6-extras-including-unpatched-broken-XML.oval.xml.bz2",
-			},
-			goldenFiles:      map[string]string{},
-			expectedErrorMsg: "failed to unmarshal Red Hat OVAL v2 XML: XML syntax error on line 411: element",
+			name:    "broken XML",
+			dir:     "testdata/broken-xml",
+			wantErr: "failed to unmarshal Red Hat OVAL v2 XML: XML syntax error on line 411: element",
 		},
 	}
 	for _, tc := range testCases {
-		dataPath := "/security/data/oval/v2"
 		t.Run(tc.name, func(t *testing.T) {
-			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				p := strings.TrimPrefix(r.URL.Path, dataPath)
-				filePath, ok := tc.bzip2FileNames[p]
-				if !ok {
-					http.NotFound(w, r)
-					return
-				}
-				b, err := ioutil.ReadFile(filePath)
-				require.NoError(t, err, tc.name)
-
-				_, err = w.Write(b)
-				assert.NoError(t, err, tc.name)
-			}))
+			ts := httptest.NewServer(http.FileServer(http.Dir(tc.dir)))
 			defer ts.Close()
 
+			tmpDir := "/tmp" // It is a virtual filesystem of afero.
 			appFs := afero.NewMemMapFs()
 			c := Config{
-				VulnListDir: "/tmp",
-				URLFormat:   ts.URL + dataPath + "/%s",
-				AppFs:       appFs,
-				Retry:       0,
-			}
-			err := c.Update()
-			switch {
-			case tc.expectedErrorMsg != "":
-				assert.Contains(t, err.Error(), tc.expectedErrorMsg, tc.name)
-			default:
-				assert.NoError(t, err, tc.name)
+				VulnListDir:  tmpDir,
+				URLFormat:    ts.URL + "/%s",
+				RepoToCpeURL: ts.URL + "/repository-to-cpe.json",
+				AppFs:        appFs,
+				Retry:        0,
 			}
 
+			err := c.Update()
+			if tc.wantErr != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tc.wantErr)
+				return
+			}
+
+			require.NoError(t, err, tc.name)
+
 			fileCount := 0
-			err = afero.Walk(appFs, "/", func(path string, info os.FileInfo, err error) error {
+			root := tmpDir + "/oval"
+			err = afero.Walk(appFs, root, func(path string, info os.FileInfo, err error) error {
 				if err != nil {
 					return err
-				}
-				if info.IsDir() {
+				} else if info.IsDir() {
 					return nil
 				}
 				fileCount++
 
-				actual, err := afero.ReadFile(appFs, path)
+				got, err := afero.ReadFile(appFs, path)
 				assert.NoError(t, err, tc.name)
 
-				goldenPath, ok := tc.goldenFiles[path]
-				assert.True(t, ok, tc.name)
+				rel, err := filepath.Rel(root, path)
+				require.NoError(t, err)
+
+				rel = strings.ReplaceAll(rel, ":", "-")
+				goldenPath := filepath.Join("testdata", "golden", rel)
 				if *update {
-					err = ioutil.WriteFile(goldenPath, actual, 0666)
-					require.NoError(t, err, tc.name)
+					err = os.WriteFile(goldenPath, got, 0666)
+					require.NoError(t, err, goldenPath)
 				}
-				expected, err := ioutil.ReadFile(goldenPath)
+				want, err := os.ReadFile(goldenPath)
 				assert.NoError(t, err, tc.name)
 
-				assert.Equal(t, string(expected), string(actual), path)
+				assert.JSONEq(t, string(want), string(got), path)
 
 				return nil
 			})
-			assert.Equal(t, len(tc.goldenFiles), fileCount, tc.name)
 			assert.NoError(t, err, tc.name)
+			assert.Equal(t, tc.wantFiles, fileCount, tc.name)
 		})
 	}
 }
 
 func TestConfig_saveRHSAPerYear(t *testing.T) {
 	testCases := []struct {
-		name          string
-		rhsaID        string
-		inputData     Definition
-		expectedError error
+		name    string
+		rhsaID  string
+		wantErr error
 	}{
 		{
-			name:      "happy path",
-			rhsaID:    "RHSA-2018:0094",
-			inputData: Definition{},
+			name:   "happy path",
+			rhsaID: "RHSA-2018:0094",
 		},
 		{
-			name:          "sad path: invalid RHSA-ID format",
-			rhsaID:        "foobarbaz",
-			inputData:     Definition{},
-			expectedError: errors.New("invalid RHSA-ID format"),
+			name:    "sad path: invalid RHSA-ID format",
+			rhsaID:  "foobarbaz",
+			wantErr: errors.New("invalid RHSA-ID format"),
 		},
 	}
 
@@ -181,15 +127,10 @@ func TestConfig_saveRHSAPerYear(t *testing.T) {
 			AppFs: afero.NewMemMapFs(),
 		}
 
-		d, _ := ioutil.TempDir("", "TestConfig_saveRHSAPerYear-*")
-		defer func() {
-			_ = os.RemoveAll(d)
-		}()
-
-		err := c.saveAdvisoryPerYear(d, tc.rhsaID, tc.inputData)
+		err := c.saveAdvisoryPerYear("/tmp", tc.rhsaID, Definition{})
 		switch {
-		case tc.expectedError != nil:
-			assert.Equal(t, tc.expectedError.Error(), err.Error(), tc.name)
+		case tc.wantErr != nil:
+			assert.Equal(t, tc.wantErr.Error(), err.Error(), tc.name)
 		default:
 			assert.NoError(t, err, tc.name)
 		}

redhat/oval/testdata/broken-xml/PULP_MANIFEST

@@ -0,0 +1 @@
+RHEL6/rhel-6-extras-including-unpatched.oval.xml.bz2,6a4e05e6c5ef90d23c4e1752ac5bf247e075d17b76c0f60f6aa02e68daa5a6a4,6347

redhat/oval/testdata/broken-xml/repository-to-cpe.json

@@ -0,0 +1,18 @@
+{
+  "data": {
+    "3scale-amp-2-for-rhel-8-ppc64le-debug-rpms": {
+      "cpes": [
+        "cpe:/a:redhat:3scale_amp:2.11::el8",
+        "cpe:/a:redhat:3scale_amp:2.12::el8",
+        "cpe:/a:redhat:3scale_amp:2.8::el8"
+      ]
+    },
+    "3scale-amp-2-for-rhel-8-ppc64le-rpms": {
+      "cpes": [
+        "cpe:/a:redhat:3scale_amp:2.11::el8",
+        "cpe:/a:redhat:3scale_amp:2.12::el8",
+        "cpe:/a:redhat:3scale_amp:2.8::el8"
+      ]
+    }
+  }
+}

redhat/oval/testdata/golden/redhat-cpe/repository-to-cpe.json

@@ -0,0 +1,12 @@
+{
+  "3scale-amp-2-for-rhel-8-ppc64le-debug-rpms": [
+    "cpe:/a:redhat:3scale_amp:2.11::el8",
+    "cpe:/a:redhat:3scale_amp:2.12::el8",
+    "cpe:/a:redhat:3scale_amp:2.8::el8"
+  ],
+  "3scale-amp-2-for-rhel-8-ppc64le-rpms": [
+    "cpe:/a:redhat:3scale_amp:2.11::el8",
+    "cpe:/a:redhat:3scale_amp:2.12::el8",
+    "cpe:/a:redhat:3scale_amp:2.8::el8"
+  ]
+}

redhat/oval/testdata/happy/repository-to-cpe.json

@@ -0,0 +1,18 @@
+{
+  "data": {
+    "3scale-amp-2-for-rhel-8-ppc64le-debug-rpms": {
+      "cpes": [
+        "cpe:/a:redhat:3scale_amp:2.11::el8",
+        "cpe:/a:redhat:3scale_amp:2.12::el8",
+        "cpe:/a:redhat:3scale_amp:2.8::el8"
+      ]
+    },
+    "3scale-amp-2-for-rhel-8-ppc64le-rpms": {
+      "cpes": [
+        "cpe:/a:redhat:3scale_amp:2.11::el8",
+        "cpe:/a:redhat:3scale_amp:2.12::el8",
+        "cpe:/a:redhat:3scale_amp:2.8::el8"
+      ]
+    }
+  }
+}

redhat/oval/testdata/invalid-bzip2/PULP_MANIFEST

@@ -0,0 +1 @@
+RHEL6/rhel-6-extras-including-unpatched.oval.xml.bz2,6a4e05e6c5ef90d23c4e1752ac5bf247e075d17b76c0f60f6aa02e68daa5a6a4,6347

redhat/oval/testdata/invalid-bzip2/repository-to-cpe.json

@@ -0,0 +1,18 @@
+{
+  "data": {
+    "3scale-amp-2-for-rhel-8-ppc64le-debug-rpms": {
+      "cpes": [
+        "cpe:/a:redhat:3scale_amp:2.11::el8",
+        "cpe:/a:redhat:3scale_amp:2.12::el8",
+        "cpe:/a:redhat:3scale_amp:2.8::el8"
+      ]
+    },
+    "3scale-amp-2-for-rhel-8-ppc64le-rpms": {
+      "cpes": [
+        "cpe:/a:redhat:3scale_amp:2.11::el8",
+        "cpe:/a:redhat:3scale_amp:2.12::el8",
+        "cpe:/a:redhat:3scale_amp:2.8::el8"
+      ]
+    }
+  }
+}

redhat/oval/testdata/missing-oval/PULP_MANIFEST

@@ -0,0 +1,3 @@
+RHEL6/rhel-6-extras-including-unpatched.oval.xml.bz2,6a4e05e6c5ef90d23c4e1752ac5bf247e075d17b76c0f60f6aa02e68daa5a6a4,6347
+RHEL7/dotnet-3.1-including-unpatched.oval.xml.bz2,3afa7c45c0ccd21444a61f236c40a417386e09ce278cf85b049bc79d02d3c493,5817
+RHEL8/ansible-2-including-unpatched.oval.xml.bz2,d695814ee6ac7de65106ae26d7da85e6efe68ea5e951fefef5f2327393e3041c,5501

redhat/oval/testdata/missing-oval/repository-to-cpe.json

@@ -0,0 +1,18 @@
+{
+  "data": {
+    "3scale-amp-2-for-rhel-8-ppc64le-debug-rpms": {
+      "cpes": [
+        "cpe:/a:redhat:3scale_amp:2.11::el8",
+        "cpe:/a:redhat:3scale_amp:2.12::el8",
+        "cpe:/a:redhat:3scale_amp:2.8::el8"
+      ]
+    },
+    "3scale-amp-2-for-rhel-8-ppc64le-rpms": {
+      "cpes": [
+        "cpe:/a:redhat:3scale_amp:2.11::el8",
+        "cpe:/a:redhat:3scale_amp:2.12::el8",
+        "cpe:/a:redhat:3scale_amp:2.8::el8"
+      ]
+    }
+  }
+}

redhat/oval/types.go

@@ -298,3 +298,9 @@ type UnameState struct {
 	Version   string    `xml:"version,attr" json:",omitempty"`
 	OsRelease OsRelease `xml:"os_release" json:",omitempty"`
 }
+
+type repositoryToCPE struct {
+	Data map[string]struct {
+		Cpes []string `json:"cpes"`
+	} `json:"data"`
+}