fix(rocky): use minor part in version number (#154)

Co-authored-by: knqyf263 <knqyf263@gmail.com>

rocky/rocky.go

@@ -11,8 +11,10 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
+	"github.com/PuerkitoBio/goquery"
 	"github.com/aquasecurity/vuln-list-update/utils"
 	"github.com/cheggaaa/pb/v3"
 	"golang.org/x/xerrors"
@@ -24,10 +26,12 @@ const (
 )
 
 var (
-	urlFormat       = "https://download.rockylinux.org/pub/rocky/%s/%s/%s/os/"
+	baseUrl       = "https://download.rockylinux.org/pub/rocky"
-	defaultReleases = []string{"8"}
+	urlFormat     = "%s/%s/%s/%s/os/"
-	defaultRepos    = []string{"BaseOS", "AppStream", "extras"}
+	defaultRepos  = []string{"BaseOS", "AppStream", "extras"}
-	defaultArches   = []string{"x86_64", "aarch64"}
+	defaultArches = []string{"x86_64", "aarch64"}
+
+	releaseRegex = regexp.MustCompile(`\d+.\d+`)
 )
 
 // RepoMd has repomd data
@@ -89,22 +93,22 @@ type Package struct {
 }
 
 type options struct {
-	url      string
+	baseUrl   string
-	dir      string
+	urlFormat string
-	retry    int
+	dir       string
-	releases []string
+	retry     int
-	repos    []string
+	repos     []string
-	arches   []string
+	arches    []string
 }
 
 type option func(*options)
 
-func With(url, dir string, retry int, releases, repos, arches []string) option {
+func With(baseUrl, urlFormat, dir string, retry int, repos, arches []string) option {
 	return func(opts *options) {
-		opts.url = url
+		opts.baseUrl = baseUrl
+		opts.urlFormat = urlFormat
 		opts.dir = dir
 		opts.retry = retry
-		opts.releases = releases
 		opts.repos = repos
 		opts.arches = arches
 	}
@@ -116,12 +120,12 @@ type Config struct {
 
 func NewConfig(opts ...option) Config {
 	o := &options{
-		url:      urlFormat,
+		baseUrl:   baseUrl,
-		dir:      filepath.Join(utils.VulnListDir(), rockyDir),
+		urlFormat: urlFormat,
-		retry:    retry,
+		dir:       filepath.Join(utils.VulnListDir(), rockyDir),
-		releases: defaultReleases,
+		retry:     retry,
-		repos:    defaultRepos,
+		repos:     defaultRepos,
-		arches:   defaultArches,
+		arches:    defaultArches,
 	}
 	for _, opt := range opts {
 		opt(o)
@@ -133,11 +137,17 @@ func NewConfig(opts ...option) Config {
 }
 
 func (c Config) Update() error {
-	for _, release := range c.releases {
+	// "8" is an alias of the latest release that doesn't contain old security advisories,
+	// so we have to get all available minor releases like 8.5 and 8.6 so that we can have all the advisories.
+	releases, err := c.getReleasesList()
+	if err != nil {
+		return xerrors.Errorf("failed to get a list of Rocky Linux releases: %w", err)
+	}
+	for _, release := range releases {
 		for _, repo := range c.repos {
 			for _, arch := range c.arches {
 				log.Printf("Fetching Rocky Linux %s %s %s data...", release, repo, arch)
-				if err := c.update(release, repo, arch); err != nil {
+				if err = c.update(release, repo, arch); err != nil {
 					return xerrors.Errorf("failed to update security advisories of Rocky Linux %s %s %s: %w", release, repo, arch, err)
 				}
 			}
@@ -156,7 +166,7 @@ func (c Config) update(release, repo, arch string) error {
 		return xerrors.Errorf("failed to mkdir: %w", err)
 	}
 
-	u, err := url.Parse(fmt.Sprintf(c.url, release, repo, arch))
+	u, err := url.Parse(fmt.Sprintf(c.urlFormat, c.baseUrl, release, repo, arch))
 	if err != nil {
 		return xerrors.Errorf("failed to parse root url: %w", err)
 	}
@@ -188,14 +198,14 @@ func (c Config) update(release, repo, arch string) error {
 	for year, errata := range secErrata {
 		log.Printf("Write Errata for Rocky Linux %s %s %s %s", release, repo, arch, year)
 
-		if err := os.MkdirAll(filepath.Join(dirPath, year), os.ModePerm); err != nil {
+		if err = os.MkdirAll(filepath.Join(dirPath, year), os.ModePerm); err != nil {
 			return xerrors.Errorf("failed to mkdir: %w", err)
 		}
 
 		bar := pb.StartNew(len(errata))
 		for _, erratum := range errata {
 			jsonPath := filepath.Join(dirPath, year, fmt.Sprintf("%s.json", erratum.ID))
-			if err := utils.Write(jsonPath, erratum); err != nil {
+			if err = utils.Write(jsonPath, erratum); err != nil {
 				return xerrors.Errorf("failed to write Rocky Linux CVE details: %w", err)
 			}
 			bar.Increment()
@@ -215,7 +225,7 @@ func (c Config) fetchUpdateInfoPath(repomdURL string) (updateInfoPath string, er
 	}
 
 	var repoMd RepoMd
-	if err := xml.NewDecoder(bytes.NewBuffer(res)).Decode(&repoMd); err != nil {
+	if err = xml.NewDecoder(bytes.NewBuffer(res)).Decode(&repoMd); err != nil {
 		return "", xerrors.Errorf("failed to decode repomd.xml: %w", err)
 	}
 
@@ -239,7 +249,7 @@ func (c Config) fetchUpdateInfo(url string) (*UpdateInfo, error) {
 	defer r.Close()
 
 	var updateInfo UpdateInfo
-	if err := xml.NewDecoder(r).Decode(&updateInfo); err != nil {
+	if err = xml.NewDecoder(r).Decode(&updateInfo); err != nil {
 		return nil, err
 	}
 	for i, alas := range updateInfo.RLSAList {
@@ -253,3 +263,27 @@ func (c Config) fetchUpdateInfo(url string) (*UpdateInfo, error) {
 	}
 	return &updateInfo, nil
 }
+
+func (c Config) getReleasesList() ([]string, error) {
+	b, err := utils.FetchURL(c.baseUrl, "", c.retry)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to get list of releases: %w", err)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(b))
+	if err != nil {
+		return nil, xerrors.Errorf("failed to read list of releases: %w", err)
+	}
+
+	var releases []string
+	doc.Find("a").Each(func(i int, s *goquery.Selection) {
+		if release := releaseRegex.FindString(s.Text()); release != "" {
+			releases = append(releases, release)
+		}
+	})
+
+	if len(releases) == 0 {
+		return nil, xerrors.Errorf("failed to get list of releases: list is empty")
+	}
+	return releases, nil
+}

rocky/rocky_test.go

@@ -1,7 +1,6 @@
 package rocky_test
 
 import (
-	"errors"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -11,57 +10,72 @@ import (
 	"github.com/aquasecurity/vuln-list-update/rocky"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"golang.org/x/xerrors"
 )
 
 func Test_Update(t *testing.T) {
 	tests := []struct {
-		name          string
+		name             string
-		rootDir       string
+		releasesFilePath string
-		repository    []string
+		rootDir          string
-		expectedError error
+		repository       []string
+		wantErr          string
 	}{
 		{
-			name:          "happy path",
+			name:             "happy path",
-			rootDir:       "testdata/fixtures/happy",
+			releasesFilePath: "testdata/fixtures/releases/happy.html",
-			repository:    []string{"BaseOS"},
+			rootDir:          "testdata/fixtures/happy",
-			expectedError: nil,
+			repository:       []string{"BaseOS"},
 		},
 		{
-			name:          "bad repomd response",
+			name:             "bad repomd response",
-			rootDir:       "testdata/fixtures/repomd_invalid",
+			releasesFilePath: "testdata/fixtures/releases/happy.html",
-			repository:    []string{"BaseOS"},
+			rootDir:          "testdata/fixtures/repomd_invalid",
-			expectedError: xerrors.Errorf("failed to update security advisories of Rocky Linux 8 BaseOS x86_64: %w", errors.New("failed to fetch updateInfo path from repomd.xml")),
+			repository:       []string{"BaseOS"},
+			wantErr:          "failed to fetch updateInfo path from repomd.xml",
 		},
 		{
-			name:          "bad updateInfo response",
+			name:             "bad updateInfo response",
-			rootDir:       "testdata/fixtures/updateinfo_invalid",
+			releasesFilePath: "testdata/fixtures/releases/happy.html",
-			repository:    []string{"BaseOS"},
+			rootDir:          "testdata/fixtures/updateinfo_invalid",
-			expectedError: xerrors.Errorf("failed to update security advisories of Rocky Linux 8 BaseOS x86_64: %w", errors.New("failed to fetch updateInfo")),
+			repository:       []string{"BaseOS"},
+			wantErr:          "failed to fetch updateInfo",
 		},
 		{
-			name:          "no updateInfo field(BaseOS)",
+			name:             "no updateInfo field(BaseOS)",
-			rootDir:       "testdata/fixtures/no_updateinfo_field",
+			releasesFilePath: "testdata/fixtures/releases/happy.html",
-			repository:    []string{"BaseOS"},
+			rootDir:          "testdata/fixtures/no_updateinfo_field",
-			expectedError: xerrors.Errorf("failed to update security advisories of Rocky Linux 8 BaseOS x86_64: %w", xerrors.Errorf("failed to fetch updateInfo path from repomd.xml: %w", rocky.ErrorNoUpdateInfoField)),
+			repository:       []string{"BaseOS"},
+			wantErr:          rocky.ErrorNoUpdateInfoField.Error(),
 		},
 		{
-			name:          "no updateInfo field(extras)",
+			name:             "no updateInfo field(extras)",
-			rootDir:       "testdata/fixtures/no_updateinfo_field",
+			releasesFilePath: "testdata/fixtures/releases/happy.html",
-			repository:    []string{"extras"},
+			rootDir:          "testdata/fixtures/no_updateinfo_field",
-			expectedError: nil,
+			repository:       []string{"extras"},
+		},
+		{
+			name:             "empty list of releases",
+			releasesFilePath: "testdata/fixtures/releases/empty.html",
+			repository:       []string{"BaseOS"},
+			wantErr:          "list is empty",
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tsUpdateInfoURL := httptest.NewServer(http.StripPrefix("/pub/rocky/8/BaseOS/x86_64/os/repodata/", http.FileServer(http.Dir(tt.rootDir))))
+			mux := http.NewServeMux()
+			mux.Handle("/pub/rocky/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				http.ServeFile(w, r, tt.releasesFilePath)
+			}))
+			mux.Handle("/pub/rocky/8.5/extras/x86_64/os/repodata/", http.StripPrefix("/pub/rocky/8.5/extras/x86_64/os/repodata/", http.FileServer(http.Dir(tt.rootDir))))
+			mux.Handle("/pub/rocky/8.5/BaseOS/x86_64/os/repodata/", http.StripPrefix("/pub/rocky/8.5/BaseOS/x86_64/os/repodata/", http.FileServer(http.Dir(tt.rootDir))))
+			tsUpdateInfoURL := httptest.NewServer(mux)
 			defer tsUpdateInfoURL.Close()
 
 			dir := t.TempDir()
-			rc := rocky.NewConfig(rocky.With(tsUpdateInfoURL.URL+"/pub/rocky/%s/%s/%s/os/", dir, 0, []string{"8"}, tt.repository, []string{"x86_64"}))
+			rc := rocky.NewConfig(rocky.With(tsUpdateInfoURL.URL+"/pub/rocky", "%s/%s/%s/%s/os/", dir, 0, tt.repository, []string{"x86_64"}))
-			if err := rc.Update(); tt.expectedError != nil {
+			if err := rc.Update(); tt.wantErr != "" {
 				require.Error(t, err)
-				assert.Contains(t, err.Error(), tt.expectedError.Error())
+				assert.Contains(t, err.Error(), tt.wantErr)
 				return
 			}
 

rocky/testdata/fixtures/releases/empty.html

@@ -0,0 +1,13 @@
+<html>
+
+<head>
+    <title>Index of /pub/rocky/</title>
+</head>
+
+<body bgcolor="white">
+<h1>Index of /pub/rocky/</h1>
+<hr>
+<hr>
+</body>
+
+</html>

rocky/testdata/fixtures/releases/happy.html

@@ -0,0 +1,17 @@
+<html>
+
+<head>
+    <title>Index of /pub/rocky/</title>
+</head>
+
+<body bgcolor="white">
+<h1>Index of /pub/rocky/</h1>
+<hr>
+<pre><a href="../">../</a>
+<a href="8/">8/</a>
+<a href="8.5/">8.5/</a>
+</pre>
+<hr>
+</body>
+
+</html>