From b19cb98d6ac9bf5f595c70f10f58c30ef9b589c2 Mon Sep 17 00:00:00 2001 From: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Date: Thu, 26 May 2022 10:31:24 +0600 Subject: [PATCH] fix(rocky): use minor part in version number (#154) Co-authored-by: knqyf263 --- rocky/rocky.go | 86 ++++++++++++++------- rocky/rocky_test.go | 74 +++++++++++------- rocky/testdata/fixtures/releases/empty.html | 13 ++++ rocky/testdata/fixtures/releases/happy.html | 17 ++++ 4 files changed, 134 insertions(+), 56 deletions(-) create mode 100644 rocky/testdata/fixtures/releases/empty.html create mode 100644 rocky/testdata/fixtures/releases/happy.html diff --git a/rocky/rocky.go b/rocky/rocky.go index 84dbf65..732c278 100644 --- a/rocky/rocky.go +++ b/rocky/rocky.go @@ -11,8 +11,10 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" + "github.com/PuerkitoBio/goquery" "github.com/aquasecurity/vuln-list-update/utils" "github.com/cheggaaa/pb/v3" "golang.org/x/xerrors" @@ -24,10 +26,12 @@ const ( ) var ( - urlFormat = "https://download.rockylinux.org/pub/rocky/%s/%s/%s/os/" - defaultReleases = []string{"8"} - defaultRepos = []string{"BaseOS", "AppStream", "extras"} - defaultArches = []string{"x86_64", "aarch64"} + baseUrl = "https://download.rockylinux.org/pub/rocky" + urlFormat = "%s/%s/%s/%s/os/" + defaultRepos = []string{"BaseOS", "AppStream", "extras"} + defaultArches = []string{"x86_64", "aarch64"} + + releaseRegex = regexp.MustCompile(`\d+.\d+`) ) // RepoMd has repomd data @@ -89,22 +93,22 @@ type Package struct { } type options struct { - url string - dir string - retry int - releases []string - repos []string - arches []string + baseUrl string + urlFormat string + dir string + retry int + repos []string + arches []string } type option func(*options) -func With(url, dir string, retry int, releases, repos, arches []string) option { +func With(baseUrl, urlFormat, dir string, retry int, repos, arches []string) option { return func(opts *options) { - opts.url = url + opts.baseUrl = baseUrl + opts.urlFormat = urlFormat opts.dir = dir opts.retry = retry - opts.releases = releases opts.repos = repos opts.arches = arches } @@ -116,12 +120,12 @@ type Config struct { func NewConfig(opts ...option) Config { o := &options{ - url: urlFormat, - dir: filepath.Join(utils.VulnListDir(), rockyDir), - retry: retry, - releases: defaultReleases, - repos: defaultRepos, - arches: defaultArches, + baseUrl: baseUrl, + urlFormat: urlFormat, + dir: filepath.Join(utils.VulnListDir(), rockyDir), + retry: retry, + repos: defaultRepos, + arches: defaultArches, } for _, opt := range opts { opt(o) @@ -133,11 +137,17 @@ func NewConfig(opts ...option) Config { } func (c Config) Update() error { - for _, release := range c.releases { + // "8" is an alias of the latest release that doesn't contain old security advisories, + // so we have to get all available minor releases like 8.5 and 8.6 so that we can have all the advisories. + releases, err := c.getReleasesList() + if err != nil { + return xerrors.Errorf("failed to get a list of Rocky Linux releases: %w", err) + } + for _, release := range releases { for _, repo := range c.repos { for _, arch := range c.arches { log.Printf("Fetching Rocky Linux %s %s %s data...", release, repo, arch) - if err := c.update(release, repo, arch); err != nil { + if err = c.update(release, repo, arch); err != nil { return xerrors.Errorf("failed to update security advisories of Rocky Linux %s %s %s: %w", release, repo, arch, err) } } @@ -156,7 +166,7 @@ func (c Config) update(release, repo, arch string) error { return xerrors.Errorf("failed to mkdir: %w", err) } - u, err := url.Parse(fmt.Sprintf(c.url, release, repo, arch)) + u, err := url.Parse(fmt.Sprintf(c.urlFormat, c.baseUrl, release, repo, arch)) if err != nil { return xerrors.Errorf("failed to parse root url: %w", err) } @@ -188,14 +198,14 @@ func (c Config) update(release, repo, arch string) error { for year, errata := range secErrata { log.Printf("Write Errata for Rocky Linux %s %s %s %s", release, repo, arch, year) - if err := os.MkdirAll(filepath.Join(dirPath, year), os.ModePerm); err != nil { + if err = os.MkdirAll(filepath.Join(dirPath, year), os.ModePerm); err != nil { return xerrors.Errorf("failed to mkdir: %w", err) } bar := pb.StartNew(len(errata)) for _, erratum := range errata { jsonPath := filepath.Join(dirPath, year, fmt.Sprintf("%s.json", erratum.ID)) - if err := utils.Write(jsonPath, erratum); err != nil { + if err = utils.Write(jsonPath, erratum); err != nil { return xerrors.Errorf("failed to write Rocky Linux CVE details: %w", err) } bar.Increment() @@ -215,7 +225,7 @@ func (c Config) fetchUpdateInfoPath(repomdURL string) (updateInfoPath string, er } var repoMd RepoMd - if err := xml.NewDecoder(bytes.NewBuffer(res)).Decode(&repoMd); err != nil { + if err = xml.NewDecoder(bytes.NewBuffer(res)).Decode(&repoMd); err != nil { return "", xerrors.Errorf("failed to decode repomd.xml: %w", err) } @@ -239,7 +249,7 @@ func (c Config) fetchUpdateInfo(url string) (*UpdateInfo, error) { defer r.Close() var updateInfo UpdateInfo - if err := xml.NewDecoder(r).Decode(&updateInfo); err != nil { + if err = xml.NewDecoder(r).Decode(&updateInfo); err != nil { return nil, err } for i, alas := range updateInfo.RLSAList { @@ -253,3 +263,27 @@ func (c Config) fetchUpdateInfo(url string) (*UpdateInfo, error) { } return &updateInfo, nil } + +func (c Config) getReleasesList() ([]string, error) { + b, err := utils.FetchURL(c.baseUrl, "", c.retry) + if err != nil { + return nil, xerrors.Errorf("failed to get list of releases: %w", err) + } + + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(b)) + if err != nil { + return nil, xerrors.Errorf("failed to read list of releases: %w", err) + } + + var releases []string + doc.Find("a").Each(func(i int, s *goquery.Selection) { + if release := releaseRegex.FindString(s.Text()); release != "" { + releases = append(releases, release) + } + }) + + if len(releases) == 0 { + return nil, xerrors.Errorf("failed to get list of releases: list is empty") + } + return releases, nil +} diff --git a/rocky/rocky_test.go b/rocky/rocky_test.go index 706d67c..a04beb4 100644 --- a/rocky/rocky_test.go +++ b/rocky/rocky_test.go @@ -1,7 +1,6 @@ package rocky_test import ( - "errors" "net/http" "net/http/httptest" "os" @@ -11,57 +10,72 @@ import ( "github.com/aquasecurity/vuln-list-update/rocky" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/xerrors" ) func Test_Update(t *testing.T) { tests := []struct { - name string - rootDir string - repository []string - expectedError error + name string + releasesFilePath string + rootDir string + repository []string + wantErr string }{ { - name: "happy path", - rootDir: "testdata/fixtures/happy", - repository: []string{"BaseOS"}, - expectedError: nil, + name: "happy path", + releasesFilePath: "testdata/fixtures/releases/happy.html", + rootDir: "testdata/fixtures/happy", + repository: []string{"BaseOS"}, }, { - name: "bad repomd response", - rootDir: "testdata/fixtures/repomd_invalid", - repository: []string{"BaseOS"}, - expectedError: xerrors.Errorf("failed to update security advisories of Rocky Linux 8 BaseOS x86_64: %w", errors.New("failed to fetch updateInfo path from repomd.xml")), + name: "bad repomd response", + releasesFilePath: "testdata/fixtures/releases/happy.html", + rootDir: "testdata/fixtures/repomd_invalid", + repository: []string{"BaseOS"}, + wantErr: "failed to fetch updateInfo path from repomd.xml", }, { - name: "bad updateInfo response", - rootDir: "testdata/fixtures/updateinfo_invalid", - repository: []string{"BaseOS"}, - expectedError: xerrors.Errorf("failed to update security advisories of Rocky Linux 8 BaseOS x86_64: %w", errors.New("failed to fetch updateInfo")), + name: "bad updateInfo response", + releasesFilePath: "testdata/fixtures/releases/happy.html", + rootDir: "testdata/fixtures/updateinfo_invalid", + repository: []string{"BaseOS"}, + wantErr: "failed to fetch updateInfo", }, { - name: "no updateInfo field(BaseOS)", - rootDir: "testdata/fixtures/no_updateinfo_field", - repository: []string{"BaseOS"}, - expectedError: xerrors.Errorf("failed to update security advisories of Rocky Linux 8 BaseOS x86_64: %w", xerrors.Errorf("failed to fetch updateInfo path from repomd.xml: %w", rocky.ErrorNoUpdateInfoField)), + name: "no updateInfo field(BaseOS)", + releasesFilePath: "testdata/fixtures/releases/happy.html", + rootDir: "testdata/fixtures/no_updateinfo_field", + repository: []string{"BaseOS"}, + wantErr: rocky.ErrorNoUpdateInfoField.Error(), }, { - name: "no updateInfo field(extras)", - rootDir: "testdata/fixtures/no_updateinfo_field", - repository: []string{"extras"}, - expectedError: nil, + name: "no updateInfo field(extras)", + releasesFilePath: "testdata/fixtures/releases/happy.html", + rootDir: "testdata/fixtures/no_updateinfo_field", + repository: []string{"extras"}, + }, + { + name: "empty list of releases", + releasesFilePath: "testdata/fixtures/releases/empty.html", + repository: []string{"BaseOS"}, + wantErr: "list is empty", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tsUpdateInfoURL := httptest.NewServer(http.StripPrefix("/pub/rocky/8/BaseOS/x86_64/os/repodata/", http.FileServer(http.Dir(tt.rootDir)))) + mux := http.NewServeMux() + mux.Handle("/pub/rocky/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, tt.releasesFilePath) + })) + mux.Handle("/pub/rocky/8.5/extras/x86_64/os/repodata/", http.StripPrefix("/pub/rocky/8.5/extras/x86_64/os/repodata/", http.FileServer(http.Dir(tt.rootDir)))) + mux.Handle("/pub/rocky/8.5/BaseOS/x86_64/os/repodata/", http.StripPrefix("/pub/rocky/8.5/BaseOS/x86_64/os/repodata/", http.FileServer(http.Dir(tt.rootDir)))) + tsUpdateInfoURL := httptest.NewServer(mux) defer tsUpdateInfoURL.Close() dir := t.TempDir() - rc := rocky.NewConfig(rocky.With(tsUpdateInfoURL.URL+"/pub/rocky/%s/%s/%s/os/", dir, 0, []string{"8"}, tt.repository, []string{"x86_64"})) - if err := rc.Update(); tt.expectedError != nil { + rc := rocky.NewConfig(rocky.With(tsUpdateInfoURL.URL+"/pub/rocky", "%s/%s/%s/%s/os/", dir, 0, tt.repository, []string{"x86_64"})) + if err := rc.Update(); tt.wantErr != "" { require.Error(t, err) - assert.Contains(t, err.Error(), tt.expectedError.Error()) + assert.Contains(t, err.Error(), tt.wantErr) return } diff --git a/rocky/testdata/fixtures/releases/empty.html b/rocky/testdata/fixtures/releases/empty.html new file mode 100644 index 0000000..0e069dc --- /dev/null +++ b/rocky/testdata/fixtures/releases/empty.html @@ -0,0 +1,13 @@ + + + + Index of /pub/rocky/ + + + +

Index of /pub/rocky/

+
+
+ + + \ No newline at end of file diff --git a/rocky/testdata/fixtures/releases/happy.html b/rocky/testdata/fixtures/releases/happy.html new file mode 100644 index 0000000..9674a02 --- /dev/null +++ b/rocky/testdata/fixtures/releases/happy.html @@ -0,0 +1,17 @@ + + + + Index of /pub/rocky/ + + + +

Index of /pub/rocky/

+
+
../
+8/
+8.5/
+
+
+ + + \ No newline at end of file