pepelyaevip/vuln-list-update
1
0
Fork 0
Code Pull Requests Actions Packages Releases Activity

fix(rocky): use minor part in version number (#154)

Browse Source 
Co-authored-by: knqyf263 <knqyf263@gmail.com>
This commit is contained in:
DmitriyLewen 2022-05-26 10:31:24 +06:00 committed by GitHub
parent e803f5cdec
commit b19cb98d6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 134 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
rocky/rocky.go
View File

@ -11,8 +11,10 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/aquasecurity/vuln-list-update/utils"
"github.com/cheggaaa/pb/v3"
"golang.org/x/xerrors"
@ -24,10 +26,12 @@ const (
)
var (
urlFormat = "https://download.rockylinux.org/pub/rocky/%s/%s/%s/os/"
defaultReleases = []string{"8"}
baseUrl = "https://download.rockylinux.org/pub/rocky"
urlFormat = "%s/%s/%s/%s/os/"
defaultRepos = []string{"BaseOS", "AppStream", "extras"}
defaultArches = []string{"x86_64", "aarch64"}
releaseRegex = regexp.MustCompile(`\d+.\d+`)
)
// RepoMd has repomd data
@ -89,22 +93,22 @@ type Package struct {
}
type options struct {
url string
baseUrl string
urlFormat string
dir string
retry int
releases []string
repos []string
arches []string
}
type option func(*options)
func With(url, dir string, retry int, releases, repos, arches []string) option {
func With(baseUrl, urlFormat, dir string, retry int, repos, arches []string) option {
return func(opts *options) {
opts.url = url
opts.baseUrl = baseUrl
opts.urlFormat = urlFormat
opts.dir = dir
opts.retry = retry
opts.releases = releases
opts.repos = repos
opts.arches = arches
}
@ -116,10 +120,10 @@ type Config struct {
func NewConfig(opts ...option) Config {
o := &options{
url: urlFormat,
baseUrl: baseUrl,
urlFormat: urlFormat,
dir: filepath.Join(utils.VulnListDir(), rockyDir),
retry: retry,
releases: defaultReleases,
repos: defaultRepos,
arches: defaultArches,
}
@ -133,11 +137,17 @@ func NewConfig(opts ...option) Config {
}
func (c Config) Update() error {
for _, release := range c.releases {
// "8" is an alias of the latest release that doesn't contain old security advisories,
// so we have to get all available minor releases like 8.5 and 8.6 so that we can have all the advisories.
releases, err := c.getReleasesList()
if err != nil {
return xerrors.Errorf("failed to get a list of Rocky Linux releases: %w", err)
}
for _, release := range releases {
for _, repo := range c.repos {
for _, arch := range c.arches {
log.Printf("Fetching Rocky Linux %s %s %s data...", release, repo, arch)
if err := c.update(release, repo, arch); err != nil {
if err = c.update(release, repo, arch); err != nil {
return xerrors.Errorf("failed to update security advisories of Rocky Linux %s %s %s: %w", release, repo, arch, err)
}
}
@ -156,7 +166,7 @@ func (c Config) update(release, repo, arch string) error {
return xerrors.Errorf("failed to mkdir: %w", err)
}
u, err := url.Parse(fmt.Sprintf(c.url, release, repo, arch))
u, err := url.Parse(fmt.Sprintf(c.urlFormat, c.baseUrl, release, repo, arch))
if err != nil {
return xerrors.Errorf("failed to parse root url: %w", err)
}
@ -188,14 +198,14 @@ func (c Config) update(release, repo, arch string) error {
for year, errata := range secErrata {
log.Printf("Write Errata for Rocky Linux %s %s %s %s", release, repo, arch, year)
if err := os.MkdirAll(filepath.Join(dirPath, year), os.ModePerm); err != nil {
if err = os.MkdirAll(filepath.Join(dirPath, year), os.ModePerm); err != nil {
return xerrors.Errorf("failed to mkdir: %w", err)
}
bar := pb.StartNew(len(errata))
for _, erratum := range errata {
jsonPath := filepath.Join(dirPath, year, fmt.Sprintf("%s.json", erratum.ID))
if err := utils.Write(jsonPath, erratum); err != nil {
if err = utils.Write(jsonPath, erratum); err != nil {
return xerrors.Errorf("failed to write Rocky Linux CVE details: %w", err)
}
bar.Increment()
@ -215,7 +225,7 @@ func (c Config) fetchUpdateInfoPath(repomdURL string) (updateInfoPath string, er
}
var repoMd RepoMd
if err := xml.NewDecoder(bytes.NewBuffer(res)).Decode(&repoMd); err != nil {
if err = xml.NewDecoder(bytes.NewBuffer(res)).Decode(&repoMd); err != nil {
return "", xerrors.Errorf("failed to decode repomd.xml: %w", err)
}
@ -239,7 +249,7 @@ func (c Config) fetchUpdateInfo(url string) (*UpdateInfo, error) {
defer r.Close()
var updateInfo UpdateInfo
if err := xml.NewDecoder(r).Decode(&updateInfo); err != nil {
if err = xml.NewDecoder(r).Decode(&updateInfo); err != nil {
return nil, err
}
for i, alas := range updateInfo.RLSAList {
@ -253,3 +263,27 @@ func (c Config) fetchUpdateInfo(url string) (*UpdateInfo, error) {
}
return &updateInfo, nil
}
func (c Config) getReleasesList() ([]string, error) {
b, err := utils.FetchURL(c.baseUrl, "", c.retry)
if err != nil {
return nil, xerrors.Errorf("failed to get list of releases: %w", err)
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(b))
if err != nil {
return nil, xerrors.Errorf("failed to read list of releases: %w", err)
}
var releases []string
doc.Find("a").Each(func(i int, s *goquery.Selection) {
if release := releaseRegex.FindString(s.Text()); release != "" {
releases = append(releases, release)
}
})
if len(releases) == 0 {
return nil, xerrors.Errorf("failed to get list of releases: list is empty")
}
return releases, nil
}

38
rocky/rocky_test.go
View File

@ -1,7 +1,6 @@
package rocky_test
import (
"errors"
"net/http"
"net/http/httptest"
"os"
@ -11,57 +10,72 @@ import (
"github.com/aquasecurity/vuln-list-update/rocky"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
)
func Test_Update(t *testing.T) {
tests := []struct {
name string
releasesFilePath string
rootDir string
repository []string
expectedError error
wantErr string
}{
{
name: "happy path",
releasesFilePath: "testdata/fixtures/releases/happy.html",
rootDir: "testdata/fixtures/happy",
repository: []string{"BaseOS"},
expectedError: nil,
},
{
name: "bad repomd response",
releasesFilePath: "testdata/fixtures/releases/happy.html",
rootDir: "testdata/fixtures/repomd_invalid",
repository: []string{"BaseOS"},
expectedError: xerrors.Errorf("failed to update security advisories of Rocky Linux 8 BaseOS x86_64: %w", errors.New("failed to fetch updateInfo path from repomd.xml")),
wantErr: "failed to fetch updateInfo path from repomd.xml",
},
{
name: "bad updateInfo response",
releasesFilePath: "testdata/fixtures/releases/happy.html",
rootDir: "testdata/fixtures/updateinfo_invalid",
repository: []string{"BaseOS"},
expectedError: xerrors.Errorf("failed to update security advisories of Rocky Linux 8 BaseOS x86_64: %w", errors.New("failed to fetch updateInfo")),
wantErr: "failed to fetch updateInfo",
},
{
name: "no updateInfo field(BaseOS)",
releasesFilePath: "testdata/fixtures/releases/happy.html",
rootDir: "testdata/fixtures/no_updateinfo_field",
repository: []string{"BaseOS"},
expectedError: xerrors.Errorf("failed to update security advisories of Rocky Linux 8 BaseOS x86_64: %w", xerrors.Errorf("failed to fetch updateInfo path from repomd.xml: %w", rocky.ErrorNoUpdateInfoField)),
wantErr: rocky.ErrorNoUpdateInfoField.Error(),
},
{
name: "no updateInfo field(extras)",
releasesFilePath: "testdata/fixtures/releases/happy.html",
rootDir: "testdata/fixtures/no_updateinfo_field",
repository: []string{"extras"},
expectedError: nil,
},
{
name: "empty list of releases",
releasesFilePath: "testdata/fixtures/releases/empty.html",
repository: []string{"BaseOS"},
wantErr: "list is empty",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tsUpdateInfoURL := httptest.NewServer(http.StripPrefix("/pub/rocky/8/BaseOS/x86_64/os/repodata/", http.FileServer(http.Dir(tt.rootDir))))
mux := http.NewServeMux()
mux.Handle("/pub/rocky/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, tt.releasesFilePath)
}))
mux.Handle("/pub/rocky/8.5/extras/x86_64/os/repodata/", http.StripPrefix("/pub/rocky/8.5/extras/x86_64/os/repodata/", http.FileServer(http.Dir(tt.rootDir))))
mux.Handle("/pub/rocky/8.5/BaseOS/x86_64/os/repodata/", http.StripPrefix("/pub/rocky/8.5/BaseOS/x86_64/os/repodata/", http.FileServer(http.Dir(tt.rootDir))))
tsUpdateInfoURL := httptest.NewServer(mux)
defer tsUpdateInfoURL.Close()
dir := t.TempDir()
rc := rocky.NewConfig(rocky.With(tsUpdateInfoURL.URL+"/pub/rocky/%s/%s/%s/os/", dir, 0, []string{"8"}, tt.repository, []string{"x86_64"}))
if err := rc.Update(); tt.expectedError != nil {
rc := rocky.NewConfig(rocky.With(tsUpdateInfoURL.URL+"/pub/rocky", "%s/%s/%s/%s/os/", dir, 0, tt.repository, []string{"x86_64"}))
if err := rc.Update(); tt.wantErr != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), tt.expectedError.Error())
assert.Contains(t, err.Error(), tt.wantErr)
return
}

13
rocky/testdata/fixtures/releases/empty.html vendored Normal file
View File

@ -0,0 +1,13 @@
<html>
<head>
<title>Index of /pub/rocky/</title>
</head>
<body bgcolor="white">
<h1>Index of /pub/rocky/</h1>
<hr>
<hr>
</body>
</html>

17
rocky/testdata/fixtures/releases/happy.html vendored Normal file
View File

@ -0,0 +1,17 @@
<html>
<head>
<title>Index of /pub/rocky/</title>
</head>
<body bgcolor="white">
<h1>Index of /pub/rocky/</h1>
<hr>
<pre><a href="../">../</a>
<a href="8/">8/</a>
<a href="8.5/">8.5/</a>
</pre>
<hr>
</body>
</html>