diff --git a/debian/debian.go b/debian/debian.go index d4ac5d7..c410e8f 100644 --- a/debian/debian.go +++ b/debian/debian.go @@ -13,24 +13,47 @@ import ( ) const ( - debianDir = "debian" + debianDir = "debian" + securityTrackerURL = "https://security-tracker.debian.org/tracker/data/json" + retry = 5 ) type DebianJSON map[string]DebianCveMap type DebianCveMap map[string]interface{} -func Update() error { +type Client struct { + URL string + VulnListDir string + Retry int +} + +func NewClient() *Client { + return &Client{ + URL: securityTrackerURL, + VulnListDir: utils.VulnListDir(), + Retry: retry, + } +} + +func (dc Client) Update() error { log.Println("Fetching Debian data...") - vulns, err := retrieveDebianCveDetails() + vulns, err := dc.retrieveDebianCveDetails() if err != nil { return xerrors.Errorf("failed to retrieve Debian CVE details: %w", err) } + log.Println("Removing old data...") + if err = os.RemoveAll(filepath.Join(dc.VulnListDir, debianDir)); err != nil { + return xerrors.Errorf("failed to remove Debian dir: %w", err) + } + + // Save all JSON files + log.Println("Saving new data...") bar := pb.StartNew(len(vulns)) for pkgName, cves := range vulns { for cveID, cve := range cves { - dir := filepath.Join(utils.VulnListDir(), debianDir, pkgName) + dir := filepath.Join(dc.VulnListDir, debianDir, pkgName) if err := os.MkdirAll(dir, os.ModePerm); err != nil { return xerrors.Errorf("failed to create the directory: %w", err) } @@ -45,10 +68,8 @@ func Update() error { return nil } -// https://security-tracker.debian.org/tracker/data/json -func retrieveDebianCveDetails() (vulns DebianJSON, err error) { - url := "https://security-tracker.debian.org/tracker/data/json" - cveJSON, err := utils.FetchURL(url, "", 5) +func (dc Client) retrieveDebianCveDetails() (vulns DebianJSON, err error) { + cveJSON, err := utils.FetchURL(dc.URL, "", dc.Retry) if err != nil { return nil, xerrors.Errorf("failed to fetch cve data from Debian. err: %w", err) } diff --git a/debian/debian_test.go b/debian/debian_test.go new file mode 100644 index 0000000..d554a8c --- /dev/null +++ b/debian/debian_test.go @@ -0,0 +1,134 @@ +package debian_test + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "golang.org/x/xerrors" + + "github.com/aquasecurity/vuln-list-update/debian" + + "github.com/stretchr/testify/assert" +) + +func TestClient_Update(t *testing.T) { + testCases := []struct { + name string + version string + existedFiles []string + jsonFileName string + path string + expectedError string + }{ + { + name: "happy path", + jsonFileName: "testdata/fixtures/debian.json", + path: "debian.json", + expectedError: "", + }, + { + name: "remove old files", + existedFiles: []string{"CVE-0000-0000", "CVE-3000-0000"}, + jsonFileName: "testdata/fixtures/debian.json", + path: "debian.json", + expectedError: "", + }, + { + name: "invalid JSON", + jsonFileName: "testdata/fixtures/invalid.json", + path: "invalid.json", + expectedError: "invalid character 'i' looking for beginning of value", + }, + { + name: "404", + jsonFileName: "testdata/fixtures/debian.json", + path: "404.html", + expectedError: "HTTP error. status code: 404", + }, + } + + for _, tc := range testCases { + //t.Run(tc.name, func(t *testing.T) { + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, ".json"): + j, _ := ioutil.ReadFile(tc.jsonFileName) + _, _ = w.Write(j) + case strings.HasSuffix(r.URL.Path, "404.html"): + http.NotFound(w, r) + default: + assert.Fail(t, "bad URL requested: ", r.URL.Path, tc.name) + } + })) + defer testServer.Close() + + fmt.Println(path.Join(testServer.URL, tc.path)) + dir, err := ioutil.TempDir("", "debian") + assert.NoError(t, err, "failed to create temp dir") + defer os.RemoveAll(dir) + + // These files must be removed + if len(tc.existedFiles) > 0 { + d := filepath.Join(dir, "debian") + _ = os.Mkdir(d, 0777) + for _, fileName := range tc.existedFiles { + err = ioutil.WriteFile(filepath.Join(d, fileName), []byte("test"), 0666) + assert.Nil(t, err, "failed to write the file") + } + } + + u, err := url.Parse(testServer.URL) + assert.NoError(t, err, "URL parse error") + u.Path = path.Join(u.Path, tc.path) + + client := debian.Client{ + URL: u.String(), + VulnListDir: dir, + Retry: 0, + } + err = client.Update() + switch { + case tc.expectedError != "": + assert.Contains(t, err.Error(), tc.expectedError, tc.name) + default: + assert.NoError(t, err, tc.name) + } + + // TODO: Expose utils with an interface so this can self contain Write() + // Compare got and golden + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return xerrors.Errorf("walk error: %w", err) + } + if info.IsDir() { + return nil + } + + // Before: /var/folders/j7/pvz71jxn637dqd96gm80nhwm0000gn/T/debian676766850/debian/prototypejs/CVE-2007-2383.json + // After: testdata/goldens/debian/prototypejs/CVE-2007-2383.json.golden + paths := strings.Split(path, string(os.PathSeparator)) + p := filepath.Join(paths[len(paths)-3:]...) + golden := filepath.Join("testdata", "goldens", p+".golden") + + want, err := ioutil.ReadFile(golden) + assert.NoError(t, err, "failed to open the golden file") + + got, err := ioutil.ReadFile(path) + assert.NoError(t, err, "failed to open the result file") + + assert.Equal(t, string(want), string(got)) + + return nil + }) + assert.NoError(t, err, "filepath walk error") + //}) + } +} diff --git a/debian/testdata/fixtures/debian.json b/debian/testdata/fixtures/debian.json new file mode 100644 index 0000000..72b84b6 --- /dev/null +++ b/debian/testdata/fixtures/debian.json @@ -0,0 +1,145 @@ +{ + "prototypejs": { + "CVE-2008-7220": { + "scope": "remote", + "debianbug": 555217, + "description": "Unspecified vulnerability in Prototype JavaScript framework (prototypejs) before 1.6.0.2 allows attackers to make \"cross-site ajax requests\" via unknown vectors.", + "releases": { + "stretch": { + "status": "resolved", + "repositories": { + "stretch": "1.7.1-3" + }, + "fixed_version": "1.6.0.2-1", + "urgency": "high**" + }, + "jessie": { + "status": "resolved", + "repositories": { + "jessie": "1.7.1-3" + }, + "fixed_version": "1.6.0.2-1", + "urgency": "high**" + }, + "sid": { + "status": "resolved", + "repositories": { + "sid": "1.7.1-3" + }, + "fixed_version": "1.6.0.2-1", + "urgency": "high**" + }, + "bullseye": { + "status": "resolved", + "repositories": { + "bullseye": "1.7.1-3" + }, + "fixed_version": "1.6.0.2-1", + "urgency": "high**" + }, + "buster": { + "status": "resolved", + "repositories": { + "buster": "1.7.1-3" + }, + "fixed_version": "1.6.0.2-1", + "urgency": "high**" + } + } + }, + "CVE-2007-2383": { + "scope": "remote", + "debianbug": 555217, + "description": "The Prototype (prototypejs) framework before 1.5.1 RC3 exchanges data using JavaScript Object Notation (JSON) without an associated protection scheme, which allows remote attackers to obtain the data via a web page that retrieves the data through a URL in the SRC attribute of a SCRIPT element and captures the data using other JavaScript code, aka \"JavaScript Hijacking.\"", + "releases": { + "stretch": { + "status": "resolved", + "repositories": { + "stretch": "1.7.1-3" + }, + "fixed_version": "0", + "urgency": "unimportant" + }, + "jessie": { + "status": "resolved", + "repositories": { + "jessie": "1.7.1-3" + }, + "fixed_version": "0", + "urgency": "unimportant" + }, + "sid": { + "status": "resolved", + "repositories": { + "sid": "1.7.1-3" + }, + "fixed_version": "0", + "urgency": "unimportant" + }, + "bullseye": { + "status": "resolved", + "repositories": { + "bullseye": "1.7.1-3" + }, + "fixed_version": "0", + "urgency": "unimportant" + }, + "buster": { + "status": "resolved", + "repositories": { + "buster": "1.7.1-3" + }, + "fixed_version": "0", + "urgency": "unimportant" + } + } + } + }, + "python-scipy": { + "CVE-2013-4251": { + "debianbug": 726093, + "releases": { + "stretch": { + "status": "resolved", + "repositories": { + "stretch": "0.18.1-2" + }, + "fixed_version": "0.12.0-3", + "urgency": "not yet assigned" + }, + "jessie": { + "status": "resolved", + "repositories": { + "jessie": "0.14.0-2" + }, + "fixed_version": "0.12.0-3", + "urgency": "not yet assigned" + }, + "sid": { + "status": "resolved", + "repositories": { + "sid": "1.2.2-7" + }, + "fixed_version": "0.12.0-3", + "urgency": "not yet assigned" + }, + "bullseye": { + "status": "resolved", + "repositories": { + "bullseye": "1.2.2-7" + }, + "fixed_version": "0.12.0-3", + "urgency": "not yet assigned" + }, + "buster": { + "status": "resolved", + "repositories": { + "buster": "1.1.0-7" + }, + "fixed_version": "0.12.0-3", + "urgency": "not yet assigned" + } + } + } + } +} diff --git a/debian/testdata/fixtures/invalid.json b/debian/testdata/fixtures/invalid.json new file mode 100644 index 0000000..e466dcb --- /dev/null +++ b/debian/testdata/fixtures/invalid.json @@ -0,0 +1 @@ +invalid \ No newline at end of file diff --git a/debian/testdata/goldens/debian/prototypejs/CVE-2007-2383.json.golden b/debian/testdata/goldens/debian/prototypejs/CVE-2007-2383.json.golden new file mode 100644 index 0000000..d1b366e --- /dev/null +++ b/debian/testdata/goldens/debian/prototypejs/CVE-2007-2383.json.golden @@ -0,0 +1,47 @@ +{ + "debianbug": 555217, + "description": "The Prototype (prototypejs) framework before 1.5.1 RC3 exchanges data using JavaScript Object Notation (JSON) without an associated protection scheme, which allows remote attackers to obtain the data via a web page that retrieves the data through a URL in the SRC attribute of a SCRIPT element and captures the data using other JavaScript code, aka \"JavaScript Hijacking.\"", + "releases": { + "bullseye": { + "fixed_version": "0", + "repositories": { + "bullseye": "1.7.1-3" + }, + "status": "resolved", + "urgency": "unimportant" + }, + "buster": { + "fixed_version": "0", + "repositories": { + "buster": "1.7.1-3" + }, + "status": "resolved", + "urgency": "unimportant" + }, + "jessie": { + "fixed_version": "0", + "repositories": { + "jessie": "1.7.1-3" + }, + "status": "resolved", + "urgency": "unimportant" + }, + "sid": { + "fixed_version": "0", + "repositories": { + "sid": "1.7.1-3" + }, + "status": "resolved", + "urgency": "unimportant" + }, + "stretch": { + "fixed_version": "0", + "repositories": { + "stretch": "1.7.1-3" + }, + "status": "resolved", + "urgency": "unimportant" + } + }, + "scope": "remote" +} \ No newline at end of file diff --git a/debian/testdata/goldens/debian/prototypejs/CVE-2008-7220.json.golden b/debian/testdata/goldens/debian/prototypejs/CVE-2008-7220.json.golden new file mode 100644 index 0000000..6e52b22 --- /dev/null +++ b/debian/testdata/goldens/debian/prototypejs/CVE-2008-7220.json.golden @@ -0,0 +1,47 @@ +{ + "debianbug": 555217, + "description": "Unspecified vulnerability in Prototype JavaScript framework (prototypejs) before 1.6.0.2 allows attackers to make \"cross-site ajax requests\" via unknown vectors.", + "releases": { + "bullseye": { + "fixed_version": "1.6.0.2-1", + "repositories": { + "bullseye": "1.7.1-3" + }, + "status": "resolved", + "urgency": "high**" + }, + "buster": { + "fixed_version": "1.6.0.2-1", + "repositories": { + "buster": "1.7.1-3" + }, + "status": "resolved", + "urgency": "high**" + }, + "jessie": { + "fixed_version": "1.6.0.2-1", + "repositories": { + "jessie": "1.7.1-3" + }, + "status": "resolved", + "urgency": "high**" + }, + "sid": { + "fixed_version": "1.6.0.2-1", + "repositories": { + "sid": "1.7.1-3" + }, + "status": "resolved", + "urgency": "high**" + }, + "stretch": { + "fixed_version": "1.6.0.2-1", + "repositories": { + "stretch": "1.7.1-3" + }, + "status": "resolved", + "urgency": "high**" + } + }, + "scope": "remote" +} \ No newline at end of file diff --git a/debian/testdata/goldens/debian/python-scipy/CVE-2013-4251.json.golden b/debian/testdata/goldens/debian/python-scipy/CVE-2013-4251.json.golden new file mode 100644 index 0000000..0f7e263 --- /dev/null +++ b/debian/testdata/goldens/debian/python-scipy/CVE-2013-4251.json.golden @@ -0,0 +1,45 @@ +{ + "debianbug": 726093, + "releases": { + "bullseye": { + "fixed_version": "0.12.0-3", + "repositories": { + "bullseye": "1.2.2-7" + }, + "status": "resolved", + "urgency": "not yet assigned" + }, + "buster": { + "fixed_version": "0.12.0-3", + "repositories": { + "buster": "1.1.0-7" + }, + "status": "resolved", + "urgency": "not yet assigned" + }, + "jessie": { + "fixed_version": "0.12.0-3", + "repositories": { + "jessie": "0.14.0-2" + }, + "status": "resolved", + "urgency": "not yet assigned" + }, + "sid": { + "fixed_version": "0.12.0-3", + "repositories": { + "sid": "1.2.2-7" + }, + "status": "resolved", + "urgency": "not yet assigned" + }, + "stretch": { + "fixed_version": "0.12.0-3", + "repositories": { + "stretch": "0.18.1-2" + }, + "status": "resolved", + "urgency": "not yet assigned" + } + } +} \ No newline at end of file diff --git a/main.go b/main.go index 85dd79e..a3e429f 100644 --- a/main.go +++ b/main.go @@ -84,7 +84,8 @@ func run() error { } commitMsg = "RedHat " + *years case "debian": - if err := debian.Update(); err != nil { + dc := debian.NewClient() + if err := dc.Update(); err != nil { return xerrors.Errorf("error in Debian update: %w", err) } commitMsg = "Debian Security Bug Tracker" diff --git a/utils/utils.go b/utils/utils.go index 242d81d..9f2a27a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -101,14 +101,16 @@ func TrimSpaceNewline(str string) string { // FetchURL returns HTTP response body with retry func FetchURL(url, apikey string, retry int) (res []byte, err error) { - for i := 1; i <= retry; i++ { + for i := 0; i <= retry; i++ { + if i > 0 { + wait := math.Pow(float64(i), 2) + float64(randInt()%10) + log.Printf("retry after %f seconds\n", wait) + time.Sleep(time.Duration(time.Duration(wait) * time.Second)) + } res, err = fetchURL(url, apikey) if err == nil { return res, nil } - wait := math.Pow(float64(i), 2) + float64(randInt()%10) - log.Printf("retry after %f seconds\n", wait) - time.Sleep(time.Duration(time.Duration(wait) * time.Second)) } return nil, xerrors.Errorf("failed to fetch URL: %w", err) } @@ -123,12 +125,12 @@ func fetchURL(url, apikey string) ([]byte, error) { if apikey != "" { req.Header.Add("api-key", apikey) } - resp, body, err := req.Type("text").EndBytes() - if err != nil { - return nil, xerrors.Errorf("HTTP error. url: %s, err: %w", url, err) + resp, body, errs := req.Type("text").EndBytes() + if len(errs) > 0 { + return nil, xerrors.Errorf("HTTP error. url: %s, err: %w", url, errs[0]) } if resp.StatusCode != 200 { - return nil, xerrors.Errorf("HTTP error. status code: %d, url: %s, err: %w", resp.StatusCode, url, err) + return nil, xerrors.Errorf("HTTP error. status code: %d, url: %s", resp.StatusCode, url) } return body, nil }