From 71d82581c985ec43b7f345a3bad933f2d480cc1d Mon Sep 17 00:00:00 2001 From: santhosh1729 <84763859+santhosh1729@users.noreply.github.com> Date: Sun, 6 Jun 2021 21:44:28 +0530 Subject: [PATCH] feat: add Arch Linux Tracker (#88) * Arch Linux Tracker to collect vulnerabilities * refactor(arch) * chore: go mod tidy Co-authored-by: knqyf263 --- arch-linux/archlinux.go | 115 +++++++++++++++++++++++++ arch-linux/archlinux_test.go | 81 +++++++++++++++++ arch-linux/testdata/archlinux.json | 38 ++++++++ arch-linux/testdata/golden/AVG-20.json | 17 ++++ arch-linux/testdata/golden/AVG-4.json | 17 ++++ arch-linux/testdata/invalid.json | 1 + go.mod | 3 +- go.sum | 19 ++-- main.go | 9 +- 9 files changed, 289 insertions(+), 11 deletions(-) create mode 100644 arch-linux/archlinux.go create mode 100644 arch-linux/archlinux_test.go create mode 100644 arch-linux/testdata/archlinux.json create mode 100644 arch-linux/testdata/golden/AVG-20.json create mode 100644 arch-linux/testdata/golden/AVG-4.json create mode 100644 arch-linux/testdata/invalid.json diff --git a/arch-linux/archlinux.go b/arch-linux/archlinux.go new file mode 100644 index 0000000..a342263 --- /dev/null +++ b/arch-linux/archlinux.go @@ -0,0 +1,115 @@ +package arch + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/aquasecurity/vuln-list-update/utils" + + "github.com/cheggaaa/pb/v3" + "golang.org/x/xerrors" +) + +const ( + archLinuxDir = "arch-linux" + securityTrackerURL = "https://security.archlinux.org/json" + retry = 3 +) + +type securityGroups []struct { + Name string `json:"name"` + Packages []string `json:"packages"` + Status string `json:"status"` + Severity string `json:"severity"` + Type string `json:"type"` + Affected string `json:"affected"` + Fixed string `json:"fixed"` + Issues []string `json:"issues"` + Advisories []string `json:"advisories"` +} + +type options struct { + url string + dir string + retry int +} + +type option func(*options) + +func WithURL(url string) option { + return func(opts *options) { opts.url = url } +} + +func WithDir(dir string) option { + return func(opts *options) { opts.dir = dir } +} + +func WithRetry(retry int) option { + return func(opts *options) { opts.retry = retry } +} + +type ArchLinux struct { + *options +} + +func NewArchLinux(opts ...option) ArchLinux { + o := &options{ + url: securityTrackerURL, + dir: filepath.Join(utils.VulnListDir(), archLinuxDir), + retry: retry, + } + + for _, opt := range opts { + opt(o) + } + + return ArchLinux{ + options: o, + } +} + +func (al ArchLinux) Update() error { + log.Println("Fetching Arch Linux data...") + asgs, err := al.retrieveSecurityGroups() + if err != nil { + return xerrors.Errorf("failed to retrieve Arch Linux Security Groups: %w", err) + } + + log.Printf("Removing old dir (%s)...", al.dir) + if err = os.RemoveAll(al.dir); err != nil { + return xerrors.Errorf("failed to remove Arch Linux dir: %w", err) + } + + // Save all JSON files + log.Println("Saving new data...") + bar := pb.StartNew(len(asgs)) + if err = os.MkdirAll(al.dir, os.ModePerm); err != nil { + return xerrors.Errorf("failed to create the directory: %w", err) + } + for _, asg := range asgs { + filePath := filepath.Join(al.dir, fmt.Sprintf("%s.json", asg.Name)) + if err = utils.Write(filePath, asg); err != nil { + return xerrors.Errorf("failed to write Debian CVE details: %w", err) + } + bar.Increment() + } + bar.Finish() + + return nil +} + +func (al ArchLinux) retrieveSecurityGroups() (securityGroups, error) { + secJSON, err := utils.FetchURL(al.url, "", al.retry) + if err != nil { + return nil, xerrors.Errorf("failed to fetch cve data from Arch Linux. err: %w", err) + } + + var asgs securityGroups + if err = json.Unmarshal(secJSON, &asgs); err != nil { + return nil, xerrors.Errorf("json unmarshal error: %w", err) + } + return asgs, nil +} diff --git a/arch-linux/archlinux_test.go b/arch-linux/archlinux_test.go new file mode 100644 index 0000000..d2a861e --- /dev/null +++ b/arch-linux/archlinux_test.go @@ -0,0 +1,81 @@ +package arch_test + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/aquasecurity/vuln-list-update/arch-linux" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestUpdate(t *testing.T) { + tests := []struct { + name string + inputJSONFile string + wantErr string + }{ + { + name: "happy path", + inputJSONFile: "testdata/archlinux.json", + }, + { + name: "sad path, unreachable Arch Linux service", + wantErr: "connection refused", + }, + { + name: "sad path, invalid json", + inputJSONFile: "testdata/invalid.json", + wantErr: "json unmarshal error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := os.ReadFile(tt.inputJSONFile) + require.NoError(t, err) + + _, _ = io.WriteString(w, string(b)) + })) + defer ts.Close() + + // Intentionally close to induce network errors + if tt.inputJSONFile == "" { + ts.Close() + } + + dir := t.TempDir() + c := arch.NewArchLinux(arch.WithURL(ts.URL), arch.WithDir(filepath.Join(dir)), arch.WithRetry(0)) + err := c.Update() + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + + for _, e := range entries { + if e.IsDir() { + continue + } + + filePath := e.Name() + gotJSON, err := os.ReadFile(filepath.Join(dir, filePath)) + require.NoError(t, err) + + wantJSON, err := os.ReadFile(filepath.Join("testdata", "golden", filepath.Base(filePath))) + require.NoError(t, err) + + assert.JSONEq(t, string(wantJSON), string(gotJSON)) + } + }) + } +} diff --git a/arch-linux/testdata/archlinux.json b/arch-linux/testdata/archlinux.json new file mode 100644 index 0000000..fde7cc2 --- /dev/null +++ b/arch-linux/testdata/archlinux.json @@ -0,0 +1,38 @@ +[ + { + "name":"AVG-20", + "packages":[ + "curl" + ], + "status":"Fixed", + "severity":"Low", + "type":"denial of service", + "affected":"7.50.2-1", + "fixed":"7.50.3-1", + "ticket":null, + "issues":[ + "CVE-2016-7167" + ], + "advisories":[ + "ASA-201609-19" + ] + }, + { + "name":"AVG-4", + "packages":[ + "bzip2" + ], + "status":"Fixed", + "severity":"Low", + "type":"denial of service", + "affected":"1.0.6-5", + "fixed":"1.0.6-6", + "ticket":null, + "issues":[ + "CVE-2016-3189" + ], + "advisories":[ + "ASA-201702-19" + ] + } +] \ No newline at end of file diff --git a/arch-linux/testdata/golden/AVG-20.json b/arch-linux/testdata/golden/AVG-20.json new file mode 100644 index 0000000..8b606b8 --- /dev/null +++ b/arch-linux/testdata/golden/AVG-20.json @@ -0,0 +1,17 @@ +{ + "name":"AVG-20", + "packages":[ + "curl" + ], + "status":"Fixed", + "severity":"Low", + "type":"denial of service", + "affected":"7.50.2-1", + "fixed":"7.50.3-1", + "issues":[ + "CVE-2016-7167" + ], + "advisories":[ + "ASA-201609-19" + ] +} \ No newline at end of file diff --git a/arch-linux/testdata/golden/AVG-4.json b/arch-linux/testdata/golden/AVG-4.json new file mode 100644 index 0000000..c5f5b41 --- /dev/null +++ b/arch-linux/testdata/golden/AVG-4.json @@ -0,0 +1,17 @@ +{ + "name":"AVG-4", + "packages":[ + "bzip2" + ], + "status":"Fixed", + "severity":"Low", + "type":"denial of service", + "affected":"1.0.6-5", + "fixed":"1.0.6-6", + "issues":[ + "CVE-2016-3189" + ], + "advisories":[ + "ASA-201702-19" + ] +} \ No newline at end of file diff --git a/arch-linux/testdata/invalid.json b/arch-linux/testdata/invalid.json new file mode 100644 index 0000000..6b7f849 --- /dev/null +++ b/arch-linux/testdata/invalid.json @@ -0,0 +1 @@ +invalid json \ No newline at end of file diff --git a/go.mod b/go.mod index d16c3fe..226cfb3 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,12 @@ require ( github.com/PuerkitoBio/goquery v1.6.0 github.com/araddon/dateparse v0.0.0-20190426192744-0d74ffceef83 github.com/cheggaaa/pb v2.0.7+incompatible + github.com/cheggaaa/pb/v3 v3.0.8 github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect - github.com/fatih/color v1.10.0 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-jsonpointer v0.0.0-20180225143300-37667080efed - github.com/mattn/go-runewidth v0.0.7 // indirect github.com/parnurzeal/gorequest v0.2.16 github.com/pkg/errors v0.8.0 // indirect github.com/shurcooL/githubv4 v0.0.0-20191127044304-8f68eb5628d0 diff --git a/go.sum b/go.sum index 3e09366..f4f37e5 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,16 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/araddon/dateparse v0.0.0-20190426192744-0d74ffceef83 h1:ukTLOeMC0aVxbJWVg6hOsVJ0VPIo8w++PbNsze/pqF8= github.com/araddon/dateparse v0.0.0-20190426192744-0d74ffceef83/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= github.com/cheggaaa/pb v2.0.7+incompatible h1:gLKifR1UkZ/kLkda5gC0K6c8g+jU2sINPtBeOiNlMhU= github.com/cheggaaa/pb v2.0.7+incompatible/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= +github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA= +github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f h1:8GDPb0tCY8LQ+OJ3dbHb5sA6YZWXFORQYZx5sdsTlMs= @@ -34,14 +38,17 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-jsonpointer v0.0.0-20180225143300-37667080efed h1:fCWISZq4YN4ulCJx7x0KB15rqxLEe3mtNJL8cSOGKZU= github.com/mattn/go-jsonpointer v0.0.0-20180225143300-37667080efed/go.mod h1:SDJ4hurDYyQ9/7nc+eCYtXqdufgK4Cq9TJlwPklqEYA= -github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ= github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/shurcooL/githubv4 v0.0.0-20191127044304-8f68eb5628d0 h1:T9uus1QvcPgeLShS30YOnnzk3r9Vvygp45muhlrufgY= github.com/shurcooL/githubv4 v0.0.0-20191127044304-8f68eb5628d0/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= @@ -55,16 +62,13 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -73,11 +77,10 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/main.go b/main.go index eb621ff..1240c06 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( "github.com/aquasecurity/vuln-list-update/alpine" "github.com/aquasecurity/vuln-list-update/amazon" + arch_linux "github.com/aquasecurity/vuln-list-update/arch-linux" "github.com/aquasecurity/vuln-list-update/cwe" debianoval "github.com/aquasecurity/vuln-list-update/debian/oval" "github.com/aquasecurity/vuln-list-update/debian/tracker" @@ -40,7 +41,7 @@ const ( var ( target = flag.String("target", "", "update target (nvd, alpine, redhat, redhat-oval, "+ - "debian, debian-oval, ubuntu, amazon, oracle-oval, suse-cvrf, photon, ghsa, glad, cwe)") + "debian, debian-oval, ubuntu, amazon, oracle-oval, suse-cvrf, photon, arch-linux, ghsa, glad, cwe)") years = flag.String("years", "", "update years (only redhat)") ) @@ -178,6 +179,12 @@ func run() error { return xerrors.Errorf("error in CWE update: %w", err) } commitMsg = "CWE Advisories" + case "arch-linux": + al := arch_linux.NewArchLinux() + if err := al.Update(); err != nil { + return xerrors.Errorf("error in CWE update: %w", err) + } + commitMsg = "Arch Linux Security Tracker" default: return xerrors.New("unknown target") }