diff --git a/alt/alt.go b/alt/alt.go new file mode 100644 index 0000000..9417b62 --- /dev/null +++ b/alt/alt.go @@ -0,0 +1,152 @@ +package alt + +import ( + "archive/zip" + "bytes" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + "github.com/cheggaaa/pb/v3" + "github.com/spf13/afero" + "golang.org/x/xerrors" + + "github.com/aquasecurity/vuln-list-update/utils" +) + +const ( + altDir = "oval" + + branchURL = "https://rdb.altlinux.org/api/errata/export/oval/%s" + branchListURL = "https://rdb.altlinux.org/api/errata/export/oval/branches" + + retry = 5 +) + +type Config struct { + VulnListDir string + BranchURL string + BranchListURL string + AppFs afero.Fs + Retry int +} + +func NewConfig() Config { + return Config{ + VulnListDir: utils.VulnListDir(), + BranchURL: branchURL, + BranchListURL: branchListURL, + AppFs: afero.NewOsFs(), + Retry: retry, + } +} + +type BranchList struct { + Length int + Branches []string +} + +func (c Config) Update() error { + dirPath := filepath.Join(c.VulnListDir, altDir) + log.Printf("Remove ALT's OVAL directoty: %s", dirPath) + if err := os.RemoveAll(dirPath); err != nil { + return xerrors.Errorf("failed to remove ALT's OVAL directory: %w", err) + } + + log.Println("Fetching ALT's OVAL branch list...") + branchList, err := c.fetchBranchList() + if err != nil { + return err + } + + for _, branch := range branchList.Branches { + log.Printf("Fetching ALT's OVAL branch: %s", branch) + if err := c.updateOVAL(branch); err != nil { + return err + } + } + + return nil +} + +func (c Config) fetchBranchList() (BranchList, error) { + resp, err := utils.FetchURL(c.BranchListURL, "", c.Retry) + if err != nil { + return BranchList{}, xerrors.Errorf("failed to get ALT's OVAL branch list: %w", err) + } + + var branchList BranchList + if err := json.Unmarshal(resp, &branchList); err != nil { + return BranchList{}, xerrors.Errorf("failed to unmarshal branch list JSON response: %w", err) + } + + return branchList, nil +} + +func (c Config) updateOVAL(branch string) error { + ovalURL := fmt.Sprintf(c.BranchURL, branch) + + resp, err := utils.FetchURL(ovalURL, "", c.Retry) + if err != nil { + return xerrors.Errorf("failed to get ALT's OVAL branch archive: %w", err) + } + + reader, err := zip.NewReader(bytes.NewReader(resp), int64(len(resp))) + if err != nil { + return xerrors.Errorf("failed to init zip reader: %w", err) + } + + pbar := pb.StartNew(len(reader.File)) + for _, file := range reader.File { + var oval OVALDefinitions + rc, err := file.Open() + if err != nil { + return xerrors.Errorf("failed to open file: %w", err) + } + content, err := io.ReadAll(rc) + if err != nil { + rc.Close() + return xerrors.Errorf("failed to read file content: %w", err) + } + + err = xml.Unmarshal(content, &oval) + if err != nil { + rc.Close() + return xerrors.Errorf("failed to unmarshal ALT's OVAL xml: %w", err) + } + + ovalName := strings.TrimSuffix(file.Name, ".xml") + ovalPath := filepath.Join(c.VulnListDir, altDir, branch, ovalName) + + if err := utils.WriteJSON(c.AppFs, ovalPath, "tests.json", oval.Tests); err != nil { + rc.Close() + return xerrors.Errorf("failed to write tests.json: %w", err) + } + + if err := utils.WriteJSON(c.AppFs, ovalPath, "objects.json", oval.Objects); err != nil { + rc.Close() + return xerrors.Errorf("failed to write objects.json: %w", err) + } + + if err = utils.WriteJSON(c.AppFs, ovalPath, "states.json", oval.States); err != nil { + rc.Close() + return xerrors.Errorf("failed to write states: %w", err) + } + + if err = utils.WriteJSON(c.AppFs, ovalPath, "definitions.json", oval.Definitions); err != nil { + rc.Close() + return xerrors.Errorf("failed to write definitions: %w", err) + } + + pbar.Increment() + rc.Close() + } + + pbar.Finish() + return nil +} diff --git a/alt/alt_test.go b/alt/alt_test.go new file mode 100644 index 0000000..976e26f --- /dev/null +++ b/alt/alt_test.go @@ -0,0 +1,61 @@ +package alt + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfig_Update(t *testing.T) { + testCases := []struct { + name string + dir string + wantFiles int + wantErr string + }{ + { + name: "happy path", + dir: "testdata/happy", + }, + { + name: "404", + dir: "testdata/missing-oval", + wantErr: "failed to get ALT's OVAL branch archive: failed to fetch URL: HTTP error. status code: 404, url:", + }, + { + name: "broken XML", + dir: "testdata/broken", + wantErr: "failed to unmarshal ALT's OVAL xml: XML syntax error on line 4: element closed by ", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ts := httptest.NewServer(http.FileServer(http.Dir(tc.dir))) + defer ts.Close() + + tmpDir := "/tmp" // It is a virtual filesystem of afero. + appFs := afero.NewMemMapFs() + c := Config{ + VulnListDir: tmpDir, + BranchURL: ts.URL + "/%s/oval_definitions.zip", + BranchListURL: ts.URL + "/branches.json", + AppFs: appFs, + Retry: 0, + } + + err := c.Update() + if tc.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + return + } + + require.NoError(t, err, tc.name) + assert.NoError(t, err, tc.name) + }) + } +} diff --git a/alt/testdata/broken/branches.json b/alt/testdata/broken/branches.json new file mode 100644 index 0000000..3295ef8 --- /dev/null +++ b/alt/testdata/broken/branches.json @@ -0,0 +1 @@ +{"length": 3, "branches": ["p9", "p10", "c9f2"]} diff --git a/alt/testdata/broken/p9/oval_definitions.zip b/alt/testdata/broken/p9/oval_definitions.zip new file mode 100644 index 0000000..23cb86e Binary files /dev/null and b/alt/testdata/broken/p9/oval_definitions.zip differ diff --git a/alt/testdata/happy/branches.json b/alt/testdata/happy/branches.json new file mode 100644 index 0000000..3295ef8 --- /dev/null +++ b/alt/testdata/happy/branches.json @@ -0,0 +1 @@ +{"length": 3, "branches": ["p9", "p10", "c9f2"]} diff --git a/alt/testdata/happy/c9f2/oval_definitions.zip b/alt/testdata/happy/c9f2/oval_definitions.zip new file mode 100644 index 0000000..f0c824e Binary files /dev/null and b/alt/testdata/happy/c9f2/oval_definitions.zip differ diff --git a/alt/testdata/happy/p10/oval_definitions.zip b/alt/testdata/happy/p10/oval_definitions.zip new file mode 100644 index 0000000..45e3f88 Binary files /dev/null and b/alt/testdata/happy/p10/oval_definitions.zip differ diff --git a/alt/testdata/happy/p9/oval_definitions.zip b/alt/testdata/happy/p9/oval_definitions.zip new file mode 100644 index 0000000..5119955 Binary files /dev/null and b/alt/testdata/happy/p9/oval_definitions.zip differ diff --git a/alt/testdata/missing-oval/branches.json b/alt/testdata/missing-oval/branches.json new file mode 100644 index 0000000..3295ef8 --- /dev/null +++ b/alt/testdata/missing-oval/branches.json @@ -0,0 +1 @@ +{"length": 3, "branches": ["p9", "p10", "c9f2"]} diff --git a/alt/types.go b/alt/types.go new file mode 100644 index 0000000..d373dd2 --- /dev/null +++ b/alt/types.go @@ -0,0 +1,230 @@ +package alt + +import "encoding/xml" + +type OVALDefinitions struct { + XMLName xml.Name `xml:"oval_definitions"` + Generator Generator `xml:"generator"` + + Definitions Definitions `xml:"definitions"` + Tests Tests `xml:"tests"` + Objects Objects `xml:"objects"` + States States `xml:"states"` +} + +type Generator struct { + Timestamp string `xml:"timestamp"` + ProductName string `xml:"product_name"` + SchemaVersion string `xml:"schema_version"` +} + +type Definitions struct { + Definition []Definition `xml:"definition" json:",omitempty"` +} + +type Definition struct { + ID string `xml:"id,attr" json:",omitempty"` + Version string `xml:"version,attr" json:",omitempty"` + Class string `xml:"class,attr" json:",omitempty"` + Metadata Metadata `xml:"metadata" json:",omitempty"` + Criteria Criteria `xml:"criteria" json:",omitempty"` +} + +type Metadata struct { + Title string `xml:"title" json:",omitempty"` + AffectedList []Affected `xml:"affected" json:",omitempty"` + References []Reference `xml:"reference" json:",omitempty"` + Description string `xml:"description" json:",omitempty"` + Advisory Advisory `xml:"advisory" json:",omitempty"` +} + +type Affected struct { + Family string `xml:"family,attr" json:",omitempty"` + Platforms []string `xml:"platform" json:",omitempty"` + Products []string `xml:"product" json:",omitempty"` +} + +type Reference struct { + RefID string `xml:"ref_id,attr" json:",omitempty"` + RefURL string `xml:"ref_url,attr" json:",omitempty"` + Source string `xml:"source,attr" json:",omitempty"` +} + +type Advisory struct { + From string `xml:"from,attr" json:",omitempty"` + Severity string `xml:"severity" json:",omitempty"` + Rights string `xml:"rights" json:",omitempty"` + Issued Issued `xml:"issued" json:",omitempty"` + Updated Updated `xml:"updated" json:",omitempty"` + BDUs []CVE `xml:"bdu" json:""` + CVEs []CVE `xml:"cve" json:",omitempty"` + Bugzilla []Bugzilla `xml:"bugzilla" json:",omitempty"` + AffectedCPEs AffectedCPEs `xml:"affected_cpe_list" json:",omitempty"` +} + +type Bugzilla struct { + ID string `xml:"id,attr" json:",omitempty"` + Href string `xml:"href,attr" json:",omitempty"` + Data string `xml:",chardata" json:",omitempty"` +} + +type Issued struct { + Date string `xml:"date,attr" json:",omitempty"` +} + +type Updated struct { + Date string `xml:"date,attr" json:",omitempty"` +} + +type CVE struct { + ID string `xml:",chardata" json:",omitempty"` + CVSS string `xml:"cvss,attr" json:",omitempty"` + CVSS3 string `xml:"cvss3,attr" json:",omitempty"` + CWE string `xml:"cwe,attr" json:",omitempty"` + Href string `xml:"href,attr" json:",omitempty"` + Impact string `xml:"impact,attr" json:",omitempty"` + Public string `xml:"public,attr" json:",omitempty"` +} + +type AffectedCPEs struct { + CPEs []string `xml:"cpe" json:",omitempty"` +} + +type Criteria struct { + Operator string `xml:"operator,attr" json:",omitempty"` + Criterions []Criterion `xml:"criterion" json:",omitempty"` + Criterias []Criteria `xml:"criteria" json:",omitempty"` +} + +type Criterion struct { + TestRef string `xml:"test_ref,attr" json:",omitempty"` + Comment string `xml:"comment,attr" json:",omitempty"` +} + +type Tests struct { + TextFileContent54Tests []TextFileContent54Test `xml:"textfilecontent54_test" json:",omitempty"` + RPMInfoTests []RPMInfoTest `xml:"rpminfo_test" json:",omitempty"` +} + +type TextFileContent54Test struct { + ID string `xml:"id,attr" json:",omitempty"` + Version string `xml:"version,attr" json:",omitempty"` + Check string `xml:"check,attr" json:",omitempty"` + Comment string `xml:"comment,attr" json:",omitempty"` + Object Object `xml:"object" json:",omitempty"` + State State `xml:"state" json:",omitempty"` +} + +type State struct { + StateRef string `xml:"state_ref,attr" json:",omitempty"` + Text string `xml:"state" json:",omitempty"` +} + +type Object struct { + ObjectRef string `xml:"object_ref,attr" json:",omitempty"` + Text string `xml:"object" json:",omitempty"` +} + +type RPMInfoTest struct { + ID string `xml:"id,attr" json:",omitempty"` + Version string `xml:"version,attr" json:",omitempty"` + Check string `xml:"check,attr" json:",omitempty"` + Comment string `xml:"comment,attr" json:",omitempty"` + Object Object `xml:"object" json:",omitempty"` + State State `xml:"state" json:",omitempty"` +} + +type RPMInfoObject struct { + ID string `xml:"id,attr" json:",omitempty"` + Version string `xml:"version,attr" json:",omitempty"` + Comment string `xml:"comment,attr" json:",omitempty"` + Name string `xml:"name" json:",omitempty"` +} + +type RPMInfoState struct { + ID string `xml:"id,attr" json:",omitempty"` + Version string `xml:"version,attr" json:",omitempty"` + Comment string `xml:"comment,attr" json:",omitempty"` + Arch Arch `xml:"arch" json:",omitempty"` + EVR EVR `xml:"evr" json:",omitempty"` + Subexpression Subexpression `xml:"subexpression" json:",omitempty"` +} + +type Arch struct { + Text string `xml:",chardata" json:",omitempty"` + Datatype string `xml:"datatype,attr" json:",omitempty"` + Operation string `xml:"operation,attr" json:",omitempty"` +} + +type EVR struct { + Text string `xml:",chardata" json:",omitempty"` + Datatype string `xml:"datatype,attr" json:",omitempty"` + Operation string `xml:"operation,attr" json:",omitempty"` +} + +type Subexpression struct { + Operation string `xml:"operation,attr" json:",omitempty"` + Text string `xml:",chardata" json:",omitempty"` +} + +type Objects struct { + TextFileContent54Objects []TextFileContent54Object `xml:"textfilecontent54_object" json:",omitempty"` + RPMInfoObjects []RPMInfoObject `xml:"rpminfo_object" json:",omitempty"` +} + +type TextFileContent54Object struct { + ID string `xml:"id,attr" json:",omitempty"` + Version string `xml:"version,attr" json:",omitempty"` + Comment string `xml:"comment,attr" json:",omitempty"` + Path Path `xml:"path" json:",omitempty"` + Filepath Filepath `xml:"filepath" json:",omitempty"` + Pattern Pattern `xml:"pattern" json:",omitempty"` + Instance Instance `xml:"instance" json:",omitempty"` +} + +type Path struct { + Datatype string `xml:"datatype,attr" json:",omitempty"` + Text string `xml:",chardata" json:",omitempty"` +} + +type Filepath struct { + Datatype string `xml:"datatype,attr" json:",omitempty"` + Text string `xml:",chardata" json:",omitempty"` +} + +type Pattern struct { + Datatype string `xml:"datatype,attr" json:",omitempty"` + Operation string `xml:"operation,attr" json:",omitempty"` + Text string `xml:",chardata" json:",omitempty"` +} + +type Instance struct { + Datatype string `xml:"datatype,attr" json:",omitempty"` + Text string `xml:",chardata" json:",omitempty"` +} + +type Name struct { + Text string `xml:",chardata" json:",omitempty"` + Operation string `xml:"operation,attr" json:",omitempty"` +} + +type States struct { + TextFileContent54State []TextFileContent54State `xml:"textfilecontent54_state" json:",omitempty"` + RPMInfoStates []RPMInfoState `xml:"rpminfo_state" json:",omitempty"` +} + +type Version struct { + Text string `xml:",chardata" json:",omitempty"` + Operation string `xml:"operation,attr" json:",omitempty"` +} + +type TextFileContent54State struct { + ID string `xml:"id,attr" json:",omitempty"` + Version string `xml:"version,attr" json:",omitempty"` + Text Text `xml:"text" json:",omitempty"` +} + +type Text struct { + Text string `xml:",chardata" json:",omitempty"` + Operation string `xml:"operation,attr" json:",omitempty"` +} diff --git a/main.go b/main.go index 9e8be68..0a3bb7a 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "github.com/aquasecurity/vuln-list-update/alma" "github.com/aquasecurity/vuln-list-update/alpine" alpineunfixed "github.com/aquasecurity/vuln-list-update/alpine-unfixed" + "github.com/aquasecurity/vuln-list-update/alt" "github.com/aquasecurity/vuln-list-update/amazon" arch_linux "github.com/aquasecurity/vuln-list-update/arch" "github.com/aquasecurity/vuln-list-update/chainguard" @@ -38,7 +39,7 @@ import ( var ( target = flag.String("target", "", "update target (nvd, alpine, alpine-unfixed, redhat, redhat-oval, "+ - "debian, ubuntu, amazon, oracle-oval, suse-cvrf, photon, arch-linux, ghsa, glad, cwe, osv, mariner, kevc, wolfi, chainguard, k8s)") + "debian, ubuntu, amazon, oracle-oval, suse-cvrf, photon, arch-linux, ghsa, glad, cwe, osv, mariner, kevc, wolfi, chainguard, k8s, alt)") vulnListDir = flag.String("vuln-list-dir", "", "vuln-list dir") targetUri = flag.String("target-uri", "", "alternative repository URI (only glad)") targetBranch = flag.String("target-branch", "", "alternative repository branch (only glad)") @@ -176,6 +177,11 @@ func run() error { if err := ku.Update(); err != nil { return xerrors.Errorf("k8s update error: %w", err) } + case "alt": + alt := alt.NewConfig() + if err := alt.Update(); err != nil { + return xerrors.Errorf("ALT update error: %w", err) + } default: return xerrors.New("unknown target") }