BREAKING CHANGE: extract Debian security advisories to separate repo (#219)

* use vuln-list-debian

* test: fix

* rename dir

* refactor: split dirs per year

.github/workflows/debian.yml

@@ -0,0 +1,40 @@
+name: Update vuln-list-debian repo
+on:
+  schedule:
+  - cron: "0 */6 * * *"
+  workflow_dispatch:
+
+jobs:
+  update:
+    name: Update vuln-list-debian
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+      VULN_LIST_DIR: "vuln-list-debian"
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version-file: go.mod
+
+    - name: Check out vuln-list-debian repo
+      uses: actions/checkout@v3
+      with:
+        repository: ${{ github.repository_owner }}/${{ env.VULN_LIST_DIR }}
+        token: ${{ secrets.ACCESS_TOKEN }}
+        path: ${{ env.VULN_LIST_DIR }}
+
+    - name: Setup github user email and name
+      run: |
+        git config --global user.email "action@github.com"
+        git config --global user.name "GitHub Action"
+
+    - name: Compile vuln-list-update
+      run: go build -o vuln-list-update .
+
+    - if: always()
+      name: Debian Security Bug Tracker
+      run: ./update.sh debian "Debian Security Bug Tracker"

.github/workflows/update.yml

@@ -47,10 +47,6 @@ jobs:
       name: Alpine Unfixed Vulnerability Tracker
       run: ./update.sh alpine-unfixed "Alpine Secshfixes Tracker"
 
-    - if: always()
-      name: Debian Security Bug Tracker
-      run: ./update.sh debian "Debian Security Bug Tracker"
-
     - if: always()
       name: Ubuntu CVE Tracker
       run: ./update.sh ubuntu "Ubuntu CVE Tracker"

debian/tracker/debian.go

@@ -19,14 +19,18 @@ import (
 )
 
 const (
-	debianDir          = "debian"
+	trackerDir         = "tracker"
 	securityTrackerURL = "https://salsa.debian.org/security-tracker-team/security-tracker/-/archive/master/security-tracker-master.tar.gz//security-tracker-master"
 	sourcesURL         = "https://ftp.debian.org/debian/dists/%s/%s/source/Sources.gz"
 	securitySourcesURL = "https://security.debian.org/debian-security/dists/%s/updates/%s/source/Sources.gz"
 )
 
 var (
-	repos = []string{"main", "contrib", "non-free"}
+	repos = []string{
+		"main",
+		"contrib",
+		"non-free",
+	}
 )
 
 type Bug struct {
@@ -91,8 +95,12 @@ func NewClient(opts ...option) Client {
 	}
 
 	return Client{
-		options:       o,
-		parsers:       []listParser{cveList{}, dlaList{}, dsaList{}},
+		options: o,
+		parsers: []listParser{
+			cveList{},
+			dlaList{},
+			dsaList{},
+		},
 		annDispatcher: newAnnotationDispatcher(),
 	}
 }
@@ -101,7 +109,7 @@ func (c Client) Update() error {
 	ctx := context.Background()
 
 	log.Println("Removing old Debian data...")
-	if err := os.RemoveAll(filepath.Join(c.vulnListDir, debianDir)); err != nil {
+	if err := os.RemoveAll(filepath.Join(c.vulnListDir, trackerDir)); err != nil {
 		return xerrors.Errorf("failed to remove Debian dir: %w", err)
 	}
 
@@ -130,7 +138,7 @@ func (c Client) Update() error {
 		return xerrors.Errorf("failed to update distributions: %w", err)
 	}
 
-	distributionJSON := filepath.Join(c.vulnListDir, debianDir, "distributions.json")
+	distributionJSON := filepath.Join(c.vulnListDir, trackerDir, "distributions.json")
 	if err = utils.Write(distributionJSON, dists); err != nil {
 		return xerrors.Errorf("unable to write %s: %w", distributionJSON, err)
 	}
@@ -148,10 +156,17 @@ func (c Client) update(dirname string, bugs []Bug) error {
 	log.Printf("Saving Debian %s data...", dirname)
 	bar := pb.StartNew(len(bugs))
 	for _, bug := range bugs {
-		fileName := fmt.Sprintf("%s.json", bug.Header.ID)
-		filePath := filepath.Join(c.vulnListDir, debianDir, dirname, fileName)
-		if err := utils.Write(filePath, bug); err != nil {
-			return xerrors.Errorf("debian: write error (%s): %w", filePath, err)
+		dir := filepath.Join(c.vulnListDir, trackerDir, dirname)
+		if dirname == "CVE" {
+			if err := utils.SaveCVEPerYear(dir, bug.Header.ID, bug); err != nil {
+				return xerrors.Errorf("debian: failed to save CVE per year: %w", err)
+			}
+		} else {
+			fileName := fmt.Sprintf("%s.json", bug.Header.ID)
+			filePath := filepath.Join(dir, fileName)
+			if err := utils.Write(filePath, bug); err != nil {
+				return xerrors.Errorf("debian: write error (%s): %w", filePath, err)
+			}
 		}
 		bar.Increment()
 	}
@@ -260,7 +275,10 @@ func shouldStore(anns []*Annotation) bool {
 }
 
 func (c Client) updateSources(ctx context.Context, dists map[string]Distribution) error {
-	for target, baseURL := range map[string]string{"source": c.sourcesURL, "updates-source": c.securitySourcesURL} {
+	for target, baseURL := range map[string]string{
+		"source":         c.sourcesURL,
+		"updates-source": c.securitySourcesURL,
+	} {
 		for code := range dists {
 			for _, r := range repos {
 				log.Printf("Updating %s %s/%s", target, code, r)
@@ -270,9 +288,16 @@ func (c Client) updateSources(ctx context.Context, dists map[string]Distribution
 					return xerrors.Errorf("unable to fetch sources: %w", err)
 				}
 
-				filePath := filepath.Join(c.vulnListDir, debianDir, target, code, r, "Sources.json")
-				if err = utils.Write(filePath, headers); err != nil {
-					return xerrors.Errorf("source write error: %w", err)
+				for _, header := range headers {
+					name := header.Get("Package")
+					if name == "" {
+						continue
+					}
+
+					filePath := filepath.Join(c.vulnListDir, trackerDir, target, code, r, name[:1], name+".json")
+					if err = utils.Write(filePath, header); err != nil {
+						return xerrors.Errorf("source write error: %w", err)
+					}
 				}
 			}
 		}

debian/tracker/debian_test.go

@@ -25,7 +25,7 @@ func TestClient_Update(t *testing.T) {
 		securitySourcesPath string
 		wantBugs            map[string]tracker.Bug
 		wantDists           map[string]tracker.Distribution
-		wantSources         map[string][]pkgDetail
+		wantSources         map[string]pkgDetail
 		wantErr             string
 	}{
 		{
@@ -46,7 +46,12 @@ func TestClient_Update(t *testing.T) {
 							Original: "{CVE-2021-29969 CVE-2021-29970 CVE-2021-29976 CVE-2021-30547}",
 							Line:     2,
 							Type:     "xref",
-							Bugs:     []string{"CVE-2021-29969", "CVE-2021-29970", "CVE-2021-29976", "CVE-2021-30547"},
+							Bugs: []string{
+								"CVE-2021-29969",
+								"CVE-2021-29970",
+								"CVE-2021-29976",
+								"CVE-2021-30547",
+							},
 						},
 						{
 							Original: "[stretch] - thunderbird 1:78.12.0-1~deb9u1",
@@ -71,7 +76,10 @@ func TestClient_Update(t *testing.T) {
 							Original: "{CVE-2019-10192 CVE-2019-10193}",
 							Line:     2,
 							Type:     "xref",
-							Bugs:     []string{"CVE-2019-10192", "CVE-2019-10193"},
+							Bugs: []string{
+								"CVE-2019-10192",
+								"CVE-2019-10193",
+							},
 						},
 						{
 							Original: "[stretch] - redis 3:3.2.6-3+deb9u3",
@@ -93,7 +101,7 @@ func TestClient_Update(t *testing.T) {
 						},
 					},
 				},
-				filepath.Join("CVE", "CVE-2021-36373.json"): {
+				filepath.Join("CVE", "2021", "CVE-2021-36373.json"): {
 					Header: &tracker.Header{
 						Original:    "CVE-2021-36373 (When reading a specially crafted TAR archive an Apache Ant build can b ...)",
 						Line:        5,
@@ -123,7 +131,7 @@ func TestClient_Update(t *testing.T) {
 						},
 					},
 				},
-				filepath.Join("CVE", "CVE-2021-36367.json"): {
+				filepath.Join("CVE", "2021", "CVE-2021-36367.json"): {
 					Header: &tracker.Header{
 						Original:    "CVE-2021-36367 (PuTTY through 0.75 proceeds with establishing an SSH session even if i ...)",
 						Line:        10,
@@ -174,70 +182,54 @@ func TestClient_Update(t *testing.T) {
 					Contact:      "team@security.debian.org",
 				},
 			},
-			wantSources: map[string][]pkgDetail{
-				filepath.Join("source", "stretch", "main", "Sources.json"): {
-					{
-						Package: []string{"0ad"},
-						Version: []string{"0.0.21-2"},
-					},
-					{
-						Package: []string{"0ad-data"},
-						Version: []string{"0.0.21-1"},
-					},
+			wantSources: map[string]pkgDetail{
+				filepath.Join("source", "stretch", "main", "0", "0ad.json"): {
+					Package: []string{"0ad"},
+					Version: []string{"0.0.21-2"},
 				},
-				filepath.Join("updates-source", "stretch", "main", "Sources.json"): {
-					{
-						Package: []string{"0ad"},
-						Version: []string{"0.0.21-2"},
-					},
-					{
-						Package: []string{"0ad-data"},
-						Version: []string{"0.0.21-1"},
-					},
+				filepath.Join("source", "stretch", "main", "0", "0ad-data.json"): {
+					Package: []string{"0ad-data"},
+					Version: []string{"0.0.21-1"},
 				},
-				filepath.Join("source", "stretch", "contrib", "Sources.json"): {
-					{
-						Package: []string{"alien-arena"},
-						Version: []string{"7.66+dfsg-3"},
-					},
+				filepath.Join("updates-source", "stretch", "main", "0", "0ad.json"): {
+					Package: []string{"0ad"},
+					Version: []string{"0.0.21-2"},
 				},
-				filepath.Join("updates-source", "stretch", "contrib", "Sources.json"): {
-					{
-						Package: []string{"alien-arena"},
-						Version: []string{"7.66+dfsg-3"},
-					},
+				filepath.Join("updates-source", "stretch", "main", "0", "0ad-data.json"): {
+					Package: []string{"0ad-data"},
+					Version: []string{"0.0.21-1"},
 				},
-				filepath.Join("source", "buster", "main", "Sources.json"): {
-					{
-						Package: []string{"zzz-to-char"},
-						Version: []string{"0.1.3-2"},
-					},
-					{
-						Package: []string{"zzzeeksphinx"},
-						Version: []string{"1.0.20-2"},
-					},
+				filepath.Join("source", "stretch", "contrib", "a", "alien-arena.json"): {
+					Package: []string{"alien-arena"},
+					Version: []string{"7.66+dfsg-3"},
 				},
-				filepath.Join("updates-source", "buster", "main", "Sources.json"): {
-					{
-						Package: []string{"zzz-to-char"},
-						Version: []string{"0.1.3-3"},
-					},
+				filepath.Join("updates-source", "stretch", "contrib", "a", "alien-arena.json"): {
+					Package: []string{"alien-arena"},
+					Version: []string{"7.66+dfsg-3"},
 				},
-				filepath.Join("source", "buster", "contrib", "Sources.json"): {
-					{
-						Package: []string{"zfs-auto-snapshot"},
-						Version: []string{"1.2.4-2"},
-					},
-					{
-						Package: []string{"zfs-linux"},
-						Version: []string{"0.7.12-2+deb10u2"},
-					},
+				filepath.Join("source", "buster", "main", "z", "zzz-to-char.json"): {
+					Package: []string{"zzz-to-char"},
+					Version: []string{"0.1.3-2"},
 				},
-				filepath.Join("updates-source", "buster", "contrib", "Sources.json"): {
-					{
-						Package: []string{"zfs-linux"},
-						Version: []string{"0.7.12-2+deb10u3"},
-					},
+				filepath.Join("source", "buster", "main", "z", "zzzeeksphinx.json"): {
+					Package: []string{"zzzeeksphinx"},
+					Version: []string{"1.0.20-2"},
+				},
+				filepath.Join("updates-source", "buster", "main", "z", "zzz-to-char.json"): {
+					Package: []string{"zzz-to-char"},
+					Version: []string{"0.1.3-3"},
+				},
+				filepath.Join("source", "buster", "contrib", "z", "zfs-auto-snapshot.json"): {
+					Package: []string{"zfs-auto-snapshot"},
+					Version: []string{"1.2.4-2"},
+				},
+				filepath.Join("source", "buster", "contrib", "z", "zfs-linux.json"): {
+					Package: []string{"zfs-linux"},
+					Version: []string{"0.7.12-2+deb10u2"},
+				},
+				filepath.Join("updates-source", "buster", "contrib", "z", "zfs-linux.json"): {
+					Package: []string{"zfs-linux"},
+					Version: []string{"0.7.12-2+deb10u3"},
 				},
 			},
 		},
@@ -264,21 +256,21 @@ func TestClient_Update(t *testing.T) {
 			// Compare CVE/list, DLA/list, and DSA/list
 			for name, want := range tt.wantBugs {
 				var got tracker.Bug
-				filePath := filepath.Join(tmpDir, "debian", name)
+				filePath := filepath.Join(tmpDir, "tracker", name)
 				compare(t, filePath, &got, &want)
 			}
 
 			// Compare distributions.json
 			{
 				var got map[string]tracker.Distribution
-				filePath := filepath.Join(tmpDir, "debian", "distributions.json")
+				filePath := filepath.Join(tmpDir, "tracker", "distributions.json")
 				compare(t, filePath, &got, &tt.wantDists)
 			}
 
 			// Compare Sources
 			for name, want := range tt.wantSources {
-				var got []pkgDetail
-				filePath := filepath.Join(tmpDir, "debian", name)
+				var got pkgDetail
+				filePath := filepath.Join(tmpDir, "tracker", name)
 				compare(t, filePath, &got, &want)
 			}
 		})

main.go

@@ -39,7 +39,7 @@ import (
 
 var (
 	target = flag.String("target", "", "update target (nvd, alpine, alpine-unfixed, redhat, redhat-oval, "+
-		"debian, debian-oval, ubuntu, amazon, oracle-oval, suse-cvrf, photon, arch-linux, ghsa, glad, cwe, osv, go-vulndb, mariner, kevc, wolfi, chainguard)")
+		"debian, ubuntu, amazon, oracle-oval, suse-cvrf, photon, arch-linux, ghsa, glad, cwe, osv, go-vulndb, mariner, kevc, wolfi, chainguard)")
 	vulnListDir  = flag.String("vuln-list-dir", "", "vuln-list dir")
 	targetUri    = flag.String("target-uri", "", "alternative repository URI (only glad)")
 	targetBranch = flag.String("target-branch", "", "alternative repository branch (only glad)")