alt/zot-wo-auth
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(metadb): fix unexpected panic when dereferencing map fields (#1993)

Browse Source 
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae 2023-11-02 23:35:49 +02:00 committed by GitHub
parent 9074f8483b
commit 6a66a9b9b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 248 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

398
pkg/meta/boltdb/boltdb.go
View File

@ -148,21 +148,9 @@ func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference
return err
}
repoMetaBlob := repoBuck.Get([]byte(repo))
protoRepoMeta := &proto_go.RepoMeta{
Name: repo,
Tags: map[string]*proto_go.TagDescriptor{"": {}}, // This is done so Protobuf can initialize a non-nil map
Statistics: map[string]*proto_go.DescriptorStatistics{"": {}},
Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}},
Referrers: map[string]*proto_go.ReferrersInfo{"": {}},
}
if len(repoMetaBlob) > 0 {
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
if err != nil {
return err
}
protoRepoMeta, err := getProtoRepoMeta(repo, repoBuck)
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
return err
}
// 2. Referrers
@ -227,17 +215,11 @@ func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference
}
// 4. Blobs
repoBlobsBytes := repoBlobsBuck.Get([]byte(protoRepoMeta.Name))
repoBlobsBytes := repoBlobsBuck.Get([]byte(repo))
repoBlobs := &proto_go.RepoBlobs{}
if len(repoBlobsBytes) == 0 {
repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo)
} else {
err := proto.Unmarshal(repoBlobsBytes, repoBlobs)
if err != nil {
return err
}
repoBlobs, err := unmarshalProtoRepoBlobs(repo, repoBlobsBytes)
if err != nil {
return err
}
protoRepoMeta, repoBlobs, err = common.AddImageMetaToRepoMeta(protoRepoMeta, repoBlobs, reference, imageMeta)
@ -245,27 +227,97 @@ func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference
return err
}
repoBlobsBytes, err = proto.Marshal(repoBlobs)
err = setProtoRepoBlobs(repoBlobs, repoBlobsBuck)
if err != nil {
return err
}
err = repoBlobsBuck.Put([]byte(protoRepoMeta.Name), repoBlobsBytes)
if err != nil {
return err
}
repoMetaBlob, err = proto.Marshal(protoRepoMeta)
if err != nil {
return err
}
return repoBuck.Put([]byte(repo), repoMetaBlob)
return setProtoRepoMeta(protoRepoMeta, repoBuck)
})
return err
}
func unmarshalProtoRepoBlobs(repo string, repoBlobsBytes []byte) (*proto_go.RepoBlobs, error) {
repoBlobs := &proto_go.RepoBlobs{
Name: repo,
}
if len(repoBlobsBytes) > 0 {
err := proto.Unmarshal(repoBlobsBytes, repoBlobs)
if err != nil {
return nil, err
}
}
if repoBlobs.Blobs == nil {
repoBlobs.Blobs = map[string]*proto_go.BlobInfo{"": {}}
}
return repoBlobs, nil
}
func setProtoRepoBlobs(repoBlobs *proto_go.RepoBlobs, repoBlobsBuck *bbolt.Bucket) error {
repoBlobsBytes, err := proto.Marshal(repoBlobs)
if err != nil {
return err
}
return repoBlobsBuck.Put([]byte(repoBlobs.Name), repoBlobsBytes)
}
func getProtoRepoMeta(repo string, repoMetaBuck *bbolt.Bucket) (*proto_go.RepoMeta, error) {
repoMetaBlob := repoMetaBuck.Get([]byte(repo))
return unmarshalProtoRepoMeta(repo, repoMetaBlob)
}
// unmarshalProtoRepoMeta will unmarshal the repoMeta blob and initialize nil maps. If the blob is empty
// an empty initialized object is returned.
func unmarshalProtoRepoMeta(repo string, repoMetaBlob []byte) (*proto_go.RepoMeta, error) {
protoRepoMeta := &proto_go.RepoMeta{
Name: repo,
}
if len(repoMetaBlob) > 0 {
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
if err != nil {
return nil, err
}
}
if protoRepoMeta.Tags == nil {
protoRepoMeta.Tags = map[string]*proto_go.TagDescriptor{"": {}}
}
if protoRepoMeta.Statistics == nil {
protoRepoMeta.Statistics = map[string]*proto_go.DescriptorStatistics{"": {}}
}
if protoRepoMeta.Signatures == nil {
protoRepoMeta.Signatures = map[string]*proto_go.ManifestSignatures{"": {}}
}
if protoRepoMeta.Referrers == nil {
protoRepoMeta.Referrers = map[string]*proto_go.ReferrersInfo{"": {}}
}
if len(repoMetaBlob) == 0 {
return protoRepoMeta, zerr.ErrRepoMetaNotFound
}
return protoRepoMeta, nil
}
func setProtoRepoMeta(repoMeta *proto_go.RepoMeta, repoBuck *bbolt.Bucket) error {
repoMetaBlob, err := proto.Marshal(repoMeta)
if err != nil {
return err
}
return repoBuck.Put([]byte(repoMeta.Name), repoMetaBlob)
}
func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string,
) (map[string]mTypes.ImageMeta, error) {
imageMetaMap := map[string]mTypes.ImageMeta{}
@ -274,7 +326,7 @@ func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string,
imageBuck := transaction.Bucket([]byte(ImageMetaBuck))
for _, digest := range digests {
protoImageMeta, err := fetchProtoImageMeta(imageBuck, digest)
protoImageMeta, err := getProtoImageMeta(imageBuck, digest)
if err != nil {
return err
}
@ -283,7 +335,7 @@ func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string,
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests {
imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest)
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
if err != nil {
return err
}
@ -326,9 +378,7 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string,
continue
}
var protoRepoMeta proto_go.RepoMeta
err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob)
if err != nil {
return err
}
@ -339,7 +389,7 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string,
protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name)
protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name)
repos = append(repos, mConvert.GetRepoMeta(&protoRepoMeta))
repos = append(repos, mConvert.GetRepoMeta(protoRepoMeta))
}
return nil
@ -348,7 +398,7 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string,
return repos, err
}
func fetchProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageMeta, error) {
func getProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageMeta, error) {
imageMetaBlob := imageBuck.Get([]byte(digest))
if len(imageMetaBlob) == 0 {
@ -393,9 +443,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
return err
}
protoRepoMeta := &proto_go.RepoMeta{}
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob)
if err != nil {
return err
}
@ -416,7 +464,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
imageManifestData, err := fetchProtoImageMeta(imageBuck, manifestDigest)
imageManifestData, err := getProtoImageMeta(imageBuck, manifestDigest)
if err != nil {
return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w",
manifestDigest, err)
@ -426,7 +474,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
imageIndexData, err := fetchProtoImageMeta(imageBuck, indexDigest)
imageIndexData, err := getProtoImageMeta(imageBuck, indexDigest)
if err != nil {
return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w",
indexDigest, err)
@ -435,7 +483,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests))
for _, manifest := range imageIndexData.Index.Index.Manifests {
imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest)
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
if err != nil {
return err
}
@ -483,9 +531,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe
continue
}
protoRepoMeta := &proto_go.RepoMeta{}
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob)
if err != nil {
viewError = errors.Join(viewError, err)
@ -506,7 +552,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
imageManifestData, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest)
imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest)
if err != nil {
viewError = errors.Join(viewError, err)
@ -521,7 +567,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
imageIndexData, err := fetchProtoImageMeta(imageMetaBuck, indexDigest)
imageIndexData, err := getProtoImageMeta(imageMetaBuck, indexDigest)
if err != nil {
viewError = errors.Join(viewError, err)
@ -533,7 +579,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe
for _, manifest := range imageIndexData.Index.Index.Manifests {
manifestDigest := manifest.Digest
imageManifestData, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest)
imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest)
if err != nil {
viewError = errors.Join(viewError, err)
@ -588,9 +634,7 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepo
continue
}
repoMeta := proto_go.RepoMeta{}
err := proto.Unmarshal(repoMetaBlob, &repoMeta)
repoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob)
if err != nil {
return err
}
@ -598,7 +642,7 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepo
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
fullRepoMeta := mConvert.GetRepoMeta(&repoMeta)
fullRepoMeta := mConvert.GetRepoMeta(repoMeta)
if filter(fullRepoMeta) {
repos = append(repos, fullRepoMeta)
@ -615,7 +659,7 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepo
}
func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMeta, error) {
var protoRepoMeta proto_go.RepoMeta
var protoRepoMeta *proto_go.RepoMeta
err := bdw.DB.View(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck))
@ -624,13 +668,9 @@ func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMet
repoMetaBlob := buck.Get([]byte(repo))
// object not found
if repoMetaBlob == nil {
return zerr.ErrRepoMetaNotFound
}
var err error
// object found
err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta)
protoRepoMeta, err = unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil {
return err
}
@ -642,7 +682,7 @@ func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMet
return nil
})
return mConvert.GetRepoMeta(&protoRepoMeta), err
return mConvert.GetRepoMeta(protoRepoMeta), err
}
func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string) (mTypes.FullImageMeta, error) {
@ -662,8 +702,9 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string
return zerr.ErrRepoMetaNotFound
}
// object found
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
var err error
protoRepoMeta, err = unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil {
return err
}
@ -677,7 +718,7 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string
return zerr.ErrImageMetaNotFound
}
protoImageMeta, err = fetchProtoImageMeta(imageBuck, descriptor.Digest)
protoImageMeta, err = getProtoImageMeta(imageBuck, descriptor.Digest)
if err != nil {
return err
}
@ -686,7 +727,7 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests {
imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest)
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
if err != nil {
return err
}
@ -709,7 +750,7 @@ func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error
err := bdw.DB.View(func(tx *bbolt.Tx) error {
imageBuck := tx.Bucket([]byte(ImageMetaBuck))
protoImageMeta, err := fetchProtoImageMeta(imageBuck, digest.String())
protoImageMeta, err := getProtoImageMeta(imageBuck, digest.String())
if err != nil {
return err
}
@ -718,7 +759,7 @@ func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests {
imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest)
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
if err != nil {
return err
}
@ -751,16 +792,14 @@ func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta
continue
}
protoRepoMeta := proto_go.RepoMeta{}
err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob)
if err != nil {
return err
}
delete(protoRepoMeta.Tags, "")
repoMeta := mConvert.GetRepoMeta(&protoRepoMeta)
repoMeta := mConvert.GetRepoMeta(protoRepoMeta)
if filter(repoMeta) {
foundRepos = append(foundRepos, repoMeta)
@ -777,9 +816,9 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige
sygMeta mTypes.SignatureMetadata,
) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo))
repoMetaBlob := repoMetaBuck.Get([]byte(repo))
if len(repoMetaBlob) == 0 {
var err error
@ -810,12 +849,10 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
return repoMetaBuck.Put([]byte(repo), repoMetaBlob)
}
protoRepoMeta := &proto_go.RepoMeta{}
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil {
return err
}
@ -852,12 +889,7 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige
manifestSignatures.Map[sygMeta.SignatureType] = signatureSlice
protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures
repoMetaBlob, err = proto.Marshal(protoRepoMeta)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
})
return err
@ -867,16 +899,14 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di
sigMeta mTypes.SignatureMetadata,
) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo))
repoMetaBlob := repoMetaBuck.Get([]byte(repo))
if repoMetaBlob == nil {
return zerr.ErrManifestMetaNotFound
}
protoRepoMeta := proto_go.RepoMeta{}
err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil {
return err
}
@ -900,12 +930,7 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di
protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures
repoMetaBlob, err = proto.Marshal(&protoRepoMeta)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
})
return err
@ -913,28 +938,21 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di
func (bdw *BoltDB) IncrementRepoStars(repo string) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo))
repoMetaBlob := repoMetaBuck.Get([]byte(repo))
if repoMetaBlob == nil {
return zerr.ErrRepoMetaNotFound
}
var repoMeta proto_go.RepoMeta
err := proto.Unmarshal(repoMetaBlob, &repoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil {
return err
}
repoMeta.Stars++
protoRepoMeta.Stars++
repoMetaBlob, err = proto.Marshal(&repoMeta)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
})
return err
@ -942,32 +960,25 @@ func (bdw *BoltDB) IncrementRepoStars(repo string) error {
func (bdw *BoltDB) DecrementRepoStars(repo string) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo))
repoMetaBlob := repoMetaBuck.Get([]byte(repo))
if repoMetaBlob == nil {
return zerr.ErrRepoMetaNotFound
}
var repoMeta proto_go.RepoMeta
err := proto.Unmarshal(repoMetaBlob, &repoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil {
return err
}
if repoMeta.Stars == 0 {
if protoRepoMeta.Stars == 0 {
return nil
}
repoMeta.Stars--
protoRepoMeta.Stars--
repoMetaBlob, err = proto.Marshal(&repoMeta)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
})
return err
@ -996,14 +1007,8 @@ func (bdw *BoltDB) ResetRepoReferences(repo string) error {
repoMetaBlob := buck.Get([]byte(repo))
if repoMetaBlob == nil {
return nil
}
protoRepoMeta := &proto_go.RepoMeta{}
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
if err != nil {
protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
return err
}
@ -1033,18 +1038,13 @@ func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest,
buck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo))
if len(repoMetaBlob) == 0 {
return zerr.ErrRepoMetaNotFound
}
var repoMeta proto_go.RepoMeta
err := proto.Unmarshal(repoMetaBlob, &repoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil {
return err
}
referrersInfo := repoMeta.Referrers[referredDigest.String()].List
referrersInfo := protoRepoMeta.Referrers[referredDigest.String()].List
for i := range referrersInfo {
if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) {
@ -1068,16 +1068,14 @@ func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest,
func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo))
repoMetaBlob := repoMetaBuck.Get([]byte(repo))
if repoMetaBlob == nil {
return zerr.ErrRepoMetaNotFound
}
var repoMeta proto_go.RepoMeta
err := proto.Unmarshal(repoMetaBlob, &repoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil {
return err
}
@ -1086,7 +1084,7 @@ func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error {
if !common.ReferenceIsDigest(reference) {
// search digest for tag
descriptor, found := repoMeta.Tags[reference]
descriptor, found := protoRepoMeta.Tags[reference]
if !found {
return zerr.ErrManifestMetaNotFound
@ -1095,21 +1093,16 @@ func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error {
manifestDigest = descriptor.Digest
}
manifestStatistics, ok := repoMeta.Statistics[manifestDigest]
manifestStatistics, ok := protoRepoMeta.Statistics[manifestDigest]
if !ok {
return zerr.ErrManifestMetaNotFound
}
manifestStatistics.DownloadCount++
manifestStatistics.LastPullTimestamp = timestamppb.Now()
repoMeta.Statistics[manifestDigest] = manifestStatistics
protoRepoMeta.Statistics[manifestDigest] = manifestStatistics
repoMetaBlob, err = proto.Marshal(&repoMeta)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
})
return err
@ -1147,9 +1140,7 @@ func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest
return zerr.ErrRepoMetaNotFound
}
protoRepoMeta := proto_go.RepoMeta{}
err = proto.Unmarshal(repoMetaBlob, &protoRepoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil {
return err
}
@ -1188,12 +1179,7 @@ func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest
protoRepoMeta.Signatures[manifestDigest.String()] = &manifestSignatures
repoMetaBlob, err = proto.Marshal(&protoRepoMeta)
if err != nil {
return err
}
return repoBuck.Put([]byte(repo), repoMetaBlob)
return setProtoRepoMeta(protoRepoMeta, repoBuck)
})
return err
@ -1201,23 +1187,20 @@ func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest
func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
imageMetaBuck := tx.Bucket([]byte(ImageMetaBuck))
repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck))
repoMetaBlob := buck.Get([]byte(repo))
if repoMetaBlob == nil {
return nil
}
repoMeta := &proto_go.RepoMeta{}
err := proto.Unmarshal(repoMetaBlob, repoMeta)
protoRepoMeta, err := getProtoRepoMeta(repo, repoMetaBuck)
if err != nil {
if errors.Is(err, zerr.ErrRepoMetaNotFound) {
return nil
}
return err
}
imageMeta, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest.String())
protoImageMeta, err := getProtoImageMeta(imageMetaBuck, manifestDigest.String())
if err != nil {
if errors.Is(err, zerr.ErrImageMetaNotFound) {
return nil
@ -1227,12 +1210,12 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go
}
// Remove Referrers
if subject := mConvert.GetImageSubject(imageMeta); subject != nil {
if subject := mConvert.GetImageSubject(protoImageMeta); subject != nil {
referredDigest := subject.Digest.String()
refInfo := &proto_go.ReferrersInfo{}
if repoMeta.Referrers[referredDigest] != nil {
refInfo = repoMeta.Referrers[referredDigest]
if protoRepoMeta.Referrers[referredDigest] != nil {
refInfo = protoRepoMeta.Referrers[referredDigest]
}
referrers := refInfo.List
@ -1251,16 +1234,16 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go
refInfo.List = referrers
repoMeta.Referrers[referredDigest] = refInfo
protoRepoMeta.Referrers[referredDigest] = refInfo
}
if !common.ReferenceIsDigest(reference) {
delete(repoMeta.Tags, reference)
delete(protoRepoMeta.Tags, reference)
} else {
// remove all tags pointing to this digest
for tag, desc := range repoMeta.Tags {
for tag, desc := range protoRepoMeta.Tags {
if desc.Digest == reference {
delete(repoMeta.Tags, tag)
delete(protoRepoMeta.Tags, tag)
}
}
}
@ -1268,32 +1251,26 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go
/* try to find at least one tag pointing to manifestDigest
if not found then we can also remove everything related to this digest */
var foundTag bool
for _, desc := range repoMeta.Tags {
for _, desc := range protoRepoMeta.Tags {
if desc.Digest == manifestDigest.String() {
foundTag = true
}
}
if !foundTag {
delete(repoMeta.Statistics, manifestDigest.String())
delete(repoMeta.Signatures, manifestDigest.String())
delete(repoMeta.Referrers, manifestDigest.String())
delete(protoRepoMeta.Statistics, manifestDigest.String())
delete(protoRepoMeta.Signatures, manifestDigest.String())
delete(protoRepoMeta.Referrers, manifestDigest.String())
}
repoBlobsBytes := repoBlobsBuck.Get([]byte(repoMeta.Name))
repoBlobsBytes := repoBlobsBuck.Get([]byte(protoRepoMeta.Name))
repoBlobs := &proto_go.RepoBlobs{}
if len(repoBlobsBytes) == 0 {
repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo)
} else {
err := proto.Unmarshal(repoBlobsBytes, repoBlobs)
if err != nil {
return err
}
repoBlobs, err := unmarshalProtoRepoBlobs(repo, repoBlobsBytes)
if err != nil {
return err
}
repoMeta, repoBlobs, err = common.RemoveImageFromRepoMeta(repoMeta, repoBlobs, reference)
protoRepoMeta, repoBlobs, err = common.RemoveImageFromRepoMeta(protoRepoMeta, repoBlobs, reference)
if err != nil {
return err
}
@ -1303,17 +1280,12 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go
return err
}
err = repoBlobsBuck.Put([]byte(repoMeta.Name), repoBlobsBytes)
err = repoBlobsBuck.Put([]byte(protoRepoMeta.Name), repoBlobsBytes)
if err != nil {
return err
}
repoMetaBlob, err = proto.Marshal(repoMeta)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
})
return err
@ -1371,31 +1343,19 @@ func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.Togg
return zerr.ErrRepoMetaNotFound
}
repoMeta := &proto_go.RepoMeta{}
err = proto.Unmarshal(repoMetaBlob, repoMeta)
protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil {
return err
}
switch res {
case mTypes.Added:
repoMeta.Stars++
protoRepoMeta.Stars++
case mTypes.Removed:
repoMeta.Stars--
protoRepoMeta.Stars--
}
repoMetaBlob, err = proto.Marshal(repoMeta)
if err != nil {
return err
}
err = repoBuck.Put([]byte(repo), repoMetaBlob)
if err != nil {
return err
}
return nil
return setProtoRepoMeta(protoRepoMeta, repoBuck)
}); err != nil {
return mTypes.NotChanged, err
}

8
pkg/meta/convert/convert.go
View File

@ -406,6 +406,10 @@ func GetFullImageMetaFromProto(tag string, protoRepoMeta *proto_go.RepoMeta, pro
func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes.ManifestData,
) []mTypes.FullManifestMeta {
if protoRepoMeta == nil {
return []mTypes.FullManifestMeta{}
}
results := []mTypes.FullManifestMeta{}
for i := range manifestData {
@ -421,6 +425,10 @@ func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes
}
func GetRepoMeta(protoRepoMeta *proto_go.RepoMeta) mTypes.RepoMeta {
if protoRepoMeta == nil {
return mTypes.RepoMeta{}
}
repoDownloads := int32(0)
for _, descriptor := range protoRepoMeta.Tags {

91
pkg/meta/dynamodb/dynamodb.go
View File

@ -208,25 +208,45 @@ func (dwr *DynamoDB) getProtoRepoMeta(ctx context.Context, repo string) (*proto_
return nil, err
}
if resp.Item == nil {
return nil, zerr.ErrRepoMetaNotFound
protoRepoMeta := &proto_go.RepoMeta{
Name: repo,
}
blob := []byte{}
err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &blob)
if err != nil {
return nil, err
if resp.Item != nil {
err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &blob)
if err != nil {
return nil, err
}
err = proto.Unmarshal(blob, protoRepoMeta)
if err != nil {
return nil, err
}
}
repoMeta := &proto_go.RepoMeta{}
err = proto.Unmarshal(blob, repoMeta)
if err != nil {
return nil, err
if protoRepoMeta.Tags == nil {
protoRepoMeta.Tags = map[string]*proto_go.TagDescriptor{"": {}}
}
return repoMeta, nil
if protoRepoMeta.Statistics == nil {
protoRepoMeta.Statistics = map[string]*proto_go.DescriptorStatistics{"": {}}
}
if protoRepoMeta.Signatures == nil {
protoRepoMeta.Signatures = map[string]*proto_go.ManifestSignatures{"": {}}
}
if protoRepoMeta.Referrers == nil {
protoRepoMeta.Referrers = map[string]*proto_go.ReferrersInfo{"": {}}
}
if len(blob) == 0 || resp.Item == nil {
return protoRepoMeta, zerr.ErrRepoMetaNotFound
}
return protoRepoMeta, nil
}
func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, reference string,
@ -252,18 +272,8 @@ func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, referenc
}
repoMeta, err := dwr.getProtoRepoMeta(ctx, repo)
if err != nil {
if !errors.Is(err, zerr.ErrRepoMetaNotFound) {
return err
}
repoMeta = &proto_go.RepoMeta{
Name: repo,
Tags: map[string]*proto_go.TagDescriptor{"": {}},
Statistics: map[string]*proto_go.DescriptorStatistics{"": {}},
Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}},
Referrers: map[string]*proto_go.ReferrersInfo{"": {}},
}
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
return err
}
// 2. Referrers
@ -328,7 +338,7 @@ func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, referenc
}
// 4. Blobs
repoBlobs, err := dwr.getRepoBlobsInfo(repo) //nolint: contextcheck
repoBlobs, err := dwr.getProtoRepoBlobs(ctx, repo)
if err != nil {
return err
}
@ -346,8 +356,8 @@ func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, referenc
return dwr.setProtoRepoMeta(repo, repoMeta) //nolint: contextcheck
}
func (dwr *DynamoDB) getRepoBlobsInfo(repo string) (*proto_go.RepoBlobs, error) {
resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{
func (dwr *DynamoDB) getProtoRepoBlobs(ctx context.Context, repo string) (*proto_go.RepoBlobs, error) {
resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{
TableName: aws.String(dwr.RepoBlobsTablename),
Key: map[string]types.AttributeValue{
"Key": &types.AttributeValueMemberS{Value: repo},
@ -357,29 +367,28 @@ func (dwr *DynamoDB) getRepoBlobsInfo(repo string) (*proto_go.RepoBlobs, error)
return nil, err
}
if resp.Item == nil {
return &proto_go.RepoBlobs{Name: repo, Blobs: make(map[string]*proto_go.BlobInfo)}, nil
repoBlobs := &proto_go.RepoBlobs{
Name: repo,
}
repoBlobsBytes := []byte{}
err = attributevalue.Unmarshal(resp.Item["RepoBlobsInfo"], &repoBlobsBytes)
if err != nil {
return nil, err
}
repoBlobs := &proto_go.RepoBlobs{}
if len(repoBlobsBytes) == 0 {
repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo)
} else {
err := proto.Unmarshal(repoBlobsBytes, repoBlobs)
if resp.Item != nil {
err = attributevalue.Unmarshal(resp.Item["RepoBlobsInfo"], &repoBlobsBytes)
if err != nil {
return nil, err
}
if len(repoBlobsBytes) > 0 {
err := proto.Unmarshal(repoBlobsBytes, repoBlobs)
if err != nil {
return nil, err
}
}
}
if len(repoBlobs.Blobs) == 0 {
repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo)
if repoBlobs.Blobs == nil {
repoBlobs.Blobs = map[string]*proto_go.BlobInfo{"": {}}
}
return repoBlobs, nil
@ -1262,7 +1271,7 @@ func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest
delete(protoRepoMeta.Referrers, manifestDigest.String())
}
repoBlobsInfo, err := dwr.getRepoBlobsInfo(repo)
repoBlobsInfo, err := dwr.getProtoRepoBlobs(context.Background(), repo)
if err != nil {
return err
}

11
pkg/meta/meta_test.go
View File

@ -612,6 +612,17 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func
So(repoMeta2.Size, ShouldEqual, img2Size)
})
Convey("Set, delete and set again", func() {
err := metaDB.SetRepoReference(ctx, repo1, tag1, imgData1)
So(err, ShouldBeNil)
err = metaDB.RemoveRepoReference(repo1, tag1, imgData1.Digest)
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repo1, tag1, imgData1)
So(err, ShouldBeNil)
})
Convey("Check repo blobs info for manifest image", func() {
image1 := CreateImageWith().RandomLayers(2, 10).
ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os1", Architecture: "arch1"}}).