Skip to content

Commit 3146e80

Browse files
authored
Merge pull request #548 from simonchrz/fix/mirror-upstream-versions
List mirror versions from the configured upstream
2 parents e141f16 + 0fc8ae7 commit 3146e80

13 files changed

Lines changed: 456 additions & 38 deletions

‎cmd/proxy/main.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -558,6 +558,13 @@ func runMirror() {
558558

559559
m := mirror.New(proxy, db, store, logger, *concurrency)
560560
m.Images = images
561+
m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient)
562+
switch s := source.(type) {
563+
case *mirror.PURLSource:
564+
s.Versions = m.Versions
565+
case *mirror.SBOMSource:
566+
s.Versions = m.Versions
567+
}
561568
if cfg.Scanning.Enabled {
562569
refuseUnscannedMirroring(m)
563570
}

‎docs/configuration.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1524,6 +1524,10 @@ Positional arguments are treated as PURLs:
15241524
proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
15251525
```
15261526

1527+
### Unversioned PURLs
1528+
1529+
A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version.
1530+
15271531
### Container images
15281532

15291533
`pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable.

‎internal/mirror/job.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,9 +209,9 @@ func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint
209209
case len(req.PURLs) > 0 && len(req.SBOM) > 0:
210210
return nil, fmt.Errorf("request must include only one of purls or sbom")
211211
case len(req.PURLs) > 0:
212-
return &PURLSource{PURLs: req.PURLs, Platforms: req.Platforms}, nil
212+
return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil
213213
case len(req.SBOM) > 0:
214-
return &SBOMSource{Data: req.SBOM, Platforms: req.Platforms}, nil
214+
return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil
215215
case req.Registry != "":
216216
return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead")
217217
default:

‎internal/mirror/mirror.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ type Mirror struct {
2727
// other unsupported ecosystem.
2828
Images ImageMirrorer
2929

30+
// Versions lists the versions of unversioned PURLs in mirror jobs and
31+
// schedules. Nil lists them from the public registries.
32+
Versions VersionLister
33+
3034
// RefusePackages, when set, fails every item Images does not take with
3135
// this reason instead of fetching it, so nothing is cached.
3236
RefusePackages string

‎internal/mirror/sbom_version_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func TestSBOMSourceVersionPrecedence(t *testing.T) {
4040
t.Run(test.name, func(t *testing.T) {
4141
data := marshalJSON(t, map[string]any{"components": []map[string]any{{"name": "app", "version": test.componentVersion, "purl": test.imagePURL}}})
4242
var got []PackageVersion
43-
source := &SBOMSource{Data: data, RegClient: registries.NewClient(registries.WithHTTPClient(&http.Client{Transport: rejectingMirrorTransport{}}))}
43+
source := &SBOMSource{Data: data, Versions: RegistryVersions{Client: registries.NewClient(registries.WithHTTPClient(&http.Client{Transport: rejectingMirrorTransport{}}))}}
4444
if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { got = append(got, pv); return nil }); err != nil {
4545
t.Fatal(err)
4646
}

‎internal/mirror/schedule.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) {
102102
}
103103
}
104104
started := time.Now()
105-
progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Platforms: schedule.Platforms})
105+
progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms})
106106
if ctx.Err() != nil {
107107
if s.Claim != nil && s.Release != nil {
108108
if err := s.Release(key); err != nil {

‎internal/mirror/source.go‎

Lines changed: 17 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ import (
55
"fmt"
66

77
"github.com/git-pkgs/purl"
8-
"github.com/git-pkgs/registries"
9-
_ "github.com/git-pkgs/registries/all"
108
"github.com/git-pkgs/sbom"
119
)
1210

@@ -44,18 +42,21 @@ type Source interface {
4442
}
4543

4644
// PURLSource yields packages from PURL strings.
47-
// Versioned PURLs produce a single item. Unversioned PURLs look up all versions from the registry.
45+
// Versioned PURLs produce a single item. Unversioned PURLs produce one item
46+
// per version Versions lists.
4847
type PURLSource struct {
49-
PURLs []string
50-
RegClient *registries.Client
48+
PURLs []string
49+
// Versions lists the versions of unversioned PURLs. Nil lists them from
50+
// the public registries.
51+
Versions VersionLister
5152
// Platforms limit which platforms of container images are mirrored.
5253
Platforms []string
5354
}
5455

5556
func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) error) error {
56-
client := s.RegClient
57-
if client == nil {
58-
client = registries.DefaultClient()
57+
versions := s.Versions
58+
if versions == nil {
59+
versions = RegistryVersions{}
5960
}
6061

6162
for _, purlStr := range s.PURLs {
@@ -85,11 +86,11 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
8586
}
8687

8788
// Unversioned: enumerate all versions
88-
versions, err := s.fetchVersions(ctx, client, ecosystem, name)
89+
listed, err := versions.ListVersions(ctx, ecosystem, name)
8990
if err != nil {
9091
return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err)
9192
}
92-
for _, v := range versions {
93+
for _, v := range listed {
9394
if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v}); err != nil {
9495
return err
9596
}
@@ -98,27 +99,13 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
9899
return nil
99100
}
100101

101-
func (s *PURLSource) fetchVersions(ctx context.Context, client *registries.Client, ecosystem, name string) ([]string, error) {
102-
reg, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", client)
103-
if err != nil {
104-
return nil, err
105-
}
106-
versions, err := reg.FetchVersions(ctx, name)
107-
if err != nil {
108-
return nil, err
109-
}
110-
result := make([]string, len(versions))
111-
for i, v := range versions {
112-
result[i] = v.Number
113-
}
114-
return result, nil
115-
}
116-
117102
// SBOMSource extracts package versions from CycloneDX or SPDX SBOM data.
118103
type SBOMSource struct {
119-
Data []byte
120-
Name string
121-
RegClient *registries.Client
104+
Data []byte
105+
Name string
106+
// Versions lists the versions of unversioned PURLs. Nil lists them from
107+
// the public registries.
108+
Versions VersionLister
122109
// Platforms limit which platforms of container images are mirrored.
123110
Platforms []string
124111
}
@@ -132,7 +119,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
132119
return fmt.Errorf("parsing SBOM: %w", err)
133120
}
134121

135-
inner := &PURLSource{PURLs: purls, RegClient: s.RegClient, Platforms: s.Platforms}
122+
inner := &PURLSource{PURLs: purls, Versions: s.Versions, Platforms: s.Platforms}
136123
return inner.Enumerate(ctx, fn)
137124
}
138125

‎internal/mirror/versions.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package mirror
2+
3+
import (
4+
"context"
5+
6+
"github.com/git-pkgs/purl"
7+
"github.com/git-pkgs/registries"
8+
_ "github.com/git-pkgs/registries/all"
9+
)
10+
11+
// VersionLister lists the versions of a package, for PURLs without one.
12+
type VersionLister interface {
13+
ListVersions(ctx context.Context, ecosystem, name string) ([]string, error)
14+
}
15+
16+
// RegistryVersions lists versions through the registries clients.
17+
type RegistryVersions struct {
18+
// Client makes the requests. Nil uses registries.DefaultClient.
19+
Client *registries.Client
20+
// URL returns the registry base URL to list a package from, by PURL
21+
// type. Nil, or an empty URL, uses the public registry.
22+
URL func(purlType, name string) (string, error)
23+
}
24+
25+
// ListVersions implements VersionLister.
26+
func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) {
27+
purlType := purl.EcosystemToPURLType(ecosystem)
28+
var baseURL string
29+
if r.URL != nil {
30+
var err error
31+
if baseURL, err = r.URL(purlType, name); err != nil {
32+
return nil, err
33+
}
34+
}
35+
client := r.Client
36+
if client == nil {
37+
client = registries.DefaultClient()
38+
}
39+
reg, err := registries.New(purlType, baseURL, client)
40+
if err != nil {
41+
return nil, err
42+
}
43+
versions, err := reg.FetchVersions(ctx, name)
44+
if err != nil {
45+
return nil, err
46+
}
47+
result := make([]string, len(versions))
48+
for i, v := range versions {
49+
result[i] = v.Number
50+
}
51+
return result, nil
52+
}

‎internal/mirror/versions_test.go‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package mirror
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"reflect"
8+
"testing"
9+
)
10+
11+
// fakeVersions lists fixed versions and records what it was asked for.
12+
type fakeVersions struct {
13+
versions []string
14+
err error
15+
asked []string
16+
}
17+
18+
func (f *fakeVersions) ListVersions(_ context.Context, ecosystem, name string) ([]string, error) {
19+
f.asked = append(f.asked, ecosystem+"/"+name)
20+
return f.versions, f.err
21+
}
22+
23+
func enumerate(t *testing.T, source Source) []PackageVersion {
24+
t.Helper()
25+
var items []PackageVersion
26+
if err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
27+
items = append(items, pv)
28+
return nil
29+
}); err != nil {
30+
t.Fatalf("Enumerate() error = %v", err)
31+
}
32+
return items
33+
}
34+
35+
func TestPURLSourceListsVersionsThroughLister(t *testing.T) {
36+
versions := &fakeVersions{versions: []string{"1.0.0", "1.1.0"}}
37+
source := &PURLSource{PURLs: []string{"pkg:npm/%40acme/widget"}, Versions: versions}
38+
39+
items := enumerate(t, source)
40+
41+
want := []PackageVersion{
42+
{Ecosystem: "npm", Name: "@acme/widget", Version: "1.0.0"},
43+
{Ecosystem: "npm", Name: "@acme/widget", Version: "1.1.0"},
44+
}
45+
if !reflect.DeepEqual(items, want) {
46+
t.Errorf("items = %v, want %v", items, want)
47+
}
48+
if !reflect.DeepEqual(versions.asked, []string{"npm/@acme/widget"}) {
49+
t.Errorf("lister asked for %v, want npm/@acme/widget", versions.asked)
50+
}
51+
}
52+
53+
func TestPURLSourceReportsListerError(t *testing.T) {
54+
source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{err: errors.New("upstream down")}}
55+
err := source.Enumerate(context.Background(), func(PackageVersion) error { return nil })
56+
if err == nil {
57+
t.Fatal("Enumerate() succeeded although listing versions failed")
58+
}
59+
}
60+
61+
func TestSBOMSourceListsVersionsThroughLister(t *testing.T) {
62+
bom := map[string]any{
63+
"bomFormat": "CycloneDX",
64+
"specVersion": "1.4",
65+
"components": []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}},
66+
}
67+
versions := &fakeVersions{versions: []string{"1.3.0"}}
68+
source := &SBOMSource{Data: marshalJSON(t, bom), Versions: versions}
69+
70+
items := enumerate(t, source)
71+
72+
if len(items) != 1 || items[0].Version != "1.3.0" {
73+
t.Errorf("items = %v, want left-pad@1.3.0 from the lister", items)
74+
}
75+
}
76+
77+
func TestSourceFromRequestUsesMirrorVersions(t *testing.T) {
78+
m := setupTestMirror(t, 1)
79+
versions := &fakeVersions{}
80+
m.Versions = versions
81+
js := NewJobStore(context.Background(), m)
82+
83+
source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}})
84+
if err != nil {
85+
t.Fatalf("sourceFromRequest() error = %v", err)
86+
}
87+
if got := source.(*PURLSource).Versions; got != versions {
88+
t.Errorf("PURL source lists versions through %v, want the mirror's lister", got)
89+
}
90+
91+
source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`)})
92+
if err != nil {
93+
t.Fatalf("sourceFromRequest() error = %v", err)
94+
}
95+
if got := source.(*SBOMSource).Versions; got != versions {
96+
t.Errorf("SBOM source lists versions through %v, want the mirror's lister", got)
97+
}
98+
}

‎internal/server/mirror_schedule.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,7 @@ func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy)
1717
return
1818
}
1919
// One worker, so a schedule never fans out against the registry.
20-
m := mirror.New(proxy, s.db, s.storage, s.logger, 1)
21-
m.Images = s.newContainerHandler(proxy)
20+
m := s.newMirror(proxy, 1)
2221
schedules := make([]mirror.Schedule, len(s.cfg.Mirror.Schedules))
2322
for i, schedule := range s.cfg.Mirror.Schedules {
2423
schedules[i] = mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()}

0 commit comments

Comments
 (0)