Skip to content

Commit 73856e6

Browse files
jabrown85jabrown85
authored
Merge pull request #305 from bojidar-bg/l1599-local-platform
Take two: Make local/docker implementation platform-aware
2 parents 65b01ef + c36b3eb commit 73856e6

6 files changed

Lines changed: 233 additions & 23 deletions

File tree

‎.github/workflows/test.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,32 @@ jobs:
2323
uses: codecov/codecov-action@v4
2424
with:
2525
token: ${{ secrets.CODECOV_TOKEN }}
26+
test-and-build-linux-containerd:
27+
runs-on: ubuntu-latest
28+
steps:
29+
- uses: actions/checkout@v4
30+
- name: Set up Docker with Containerd
31+
uses: docker/setup-docker-action@v5
32+
with:
33+
set-host: true
34+
daemon-config: |
35+
{
36+
"features": {
37+
"containerd-snapshotter": true
38+
}
39+
}
40+
- name: Set up go
41+
uses: actions/setup-go@v5
42+
with:
43+
go-version-file: 'go.mod'
44+
- name: Test
45+
run: make test
46+
env:
47+
PLATFORM_AWARE_DOCKER: 'true'
48+
- name: Upload Coverage to Codecov
49+
uses: codecov/codecov-action@v4
50+
with:
51+
token: ${{ secrets.CODECOV_TOKEN }}
2652
test-and-build-windows:
2753
runs-on: windows-2022
2854
steps:

‎local/local_test.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/google/go-containerregistry/pkg/authn"
1414
v1 "github.com/google/go-containerregistry/pkg/v1"
1515
"github.com/moby/moby/client"
16+
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
1617
"github.com/sclevine/spec"
1718
"github.com/sclevine/spec/report"
1819

@@ -291,6 +292,62 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
291292
})
292293
})
293294

295+
platformAwareWhen := when.Pend
296+
if h.DockerIsPlatformAware() {
297+
platformAwareWhen = when
298+
}
299+
platformAwareWhen("base image has multiple platforms available", func() {
300+
it("uses the matching platform", func() {
301+
// linux/arm64 busybox image
302+
multiplatformBaseImageName := "busybox"
303+
expectedArchitecture := "arm64"
304+
expectedOSVersion := ""
305+
distractingArchitecture := "amd64"
306+
distractingOSVersion := ""
307+
if daemonOS == "windows" {
308+
// windows/arm nanoserver image
309+
multiplatformBaseImageName = "mcr.microsoft.com/windows/nanoserver:10.0.17763.1040"
310+
expectedArchitecture = "arm"
311+
expectedOSVersion = "10.0.17763.1040"
312+
distractingArchitecture = "amd64"
313+
distractingOSVersion = "10.0.17763.1040"
314+
}
315+
316+
h.PullWithPlatformIfMissing(t, dockerClient, multiplatformBaseImageName, ocispec.Platform{
317+
OS: daemonOS,
318+
Architecture: expectedArchitecture,
319+
OSVersion: expectedOSVersion,
320+
})
321+
h.PullWithPlatformIfMissing(t, dockerClient, multiplatformBaseImageName, ocispec.Platform{
322+
OS: daemonOS,
323+
Architecture: distractingArchitecture,
324+
OSVersion: distractingOSVersion,
325+
})
326+
327+
img, err := local.NewImage(
328+
newTestImageName(),
329+
dockerClient,
330+
local.FromBaseImage(multiplatformBaseImageName),
331+
local.WithDefaultPlatform(imgutil.Platform{
332+
OS: daemonOS,
333+
Architecture: expectedArchitecture,
334+
OSVersion: expectedOSVersion,
335+
}),
336+
)
337+
h.AssertNil(t, err)
338+
h.AssertNil(t, img.Save())
339+
defer h.DockerRmi(dockerClient, img.Name())
340+
341+
imgArch, err := img.Architecture()
342+
h.AssertNil(t, err)
343+
h.AssertEq(t, imgArch, expectedArchitecture)
344+
345+
imgOSVersion, err := img.OSVersion()
346+
h.AssertNil(t, err)
347+
h.AssertEq(t, imgOSVersion, expectedOSVersion)
348+
})
349+
})
350+
294351
when("base image does not exist", func() {
295352
it("returns an empty image based on platform fields", func() {
296353
img, err := local.NewImage(

‎local/new.go‎

Lines changed: 87 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,14 @@ package local
33
import (
44
"context"
55
"fmt"
6+
"runtime"
67

78
cerrdefs "github.com/containerd/errdefs"
89
v1 "github.com/google/go-containerregistry/pkg/v1"
910
"github.com/moby/moby/api/types/image"
1011
"github.com/moby/moby/client"
12+
"github.com/moby/moby/client/pkg/versions"
13+
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
1114

1215
"github.com/buildpacks/imgutil"
1316
)
@@ -21,12 +24,13 @@ func NewImage(repoName string, dockerClient DockerClient, ops ...imgutil.ImageOp
2124
}
2225

2326
var err error
24-
options.Platform, err = processPlatformOption(options.Platform, dockerClient)
27+
var isPlatformAware bool
28+
options.Platform, isPlatformAware, err = processPlatformOption(options.Platform, dockerClient)
2529
if err != nil {
2630
return nil, err
2731
}
2832

29-
previousImage, err := processImageOption(options.PreviousImageRepoName, dockerClient, true)
33+
previousImage, err := processImageOption(options.PreviousImageRepoName, isPlatformAware, options.Platform, dockerClient, true)
3034
if err != nil {
3135
return nil, err
3236
}
@@ -38,7 +42,7 @@ func NewImage(repoName string, dockerClient DockerClient, ops ...imgutil.ImageOp
3842
baseIdentifier string
3943
store *Store
4044
)
41-
baseImage, err := processImageOption(options.BaseImageRepoName, dockerClient, false)
45+
baseImage, err := processImageOption(options.BaseImageRepoName, isPlatformAware, options.Platform, dockerClient, false)
4246
if err != nil {
4347
return nil, err
4448
}
@@ -47,7 +51,11 @@ func NewImage(repoName string, dockerClient DockerClient, ops ...imgutil.ImageOp
4751
baseIdentifier = baseImage.identifier
4852
store = baseImage.layerStore
4953
} else {
50-
store = NewStore(dockerClient)
54+
if isPlatformAware {
55+
store = NewStoreWithPlatform(dockerClient, options.Platform)
56+
} else {
57+
store = NewStore(dockerClient)
58+
}
5159
}
5260

5361
cnbImage, err := imgutil.NewCNBImage(*options)
@@ -64,30 +72,35 @@ func NewImage(repoName string, dockerClient DockerClient, ops ...imgutil.ImageOp
6472
}, nil
6573
}
6674

67-
func defaultPlatform(dockerClient DockerClient) (imgutil.Platform, error) {
75+
func defaultPlatform(dockerClient DockerClient) (imgutil.Platform, bool, error) {
6876
daemonInfo, err := dockerClient.ServerVersion(context.Background(), client.ServerVersionOptions{})
6977
if err != nil {
70-
return imgutil.Platform{}, err
78+
return imgutil.Platform{}, false, err
79+
}
80+
isPlatformAware := versions.GreaterThanOrEqualTo(daemonInfo.APIVersion, "1.49")
81+
// When running on a different architecture than the daemon, we want to use images matching our own architecture
82+
// https://github.com/buildpacks/lifecycle/issues/1599
83+
if isPlatformAware {
84+
return imgutil.Platform{
85+
OS: runtime.GOOS,
86+
Architecture: runtime.GOARCH,
87+
}, isPlatformAware, nil
7188
}
7289
return imgutil.Platform{
7390
OS: daemonInfo.Os,
7491
Architecture: daemonInfo.Arch,
75-
}, nil
92+
}, isPlatformAware, nil
7693
}
7794

78-
func processPlatformOption(requestedPlatform imgutil.Platform, dockerClient DockerClient) (imgutil.Platform, error) {
79-
dockerPlatform, err := defaultPlatform(dockerClient)
95+
func processPlatformOption(requestedPlatform imgutil.Platform, dockerClient DockerClient) (imgutil.Platform, bool, error) {
96+
defaultPlatform, isPlatformAware, err := defaultPlatform(dockerClient)
8097
if err != nil {
81-
return imgutil.Platform{}, err
98+
return imgutil.Platform{}, false, err
8299
}
83100
if (requestedPlatform == imgutil.Platform{}) {
84-
return dockerPlatform, nil
85-
}
86-
if requestedPlatform.OS != "" && requestedPlatform.OS != dockerPlatform.OS {
87-
return imgutil.Platform{},
88-
fmt.Errorf("invalid os: platform os %q must match the daemon os %q", requestedPlatform.OS, dockerPlatform.OS)
101+
return defaultPlatform, isPlatformAware, nil
89102
}
90-
return requestedPlatform, nil
103+
return requestedPlatform, isPlatformAware, nil
91104
}
92105

93106
type imageResult struct {
@@ -96,25 +109,52 @@ type imageResult struct {
96109
layerStore *Store
97110
}
98111

99-
func processImageOption(repoName string, dockerClient DockerClient, downloadLayersOnAccess bool) (imageResult, error) {
112+
func processImageOption(repoName string, isPlatformAware bool, platform imgutil.Platform, dockerClient DockerClient, downloadLayersOnAccess bool) (imageResult, error) {
100113
if repoName == "" {
101114
return imageResult{}, nil
102115
}
103116
inspect, history, err := getInspectAndHistory(repoName, dockerClient)
104117
if err != nil {
105118
return imageResult{}, err
106119
}
120+
107121
if inspect == nil {
108122
return imageResult{}, nil
109123
}
110-
layerStore := NewStore(dockerClient)
124+
125+
// Always use the platform-unaware image ID
126+
identifier := inspect.ID
127+
128+
// Try using the platform-specific inspected value if possible, otherwise fall back to the generic inspect
129+
if isPlatformAware {
130+
platformInspect, platformHistory, err := getPlatformAwareInspectAndHistory(repoName, platform, dockerClient)
131+
if err != nil {
132+
return imageResult{}, err
133+
}
134+
if platformInspect != nil && platformHistory != nil {
135+
inspect = platformInspect
136+
history = platformHistory
137+
}
138+
}
139+
140+
var layerStore *Store
141+
if isPlatformAware {
142+
layerStore = NewStoreWithPlatform(dockerClient, imgutil.Platform{
143+
Architecture: inspect.Architecture,
144+
OS: inspect.Os,
145+
OSVersion: inspect.OsVersion,
146+
Variant: inspect.Variant,
147+
})
148+
} else {
149+
layerStore = NewStore(dockerClient)
150+
}
111151
v1Image, err := newV1ImageFacadeFromInspect(*inspect, history, layerStore, downloadLayersOnAccess)
112152
if err != nil {
113153
return imageResult{}, err
114154
}
115155
return imageResult{
116156
image: v1Image,
117-
identifier: inspect.ID,
157+
identifier: identifier,
118158
layerStore: layerStore,
119159
}, nil
120160
}
@@ -131,5 +171,33 @@ func getInspectAndHistory(repoName string, dockerClient DockerClient) (*image.In
131171
if err != nil {
132172
return nil, nil, fmt.Errorf("get history for image %q: %w", repoName, err)
133173
}
174+
134175
return &inspect.InspectResponse, historyResult.Items, nil
135176
}
177+
178+
func getPlatformAwareInspectAndHistory(repoName string, platform imgutil.Platform, dockerClient DockerClient) (*image.InspectResponse, []image.HistoryResponseItem, error) {
179+
ociPlatform := ocispec.Platform{
180+
Architecture: platform.Architecture,
181+
OS: platform.OS,
182+
OSVersion: platform.OSVersion,
183+
Variant: platform.Variant,
184+
}
185+
186+
platformHistoryResult, err := dockerClient.ImageHistory(context.Background(), repoName, client.ImageHistoryWithPlatform(ociPlatform))
187+
if err != nil {
188+
if cerrdefs.IsNotFound(err) {
189+
return nil, nil, nil
190+
}
191+
return nil, nil, fmt.Errorf("get history for image %q: %w", repoName, err)
192+
}
193+
194+
platformInspect, err := dockerClient.ImageInspect(context.Background(), repoName, client.ImageInspectWithPlatform(&ociPlatform))
195+
if err != nil {
196+
if cerrdefs.IsNotFound(err) {
197+
return nil, nil, nil
198+
}
199+
return nil, nil, fmt.Errorf("inspecting platform-specific image %q: %w", repoName, err)
200+
}
201+
202+
return &platformInspect.InspectResponse, platformHistoryResult.Items, nil
203+
}

‎local/store.go‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/moby/moby/api/types/image"
2020
"github.com/moby/moby/api/types/jsonstream"
2121
"github.com/moby/moby/client"
22+
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
2223
"golang.org/x/sync/errgroup"
2324

2425
"github.com/buildpacks/imgutil"
@@ -33,6 +34,7 @@ type Store struct {
3334
// optional
3435
downloadOnce *sync.Once
3536
onDiskLayersByDiffID map[v1.Hash]annotatedLayer
37+
platform imgutil.Platform
3638
}
3739

3840
// DockerClient is subset of client.APIClient required by this package.
@@ -60,6 +62,15 @@ func NewStore(dockerClient DockerClient) *Store {
6062
}
6163
}
6264

65+
func NewStoreWithPlatform(dockerClient DockerClient, platform imgutil.Platform) *Store {
66+
return &Store{
67+
dockerClient: dockerClient,
68+
downloadOnce: &sync.Once{},
69+
onDiskLayersByDiffID: make(map[v1.Hash]annotatedLayer),
70+
platform: platform,
71+
}
72+
}
73+
6374
// images
6475

6576
func (s *Store) Contains(identifier string) bool {
@@ -408,7 +419,22 @@ func (s *Store) doDownloadLayersFor(identifier string) error {
408419
}
409420
ctx := context.Background()
410421

411-
imageReader, err := s.dockerClient.ImageSave(ctx, []string{identifier})
422+
var imageReader client.ImageSaveResult
423+
var err error
424+
425+
if s.platform != (imgutil.Platform{}) {
426+
// Download right platform on platform-aware Docker
427+
ociPlatform := ocispec.Platform{
428+
Architecture: s.platform.Architecture,
429+
OS: s.platform.OS,
430+
OSVersion: s.platform.OSVersion,
431+
Variant: s.platform.Variant,
432+
}
433+
imageReader, err = s.dockerClient.ImageSave(ctx, []string{identifier}, client.ImageSaveWithPlatforms(ociPlatform))
434+
} else {
435+
imageReader, err = s.dockerClient.ImageSave(ctx, []string{identifier})
436+
}
437+
412438
if err != nil {
413439
return fmt.Errorf("saving image with ID %q from the docker daemon: %w", identifier, err)
414440
}

‎local/v1_facade.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@ func newV1ImageFacadeFromInspect(dockerInspect image.InspectResponse, history []
2727
return nil, err
2828
}
2929
configFile := &v1.ConfigFile{
30-
Architecture: dockerInspect.Architecture, // FIXME: this should come from options.Platform
30+
Architecture: dockerInspect.Architecture,
3131
Author: dockerInspect.Author,
3232
Created: toV1Time(dockerInspect.Created),
3333
History: imgutil.NormalizedHistory(toV1History(history), len(dockerInspect.RootFS.Layers)),
3434
OS: dockerInspect.Os,
3535
RootFS: rootFS,
3636
Config: toV1Config(dockerInspect.Config),
37-
OSVersion: dockerInspect.OsVersion, // FIXME: this should come from options.Platform
38-
Variant: dockerInspect.Variant, // FIXME: this should come from options.Platform
37+
OSVersion: dockerInspect.OsVersion,
38+
Variant: dockerInspect.Variant,
3939
}
4040
layersToSet := newEmptyLayerListFrom(configFile, downloadLayersOnAccess, withStore, dockerInspect.ID)
4141
return imageFrom(layersToSet, configFile, imgutil.DockerTypes) // FIXME: this should be configurable with options.MediaTypes

0 commit comments

Comments
 (0)