Skip to content

Commit 37443a9

Browse files
vsochvsoch
authored
Merge pull request #8 from supercontainers/add-check-prototype
add example prototype for check
2 parents 83d8d43 + 5ac1cd0 commit 37443a9

File tree

13 files changed

+332
-3
lines changed

13 files changed

+332
-3
lines changed

‎.github/LICENSE-SYSINFO

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
The MIT License (MIT)
2+
3+
Copyright © 2016 Zlatko Čalušić
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎cmd/compspec/check/check.go

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package check
2+
3+
import (
4+
"fmt"
5+
"os"
6+
7+
"github.com/supercontainers/compspec-go/pkg/oras"
8+
"github.com/supercontainers/compspec-go/pkg/types"
9+
"sigs.k8s.io/yaml"
10+
)
11+
12+
var (
13+
defaultMediaType = "application/org.supercontainers.compspec"
14+
)
15+
16+
// loadManifest loads the manifest into a ManifestList
17+
func loadManifest(filename string) (*types.ManifestList, error) {
18+
m := types.ManifestList{}
19+
yamlFile, err := os.ReadFile(filename)
20+
if err != nil {
21+
return &m, err
22+
}
23+
24+
err = yaml.Unmarshal(yamlFile, &m)
25+
if err != nil {
26+
return &m, err
27+
}
28+
return &m, nil
29+
}
30+
31+
// Run will check a manifest list of artifacts against a host machine
32+
// For now, the host machine parameters will be provided as flags
33+
func Run(manifestFile string, hostFields []string, mediaType string) error {
34+
35+
// Default media type if one not provided
36+
if mediaType == "" {
37+
mediaType = defaultMediaType
38+
}
39+
40+
// Cut out early if a spec not provided
41+
if manifestFile == "" {
42+
return fmt.Errorf("A manifest file input -i/--input is required")
43+
}
44+
manifestList, err := loadManifest(manifestFile)
45+
if err != nil {
46+
return err
47+
}
48+
fmt.Println(manifestList)
49+
50+
// Load the compatibility specs into a lookup by image
51+
// This assumes we allow one image per compability spec, not sure
52+
// if there is a use case to have an image twice with two (sounds weird)
53+
lookup := map[string]types.CompatibilityRequest{}
54+
for _, item := range manifestList.Images {
55+
compspec, err := oras.LoadArtifact(item.Artifact, mediaType)
56+
if err != nil {
57+
fmt.Printf("warning, there was an issue loading the artifact for %s, skipping\n", item.Name)
58+
}
59+
lookup[item.Name] = compspec
60+
}
61+
62+
// TODO we will take this set of requests, load them into a graph,
63+
// and then query the graph based on user preferences (the host fields)
64+
// that are provided that describe the host we want to match compatibility with
65+
return nil
66+
}

‎cmd/compspec/compspec.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"os"
77

88
"github.com/akamensky/argparse"
9+
"github.com/supercontainers/compspec-go/cmd/compspec/check"
910
"github.com/supercontainers/compspec-go/cmd/compspec/create"
1011
"github.com/supercontainers/compspec-go/cmd/compspec/extract"
1112
"github.com/supercontainers/compspec-go/cmd/compspec/list"
@@ -32,17 +33,23 @@ func main() {
3233
extractCmd := parser.NewCommand("extract", "Run one or more extractors")
3334
listCmd := parser.NewCommand("list", "List plugins and known sections")
3435
createCmd := parser.NewCommand("create", "Create a compatibility artifact for the current host according to a definition")
36+
checkCmd := parser.NewCommand("check", "Check a manifest of container images / artifact pairs against a set of host fields")
3537

3638
// Shared arguments (likely this will break into check and extract, shared for now)
3739
pluginNames := parser.StringList("n", "name", &argparse.Options{Help: "One or more specific plugins to target names"})
3840

3941
// Extract arguments
4042
filename := extractCmd.String("o", "out", &argparse.Options{Help: "Save extraction to json file"})
4143

44+
// Check arguments
45+
hostFields := checkCmd.StringList("a", "append", &argparse.Options{Help: "Append one or more host fields to include in the check"})
46+
manifestFile := checkCmd.String("i", "in", &argparse.Options{Required: true, Help: "Input manifest list yaml that contains pairs of images and artifacts"})
47+
4248
// Create arguments
43-
options := parser.StringList("a", "append", &argparse.Options{Help: "One or more custom metadata fields to append"})
49+
options := parser.StringList("a", "append", &argparse.Options{Help: "Append one or more custom metadata fields to append"})
4450
specname := createCmd.String("i", "in", &argparse.Options{Required: true, Help: "Input yaml that contains spec for creation"})
4551
specfile := createCmd.String("o", "out", &argparse.Options{Help: "Save compatibility json artifact to this file"})
52+
mediaType := createCmd.String("m", "media-type", &argparse.Options{Help: "The expected media-type for the compatibility artifact"})
4653

4754
// Now parse the arguments
4855
err := parser.Parse(os.Args)
@@ -62,6 +69,12 @@ func main() {
6269
if err != nil {
6370
log.Fatal(err.Error())
6471
}
72+
} else if checkCmd.Happened() {
73+
err := check.Run(*manifestFile, *hostFields, *mediaType)
74+
if err != nil {
75+
log.Fatal(err.Error())
76+
}
77+
6578
} else if listCmd.Happened() {
6679
err := list.Run(*pluginNames)
6780
if err != nil {

‎docs/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ This is early documentation that will be converted eventually to something prett
99
## Thanks and Previous Art
1010

1111
- I learned about kernel parsing from [mfranczy/compat](https://github.com/mfranczy/compat)
12+
- The graph design is based on the (now archived) [kraph](https://github.com/milosgajdos/kraph), released under Apache-2
1213

‎docs/usage.md

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,81 @@ For now we will manually remember the pairing, at least until the compatibility
136136

137137
Check is the command you would use to check a potential host against one or more existing artifacts.
138138
For a small experiment of using create against a set of containers and then testing how to do a check, we are going to place content
139-
in [examples/check-lammps](examples/check-lammps).
139+
in [examples/check-lammps](examples/check-lammps). As an example, we might use the manifest in that directory to run a check.
140+
Note that since most use cases aren't checking the images in the manifest list against the host running the command, we instead
141+
provide the parameters about the expected runtime host to them.
142+
143+
```bash
144+
./bin/compspec check -i ./examples/check-lammps/manifest.yaml
145+
```
146+
147+
<details>
148+
149+
<summary>Example check output</summary>
150+
151+
```console
152+
&{[{ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64 ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64-compspec}]}
153+
Found digest: sha256:b0382d21a2e734ffd3b39160443384fdd44ee7b38e99197f1c832dd73216af2d for intel-mpi-rocky-9-amd64-compspec
154+
{
155+
"version": "0.0.0",
156+
"kind": "CompatibilitySpec",
157+
"metadata": {
158+
"name": "lammps-prototype",
159+
"jsonSchema": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
160+
},
161+
"compatibilities": [
162+
{
163+
"name": "org.supercontainers.mpi",
164+
"version": "0.0.0",
165+
"annotations": {
166+
"implementation": "intel-mpi",
167+
"version": "2021.8"
168+
}
169+
},
170+
{
171+
"name": "org.supercontainers.os",
172+
"version": "0.0.0",
173+
"annotations": {
174+
"name": "Rocky Linux 9.3 (Blue Onyx)",
175+
"release": "9.3",
176+
"vendor": "rocky",
177+
"version": "9.3"
178+
}
179+
},
180+
{
181+
"name": "org.supercontainers.hardware.gpu",
182+
"version": "0.0.0",
183+
"annotations": {
184+
"available": "no"
185+
}
186+
},
187+
{
188+
"name": "io.archspec.cpu",
189+
"version": "0.0.0",
190+
"annotations": {
191+
"model": "13th Gen Intel(R) Core(TM) i5-1335U",
192+
"target": "amd64",
193+
"vendor": "GenuineIntel"
194+
}
195+
}
196+
]
197+
}
198+
{0.0.0 CompatibilitySpec {lammps-prototype https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json} [{org.supercontainers.mpi 0.0.0 map[implementation:intel-mpi version:2021.8]} {org.supercontainers.os 0.0.0 map[name:Rocky Linux 9.3 (Blue Onyx) release:9.3 vendor:rocky version:9.3]} {org.supercontainers.hardware.gpu 0.0.0 map[available:no]} {io.archspec.cpu 0.0.0 map[model:13th Gen Intel(R) Core(TM) i5-1335U target:amd64 vendor:GenuineIntel]}]}
199+
```
200+
201+
</details>
202+
203+
Note that if you provide (append) zero host fields with `-a` for each, you will basically get back the listing of ordered images.
204+
The command logic and (very simple) algortithm works as follows.
205+
206+
1. Read in all entries from the list
207+
2. Retrieve their artifacts, look for the "application/org.supercontainers.compspec" layer media type to identify it.
208+
3. Retrieve it (within the application) and parse into the compatibility metadata
209+
4. Create a flat graph with a node for each image, and annotations as the attributes
210+
5. Search the graph based on the provided preferences
211+
212+
Note that the current prototype knows how to read in the manifest and print out the json. I am going to play around with graphs (and making a library) soon that we can use here.
213+
140214

141215
## Extract
142216

‎examples/check-lammps/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ cmd=". /etc/profile && /tmp/data/generate-artifact.sh no /tmp/data/specs/compspe
3131
docker run -v $PWD:/tmp/data -it ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64 /bin/bash -c "$cmd"
3232

3333
# This generates ./specs/compspec-intel-mpi-rocky-9-amd64.json, let's push to a registry with oras
34-
oras push ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64-compspec --artifact-type application/org.supercontainers.compspec ./specs/compspec-intel-mpi-rocky-9-amd64.json
34+
oras push ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64-compspec-test --artifact-type application/org.supercontainers.compspec ./specs/compspec-intel-mpi-rocky-9-amd64.json:application/org.supercontainers.compspec
3535
```
3636

3737
Here is how we might see it:

‎examples/check-lammps/manifest.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Check manifest
2+
# This is an example manifest to manually pair container images with their compatibility artifacts
3+
# note that the artifact layer type needs to be "application/org.supercontainers.compspec" to be discovered
4+
images:
5+
- name: ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64
6+
artifact: ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64-compspec

‎examples/generated-compatibility-spec.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,16 @@
1414
"version": "4.1.1"
1515
}
1616
},
17+
{
18+
"name": "org.supercontainers.os",
19+
"version": "0.0.0",
20+
"annotations": {
21+
"name": "Ubuntu 22.04.3 LTS",
22+
"release": "22.04.3",
23+
"vendor": "ubuntu",
24+
"version": "22.04"
25+
}
26+
},
1727
{
1828
"name": "org.supercontainers.hardware.gpu",
1929
"version": "0.0.0",

‎go.mod

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,18 @@ require (
77
github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8
88
github.com/jedib0t/go-pretty/v6 v6.5.3
99
github.com/moby/moby v25.0.0+incompatible
10+
github.com/opencontainers/image-spec v1.1.0-rc5
1011
golang.org/x/sys v0.16.0
12+
oras.land/oras-go/v2 v2.3.1
1113
sigs.k8s.io/yaml v1.4.0
1214
)
1315

1416
require (
1517
github.com/containerd/log v0.1.0 // indirect
1618
github.com/mattn/go-runewidth v0.0.15 // indirect
19+
github.com/opencontainers/go-digest v1.0.0 // indirect
1720
github.com/rivo/uniseg v0.2.0 // indirect
1821
github.com/sirupsen/logrus v1.9.3 // indirect
22+
golang.org/x/sync v0.4.0 // indirect
1923
gotest.tools/v3 v3.5.1 // indirect
2024
)

‎go.sum

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
1515
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
1616
github.com/moby/moby v25.0.0+incompatible h1:KIFudkwXNK+kBrnCxWZNwhEf/jJzdjQAP7EF/awywMI=
1717
github.com/moby/moby v25.0.0+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc=
18+
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
19+
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
20+
github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
21+
github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
1822
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
1923
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
2024
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
@@ -24,6 +28,8 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs
2428
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
2529
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
2630
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
31+
golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
32+
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
2733
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
2834
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
2935
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
@@ -33,5 +39,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
3339
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
3440
gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
3541
gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
42+
oras.land/oras-go/v2 v2.3.1 h1:lUC6q8RkeRReANEERLfH86iwGn55lbSWP20egdFHVec=
43+
oras.land/oras-go/v2 v2.3.1/go.mod h1:5AQXVEu1X/FKp1F9DMOb5ZItZBOa0y5dha0yCm4NR9c=
3644
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
3745
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

0 commit comments

Comments
 (0)