Skip to content

Commit 9f28d93

Browse files
authored
feat: initial commit for erofs support (#25)
* feat: initial commit for erofs support Before this commit, only squashfs was supported. However, there are other filesystems such as erofs that fit the same theme, and additional filesystem support requires refactoring and exposing a more generic filesystem interface. pkg/fs/fs.go - Filesystem interface pkg/squashfs - squashfs pkg/erofs - erofs pkg/common - filesystem-agnostic common routines pkg/verity - verity routines Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> * fix: disable "advanced/incompatible" features for now Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> * fix!: drop the double invalid '+' in layer media-type BREAKING-CHANGE: the layer media-type no longer contains "+verity" For a layer media-type, we add the fstype+compression+verity_present. Only one '+' is allowed as per following RFC. https://datatracker.ietf.org/doc/html/rfc6838#section-4.2 Instead just rely on "root_hash" annotation. Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> * test: add a stacker.yaml for tests * fix: rebasing after Mike's changes Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> * fix: some more refactoring * fix: remove notify support from erofs * fix: try a released stacker version Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com> --------- Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com> Signed-off-by: Ramkumar Chinchani <rchincha.dev@gmail.com>
1 parent fb656de commit 9f28d93

41 files changed

Lines changed: 1808 additions & 595 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/build.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
sudo apt-get update
2121
sudo apt-get install bats fuse3 make libcryptsetup-dev libgpgme-dev \
2222
libcap-dev lxc libdevmapper-dev libacl1-dev libarchive-tools \
23-
squashfuse squashfs-tools
23+
squashfuse squashfs-tools erofs-utils
2424
- name: setup lxc
2525
run: |
2626
chmod ugo+x $HOME

Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ VERSION_LDFLAGS=-X main.Version=$(MAIN_VERSION)
1212
BATS = $(TOOLS_D)/bin/bats
1313
BATS_VERSION := v1.10.0
1414
STACKER = $(TOOLS_D)/bin/stacker
15-
STACKER_VERSION := v1.0.0
15+
STACKER_VERSION := v1.1.0-erofs
1616
TOOLS_D := $(ROOT)/tools
1717
GOCOVERDIR ?= $(ROOT)
1818

@@ -36,7 +36,7 @@ gotest: $(GO_SRC)
3636

3737
$(STACKER):
3838
mkdir -p $(TOOLS_D)/bin
39-
wget --progress=dot:giga https://github.com/project-stacker/stacker/releases/download/$(STACKER_VERSION)/stacker
39+
wget --progress=dot:giga https://github.com/rchincha/stacker/releases/download/$(STACKER_VERSION)/stacker
4040
chmod +x stacker
4141
cp stacker $(TOOLS_D)/bin/
4242

atomfs.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
package atomfs

cmd/atomfs/mount.go

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,8 @@ import (
99

1010
"github.com/pkg/errors"
1111
"github.com/urfave/cli"
12-
13-
"machinerun.io/atomfs"
14-
"machinerun.io/atomfs/squashfs"
12+
"machinerun.io/atomfs/pkg/common"
13+
"machinerun.io/atomfs/pkg/molecule"
1514
)
1615

1716
var mountCmd = cli.Command{
@@ -51,7 +50,7 @@ func findImage(ctx *cli.Context) (string, string, error) {
5150
}
5251
ocidir := r[0]
5352
tag := r[1]
54-
if !atomfs.PathExists(ocidir) {
53+
if !common.PathExists(ocidir) {
5554
return "", "", fmt.Errorf("oci directory %s does not exist: %w", ocidir, mountUsage(ctx.App.Name))
5655
}
5756
return ocidir, tag, nil
@@ -94,7 +93,7 @@ func doMount(ctx *cli.Context) error {
9493
return fmt.Errorf("--persist requires an argument")
9594
}
9695
}
97-
opts := atomfs.MountOCIOpts{
96+
opts := molecule.MountOCIOpts{
9897
OCIDir: absOCIDir,
9998
Tag: tag,
10099
Target: absTarget,
@@ -104,7 +103,7 @@ func doMount(ctx *cli.Context) error {
104103
MetadataDir: ctx.String("metadir"), // nil here means /run/atomfs
105104
}
106105

107-
mol, err := atomfs.BuildMoleculeFromOCI(opts)
106+
mol, err := molecule.BuildMoleculeFromOCI(opts)
108107
if err != nil {
109108
return errors.Wrapf(err, "couldn't build molecule with opts %+v", opts)
110109
}
@@ -132,7 +131,7 @@ func amPrivileged() bool {
132131

133132
func squashUmount(p string) error {
134133
if amPrivileged() {
135-
return squashfs.Umount(p)
134+
return common.Umount(p)
136135
}
137136
return RunCommand("fusermount", "-u", p)
138137
}

cmd/atomfs/umount.go

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@ import (
55
"path/filepath"
66

77
"github.com/urfave/cli"
8-
"machinerun.io/atomfs"
9-
"machinerun.io/atomfs/mount"
8+
"machinerun.io/atomfs/pkg/molecule"
109
)
1110

1211
var umountCmd = cli.Command{
@@ -26,11 +25,6 @@ func umountUsage(me string) error {
2625
return fmt.Errorf("Usage: %s umount mountpoint", me)
2726
}
2827

29-
func isMountpoint(p string) bool {
30-
mounted, err := mount.IsMountpoint(p)
31-
return err == nil && mounted
32-
}
33-
3428
func doUmount(ctx *cli.Context) error {
3529
if ctx.NArg() < 1 {
3630
return umountUsage(ctx.App.Name)
@@ -46,5 +40,5 @@ func doUmount(ctx *cli.Context) error {
4640
}
4741
}
4842

49-
return atomfs.UmountWithMetadir(mountpoint, ctx.String("metadir"))
43+
return molecule.UmountWithMetadir(mountpoint, ctx.String("metadir"))
5044
}

cmd/atomfs/verify.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@ import (
66
"strings"
77

88
"github.com/urfave/cli"
9-
"machinerun.io/atomfs"
10-
"machinerun.io/atomfs/log"
11-
"machinerun.io/atomfs/mount"
12-
"machinerun.io/atomfs/squashfs"
9+
"machinerun.io/atomfs/pkg/common"
10+
"machinerun.io/atomfs/pkg/log"
11+
"machinerun.io/atomfs/pkg/mount"
12+
"machinerun.io/atomfs/pkg/verity"
1313
)
1414

1515
var verifyCmd = cli.Command{
@@ -45,16 +45,16 @@ func doVerify(ctx *cli.Context) error {
4545
}
4646
}
4747

48-
if !isMountpoint(mountpoint) {
48+
if !common.IsMountpoint(mountpoint) {
4949
return fmt.Errorf("%s is not a mountpoint", mountpoint)
5050
}
5151

52-
mountNSName, err := atomfs.GetMountNSName()
52+
mountNSName, err := common.GetMountNSName()
5353
if err != nil {
5454
return err
5555
}
5656

57-
metadir := filepath.Join(atomfs.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, atomfs.ReplacePathSeparators(mountpoint))
57+
metadir := filepath.Join(common.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, common.ReplacePathSeparators(mountpoint))
5858
mountsdir := filepath.Join(metadir, "mounts")
5959

6060
mounts, err := mount.ParseMounts("/proc/self/mountinfo")
@@ -83,7 +83,7 @@ func doVerify(ctx *cli.Context) error {
8383
continue
8484
}
8585
checkedCount = checkedCount + 1
86-
err = squashfs.ConfirmExistingVerityDeviceCurrentValidity(m.Source)
86+
err = verity.ConfirmExistingVerityDeviceCurrentValidity(m.Source)
8787
if err != nil {
8888
fmt.Printf("%s: CORRUPTION FOUND\n", m.Source)
8989
allOK = false

go.mod

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db
77
github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6
88
github.com/opencontainers/go-digest v1.0.0
9-
github.com/opencontainers/image-spec v1.1.0-rc2
9+
github.com/opencontainers/image-spec v1.1.0
1010
github.com/opencontainers/runc v1.2.3 // indirect
1111
github.com/opencontainers/umoci v0.4.8-0.20220412065115-12453f247749
1212
github.com/pkg/errors v0.9.1
@@ -22,7 +22,6 @@ require (
2222
github.com/cyphar/filepath-securejoin v0.3.5 // indirect
2323
github.com/davecgh/go-spew v1.1.1 // indirect
2424
github.com/docker/go-units v0.5.0 // indirect
25-
github.com/google/go-cmp v0.5.6 // indirect
2625
github.com/klauspost/compress v1.15.15 // indirect
2726
github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
2827
github.com/moby/sys/user v0.3.0 // indirect

go.sum

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,8 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
4343
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
4444
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
4545
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
46+
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
4647
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
47-
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
48-
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
4948
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
5049
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
5150
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -88,8 +87,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
8887
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
8988
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
9089
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
91-
github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
92-
github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
90+
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
91+
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
9392
github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
9493
github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80=
9594
github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM=

pkg/common/common_test.go

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package common
2+
3+
import (
4+
"os"
5+
"testing"
6+
7+
"github.com/stretchr/testify/assert"
8+
)
9+
10+
type uidmapTestcase struct {
11+
uidmap string
12+
expected bool
13+
}
14+
15+
var uidmapTests = []uidmapTestcase{
16+
{
17+
uidmap: ` 0 0 4294967295`,
18+
expected: true,
19+
},
20+
{
21+
uidmap: ` 0 0 1000
22+
2000 2000 1`,
23+
expected: false,
24+
},
25+
{
26+
uidmap: ` 0 0 1000`,
27+
expected: false,
28+
},
29+
{
30+
uidmap: ` 10 0 4294967295`,
31+
expected: false,
32+
},
33+
{
34+
uidmap: ` 0 10 4294967295`,
35+
expected: false,
36+
},
37+
{
38+
uidmap: ` 0 0 1`,
39+
expected: false,
40+
},
41+
}
42+
43+
func TestAmHostRoot(t *testing.T) {
44+
t.Parallel()
45+
assert := assert.New(t)
46+
for _, testcase := range uidmapTests {
47+
v := uidmapIsHost(testcase.uidmap)
48+
assert.Equal(v, testcase.expected)
49+
}
50+
}
51+
52+
func TestIsEmpytDir(t *testing.T) {
53+
t.Parallel()
54+
assert := assert.New(t)
55+
v, e := IsEmptyDir("/")
56+
assert.NoError(e)
57+
assert.False(v)
58+
59+
v, e = IsEmptyDir("/root")
60+
assert.Error(e)
61+
62+
dname, err := os.MkdirTemp("", "squashfs_empty_test_dir")
63+
assert.NoError(err)
64+
defer os.RemoveAll(dname)
65+
66+
v, e = IsEmptyDir(dname)
67+
assert.NoError(e)
68+
assert.True(v)
69+
}

pkg/common/exclude.go

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package common
2+
3+
import (
4+
"bytes"
5+
"path"
6+
"path/filepath"
7+
"strings"
8+
)
9+
10+
// ExcludePaths represents a list of paths to exclude in a filesystem listing.
11+
// Users should do something like filepath.Walk() over the whole filesystem,
12+
// calling AddExclude() or AddInclude() based on whether they want to include
13+
// or exclude a particular file. Note that if e.g. /usr is excluded, then
14+
// everyting underneath is also implicitly excluded. The
15+
// AddExclude()/AddInclude() methods do the math to figure out what is the
16+
// correct set of things to exclude or include based on what paths have been
17+
// previously included or excluded.
18+
type ExcludePaths struct {
19+
exclude map[string]bool
20+
include []string
21+
}
22+
23+
func NewExcludePaths() *ExcludePaths {
24+
return &ExcludePaths{
25+
exclude: map[string]bool{},
26+
include: []string{},
27+
}
28+
}
29+
30+
func (eps *ExcludePaths) AddExclude(p string) {
31+
for _, inc := range eps.include {
32+
// If /usr/bin/ls has changed but /usr hasn't, we don't want to list
33+
// /usr in the include paths any more, so let's be sure to only
34+
// add things which aren't prefixes.
35+
if strings.HasPrefix(inc, p) {
36+
return
37+
}
38+
}
39+
eps.exclude[p] = true
40+
}
41+
42+
func (eps *ExcludePaths) AddInclude(orig string, isDir bool) {
43+
// First, remove this thing and all its parents from exclude.
44+
p := orig
45+
46+
// normalize to the first dir
47+
if !isDir {
48+
p = path.Dir(p)
49+
}
50+
for {
51+
// our paths are all absolute, so this is a base case
52+
if p == "/" {
53+
break
54+
}
55+
56+
delete(eps.exclude, p)
57+
p = filepath.Dir(p)
58+
}
59+
60+
// now add it to the list of includes, so we don't accidentally re-add
61+
// anything above.
62+
eps.include = append(eps.include, orig)
63+
}
64+
65+
func (eps *ExcludePaths) String() (string, error) {
66+
var buf bytes.Buffer
67+
for p := range eps.exclude {
68+
_, err := buf.WriteString(p)
69+
if err != nil {
70+
return "", err
71+
}
72+
_, err = buf.WriteString("\n")
73+
if err != nil {
74+
return "", err
75+
}
76+
}
77+
78+
_, err := buf.WriteString("\n")
79+
if err != nil {
80+
return "", err
81+
}
82+
83+
return buf.String(), nil
84+
}

0 commit comments

Comments
 (0)