Skip to content

Commit 64e4ef2

Browse files
cfergeaupraveenkumar
authored andcommitted
Issue #509: build: Add build-time tool to embed misc binaries in crc
If we want crc to work in disconnected environment, we cannot assume that we can download the misc binaries that we need at setup time. This commit adds a "crc-embedder" tool which knows how to download and embed the various binaries that are needed for each of our supported platforms. The subsequent commits will then change the preflight checks to make use of these embedded binaries if they are available, but we'll still download them if they are not embedded #509
1 parent 6f57e39 commit 64e4ef2

10 files changed

Lines changed: 321 additions & 17 deletions

File tree

Makefile

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,12 @@ DOCS_BUILD_TARGET ?= /docs/source/getting-started/master.adoc
3131

3232
GOOS ?= $(shell go env GOOS)
3333
GOARCH ?= $(shell go env GOARCH)
34+
HOST_BUILD_DIR=$(BUILD_DIR)/$(GOOS)-$(GOARCH)
3435
GOPATH ?= $(shell go env GOPATH)
3536
ORG := github.com/code-ready
3637
REPOPATH ?= $(ORG)/crc
3738

38-
PACKAGES := go list ./...
39+
PACKAGES := go list --tags build ./...
3940
SOURCES := $(shell git ls-files *.go ":^vendor")
4041

4142
RELEASE_INFO := release-info.json
@@ -72,10 +73,6 @@ default: install
7273
vendor:
7374
GO111MODULE=on go mod vendor
7475

75-
# Get binappend
76-
$(GOPATH)/bin/binappend-cli:
77-
GO111MODULE=off go get -u github.com/yourfin/binappend-cli
78-
7976
# Start of the actual build targets
8077

8178
.PHONY: install
@@ -91,12 +88,15 @@ $(BUILD_DIR)/linux-amd64/crc: $(SOURCES)
9188
$(BUILD_DIR)/windows-amd64/crc.exe: $(SOURCES)
9289
GOARCH=amd64 GOOS=windows go build -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/windows-amd64/crc.exe ./cmd/crc
9390

91+
$(HOST_BUILD_DIR)/crc-embedder: $(SOURCES)
92+
go build --tags="build" -ldflags="$(LDFLAGS)" -o $(HOST_BUILD_DIR)/crc-embedder ./cmd/crc-embedder
93+
9494
.PHONY: cross ## Cross compiles all binaries
9595
cross: $(BUILD_DIR)/macos-amd64/crc $(BUILD_DIR)/linux-amd64/crc $(BUILD_DIR)/windows-amd64/crc.exe
9696

9797
.PHONY: test
9898
test:
99-
go test -v -ldflags="$(VERSION_VARIABLES)" $(shell $(PACKAGES))
99+
go test --tags build -v -ldflags="$(VERSION_VARIABLES)" $(shell $(PACKAGES))
100100

101101
.PHONY: build_docs
102102
build_docs:
@@ -148,21 +148,22 @@ $(GOPATH)/bin/golangci-lint:
148148
GO111MODULE=on go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.17.1
149149

150150
# Run golangci-lint against code
151+
GOLANGCI_FLAGS=--build-tags build
151152
.PHONY: lint cross-lint
152153
lint: $(GOPATH)/bin/golangci-lint
153-
$(GOPATH)/bin/golangci-lint run
154+
$(GOPATH)/bin/golangci-lint $(GOLANGCI_FLAGS) run
154155

155156
cross-lint: $(GOPATH)/bin/golangci-lint
156-
GOOS=darwin $(GOPATH)/bin/golangci-lint run
157-
GOOS=linux $(GOPATH)/bin/golangci-lint run
158-
GOOS=windows $(GOPATH)/bin/golangci-lint run
157+
GOOS=darwin $(GOPATH)/bin/golangci-lint $(GOLANGCI_FLAGS) run
158+
GOOS=linux $(GOPATH)/bin/golangci-lint $(GOLANGCI_FLAGS) run
159+
GOOS=windows $(GOPATH)/bin/golangci-lint $(GOLANGCI_FLAGS) run
159160

160161
$(GOPATH)/bin/gosec:
161162
GO111MODULE=on go get github.com/securego/gosec/cmd/gosec@2.0.0
162163

163164
# Run gosec against code
164165
gosec: $(GOPATH)/bin/gosec
165-
$(GOPATH)/bin/gosec -tests -severity medium ./...
166+
$(GOPATH)/bin/gosec -tags build -tests -severity medium ./...
166167

167168
.PHONY: gen_release_info
168169
gen_release_info:
@@ -199,7 +200,7 @@ check_bundledir:
199200
@$(call check_defined, BUNDLE_DIR, "Embedding bundle requires BUNDLE_DIR set to a directory containing CRC bundles for all hypervisors")
200201

201202
embed_bundle: LDFLAGS += $(BUNDLE_EMBEDDED)
202-
embed_bundle: clean cross $(GOPATH)/bin/binappend-cli check_bundledir $(HYPERKIT_BUNDLENAME) $(HYPERV_BUNDLENAME) $(LIBVIRT_BUNDLENAME)
203-
$(GOPATH)/bin/binappend-cli write $(BUILD_DIR)/linux-amd64/crc $(LIBVIRT_BUNDLENAME)
204-
$(GOPATH)/bin/binappend-cli write $(BUILD_DIR)/macos-amd64/crc $(HYPERKIT_BUNDLENAME)
205-
$(GOPATH)/bin/binappend-cli write $(BUILD_DIR)/windows-amd64/crc.exe $(HYPERV_BUNDLENAME)
203+
embed_bundle: clean cross $(HOST_BUILD_DIR)/crc-embedder check_bundledir $(HYPERKIT_BUNDLENAME) $(HYPERV_BUNDLENAME) $(LIBVIRT_BUNDLENAME)
204+
$(HOST_BUILD_DIR)/crc-embedder embed --log-level debug --goos=darwin --bundle-dir=$(BUNDLE_DIR) $(BUILD_DIR)/macos-amd64/crc
205+
$(HOST_BUILD_DIR)/crc-embedder embed --log-level debug --goos=linux --bundle-dir=$(BUNDLE_DIR) $(BUILD_DIR)/linux-amd64/crc
206+
$(HOST_BUILD_DIR)/crc-embedder embed --log-level debug --goos=windows --bundle-dir=$(BUNDLE_DIR) $(BUILD_DIR)/windows-amd64/crc.exe

cmd/crc-embedder/cmd/embed.go

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"io/ioutil"
6+
"os"
7+
"path"
8+
"runtime"
9+
10+
"github.com/code-ready/crc/pkg/crc/constants"
11+
"github.com/code-ready/crc/pkg/crc/logging"
12+
"github.com/code-ready/crc/pkg/download"
13+
14+
"github.com/code-ready/crc/pkg/crc/machine/hyperkit"
15+
"github.com/code-ready/crc/pkg/crc/machine/libvirt"
16+
17+
"github.com/YourFin/binappend"
18+
"github.com/spf13/cobra"
19+
)
20+
21+
var (
22+
bundleDir string
23+
goos string
24+
)
25+
26+
func init() {
27+
embedCmd.Flags().StringVar(&bundleDir, "bundle-dir", constants.MachineCacheDir, "Directory where the OpenShift bundle can be found")
28+
embedCmd.Flags().StringVar(&goos, "goos", runtime.GOOS, "Target platform (darwin, linux or windows)")
29+
rootCmd.AddCommand(embedCmd)
30+
}
31+
32+
var embedCmd = &cobra.Command{
33+
Use: "embed",
34+
Short: "Embed data files in crc binary",
35+
Long: `Embed the OpenShift bundle and the binaries needed at runtime in the crc binary`,
36+
Run: func(cmd *cobra.Command, args []string) {
37+
runEmbed(args)
38+
},
39+
}
40+
41+
func runEmbed(args []string) {
42+
if len(args) != 1 {
43+
logging.Fatalf("embed takes exactly one argument")
44+
}
45+
binaryPath := args[0]
46+
destDir, err := ioutil.TempDir("", "crc-embedder")
47+
if err != nil {
48+
logging.Errorf(fmt.Sprintf("Failed to create temporary directory: %v", err))
49+
}
50+
defer os.RemoveAll(destDir)
51+
downloadedFiles, err := downloadDataFiles(goos, destDir)
52+
if err != nil {
53+
logging.Errorf(fmt.Sprintf("Failed to download data files: %v", err))
54+
}
55+
56+
bundlePath := path.Join(bundleDir, constants.GetDefaultBundleForOs(goos))
57+
downloadedFiles = append(downloadedFiles, bundlePath)
58+
err = embedFiles(binaryPath, downloadedFiles)
59+
if err != nil {
60+
logging.Errorf(fmt.Sprintf("Failed to embed data files: %v", err))
61+
}
62+
}
63+
64+
func embedFiles(binary string, filenames []string) error {
65+
appender, err := binappend.MakeAppender(binary)
66+
if err != nil {
67+
return err
68+
}
69+
defer appender.Close()
70+
for _, filename := range filenames {
71+
logging.Debugf("Embedding %s in %s", filename, binary)
72+
f, err := os.Open(filename) // #nosec G304
73+
if err != nil {
74+
return fmt.Errorf("Failed to open %s: %v", filename, err)
75+
}
76+
defer f.Close()
77+
78+
err = appender.AppendStreamReader(path.Base(filename), f, false)
79+
if err != nil {
80+
return fmt.Errorf("Failed to append %s to %s: %v", filename, binary, err)
81+
}
82+
}
83+
84+
return nil
85+
}
86+
87+
var (
88+
dataFileUrls = map[string][]string{
89+
"darwin": []string{
90+
hyperkit.MachineDriverDownloadUrl,
91+
hyperkit.HyperkitDownloadUrl,
92+
constants.GetOcUrlForOs("darwin"),
93+
},
94+
"linux": []string{
95+
libvirt.MachineDriverDownloadUrl,
96+
constants.GetOcUrlForOs("linux"),
97+
},
98+
"windows": []string{
99+
constants.GetOcUrlForOs("windows"),
100+
},
101+
}
102+
)
103+
104+
func downloadDataFiles(goos string, destDir string) ([]string, error) {
105+
downloadedFiles := []string{}
106+
downloads := dataFileUrls[goos]
107+
for _, url := range downloads {
108+
filename, err := download.Download(url, destDir, 0644)
109+
if err != nil {
110+
return nil, err
111+
}
112+
downloadedFiles = append(downloadedFiles, filename)
113+
}
114+
115+
return downloadedFiles, nil
116+
}

cmd/crc-embedder/cmd/extract.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package cmd
2+
3+
import (
4+
"github.com/code-ready/crc/pkg/crc/logging"
5+
"github.com/code-ready/crc/pkg/crc/output"
6+
"github.com/code-ready/crc/pkg/embed"
7+
8+
"github.com/spf13/cobra"
9+
)
10+
11+
func init() {
12+
rootCmd.AddCommand(extractCmd)
13+
}
14+
15+
var extractCmd = &cobra.Command{
16+
Use: "extract",
17+
Short: "Extract data file embedded in the crc binary",
18+
Long: `Extract a data file wich is embedded in the crc binary`,
19+
Run: func(cmd *cobra.Command, args []string) {
20+
runExtract(args)
21+
},
22+
}
23+
24+
func runExtract(args []string) {
25+
if len(args) != 3 {
26+
logging.Fatalf("extract takes exactly three arguments")
27+
}
28+
binaryPath := args[0]
29+
embedName := args[1]
30+
destFile := args[2]
31+
err := embed.ExtractFromBinary(binaryPath, embedName, destFile)
32+
if err != nil {
33+
logging.Fatalf("Could not extract data embedded in %s: %v", binaryPath, err)
34+
}
35+
output.Outf("Successfully copied embedded '%s' from %s to %s: %v", embedName, binaryPath, destFile, err)
36+
}

cmd/crc-embedder/cmd/list.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package cmd
2+
3+
import (
4+
"github.com/code-ready/crc/pkg/crc/logging"
5+
"github.com/code-ready/crc/pkg/crc/output"
6+
7+
"github.com/YourFin/binappend"
8+
"github.com/spf13/cobra"
9+
)
10+
11+
func init() {
12+
rootCmd.AddCommand(listCmd)
13+
}
14+
15+
var listCmd = &cobra.Command{
16+
Use: "list",
17+
Short: "List data files embedded in the crc binary",
18+
Long: `List all the data files wich were embedded in the crc binary`,
19+
Run: func(cmd *cobra.Command, args []string) {
20+
runList(args)
21+
},
22+
}
23+
24+
func runList(args []string) {
25+
if len(args) != 1 {
26+
logging.Fatalf("list takes exactly one argument")
27+
}
28+
binaryPath := args[0]
29+
extractor, err := binappend.MakeExtractor(binaryPath)
30+
if err != nil {
31+
logging.Fatalf("Could not access data embedded in %s: %v", binaryPath, err)
32+
}
33+
output.Outf("Data files embedded in %s:\n", binaryPath)
34+
for _, name := range extractor.AvalibleData() {
35+
output.Outln("\t", name)
36+
}
37+
}

cmd/crc-embedder/cmd/root.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package cmd
2+
3+
import (
4+
"github.com/code-ready/crc/pkg/crc/constants"
5+
"github.com/code-ready/crc/pkg/crc/logging"
6+
"github.com/code-ready/crc/pkg/crc/output"
7+
8+
"github.com/spf13/cobra"
9+
)
10+
11+
var rootCmd = &cobra.Command{
12+
Use: "crc-embedder [list|embed|extract]",
13+
Short: "Build helper for crc for binary embedding",
14+
Long: `crc-embedder is a command line utility for listing or appending binary data
15+
when building the crc binary for release`,
16+
PersistentPreRun: func(cmd *cobra.Command, args []string) {
17+
runPrerun()
18+
},
19+
Run: func(cmd *cobra.Command, args []string) {
20+
runRoot()
21+
_ = cmd.Help()
22+
},
23+
PersistentPostRun: func(cmd *cobra.Command, args []string) {
24+
runPostrun()
25+
},
26+
}
27+
28+
func init() {
29+
rootCmd.PersistentFlags().StringVar(&logging.LogLevel, "log-level", constants.DefaultLogLevel, "log level (e.g. \"debug | info | warn | error\")")
30+
}
31+
32+
func Execute() {
33+
if err := rootCmd.Execute(); err != nil {
34+
logging.Fatal(err)
35+
}
36+
}
37+
38+
func runPrerun() {
39+
// Setting up logrus
40+
logging.InitLogrus(logging.LogLevel)
41+
logging.SetupFileHook()
42+
}
43+
44+
func runRoot() {
45+
output.Outln("No command given")
46+
output.Outln("")
47+
}
48+
49+
func runPostrun() {
50+
logging.CloseLogging()
51+
}

cmd/crc-embedder/crc-embedder.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package main
2+
3+
import (
4+
"github.com/code-ready/crc/cmd/crc-embedder/cmd"
5+
)
6+
7+
func main() {
8+
cmd.Execute()
9+
}

go.sum

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@ github.com/code-ready/clicumber v0.0.0-20190801094041-02a92c8f7ece h1:ZuvFtsdv+H
1919
github.com/code-ready/clicumber v0.0.0-20190801094041-02a92c8f7ece/go.mod h1:bjcKpFX8OebrlwSDgDAf0KAqpdVPGQssQk+IcVwlcog=
2020
github.com/code-ready/goodhosts v0.0.0-20190712111040-f090f3f77c26 h1:rNdqNBHRAAuGv68zZCX0rQAekl0g4qwnseA8OFM2j54=
2121
github.com/code-ready/goodhosts v0.0.0-20190712111040-f090f3f77c26/go.mod h1:kc1Z2UDuIlxPOo7VWCIkYPOaD55KPX0t5qS5cmLBe8Y=
22-
github.com/code-ready/machine v0.0.0-20190904103148-0fde37b5d95b h1:uDsea77LGyGm5kDFi4N8xyi2k06dlCKvK09or3BGsUA=
23-
github.com/code-ready/machine v0.0.0-20190904103148-0fde37b5d95b/go.mod h1:x/y5No186t4X+Z9fQafLByNoo+K4MyNzxgcKKq76N+U=
2422
github.com/code-ready/machine v0.0.0-20191115055627-b284f794e910 h1:roFnY4xZrCGYgzxymOxCONENB6VOZICijEPsN+2x5zw=
2523
github.com/code-ready/machine v0.0.0-20191115055627-b284f794e910/go.mod h1:x/y5No186t4X+Z9fQafLByNoo+K4MyNzxgcKKq76N+U=
2624
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=

pkg/crc/machine/hyperkit/constants_darwin.go renamed to pkg/crc/machine/hyperkit/constants.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
//+build darwin build
2+
13
package hyperkit
24

35
import "fmt"

pkg/crc/machine/libvirt/constants_linux.go renamed to pkg/crc/machine/libvirt/constants.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
//+build linux build
2+
13
package libvirt
24

35
import "fmt"

0 commit comments

Comments
 (0)