Skip to content
This repository was archived by the owner on Jul 14, 2021. It is now read-only.

Commit 78fc802

Browse files
klingtnetklingtnet
committed
Implement embed
embed is yet another tool for embedding static files in a Go binary. I was not satisfied with any of the existing tools for embedding static content, either they lacked functionality like including more than a single file or folder or their APIs were cumbersome to use. Thus I decided to implement a file embedding tool myself. The original implementation took about an hour and consistent of two files, one for the binary and another one to support SQL schema migrations from embedded content. You can find the original implementation here [1]. Like often the 80:20 rule fit here as well and setting up tests, CI, making the library importable and writing some sentences of documentation took five times as long as writing the initial implementation. However, I still like how it turned out in the end and think that it is pretty usable. I know that this will be redundant when the file embedding proposal lands in Go 1.17 but there still a couple of months left until this happens. [1]: https://gist.github.com/klingtnet/b66ecace3e87b10972245fec7e4c3fc5
0 parents  commit 78fc802

30 files changed

+1895
-0
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: CI
2+
3+
on: [push, pull_request]
4+
5+
jobs:
6+
build:
7+
strategy:
8+
matrix:
9+
go-version: [1.14.x, 1.15.x]
10+
platform: [ubuntu-latest, macos-latest, windows-latest]
11+
runs-on: ${{ matrix.platform }}
12+
steps:
13+
- name: Install Go
14+
uses: actions/setup-go@v2
15+
with:
16+
go-version: ${{ matrix.go-version }}
17+
- name: Checkout code
18+
uses: actions/checkout@v2
19+
- name: Test
20+
run: |
21+
go test -race ./...
22+
go build .
23+

‎.github/workflows/release.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: Release
2+
on:
3+
push:
4+
tags: "v*"
5+
jobs:
6+
build:
7+
name: Create Release
8+
runs-on: ubuntu-latest
9+
steps:
10+
- name: Checkout code
11+
uses: actions/checkout@v2
12+
- name: Install Go
13+
uses: actions/setup-go@v2
14+
with:
15+
go-version: 1.15.x
16+
- name: Build
17+
run: |
18+
make cross
19+
- name: Create Release
20+
id: create_release
21+
uses: actions/create-release@v1
22+
env:
23+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
24+
with:
25+
tag_name: ${{ github.ref }}
26+
release_name: Release ${{ github.ref }}
27+
draft: false
28+
prerelease: false
29+
- name: Upload Linux Build
30+
id: upload-linux-build
31+
uses: actions/upload-release-asset@v1
32+
env:
33+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
34+
with:
35+
upload_url: ${{ steps.create_release.outputs.upload_url }}
36+
asset_path: ./embed
37+
asset_name: embed
38+
asset_content_type: application/octet-stream
39+
- name: Upload Windows Build
40+
id: upload-windows-build
41+
uses: actions/upload-release-asset@v1
42+
env:
43+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
44+
with:
45+
upload_url: ${{ steps.create_release.outputs.upload_url }}
46+
asset_path: ./embed.exe
47+
asset_name: embed.exe
48+
asset_content_type: application/octet-stream
49+
- name: Upload Mac Build
50+
id: upload-mac-build
51+
uses: actions/upload-release-asset@v1
52+
env:
53+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
54+
with:
55+
upload_url: ${{ steps.create_release.outputs.upload_url }}
56+
asset_path: ./embed.mac
57+
asset_name: embed.mac
58+
asset_content_type: application/octet-stream
59+
- name: Upload Raspberry Pi Build
60+
id: upload-raspberry-pi-build
61+
uses: actions/upload-release-asset@v1
62+
env:
63+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
64+
with:
65+
upload_url: ${{ steps.create_release.outputs.upload_url }}
66+
asset_path: ./embed.pi
67+
asset_name: embed.pi
68+
asset_content_type: application/octet-stream

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/embed
2+
/embed.pi
3+
/embed.mac
4+
/embed.exe
5+
/.vscode/
6+

‎LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2020 Andreas Linz
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎Makefile‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
.PHONY: clean test
2+
3+
VERSION:=$(shell git describe --always --tags)
4+
GO_FILES:=$(wildcard *.go)
5+
6+
cross: $(GO_FILES) embed
7+
GOOS=windows go build -ldflags "-X main.Version=$(VERSION)" ./cmd/embed
8+
GOOS=darwin go build -o embed.mac -ldflags "-X main.Version=$(VERSION)" ./cmd/embed
9+
GOOS=linux GOARCH=arm go build -o embed.pi -ldflags "-X main.Version=$(VERSION)" ./cmd/embed
10+
11+
embed: test $(GO_FILES)
12+
go build -o $@ -ldflags "-X main.Version=$(VERSION)" ./cmd/embed
13+
14+
install: embed
15+
install -Dm 0755 embed ~/.local/bin/embed
16+
17+
test:
18+
go run ./cmd/embed --package internal --destination internal/embeds.go --include internal/testdata
19+
go test ./...
20+
21+
clean:
22+
git clean -fd

‎README.md‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# embed
2+
3+
![CI](https://github.com/klingtnet/embed/workflows/CI/badge.svg)
4+
5+
- [Documentation](https://pkg.go.dev/github.com/klingtnet/embed)
6+
- [Releases](https://github.com/klingtnet/embed/releases)
7+
8+
embed is a tool for embedding static content in your Go application.
9+
10+
It provides three methods, listing embedded files and getting their content as `[]byte` or `string`. If you need a `io.Writer` just wrap the `[]byte` content in a `bytes.NewBuffer`.
11+
12+
The motivation for building yet another static file embedding tool for Go was that I was not satisified with any of the existing tools, they either had inconvenient APIs or did not support to include more than a single folder or file.
13+
14+
Please note that this tool, as well as most other static file embedding tools, will be redundant as soon as the proposal to [add support for embedded files](https://github.com/golang/go/issues/41191) lands in `go/cmd`.
15+
16+
## Usage
17+
18+
You can run the tool with `go run github.com/klingtnet/embed/cmd/embed` or by downloading a precompiled binary from the [releases page](https://github.com/klingtnet/embed/releases).
19+
20+
```sh
21+
$ ./embed
22+
NAME:
23+
embed - A new cli application
24+
25+
USAGE:
26+
embed [global options] command [command options] [arguments...]
27+
28+
COMMANDS:
29+
help, h Shows a list of commands or help for one command
30+
31+
GLOBAL OPTIONS:
32+
--package value, -p value name of the package the generated Go file is associated to (default: "main")
33+
--destination value, --dest value, -d value where to store the generated Go file (default: "embeds.go")
34+
--include value, -i value paths to embed, directories are stored recursively (can be used multiple times)
35+
--help, -h show help (default: false)
36+
```
37+
38+
Running `embed --include assets --include views` will create a file `embeds.go` (you can change the destination) that bundles all files from the assets and views directory. In your application you can then use `embeds.File("assets/my-asset.png")` to get the contents of an embedded file. For an example of such a generated file see [`internal/embeds.go`](https://github.com/klingtnet/embed/blob/master/internal/embeds.go).
39+
40+
## golang-migrate driver
41+
42+
The package also provides a migration source driver for [golang-migrate](https://github.com/golang-migrate/migrate).
43+
For a usage example refer to [`examples/migrate/migrate.go`](https://github.com/klingtnet/embed/blob/master/examples/migrate/migrate.go).

‎cmd/embed/embed.go‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"encoding/base64"
7+
"fmt"
8+
"go/format"
9+
"io/ioutil"
10+
"log"
11+
"os"
12+
"path/filepath"
13+
"text/template"
14+
15+
"github.com/urfave/cli/v2"
16+
)
17+
18+
func pathToVar(path string) string {
19+
return fmt.Sprintf("file%x", []byte(path))
20+
}
21+
22+
func encodeFile(data []byte) string {
23+
return base64.RawStdEncoding.EncodeToString(data)
24+
}
25+
26+
var (
27+
fileTemplate = template.Must(template.New("").Funcs(template.FuncMap{"pathToVar": pathToVar, "encode": encodeFile}).Parse(`package {{ .Package }}
28+
29+
import (
30+
"encoding/base64"
31+
"sort"
32+
)
33+
34+
const (
35+
{{- range $path, $data := .Files }}
36+
{{ pathToVar $path }} = "{{ encode $data }}"
37+
{{- end }}
38+
)
39+
40+
// Embedded implements github.com/klingtnet/embed/Embed .
41+
type Embedded struct {
42+
embedMap map[string]string
43+
}
44+
45+
// Embeds stores the embedded data.
46+
var Embeds = Embedded {
47+
embedMap: map[string]string{
48+
{{- range $path, $_ := .Files }}
49+
"{{ $path }}": {{ pathToVar $path }},
50+
{{- end }}
51+
},
52+
}
53+
54+
// Files implements github.com/klingtnet/embed/Embed .
55+
func (e Embedded) Files() []string {
56+
var fs []string
57+
for f := range e.embedMap {
58+
fs = append(fs,f)
59+
}
60+
sort.Strings(fs)
61+
return fs
62+
}
63+
64+
// File implements github.com/klingtnet/embed/Embed .
65+
func (e Embedded) File(path string) []byte {
66+
file, ok := e.embedMap[path]
67+
if !ok {
68+
return nil
69+
}
70+
d, err := base64.RawStdEncoding.DecodeString(file)
71+
if err != nil {
72+
panic(err)
73+
}
74+
return d
75+
}
76+
77+
// FileString implements github.com/klingtnet/embed/Embed .
78+
func (e Embedded) FileString(path string) string {
79+
return string(e.File(path))
80+
}
81+
`))
82+
)
83+
84+
func readFile(path string) (data []byte, err error) {
85+
f, err := os.Open(path)
86+
if err != nil {
87+
return
88+
}
89+
defer f.Close()
90+
data, err = ioutil.ReadAll(f)
91+
return
92+
}
93+
94+
func embedAction(c *cli.Context) error {
95+
return embed(c.Context, c.StringSlice("include"), c.String("package"), c.String("destination"))
96+
}
97+
98+
func embed(ctx context.Context, includes []string, packageName, destinationPath string) error {
99+
files := make(map[string][]byte)
100+
101+
for _, includePath := range includes {
102+
info, err := os.Stat(includePath)
103+
if err != nil {
104+
return fmt.Errorf("stat: %w", err)
105+
}
106+
if info.IsDir() {
107+
walkFn := func(path string, info os.FileInfo, err error) error {
108+
if err != nil {
109+
return err
110+
}
111+
if info.IsDir() {
112+
return nil
113+
}
114+
data, err := readFile(path)
115+
if err != nil {
116+
return fmt.Errorf("readFile: %w", err)
117+
}
118+
files[path] = data
119+
120+
return nil
121+
}
122+
err = filepath.Walk(includePath, walkFn)
123+
if err != nil {
124+
return fmt.Errorf("filepath.Walk: %w", err)
125+
}
126+
} else {
127+
data, err := readFile(includePath)
128+
if err != nil {
129+
return fmt.Errorf("readFile: %w", err)
130+
}
131+
files[includePath] = data
132+
}
133+
}
134+
135+
templateData := struct {
136+
Package string
137+
Files map[string][]byte
138+
}{
139+
Package: packageName,
140+
Files: files,
141+
}
142+
143+
buf := bytes.NewBuffer(nil)
144+
err := fileTemplate.Execute(buf, templateData)
145+
if err != nil {
146+
return fmt.Errorf("fileTemplate.Execute: %w", err)
147+
}
148+
source, err := format.Source(buf.Bytes())
149+
if err != nil {
150+
return fmt.Errorf("format.Source: %w", err)
151+
}
152+
dest, err := os.OpenFile(destinationPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
153+
if err != nil {
154+
return fmt.Errorf("os.OpenFile %q: %w", destinationPath, err)
155+
}
156+
defer dest.Close()
157+
_, err = dest.Write(source)
158+
if err != nil {
159+
return fmt.Errorf("dest.Write: %w", err)
160+
}
161+
return nil
162+
}
163+
164+
// Version is the build version.
165+
// The actual version is set on build time.
166+
var Version = "unset"
167+
168+
func main() {
169+
app := cli.App{
170+
Name: "embed",
171+
Version: Version,
172+
Flags: []cli.Flag{
173+
&cli.StringFlag{
174+
Name: "package",
175+
Aliases: []string{"p"},
176+
Usage: "name of the package the generated Go file is associated to",
177+
Value: "main",
178+
},
179+
&cli.StringFlag{
180+
Name: "destination",
181+
Aliases: []string{"dest", "d"},
182+
Usage: "where to store the generated Go file",
183+
Value: "embeds.go",
184+
},
185+
&cli.StringSliceFlag{
186+
Name: "include",
187+
Aliases: []string{"i"},
188+
Usage: "paths to embed, directories are stored recursively (can be used multiple times)",
189+
Required: true,
190+
},
191+
},
192+
Action: embedAction,
193+
}
194+
err := app.Run(os.Args)
195+
if err != nil {
196+
log.Fatal(err)
197+
}
198+
}

0 commit comments

Comments
 (0)