Skip to content
This repository was archived by the owner on Feb 1, 2024. It is now read-only.

Commit b0d608f

Browse files
nikhilsarafnikhilsaraf
authored
automate creation of pre-compiled binary for ccxt-rest (#311)
* 1 - check for pkg installed, download and untar ccxt source code * 2 - split ccxt_bin_gen script from fs_bin_gen script * 3 - move to separate folders in ./scripts * 4 - run npm install before running pkg tool; copy dependency files to output directory * 5 - build script should check build result after installing web dependencies and generating static files * 6 - wrap underlying error when pkg check fails * 7 - check node version before generating ccxt binary * 8 - separate -g flag on build script to generate ccxt binary * 9 - remove /downloads from path of ccxt binary output * 10 - set silent registrations on kelpos and prettify ccxt_bin_gen log output * 11 - zip output of ccxt binary * 12 - check result of running ccxt_bin_gen in main build script * 13 - keep zipped output file in it's own folder and remove inner folder hierarchy * 14 - clean up directory that was zipped * 15 - remove fetch logic of ccxt binary from build script * update circle config to install yarn * 17 - install yarn after usinng correct version of node * 18 - command to use yarn in same shell * 19 - update command to use yarn and move setup of nvm and yarn to install_deps section * 20 - use nvm version 10.17 * 21 - move filesystem_vfsdata_dev to avoid package conflict * 22 - ReplaceAll is only available from go10.12 onwards
1 parent 6062a2d commit b0d608f

File tree

9 files changed

+295
-32
lines changed

9 files changed

+295
-32
lines changed

‎.circleci/config.yml‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ commands:
55
- checkout
66
- restore_cache:
77
keys:
8-
- v2-pkg-cache
9-
- v1-node-cache
8+
- v3-pkg-cache
9+
- v5-node-cache
1010
- run:
1111
name: Install glide
1212
command: curl https://glide.sh/get | sh
@@ -19,9 +19,20 @@ commands:
1919
- run:
2020
name: Install astilectron-bundler
2121
command: go install github.com/asticode/go-astilectron-bundler
22-
22+
- run:
23+
name: Install nvm
24+
command: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
25+
- run:
26+
name: Use Node 10.5
27+
command: echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV && echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV && source $BASH_ENV && nvm install 10.17 && nvm use 10.17
28+
- run:
29+
name: Install yarn
30+
command: curl -o- -L https://yarnpkg.com/install.sh | bash
31+
- run:
32+
name: Use yarn
33+
command: echo 'export PATH="$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH"' >> $BASH_ENV && source $BASH_ENV
2334
- save_cache:
24-
key: v2-pkg-cache
35+
key: v3-pkg-cache
2536
paths:
2637
- "/go/src/github.com/stellar/kelp/vendor"
2738

@@ -33,17 +44,11 @@ commands:
3344

3445
build_kelp:
3546
steps:
36-
- run:
37-
name: Install nvm
38-
command: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
39-
- run:
40-
name: Use Node 10.5
41-
command: echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV && echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV && source $BASH_ENV && nvm install 10.5 && nvm use 10.5
4247
- run:
4348
name: Build Kelp
4449
command: ./scripts/build.sh
4550
- save_cache:
46-
key: v1-node-cache
51+
key: v5-node-cache
4752
paths:
4853
- "/go/src/github.com/stellar/kelp/gui/web/node_modules"
4954
- run:

‎scripts/build.sh‎

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
#!/bin/bash
22

33
function usage() {
4-
echo "Usage: $0 [flags]"
4+
echo "Usage: $0 [flags] [flag-fields]"
55
echo ""
66
echo "Flags:"
7-
echo " -d, --deploy prepare tar archives in build/, only works on a tagged commit in the format v1.0.0 or v1.0.0-rc1"
8-
echo " -t, --test-deploy test prepare tar archives in build/ for your native platform only"
9-
echo " -h, --help show this help info"
10-
exit 1
7+
echo " -d, --deploy prepare tar archives in build/, only works on a tagged commit in the format v1.0.0 or v1.0.0-rc1"
8+
echo " -t, --test-deploy test prepare tar archives in build/ for your native platform only"
9+
echo " -g, --gen-ccxt generate binary for ccxt-rest executable for to be uploaded to GitHub for use in building kelp binary, takes in arguments (linux, darwin, windows)"
10+
echo " -h, --help show this help info"
1111
}
1212

1313
function install_web_dependencies() {
@@ -16,6 +16,7 @@ function install_web_dependencies() {
1616
cd $CURRENT_DIR/gui/web
1717

1818
yarn install
19+
check_build_result $?
1920

2021
cd $CURRENT_DIR
2122
echo "... finished installing web dependencies"
@@ -28,6 +29,7 @@ function generate_static_web_files() {
2829
cd $CURRENT_DIR/gui/web
2930

3031
yarn build
32+
check_build_result $?
3133

3234
cd $CURRENT_DIR
3335
echo "... finished generating contents of gui/web/build"
@@ -44,7 +46,16 @@ function check_build_result() {
4446
fi
4547
}
4648

47-
if [[ ($# -gt 1 || $(basename $("pwd")) != "kelp") ]]
49+
# takes in the GOOS for which to build
50+
function gen_ccxt_binary() {
51+
echo "generating ccxt binary for GOOS=$1"
52+
echo ""
53+
go run ./scripts/ccxt_bin_gen/ccxt_bin_gen.go -goos $1
54+
check_build_result $?
55+
echo "successful"
56+
}
57+
58+
if [[ $(basename $("pwd")) != "kelp" ]]
4859
then
4960
echo "need to invoke from the root 'kelp' directory"
5061
exit 1
@@ -60,10 +71,36 @@ elif [[ ($# -eq 1 && ("$1" == "-t" || "$1" == "--test-deploy")) ]]; then
6071
IS_TEST_MODE=1
6172
elif [[ ($# -eq 1 && ("$1" == "-h" || "$1" == "--help")) ]]; then
6273
usage
63-
elif [[ $# -eq 1 ]]; then
74+
exit 0
75+
elif [[ (($# -eq 1 || $# -eq 2) && ("$1" == "-g" || "$1" == "--gen-ccxt")) ]]; then
76+
if [[ $# -eq 1 ]]; then
77+
echo "the $1 flag needs to be followed by the GOOS for which to build the ccxt binary"
78+
echo ""
79+
usage
80+
exit 1
81+
fi
82+
83+
if [[ $# -eq 2 ]]; then
84+
if [[ "$2" == "linux" || "$2" == "darwin" || "$2" == "windows" ]]; then
85+
gen_ccxt_binary $2
86+
echo ""
87+
echo "BUILD SUCCESSFUL"
88+
exit 0
89+
else
90+
echo "invalid GOOS type passed in: $2"
91+
echo ""
92+
usage
93+
exit 1
94+
fi
95+
fi
96+
6497
usage
65-
else
98+
exit 1
99+
elif [[ $# -eq 0 ]]; then
66100
ENV=dev
101+
else
102+
usage
103+
exit 1
67104
fi
68105

69106
# version is git tag if it's available, otherwise git hash
@@ -120,7 +157,7 @@ fi
120157

121158
echo ""
122159
echo "embedding contents of gui/web/build into a .go file (env=$ENV) ..."
123-
go run ./scripts/fs_bin_gen.go -env $ENV
160+
go run ./scripts/fs_bin_gen/fs_bin_gen.go -env $ENV
124161
check_build_result $?
125162
echo "... finished embedding contents of gui/web/build into a .go file (env=$ENV)"
126163
echo ""

‎scripts/ccxt_bin_gen/ccxt_bin_gen.go‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"fmt"
6+
"log"
7+
"path/filepath"
8+
"regexp"
9+
"strings"
10+
11+
"github.com/pkg/errors"
12+
"github.com/stellar/kelp/support/kelpos"
13+
"github.com/stellar/kelp/support/networking"
14+
)
15+
16+
const kelpPrefsDirectory = "build"
17+
const ccxtDownloadURL = "https://github.com/ccxt-rest/ccxt-rest/archive/v0.0.4.tar.gz"
18+
const ccxtDownloadFilename = "ccxt-rest_v0.0.4.tar.gz"
19+
const ccxtUntaredDirName = "ccxt-rest-0.0.4"
20+
const ccxtBinOutputDir = "bin"
21+
const nodeVersionMatchRegex = "v8.[0-9]+.[0-9]+"
22+
23+
func main() {
24+
goosP := flag.String("goos", "", "GOOS for which to build")
25+
flag.Parse()
26+
goos := *goosP
27+
28+
pkgos := ""
29+
if goos == "darwin" {
30+
pkgos = "macos"
31+
} else if goos == "linux" {
32+
pkgos = "linux"
33+
} else if goos == "windows" {
34+
pkgos = "win"
35+
} else {
36+
panic("unsupported goos flag: " + goos)
37+
}
38+
39+
kos := kelpos.GetKelpOS()
40+
kos.SetSilentRegistrations()
41+
42+
zipFoldername := fmt.Sprintf("ccxt-rest_%s-x64", goos)
43+
generateCcxtBinary(kos, pkgos, zipFoldername)
44+
}
45+
46+
func checkNodeVersion(kos *kelpos.KelpOS) {
47+
fmt.Printf("checking node version ... ")
48+
49+
version, e := kos.Blocking("node", "node -v")
50+
if e != nil {
51+
log.Fatal(errors.Wrap(e, "ensure that the `pkg` tool is installed correctly. You can get it from here https://github.com/zeit/pkg or by running `npm install -g pkg`"))
52+
}
53+
54+
match, e := regexp.Match(nodeVersionMatchRegex, version)
55+
if e != nil {
56+
log.Fatal(errors.Wrap(e, "could not match regex against node version"))
57+
}
58+
if !match {
59+
log.Fatal("node version will fail to compile a successful binary because of the requirements of ccxt-rest's dependencies, should use v8.x.x instead of " + string(version))
60+
}
61+
62+
fmt.Printf("valid\n")
63+
}
64+
65+
func checkPkgTool(kos *kelpos.KelpOS) {
66+
fmt.Printf("checking for presence of `pkg` tool ... ")
67+
_, e := kos.Blocking("pkg", "pkg -v")
68+
if e != nil {
69+
log.Fatal(errors.Wrap(e, "ensure that the `pkg` tool is installed correctly. You can get it from here https://github.com/zeit/pkg or by running `npm install -g pkg`"))
70+
}
71+
fmt.Printf("done\n")
72+
}
73+
74+
func downloadCcxtSource(kos *kelpos.KelpOS, downloadDir string) {
75+
fmt.Printf("making directory where we can download ccxt file %s ... ", downloadDir)
76+
e := kos.Mkdir(downloadDir)
77+
if e != nil {
78+
log.Fatal(errors.Wrap(e, "could not make directory for downloadDir "+downloadDir))
79+
}
80+
fmt.Printf("done\n")
81+
82+
fmt.Printf("downloading file from URL %s ... ", ccxtDownloadURL)
83+
downloadFilePath := filepath.Join(downloadDir, ccxtDownloadFilename)
84+
e = networking.DownloadFile(ccxtDownloadURL, downloadFilePath)
85+
if e != nil {
86+
log.Fatal(errors.Wrap(e, "could not download ccxt tar.gz file"))
87+
}
88+
fmt.Printf("done\n")
89+
90+
fmt.Printf("untaring file %s ... ", downloadFilePath)
91+
_, e = kos.Blocking("tar", fmt.Sprintf("tar xvf %s -C %s", downloadFilePath, downloadDir))
92+
if e != nil {
93+
log.Fatal(errors.Wrap(e, "could not untar ccxt file"))
94+
}
95+
fmt.Printf("done\n")
96+
}
97+
98+
func npmInstall(kos *kelpos.KelpOS, installDir string) {
99+
fmt.Printf("running npm install on directory %s ... ", installDir)
100+
npmCmd := fmt.Sprintf("cd %s && npm install && cd -", installDir)
101+
_, e := kos.Blocking("npm", npmCmd)
102+
if e != nil {
103+
log.Fatal(errors.Wrap(e, "failed to run npm install"))
104+
}
105+
fmt.Printf("done\n")
106+
}
107+
108+
// pkg --targets node8-linux-x64 build/ccxt/ccxt-rest-0.0.4
109+
func runPkgTool(kos *kelpos.KelpOS, sourceDir string, outDir string, pkgos string) {
110+
target := fmt.Sprintf("node8-%s-x64", pkgos)
111+
112+
fmt.Printf("running pkg tool on source directory %s with output directory as %s on target platform %s ... ", sourceDir, outDir, target)
113+
pkgCommand := fmt.Sprintf("pkg --out-path %s --targets %s %s", outDir, target, sourceDir)
114+
outputBytes, e := kos.Blocking("pkg", pkgCommand)
115+
if e != nil {
116+
log.Fatal(errors.Wrap(e, "failed to run pkg tool"))
117+
}
118+
fmt.Printf("done\n")
119+
120+
copyDependencyFiles(kos, outDir, string(outputBytes))
121+
}
122+
123+
func copyDependencyFiles(kos *kelpos.KelpOS, outDir string, pkgCmdOutput string) {
124+
fmt.Println()
125+
fmt.Printf("copying dependency files to the output directory %s ...\n", outDir)
126+
for _, line := range strings.Split(pkgCmdOutput, "\n") {
127+
if !strings.Contains(line, "node_modules") {
128+
continue
129+
}
130+
filename := strings.TrimSpace(strings.Replace(line, "(MISSING)", "", -1))
131+
132+
fmt.Printf(" copying file %s to the output directory %s ... ", filename, outDir)
133+
cpCmd := fmt.Sprintf("cp %s %s", filename, outDir)
134+
_, e := kos.Blocking("cp", cpCmd)
135+
if e != nil {
136+
log.Fatal(errors.Wrap(e, "failed to copy dependency file "+filename))
137+
}
138+
fmt.Printf("done\n")
139+
}
140+
fmt.Printf("done\n")
141+
fmt.Println()
142+
}
143+
144+
func mkDir(kos *kelpos.KelpOS, zipDir string) {
145+
fmt.Printf("making directory %s ... ", zipDir)
146+
e := kos.Mkdir(zipDir)
147+
if e != nil {
148+
log.Fatal(errors.Wrap(e, "unable to make directory "+zipDir))
149+
}
150+
fmt.Printf("done\n")
151+
}
152+
153+
func zipOutput(kos *kelpos.KelpOS, ccxtDir string, sourceDir string, zipFoldername string, zipOutDir string) {
154+
zipFilename := zipFoldername + ".zip"
155+
fmt.Printf("zipping directory %s as file %s ... ", filepath.Join(ccxtDir, ccxtBinOutputDir), zipFilename)
156+
zipCmd := fmt.Sprintf("cd %s && mv %s %s && zip -rq %s %s && cd - && mv %s %s", ccxtDir, ccxtBinOutputDir, zipFoldername, zipFilename, zipFoldername, filepath.Join(ccxtDir, zipFilename), zipOutDir)
157+
_, e := kos.Blocking("zip", zipCmd)
158+
if e != nil {
159+
log.Fatal(errors.Wrap(e, "unable to zip folder with ccxt binary and dependencies"))
160+
}
161+
fmt.Printf("done\n")
162+
163+
zipDirPath := filepath.Join(ccxtDir, zipFoldername)
164+
fmt.Printf("clean up zipped directory %s ... ", zipDirPath)
165+
cleanupCmd := fmt.Sprintf("rm %s/* && rmdir %s", zipDirPath, zipDirPath)
166+
_, e = kos.Blocking("zip", cleanupCmd)
167+
if e != nil {
168+
log.Fatal(errors.Wrap(e, fmt.Sprintf("unable to cleanup zip folder %s with ccxt binary and dependencies", zipDirPath)))
169+
}
170+
fmt.Printf("done\n")
171+
}
172+
173+
func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string, zipFoldername string) {
174+
checkNodeVersion(kos)
175+
checkPkgTool(kos)
176+
177+
ccxtDir := filepath.Join(kelpPrefsDirectory, "ccxt")
178+
sourceDir := filepath.Join(ccxtDir, ccxtUntaredDirName)
179+
outDir := filepath.Join(ccxtDir, ccxtBinOutputDir)
180+
zipOutDir := filepath.Join(ccxtDir, "zipped")
181+
182+
downloadCcxtSource(kos, ccxtDir)
183+
npmInstall(kos, sourceDir)
184+
runPkgTool(kos, sourceDir, outDir, pkgos)
185+
mkDir(kos, zipOutDir)
186+
zipOutput(kos, ccxtDir, outDir, zipFoldername, zipOutDir)
187+
}

‎scripts/clean.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ fi
1818

1919
echo "removing files ..."
2020
rm -vrf bin
21-
rm -vrf build
21+
delete_large_dir build
2222
delete_large_dir gui/web/build
2323
delete_large_dir gui/web/node_modules
2424
rm -vf gui/filesystem_vfsdata.go

scripts/fs_bin_gen.go renamed to scripts/fs_bin_gen/fs_bin_gen.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"github.com/shurcooL/vfsgen"
1010
)
1111

12-
const fsDev_filename = "./scripts/fs_gen/filesystem_vfsdata_dev.go"
12+
const fsDev_filename = "./scripts/fs_bin_gen/gui/filesystem_vfsdata_dev.go"
1313
const fs_filename = "./gui/filesystem_vfsdata.go"
1414

1515
func main() {
@@ -18,15 +18,15 @@ func main() {
1818
env := *envP
1919

2020
if env == "dev" {
21-
generateDev()
21+
generateWeb_Dev()
2222
} else if env == "release" {
23-
generateRelease()
23+
generateWeb_Release()
2424
} else {
2525
panic("unrecognized env flag: " + env)
2626
}
2727
}
2828

29-
func generateRelease() {
29+
func generateWeb_Release() {
3030
fs := http.Dir("./gui/web/build")
3131
e := vfsgen.Generate(fs, vfsgen.Options{
3232
Filename: fs_filename,
@@ -39,7 +39,7 @@ func generateRelease() {
3939
}
4040
}
4141

42-
func generateDev() {
42+
func generateWeb_Dev() {
4343
c := exec.Command("cp", fsDev_filename, fs_filename)
4444
e := c.Run()
4545
if e != nil {

scripts/fs_gen/filesystem_vfsdata_dev.go renamed to scripts/fs_bin_gen/gui/filesystem_vfsdata_dev.go

File renamed without changes.

0 commit comments

Comments
 (0)