diff --git a/.gitignore b/.gitignore index a33d539b..a54b7129 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,13 @@ omnibus/pkg/ +omnibus/crystal-darwin-universal omnibus/.bundle -omnibus/bin -omnibus/crystal-darwin-* -omnibus/shards-darwin-* omnibus/vendor +omnibus/bin docs/build/ darwin/build/ +darwin/tmp/ snapcraft/snap/snapcraft.yaml snapcraft/*.snap diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..d21bc490 --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +.PHONY: docker-image +docker-image: + make -C linux + DOCKER_BUILDKIT=1 docker build -f Dockerfile -t opentracing-contrib/nginx-opentracing --target final . + +.PHONY: test +test: + ./ci/system_testing.sh + +.PHONY: clean +clean: + rm -fr test-log + rm -fr omnibus/local + rm -fr omnibus/crystal-* + make -C darwin clean + rm -fr ~/tmp + rm -fr ~/.cache/crystal + rm -fr ~/.cache/shards + +# https://github.com/chef/omnibus +.PHONY: setup +omnibus_setup: + cd omnibus && bundle install --binstubs --path vendor/bundler + +.PHONY: darwin +darwin: omnibus_setup + FORCE_GIT_TAGGED=0 CRYSTAL_REPO=https://github.com/crystal-lang/crystal CRYSTAL_SRC=https://github.com/crystal-lang/crystal CRYSTAL_SHA1=master make -C darwin CRYSTAL_VERSION=1.10.1 PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=https://github.com/crystal-lang/crystal/releases/download/1.10.1/crystal-1.10.1-1-darwin-universal.tar.gz diff --git a/darwin/Makefile b/darwin/Makefile index 584a93ef..2eea1849 100644 --- a/darwin/Makefile +++ b/darwin/Makefile @@ -9,18 +9,19 @@ CRYSTAL_SHA1 ?= $(CRYSTAL_VERSION) ## Git tag/branch/sha1 to checkout and build PACKAGE_ITERATION ?= 1 FORCE_GIT_TAGGED ?= 1 ## Require build to be based on git tag/branch -PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ ?= ## url to crystal-{version}-{package}-darwin-x86_64.tar.gz +PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ ?= ## url to crystal-{version}-{package}-darwin-universal.tar.gz OUTPUT_DIR = build # mimics the tgz_package.rb version mangling DARWIN_ARCH = $(shell uname -m) DARWIN_PREFIX = crystal-$(CRYSTAL_VERSION)-$(PACKAGE_ITERATION) -DARWIN_NAME = $(DARWIN_PREFIX)-darwin-$(DARWIN_ARCH).tar.gz +DARWIN_NAME = $(DARWIN_PREFIX)-darwin-universal.tar.gz DARWIN_PKG_NAME = $(DARWIN_PREFIX).$(DARWIN_ARCH).pkg +DARWIN_PKG_NAME_UNIVERSAL = $(DARWIN_PREFIX).universal.pkg .PHONY: all -all: darwin-previous $(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME) ## Build compressed omnibus and distribution packages [default] +all: darwin-previous $(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL) ## Build compressed omnibus and distribution packages [default] .PHONY: help help: ## Show this help @@ -40,20 +41,19 @@ help: ## Show this help awk 'BEGIN {FS = "## "}; /^## [a-zA-Z_-]/ {printf " \033[36m%s\033[0m\n", $$2}; /^## / {printf " %s\n", $$2}' .PHONY: darwin-previous -darwin-previous: $(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal ## download previous crystal darwin release +darwin-previous: $(CURDIR)/../omnibus/crystal-darwin-universal/embedded/bin/crystal ## download previous crystal darwin release -# Once there are prior builds for arm64, this can use DARWIN_ARCH -$(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal: - curl -L -o /tmp/crystal-darwin-x86_64.tar.gz $(PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ) \ - && mkdir -p $(CURDIR)/../omnibus/crystal-darwin-x86_64 \ - && tar xfz /tmp/crystal-darwin-x86_64.tar.gz -C $(CURDIR)/../omnibus/crystal-darwin-x86_64 --strip-components=1 \ - && rm /tmp/crystal-darwin-x86_64.tar.gz \ - && chmod +x $(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal +$(CURDIR)/../omnibus/crystal-darwin-universal/embedded/bin/crystal: + curl -L -o /tmp/crystal-darwin.tar.gz $(PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ) \ + && mkdir -p $(CURDIR)/../omnibus/crystal-darwin-universal \ + && tar xfz /tmp/crystal-darwin.tar.gz -C $(CURDIR)/../omnibus/crystal-darwin-universal --strip-components=1 \ + && rm /tmp/crystal-darwin.tar.gz \ + && chmod +x $(CURDIR)/../omnibus/crystal-darwin-universal/embedded/bin/crystal -$(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME): ## Build omnibus crystal project +$(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL): ## Build omnibus crystal project ifeq ($(FORCE_GIT_TAGGED), 0) rm -Rf $(CURDIR)/tmp && mkdir -p $(CURDIR)/tmp && cd $(CURDIR)/tmp \ - && git clone "$(CRYSTAL_REPO)" \ + && git clone $(CRYSTAL_REPO) \ && cd crystal \ && git checkout $(CRYSTAL_SHA1) \ && git checkout -b $(CRYSTAL_VERSION) @@ -65,9 +65,9 @@ endif && export MACOSX_DEPLOYMENT_TARGET=11.0 \ && export SDKROOT=$(shell xcrun --sdk macosx --show-sdk-path) \ && bundle exec omnibus clean crystal shards \ - && bundle exec omnibus build crystal \ - && cp ./pkg/$(DARWIN_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(subst x86_64,universal,$(DARWIN_NAME)) \ - && cp ./pkg/$(DARWIN_PKG_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(subst x86_64,universal,$(DARWIN_PKG_NAME)) + && bundle exec omnibus build crystal --log-level debug \ + && cp ./pkg/$(DARWIN_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(DARWIN_NAME) \ + && cp ./pkg/$(DARWIN_PKG_NAME) $(CURDIR)/$(OUTPUT_DIR)/$(DARWIN_PKG_NAME_UNIVERSAL) .PHONY: clean clean: ## Clean up build directory @@ -75,6 +75,6 @@ clean: ## Clean up build directory rm -Rf $(CURDIR)/tmp rm -Rf $(CURDIR)/../omnibus/pkg/crystal-* rm -Rf $(CURDIR)/../omnibus/pkg/version-* - rm -Rf $(CURDIR)/../omnibus/crystal-darwin-* + rm -Rf $(CURDIR)/../omnibus/crystal-darwin-universal* rm -Rf /var/cache/omnibus/* rm -Rf /opt/crystal/* diff --git a/docker/Containerfile b/docker/Containerfile new file mode 100644 index 00000000..e69de29b diff --git a/omnibus/config/projects/crystal.rb b/omnibus/config/projects/crystal.rb index a2b1e1eb..5f8f642e 100644 --- a/omnibus/config/projects/crystal.rb +++ b/omnibus/config/projects/crystal.rb @@ -2,7 +2,7 @@ maintainer 'Juan Wajnerman' homepage 'http://crystal-lang.org/' -install_dir '/opt/crystal' +install_dir File.join(ENV.fetch('HOME'), '/tmp/crystal') build_version do source :version, from_dependency: 'crystal' end diff --git a/omnibus/config/projects/llvm.rb b/omnibus/config/projects/llvm.rb index e0c0032b..92099ec3 100644 --- a/omnibus/config/projects/llvm.rb +++ b/omnibus/config/projects/llvm.rb @@ -2,7 +2,7 @@ maintainer 'Juan Wajnerman' homepage 'http://llvm.org/' -install_dir '/opt/llvm' +install_dir File.join(ENV.fetch('HOME'), '/tmp/llvm') build_version do source :version, from_dependency: 'llvm' end diff --git a/omnibus/config/software/crystal.rb b/omnibus/config/software/crystal.rb index 4a31e3cb..f5d84a3c 100644 --- a/omnibus/config/software/crystal.rb +++ b/omnibus/config/software/crystal.rb @@ -1,17 +1,13 @@ CRYSTAL_VERSION = ENV['CRYSTAL_VERSION'] CRYSTAL_SHA1 = ENV['CRYSTAL_SHA1'] FIRST_RUN = ENV["FIRST_RUN"] -CRYSTAL_SRC = (ENV['CRYSTAL_SRC'] || "").strip +CRYSTAL_SRC = ENV.fetch('CRYSTAL_SRC', 'https://github.com/crystal-lang/crystal') name "crystal" default_version CRYSTAL_VERSION skip_transitive_dependency_licensing true -if CRYSTAL_SRC.empty? - source git: "https://github.com/crystal-lang/crystal" -else - source git: CRYSTAL_SRC -end +source git: CRYSTAL_SRC dependency "pcre2" dependency "bdw-gc" @@ -32,60 +28,113 @@ output_path = "#{install_dir}/embedded/bin" output_bin = "#{output_path}/crystal" +output_bin_x86_64 = "#{output_path}/crystal_x86_64" +output_bin_arm = "#{output_path}/crystal_arm" -if FIRST_RUN - env["PATH"] = "#{project_dir}/deps:#{env["PATH"]}" -else - env["PATH"] = "#{llvm_bin.project_dir}/bin:#{project_dir}/deps:#{env["PATH"]}" +path_env = "#{project_dir}/deps:#{env["PATH"]}" +unless FIRST_RUN + path_env = "#{llvm_bin.project_dir}/bin:#{path_env}" end +env["PATH"] = path_env + +# raise Omnibus::Config.cache_dir.inspect if macos? || mac_os_x? - env["CRYSTAL_PATH"] = "lib:/private/var/cache/omnibus/src/crystal/src" + env["CRYSTAL_PATH"] = "lib:/private/var/cache/omnibus/src/crystal/src:#{project_dir}/src" else env["CRYSTAL_PATH"] = "lib:#{project_dir}/src" end build do - command "git checkout #{CRYSTAL_SHA1}", cwd: project_dir + block { puts "\n===> Starting build phase for crystal from #{Dir.pwd} ===\n\n" } + command "echo ' Sources are located in #{project_dir}'", env: env.dup + + command "git checkout '#{CRYSTAL_SHA1}'", cwd: project_dir + block { puts "\n===== 1. Build native crystal bin with embedded universal crystal binary\n\n" } mkdir "#{project_dir}/deps" - make "deps", env: env + make "deps", env: env.dup mkdir ".build" - command "echo #{Dir.pwd}", env: env + block { raise "Could not find embedded/bin/crystal" unless File.exist?("#{Dir.pwd}/crystal-darwin-universal/embedded/bin/crystal") } + copy "#{Dir.pwd}/crystal-darwin-universal/embedded/bin/crystal", ".build/crystal" + command ".build/crystal --version", env: env.dup + command "file .build/crystal", env: env.dup + + # Compile native crflags = "--no-debug" + command "make crystal stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env.dup + block "Testing the result file" do + puts "===== >>> Testing the result file #{output_bin}" + raise "Could not build native crystal: #{output_bin}" unless File.exist?("#{output_bin}") + end + # TODO: Add validation of command output + command "file #{output_bin}", env: env.dup + + block { puts "\n===== 2. Restore compiler with cross-compile support\n\n" } + # Restore compiler w/ cross-compile support + move "#{output_bin}", ".build/crystal" + command ".build/crystal --version", env: env.dup + + # Clean up + make "clean_cache clean", env: env - copy "#{Dir.pwd}/crystal-#{ohai['os']}-#{ohai['kernel']['machine']}/embedded/bin/crystal", ".build/crystal" + block { puts "\n===== 3. Building crystal x86_64 version\n\n" } - # Compile for Intel - command "make crystal stats=true release=true FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env - move output_bin, "#{output_bin}_x86_64" + original_CXXFLAGS_env = env["CXXFLAGS"].dup + original_LDFLAGS_env = env["LDFLAGS"].dup + + # Build for x86_64 + env["CXXFLAGS"] = original_CXXFLAGS_env + " -target x86_64-apple-darwin" + env["LDFLAGS"] = original_LDFLAGS_env + " -v -target x86_64-apple-darwin" + make "deps", env: env.dup + + make "crystal verbose=true stats=true release=true target=x86_64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=x86_64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env + command "clang #{output_path}/crystal.o -o #{output_bin_x86_64} -target x86_64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env + + # Assertion + block { raise "Could not build #{output_bin_x86_64}" unless File.exist?(output_bin_x86_64) } + command "file #{output_bin_x86_64}", env: env # Clean up + delete "#{output_path}/crystal.o" make "clean_cache clean", env: env - # Restore x86_64 compiler w/ cross-compile support - mkdir ".build" - copy "#{output_bin}_x86_64", ".build/crystal" + # Build for arm64 + block { puts "\n===== 4. Building crystal arm64 version\n\n" } # Compile for ARM64. Apple's clang only understands arm64, LLVM uses aarch64, # so we need to sub out aarch64 in our calls to Apple tools - env["CXXFLAGS"] << " -target arm64-apple-darwin" - make "deps", env: env + env["CXXFLAGS"] = original_CXXFLAGS_env + " -target arm64-apple-darwin" + env["LDFLAGS"] = original_LDFLAGS_env + " -v -target arm64-apple-darwin" + make "deps", env: env.dup + make "crystal verbose=true stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env + command "clang #{output_path}/crystal.o -o #{output_bin_arm} -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env - make "crystal stats=true release=true target=aarch64-apple-darwin FLAGS=\"#{crflags}\" CRYSTAL_CONFIG_TARGET=aarch64-apple-darwin CRYSTAL_CONFIG_LIBRARY_PATH= O=#{output_path}", env: env + # Assertion + block { raise "Could not build #{output_bin_arm}" unless File.exist?(output_bin_arm) } + command "file #{output_bin_arm}", env: env - command "clang #{output_path}/crystal.o -o #{output_bin}_arm64 -target arm64-apple-darwin src/llvm/ext/llvm_ext.o `llvm-config --libs --system-libs --ldflags 2>/dev/null` -lstdc++ -lpcre2-8 -lgc -lpthread -levent -liconv -ldl -v", env: env + # Clean up delete "#{output_path}/crystal.o" # Lipo them up - command "lipo -create -output #{output_bin} #{output_bin}_x86_64 #{output_bin}_arm64" - delete "#{output_bin}_x86_64" - delete "#{output_bin}_arm64" + block { puts "\n===== 5. Combine x86_64 and arm64 binaries in single crystal universal binary\n\n" } + command "lipo -create -output #{output_bin} #{output_bin_x86_64} #{output_bin_arm}" block do - raise "Could not build crystal" unless File.exist?(output_bin) + puts "===== >>> Testing the result file #{output_bin}" + raise "Could not build universal crystal #{output_bin}" unless File.exist?(output_bin) + end + # TODO: Add validation of command output + command "file #{output_bin}", env: env.dup + + # Clean up + delete output_bin_x86_64 + delete output_bin_arm + make "clean_cache clean", env: env + block do if macos? || mac_os_x? otool_libs = `otool -L #{output_bin}` if otool_libs.include?("/usr/local/lib") || otool_libs.include?('/opt/homebrew/lib') @@ -103,4 +152,6 @@ dest: "#{install_dir}/bin/crystal", mode: 0755, vars: { install_dir: install_dir } + + block { puts "\n===< Crystal successfully built\n\n" } end diff --git a/omnibus/config/software/llvm_bin.rb b/omnibus/config/software/llvm_bin.rb index eaa62b1f..24078da1 100644 --- a/omnibus/config/software/llvm_bin.rb +++ b/omnibus/config/software/llvm_bin.rb @@ -18,7 +18,11 @@ raise "llvm_bin not supported" end -source url: "http://crystal-lang.s3.amazonaws.com/llvm/llvm-#{version}-#{ohai['os']}-#{ohai['kernel']['machine']}.tar.gz", +platform = ohai['os'] +arch = ohai['kernel']['machine'] +arch = "x86_64" + +source url: "http://crystal-lang.s3.amazonaws.com/llvm/llvm-#{version}-#{platform}-#{arch}.tar.gz", md5: source_md5 relative_path "llvm-#{version}" diff --git a/omnibus/config/software/shards.rb b/omnibus/config/software/shards.rb index 2b6448ad..e1e17642 100644 --- a/omnibus/config/software/shards.rb +++ b/omnibus/config/software/shards.rb @@ -82,22 +82,57 @@ env["CPPFLAGS"] = env["CPPFLAGS"].gsub("-arch arm64 -arch x86_64", "") build do - crflags = "--no-debug --release" + block { puts "\n=== Starting build phase for shards from #{project_dir} ===\n\n" } + + make "clean", env: env + command "#{install_dir}/bin/crystal clear_cache" - # Build for Intel - make "bin/shards SHARDS=false CRYSTAL=#{install_dir}/bin/crystal FLAGS='#{crflags}'", env: env - move "bin/shards", "bin/shards_x86_64" + # Build for x86_64 + crflags = "--no-debug --release" + block { puts "\n===== 1. Building shards x86_64 version\n\n" } + crflags_x86_64 = crflags + " --cross-compile --target x86_64-apple-darwin" + make "bin/shards SHARDS=false CRYSTAL=#{install_dir}/bin/crystal FLAGS='#{crflags_x86_64}'", env: env + + output_bin = "#{project_dir}/bin/shards" + output_bin_x86_64 = "#{output_bin}_x86_64" + target_x86_64 = 'x86_64-apple-darwin' + command "clang bin/shards.o -o #{output_bin_x86_64} -v -target #{target_x86_64} -L#{install_dir}/embedded/lib -lyaml -lpcre2-8 -lgc -lpthread -levent -liconv -ldl", env: env + block "Testing the result file" do + puts "===== >>> Testing the result file #{output_bin_x86_64} in #{project_dir}}" + raise "Could not build #{output_bin_x86_64}" unless File.exist?(output_bin_x86_64) + end + # TODO: Add a validation of the output to check archs + command "file #{output_bin_x86_64}", env: env.dup # Clean make "clean", env: env - - # Build for ARM64 - crflags += " --cross-compile --target aarch64-apple-darwin" - make "bin/shards SHARDS=false CRYSTAL=#{install_dir}/bin/crystal FLAGS='#{crflags}'", env: env - command "clang bin/shards.o -o bin/shards_arm64 -target arm64-apple-darwin -L#{install_dir}/embedded/lib -lyaml -lpcre2-8 -lgc -lpthread -levent -liconv -ldl", env: env + command "#{install_dir}/bin/crystal clear_cache" + + # Build for arm64 + block { puts "\n===== 2. Building shards arm64 version\n\n" } + crflags_arm = crflags + " --cross-compile --target aarch64-apple-darwin" + make "bin/shards SHARDS=false CRYSTAL=#{install_dir}/bin/crystal FLAGS='#{crflags_arm}'", env: env + output_bin_arm64 = "#{output_bin}_arm64" + target_arm64 = 'arm64-apple-darwin' + command "clang bin/shards.o -o #{output_bin_arm64} -v -target #{target_arm64} -L#{install_dir}/embedded/lib -lyaml -lpcre2-8 -lgc -lpthread -levent -liconv -ldl", env: env + block "Testing the result file" do + puts "===== >>> Testing the result file #{output_bin_arm64}" + raise "Could not build #{output_bin_arm64}" unless File.exist?("#{output_bin_arm64}") + end + # TODO: Add a validation of the output to check archs + command "file #{output_bin_arm64}", env: env.dup # Lipo them up - command "lipo -create -output bin/shards bin/shards_x86_64 bin/shards_arm64" + block { puts "\n===== 3. Combine x86_64 and arm64 binaries in a single universal binary\n\n" } + command "lipo -create -output #{output_bin} #{output_bin_x86_64} #{output_bin_arm64}" + block "Testing the result file" do + puts "===== >>> Testing the result file #{output_bin}" + raise "Could not build #{output_bin}: #{output_bin}" unless File.exist?("#{output_bin}") + end + # TODO: Add a validation of the output to check archs + command "file #{output_bin}", env: env.dup copy "bin/shards", "#{install_dir}/embedded/bin/shards" + + block { puts "\n===< Shards successfully built\n\n" } end diff --git a/omnibus/config/software/tgz_package.rb b/omnibus/config/software/tgz_package.rb index 200754f8..b550974c 100644 --- a/omnibus/config/software/tgz_package.rb +++ b/omnibus/config/software/tgz_package.rb @@ -7,7 +7,7 @@ destination = File.expand_path('pkg', Omnibus::Config.project_root) version = "#{project.build_version}-#{project.build_iteration}" version.gsub!("/", "-") - tgz_name = "#{project.name}-#{version}-#{ohai['os']}-#{ohai['kernel']['machine']}.tar.gz" + tgz_name = "#{project.name}-#{version}-#{ohai['os']}-universal.tar.gz" if macos? || mac_os_x? transform = "-s /./#{project.name}-#{version}/" else @@ -16,5 +16,8 @@ command "tar czf #{destination}/#{tgz_name} #{transform} -C #{install_dir} .", env: {"COPYFILE_DISABLE" => "1"} + + # NOTE: hack to bypass `git_caching` to modify anything in the working directory to create a commit + command "date > #{install_dir}/tgz_package.log" end end diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb index 47a9fad4..2abb9f2d 100644 --- a/omnibus/omnibus.rb +++ b/omnibus/omnibus.rb @@ -17,12 +17,12 @@ # # Uncomment this line to change the default base directory to "local" # ------------------------------------------------------------------- -# base_dir './local' +base_dir File.join(ENV.fetch('HOME'), '/tmp/omnibus') # # Alternatively you can tune the individual values # ------------------------------------------------ # cache_dir './local/omnibus/cache' -# git_cache_dir './local/omnibus/cache/install_path' +# git_cache_dir './local/omnibus/cache/{install_path}' # source_dir './local/omnibus/src' # build_dir './local/omnibus/build' # package_dir './local/omnibus/pkg'