Merge remote-tracking branch 'origin/master' into bump-llvm-10.0.0_3

bcardiff · bcardiff · commit 29850cf7a40a · 2021-07-25T12:49:09.000-03:00
diff --git a/docker/Makefile b/docker/Makefile
@@ -13,11 +13,14 @@ CRYSTAL_TARGZ ?=     ## Which crystal.tar.gz file to install in docker images (a
 DOCKER_TAG ?=	     ## How to tag the docker image (examples: `0.27.2`, `nightly20190307`). `-build` will be appended for build images.
 DOCKER_REPOSITORY ?= ## Docker hub repository to commit image
 
+GC_VERSION = v8.0.4
+LIBATOMIC_OPS_VERSION = v7.6.10
+
 OUTPUT_DIR := build
 BUILD_CONTEXT := $(CURDIR)/build-context
 BUILD_ARGS_UBUNTU64 := --build-arg crystal_deb=crystal.deb $(BUILD_CONTEXT)/ubuntu64 --build-arg base_docker_image=ubuntu:focal
 BUILD_ARGS_UBUNTU32 := --build-arg crystal_deb=crystal.deb $(BUILD_CONTEXT)/ubuntu32 --build-arg base_docker_image=i386/ubuntu:bionic
-BUILD_ARGS_ALPINE := --build-arg crystal_targz=crystal.tar.gz $(BUILD_CONTEXT)/alpine
+BUILD_ARGS_ALPINE := --build-arg crystal_targz=crystal.tar.gz $(BUILD_CONTEXT)/alpine --build-arg gc_version=$(GC_VERSION) --build-arg libatomic_ops_version=$(LIBATOMIC_OPS_VERSION)
 DOCKER_TAG_UBUNTU := $(DOCKER_REPOSITORY):$(DOCKER_TAG)
 DOCKER_TAG_ALPINE := $(DOCKER_REPOSITORY):$(DOCKER_TAG)-alpine
 
@@ -53,6 +56,8 @@ $(BUILD_CONTEXT)/ubuntu64: ubuntu.Dockerfile $(BUILD_CONTEXT)/ubuntu64/crystal.d
 
 $(BUILD_CONTEXT)/alpine: alpine.Dockerfile $(BUILD_CONTEXT)/alpine/crystal.tar.gz
 	cp alpine.Dockerfile $@/Dockerfile
+	mkdir $@/files/
+	cp ../linux/files/feature-thread-stackbottom-upstream.patch $@/files/feature-thread-stackbottom-upstream.patch
 
 %/crystal.deb:
 	mkdir -p $(shell dirname $@)
diff --git a/docker/alpine.Dockerfile b/docker/alpine.Dockerfile
@@ -3,11 +3,33 @@ FROM alpine:3.13 as runtime
 RUN \
   apk add --update --no-cache --force-overwrite \
     # core dependencies
-    gc-dev gcc gmp-dev libatomic_ops libevent-static musl-dev pcre-dev \
+    gcc gmp-dev libevent-static musl-dev pcre-dev \
     # stdlib dependencies
     libxml2-dev openssl-dev openssl-libs-static tzdata yaml-static zlib-static \
     # dev tools
-    make git
+    make git \
+    # build libgc dependencies
+    autoconf automake libtool patch
+
+# Build libgc
+ARG gc_version
+ARG libatomic_ops_version
+COPY files/feature-thread-stackbottom-upstream.patch /tmp/
+RUN git clone https://github.com/ivmai/bdwgc \
+ && cd bdwgc \
+ && git checkout ${gc_version} \
+ && git clone https://github.com/ivmai/libatomic_ops \
+ && (cd libatomic_ops && git checkout ${libatomic_ops_version}) \
+ \
+ && patch -p1 < /tmp/feature-thread-stackbottom-upstream.patch \
+ \
+ && ./autogen.sh \
+ && ./configure --disable-debug --disable-shared --enable-large-config \
+ && make -j$(nproc) CFLAGS=-DNO_GETCONTEXT \
+ && make install
+
+# Remove build tools from image now that libgc is built
+RUN apk del -r --purge autoconf automake libtool patch
 
 ARG crystal_targz
 COPY ${crystal_targz} /tmp/crystal.tar.gz
diff --git a/linux/Dockerfile b/linux/Dockerfile
@@ -88,6 +88,8 @@ ARG musl_target
 RUN git clone https://github.com/crystal-lang/shards \
  && cd shards \
  && git checkout ${shards_version} \
+ # FIXME: Remove this workaround for https://github.com/crystal-lang/crystal/issues/10861
+ && touch shard.lock \
  && make SHARDS=false CRYSTAL=/crystal/bin/crystal \
          FLAGS="--stats --target ${musl_target} --static ${release:+--release}" \
  \
diff --git a/linux/Makefile b/linux/Makefile
@@ -19,7 +19,7 @@ PACKAGE_MAINTAINER = Crystal Team <crystal@manas.tech>
 PREVIOUS_CRYSTAL_RELEASE_LINUX64_TARGZ ?=  ## url to crystal-{version}-{package}-linux-x86_64.tar.gz
 PREVIOUS_CRYSTAL_RELEASE_LINUX32_TARGZ ?=  ## url to crystal-{version}-{package}-linux-i686.tar.gz
 
-SHARDS_VERSION = v0.14.1
+SHARDS_VERSION = v0.15.0
 GC_VERSION = v8.0.4
 LIBATOMIC_OPS_VERSION = v7.6.10
 
diff --git a/omnibus/config/software/shards.rb b/omnibus/config/software/shards.rb
@@ -1,4 +1,4 @@
-SHARDS_VERSION = "0.14.1"
+SHARDS_VERSION = "0.15.0"
 
 name "shards"
 default_version SHARDS_VERSION
@@ -43,6 +43,10 @@
   source md5: "d7bdd10bb096b71428b06fc93097b3cc"
 end
 
+version "0.15.0" do
+  source md5: "696525e924350a1270eee5c221eb6c80"
+end
+
 source url: "https://github.com/crystal-lang/shards/archive/v#{version}.tar.gz"
 
 relative_path "shards-#{version}"
diff --git a/processes/crystal-CHANGELOG.md b/processes/crystal-CHANGELOG.md
@@ -0,0 +1,66 @@
+- **(breaking-change)**
+- **(deprecate)**
+- Fixed ... ([#PR], thanks @author)
+- Fixed docs ...
+- Add ...
+- **(performance)**
+- Add docs ...
+- Refactor ...
+- Fix specs ...
+
+## Language changes
+
+### Macros
+
+## Standard library
+
+### Macros
+
+### Numeric
+<!-- Int, Float, BigInt, ... -->
+
+### Text
+<!-- String, Regex, ...  -->
+
+### Collections
+<!-- Array, Hash, Enumerable, ... -->
+
+### Serialization
+<!-- YAML, JSON, ... -->
+
+### Time
+
+### Files
+
+### Networking
+<!-- HTTP, Sockets, SSL, IO -->
+
+### Logging
+
+### Crypto
+
+### Concurrency
+
+### System
+
+### Runtime
+
+### Spec
+
+## Compiler
+<!-- CLI -->
+
+### Language semantics
+<!-- Codegen issues affecting semantic -->
+
+### Debugger
+
+## Tools
+
+### Formatter
+
+### Doc generator
+
+### Playground
+
+## Others
diff --git a/processes/crystal-release.md b/processes/crystal-release.md
@@ -0,0 +1,103 @@
+# Crystal release process checklist
+
+1. Submit a Freeze/Release PR
+   * It should contain the expected date of the release (~two weeks after the PR is issued).
+   * It should be populated with updates to CHANGELOG.md and VERSION.
+2. Verify Maintenance CI in the parent commit of the changelog succeed
+   * This mean either the commit was the one used for nightly build or,
+   * A branch with `release/` prefix or `ci` word was pushed
+3. Check the artifacts pass the ecosystem test
+   * Grab latest copy of https://github.com/crystal-lang/test-ecosystem
+   * Override branches of projects if needed at ./scripts/10-clone-repos.sh
+   * Set `./crystal-versions.env` with the URL of the artifacts and with the docker image
+   * Use the different make targets described in the `./README`
+4. Merge the Changelog PR
+5. Tag & annotate the commit with the changelog using `<M.m.p>` pattern as {version} (as a pre-release directly in GH?)
+6. Wait for the build
+7. Smoke test with test-ecosystem
+8. Push changes to OBS for building linux packages
+   1. Checkout https://github.com/crystal-lang/distribution-scripts
+   2. Grab the urls (or local path) of the artifacts from circle-ci
+      * `crystal-*-linux-x86_64.tar.gz`
+      * `crystal-*-linux-i686.tar.gz`
+      * `crystal-*-docs.tar.gz`
+   3. Follow [./packages/README.md](../packages/README.md) in distribution-scripts
+      1. The following steps should run in an openSUSE distribution with osc installed (https://github.com/crystal-lang/osc-docker)
+      1. Configure build.opensuse.org credentials in ~/.oscrc
+      1. `osc branchco devel:languages:crystal crystal`
+      1. `cd home\:$OBS_USERNAME\:branches\:devel\:languages\:crystal/crystal`
+      1. Copy the updated build artifact tarballs in place
+      1. Update version in package specifications:
+         ```bash
+         VERSION=1.1.0 # or whatever version you’re releasing
+         sed -i -e "s/^Version: .*/Version: ${VERSION}-1/" *.dsc
+         sed -i -e "s/^Version: .*/Version: ${VERSION}/" *.spec
+         ```
+      1. Verify the changes with `osc diff`
+      1. Update changelog: `osc vc -m “Update devel:languages:crystal to $VERSION`
+      1. Commit changes: `osc commit -m “Update devel:languages:crystal to $VERSION`
+      1. Now OBS builds the packages. It’s best to continue in the browser now: https://build.opensuse.org/project/show/home:$OBS_USER:branches:devel:langauges:crystal/crystal
+      1. Wait for all jobs to finish and when everything is green, create a submit request against the original package (“Submit package” link in the menu bar on the package in your branch)
+9. Build and publish docker images
+   1. Checkout https://github.com/crystal-lang/crystal-dist
+   2. `$ ./dist.sh build-docker {version}`
+   3. `$ ./dist.sh push-docker {version}`
+   4. Manually push the `crystallang/crystal:{version}` image as `crystallang/crystal:latest`
+      * `$ docker tag crystallang/crystal:{version} crystallang/crystal:latest`
+      * `$ docker push crystallang/crystal:latest`
+   5. Manually build and push alpine images
+      * Go to distribution-scripts/docker
+      * `$ make clean alpine DOCKER_REPOSITORY=crystallang/crystal DOCKER_TAG={version} CRYSTAL_VERSION={version} CRYSTAL_TARGZ=~/Downloads/crystal-{version}-1-linux-x86_64.tar.gz`
+      * `$ docker push crystallang/crystal:{version}-alpine`
+      * `$ docker push crystallang/crystal:{version}-alpine-build`
+      * `$ docker tag crystallang/crystal:{version}-alpine crystallang/crystal:latest-alpine`
+      * `$ docker push crystallang/crystal:latest-alpine`
+10. Publish docs
+    1. Have s3cmd installed and AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env variables defined
+    1. `$ source /Volumes/Secured/secure/manas-aws-key-crystal-dist.sh`
+    1. Checkout https://github.com/crystal-lang/crystal-dist
+    1. Download `*-docs.tar.gz` from circle-ci artifacts
+    1. `$ ./dist.sh push-docs {version} path/to/crystal-{version}-docs.tar.gz`
+    1. `$ ./dist.sh update-docs-versions {crystal_working_copy}`
+    1. `$ ./dist.sh redirect-docs {version}`
+11. Publish snap package
+    1. You need to logged in via `$ snapcraft login`
+    1. Recent tagged release is published directly to edge channel. The CI logs the snap revision number. Otherwise the .snap file is in the artifacts.
+    1. Check the current status to find the revision of the tagged release otherwise:
+    1. `$ snapcraft status crystal`
+    1. `$ snapcraft release crystal <revision-number> beta`
+    1. `$ snapcraft release crystal <revision-number> stable`
+12. Build Github release
+    1. Copy the changelog as release notes in the tag (if not done before)
+    1. Upload the signed binaries, *.tar.gz and *.pkg artifacts
+    1. Signed binaries are in bintray/build/signed
+    1. Publish the Github release
+13. Submit a PR to update the homebrew formula in https://github.com/Homebrew/homebrew-core/blob/master/Formula/crystal.rb .
+    1. Update the previous and new version (with their respective hashes).
+    1. Try locally `$ brew install --build-from-source <source of formula>`
+    1. Create PR
+14. Update crystal-website
+    1. Post a release notes in the website
+    1. Update information of `latest_release` in https://github.com/crystal-lang/crystal-website/blob/master/_config.yml
+    1. Update link to blog in https://github.com/crystal-lang/crystal-website/blob/5905e5644fcb4863976d0ca69856edbd74091370/index.html#L26
+15. Post announcement in https://forum.crystal-lang.org/c/news/official
+16. Tweet about the release
+17. Post in Reddit
+18. Update crystal repo to use released version
+   * Edit PREVIOUS_CRYSTAL_BASE_URL in `.circleci/config.yml`
+   * Edit DOCKER_TEST_PREFIX in `bin/ci`
+   * Edit `prepare_build` on_osx download package and folder
+   * Edit ` .github/workflows/win.yml` to point to docker image
+   * Edit `shell.nix` `latestCrystalBinary` using  `nix-prefetch-url --unpack <url>`
+19. Increment VERSION file to the next minor and -dev suffix
+20. Perform uncomment/todos left in the repo
+21. Update carc.in / play.crystal-lang.org (by jhass)
+    Note: This is done after Arch repos are updated
+
+## Observable Helper
+
+Build changelog lines
+https://observablehq.com/d/035be530d554ccdf
+
+Check commit history
+https://observablehq.com/d/4937e5db876fe1d4
diff --git a/processes/shards-release.md b/processes/shards-release.md
@@ -0,0 +1,16 @@
+# Shards release process checklist
+
+1. Submit a Release PR
+   * Should contain updates to CHANGELOG, VERSION, shard.yml
+   * Should contain updates to the man files with the release date
+    `$ make clean docs SOURCE_DATE_EPOCH=$(gdate -d "YYYY-MM-DD" +"%s")` if it’s done ahead of time
+2. Merge the Changelog PR
+3. Tag & annotate the commit with the changelog using `v<M.m.p>` pattern as {version}
+   * `git tag -s vX.X.X`
+   * `git push --tags`
+4. Build Github release
+   * Copy the changelog as release notes in the tag
+5. Update distribution-scripts
+   * [linux/Makefile](../linux/Makefile)
+   * [omnibus/config/software/shards.rb](../omnibus/config/software/shards.rb)
+6. Submit a PR to update the homebrew formula in https://github.com/Homebrew/homebrew-core/blob/master/Formula/crystal.rb . Or do it on Crystal release.

Original file line number	Diff line number	Diff line change
`@@ -88,6 +88,8 @@ ARG musl_target`
`88`	`88`	`RUN git clone https://github.com/crystal-lang/shards \`
`89`	`89`	`&& cd shards \`
`90`	`90`	`&& git checkout ${shards_version} \`
	`91`	`+ # FIXME: Remove this workaround for https://github.com/crystal-lang/crystal/issues/10861`
	`92`	`+ && touch shard.lock \`
`91`	`93`	`&& make SHARDS=false CRYSTAL=/crystal/bin/crystal \`
`92`	`94`	`FLAGS="--stats --target ${musl_target} --static ${release:+--release}" \`
`93`	`95`	`\`