Merge pull request #11338 from rabbitmq/loic-make-make-maker

Making make better

Author: lhoguin

Makefile

@@ -5,7 +5,7 @@ PROJECT_DESCRIPTION = RabbitMQ Server
 # other components. If PROJECT_VERSION is unset, then an empty variable
 # is propagated and the default version will fallback to the default
 # value from rabbitmq-components.mk.
-export RABBITMQ_VERSION = $(PROJECT_VERSION)
+export RABBITMQ_VERSION := $(PROJECT_VERSION)
 
 # Release artifacts are put in $(PACKAGES_DIR).
 PACKAGES_DIR ?= $(abspath PACKAGES)
@@ -29,6 +29,7 @@ DEP_PLUGINS = rabbit_common/mk/rabbitmq-dist.mk \
 
 DISABLE_DISTCLEAN = 1
 
+ifeq ($(filter-out xref,$(MAKECMDGOALS)),)
 XREF_SCOPE = app deps
 
 # We add all the applications that are in non-standard paths
@@ -47,12 +48,19 @@ XREF_IGNORE = [ \
 
 # Include Elixir libraries in the Xref checks.
 xref: ERL_LIBS := $(ERL_LIBS):$(CURDIR)/apps:$(CURDIR)/deps:$(dir $(shell elixir --eval ":io.format '~s~n', [:code.lib_dir :elixir ]"))
+endif
 
 ifneq ($(wildcard deps/.hex/cache.erl),)
 deps:: restore-hex-cache-ets-file
 endif
 
 include rabbitmq-components.mk
+
+# Set PROJECT_VERSION, calculated in rabbitmq-components.mk,
+# in stone now, because in this Makefile we will be using it
+# multiple times (including for release file names and whatnot).
+PROJECT_VERSION := $(PROJECT_VERSION)
+
 include erlang.mk
 include mk/github-actions.mk
 include mk/bazel.mk

deps/rabbit/Makefile

@@ -247,13 +247,6 @@ ifdef TRACE_SUPERVISOR2
 RMQ_ERLC_OPTS += -DTRACE_SUPERVISOR2=true
 endif
 
-ifndef USE_PROPER_QC
-# PropEr needs to be installed for property checking
-# http://proper.softlab.ntua.gr/
-USE_PROPER_QC := $(shell $(ERL) -eval 'io:format({module, proper} =:= code:ensure_loaded(proper)), halt().')
-RMQ_ERLC_OPTS += $(if $(filter true,$(USE_PROPER_QC)),-Duse_proper_qc)
-endif
-
 # --------------------------------------------------------------------
 # Documentation.
 # --------------------------------------------------------------------

deps/rabbit_common/mk/rabbitmq-dist.mk

@@ -214,77 +214,32 @@ do-dist:: $(DIST_EZS)
 CLI_SCRIPTS_LOCK = $(CLI_SCRIPTS_DIR).lock
 CLI_ESCRIPTS_LOCK = $(CLI_ESCRIPTS_DIR).lock
 
+ifeq ($(MAKELEVEL),0)
 ifneq ($(filter-out rabbit_common amqp10_common rabbitmq_stream_common,$(PROJECT)),)
 app:: install-cli
 test-build:: install-cli
 endif
+endif
 
 install-cli: install-cli-scripts install-cli-escripts
 	@:
 
-ifeq ($(PROJECT),rabbit)
-install-cli-scripts:
-	$(gen_verbose) \
-	if command -v flock >/dev/null; then \
-		flock $(CLI_SCRIPTS_LOCK) \
-		sh -c 'mkdir -p "$(CLI_SCRIPTS_DIR)" && \
-		for file in scripts/*; do \
-			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
-			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
-		done'; \
-	elif command -v lockf >/dev/null; then \
-		lockf $(CLI_SCRIPTS_LOCK) \
-		sh -c 'mkdir -p "$(CLI_SCRIPTS_DIR)" && \
-		for file in scripts/*; do \
-			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
-			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
-		done'; \
-	else \
-		mkdir -p "$(CLI_SCRIPTS_DIR)" && \
-		for file in scripts/*; do \
-			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
-			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
-		done; \
-	fi
-else
-
 install-cli-scripts:
 	$(gen_verbose) \
 	set -e; \
-	if test -d "$(DEPS_DIR)/rabbit/scripts"; then \
-		rabbit_scripts_dir='$(DEPS_DIR)/rabbit/scripts'; \
-	elif test -d "$(DEPS_DIR)/../scripts"; then \
-		rabbit_scripts_dir='$(DEPS_DIR)/../scripts'; \
-	else \
-		echo 'rabbit/scripts directory not found' 1>&2; \
-		exit 1; \
-	fi; \
-	test -d "$$rabbit_scripts_dir"; \
+	test -d "$(DEPS_DIR)/rabbit/scripts"; \
 	if command -v flock >/dev/null; then \
 		flock $(CLI_SCRIPTS_LOCK) \
 		sh -e -c 'mkdir -p "$(CLI_SCRIPTS_DIR)" && \
-		for file in "'$$rabbit_scripts_dir'"/*; do \
-			test -f "$$file"; \
-			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
-			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
-		done'; \
+			cp -a $(DEPS_DIR)/rabbit/scripts/* $(CLI_SCRIPTS_DIR)/'; \
 	elif command -v lockf >/dev/null; then \
 		lockf $(CLI_SCRIPTS_LOCK) \
 		sh -e -c 'mkdir -p "$(CLI_SCRIPTS_DIR)" && \
-		for file in "'$$rabbit_scripts_dir'"/*; do \
-			test -f "$$file"; \
-			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
-			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
-		done'; \
+			cp -a $(DEPS_DIR)/rabbit/scripts/* $(CLI_SCRIPTS_DIR)/'; \
 	else \
 		mkdir -p "$(CLI_SCRIPTS_DIR)" && \
-		for file in "$$rabbit_scripts_dir"/*; do \
-			test -f "$$file"; \
-			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
-			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
-		done; \
+			cp -a $(DEPS_DIR)/rabbit/scripts/* $(CLI_SCRIPTS_DIR)/; \
 	fi
-endif
 
 install-cli-escripts:
 	$(gen_verbose) \

deps/rabbit_common/mk/rabbitmq-run.mk

@@ -96,16 +96,22 @@ RABBITMQ_ENABLED_PLUGINS_FILE ?= $(call node_enabled_plugins_file,$(RABBITMQ_NOD
 RABBITMQ_LOG ?= debug,+color
 export RABBITMQ_LOG
 
-# erlang.mk adds dependencies' ebin directory to ERL_LIBS. This is
-# a sane default, but we prefer to rely on the .ez archives in the
-# `plugins` directory so the plugin code is executed. The `plugins`
-# directory is added to ERL_LIBS by rabbitmq-env.
-DIST_ERL_LIBS = $(patsubst :%,%,$(patsubst %:,%,$(subst :$(APPS_DIR):,:,$(subst :$(DEPS_DIR):,:,:$(ERL_LIBS):))))
+FAST_RUN_BROKER ?= 1
+
+ifeq ($(FAST_RUN_BROKER),1)
+DIST_TARGET = $(if $(NOBUILD),,all)
+PLUGINS_FROM_DEPS_DIR = 1
+endif
 
 ifdef PLUGINS_FROM_DEPS_DIR
-RMQ_PLUGINS_DIR=$(DEPS_DIR)
+RMQ_PLUGINS_DIR = $(DEPS_DIR)
+DIST_ERL_LIBS = $(ERL_LIBS)
 else
-RMQ_PLUGINS_DIR=$(CURDIR)/$(DIST_DIR)
+RMQ_PLUGINS_DIR = $(CURDIR)/$(DIST_DIR)
+# We do not want to add apps/ or deps/ to ERL_LIBS
+# when running the release from dist. The `plugins`
+# directory is added to ERL_LIBS by rabbitmq-env.
+DIST_ERL_LIBS = $(patsubst :%,%,$(patsubst %:,%,$(subst :$(APPS_DIR):,:,$(subst :$(DEPS_DIR):,:,:$(ERL_LIBS):))))
 endif
 
 node_plugins_dir = $(if $(RABBITMQ_PLUGINS_DIR),$(RABBITMQ_PLUGINS_DIR),$(if $(EXTRA_PLUGINS_DIR),$(EXTRA_PLUGINS_DIR):$(RMQ_PLUGINS_DIR),$(RMQ_PLUGINS_DIR)))
@@ -410,6 +416,42 @@ stop-brokers stop-cluster:
 	done; \
 	wait
 
+# --------------------------------------------------------------------
+# Code reloading.
+#
+# For `make run-broker` either do:
+# * make RELOAD=1
+# * make all reload-broker        (can't do this alongside -j flag)
+# * make && make reload-broker    (fine with -j flag)
+#
+# Or if recompiling a specific application:
+# * make -C deps/rabbit RELOAD=1
+#
+# For `make start-cluster` use the `reload-cluster` target.
+# Same constraints apply as with `reload-broker`:
+# * make all reload-cluster
+# * make && make reload-cluster
+# --------------------------------------------------------------------
+
+reload-broker:
+	$(exec_verbose) ERL_LIBS="$(DIST_ERL_LIBS)" \
+		$(RABBITMQCTL) -n $(RABBITMQ_NODENAME) \
+		eval "io:format(\"~p~n\", [c:lm()])."
+
+ifeq ($(MAKELEVEL),0)
+ifdef RELOAD
+all:: reload-broker
+endif
+endif
+
+reload-cluster:
+	@for n in $$(seq $(NODES) -1 1); do \
+		nodename="rabbit-$$n@$(HOSTNAME)"; \
+		$(MAKE) reload-broker \
+			RABBITMQ_NODENAME="$$nodename" & \
+	done; \
+	wait
+
 # --------------------------------------------------------------------
 # Used by testsuites.
 # --------------------------------------------------------------------

deps/rabbitmq_auth_backend_oauth2/Makefile

@@ -6,7 +6,7 @@ BUILD_WITHOUT_QUIC=1
 export BUILD_WITHOUT_QUIC
 
 LOCAL_DEPS = inets public_key
-BUILD_DEPS = rabbit_common oauth2_client
+BUILD_DEPS = rabbit_common
 DEPS = rabbit cowlib jose base64url oauth2_client
 TEST_DEPS = cowboy rabbitmq_web_dispatch rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client rabbitmq_mqtt emqtt
 

deps/rabbitmq_cli/Makefile

@@ -78,7 +78,8 @@ app:: $(ESCRIPTS)
 	@:
 
 rabbitmqctl_srcs := mix.exs \
-		    $(shell find config lib -name "*.ex" -o -name "*.exs")
+	$(call core_find,config/,*.exs) \
+	$(call core_find,lib/,*.ex)
 
 # Elixir dependencies are fetched and compiled as part of the alias
 # `mix make_all`. We do not fetch and build them in `make deps` because

erlang.mk

@@ -17,7 +17,7 @@
 ERLANG_MK_FILENAME := $(realpath $(lastword $(MAKEFILE_LIST)))
 export ERLANG_MK_FILENAME
 
-ERLANG_MK_VERSION = 04c473a
+ERLANG_MK_VERSION = 2022.05.31-69-gb54e4b5-dirty
 ERLANG_MK_WITHOUT = 
 
 # Make 3.81 and 3.82 are deprecated.
@@ -184,8 +184,9 @@ core_http_get = curl -Lf$(if $(filter-out 0,$(V)),,s)o $(call core_native_path,$
 
 core_eq = $(and $(findstring $(1),$(2)),$(findstring $(2),$(1)))
 
-# We skip files that contain spaces or '#' because they end up causing issues.
-core_find = $(if $(wildcard $1),$(shell find $(1:%/=%) \( -type l -o -type f \) -name $(subst *,\*,$2) -not -name "*[ \#]*"))
+# We skip files that contain spaces because they end up causing issues.
+# Files that begin with a dot are already ignored by the wildcard function.
+core_find = $(foreach f,$(wildcard $(1:%/=%)/$2),$(if $(wildcard $f/.),$(call core_find,$f,$2),$(if $(wildcard $f),$f)))
 
 core_lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$(1)))))))))))))))))))))))))))
 
@@ -3564,8 +3565,10 @@ export DEPS_DIR
 REBAR_DEPS_DIR = $(DEPS_DIR)
 export REBAR_DEPS_DIR
 
+# When testing Erlang.mk and updating these, make sure
+# to delete test/test_rebar_git before running tests again.
 REBAR3_GIT ?= https://github.com/erlang/rebar3
-REBAR3_COMMIT ?= 3f563feaf1091a1980241adefa83a32dd2eebf7c # 3.20.0
+REBAR3_COMMIT ?= bde4b54248d16280b2c70a244aca3bb7566e2033 # 3.23.0
 
 CACHE_DEPS ?= 0
 
@@ -4018,7 +4021,7 @@ define dep_autopatch_rebar.erl
 			false -> ok;
 			{_, Files0} ->
 				Files = [begin
-					hd(filelib:wildcard("$(call core_native_path,$(DEPS_DIR)/$1/src/**/" ++ filename:rootname(F) ++ ".*rl")))
+					hd(filelib:wildcard("$(call core_native_path,$(DEPS_DIR)/$1/src/)**/" ++ filename:rootname(F) ++ ".*rl"))
 				end || "src/" ++ F <- Files0],
 				Names = [[" ", case lists:reverse(F) of
 					"lre." ++ Elif -> lists:reverse(Elif);
@@ -4147,8 +4150,8 @@ define dep_autopatch_rebar.erl
 					"\t$$\(CC) -o $$\@ $$\? $$\(LDFLAGS) $$\(ERL_LDFLAGS) $$\(DRV_LDFLAGS) $$\(LDLIBS) $$\(EXE_LDFLAGS)",
 					case {filename:extension(Output), $(PLATFORM)} of
 					    {[], _} -> "\n";
-					    {".so", darwin} -> "-shared\n";
-					    {".dylib", darwin} -> "-shared\n";
+					    {".so", darwin} -> " -shared\n";
+					    {".dylib", darwin} -> " -shared\n";
 					    {_, darwin} -> "\n";
 					    _ -> " -shared\n"
 					end])
@@ -4665,7 +4668,6 @@ define makedep.erl
 	end,
 	MakeDepend = fun
 		(F, Fd, Mod, StartLocation) ->
-			{ok, Filename} = file:pid2name(Fd),
 			case io:parse_erl_form(Fd, undefined, StartLocation) of
 				{ok, AbsData, EndLocation} ->
 					case AbsData of
@@ -5838,7 +5840,7 @@ else
 
 ci:: $(addprefix ci-,$(CI_OTP))
 
-ci-prepare: $(addprefix $(KERL_INSTALL_DIR)/,$(CI_OTP))
+ci-prepare: $(addprefix ci-prepare-,$(CI_OTP))
 
 ci-setup::
 	$(verbose) :
@@ -5850,7 +5852,10 @@ ci_verbose_0 = @echo " CI    " $(1);
 ci_verbose = $(ci_verbose_$(V))
 
 define ci_target
-ci-$1: $(KERL_INSTALL_DIR)/$2
+ci-prepare-$1: $(KERL_INSTALL_DIR)/$2
+	$(verbose) :
+
+ci-$1: ci-prepare-$1
 	$(verbose) $(MAKE) --no-print-directory clean
 	$(ci_verbose) \
 		PATH="$(KERL_INSTALL_DIR)/$2/bin:$(PATH)" \

rabbitmq-components.mk

@@ -14,7 +14,7 @@ endif
 PROJECT_VERSION := $(RABBITMQ_VERSION)
 
 ifeq ($(PROJECT_VERSION),)
-PROJECT_VERSION := $(shell \
+PROJECT_VERSION = $(shell \
 if test -f git-revisions.txt; then \
 	head -n1 git-revisions.txt | \
 	awk '{print $$$(words $(PROJECT_DESCRIPTION) version);}'; \