Merge #19761: build: improve sed robustness by not using sed

3de365e build: replace wingenminiupnpcstrings sed with a patch in miniupnpc package (fanquake)
bbc01a7 build: replace qtranslations lrelease sed with a patch in qt package (fanquake)
c723e41 build: replace FreeType back-compat sed with a patch in qt package (fanquake)
3aaa39d build: replace pwd sed in qt package with a patch (fanquake)
9d440f4 build: remove no-longer needed qt workaround (fanquake)
bf85eac build: remove no-longer needed qt configure workaround (fanquake)
4af59a4 build: use patch rather than sed in zeromq package (fanquake)
cc107a3 build: use patch rather than sed in native_cctools package (fanquake)
865cb23 build: use patch rather than sed in fontconfig package (fanquake)
335bd7f build: use patch rather than sed in Boost package (fanquake)
f36140d build: use patch rather than sed in bdb package (fanquake)

Pull request description:

  While using `sed` can be handy to use for a quick-fix, these instances accumulate, and can become unmaintainable. Not only that, but using sed isn't necessarily robust and it can fail silently. Most of our usage is also missing any documentation explaining why something is being done, when it should be updated/removed etc.

  Rather than relying on sed going forward, where possible, I've converted our sed usage into patches. These are easier to maintain, contain documentation, and should fail loudly when they don't apply.

  The remaining sed usage, (1 in miniupnpc, the rest in qt), are non-trivial to remove, as they are using build-time variables, or some input from the environment.

  This also steals 2 related commits out of #19716.

  Related to #16838.

ACKs for top commit:
  hebasto:
    re-ACK 3de365e, only `drop_lrelease_dependency.patch` updated. Travis makes ARM build without errors now.
  theuni:
    ACK 3de365e.

Tree-SHA512: b39afcb237e4421f9caabbd665af93fd2e749a1cdd42b1d5ee2261059120005c0e82994d315e679c317d23794eab5c7727f51cae403c94a9c4e4fd7eee9e7ee6

Author: fanquake

depends/packages/bdb.mk

@@ -4,6 +4,7 @@ $(package)_download_path=https://download.oracle.com/berkeley-db
 $(package)_file_name=db-$($(package)_version).NC.tar.gz
 $(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef
 $(package)_build_subdir=build_unix
+$(package)_patches=clang_cxx_11.patch
 
 define $(package)_set_vars
 $(package)_config_opts=--disable-shared --enable-cxx --disable-replication --enable-option-checking
@@ -14,8 +15,7 @@ $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE
 endef
 
 define $(package)_preprocess_cmds
-  sed -i.old 's/__atomic_compare_exchange/__atomic_compare_exchange_db/' dbinc/atomic.h && \
-  sed -i.old 's/atomic_init/atomic_init_db/' dbinc/atomic.h mp/mp_region.c mp/mp_mvcc.c mp/mp_fget.c mutex/mut_method.c mutex/mut_tas.c && \
+  patch -p1 < $($(package)_patch_dir)/clang_cxx_11.patch && \
   cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub dist
 endef
 

depends/packages/boost.mk

@@ -3,6 +3,7 @@ $(package)_version=1_70_0
 $(package)_download_path=https://dl.bintray.com/boostorg/release/1.70.0/source/
 $(package)_file_name=boost_$($(package)_version).tar.bz2
 $(package)_sha256_hash=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778
+$(package)_patches=unused_var_in_process.patch
 
 define $(package)_set_vars
 $(package)_config_opts_release=variant=release
@@ -31,9 +32,8 @@ $(package)_cxxflags_linux=-fPIC
 $(package)_cxxflags_android=-fPIC
 endef
 
-# Fix unused variable in boost_process, can be removed after upgrading to 1.72
 define $(package)_preprocess_cmds
-  sed -i.old "s/int ret_sig = 0;//" boost/process/detail/posix/wait_group.hpp && \
+  patch -p1 < $($(package)_patch_dir)/unused_var_in_process.patch && \
   echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$($(package)_archiver_$(host_os))\" <striper>\"$(host_STRIP)\"  <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam
 endef
 

depends/packages/fontconfig.mk

@@ -4,23 +4,23 @@ $(package)_download_path=https://www.freedesktop.org/software/fontconfig/release
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=b449a3e10c47e1d1c7a6ec6e2016cca73d3bd68fbbd4f0ae5cc6b573f7d6c7f3
 $(package)_dependencies=freetype expat
+$(package)_patches=remove_char_width_usage.patch gperf_header_regen.patch
 
 define $(package)_set_vars
   $(package)_config_opts=--disable-docs --disable-static --disable-libxml2 --disable-iconv
   $(package)_config_opts += --disable-dependency-tracking --enable-option-checking
 endef
 
+define $(package)_preprocess_cmds
+  patch -p1 < $($(package)_patch_dir)/remove_char_width_usage.patch && \
+  patch -p1 < $($(package)_patch_dir)/gperf_header_regen.patch
+endef
+
 define $(package)_config_cmds
   $($(package)_autoconf)
 endef
 
-# 2.12.1 uses CHAR_WIDTH which is reserved and clashes with some glibc versions, but newer versions of fontconfig
-# have broken makefiles which needlessly attempt to re-generate headers with gperf.
-# Instead, change all uses of CHAR_WIDTH, and disable the rule that forces header re-generation.
-# This can be removed once the upstream build is fixed.
 define $(package)_build_cmds
-  sed -i 's/CHAR_WIDTH/CHARWIDTH/g' fontconfig/fontconfig.h src/fcobjshash.gperf src/fcobjs.h src/fcobjshash.h && \
-  sed -i 's/fcobjshash.h: fcobjshash.gperf/fcobjshash.h:/' src/Makefile && \
   $(MAKE)
 endef
 

depends/packages/miniupnpc.mk

@@ -3,6 +3,7 @@ $(package)_version=2.0.20180203
 $(package)_download_path=https://miniupnp.tuxfamily.org/files/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=90dda8c7563ca6cd4a83e23b3c66dbbea89603a1675bfdb852897c2c9cc220b7
+$(package)_patches=dont_use_wingen.patch
 
 define $(package)_set_vars
 $(package)_build_opts=CC="$($(package)_cc)"
@@ -14,7 +15,7 @@ endef
 define $(package)_preprocess_cmds
   mkdir dll && \
   sed -e 's|MINIUPNPC_VERSION_STRING \"version\"|MINIUPNPC_VERSION_STRING \"$($(package)_version)\"|' -e 's|OS/version|$(host)|' miniupnpcstrings.h.in > miniupnpcstrings.h && \
-  sed -i.old "s|miniupnpcstrings.h: miniupnpcstrings.h.in wingenminiupnpcstrings|miniupnpcstrings.h: miniupnpcstrings.h.in|" Makefile.mingw
+  patch -p1 < $($(package)_patch_dir)/dont_use_wingen.patch
 endef
 
 define $(package)_build_cmds

depends/packages/native_cctools.mk

@@ -4,6 +4,8 @@ $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive
 $(package)_file_name=$($(package)_version).tar.gz
 $(package)_sha256_hash=e51995a843533a3dac155dd0c71362dd471597a2d23f13dff194c6285362f875
 $(package)_build_subdir=cctools
+$(package)_patches=ld64_disable_threading.patch
+
 ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 $(package)_clang_version=8.0.0
 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version)
@@ -78,7 +80,7 @@ endef
 define $(package)_preprocess_cmds
   CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/build.sh && \
   CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/install.sh && \
-  sed -i.old "/define HAVE_PTHREADS/d" $($(package)_build_subdir)/ld64/src/ld/InputFiles.h
+  patch -p1 < $($(package)_patch_dir)/ld64_disable_threading.patch
 endef
 
 define $(package)_config_cmds

depends/packages/qt.mk

@@ -8,7 +8,10 @@ $(package)_dependencies=zlib
 $(package)_linux_dependencies=freetype fontconfig libxcb
 $(package)_build_subdir=qtbase
 $(package)_qt_libs=corelib network widgets gui plugins testlib
-$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch fix_android_qmake_conf.patch fix_android_jni_static.patch
+$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch
+$(package)_patches+= fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch
+$(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch
+$(package)_patches+= freetype_back_compat.patch drop_lrelease_dependency.patch
 
 # Update OSX_QT_TRANSLATIONS when this is updated
 $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix)
@@ -190,11 +193,10 @@ define $(package)_extract_cmds
 endef
 
 define $(package)_preprocess_cmds
-  sed -i.old "s|FT_Get_Font_Format|FT_Get_X11_Font_Format|" qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp && \
+  patch -p1 -i $($(package)_patch_dir)/freetype_back_compat.patch && \
   sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \
-  sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \
-  sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \
-  sed -i.old -e 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' -e 's|/bin/pwd|pwd|' qtbase/configure && \
+  patch -p1 -i $($(package)_patch_dir)/drop_lrelease_dependency.patch && \
+  patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch &&\
   mkdir -p qtbase/mkspecs/macx-clang-linux &&\
   cp -f qtbase/mkspecs/macx-clang/Info.plist.lib qtbase/mkspecs/macx-clang-linux/ &&\
   cp -f qtbase/mkspecs/macx-clang/Info.plist.app qtbase/mkspecs/macx-clang-linux/ &&\

depends/packages/zeromq.mk

@@ -3,6 +3,7 @@ $(package)_version=4.3.1
 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb
+$(package)_patches=remove_libstd_link.patch
 
 define $(package)_set_vars
   $(package)_config_opts=--without-docs --disable-shared --disable-curve --disable-curve-keygen --disable-perf
@@ -15,7 +16,8 @@ define $(package)_set_vars
 endef
 
 define $(package)_preprocess_cmds
-   cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config
+  patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \
+  cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config
 endef
 
 define $(package)_config_cmds
@@ -31,6 +33,5 @@ define $(package)_stage_cmds
 endef
 
 define $(package)_postprocess_cmds
-  sed -i.old "s/ -lstdc++//" lib/pkgconfig/libzmq.pc && \
   rm -rf bin share lib/*.la
 endef

depends/patches/bdb/clang_cxx_11.patch

@@ -0,0 +1,147 @@
+commit 3311d68f11d1697565401eee6efc85c34f022ea7
+Author: fanquake <[email protected]>
+Date:   Mon Aug 17 20:03:56 2020 +0800
+
+    Fix C++11 compatibility
+
+diff --git a/dbinc/atomic.h b/dbinc/atomic.h
+index 0034dcc..7c11d4a 100644
+--- a/dbinc/atomic.h
++++ b/dbinc/atomic.h
+@@ -70,7 +70,7 @@ typedef struct {
+  * These have no memory barriers; the caller must include them when necessary.
+  */
+ #define	atomic_read(p)		((p)->value)
+-#define	atomic_init(p, val)	((p)->value = (val))
++#define	atomic_init_db(p, val)	((p)->value = (val))
+ 
+ #ifdef HAVE_ATOMIC_SUPPORT
+ 
+@@ -144,7 +144,7 @@ typedef LONG volatile *interlocked_val;
+ #define	atomic_inc(env, p)	__atomic_inc(p)
+ #define	atomic_dec(env, p)	__atomic_dec(p)
+ #define	atomic_compare_exchange(env, p, o, n)	\
+-	__atomic_compare_exchange((p), (o), (n))
++	__atomic_compare_exchange_db((p), (o), (n))
+ static inline int __atomic_inc(db_atomic_t *p)
+ {
+ 	int	temp;
+@@ -176,7 +176,7 @@ static inline int __atomic_dec(db_atomic_t *p)
+  * http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html
+  * which configure could be changed to use.
+  */
+-static inline int __atomic_compare_exchange(
++static inline int __atomic_compare_exchange_db(
+ 	db_atomic_t *p, atomic_value_t oldval, atomic_value_t newval)
+ {
+ 	atomic_value_t was;
+@@ -206,7 +206,7 @@ static inline int __atomic_compare_exchange(
+ #define	atomic_dec(env, p)	(--(p)->value)
+ #define	atomic_compare_exchange(env, p, oldval, newval)		\
+ 	(DB_ASSERT(env, atomic_read(p) == (oldval)),		\
+-	atomic_init(p, (newval)), 1)
++	atomic_init_db(p, (newval)), 1)
+ #else
+ #define atomic_inc(env, p)	__atomic_inc(env, p)
+ #define atomic_dec(env, p)	__atomic_dec(env, p)
+diff --git a/mp/mp_fget.c b/mp/mp_fget.c
+index 5fdee5a..0b75f57 100644
+--- a/mp/mp_fget.c
++++ b/mp/mp_fget.c
+@@ -617,7 +617,7 @@ alloc:		/* Allocate a new buffer header and data space. */
+ 
+ 		/* Initialize enough so we can call __memp_bhfree. */
+ 		alloc_bhp->flags = 0;
+-		atomic_init(&alloc_bhp->ref, 1);
++		atomic_init_db(&alloc_bhp->ref, 1);
+ #ifdef DIAGNOSTIC
+ 		if ((uintptr_t)alloc_bhp->buf & (sizeof(size_t) - 1)) {
+ 			__db_errx(env,
+@@ -911,7 +911,7 @@ alloc:		/* Allocate a new buffer header and data space. */
+ 			MVCC_MPROTECT(bhp->buf, mfp->stat.st_pagesize,
+ 			    PROT_READ);
+ 
+-		atomic_init(&alloc_bhp->ref, 1);
++		atomic_init_db(&alloc_bhp->ref, 1);
+ 		MUTEX_LOCK(env, alloc_bhp->mtx_buf);
+ 		alloc_bhp->priority = bhp->priority;
+ 		alloc_bhp->pgno = bhp->pgno;
+diff --git a/mp/mp_mvcc.c b/mp/mp_mvcc.c
+index 34467d2..f05aa0c 100644
+--- a/mp/mp_mvcc.c
++++ b/mp/mp_mvcc.c
+@@ -276,7 +276,7 @@ __memp_bh_freeze(dbmp, infop, hp, bhp, need_frozenp)
+ #else
+ 	memcpy(frozen_bhp, bhp, SSZA(BH, buf));
+ #endif
+-	atomic_init(&frozen_bhp->ref, 0);
++	atomic_init_db(&frozen_bhp->ref, 0);
+ 	if (mutex != MUTEX_INVALID)
+ 		frozen_bhp->mtx_buf = mutex;
+ 	else if ((ret = __mutex_alloc(env, MTX_MPOOL_BH,
+@@ -428,7 +428,7 @@ __memp_bh_thaw(dbmp, infop, hp, frozen_bhp, alloc_bhp)
+ #endif
+ 		alloc_bhp->mtx_buf = mutex;
+ 		MUTEX_LOCK(env, alloc_bhp->mtx_buf);
+-		atomic_init(&alloc_bhp->ref, 1);
++		atomic_init_db(&alloc_bhp->ref, 1);
+ 		F_CLR(alloc_bhp, BH_FROZEN);
+ 	}
+ 
+diff --git a/mp/mp_region.c b/mp/mp_region.c
+index e6cece9..ddbe906 100644
+--- a/mp/mp_region.c
++++ b/mp/mp_region.c
+@@ -224,7 +224,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg)
+ 			     MTX_MPOOL_FILE_BUCKET, 0, &htab[i].mtx_hash)) != 0)
+ 				return (ret);
+ 			SH_TAILQ_INIT(&htab[i].hash_bucket);
+-			atomic_init(&htab[i].hash_page_dirty, 0);
++			atomic_init_db(&htab[i].hash_page_dirty, 0);
+ 		}
+ 
+ 		/*
+@@ -269,7 +269,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg)
+ 		hp->mtx_hash = (mtx_base == MUTEX_INVALID) ? MUTEX_INVALID :
+ 		    mtx_base + i;
+ 		SH_TAILQ_INIT(&hp->hash_bucket);
+-		atomic_init(&hp->hash_page_dirty, 0);
++		atomic_init_db(&hp->hash_page_dirty, 0);
+ #ifdef HAVE_STATISTICS
+ 		hp->hash_io_wait = 0;
+ 		hp->hash_frozen = hp->hash_thawed = hp->hash_frozen_freed = 0;
+diff --git a/mutex/mut_method.c b/mutex/mut_method.c
+index 2588763..5c6d516 100644
+--- a/mutex/mut_method.c
++++ b/mutex/mut_method.c
+@@ -426,7 +426,7 @@ atomic_compare_exchange(env, v, oldval, newval)
+ 	MUTEX_LOCK(env, mtx);
+ 	ret = atomic_read(v) == oldval;
+ 	if (ret)
+-		atomic_init(v, newval);
++		atomic_init_db(v, newval);
+ 	MUTEX_UNLOCK(env, mtx);
+ 
+ 	return (ret);
+diff --git a/mutex/mut_tas.c b/mutex/mut_tas.c
+index f3922e0..e40fcdf 100644
+--- a/mutex/mut_tas.c
++++ b/mutex/mut_tas.c
+@@ -46,7 +46,7 @@ __db_tas_mutex_init(env, mutex, flags)
+ 
+ #ifdef HAVE_SHARED_LATCHES
+ 	if (F_ISSET(mutexp, DB_MUTEX_SHARED))
+-		atomic_init(&mutexp->sharecount, 0);
++		atomic_init_db(&mutexp->sharecount, 0);
+ 	else
+ #endif
+ 	if (MUTEX_INIT(&mutexp->tas)) {
+@@ -486,7 +486,7 @@ __db_tas_mutex_unlock(env, mutex)
+ 			F_CLR(mutexp, DB_MUTEX_LOCKED);
+ 			/* Flush flag update before zeroing count */
+ 			MEMBAR_EXIT();
+-			atomic_init(&mutexp->sharecount, 0);
++			atomic_init_db(&mutexp->sharecount, 0);
+ 		} else {
+ 			DB_ASSERT(env, sharecount > 0);
+ 			MEMBAR_EXIT();

depends/patches/boost/unused_var_in_process.patch

@@ -0,0 +1,22 @@
+commit dbd95cdaefdea95307d004f019a1c394cf9389f0
+Author: fanquake <[email protected]>
+Date:   Mon Aug 17 20:15:17 2020 +0800
+
+    Remove unused variable in Boost Process
+
+    This causes issues with our linters / CI.
+
+    Can be removed once depends Boost is 1.71.0 or later.
+
+diff --git a/boost/process/detail/posix/wait_group.hpp b/boost/process/detail/posix/wait_group.hpp
+index 9dc249803..2502d9772 100644
+--- a/boost/process/detail/posix/wait_group.hpp
++++ b/boost/process/detail/posix/wait_group.hpp
+@@ -137,7 +137,6 @@ inline bool wait_until(
+
+     do
+     {
+-        int ret_sig = 0;
+         int status;
+         if ((::waitpid(timeout_pid, &status, WNOHANG) != 0)
+             && (WIFEXITED(status) || WIFSIGNALED(status)))

depends/patches/fontconfig/gperf_header_regen.patch

@@ -0,0 +1,24 @@
+commit 7b6eb33ecd88768b28c67ce5d2d68a7eed5936b6
+Author: fanquake <[email protected]>
+Date:   Tue Aug 25 14:34:53 2020 +0800
+
+    Remove rule that causes inadvertant header regeneration
+
+    Otherwise the makefile will needlessly attempt to re-generate the
+    headers with gperf. This can be dropped once the upstream build is fixed.
+
+    See #10851.
+
+diff --git a/src/Makefile.in b/src/Makefile.in
+index f4626ad..4ae1b00 100644
+--- a/src/Makefile.in
++++ b/src/Makefile.in
+@@ -903,7 +903,7 @@ fcobjshash.gperf: fcobjshash.gperf.h fcobjs.h
+ 	' - > [email protected] && \
+ 	mv -f [email protected] $@ || ( $(RM) [email protected] && false )
+
+-fcobjshash.h: fcobjshash.gperf
++fcobjshash.h:
+ 	$(AM_V_GEN) $(GPERF) -m 100 $< > [email protected] && \
+ 	mv -f [email protected] $@ || ( $(RM) [email protected] && false )
+