Merge #19683: depends: Pin clang search paths for darwin host

196b727649 depends: Add comment about cache invalidation (Carl Dong)
949c480e52 depends: Fully determine path for darwin cctools (Carl Dong)
880660acfa depends: Fully determine path for darwin_{CC,CXX} (Carl Dong)
8033110741 depends: Quote to prevent word splitting in config.site (Carl Dong)
77b1ef89a0 depends: Remove -fuse-ld line (Carl Dong)
3007339218 depends: Pin clang search paths for darwin host (Carl Dong)
107f33d434 depends: Delay expansion of per-package vars (Carl Dong)

Pull request description:

  > Hello clang/lib/frontend,
  > I search your headers once again.
  > Because it's time for some housekeeping,
  > Within the code I was tweaking,
  > And the targets I was making with my build,
  > Are unfulfilled,
  > It's just language compliance.
  >
  > In reference works I scroll alone
  > Pages cribbed from holy tomes
  > In the details of a template
  > My code's behaviour has now found its fate
  > When my hopes were dashed as a note left it as described:
  > As undefined
  > It's not in compliance
  >
  > And from the standard text I saw
  > Ten thousand errors, maybe more
  > Threading used without locking
  > Pointers referenced after freeing
  > Linters writing warnings that coders will never fix
  > But still they tick
  > The box that claims compliance
  >
  > "Fools," said I, "you do not know"
  > Errors, like a cancer, grow
  > Hear my words that I might reach you
  > Use -Wall and it might teach you
  > But my words and compiler errors fade.
  > Schedules forbade compliance.
  >
  > And the people bowed and prayed
  > With static checking torn and frayed
  > The markets flashed out their warning
  > In the words that they were forming
  > As recruiters said "The search for more profits leads to writing stuff in CSS,
  > And node.js.
  > Without a need for compliance"

  Many thanks to ajtowns for the above contribution!

  -----

  This PR is ready for review!

  When cross-compiling for macOS, the SDK gives us the entire context/sysroot on which we should base the build. This means that we can be extremely specific w/re our search path ordering in order to avoid build problems that arise out of a user's specific environment/system setup and improve the robustness of our macOS toolchain. This PR does 2 things to this end:

  1. Unset environment variables which are known to alter search paths.
  1. Makes us (in the case of macOS builds) explicitly specify the list of system include search paths and its ordering, rather than rely on `clang`'s unreliable autodetection routine. Here is the [rabbit-hole gist](https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9).

      See the added comments in `depends/hosts/darwin.mk` for more details:

      8b8296dc70/depends/hosts/darwin.mk (L37-L60)

  We can be this specific _only_ because macOS builds are neatly contained in an SDK, **and** we are cross-compiling. Native toolchains should rely on the environment/distro/user to know how best to build for the running system.

  Note: Although the `-u` flag of `env` is not a POSIX standard flag, it seems like it is useful enough to be implemented in [coreutils](https://www.gnu.org/software/coreutils/manual/html_node/env-invocation.html), [busybox](https://busybox.net/downloads/BusyBox.html#env), [FreeBSD](https://www.freebsd.org/cgi/man.cgi?env).

ACKs for top commit:
  laanwj:
    code review ACK 196b727649

Tree-SHA512: 406442df16d9aa0aef62f9fa94f72d7e48374301f3d826bf32f183e1610942aa44a4adfac7bead1f14aded0044fac400e1328fcd933b2337e55a024f034b5013

depends/Makefile

@@ -127,6 +127,8 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)
 
 ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+# Make sure that cache is invalidated when switching between system and
+# depends-managed, pinned clang
 build_id_string+=system_clang
 $(host_arch)_$(host_os)_id_string+=system_clang
 endif
@@ -166,12 +168,6 @@ $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
 
 include funcs.mk
 
-binutils_path=$($($(host_arch)_$(host_os)_native_binutils)_prefixbin)
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
-else
-toolchain_path=
-endif
 final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)
 final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
 $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
@@ -182,15 +178,39 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
 	$(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); )
 	$(AT)touch $@
 
+# $PATH is not preserved between ./configure and make by convention. Its
+# modification and overriding at ./configure time is (as I understand it)
+# supposed to be captured by the AC_{PROG_{,OBJ}CXX,PATH_{PROG,TOOL}} macros,
+# which will expand the program names to their full absolute paths. The notable
+# exception is command line overriding: ./configure CC=clang, which skips the
+# program name expansion step, and works because the user implicitly indicates
+# with CC=clang that clang will be available in $PATH at all times, and is most
+# likely part of the user's system.
+#
+# Therefore, when we "seed the autoconf cache"/"override well-known program
+# vars" by setting AR=<blah> in our config.site, either one of two things needs
+# to be true for the build system to work correctly:
+#
+#   1. If we refer to the program by name (e.g. AR=riscv64-gnu-linux-ar), the
+#      tool needs to be available in $PATH at all times.
+#
+#   2. If the tool is _**not**_ expected to be available in $PATH at all times
+#      (such as is the case for our native_cctools binutils tools), it needs to
+#      be referred to by its absolute path, such as would be output by the
+#      AC_PATH_{PROG,TOOL} macros.
+#
+# Minor note: it is also okay to refer to tools by their absolute path even if
+# we expect them to be available in $PATH at all times, more specificity does
+# not hurt.
 $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id)
 	$(AT)@mkdir -p $(@D)
 	$(AT)sed -e 's|@HOST@|$(host)|' \
-            -e 's|@CC@|$(toolchain_path)$(host_CC)|' \
+            -e 's|@CC@|$(host_CC)|' \
-            -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \
+            -e 's|@CXX@|$(host_CXX)|' \
-            -e 's|@AR@|$(binutils_path)$(host_AR)|' \
+            -e 's|@AR@|$(host_AR)|' \
-            -e 's|@RANLIB@|$(binutils_path)$(host_RANLIB)|' \
+            -e 's|@RANLIB@|$(host_RANLIB)|' \
-            -e 's|@NM@|$(binutils_path)$(host_NM)|' \
+            -e 's|@NM@|$(host_NM)|' \
-            -e 's|@STRIP@|$(binutils_path)$(host_STRIP)|' \
+            -e 's|@STRIP@|$(host_STRIP)|' \
             -e 's|@build_os@|$(build_os)|' \
             -e 's|@host_os@|$(host_os)|' \
             -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \

depends/config.site.in

@@ -8,74 +8,74 @@ true  # Dummy command because shellcheck treats all directives before first
       # See: https://github.com/koalaman/shellcheck/wiki/Directive
 
 # shellcheck disable=SC2154
-depends_prefix="$(cd "$(dirname ${ac_site_file})/.." && pwd)"
+depends_prefix="$(cd "$(dirname "$ac_site_file")/.." && pwd)"
 
 cross_compiling=maybe
-host_alias=@HOST@
+host_alias="@HOST@"
-ac_tool_prefix=${host_alias}-
+ac_tool_prefix="${host_alias}-"
 
-if test -z $with_boost; then
+if test -z "$with_boost"; then
-  with_boost=$depends_prefix
+  with_boost="$depends_prefix"
 fi
-if test -z $with_qt_plugindir; then
+if test -z "$with_qt_plugindir"; then
-  with_qt_plugindir=$depends_prefix/plugins
+  with_qt_plugindir="${depends_prefix}/plugins"
 fi
-if test -z $with_qt_translationdir; then
+if test -z "$with_qt_translationdir"; then
-  with_qt_translationdir=$depends_prefix/translations
+  with_qt_translationdir="${depends_prefix}/translations"
 fi
-if test -z $with_qt_bindir && test -z "@no_qt@"; then
+if test -z "$with_qt_bindir" && test -z "@no_qt@"; then
-  with_qt_bindir=$depends_prefix/native/bin
+  with_qt_bindir="${depends_prefix}/native/bin"
 fi
-if test -z $with_mpgen && test -n "@multiprocess@"; then
+if test -z "$with_mpgen" && test -n "@multiprocess@"; then
-  with_mpgen=$depends_prefix/native
+  with_mpgen="${depends_prefix}/native"
 fi
 
-if test -z $with_qrencode && test -n "@no_qr@"; then
+if test -z "$with_qrencode" && test -n "@no_qr@"; then
   with_qrencode=no
 fi
 
-if test -z $enable_wallet && test -n "@no_wallet@"; then
+if test -z "$enable_wallet" && test -n "@no_wallet@"; then
   enable_wallet=no
 fi
 
-if test -z $enable_multiprocess && test -n "@multiprocess@"; then
+if test -z "$enable_multiprocess" && test -n "@multiprocess@"; then
   enable_multiprocess=yes
 fi
 
-if test -z $with_miniupnpc && test -n "@no_upnp@"; then
+if test -z "$with_miniupnpc" && test -n "@no_upnp@"; then
   with_miniupnpc=no
 fi
 
-if test -z $with_natpmp && test -n "@no_natpmp@"; then
+if test -z "$with_natpmp" && test -n "@no_natpmp@"; then
   with_natpmp=no
 fi
 
-if test -z $with_gui && test -n "@no_qt@"; then
+if test -z "$with_gui" && test -n "@no_qt@"; then
   with_gui=no
 fi
 
-if test -z $enable_zmq && test -n "@no_zmq@"; then
+if test -z "$enable_zmq" && test -n "@no_zmq@"; then
   enable_zmq=no
 fi
 
-if test x@host_os@ = xdarwin; then
+if test "x@host_os@" = xdarwin; then
   BREW=no
   PORT=no
 fi
 
-PATH=$depends_prefix/native/bin:$PATH
+PATH="${depends_prefix}/native/bin:${PATH}"
 PKG_CONFIG="$(which pkg-config) --static"
 
 # These two need to remain exported because pkg-config does not see them
 # otherwise. That means they must be unexported at the end of configure.ac to
 # avoid ruining the cache. Sigh.
-export PKG_CONFIG_PATH=$depends_prefix/share/pkgconfig:$depends_prefix/lib/pkgconfig
+export PKG_CONFIG_PATH="${depends_prefix}/share/pkgconfig:${depends_prefix}/lib/pkgconfig"
 if test -z "@allow_host_packages@"; then
-  export PKG_CONFIG_LIBDIR=$depends_prefix/lib/pkgconfig
+  export PKG_CONFIG_LIBDIR="${depends_prefix}/lib/pkgconfig"
 fi
 
-CPPFLAGS="-I$depends_prefix/include/ $CPPFLAGS"
+CPPFLAGS="-I${depends_prefix}/include/ ${CPPFLAGS}"
-LDFLAGS="-L$depends_prefix/lib $LDFLAGS"
+LDFLAGS="-L${depends_prefix}/lib ${LDFLAGS}"
 
 if test -n "@CC@" -a -z "${CC}"; then
   CC="@CC@"
@@ -86,18 +86,18 @@ fi
 PYTHONPATH="${depends_prefix}/native/lib/python3/dist-packages${PYTHONPATH:+${PATH_SEPARATOR}}${PYTHONPATH}"
 
 if test -n "@AR@"; then
-  AR=@AR@
+  AR="@AR@"
-  ac_cv_path_ac_pt_AR=${AR}
+  ac_cv_path_ac_pt_AR="${AR}"
 fi
 
 if test -n "@RANLIB@"; then
-  RANLIB=@RANLIB@
+  RANLIB="@RANLIB@"
-  ac_cv_path_ac_pt_RANLIB=${RANLIB}
+  ac_cv_path_ac_pt_RANLIB="${RANLIB}"
 fi
 
 if test -n "@NM@"; then
-  NM=@NM@
+  NM="@NM@"
-  ac_cv_path_ac_pt_NM=${NM}
+  ac_cv_path_ac_pt_NM="${NM}"
 fi
 
 if test -n "@debug@"; then
@@ -105,14 +105,14 @@ if test -n "@debug@"; then
 fi
 
 if test -n "@CFLAGS@"; then
-  CFLAGS="@CFLAGS@ $CFLAGS"
+  CFLAGS="@CFLAGS@ ${CFLAGS}"
 fi
 if test -n "@CXXFLAGS@"; then
-  CXXFLAGS="@CXXFLAGS@ $CXXFLAGS"
+  CXXFLAGS="@CXXFLAGS@ ${CXXFLAGS}"
 fi
 if test -n "@CPPFLAGS@"; then
-  CPPFLAGS="@CPPFLAGS@ $CPPFLAGS"
+  CPPFLAGS="@CPPFLAGS@ ${CPPFLAGS}"
 fi
 if test -n "@LDFLAGS@"; then
-  LDFLAGS="@LDFLAGS@ $LDFLAGS"
+  LDFLAGS="@LDFLAGS@ ${LDFLAGS}"
 fi

depends/funcs.mk

@@ -1,17 +1,23 @@
 define int_vars
 #Set defaults for vars which may be overridden per-package
-$(1)_cc=$($($(1)_type)_CC)
+$(1)_cc=$$($$($(1)_type)_CC)
-$(1)_cxx=$($($(1)_type)_CXX)
+$(1)_cxx=$$($$($(1)_type)_CXX)
-$(1)_objc=$($($(1)_type)_OBJC)
+$(1)_objc=$$($$($(1)_type)_OBJC)
-$(1)_objcxx=$($($(1)_type)_OBJCXX)
+$(1)_objcxx=$$($$($(1)_type)_OBJCXX)
-$(1)_ar=$($($(1)_type)_AR)
+$(1)_ar=$$($$($(1)_type)_AR)
-$(1)_ranlib=$($($(1)_type)_RANLIB)
+$(1)_ranlib=$$($$($(1)_type)_RANLIB)
-$(1)_libtool=$($($(1)_type)_LIBTOOL)
+$(1)_libtool=$$($$($(1)_type)_LIBTOOL)
-$(1)_nm=$($($(1)_type)_NM)
+$(1)_nm=$$($$($(1)_type)_NM)
-$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS)
+$(1)_cflags=$$($$($(1)_type)_CFLAGS) \
-$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS)
+            $$($$($(1)_type)_$$(release_type)_CFLAGS)
-$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib
+$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \
-$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include
+              $$($$($(1)_type)_$$(release_type)_CXXFLAGS)
+$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \
+             $$($$($(1)_type)_$$(release_type)_LDFLAGS) \
+             -L$$($($(1)_type)_prefix)/lib
+$(1)_cppflags=$$($$($(1)_type)_CPPFLAGS) \
+              $$($$($(1)_type)_$$(release_type)_CPPFLAGS) \
+              -I$$($$($(1)_type)_prefix)/include
 $(1)_recipe_hash:=
 endef
 

depends/hosts/darwin.mk

@@ -6,6 +6,49 @@ LD64_VERSION=530
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
 
+darwin_native_binutils=native_cctools
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang
+# from llvm.org
+
+# The native_cctools package is what provides clang when FORCE_USE_SYSTEM_CLANG
+# is empty
+darwin_native_toolchain=native_cctools
+
+clang_prog=$(build_prefix)/bin/clang
+clangxx_prog=$(clang_prog)++
+
+clang_resource_dir=$(build_prefix)/lib/clang/$(native_cctools_clang_version)
+else
+# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's
+# system
+
+darwin_native_toolchain=
+
+# We can't just use $(shell command -v clang) because GNU Make handles builtins
+# in a special way and doesn't know that `command` is a POSIX-standard builtin
+# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90.
+# At the time of writing, GNU Make v4.2.1 is still being used in supported
+# distro releases.
+#
+# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html
+clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang")
+clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++")
+
+clang_resource_dir=$(shell clang -print-resource-dir)
+endif
+
+cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL
+
+# Make-only lowercase function
+lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
+
+# For well-known tools provided by cctools, make sure that their well-known
+# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG}
+# would.
+$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL))))
+
 # Flag explanations:
 #
 #     -mlinker-version
@@ -18,7 +61,7 @@ OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-
 #         Explicitly point to our binaries (e.g. cctools) so that they are
 #         ensured to be found and preferred over other possibilities.
 #
-#     -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
+#     -stdlib=libc++ -nostdinc++ -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1
 #
 #         Forces clang to use the libc++ headers from our SDK and completely
 #         forget about the libc++ headers from the standard directories
@@ -28,8 +71,49 @@ OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-
 #         https://reviews.llvm.org/D64089, we should use that instead. Read the
 #         differential summary there for more details.
 #
-darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin
+#     -Xclang -*system<path_a> \
-darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
+#     -Xclang -*system<path_b> \
+#     -Xclang -*system<path_c> ...
+#
+#         Adds path_a, path_b, and path_c to the bottom of clang's list of
+#         include search paths. This is used to explicitly specify the list of
+#         system include search paths and its ordering, rather than rely on
+#         clang's autodetection routine. This routine has been shown to:
+#             1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1
+#                when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034)
+#             2. Fail to pickup C headers in $SYSROOT/usr/include when
+#                C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9)
+#
+#         Talking directly to cc1 with -Xclang here grants us access to specify
+#         more granular categories for these system include search paths, and we
+#         can use the correct categories that these search paths would have been
+#         placed in if the autodetection routine had worked correctly. (see:
+#         https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment)
+#
+#         Furthermore, it places these search paths after any "non-Xclang"
+#         specified search paths. This prevents any additional clang options or
+#         environment variables from coming after or in between these system
+#         include search paths, as that would be wrong in general but would also
+#         break #include_next's.
+#
+darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
+              -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
+              -u LIBRARY_PATH \
+            $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
+              -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
+              --sysroot=$(OSX_SDK) \
+              -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
+              -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
+darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
+               -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
+               -u LIBRARY_PATH \
+             $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
+               -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
+               --sysroot=$(OSX_SDK) \
+               -stdlib=libc++ -nostdinc++ \
+               -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \
+               -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
+               -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
 
 darwin_CFLAGS=-pipe
 darwin_CXXFLAGS=$(darwin_CFLAGS)
@@ -40,11 +124,4 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
 darwin_debug_CFLAGS=-O1
 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
 
-darwin_native_binutils=native_cctools
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-darwin_native_toolchain=native_cctools
-else
-darwin_native_toolchain=
-endif
-
 darwin_cmake_system=Darwin