Merge #17920: guix: Build support for macOS

f1694757dd guix: Fix typo (Carl Dong)
771c4b98a8 guix: README: Add darwin HOSTS entry (Carl Dong)
8dbf18cb1d guix: Check for macOS SDK before building anything (Carl Dong)
34b23f597e guix: Set ZERO_AR_DATE for darwin build determinism (Carl Dong)
f3835dc6a3 build: Make xorrisofs reproducible with -volume_date (Carl Dong)
c9eb4cf3a0 guix: Add support for darwin builds (Carl Dong)
37fe73a092 build: Add var printing target to src/Makefile.am (Carl Dong)

Pull request description:

  This PR brings our Guix builds on par with Gitian in terms of supported architectures.

  Reviewers: if you run a build, please submit:

  ```
  find output/ -type f -print0 | env LC_ALL=C sort -z | xargs -r0 sha256sum
  ```

  So that we can compare hashes and ensure reproducibility!

ACKs for top commit:
  fanquake:
    ACK f1694757dd - I think we can make some small usability improvements, but this is ok to merge now.

Tree-SHA512: 4af2b71654a9736467dcc681d10601c6eee37800d7847011a50585455b67b55d61742ca5604585f310a2fd75335b674e5e27dfb5169cb2f26e112aa4c411d8be

Makefile.am

@@ -141,7 +141,7 @@ $(APP_DIST_DIR)/Applications:
 $(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt
 
 $(OSX_TEMP_ISO): $(APP_DIST_EXTRAS)
-	$(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist
+	$(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH))
 
 $(OSX_DMG): $(OSX_TEMP_ISO)
 	$(DMG) dmg "$<" "$@"

contrib/guix/README.md

@@ -105,7 +105,8 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum
 
   Override the space-separated list of platform triples for which to perform a
   bootstrappable build. _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf
-  aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32")_
+  aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32
+  x86_64-apple-darwin18")_
 
 * _**SOURCES_PATH**_
 

contrib/guix/guix-build.sh

@@ -65,12 +65,13 @@ else
 fi
 
 ################
-# Check 4: Make sure that build directories do no exist
+# Check 4: Make sure that build directories do not exist
 ################
 
 # Default to building for all supported HOSTs (overridable by environment)
 export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu
-                       x86_64-w64-mingw32}"
+                       x86_64-w64-mingw32
+                       x86_64-apple-darwin18}"
 
 DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}"
 
@@ -105,9 +106,28 @@ for host in $hosts_distsrc_exists; do
 done
 exit 1
 else
+
     mkdir -p "$DISTSRC_BASE"
 fi
 
+################
+# Check 5: When building for darwin, make sure that the macOS SDK exists
+################
+
+for host in $HOSTS; do
+    case "$host" in
+        *darwin*)
+            OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=[[:space:]]\+@@g')"
+            if [ -e "$OSX_SDK" ]; then
+                echo "Found macOS SDK at '${OSX_SDK}', using..."
+            else
+                echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..."
+                exit 1
+            fi
+            ;;
+    esac
+done
+
 #########
 # Setup #
 #########

contrib/guix/libexec/build.sh

@@ -38,7 +38,28 @@ store_path() {
               --expression='s|"[[:space:]]*$||'
 }
 
-# Set environment variables to point Guix's cross-toolchain to the right
+
+# Set environment variables to point the NATIVE toolchain to the right
+# includes/libs
+NATIVE_GCC="$(store_path gcc-toolchain)"
+export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64"
+export CPATH="${NATIVE_GCC}/include"
+case "$HOST" in
+    *darwin*)
+        # When targeting darwin, some native tools built by depends require
+        # native packages not incorporated in depends
+        #
+        # libcap       required by native_cdrkit/wodim
+        # zlib, bzip2  required by native_cdrkit/genisoimage
+        for native_pkg in libcap zlib bzip2; do
+            native_pkg_store_path=$(store_path "$native_pkg")
+            export LIBRARY_PATH="${native_pkg_store_path}/lib:${LIBRARY_PATH}"
+            export CPATH="${native_pkg_store_path}/include:${CPATH}"
+        done
+        ;;
+esac
+
+# Set environment variables to point the CROSS toolchain to the right
 # includes/libs for $HOST
 case "$HOST" in
     *mingw*)
@@ -48,14 +69,18 @@ case "$HOST" in
         CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories...
         CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one)
 
-        NATIVE_GCC="$(store_path gcc-glibc-2.27-toolchain)"
+        # The search path ordering is generally:
-        export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64"
+        #    1. gcc-related search paths
-        export CPATH="${NATIVE_GCC}/include"
+        #    2. libc-related search paths
-
+        #    2. kernel-header-related search paths (not applicable to mingw-w64 hosts)
         export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include"
         export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}"
         export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib"
         ;;
+    *darwin*)
+        # The CROSS toolchain for darwin uses the SDK and ignores environment variables.
+        # See depends/hosts/darwin.mk for more details.
+        ;;
     *linux*)
         CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")"
         CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)"
@@ -64,9 +89,7 @@ case "$HOST" in
         CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories...
         CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one)
 
-        # NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because
+        export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include"
-        # the limits.h in it is missing a '#include_next <limits.h>'
-        export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include"
         export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}"
         export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib"
         ;;
@@ -77,14 +100,25 @@ esac
 # Sanity check CROSS_*_PATH directories
 IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}"
 for p in "${PATHS[@]}"; do
-    if [ ! -d "$p" ]; then
+    if [ -n "$p" ] && [ ! -d "$p" ]; then
         echo "'$p' doesn't exist or isn't a directory... Aborting..."
         exit 1
     fi
 done
 
 # Disable Guix ld auto-rpath behavior
-export GUIX_LD_WRAPPER_DISABLE_RPATH=yes
+case "$HOST" in
+    *darwin*)
+        # The auto-rpath behavior is necessary for darwin builds as some native
+        # tools built by depends refer to and depend on Guix-built native
+        # libraries
+        #
+        # After the native packages in depends are built, the ld wrapper should
+        # no longer affect our build, as clang would instead reach for
+        # x86_64-apple-darwin18-ld from cctools
+        ;;
+    *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;;
+esac
 
 # Make /usr/bin if it doesn't exist
 [ -e /usr/bin ] || mkdir -p /usr/bin
@@ -114,6 +148,16 @@ export QT_RCC_TEST=1
 export QT_RCC_SOURCE_DATE_OVERRIDE=1
 export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name"
 export TZ="UTC"
+case "$HOST" in
+    *darwin*)
+        # cctools AR, unlike GNU binutils AR, does not have a deterministic mode
+        # or a configure flag to enable determinism by default, it only
+        # understands if this env-var is set or not. See:
+        #
+        # https://github.com/tpoechtrager/cctools-port/blob/55562e4073dea0fbfd0b20e0bf69ffe6390c7f97/cctools/ar/archive.c#L334
+        export ZERO_AR_DATE=yes
+        ;;
+esac
 
 ####################
 # Depends Building #
@@ -135,7 +179,8 @@ make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \
                                    x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \
                                    x86_64_linux_NM=x86_64-linux-gnu-nm \
                                    x86_64_linux_STRIP=x86_64-linux-gnu-strip \
-                                   qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++'
+                                   qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \
+                                   FORCE_USE_SYSTEM_CLANG=1
 
 
 ###########################
@@ -169,6 +214,7 @@ HOST_CFLAGS="-O2 -g"
 case "$HOST" in
     *linux*)  HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;;
     *mingw*)  HOST_CFLAGS+=" -fno-ident" ;;
+    *darwin*) unset HOST_CFLAGS ;;
 esac
 
 # CXXFLAGS
@@ -199,8 +245,8 @@ mkdir -p "$DISTSRC"
                     --disable-maintainer-mode \
                     --disable-dependency-tracking \
                     ${CONFIGFLAGS} \
-                    CFLAGS="${HOST_CFLAGS}" \
+                    ${HOST_CFLAGS:+CFLAGS="${HOST_CFLAGS}"} \
-                    CXXFLAGS="${HOST_CXXFLAGS}" \
+                    ${HOST_CXXFLAGS:+CXXFLAGS="${HOST_CXXFLAGS}"} \
                     ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"}
 
     sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool
@@ -210,14 +256,9 @@ mkdir -p "$DISTSRC"
 
     # Perform basic ELF security checks on a series of executables.
     make -C src --jobs=1 check-security ${V:+V=1}
-
+    # Check that executables only contain allowed gcc, glibc and libstdc++
-    case "$HOST" in
+    # version symbols for Linux distro back-compatibility.
-        *linux*|*mingw*)
+    make -C src --jobs=1 check-symbols  ${V:+V=1}
-            # Check that executables only contain allowed gcc, glibc and libstdc++
-            # version symbols for Linux distro back-compatibility.
-            make -C src --jobs=1 check-symbols  ${V:+V=1}
-            ;;
-    esac
 
     # Make the os-specific installers
     case "$HOST" in
@@ -232,8 +273,39 @@ mkdir -p "$DISTSRC"
     INSTALLPATH="${PWD}/installed/${DISTNAME}"
     mkdir -p "${INSTALLPATH}"
     # Install built Bitcoin Core to $INSTALLPATH
-    make install DESTDIR="${INSTALLPATH}" ${V:+V=1}
+    case "$HOST" in
+        *darwin*)
+            make install-strip DESTDIR="${INSTALLPATH}" ${V:+V=1}
+            ;;
+        *)
+            make install DESTDIR="${INSTALLPATH}" ${V:+V=1}
+            ;;
+    esac
 
+    case "$HOST" in
+        *darwin*)
+            make osx_volname ${V:+V=1}
+            make deploydir ${V:+V=1}
+            mkdir -p "unsigned-app-${HOST}"
+            cp  --target-directory="unsigned-app-${HOST}" \
+                osx_volname \
+                contrib/macdeploy/detached-sig-{apply,create}.sh \
+                "${BASEPREFIX}/${HOST}"/native/bin/dmg
+            for util in codesign_allocate pagestuff; do
+                cp --no-target-directory {"${BASEPREFIX}/${HOST}/native/bin/${HOST}-","unsigned-app-${HOST}/"}"$util"
+            done
+            mv --target-directory="unsigned-app-${HOST}" dist
+            (
+                cd "unsigned-app-${HOST}"
+                find . -print0 \
+                    | sort --zero-terminated \
+                    | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
+                    | gzip -9n > "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" \
+                    || ( rm -f "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" && exit 1 )
+            )
+            make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg"
+            ;;
+    esac
     (
         cd installed
 
@@ -248,13 +320,18 @@ mkdir -p "$DISTSRC"
         find . -name "lib*.a" -delete
 
         # Prune pkg-config files
-        rm -r "${DISTNAME}/lib/pkgconfig"
+        rm -rf "${DISTNAME}/lib/pkgconfig"
 
-        # Split binaries and libraries from their debug symbols
+        case "$HOST" in
-        {
+            *darwin*) ;;
-            find "${DISTNAME}/bin" -type f -executable -print0
+            *)
-            find "${DISTNAME}/lib" -type f -print0
+                # Split binaries and libraries from their debug symbols
-        } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg
+                {
+                    find "${DISTNAME}/bin" -type f -executable -print0
+                    find "${DISTNAME}/lib" -type f -print0
+                } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg
+                ;;
+        esac
 
         case "$HOST" in
             *mingw*)
@@ -294,6 +371,13 @@ mkdir -p "$DISTSRC"
                     | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \
                     || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 )
                 ;;
+            *darwin*)
+                find "${DISTNAME}" -print0 \
+                    | sort --zero-terminated \
+                    | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
+                    | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" \
+                    || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" && exit 1 )
+                ;;
         esac
     )
 )

contrib/guix/manifest.scm

@@ -3,24 +3,33 @@
              (gnu packages autotools)
              (gnu packages base)
              (gnu packages bash)
+             (gnu packages cdrom)
              (gnu packages check)
+             (gnu packages cmake)
              (gnu packages commencement)
              (gnu packages compression)
              (gnu packages cross-base)
              (gnu packages file)
              (gnu packages gawk)
              (gnu packages gcc)
+             (gnu packages gnome)
+             (gnu packages image)
+             (gnu packages imagemagick)
              (gnu packages installers)
              (gnu packages linux)
+             (gnu packages llvm)
              (gnu packages mingw)
              (gnu packages perl)
              (gnu packages pkg-config)
              (gnu packages python)
              (gnu packages shells)
              (gnu packages version-control)
+             (guix build-system font)
              (guix build-system gnu)
              (guix build-system trivial)
+             (guix download)
              (guix gexp)
+             ((guix licenses) #:prefix license:)
              (guix packages)
              (guix profiles)
              (guix utils))
@@ -161,11 +170,29 @@ chain for " target " development."))
   (package-with-extra-patches base-nsis
     (search-our-patches "nsis-SConstruct-sde-support.patch")))
 
+(define-public font-tuffy
+  (package
+   (name "font-tuffy")
+   (version "20120614")
+   (source
+    (origin
+     (method url-fetch)
+     (uri (string-append "http://tulrich.com/fonts/tuffy-" version ".tar.gz"))
+     (file-name (string-append name "-" version ".tar.gz"))
+     (sha256
+      (base32
+       "02vf72bgrp30vrbfhxjw82s115z27dwfgnmmzfb0n9wfhxxfpyf6"))))
+   (build-system font-build-system)
+   (home-page "http://tulrich.com/fonts/")
+   (synopsis "The Tuffy Truetype Font Family")
+   (description
+    "Thatcher Ulrich's first outline font design. He started with the goal of producing a neutral, readable sans-serif text font. There are lots of \"expressive\" fonts out there, but he wanted to start with something very plain and clean, something he might want to actually use. ")
+   (license license:public-domain)))
 
 (packages->manifest
  (append
   (list ;; The Basics
-        bash-minimal
+        bash
         which
         coreutils
         util-linux
@@ -195,8 +222,8 @@ chain for " target " development."))
         python-3.7
         ;; Git
         git
-        ;; Native gcc 9 toolchain targeting glibc 2.27
+        ;; Native gcc 7 toolchain
-        (make-gcc-toolchain gcc-9 glibc-2.27))
+        gcc-toolchain-7)
   (let ((target (getenv "HOST")))
     (cond ((string-suffix? "-mingw32" target)
            ;; Windows
@@ -208,4 +235,6 @@ chain for " target " development."))
                                                #:base-gcc-for-libc gcc-7)))
           ((string-contains target "-linux-")
            (list (make-bitcoin-cross-toolchain target)))
+          ((string-contains target "darwin")
+           (list clang-8 libcap binutils imagemagick libtiff librsvg font-tuffy cmake-3.15.5 xorriso))
           (else '())))))

src/Makefile.am

@@ -2,6 +2,10 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+# Pattern rule to print variables, e.g. make print-top_srcdir
+print-%:
+	@echo $* = $($*)
+
 DIST_SUBDIRS = secp256k1 univalue
 
 AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS)