From 6a4337298035683a8748ae446eae90e5c5f41d1b Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 25 Aug 2023 13:50:04 +0100 Subject: [PATCH] ci: Run "Win64 native" job on GitHub Actions --- .cirrus.yml | 96 ------------------- .github/workflows/ci.yml | 196 +++++++++++++++++++++++++++++++++++++++ ci/test/wrapped-cl.bat | 1 - 3 files changed, 196 insertions(+), 97 deletions(-) delete mode 100644 ci/test/wrapped-cl.bat diff --git a/.cirrus.yml b/.cirrus.yml index 4f46da4c8e4..263d9d73de7 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -101,102 +101,6 @@ task: env: FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh" -task: - name: "Win64 native [vs2022]" - << : *FILTER_TEMPLATE - windows_container: - cpu: 6 - memory: 12G - image: cirrusci/windowsservercore:visualstudio2022 - timeout_in: 120m - env: - PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%' - PYTHONUTF8: 1 - CI_VCPKG_TAG: '2023.01.09' - VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' - VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' - CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' - WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat' - QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip' - QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.5.zip' - QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.5' - QTBASEDIR: 'C:\Qt_static' - x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' - QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml' - IgnoreWarnIntDirInTempDetected: 'true' - merge_script: - - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; } - msvc_qt_built_cache: - folder: "%QTBASEDIR%" - reupload_on_changes: false - fingerprint_script: - - echo %QT_DOWNLOAD_URL% %QT_CONFIGURE_COMMAND% - - msbuild -version - populate_script: - - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip - - mkdir C:\jom - - tar -xf C:\jom.zip -C C:\jom - - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL% - - tar -xf %QT_LOCAL_PATH% -C C:\ - - '%x64_NATIVE_TOOLS%' - - cd %QT_SOURCE_DIR% - - mkdir build - - cd build - - '%QT_CONFIGURE_COMMAND% -prefix %QTBASEDIR%' - - jom - - jom install - vcpkg_tools_cache: - folder: '%VCPKG_DOWNLOADS%\tools' - reupload_on_changes: false - fingerprint_script: - - echo %CI_VCPKG_TAG% - - msbuild -version - vcpkg_binary_cache: - folder: '%VCPKG_DEFAULT_BINARY_CACHE%' - reupload_on_changes: true - fingerprint_script: - - echo %CI_VCPKG_TAG% - - type build_msvc\vcpkg.json - - msbuild -version - populate_script: - - mkdir %VCPKG_DEFAULT_BINARY_CACHE% - ccache_cache: - folder: '%CCACHE_DIR%' - install_tools_script: - - choco install --yes --no-progress ccache --version=4.7.4 - - choco install --yes --no-progress python3 --version=3.9.6 - - pip install zmq - - ccache --version - - python -VV - install_vcpkg_script: - - cd .. - - git clone --quiet https://github.com/microsoft/vcpkg.git - - cd vcpkg - - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG% - - .\bootstrap-vcpkg -disableMetrics - - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake - - .\vcpkg integrate install - - .\vcpkg version - build_script: - - '%x64_NATIVE_TOOLS%' - - cd %CIRRUS_WORKING_DIR% - - ccache --zero-stats - - python build_msvc\msvc-autogen.py - - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL%;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo - - ccache --show-stats - check_script: - - src\test_bitcoin.exe -l test_suite - - src\bench_bitcoin.exe --sanity-check - - python test\util\test_runner.py - - python test\util\rpcauth-test.py - functional_tests_script: - # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". - # See: https://learn.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance - - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - - netsh int ipv6 set dynamicport tcp start=1025 num=64511 - # Exclude feature_dbcrash for now due to timeout - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash - task: name: 'ARM, unit tests, no functional tests' << : *GLOBAL_TASK_TEMPLATE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29b725b2540..e6e82386677 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,3 +68,199 @@ jobs: path: ${{ env.CCACHE_DIR }} # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache key: ${{ github.job }}-ccache-${{ github.run_id }} + + win64-native: + name: 'Win64 native, VS 2022' + # Use latest image, but hardcode version to avoid silent upgrades (and breaks). + # See: https://github.com/actions/runner-images#available-images. + runs-on: windows-2022 + + # No need to run on the read-only mirror, unless it is a PR. + if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' + + env: + CCACHE_MAXSIZE: '200M' + CI_CCACHE_VERSION: '4.7.5' + CI_QT_CONF: '-release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml' + CI_QT_DIR: 'qt-everywhere-src-5.15.5' + CI_QT_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip' + PYTHONUTF8: 1 + TEST_RUNNER_TIMEOUT_FACTOR: 40 + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Fix Visual Studio installation + # See: https://github.com/actions/runner-images/issues/7832#issuecomment-1617585694. + run: | + Set-Location "C:\Program Files (x86)\Microsoft Visual Studio\Installer\" + $InstallPath = "C:\Program Files\Microsoft Visual Studio\2022\Enterprise" + $componentsToRemove= @( + "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM.Spectre" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM64" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM64.Spectre" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.x86.x64" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.x86.x64.Spectre" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.Spectre" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM.Spectre" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM64" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM64.Spectre" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.Spectre" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM.Spectre" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM64" + "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM64.Spectre" + ) + [string]$workloadArgs = $componentsToRemove | ForEach-Object {" --remove " + $_} + $Arguments = ('/c', "vs_installer.exe", 'modify', '--installPath', "`"$InstallPath`"",$workloadArgs, '--quiet', '--norestart', '--nocache') + # should be run twice + $process = Start-Process -FilePath cmd.exe -ArgumentList $Arguments -Wait -PassThru -WindowStyle Hidden + $process = Start-Process -FilePath cmd.exe -ArgumentList $Arguments -Wait -PassThru -WindowStyle Hidden + + - name: Configure Developer Command Prompt for Microsoft Visual C++ + # Using microsoft/setup-msbuild is not enough. + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Check MSBuild and Qt + run: | + msbuild -version | Out-File -FilePath "$env:GITHUB_WORKSPACE\msbuild_version" + Get-Content -Path "$env:GITHUB_WORKSPACE\msbuild_version" + $env:CI_QT_URL | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_url" + $env:CI_QT_CONF | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_conf" + + - name: Restore static Qt cache + id: static-qt-cache + uses: actions/cache/restore@v3 + with: + path: C:\Qt_static + key: ${{ github.job }}-static-qt-${{ hashFiles('msbuild_version', 'qt_url', 'qt_conf') }} + + - name: Build static Qt. Download + if: steps.static-qt-cache.outputs.cache-hit != 'true' + shell: cmd + run: | + curl --location --output C:\qt-src.zip %CI_QT_URL% + choco install --yes --no-progress jom + + - name: Build static Qt. Expand source archive + if: steps.static-qt-cache.outputs.cache-hit != 'true' + shell: cmd + run: tar -xf C:\qt-src.zip -C C:\ + + - name: Build static Qt. Create build directory + if: steps.static-qt-cache.outputs.cache-hit != 'true' + run: | + Rename-Item -Path "C:\$env:CI_QT_DIR" -NewName "C:\qt-src" + New-Item -ItemType Directory -Path "C:\qt-src\build" + + - name: Build static Qt. Configure + if: steps.static-qt-cache.outputs.cache-hit != 'true' + working-directory: C:\qt-src\build + shell: cmd + run: ..\configure %CI_QT_CONF% -prefix C:\Qt_static + + - name: Build static Qt. Build + if: steps.static-qt-cache.outputs.cache-hit != 'true' + working-directory: C:\qt-src\build + shell: cmd + run: jom + + - name: Build static Qt. Install + if: steps.static-qt-cache.outputs.cache-hit != 'true' + working-directory: C:\qt-src\build + shell: cmd + run: jom install + + - name: Save static Qt cache + if: steps.static-qt-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: C:\Qt_static + key: ${{ github.job }}-static-qt-${{ hashFiles('msbuild_version', 'qt_url', 'qt_conf') }} + + - name: Ccache installation cache + id: ccache-installation-cache + uses: actions/cache@v3 + with: + path: | + C:\ProgramData\chocolatey\lib\ccache + C:\ProgramData\chocolatey\bin\ccache.exe + C:\ccache\cl.exe + key: ${{ github.job }}-ccache-installation-${{ env.CI_CCACHE_VERSION }} + + - name: Install Ccache + if: steps.ccache-installation-cache.outputs.cache-hit != 'true' + run: | + choco install --yes --no-progress ccache --version=$env:CI_CCACHE_VERSION + New-Item -ItemType Directory -Path "C:\ccache" + Copy-Item -Path "$env:ChocolateyInstall\lib\ccache\tools\ccache-$env:CI_CCACHE_VERSION-windows-x86_64\ccache.exe" -Destination "C:\ccache\cl.exe" + + - name: Restore Ccache cache + uses: actions/cache/restore@v3 + with: + path: ~/AppData/Local/ccache + key: ${{ github.job }}-ccache-${{ github.run_id }} + restore-keys: ${{ github.job }}-ccache- + + - name: Using vcpkg with MSBuild + run: | + Set-Location "$env:VCPKG_INSTALLATION_ROOT" + Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" + vcpkg --vcpkg-root "$env:VCPKG_INSTALLATION_ROOT" integrate install + git rev-parse HEAD | Out-File -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit" + Get-Content -Path "$env:GITHUB_WORKSPACE\vcpkg_commit" + + - name: vcpkg tools cache + uses: actions/cache@v3 + with: + path: C:/vcpkg/downloads/tools + key: ${{ github.job }}-vcpkg-tools + + - name: vcpkg binary cache + uses: actions/cache@v3 + with: + path: ~/AppData/Local/vcpkg/archives + key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('vcpkg_commit', 'msbuild_version', 'build_msvc/vcpkg.json') }} + + - name: Generate project files + run: py -3 build_msvc\msvc-autogen.py + + - name: Build + shell: cmd + run: | + ccache --zero-stats + msbuild build_msvc\bitcoin.sln -property:CLToolPath=C:\ccache;CLToolExe=cl.exe;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + + - name: Ccache stats + run: ccache --show-stats + + - name: Save Ccache cache + uses: actions/cache/save@v3 + if: github.event_name != 'pull_request' + with: + path: ~/AppData/Local/ccache + # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache + key: ${{ github.job }}-ccache-${{ github.run_id }} + + - name: Run unit tests + run: src\test_bitcoin.exe -l test_suite + + - name: Run benchmarks + run: src\bench_bitcoin.exe -sanity-check + + - name: Run util tests + run: py -3 test\util\test_runner.py + + - name: Run rpcauth test + run: py -3 test\util\rpcauth-test.py + + - name: Run functional tests + shell: cmd + run: py -3 test\functional\test_runner.py --ci --quiet --tmpdirprefix=%RUNNER_TEMP% --combinedlogslen=99999999 --timeout-factor=%TEST_RUNNER_TIMEOUT_FACTOR% --extended --exclude feature_dbcrash diff --git a/ci/test/wrapped-cl.bat b/ci/test/wrapped-cl.bat deleted file mode 100644 index fc2a604c580..00000000000 --- a/ci/test/wrapped-cl.bat +++ /dev/null @@ -1 +0,0 @@ -ccache cl %*