CI: Port Windows build to Azure Pipelines (#7757)
* CI: Port Windows build to Azure Pipelines from Appveyor * CI: Split Windows build into scripts * CI: Remove Appveyor * CI: Add GitHub Release deployment to Azure Windows Build * VCS: Add full branch name function to rpcs3_version The STRINGIZE macro was a little awkward, and difficult to control at configure time. Since other version information is already included, the full branch name is now added as a function. It's runtime instead of compile-time checking, but it seems worth it. * CI: Overhaul Windows setup script Previously, there was no way of forcing a re-download of cached dependencies when they were replaced by new ones. In addition, there was really no verification of downloads or cache. Now, changing a few lines at the top of the file will automagically force a cache update.
This commit is contained in:
JohnHolmesII
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh -ex
|
||||
|
||||
# From pure sh bible, strips rogue quotes
|
||||
# Does single and double quotes
|
||||
trim_quotes()
|
||||
{
|
||||
set -f
|
||||
|
||||
old_ifs=$IFS
|
||||
IFS=\"\'
|
||||
|
||||
set -- $1
|
||||
IFS=
|
||||
printf '%s\n' "$*"
|
||||
IFS=$old_ifs
|
||||
|
||||
set +f
|
||||
}
|
||||
|
||||
# BUILD_blablabla is Azure specific, so we wrap it for portability
|
||||
# The BUILD var is passed from a previous stage, so it is cleaned
|
||||
# due to a bug in Azure
|
||||
ARTIFACT_DIR="$BUILD_ARTIFACTSTAGINGDIRECTORY"
|
||||
BUILD=$(trim_quotes "$BUILD")
|
||||
|
||||
# Remove unecessary files
|
||||
rm -f ./bin/rpcs3.exp ./bin/rpcs3.lib ./bin/rpcs3.pdb
|
||||
|
||||
# Prepare compatibility database for packaging, as well as
|
||||
# certificate for ssl (auto-updater)
|
||||
curl -sL 'https://rpcs3.net/compatibility?api=v1&export' | iconv -t UTF-8 > ./bin/GuiConfigs/compat_database.dat
|
||||
curl -sL 'https://curl.haxx.se/ca/cacert.pem' > ./bin/cacert.pem
|
||||
|
||||
# Package artifacts
|
||||
7z a -m0=LZMA2 -mx9 "$BUILD" ./bin/*
|
||||
|
||||
# Generate sha256 hashes
|
||||
# Write to file for GitHub releases
|
||||
sha256sum "$BUILD" | awk '{ print $1 }' | tee "$BUILD.sha256"
|
||||
echo "$(cat "$BUILD.sha256");$(stat -c %s "$BUILD")B" > GitHubReleaseMessage.txt
|
||||
|
||||
# Move files to publishing directory
|
||||
mv -- "$BUILD" "$ARTIFACT_DIR"
|
||||
mv -- "$BUILD.sha256" "$ARTIFACT_DIR"
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/bin/sh -ex
|
||||
|
||||
# These are Azure specific, so we wrap them for portability
|
||||
REPO_NAME="$BUILD_REPOSITORY_NAME"
|
||||
REPO_BRANCH="$SYSTEM_PULLREQUEST_SOURCEBRANCH"
|
||||
PR_NUMBER="$SYSTEM_PULLREQUEST_PULLREQUESTID"
|
||||
|
||||
# Resource/dependency URLs
|
||||
# Qt mirrors can be volatile and slow, so we list 2
|
||||
#QT_HOST="http://mirrors.ocf.berkeley.edu/qt/"
|
||||
QT_HOST="http://qt.mirror.constant.com/"
|
||||
QT_URL_VER=$(echo "$QT_VER" | sed "s/\.//g")
|
||||
QT_PREFIX="online/qtsdkrepository/windows_x86/desktop/qt5_${QT_URL_VER}/qt.qt5.${QT_URL_VER}.win64_msvc2017_64/${QT_VER}-0-${QT_DATE}"
|
||||
QT_SUFFIX="-Windows-Windows_10-MSVC2017-Windows-Windows_10-X86_64.7z"
|
||||
QT_BASE_URL="${QT_HOST}${QT_PREFIX}qtbase${QT_SUFFIX}"
|
||||
QT_WINE_URL="${QT_HOST}${QT_PREFIX}qtwinextras${QT_SUFFIX}"
|
||||
QT_DECL_URL="${QT_HOST}${QT_PREFIX}qtdeclarative${QT_SUFFIX}"
|
||||
QT_TOOL_URL="${QT_HOST}${QT_PREFIX}qttools${QT_SUFFIX}"
|
||||
LLVMLIBS_URL='https://github.com/RPCS3/llvm-mirror/releases/download/custom-build-win/llvmlibs_mt.7z'
|
||||
LLVM_SHA="773273312b6bfe0c361b3780e175ab2935f8d10558431420dee2a1414ee965d0"
|
||||
GLSLANG_URL='https://github.com/RPCS3/glslang/releases/download/custom-build-win/glslanglibs_mt.7z'
|
||||
GLSLANG_SHA="b2f47a20239eddd423257c10258e1c7889a5bb7d66773460e3a87b4dbead7c1d"
|
||||
VULKAN_SDK_URL="https://sdk.lunarg.com/sdk/download/${VULKAN_VER}/windows/VulkanSDK-${VULKAN_VER}-Installer.exe"
|
||||
|
||||
DEP_URLS=" \
|
||||
$QT_BASE_URL \
|
||||
$QT_WINE_URL \
|
||||
$QT_DECL_URL \
|
||||
$QT_TOOL_URL \
|
||||
$LLVMLIBS_URL \
|
||||
$GLSLANG_URL \
|
||||
$VULKAN_SDK_URL"
|
||||
|
||||
# Azure pipelines doesn't make a cache dir if it doesn't exist, so we do it manually
|
||||
[ -d "$CACHE_DIR" ] || mkdir "$CACHE_DIR"
|
||||
|
||||
# Pull all the submodules except llvm, since it is built separately and we just download that build
|
||||
# Note: Tried to use git submodule status, but it takes over 20 seconds
|
||||
# shellcheck disable=SC2046
|
||||
git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules)
|
||||
|
||||
# Git bash doesn't have rev, so here it is
|
||||
rev()
|
||||
{
|
||||
echo "$1" | awk '{ for(i = length($0); i != 0; --i) { a = a substr($0, i, 1); } } END { print a }'
|
||||
}
|
||||
|
||||
# Usage: download_and_verify url checksum algo file
|
||||
# Check to see if its already cached, and the checksum matches. If not, download it.
|
||||
# Tries up to 3 times
|
||||
download_and_verify()
|
||||
{
|
||||
url="$1"
|
||||
correctChecksum="$2"
|
||||
algo="$3"
|
||||
fileName="$4"
|
||||
|
||||
for _ in 1 2 3; do
|
||||
[ -e "$CACHE_DIR/$fileName" ] || curl -L -o "$CACHE_DIR/$fileName" "$url"
|
||||
fileChecksum=$("${algo}sum" "$CACHE_DIR/$fileName" | awk '{ print $1 }')
|
||||
[ "$fileChecksum" = "$correctChecksum" ] && return 0
|
||||
rm
|
||||
done
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
# Some dependencies install here
|
||||
[ -d "./lib" ] || mkdir "./lib"
|
||||
|
||||
for url in $DEP_URLS; do
|
||||
# Get the filename from the URL. Breaks if urls have js args, so don't do that pls
|
||||
fileName="$(rev "$(rev "$url" | cut -d'/' -f1)")"
|
||||
[ -z "$fileName" ] && echo "Unable to parse url: $url" && exit 1
|
||||
|
||||
# shellcheck disable=SC1003
|
||||
case "$url" in
|
||||
*qt*) checksum=$(curl -L "${url}.sha1"); algo="sha1"; outDir='C:\Qt\' ;;
|
||||
*llvm*) checksum="$LLVM_SHA"; algo="sha256"; outDir="." ;;
|
||||
*glslang*) checksum="$GLSLANG_SHA"; algo=sha256; outDir="./lib/Release - LLVM-x64" ;;
|
||||
*Vulkan*)
|
||||
# Vulkan setup needs to be run in batch environment
|
||||
# Need to subshell this or else it doesn't wait
|
||||
download_and_verify "$url" "$VULKAN_SDK_SHA" "sha256" "$fileName"
|
||||
cp "$CACHE_DIR/$fileName" .
|
||||
_=$(echo "$fileName /S" | cmd)
|
||||
continue
|
||||
;;
|
||||
*) echo "Unknown url resource: $url"; exit 1 ;;
|
||||
esac
|
||||
|
||||
download_and_verify "$url" "$checksum" "$algo" "$fileName"
|
||||
7z x -y "$CACHE_DIR/$fileName" -aos -o"$outDir"
|
||||
done
|
||||
|
||||
# Gather explicit version number and number of commits
|
||||
COMM_TAG=$(awk '/version{.*}/ { printf("%d.%d.%d", $5, $6, $7) }' ./rpcs3/rpcs3_version.cpp)
|
||||
COMM_COUNT=$(git rev-list --count HEAD)
|
||||
COMM_HASH=$(git rev-parse --short=8 HEAD)
|
||||
|
||||
# Format the above into filenames
|
||||
if [ -n "$PR_NUMBER" ]; then
|
||||
AVVER="${COMM_TAG}-${COMM_HASH}"
|
||||
BUILD="rpcs3-v${AVVER}_win64.7z"
|
||||
else
|
||||
AVVER="${COMM_TAG}-${COMM_COUNT}"
|
||||
BUILD="rpcs3-v${AVVER}-${COMM_HASH}_win64.7z"
|
||||
fi
|
||||
|
||||
# Export variables for later stages of the Azure pipeline
|
||||
# Values done in this manner will appear as environment variables
|
||||
# in later stages, but are not added to environment variables
|
||||
# in *this* stage. Thank azure for that one.
|
||||
# BRANCH is used for experimental build warnings for pr builds
|
||||
# used in main_window.cpp. AVVER is used for GitHub releases.
|
||||
BRANCH="${REPO_NAME}/${REPO_BRANCH}"
|
||||
echo "##vso[task.setvariable variable=branch]$BRANCH"
|
||||
echo "##vso[task.setvariable variable=build]$BUILD"
|
||||
echo "##vso[task.setvariable variable=avver]$AVVER"
|
||||
@@ -6,6 +6,8 @@ TRAVIS_COMMIT
|
||||
BUILD_REASON
|
||||
BUILD_SOURCEVERSION
|
||||
BUILD_ARTIFACTSTAGINGDIRECTORY
|
||||
BUILD_REPOSITORY_NAME
|
||||
BUILD_SOURCEBRANCHNAME
|
||||
# Variables for Travis build matrix
|
||||
COMPILER
|
||||
DEPLOY_APPIMAGE
|
||||
|
||||
Reference in New Issue
Block a user