CI: Maintenance
- Rename .travis dir to .ci, since it isn't just for Travis - Convert Linux build scripts to posix sh - Clean up some scripts per shellcheck
This commit is contained in:
JohnHolmesII
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/sh -ex
|
||||
|
||||
# Setup Qt variables
|
||||
export QT_BASE_DIR=/opt/qt${QTVERMIN}
|
||||
export PATH=$QT_BASE_DIR/bin:$PATH
|
||||
export LD_LIBRARY_PATH=$QT_BASE_DIR/lib/x86_64-linux-gnu:$QT_BASE_DIR/lib
|
||||
|
||||
cd rpcs3 || exit 1
|
||||
|
||||
# Pull all the submodules except llvm, since it is built separately and we just download that build
|
||||
# Note: Tried to use git submodule status, but it takes over 20 seconds
|
||||
# shellcheck disable=SC2046
|
||||
git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules)
|
||||
|
||||
# Download pre-compiled llvm libs
|
||||
curl -sLO https://github.com/RPCS3/llvm-mirror/releases/download/custom-build/llvmlibs-linux.tar.gz
|
||||
mkdir llvmlibs
|
||||
tar -xzf ./llvmlibs-linux.tar.gz -C llvmlibs
|
||||
|
||||
mkdir build && cd build || exit 1
|
||||
|
||||
if [ "$COMPILER" = "gcc" ]; then
|
||||
# These are set in the dockerfile
|
||||
export CC=${GCC_BINARY}
|
||||
export CXX=${GXX_BINARY}
|
||||
export LINKER=gold
|
||||
# We need to set the following variables for LTO to link properly
|
||||
export AR=/usr/bin/gcc-ar-9
|
||||
export RANLIB=/usr/bin/gcc-ranlib-9
|
||||
export CFLAGS="-fuse-linker-plugin"
|
||||
else
|
||||
export CC=${CLANG_BINARY}
|
||||
export CXX=${CLANGXX_BINARY}
|
||||
export LINKER=lld
|
||||
export AR=/usr/bin/llvm-ar-$LLVMVER
|
||||
export RANLIB=/usr/bin/llvm-ranlib-$LLVMVER
|
||||
fi
|
||||
|
||||
export CFLAGS="$CFLAGS -fuse-ld=${LINKER}"
|
||||
|
||||
cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_LLVM_SUBMODULE=OFF -DUSE_COTIRE=OFF -DLLVM_DIR=llvmlibs/lib/cmake/llvm/ -DUSE_NATIVE_INSTRUCTIONS=OFF \
|
||||
-DCMAKE_C_FLAGS="$CFLAGS" -DCMAKE_CXX_FLAGS="$CFLAGS" -DCMAKE_AR=$AR -DCMAKE_RANLIB=$RANLIB \
|
||||
-G Ninja
|
||||
|
||||
ninja; build_status=$?;
|
||||
|
||||
cd ..
|
||||
|
||||
# If it compiled succesfully let's deploy depending on the build pipeline (Travis, Azure Pipelines).
|
||||
# Travis only deploys on master, and it publishes to GitHub releases. Azure publishes PRs as artifacts
|
||||
# only.
|
||||
{ [ "$IS_AZURE" = "true" ] ||
|
||||
{ [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; };
|
||||
} && SHOULD_DEPLOY="true" || SHOULD_DEPLOY="false"
|
||||
|
||||
if [ "$build_status" -eq 0 ] && [ "$SHOULD_DEPLOY" = "true" ]; then
|
||||
.ci/deploy-linux.sh
|
||||
fi
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/sh -ex
|
||||
|
||||
export CCACHE_SLOPPINESS=pch_defines,time_macros
|
||||
export CMAKE_PREFIX_PATH=/usr/local/opt/qt5/
|
||||
export PATH="/usr/local/opt/ccache/libexec:$PATH"
|
||||
|
||||
# Setup vulkan and gfx-rs/portability
|
||||
curl -sLO https://github.com/gfx-rs/portability/releases/download/latest/gfx-portability-macos-latest.zip
|
||||
unzip -: gfx-portability-macos-latest.zip
|
||||
curl -sLO https://github.com/KhronosGroup/Vulkan-Headers/archive/sdk-1.1.106.0.zip
|
||||
unzip -: sdk-*.zip
|
||||
mkdir vulkan-sdk
|
||||
ln -s "${PWD}"/Vulkan-Headers*/include vulkan-sdk/include
|
||||
mkdir vulkan-sdk/lib
|
||||
cp target/release/libportability.dylib vulkan-sdk/lib/libVulkan.dylib
|
||||
# Let macdeployqt locate and install Vulkan library
|
||||
install_name_tool -id "${PWD}"/vulkan-sdk/lib/libVulkan.dylib vulkan-sdk/lib/libVulkan.dylib
|
||||
export VULKAN_SDK="${PWD}/vulkan-sdk"
|
||||
|
||||
# Pull all the submodules except llvm
|
||||
# Note: Tried to use git submodule status, but it takes over 20 seconds
|
||||
# shellcheck disable=SC2046
|
||||
git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules)
|
||||
|
||||
mkdir build && cd build || exit 1
|
||||
cmake .. -DWITH_LLVM=OFF -DUSE_NATIVE_INSTRUCTIONS=OFF -G Ninja
|
||||
ninja
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/bin/sh -ex
|
||||
|
||||
cd build || exit 1
|
||||
|
||||
if [ "$DEPLOY_APPIMAGE" = "true" ]; then
|
||||
DESTDIR=appdir ninja install
|
||||
QT_APPIMAGE="linuxdeployqt.AppImage"
|
||||
|
||||
curl -sL "https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage" > "$QT_APPIMAGE"
|
||||
chmod a+x "$QT_APPIMAGE"
|
||||
"./$QT_APPIMAGE" --appimage-extract
|
||||
./squashfs-root/AppRun ./appdir/usr/share/applications/*.desktop -bundle-non-qt-libs
|
||||
ls ./appdir/usr/lib/
|
||||
rm -r ./appdir/usr/share/doc
|
||||
rm ./appdir/usr/lib/libxcb*
|
||||
cp "$(readlink -f /lib/x86_64-linux-gnu/libnsl.so.1)" ./appdir/usr/lib/libnsl.so.1
|
||||
export PATH=/rpcs3/build/squashfs-root/usr/bin/:${PATH}
|
||||
|
||||
# Embed newer libstdc++ for distros that don't come with it (ubuntu 16.04)
|
||||
mkdir -p appdir/usr/optional/ ; mkdir -p appdir/usr/optional/libstdc++/
|
||||
cp /usr/lib/x86_64-linux-gnu/libstdc++.so.6 ./appdir/usr/optional/libstdc++/
|
||||
rm ./appdir/AppRun
|
||||
curl -sL https://github.com/RPCS3/AppImageKit-checkrt/releases/download/continuous2/AppRun-patched-x86_64 -o ./appdir/AppRun
|
||||
chmod a+x ./appdir/AppRun
|
||||
curl -sL https://github.com/RPCS3/AppImageKit-checkrt/releases/download/continuous2/exec-x86_64.so -o ./appdir/usr/optional/exec.so
|
||||
|
||||
# Compile checker binary for AppImageKit-checkrt
|
||||
|
||||
# This may need updating if you update the compiler or rpcs3 uses newer c++ features
|
||||
# See https://github.com/gcc-mirror/gcc/blob/master/libstdc%2B%2B-v3/config/abi/pre/gnu.ver
|
||||
# for which definitions correlate to which CXXABI version.
|
||||
# Currently we target a minimum of GLIBCXX_3.4.26 and CXXABI_1.3.11
|
||||
printf "#include <bits/stdc++.h>\nint main(){std::make_exception_ptr(0);std::pmr::get_default_resource();}" | $CXX -x c++ -std=c++2a -o ./appdir/usr/optional/checker -
|
||||
|
||||
# Package it up and send it off
|
||||
./squashfs-root/usr/bin/appimagetool /rpcs3/build/appdir
|
||||
ls
|
||||
|
||||
COMM_TAG="$(grep 'version{.*}' ../rpcs3/rpcs3_version.cpp | awk -F[\{,] '{printf "%d.%d.%d", $2, $3, $4}')"
|
||||
COMM_COUNT="$(git rev-list --count HEAD)"
|
||||
COMM_HASH="$(git rev-parse --short=8 HEAD)"
|
||||
RPCS3_APPIMAGE="rpcs3-v${COMM_TAG}-${COMM_COUNT}-${COMM_HASH}_linux64.AppImage"
|
||||
|
||||
curl -sLO https://github.com/hcorion/uploadtool/raw/master/upload.sh
|
||||
mv ./RPCS3*.AppImage "$RPCS3_APPIMAGE"
|
||||
|
||||
# If we're building using Azure Pipelines, let's copy over the AppImage artifact
|
||||
if [ -n "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]; then
|
||||
cp "$RPCS3_APPIMAGE" ~/artifacts
|
||||
fi
|
||||
|
||||
FILESIZE=$(stat -c %s ./rpcs3*.AppImage)
|
||||
SHA256SUM=$(sha256sum ./rpcs3*.AppImage)
|
||||
if [ -n "$GITHUB_TOKEN" ]; then
|
||||
unset TRAVIS_REPO_SLUG
|
||||
REPO_SLUG=RPCS3/rpcs3-binaries-linux \
|
||||
UPLOADTOOL_BODY="$SHA256SUM;${FILESIZE}B"\
|
||||
RELEASE_NAME=build-${TRAVIS_COMMIT}\
|
||||
RELEASE_TITLE=${COMM_TAG}-${COMM_COUNT}\
|
||||
REPO_COMMIT=d812f1254a1157c80fd402f94446310560f54e5f\
|
||||
bash upload.sh rpcs3*.AppImage
|
||||
fi
|
||||
fi
|
||||
if [ "$DEPLOY_PPA" = "true" ]; then
|
||||
export DEBFULLNAME="RPCS3 Build Bot"
|
||||
fi
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh -ex
|
||||
|
||||
# BUILD_blablabla is Azure specific, so we wrap it for portability
|
||||
ARTIFACT_DIR="$BUILD_ARTIFACTSTAGINGDIRECTORY"
|
||||
|
||||
# Remove unecessary files
|
||||
rm -f ./bin/rpcs3.exp ./bin/rpcs3.lib ./bin/rpcs3.pdb
|
||||
|
||||
# Prepare compatibility database for packaging, as well as
|
||||
# certificate for ssl (auto-updater)
|
||||
curl -sL 'https://rpcs3.net/compatibility?api=v1&export' | iconv -t UTF-8 > ./bin/GuiConfigs/compat_database.dat
|
||||
curl -sL 'https://curl.haxx.se/ca/cacert.pem' > ./bin/cacert.pem
|
||||
|
||||
# Package artifacts
|
||||
7z a -m0=LZMA2 -mx9 "$BUILD" ./bin/*
|
||||
|
||||
# Generate sha256 hashes
|
||||
# Write to file for GitHub releases
|
||||
sha256sum "$BUILD" | awk '{ print $1 }' | tee "$BUILD.sha256"
|
||||
echo "$(cat "$BUILD.sha256");$(stat -c %s "$BUILD")B" > GitHubReleaseMessage.txt
|
||||
|
||||
# Move files to publishing directory
|
||||
mv -- "$BUILD" "$ARTIFACT_DIR"
|
||||
mv -- "$BUILD.sha256" "$ARTIFACT_DIR"
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh -e
|
||||
|
||||
# Export variables for later stages of the Azure pipeline
|
||||
# Values done in this manner will appear as environment variables
|
||||
# in later stages.
|
||||
|
||||
# From pure-sh-bible
|
||||
# Setting 'IFS' tells 'read' where to split the string.
|
||||
while IFS='=' read -r key val; do
|
||||
# Skip over lines containing comments.
|
||||
[ "${key##\#*}" ] || continue
|
||||
echo "##vso[task.setvariable variable=$key]$val"
|
||||
done < ".ci/azure-vars.env"
|
||||
Executable
+116
@@ -0,0 +1,116 @@
|
||||
#!/bin/sh -ex
|
||||
|
||||
# These are Azure specific, so we wrap them for portability
|
||||
REPO_NAME="$BUILD_REPOSITORY_NAME"
|
||||
REPO_BRANCH="$SYSTEM_PULLREQUEST_SOURCEBRANCH"
|
||||
PR_NUMBER="$SYSTEM_PULLREQUEST_PULLREQUESTID"
|
||||
|
||||
# Resource/dependency URLs
|
||||
# Qt mirrors can be volatile and slow, so we list 2
|
||||
#QT_HOST="http://mirrors.ocf.berkeley.edu/qt/"
|
||||
QT_HOST="http://qt.mirror.constant.com/"
|
||||
QT_URL_VER=$(echo "$QT_VER" | sed "s/\.//g")
|
||||
QT_PREFIX="online/qtsdkrepository/windows_x86/desktop/qt5_${QT_URL_VER}/qt.qt5.${QT_URL_VER}.win64_msvc2017_64/${QT_VER}-0-${QT_DATE}"
|
||||
QT_SUFFIX="-Windows-Windows_10-MSVC2017-Windows-Windows_10-X86_64.7z"
|
||||
QT_BASE_URL="${QT_HOST}${QT_PREFIX}qtbase${QT_SUFFIX}"
|
||||
QT_WINE_URL="${QT_HOST}${QT_PREFIX}qtwinextras${QT_SUFFIX}"
|
||||
QT_DECL_URL="${QT_HOST}${QT_PREFIX}qtdeclarative${QT_SUFFIX}"
|
||||
QT_TOOL_URL="${QT_HOST}${QT_PREFIX}qttools${QT_SUFFIX}"
|
||||
LLVMLIBS_URL='https://github.com/RPCS3/llvm-mirror/releases/download/custom-build-win/llvmlibs_mt.7z'
|
||||
LLVM_SHA="773273312b6bfe0c361b3780e175ab2935f8d10558431420dee2a1414ee965d0"
|
||||
GLSLANG_URL='https://github.com/RPCS3/glslang/releases/download/custom-build-win/glslanglibs_mt.7z'
|
||||
GLSLANG_SHA="b2f47a20239eddd423257c10258e1c7889a5bb7d66773460e3a87b4dbead7c1d"
|
||||
VULKAN_SDK_URL="https://sdk.lunarg.com/sdk/download/${VULKAN_VER}/windows/VulkanSDK-${VULKAN_VER}-Installer.exe"
|
||||
|
||||
DEP_URLS=" \
|
||||
$QT_BASE_URL \
|
||||
$QT_WINE_URL \
|
||||
$QT_DECL_URL \
|
||||
$QT_TOOL_URL \
|
||||
$LLVMLIBS_URL \
|
||||
$GLSLANG_URL \
|
||||
$VULKAN_SDK_URL"
|
||||
|
||||
# Azure pipelines doesn't make a cache dir if it doesn't exist, so we do it manually
|
||||
[ -d "$CACHE_DIR" ] || mkdir "$CACHE_DIR"
|
||||
|
||||
# Pull all the submodules except llvm, since it is built separately and we just download that build
|
||||
# Note: Tried to use git submodule status, but it takes over 20 seconds
|
||||
# shellcheck disable=SC2046
|
||||
git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules)
|
||||
|
||||
# Git bash doesn't have rev, so here it is
|
||||
rev()
|
||||
{
|
||||
echo "$1" | awk '{ for(i = length($0); i != 0; --i) { a = a substr($0, i, 1); } } END { print a }'
|
||||
}
|
||||
|
||||
# Usage: download_and_verify url checksum algo file
|
||||
# Check to see if a file is already cached, and the checksum matches. If not, download it.
|
||||
# Tries up to 3 times
|
||||
download_and_verify()
|
||||
{
|
||||
url="$1"
|
||||
correctChecksum="$2"
|
||||
algo="$3"
|
||||
fileName="$4"
|
||||
|
||||
for _ in 1 2 3; do
|
||||
[ -e "$CACHE_DIR/$fileName" ] || curl -L -o "$CACHE_DIR/$fileName" "$url"
|
||||
fileChecksum=$("${algo}sum" "$CACHE_DIR/$fileName" | awk '{ print $1 }')
|
||||
[ "$fileChecksum" = "$correctChecksum" ] && return 0
|
||||
rm "$CACHE_DIR/$fileName"
|
||||
done
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
# Some dependencies install here
|
||||
[ -d "./lib" ] || mkdir "./lib"
|
||||
|
||||
for url in $DEP_URLS; do
|
||||
# Get the filename from the URL. Breaks if urls have js args, so don't do that pls
|
||||
fileName="$(rev "$(rev "$url" | cut -d'/' -f1)")"
|
||||
[ -z "$fileName" ] && echo "Unable to parse url: $url" && exit 1
|
||||
|
||||
# shellcheck disable=SC1003
|
||||
case "$url" in
|
||||
*qt*) checksum=$(curl -L "${url}.sha1"); algo="sha1"; outDir='C:\Qt\' ;;
|
||||
*llvm*) checksum="$LLVM_SHA"; algo="sha256"; outDir="." ;;
|
||||
*glslang*) checksum="$GLSLANG_SHA"; algo=sha256; outDir="./lib/Release - LLVM-x64" ;;
|
||||
*Vulkan*)
|
||||
# Vulkan setup needs to be run in batch environment
|
||||
# Need to subshell this or else it doesn't wait
|
||||
download_and_verify "$url" "$VULKAN_SDK_SHA" "sha256" "$fileName"
|
||||
cp "$CACHE_DIR/$fileName" .
|
||||
_=$(echo "$fileName /S" | cmd)
|
||||
continue
|
||||
;;
|
||||
*) echo "Unknown url resource: $url"; exit 1 ;;
|
||||
esac
|
||||
|
||||
download_and_verify "$url" "$checksum" "$algo" "$fileName"
|
||||
7z x -y "$CACHE_DIR/$fileName" -aos -o"$outDir"
|
||||
done
|
||||
|
||||
# Gather explicit version number and number of commits
|
||||
COMM_TAG=$(awk '/version{.*}/ { printf("%d.%d.%d", $5, $6, $7) }' ./rpcs3/rpcs3_version.cpp)
|
||||
COMM_COUNT=$(git rev-list --count HEAD)
|
||||
COMM_HASH=$(git rev-parse --short=8 HEAD)
|
||||
|
||||
# Format the above into filenames
|
||||
if [ -n "$PR_NUMBER" ]; then
|
||||
AVVER="${COMM_TAG}-${COMM_HASH}"
|
||||
BUILD="rpcs3-v${AVVER}_win64.7z"
|
||||
else
|
||||
AVVER="${COMM_TAG}-${COMM_COUNT}"
|
||||
BUILD="rpcs3-v${AVVER}-${COMM_HASH}_win64.7z"
|
||||
fi
|
||||
|
||||
# BRANCH is used for experimental build warnings for pr builds, used in main_window.cpp.
|
||||
# BUILD is the name of the release artifact
|
||||
# AVVER is used for GitHub releases, it is the version number.
|
||||
BRANCH="${REPO_NAME}/${REPO_BRANCH}"
|
||||
echo "BRANCH=$BRANCH" > .ci/azure-vars.env
|
||||
echo "BUILD=$BUILD" >> .ci/azure-vars.env
|
||||
echo "AVVER=$AVVER" >> .ci/azure-vars.env
|
||||
@@ -0,0 +1,16 @@
|
||||
# Variables set by Travis CI
|
||||
TRAVIS_PULL_REQUEST
|
||||
TRAVIS_BRANCH
|
||||
TRAVIS_COMMIT
|
||||
# Variables set by Azure Pipelines
|
||||
IS_AZURE
|
||||
BUILD_REASON
|
||||
BUILD_SOURCEVERSION
|
||||
BUILD_ARTIFACTSTAGINGDIRECTORY
|
||||
BUILD_REPOSITORY_NAME
|
||||
BUILD_SOURCEBRANCHNAME
|
||||
# Variables for Travis build matrix
|
||||
COMPILER
|
||||
DEPLOY_APPIMAGE
|
||||
# Private Travis CI variables
|
||||
GITHUB_TOKEN
|
||||
Reference in New Issue
Block a user