Merge pull request #3706 from liushuyu/appveyor-cache

Use Travis CI to build MinGW packages

.travis.yml

@@ -48,6 +48,19 @@ matrix:
       services: docker
       cache: ccache
       script: "./.travis/linux-frozen/build.sh"
+    - os: linux
+      env: NAME="MinGW build"
+      sudo: required
+      dist: trusty
+      services: docker
+      addons:
+        apt:
+          packages:
+            - p7zip-full
+      install: "./.travis/linux-mingw/deps.sh"
+      script: "./.travis/linux-mingw/build.sh"
+      after_success: "./.travis/linux-mingw/upload.sh"
+      cache: ccache
 
 deploy:
   provider: releases

.travis/common/post-upload.sh

@@ -11,6 +11,9 @@ if [ -z $TRAVIS_TAG ]; then
     RELEASE_NAME=head
 else
     RELEASE_NAME=$(echo $TRAVIS_TAG | cut -d- -f1)
+    if [ "$NAME" = "MinGW build" ]; then
+        RELEASE_NAME="${RELEASE_NAME}-mingw"
+    fi
 fi
 
 mv "$REV_NAME" $RELEASE_NAME

.travis/linux-mingw/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -ex
+mkdir "$HOME/.ccache" || true
+docker run --env-file .travis/common/travis-ci.env -e ENABLE_COMPATIBILITY_REPORTING -v $(pwd):/citra -v "$HOME/.ccache":/root/.ccache ubuntu:18.04 /bin/bash -ex /citra/.travis/linux-mingw/docker.sh

.travis/linux-mingw/deps.sh

@@ -0,0 +1,3 @@
+#!/bin/sh -ex
+
+docker pull ubuntu:18.04

.travis/linux-mingw/docker.sh

@@ -0,0 +1,48 @@
+#!/bin/bash -ex
+
+cd /citra
+MINGW_PACKAGES="sdl2-mingw-w64 qt5base-mingw-w64 qt5tools-mingw-w64 libsamplerate-mingw-w64 qt5multimedia-mingw-w64"
+apt-get update
+apt-get install -y gpg wget git python3-pip ccache g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64 mingw-w64-tools cmake
+# HACK: the repository does not contain necessary packages for 18.04, we'll use packages for 17.10 for now
+echo 'deb http://ppa.launchpad.net/tobydox/mingw-w64/ubuntu artful main ' > /etc/apt/sources.list.d/extras.list
+apt-key adv --keyserver keyserver.ubuntu.com --recv '72931B477E22FEFD47F8DECE02FE5F12ADDE29B2'
+apt-get update
+apt-get install -y ${MINGW_PACKAGES}
+
+# fix a problem in current MinGW headers
+wget -q https://github.com/Alexpux/mingw-w64/raw/d0d7f784833bbb0b2d279310ddc6afb52fe47a46/mingw-w64-headers/crt/errno.h -O /usr/x86_64-w64-mingw32/include/errno.h
+# override Travis CI unreasonable ccache size
+echo 'max_size = 3.0G' > "$HOME/.ccache/ccache.conf"
+
+mkdir build && cd build
+cmake .. -DCMAKE_TOOLCHAIN_FILE="$(pwd)/../CMakeModules/MinGWCross.cmake" -DUSE_CCACHE=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_QT_TRANSLATION=ON -DCITRA_ENABLE_COMPATIBILITY_REPORTING=${ENABLE_COMPATIBILITY_REPORTING:-"OFF"} -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON
+make -j4
+
+echo "Tests skipped"
+#ctest -VV -C Release
+
+ccache -s
+
+echo 'Prepare binaries...'
+cd ..
+mkdir package
+
+QT_PLATFORM_DLL_PATH='/usr/x86_64-w64-mingw32/lib/qt5/plugins/platforms/'
+find build/ -name "citra*.exe" -exec cp {} 'package' \;
+
+# copy Qt plugins
+mkdir package/platforms
+cp "${QT_PLATFORM_DLL_PATH}/qwindows.dll" package/platforms/
+cp -rv "${QT_PLATFORM_DLL_PATH}/../mediaservice/" package/
+cp -rv "${QT_PLATFORM_DLL_PATH}/../imageformats/" package/
+rm -f package/mediaservice/*d.dll
+
+for i in package/*.exe; do
+  # we need to process pdb here, however, cv2pdb
+  # does not work here, so we just simply strip all the debug symbols
+  x86_64-w64-mingw32-strip "${i}"
+done
+
+pip3 install pefile
+python3 .travis/linux-mingw/scan_dll.py package/*.exe "package/"

.travis/linux-mingw/scan_dll.py

@@ -0,0 +1,106 @@
+import pefile
+import sys
+import re
+import os
+import queue
+import shutil
+
+# constant definitions
+KNOWN_SYS_DLLS = ['WINMM.DLL', 'MSVCRT.DLL', 'VERSION.DLL', 'MPR.DLL',
+                  'DWMAPI.DLL', 'UXTHEME.DLL', 'DNSAPI.DLL', 'IPHLPAPI.DLL']
+# below is for Ubuntu 18.04 with specified PPA enabled, if you are using
+# other distro or different repositories, change the following accordingly
+DLL_PATH = [
+    '/usr/x86_64-w64-mingw32/bin/',
+    '/usr/x86_64-w64-mingw32/lib/',
+    '/usr/lib/gcc/x86_64-w64-mingw32/7.3-posix/'
+]
+
+missing = []
+
+
+def parse_imports(file_name):
+    results = []
+    pe = pefile.PE(file_name, fast_load=True)
+    pe.parse_data_directories()
+
+    for entry in pe.DIRECTORY_ENTRY_IMPORT:
+        current = entry.dll.decode()
+        current_u = current.upper()  # b/c Windows is often case insensitive
+        # here we filter out system dlls
+        # dll w/ names like *32.dll are likely to be system dlls
+        if current_u.upper() not in KNOWN_SYS_DLLS and not re.match(string=current_u, pattern=r'.*32\.DLL'):
+            results.append(current)
+
+    return results
+
+
+def parse_imports_recursive(file_name, path_list=[]):
+    q = queue.Queue()  # create a FIFO queue
+    # file_name can be a string or a list for the convience
+    if isinstance(file_name, str):
+        q.put(file_name)
+    elif isinstance(file_name, list):
+        for i in file_name:
+            q.put(i)
+    full_list = []
+    while q.qsize():
+        current = q.get_nowait()
+        print('> %s' % current)
+        deps = parse_imports(current)
+        # if this dll does not have any import, ignore it
+        if not deps:
+            continue
+        for dep in deps:
+            # the dependency already included in the list, skip
+            if dep in full_list:
+                continue
+            # find the requested dll in the provided paths
+            full_path = find_dll(dep)
+            if not full_path:
+                missing.append(dep)
+                continue
+            full_list.append(dep)
+            q.put(full_path)
+            path_list.append(full_path)
+    return full_list
+
+
+def find_dll(name):
+    for path in DLL_PATH:
+        for root, _, files in os.walk(path):
+            for f in files:
+                if name.lower() == f.lower():
+                    return os.path.join(root, f)
+
+
+def deploy(name, dst, dry_run=False):
+    dlls_path = []
+    parse_imports_recursive(name, dlls_path)
+    for dll_entry in dlls_path:
+        if not dry_run:
+            shutil.copy(dll_entry, dst)
+        else:
+            print('[Dry-Run] Copy %s to %s' % (dll_entry, dst))
+    print('Deploy completed.')
+    return dlls_path
+
+
+def main():
+    if len(sys.argv) < 3:
+        print('Usage: %s [files to examine ...] [target deploy directory]')
+        return 1
+    to_deploy = sys.argv[1:-1]
+    tgt_dir = sys.argv[-1]
+    if not os.path.isdir(tgt_dir):
+        print('%s is not a directory.' % tgt_dir)
+        return 1
+    print('Scanning dependencies...')
+    deploy(to_deploy, tgt_dir)
+    if missing:
+        print('Following DLLs are not found: %s' % ('\n'.join(missing)))
+    return 0
+
+
+if __name__ == '__main__':
+    main()

.travis/linux-mingw/upload.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -ex
+
+. .travis/common/pre-upload.sh
+
+REV_NAME="citra-windows-mingw-${GITDATE}-${GITREV}"
+ARCHIVE_NAME="${REV_NAME}.tar.gz"
+COMPRESSION_FLAGS="-czvf"
+
+mkdir "$REV_NAME"
+# get around the permission issues
+cp -r package/* "$REV_NAME"
+
+. .travis/common/post-upload.sh

CMakeModules/MinGWCross.cmake

@@ -0,0 +1,54 @@
+SET(MINGW_PREFIX   /usr/x86_64-w64-mingw32/)
+SET(CMAKE_SYSTEM_NAME               Windows)
+SET(CMAKE_SYSTEM_PROCESSOR           x86_64)
+# Actually a hack, w/o this will cause some strange errors
+SET(CMAKE_HOST_WIN32                 TRUE)
+
+
+SET(CMAKE_FIND_ROOT_PATH            ${MINGW_PREFIX})
+SET(SDL2_PATH                       ${MINGW_PREFIX})
+SET(MINGW_TOOL_PREFIX               ${CMAKE_SYSTEM_PROCESSOR}-w64-mingw32-)
+
+# Specify the cross compiler
+SET(CMAKE_C_COMPILER            ${MINGW_TOOL_PREFIX}gcc-posix)
+SET(CMAKE_CXX_COMPILER          ${MINGW_TOOL_PREFIX}g++-posix)
+SET(CMAKE_RC_COMPILER           ${MINGW_TOOL_PREFIX}windres)
+
+# Mingw tools
+SET(STRIP                       ${MINGW_TOOL_PREFIX}strip)
+SET(WINDRES                     ${MINGW_TOOL_PREFIX}windres)
+SET(ENV{PKG_CONFIG}             ${MINGW_TOOL_PREFIX}pkg-config)
+
+# ccache wrapper
+OPTION(USE_CCACHE "Use ccache for compilation" OFF)
+IF(USE_CCACHE)
+    FIND_PROGRAM(CCACHE ccache)
+    IF (CCACHE)
+        MESSAGE(STATUS "Using ccache found in PATH")
+        SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE})
+        SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE})
+    ELSE(CCACHE)
+        MESSAGE(WARNING "USE_CCACHE enabled, but no ccache found")
+    ENDIF(CCACHE)
+ENDIF(USE_CCACHE)
+
+# Search for programs in the build host directories
+SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+
+
+# Echo modified cmake vars to screen for debugging purposes
+IF(NOT DEFINED ENV{MINGW_DEBUG_INFO})
+	MESSAGE("")
+	MESSAGE("Custom cmake vars: (blank = system default)")
+	MESSAGE("-----------------------------------------")
+	MESSAGE("* CMAKE_C_COMPILER                     : ${CMAKE_C_COMPILER}")
+	MESSAGE("* CMAKE_CXX_COMPILER                   : ${CMAKE_CXX_COMPILER}")
+	MESSAGE("* CMAKE_RC_COMPILER                    : ${CMAKE_RC_COMPILER}")
+	MESSAGE("* WINDRES                              : ${WINDRES}")
+	MESSAGE("* ENV{PKG_CONFIG}                      : $ENV{PKG_CONFIG}")
+	MESSAGE("* STRIP                                : ${STRIP}")
+  MESSAGE("* USE_CCACHE                           : ${USE_CCACHE}")
+	MESSAGE("")
+	# So that the debug info only appears once
+	SET(ENV{MINGW_DEBUG_INFO} SHOWN)
+ENDIF()

appveyor.yml

@@ -171,17 +171,6 @@ artifacts:
   - path: $(BUILD_UPDATE)
     name: update
 
-deploy:
-  provider: GitHub
-  release: $(appveyor_repo_tag_name)
-  auth_token:
-    secure: "dbpsMC/MgPKWFNJCXpQl4cR8FYhepkPLjgNp/pRMktZ8oLKTqPYErfreaIxb/4P1"
-  artifact: update,build
-  draft: false
-  prerelease: false
-  on:
-    appveyor_repo_tag: true
-
 notifications:
   - provider: Webhook
     url: https://api.citra-emu.org/code/appveyor/notify

externals/libressl

@@ -1 +1 @@
-Subproject commit cbb1dcab9a566d0aa0036339f21f2e37012b2507
+Subproject commit 7d01cb01cb1a926ecb4c9c98b107ef3c26f59dfb