[coverage] Automatic merger for LLVM profile data

CMakeLists.txt

@@ -58,6 +58,11 @@ option(SWIFT_INCLUDE_DOCS
     "Create targets for building docs."
     TRUE)
 
+set(SWIFT_ANALYZE_CODE_COVERAGE FALSE CACHE STRING
+    "Build Swift with code coverage instrumenting enabled [FALSE, NOT-MERGED, MERGED]")
+set_property(CACHE SWIFT_ANALYZE_CODE_COVERAGE PROPERTY
+    STRINGS FALSE "NOT-MERGED" "MERGED")
+
 set(SWIFT_VERSION "3.0" CACHE STRING
     "The user-visible version of the Swift compiler")
 set(SWIFT_VENDOR "" CACHE STRING

cmake/modules/AddSwift.cmake

@@ -64,7 +64,7 @@ function(_add_variant_c_compile_link_flags
         "-m${SWIFT_SDK_${sdk}_VERSION_MIN_NAME}-version-min=${SWIFT_SDK_${sdk}_DEPLOYMENT_VERSION}")
 
     if(analyze_code_coverage)
-      list(APPEND result "-fprofile-instr-generate=swift-%p.profraw"
+      list(APPEND result "-fprofile-instr-generate"
                          "-fcoverage-mapping")
     endif()
   endif()
@@ -111,7 +111,7 @@ function(_add_variant_c_compile_flags
   endif()
 
   if(analyze_code_coverage)
-    list(APPEND result "-fprofile-instr-generate=swift-%p.profraw"
+    list(APPEND result "-fprofile-instr-generate"
                        "-fcoverage-mapping")
   endif()
 

test/CMakeLists.txt

@@ -117,6 +117,7 @@ if(PYTHONINTERP_FOUND)
 
   set(TEST_MODES optimize_none optimize optimize_unchecked)
 
+
   foreach(SDK ${SWIFT_SDKS})
     foreach(ARCH ${SWIFT_SDK_${SDK}_ARCHITECTURES})
       foreach(TEST_MODE ${TEST_MODES})
@@ -196,30 +197,52 @@ if(PYTHONINTERP_FOUND)
             "${CMAKE_CURRENT_SOURCE_DIR}/../validation-test/lit.site.cfg.in"
             "${validation_test_bin_dir}/lit.site.cfg"
             "validation-test${VARIANT_SUFFIX}.lit.site.cfg")
+        set(profdata_merge_worker
+            "${CMAKE_CURRENT_SOURCE_DIR}/../utils/profdata_merge/main.py")
+
+        if(SWIFT_ANALYZE_CODE_COVERAGE STREQUAL "MERGED")
+          set(command_profdata_merge_start
+            COMMAND "${PYTHON_EXECUTABLE}" "${profdata_merge_worker}" start
+                -o "${swift_test_results_dir}"
+                -l "${swift_test_results_dir}/profdata_merge.log")
+          set(command_profdata_merge_stop
+            COMMAND "${PYTHON_EXECUTABLE}" "${profdata_merge_worker}" stop)
+        else()
+          set(command_profdata_merge_start)
+          set(command_profdata_merge_stop)
+        endif()
 
         add_custom_target("check-swift${test_mode_target_suffix}${VARIANT_SUFFIX}"
             ${command_upload_stdlib}
             ${command_clean_test_results_dir}
+            ${command_profdata_merge_start}
             COMMAND ${lit_command} "${test_bin_dir}"
+            ${command_profdata_merge_stop}
             DEPENDS ${test_dependencies}
             COMMENT "Running Swift tests for ${VARIANT_TRIPLE}"
             ${cmake_3_2_USES_TERMINAL})
 
         add_custom_target("check-swift-validation${test_mode_target_suffix}${VARIANT_SUFFIX}"
             ${command_upload_stdlib}
             ${command_clean_test_results_dir}
+            ${command_profdata_merge_start}
             COMMAND ${lit_command} "${validation_test_bin_dir}"
+            ${command_profdata_merge_stop}
             DEPENDS ${test_dependencies} ${validation_test_dependencies}
             COMMENT "Running Swift validation tests for ${VARIANT_TRIPLE}"
             ${cmake_3_2_USES_TERMINAL})
 
         add_custom_target("check-swift-all${test_mode_target_suffix}${VARIANT_SUFFIX}"
             ${command_upload_stdlib}
             ${command_clean_test_results_dir}
+            ${command_profdata_merge_start}
             COMMAND ${lit_command} "${validation_test_bin_dir}" "${test_bin_dir}"
+            ${command_profdata_merge_stop}
             DEPENDS ${test_dependencies} ${validation_test_dependencies}
             COMMENT "Running all Swift tests for ${VARIANT_TRIPLE}"
             ${cmake_3_2_USES_TERMINAL})
+
+
       endforeach()
     endforeach()
   endforeach()
@@ -245,6 +268,7 @@ if(PYTHONINTERP_FOUND)
 
     add_custom_target(check-swift-all${test_mode_target_suffix}
         DEPENDS "check-swift-all${test_mode_target_suffix}${SWIFT_PRIMARY_VARIANT_SUFFIX}")
+
   endforeach()
 
 endif()

test/Unit/lit.site.cfg.in

@@ -8,5 +8,7 @@ config.build_mode = lit_config.params.get('build_mode', "@LLVM_BUILD_MODE@")
 config.swift_obj_root = "@SWIFT_BINARY_DIR@"
 config.target_triple = "@TARGET_TRIPLE@"
 
+config.coverage_mode = "@SWIFT_ANALYZE_CODE_COVERAGE@"
+
 # Let the main config do the real work.
 lit_config.load_config(config, "@SWIFT_SOURCE_DIR@/test/Unit/lit.cfg")

test/lit.cfg

@@ -24,7 +24,10 @@ import re
 import subprocess
 import sys
 import tempfile
+import socket
+import glob
 
+import lit
 import lit.formats
 import lit.util
 
@@ -129,6 +132,43 @@ if config.test_exec_root is None:
 
 ###
 
+class SwiftTest(lit.formats.ShTest, object):
+    def __init__(self, coverage_mode=None, execute_external=True):
+        super(SwiftTest, self).__init__(execute_external=execute_external)
+        if coverage_mode == "FALSE":
+            self.coverage_mode = None
+        else:
+            self.coverage_mode = coverage_mode
+
+    def profdir_for_test(self, test):
+        _, tmp_base = lit.TestRunner.getTempPaths(test)
+        return tmp_base + ".profdir"
+
+    def before_test(self, test, litConfig):
+        if self.coverage_mode:
+            profdir = self.profdir_for_test(test)
+            if not os.path.exists(profdir):
+                os.makedirs(profdir)
+
+            test.config.environment["LLVM_PROFILE_FILE"] = \
+                os.path.join(profdir, "swift-%p.profraw")
+
+    def after_test(self, test, litConfig, result):
+        if self.coverage_mode == "MERGED":
+            profdir = self.profdir_for_test(test)
+            files = glob.glob(os.path.join(profdir, "swift-*.profraw"))
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.connect(('localhost', 12400))
+            sock.send("\n".join(files))
+            sock.close()
+        return result
+
+
+    def execute(self, test, litConfig):
+        self.before_test(test, litConfig)
+        result = super(SwiftTest, self).execute(test, litConfig)
+        return self.after_test(test, litConfig, result)
+
 # name: The name of this test suite.
 config.name = 'Swift'
 
@@ -138,7 +178,7 @@ if platform.system() == 'Darwin':
     config.environment['TOOLCHAINS'] = 'default'
 
 # testFormat: The test format to use to interpret tests.
-config.test_format = lit.formats.ShTest(execute_external=True)
+config.test_format = SwiftTest(coverage_mode=config.coverage_mode)
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = ['.swift', '.ll', '.sil', '.gyb', '.m']

test/lit.site.cfg.in

@@ -18,6 +18,8 @@ config.variant_sdk = "@VARIANT_SDK@"
 config.swiftlib_dir = "@LIT_SWIFTLIB_DIR@"
 config.darwin_xcrun_toolchain = "@SWIFT_DARWIN_XCRUN_TOOLCHAIN@"
 
+config.coverage_mode = "@SWIFT_ANALYZE_CODE_COVERAGE@"
+
 if "@SWIFT_ASAN_BUILD@" == "TRUE":
     config.available_features.add("asan")
 else:
@@ -42,11 +44,6 @@ if "@SWIFT_OPTIMIZED@" == "TRUE":
 if "@SWIFT_HAVE_WORKING_STD_REGEX@" == "FALSE":
     config.available_features.add('broken_std_regex')
 
-if "@SWIFT_ANALYZE_CODE_COVERAGE@" == "TRUE":
-    lit_config.useValgrind = True
-    lit_config.valgrindArgs = [os.path.join(config.swift_src_root,
-                                            "utils/use_profdir.py")]
-
 # Let the main config do the real work.
 if config.test_exec_root is None:
     config.test_exec_root = os.path.dirname(os.path.realpath(__file__))

tools/SourceKit/CMakeLists.txt

@@ -241,7 +241,7 @@ macro(add_sourcekit_executable name)
 
       if(SWIFT_ANALYZE_CODE_COVERAGE)
         set_property(TARGET "${name}" APPEND_STRING PROPERTY
-            LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+          LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
       endif()
     endif()
   endif()

tools/SourceKit/tools/complete-test/CMakeLists.txt

@@ -17,7 +17,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 
   if(SWIFT_ANALYZE_CODE_COVERAGE)
     set_property(TARGET complete-test APPEND_STRING PROPERTY
-        LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+      LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
   endif()
 endif()
 

tools/SourceKit/tools/sourcekitd-repl/CMakeLists.txt

@@ -11,7 +11,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 
   if(SWIFT_ANALYZE_CODE_COVERAGE)
     set_property(TARGET sourcekitd-repl APPEND_STRING PROPERTY
-        LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+      LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
   endif()
 endif()
 

tools/SourceKit/tools/sourcekitd-test/CMakeLists.txt

@@ -25,7 +25,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 
   if(SWIFT_ANALYZE_CODE_COVERAGE)
     set_property(TARGET sourcekitd-test APPEND_STRING PROPERTY
-        LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+      LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
   endif()
 endif()
 

unittests/CMakeLists.txt

@@ -29,7 +29,7 @@ function(add_swift_unittest test_dirname)
 
     if(SWIFT_ANALYZE_CODE_COVERAGE)
       set_property(TARGET "${test_dirname}" APPEND_STRING PROPERTY
-          LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+        LINK_FLAGS " -fprofile-instr-generate -fcoverage-mapping")
     endif()
   endif()
 endfunction()

utils/build-script

@@ -512,11 +512,11 @@ also build for Apple watchos, but disallow tests that require an watchOS device"
         action="store_true")
 
     parser.add_argument("--swift-analyze-code-coverage",
-        help="enable code coverage analysis in Swift",
-        action="store_const",
-        const=True,
-        dest="swift_analyze_code_coverage",
-        default=False)
+        help="""enable code coverage analysis in Swift (false, not-merged,
+        merged).""",
+        choices=["false", "not-merged", "merged"],
+        default="false", # so CMake can see the inert mode as a false value
+        dest="swift_analyze_code_coverage")
 
     parser.add_argument("--build-subdir",
         help="""
@@ -726,7 +726,7 @@ the number of parallel build jobs to use""",
         swift_build_dir_label = args.swift_build_variant
         if args.swift_assertions:
             swift_build_dir_label += "Assert"
-        if args.swift_analyze_code_coverage:
+        if args.swift_analyze_code_coverage != "false":
             swift_build_dir_label += "Coverage"
 
         swift_stdlib_build_dir_label = args.swift_stdlib_build_variant

utils/build-script-impl

@@ -71,7 +71,7 @@ KNOWN_SETTINGS=(
     llvm-enable-assertions      "1"              "enable assertions in LLVM and Clang"
     swift-build-type            "Debug"          "the CMake build variant for Swift"
     swift-enable-assertions     "1"              "enable assertions in Swift"
-    swift-analyze-code-coverage "0"              "enable code coverage analysis in Swift"
+    swift-analyze-code-coverage "not-merged"     "Code coverage analysis mode for Swift (false, not-merged, merged). Defaults to false if the argument is not present, and not-merged if the argument is present without a modifier."
     swift-stdlib-build-type     "Debug"          "the CMake build variant for Swift"
     swift-stdlib-enable-assertions "1"           "enable assertions in Swift"
     lldb-build-type             "Debug"          "the CMake build variant for LLDB"
@@ -1600,7 +1600,7 @@ for deployment_target in "${HOST_TARGET}" "${CROSS_TOOLS_DEPLOYMENT_TARGETS[@]}"
                     -DCMAKE_CXX_FLAGS="$(swift_c_flags ${deployment_target})"
                     -DCMAKE_BUILD_TYPE:STRING="${SWIFT_BUILD_TYPE}"
                     -DLLVM_ENABLE_ASSERTIONS:BOOL=$(true_false "${SWIFT_ENABLE_ASSERTIONS}")
-                    -DSWIFT_ANALYZE_CODE_COVERAGE:BOOL=$(true_false "${SWIFT_ANALYZE_CODE_COVERAGE}")
+                    -DSWIFT_ANALYZE_CODE_COVERAGE:STRING=$(toupper "${SWIFT_ANALYZE_CODE_COVERAGE}")
                     -DSWIFT_STDLIB_BUILD_TYPE:STRING="${SWIFT_STDLIB_BUILD_TYPE}"
                     -DSWIFT_STDLIB_ASSERTIONS:BOOL=$(true_false "${SWIFT_STDLIB_ENABLE_ASSERTIONS}")
                     -DSWIFT_NATIVE_LLVM_TOOLS_PATH:STRING="${native_llvm_tools_path}"

utils/profdata_merge/README.md

@@ -0,0 +1,17 @@
+# Profdata Merge
+
+Because LLVM's instrumented builds produce a `profraw` file every time a
+they are executed, the Swift test suite, when run with coverage enabled,
+produces thousands of 30MB profile files.
+
+When build-script is run with `--swift-analyze-code-coverage merged`, this
+module will get called before the tests run and spin up a small server that
+runs alongside the test suite. The server accepts filenames that it receives
+after each test directory is finished executing, and feeds them to 10 worker
+processes which concurrently merge those files together. Doing so means the
+test results directory doesn't balloon in size, as the files are merged together
+and removed often.
+
+Once the test suite is finished, it sends a crafted message to the server that
+tells it to merge any files left in the queue, then merges all the workers
+together into one final file.

utils/profdata_merge/config.py

@@ -0,0 +1,27 @@
+# utils/profdata_merge/config.py
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+# This file contains the data structure that transforms arguments into usable
+# values
+
+import tempfile
+import os
+
+
+class Config():
+    """A class to store configuration information specified by command-line
+    arguments."""
+    def __init__(self, out_dir, no_remove_files):
+        self.out_dir = out_dir
+        self.tmp_dir = tempfile.mkdtemp()
+        self.pid_file_path = os.path.join(self.out_dir,
+                                          "profdata_merge_worker.pid")
+        self.final_profdata_path = os.path.join(self.out_dir, "swift.profdata")
+        self.remove_files = not no_remove_files

utils/profdata_merge/main.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+# utils/profdata_merge/main.py
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+# This module is used to prevent profile data filling up available disk space
+# by listening for profile data and merging them into a universal profdata
+# file while tests are executing.
+# This file invokes the runner after parsing arguments.
+
+from __future__ import print_function
+import sys
+import argparse
+import tempfile
+import logging
+
+import runner
+
+SERVER_ADDRESS = ('localhost', 12400)
+TESTS_FINISHED_SENTINEL = "PROFDATA_MERGE_WORKER_TESTS_FINISHED_SENTINEL"
+
+if __name__ == "__main__":
+    if sys.platform != "darwin":
+        sys.exit("Error: The profile data merge worker requires OS X.")
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-l", "--log-file",
+                        help="The file to write logs in debug mode.")
+
+    subparsers = parser.add_subparsers()
+
+    start = subparsers.add_parser("start")
+    start.add_argument("-d", "--debug",
+                       help="Run in foreground and report status.",
+                       action="store_true")
+    start.add_argument("-o", "--output-dir",
+                       help=("The directory to write the PID file" +
+                             "and final profdata file."),
+                       default=tempfile.gettempdir())
+    start.add_argument("--no-remove",
+                       action="store_true",
+                       help="Don't remove profraw files after merging them.")
+    start.set_defaults(func=runner.start_server)
+
+    stop = subparsers.add_parser("stop")
+    stop.set_defaults(func=runner.stop_server)
+
+    args = parser.parse_args()
+
+    log_args = {'level': logging.DEBUG}
+    if args.log_file:
+        log_args['filename'] = args.log_file
+
+    logging.basicConfig(**log_args)
+
+    args.func(args)