diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 8f67136..7365969 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -49,66 +49,80 @@ extension FileManager { } } -final class EndToEndTests: XCTestCase { - private let testcases = [ - #""" - // Default program generated by swift package init - print("Hello, world!") - """#, - #""" - // Check that libc_nonshared.a is linked properly - import Foundation - - func fin() -> Void { - print("exiting") - } - - atexit(fin) - """#, - ] - - private let logger = Logger(label: "swift-sdk-generator") - - // Building an SDK requires running the sdk-generator with `swift run swift-sdk-generator`. - // This takes a lock on `.build`, but if the tests are being run by `swift test` the outer Swift Package Manager - // instance will already hold this lock, causing the test to deadlock. We can work around this by giving - // the `swift run swift-sdk-generator` instance its own scratch directory. - func buildSDK(inDirectory packageDirectory: FilePath, scratchPath: String, withArguments runArguments: String) async throws -> String { - let generatorOutput = try await Shell.readStdout( - "cd \(packageDirectory) && swift run --scratch-path \"\(scratchPath)\" swift-sdk-generator make-linux-sdk \(runArguments)" - ) - - let installCommand = try XCTUnwrap(generatorOutput.split(separator: "\n").first { - $0.contains("swift experimental-sdk install") - }) +// Building an SDK requires running the sdk-generator with `swift run swift-sdk-generator`. +// This takes a lock on `.build`, but if the tests are being run by `swift test` the outer Swift Package Manager +// instance will already hold this lock, causing the test to deadlock. We can work around this by giving +// the `swift run swift-sdk-generator` instance its own scratch directory. +func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: String) async throws -> String { + var logger = logger + logger[metadataKey: "runArguments"] = "\"\(runArguments)\"" + logger[metadataKey: "scratchPath"] = "\(scratchPath)" + + logger.info("Building SDK") + + var packageDirectory = FilePath(#filePath) + packageDirectory.removeLastComponent() + packageDirectory.removeLastComponent() + + let generatorOutput = try await Shell.readStdout( + "cd \(packageDirectory) && swift run --scratch-path \"\(scratchPath)\" swift-sdk-generator make-linux-sdk \(runArguments)" + ) + logger.info("Finished building SDK") + + let installCommand = try XCTUnwrap(generatorOutput.split(separator: "\n").first { + $0.contains("swift experimental-sdk install") + }) + + let bundleName = try XCTUnwrap( + FilePath(String(XCTUnwrap(installCommand.split(separator: " ").last))).components.last + ).stem + logger[metadataKey: "bundleName"] = "\(bundleName)" + + logger.info("Checking installed SDKs") + let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components(separatedBy: "\n") + + // Make sure this bundle hasn't been installed already. + if installedSDKs.contains(bundleName) { + logger.info("Removing existing SDK") + try await Shell.run("swift experimental-sdk remove \(bundleName)") + } - let bundleName = try XCTUnwrap( - FilePath(String(XCTUnwrap(installCommand.split(separator: " ").last))).components.last - ).stem + logger.info("Installing new SDK") + let installOutput = try await Shell.readStdout(String(installCommand)) + XCTAssertTrue(installOutput.contains("successfully installed")) - let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components(separatedBy: "\n") + return bundleName +} - // Make sure this bundle hasn't been installed already. - if installedSDKs.contains(bundleName) { - try await Shell.run("swift experimental-sdk remove \(bundleName)") +private let testcases = [ + #""" + // Default program generated by swift package init + print("Hello, world!") + """#, + #""" + // Check that libc_nonshared.a is linked properly + import Foundation + + func fin() -> Void { + print("exiting") } - let installOutput = try await Shell.readStdout(String(installCommand)) - XCTAssertTrue(installOutput.contains("successfully installed")) + atexit(fin) + """#, +] - return bundleName - } +final class RepeatedBuildTests: XCTestCase { + private let logger = Logger(label: "swift-sdk-generator") - func testPackageInitExecutable() async throws { + func testRepeatedSDKBuilds() async throws { if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") } - var packageDirectory = FilePath(#filePath) - packageDirectory.removeLastComponent() - packageDirectory.removeLastComponent() + var logger = logger + logger[metadataKey: "testcase"] = "testRepeatedSDKBuilds" - // Do multiple runs with different sets of arguments. + // Test that an existing SDK can be rebuilt without cleaning up. // Test with no arguments by default: var possibleArguments = [""] do { @@ -125,66 +139,268 @@ final class EndToEndTests: XCTestCase { continue } - let bundleName = try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - try await buildSDK(inDirectory: packageDirectory, scratchPath: tempDir.path, withArguments: runArguments) + try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in + let _ = try await buildSDK(logger, scratchPath: tempDir.path, withArguments: runArguments) + let _ = try await buildSDK(logger, scratchPath: tempDir.path, withArguments: runArguments) } + } + } +} - for testcase in self.testcases { - try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - let testPackageURL = tempDir.appendingPathComponent("swift-sdk-generator-test") - let testPackageDir = FilePath(testPackageURL.path) - try FileManager.default.createDirectory(atPath: testPackageDir.string, withIntermediateDirectories: true) +// SDKConfiguration represents an SDK build configuration and can construct the corresponding SDK generator arguments +struct SDKConfiguration { + var swiftVersion: String + var linuxDistributionName: String + var architecture: String + var withDocker: Bool - try await Shell.run("swift package --package-path \(testPackageDir) init --type executable") - let main_swift = testPackageURL.appendingPathComponent("Sources/main.swift") - try testcase.write(to: main_swift, atomically: true, encoding: .utf8) + var bundleName: String { "\(linuxDistributionName)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" } - var buildOutput = try await Shell.readStdout( - "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName)" - ) - XCTAssertTrue(buildOutput.contains("Build complete!")) + func withDocker(_ enabled: Bool = true) -> SDKConfiguration { + var res = self + res.withDocker = enabled + return res + } - try await Shell.run("rm -rf \(testPackageDir.appending(".build"))") + func withArchitecture(_ arch: String) -> SDKConfiguration { + var res = self + res.architecture = arch + return res + } - buildOutput = try await Shell.readStdout( - "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName) --static-swift-stdlib" - ) - XCTAssertTrue(buildOutput.contains("Build complete!")) - } - } - } + var sdkGeneratorArguments: String { + return [ + "--sdk-name \(bundleName)", + withDocker ? "--with-docker" : nil, + "--swift-version \(swiftVersion)-RELEASE", + "--target \(architecture)-unknown-linux-gnu", + "--linux-distribution-name \(linuxDistributionName)" + ].compactMap{ $0 }.joined(separator: " ") } +} - func testRepeatedSDKBuilds() async throws { - if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { - throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") - } +// Skip slow tests unless an environment variable is set +func skipSlow() throws { + try XCTSkipUnless( + ProcessInfo.processInfo.environment.keys.contains("SWIFT_SDK_GENERATOR_RUN_SLOW_TESTS"), + "Skipping slow test because SWIFT_SDK_GENERATOR_RUN_SLOW_TESTS is not set" + ) +} - var packageDirectory = FilePath(#filePath) - packageDirectory.removeLastComponent() - packageDirectory.removeLastComponent() +// Skip known failing tests unless an environment variable is set +func skipBroken(_ message: String) throws { + try XCTSkipUnless( + ProcessInfo.processInfo.environment.keys.contains("SWIFT_SDK_GENERATOR_RUN_BROKEN_TESTS"), + "Skipping broken test because SWIFT_SDK_GENERATOR_RUN_BROKEN_TESTS is not set: \(message)" + ) +} - // Test that an existing SDK can be rebuilt without cleaning up. - // Test with no arguments by default: - var possibleArguments = [""] +func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempDir: URL) async throws { + let testPackageURL = tempDir.appendingPathComponent("swift-sdk-generator-test") + let testPackageDir = FilePath(testPackageURL.path) + try FileManager.default.createDirectory(atPath: testPackageDir.string, withIntermediateDirectories: true) + + logger.info("Creating test project") + try await Shell.run("swift package --package-path \(testPackageDir) init --type executable") + let main_swift = testPackageURL.appendingPathComponent("Sources/main.swift") + try testcase.write(to: main_swift, atomically: true, encoding: .utf8) + + logger.info("Building test project") + var buildOutput = try await Shell.readStdout( + "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName)" + ) + XCTAssertTrue(buildOutput.contains("Build complete!")) + logger.info("Test project built successfully") + + try await Shell.run("rm -rf \(testPackageDir.appending(".build"))") + + logger.info("Building test project with static-swift-stdlib") + buildOutput = try await Shell.readStdout( + "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName) --static-swift-stdlib" + ) + XCTAssertTrue(buildOutput.contains("Build complete!")) + logger.info("Test project built successfully") +} + +func buildTestcases(config: SDKConfiguration) async throws { + var logger = Logger(label: "EndToEndTests") + logger[metadataKey: "testcase"] = "testPackageInitExecutable" + + if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { + throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") + } + + if config.withDocker { do { try await Shell.run("docker ps") - possibleArguments.append("--with-docker --linux-distribution-name rhel --linux-distribution-version ubi9") } catch { - self.logger.warning("Docker CLI does not seem to be working, skipping tests that involve Docker.") + throw XCTSkip("Container runtime is not available - skipping tests which require it") } + } - for runArguments in possibleArguments { - if runArguments.contains("rhel") { - // Temporarily skip the RHEL-based SDK. XCTSkip() is not suitable as it would skipping the entire test case - logger.warning("RHEL-based SDKs currently do not work with Swift 6.0: https://github.com/swiftlang/swift-sdk-generator/issues/138") - continue - } + let bundleName = try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in + try await buildSDK(logger, scratchPath: tempDir.path, withArguments: config.sdkGeneratorArguments) + } - try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - let _ = try await buildSDK(inDirectory: packageDirectory, scratchPath: tempDir.path, withArguments: runArguments) - let _ = try await buildSDK(inDirectory: packageDirectory, scratchPath: tempDir.path, withArguments: runArguments) - } + logger.info("Built SDK") + + for testcase in testcases { + try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in + try await buildTestcase(logger, testcase: testcase, bundleName: bundleName, tempDir: tempDir) } } } + +final class Swift59_UbuntuEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.9.2", + linuxDistributionName: "ubuntu", + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/147") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +} + +final class Swift510_UbuntuEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.10.1", + linuxDistributionName: "ubuntu", + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/147") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +} + +final class Swift60_UbuntuEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.0.2", + linuxDistributionName: "ubuntu", + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +} + +final class Swift59_RHELEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.9.2", + linuxDistributionName: "rhel", + architecture: "aarch64", + withDocker: true // RHEL-based SDKs can only be built from containers + ) + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/147") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +} + +final class Swift510_RHELEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.10.1", + linuxDistributionName: "rhel", + architecture: "aarch64", + withDocker: true // RHEL-based SDKs can only be built from containers + ) + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/147") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +} + +final class Swift60_RHELEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.0.2", + linuxDistributionName: "rhel", + architecture: "aarch64", + withDocker: true // RHEL-based SDKs can only be built from containers + ) + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/147") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +}