diff --git a/Sources/SwiftBuildSupport/PIFBuilder.swift b/Sources/SwiftBuildSupport/PIFBuilder.swift index 6e304910499..37b626d3e53 100644 --- a/Sources/SwiftBuildSupport/PIFBuilder.swift +++ b/Sources/SwiftBuildSupport/PIFBuilder.swift @@ -202,8 +202,11 @@ public final class PIFBuilder { return accessibleToolsPerPlugin } - /// Constructs a `PIF.TopLevelObject` representing the package graph. - package func constructPIF(buildParameters: BuildParameters) async throws -> PIF.TopLevelObject { + /// Constructs all `PackagePIFBuilder` objects used by the `constructPIF` function. + /// In particular, this is useful for unit testing the complex `PIFBuilder`class. + package func makePIFBuilders( + buildParameters: BuildParameters + ) async throws -> [(ResolvedPackage, PackagePIFBuilder, any PackagePIFBuilder.BuildDelegate)] { let pluginScriptRunner = self.parameters.pluginScriptRunner let outputDir = self.parameters.pluginWorkingDirectory.appending("outputs") @@ -218,225 +221,237 @@ public final class PIFBuilder { hostTriple: try pluginScriptRunner.hostTriple ) - return try await memoize(to: &self.cachedPIF) { - guard let rootPackage = self.graph.rootPackages.only else { - if self.graph.rootPackages.isEmpty { - throw PIFGenerationError.rootPackageNotFound - } else { - throw PIFGenerationError.multipleRootPackagesFound - } - } - - let sortedPackages = self.graph.packages - .sorted { $0.manifest.displayName < $1.manifest.displayName } // TODO: use identity instead? - - var packagesAndProjects: [(ResolvedPackage, ProjectModel.Project)] = [] - - for package in sortedPackages { - var buildToolPluginResultsByTargetName: [String: [PackagePIFBuilder.BuildToolPluginInvocationResult]] = [:] - - for module in package.modules { - // Apply each build tool plugin used by the target in order, - // creating a list of results (one for each plugin usage). - var buildToolPluginResults: [BuildToolPluginInvocationResult] = [] - var buildCommands: [PackagePIFBuilder.CustomBuildCommand] = [] - var prebuildCommands: [BuildToolPluginInvocationResult.PrebuildCommand] = [] - - for plugin in module.pluginDependencies(satisfying: buildParameters.buildEnvironment) { - let pluginModule = plugin.underlying as! PluginModule - - // Determine the tools to which this plugin has access, and create a name-to-path mapping from tool - // names to the corresponding paths. Built tools are assumed to be in the build tools directory. - guard let accessibleTools = availablePluginTools[plugin.id] else { - throw InternalError("No tools found for plugin \(plugin.name)") - } + let sortedPackages = self.graph.packages + .sorted { $0.manifest.displayName < $1.manifest.displayName } // TODO: use identity instead? - // Assign a plugin working directory based on the package, target, and plugin. - let pluginOutputDir = outputDir.appending( - components: [ - package.identity.description, - module.name, - buildParameters.destination == .host ? "tools" : "destination", - plugin.name, - ] - ) + var packagesAndBuilders: [(ResolvedPackage, PackagePIFBuilder, any PackagePIFBuilder.BuildDelegate)] = [] - // Determine the set of directories under which plugins are allowed to write. - // We always include just the output directory, and for now there is no possibility - // of opting into others. - let writableDirectories = [outputDir] - - // Determine a set of further directories under which plugins are never allowed - // to write, even if they are covered by other rules (such as being able to write - // to the temporary directory). - let readOnlyDirectories = [package.path] - - // In tools version 6.0 and newer, we vend the list of files generated by previous plugins. - let pluginDerivedSources: Sources - let pluginDerivedResources: [Resource] - if package.manifest.toolsVersion >= .v6_0 { - // Set up dummy observability because we don't want to emit diagnostics for this before the actual - // build. - let observability = ObservabilitySystem { _, _ in } - // Compute the generated files based on all results we have computed so far. - (pluginDerivedSources, pluginDerivedResources) = ModulesGraph.computePluginGeneratedFiles( - target: module, - toolsVersion: package.manifest.toolsVersion, - additionalFileRules: self.parameters.additionalFileRules, - buildParameters: buildParameters, - buildToolPluginInvocationResults: buildToolPluginResults, - prebuildCommandResults: [], - observabilityScope: observability.topScope - ) - } else { - pluginDerivedSources = .init(paths: [], root: package.path) - pluginDerivedResources = [] - } + for package in sortedPackages { + var buildToolPluginResultsByTargetName: [String: [PackagePIFBuilder.BuildToolPluginInvocationResult]] = [:] - let result = try await pluginModule.invoke( - module: plugin, - action: .createBuildToolCommands( - package: package, - target: module, - pluginGeneratedSources: pluginDerivedSources.paths, - pluginGeneratedResources: pluginDerivedResources.map(\.path) - ), - buildEnvironment: buildParameters.buildEnvironment, - scriptRunner: pluginScriptRunner, - workingDirectory: package.path, - outputDirectory: pluginOutputDir, - toolSearchDirectories: [buildParameters.toolchain.swiftCompilerPath.parentDirectory], - accessibleTools: accessibleTools, - writableDirectories: writableDirectories, - readOnlyDirectories: readOnlyDirectories, - allowNetworkConnections: [], - pkgConfigDirectories: self.parameters.pkgConfigDirectories, - sdkRootPath: buildParameters.toolchain.sdkRootPath, - fileSystem: fileSystem, - modulesGraph: self.graph, - observabilityScope: observabilityScope - ) + for module in package.modules { + // Apply each build tool plugin used by the target in order, + // creating a list of results (one for each plugin usage). + var buildToolPluginResults: [BuildToolPluginInvocationResult] = [] + var buildCommands: [PackagePIFBuilder.CustomBuildCommand] = [] + var prebuildCommands: [BuildToolPluginInvocationResult.PrebuildCommand] = [] - buildToolPluginResults.append(result) + for plugin in module.pluginDependencies(satisfying: buildParameters.buildEnvironment) { + let pluginModule = plugin.underlying as! PluginModule - let diagnosticsEmitter = observabilityScope.makeDiagnosticsEmitter { - var metadata = ObservabilityMetadata() - metadata.moduleName = module.name - metadata.pluginName = result.plugin.name - return metadata - } - - for line in result.textOutput.split(whereSeparator: { $0.isNewline }) { - diagnosticsEmitter.emit(info: line) - } + // Determine the tools to which this plugin has access, and create a name-to-path mapping from tool + // names to the corresponding paths. Built tools are assumed to be in the build tools directory. + guard let accessibleTools = availablePluginTools[plugin.id] else { + throw InternalError("No tools found for plugin \(plugin.name)") + } - for diag in result.diagnostics { - diagnosticsEmitter.emit(diag) - } + // Assign a plugin working directory based on the package, target, and plugin. + let pluginOutputDir = outputDir.appending( + components: [ + package.identity.description, + module.name, + buildParameters.destination == .host ? "tools" : "destination", + plugin.name, + ] + ) - prebuildCommands.append(contentsOf: result.prebuildCommands) - - buildCommands.append(contentsOf: result.buildCommands.map( { buildCommand in - var newEnv: Environment = buildCommand.configuration.environment - - // FIXME: This is largely a workaround for improper rpath setup on Linux. It should be - // removed once the Swift Build backend switches to use swiftc as the linker driver - // for targets with Swift sources. For now, limit the scope to non-macOS, so that - // plugins do not inadvertently use the toolchain stdlib instead of the OS stdlib - // when built with a Swift.org toolchain. - #if !os(macOS) - let runtimeLibPaths = buildParameters.toolchain.runtimeLibraryPaths - - // Add paths to swift standard runtime libraries to the library path so that they can be found at runtime - for libPath in runtimeLibPaths { - newEnv.appendPath(key: .libraryPath, value: libPath.pathString) - } - #endif - - // Append the system path at the end so that necessary system tool paths can be found - if let pathValue = Environment.current[EnvironmentKey.path] { - newEnv.appendPath(key: .path, value: pathValue) - } - - let writableDirectories: [AbsolutePath] = [pluginOutputDir] - - return PackagePIFBuilder.CustomBuildCommand( - displayName: buildCommand.configuration.displayName, - executable: buildCommand.configuration.executable.pathString, - arguments: buildCommand.configuration.arguments, - environment: .init(newEnv), - workingDir: package.path, - inputPaths: buildCommand.inputFiles, - outputPaths: buildCommand.outputFiles.map(\.pathString), - sandboxProfile: - self.parameters.disableSandbox ? - nil : - .init( - strictness: .writableTemporaryDirectory, - writableDirectories: writableDirectories, - readOnlyDirectories: buildCommand.inputFiles - ) - ) - })) + // Determine the set of directories under which plugins are allowed to write. + // We always include just the output directory, and for now there is no possibility + // of opting into others. + let writableDirectories = [outputDir] + + // Determine a set of further directories under which plugins are never allowed + // to write, even if they are covered by other rules (such as being able to write + // to the temporary directory). + let readOnlyDirectories = [package.path] + + // In tools version 6.0 and newer, we vend the list of files generated by previous plugins. + let pluginDerivedSources: Sources + let pluginDerivedResources: [Resource] + if package.manifest.toolsVersion >= .v6_0 { + // Set up dummy observability because we don't want to emit diagnostics for this before the actual + // build. + let observability = ObservabilitySystem { _, _ in } + // Compute the generated files based on all results we have computed so far. + (pluginDerivedSources, pluginDerivedResources) = ModulesGraph.computePluginGeneratedFiles( + target: module, + toolsVersion: package.manifest.toolsVersion, + additionalFileRules: self.parameters.additionalFileRules, + buildParameters: buildParameters, + buildToolPluginInvocationResults: buildToolPluginResults, + prebuildCommandResults: [], + observabilityScope: observability.topScope + ) + } else { + pluginDerivedSources = .init(paths: [], root: package.path) + pluginDerivedResources = [] } - // Run the prebuild commands generated from the plugin invocation now for this module. This will - // also give use the derived source code files needed for PIF generation. - let runResults = try Self.runPluginCommands( - using: self.pluginConfiguration, - for: buildToolPluginResults, + let result = try await pluginModule.invoke( + module: plugin, + action: .createBuildToolCommands( + package: package, + target: module, + pluginGeneratedSources: pluginDerivedSources.paths, + pluginGeneratedResources: pluginDerivedResources.map(\.path) + ), + buildEnvironment: buildParameters.buildEnvironment, + scriptRunner: pluginScriptRunner, + workingDirectory: package.path, + outputDirectory: pluginOutputDir, + toolSearchDirectories: [buildParameters.toolchain.swiftCompilerPath.parentDirectory], + accessibleTools: accessibleTools, + writableDirectories: writableDirectories, + readOnlyDirectories: readOnlyDirectories, + allowNetworkConnections: [], + pkgConfigDirectories: self.parameters.pkgConfigDirectories, + sdkRootPath: buildParameters.toolchain.sdkRootPath, fileSystem: fileSystem, + modulesGraph: self.graph, observabilityScope: observabilityScope ) - let result = PackagePIFBuilder.BuildToolPluginInvocationResult( - prebuildCommandOutputPaths: runResults.flatMap( { $0.derivedFiles }), - buildCommands: buildCommands - ) + buildToolPluginResults.append(result) - // Add a BuildToolPluginInvocationResult to the mapping. - if var existingResults = buildToolPluginResultsByTargetName[module.name] { - existingResults.append(result) - } else { - buildToolPluginResultsByTargetName[module.name] = [result] + let diagnosticsEmitter = observabilityScope.makeDiagnosticsEmitter { + var metadata = ObservabilityMetadata() + metadata.moduleName = module.name + metadata.pluginName = result.plugin.name + return metadata + } + + for line in result.textOutput.split(whereSeparator: { $0.isNewline }) { + diagnosticsEmitter.emit(info: line) + } + + for diag in result.diagnostics { + diagnosticsEmitter.emit(diag) } + + prebuildCommands.append(contentsOf: result.prebuildCommands) + + buildCommands.append(contentsOf: result.buildCommands.map( { buildCommand in + var newEnv: Environment = buildCommand.configuration.environment + + // FIXME: This is largely a workaround for improper rpath setup on Linux. It should be + // removed once the Swift Build backend switches to use swiftc as the linker driver + // for targets with Swift sources. For now, limit the scope to non-macOS, so that + // plugins do not inadvertently use the toolchain stdlib instead of the OS stdlib + // when built with a Swift.org toolchain. + #if !os(macOS) + let runtimeLibPaths = buildParameters.toolchain.runtimeLibraryPaths + + // Add paths to swift standard runtime libraries to the library path so that they can be found at runtime + for libPath in runtimeLibPaths { + newEnv.appendPath(key: .libraryPath, value: libPath.pathString) + } + #endif + + // Append the system path at the end so that necessary system tool paths can be found + if let pathValue = Environment.current[EnvironmentKey.path] { + newEnv.appendPath(key: .path, value: pathValue) + } + + let writableDirectories: [AbsolutePath] = [pluginOutputDir] + + return PackagePIFBuilder.CustomBuildCommand( + displayName: buildCommand.configuration.displayName, + executable: buildCommand.configuration.executable.pathString, + arguments: buildCommand.configuration.arguments, + environment: .init(newEnv), + workingDir: package.path, + inputPaths: buildCommand.inputFiles, + outputPaths: buildCommand.outputFiles.map(\.pathString), + sandboxProfile: + self.parameters.disableSandbox ? + nil : + .init( + strictness: .writableTemporaryDirectory, + writableDirectories: writableDirectories, + readOnlyDirectories: buildCommand.inputFiles + ) + ) + })) } - let packagePIFBuilderDelegate = PackagePIFBuilderDelegate( - package: package + // Run the prebuild commands generated from the plugin invocation now for this module. This will + // also give use the derived source code files needed for PIF generation. + let runResults = try Self.runPluginCommands( + using: self.pluginConfiguration, + for: buildToolPluginResults, + fileSystem: fileSystem, + observabilityScope: observabilityScope ) - let packagePIFBuilder = PackagePIFBuilder( - modulesGraph: self.graph, - resolvedPackage: package, - packageManifest: package.manifest, - delegate: packagePIFBuilderDelegate, - buildToolPluginResultsByTargetName: buildToolPluginResultsByTargetName, - createDylibForDynamicProducts: self.parameters.shouldCreateDylibForDynamicProducts, - addLocalRpaths: self.parameters.addLocalRpaths, - packageDisplayVersion: package.manifest.displayName, - fileSystem: self.fileSystem, - observabilityScope: self.observabilityScope + + let result = PackagePIFBuilder.BuildToolPluginInvocationResult( + prebuildCommandOutputPaths: runResults.flatMap( { $0.derivedFiles }), + buildCommands: buildCommands ) - - try packagePIFBuilder.build() - packagesAndProjects.append((package, packagePIFBuilder.pifProject)) + + // Add a BuildToolPluginInvocationResult to the mapping. + if var existingResults = buildToolPluginResultsByTargetName[module.name] { + existingResults.append(result) + } else { + buildToolPluginResultsByTargetName[module.name] = [result] + } } - - var projects = packagesAndProjects.map(\.1) - projects.append( - try buildAggregateProject( - packagesAndProjects: packagesAndProjects, + + let packagePIFBuilderDelegate = PackagePIFBuilderDelegate( + package: package + ) + let packagePIFBuilder = PackagePIFBuilder( + modulesGraph: self.graph, + resolvedPackage: package, + packageManifest: package.manifest, + delegate: packagePIFBuilderDelegate, + buildToolPluginResultsByTargetName: buildToolPluginResultsByTargetName, + createDylibForDynamicProducts: self.parameters.shouldCreateDylibForDynamicProducts, + addLocalRpaths: self.parameters.addLocalRpaths, + packageDisplayVersion: package.manifest.displayName, + fileSystem: self.fileSystem, + observabilityScope: self.observabilityScope + ) + + packagesAndBuilders.append((package, packagePIFBuilder, packagePIFBuilderDelegate)) + } + + return packagesAndBuilders + } + + /// Constructs a `PIF.TopLevelObject` representing the package graph. + package func constructPIF(buildParameters: BuildParameters) async throws -> PIF.TopLevelObject { + return try await memoize(to: &self.cachedPIF) { + let packagesAndPIFBuilders = try await makePIFBuilders(buildParameters: buildParameters) + + let packagesAndPIFProjects = try packagesAndPIFBuilders.map { (package, pifBuilder, _) in + try pifBuilder.build() + let pifProject: ProjectModel.Project = pifBuilder.pifProject + return (package, pifProject) + } + + var pifProjects: [ProjectModel.Project] = packagesAndPIFProjects.map(\.1) + pifProjects.append( + try buildAggregatePIFProject( + packagesAndProjects: packagesAndPIFProjects, observabilityScope: observabilityScope, modulesGraph: graph, buildParameters: buildParameters ) ) + guard let rootPackage = self.graph.rootPackages.only else { + if self.graph.rootPackages.isEmpty { + throw PIFGenerationError.rootPackageNotFound + } else { + throw PIFGenerationError.multipleRootPackagesFound + } + } + let workspace = PIF.Workspace( id: "Workspace:\(rootPackage.path.pathString)", name: rootPackage.manifest.displayName, // TODO: use identity instead? path: rootPackage.path, - projects: projects + projects: pifProjects ) return PIF.TopLevelObject(workspace: workspace) @@ -625,7 +640,7 @@ fileprivate final class PackagePIFBuilderDelegate: PackagePIFBuilder.BuildDelega } } -fileprivate func buildAggregateProject( +fileprivate func buildAggregatePIFProject( packagesAndProjects: [(package: ResolvedPackage, project: ProjectModel.Project)], observabilityScope: ObservabilityScope, modulesGraph: ModulesGraph,