diff --git a/include/swift/AST/FineGrainedDependencies.h b/include/swift/AST/FineGrainedDependencies.h index 49f0b755f9e39..5f3424a189e84 100644 --- a/include/swift/AST/FineGrainedDependencies.h +++ b/include/swift/AST/FineGrainedDependencies.h @@ -721,7 +721,8 @@ class SourceFileDepGraphNode : public DepGraphNode { } std::string humanReadableName() const { - return DepGraphNode::humanReadableName("here"); + return DepGraphNode::humanReadableName(getIsProvides() ? "here" + : "somewhere else"); } bool verify() const { @@ -883,6 +884,8 @@ class SourceFileDepGraph { bool verifySequenceNumber() const; + void emitDotFile(StringRef outputPath, DiagnosticEngine &diags); + private: void addNode(SourceFileDepGraphNode *n) { n->setSequenceNumber(allNodes.size()); diff --git a/include/swift/Driver/FineGrainedDependencyDriverGraph.h b/include/swift/Driver/FineGrainedDependencyDriverGraph.h index 9d1e9954d2c15..36ab089d6c029 100644 --- a/include/swift/Driver/FineGrainedDependencyDriverGraph.h +++ b/include/swift/Driver/FineGrainedDependencyDriverGraph.h @@ -123,6 +123,8 @@ class ModuleDepGraphNode : public DepGraphNode { return DepGraphNode::humanReadableName(where); } + void dump(raw_ostream &) const; + SWIFT_DEBUG_DUMP; }; @@ -312,9 +314,11 @@ class ModuleDepGraph { } /// For unit tests. - ModuleDepGraph(const bool EnableTypeFingerprints) - : ModuleDepGraph(true, false, EnableTypeFingerprints, false, nullptr) {} - + ModuleDepGraph(const bool EnableTypeFingerprints, + const bool EmitDotFilesForDebugging = false) + : ModuleDepGraph( + true, /*emitFineGrainedDependencyDotFileAfterEveryImport=*/ + EmitDotFilesForDebugging, EnableTypeFingerprints, false, nullptr) {} //============================================================================ // MARK: ModuleDepGraph - updating from a switdeps file @@ -328,18 +332,18 @@ class ModuleDepGraph { /// compensates. Changes loadFromPath(const driver::Job *, StringRef, DiagnosticEngine &); - Changes loadFromSourceFileDepGraph(const driver::Job *cmd, - const SourceFileDepGraph &); + const SourceFileDepGraph &, + DiagnosticEngine &); - /// Also for unit tests - Changes - simulateLoad(const driver::Job *cmd, - llvm::StringMap> simpleNames, - llvm::StringMap>> - compoundNames = {}, - const bool includePrivateDeps = false, - const bool hadCompilationError = false); + /// Also for unit tests + Changes + simulateLoad(const driver::Job *cmd, + llvm::StringMap> simpleNames, + llvm::StringMap>> + compoundNames = {}, + const bool includePrivateDeps = false, + const bool hadCompilationError = false); private: @@ -347,7 +351,8 @@ class ModuleDepGraph { /// and integrate it into the ModuleDepGraph. /// Used both the first time, and to reload the SourceFileDepGraph. /// If any changes were observed, indicate same in the return vale. - Changes loadFromBuffer(const driver::Job *, llvm::MemoryBuffer &); + Changes loadFromBuffer(const driver::Job *, llvm::MemoryBuffer &, + DiagnosticEngine &); /// Integrate a SourceFileDepGraph into the receiver. /// Integration happens when the driver needs to read SourceFileDepGraph. @@ -365,8 +370,9 @@ class ModuleDepGraph { const SourceFileDepGraphNode *integrand) const; /// Integrate the \p integrand into the receiver. - /// Return the changed node if any.. - NullablePtr + /// If an illegal value was found, return \c None, otherwise + /// return the changed node if any.. + Optional> integrateSourceFileDepGraphNode(const SourceFileDepGraph &g, const SourceFileDepGraphNode *integrand, const PreexistingNodeIfAny preexistingMatch, @@ -392,6 +398,12 @@ class ModuleDepGraph { /// After importing a provides node from the frontend, record its /// dependencies. /// Return true if moduleUseNode picks up a new external-dependency + /// + /// \param g The source file graph being integrated into the module graph + /// \param sourceFileUseNode The source file node just integrated, which may + /// also be a use (i.e. a "depends", a declaration used by something else) + /// \param moduleUseNode The module file node corresponding to the \c + /// sourceFileUseNode bool recordWhatUseDependsUpon(const SourceFileDepGraph &g, const SourceFileDepGraphNode *sourceFileUseNode, ModuleDepGraphNode *moduleUseNode); @@ -486,17 +498,8 @@ class ModuleDepGraph { bool haveAnyNodesBeenTraversedIn(const driver::Job *) const; - /// Given a "cascading" job, that is a job whose dependents must be recompiled - /// when this job is recompiled, Compute two sets of jobs: - /// 1. Return value (via visited) is the set of jobs needing recompilation - /// after this one, and - /// 2. Jobs not previously known to need dependencies reexamined after they - /// are recompiled. - /// - /// Returns jobs to be run because of changes to any/ever node in the - /// argument. Only return jobs marked that were previously unmarked, assuming - /// previously marked jobs are already scheduled. - /// TODO: rewrite above comment + /// Find all jobs (possibly including the argument) requiring recompilation + /// assuming that every entity in \p jobToBeRecompiled has changed. std::vector findJobsToRecompileWhenWholeJobChanges(const driver::Job *jobToBeRecompiled); diff --git a/lib/AST/FineGrainedDependencies.cpp b/lib/AST/FineGrainedDependencies.cpp index 69256f1cffc93..278fa6538a75e 100644 --- a/lib/AST/FineGrainedDependencies.cpp +++ b/lib/AST/FineGrainedDependencies.cpp @@ -15,10 +15,15 @@ #include "swift/AST/FineGrainedDependencies.h" // may not all be needed +#include "swift/AST/DiagnosticEngine.h" +#include "swift/AST/DiagnosticsCommon.h" +#include "swift/AST/DiagnosticsFrontend.h" +#include "swift/AST/FileSystem.h" #include "swift/Basic/FileSystem.h" #include "swift/Basic/LLVM.h" #include "swift/Demangling/Demangle.h" #include "swift/Frontend/FrontendOptions.h" + #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" @@ -34,6 +39,28 @@ using namespace swift; using namespace fine_grained_dependencies; +//============================================================================== +// MARK: Emitting and reading SourceFileDepGraph +//============================================================================== + +Optional SourceFileDepGraph::loadFromPath(StringRef path) { + auto bufferOrError = llvm::MemoryBuffer::getFile(path); + if (!bufferOrError) + return None; + return loadFromBuffer(*bufferOrError.get()); +} + +Optional +SourceFileDepGraph::loadFromBuffer(llvm::MemoryBuffer &buffer) { + SourceFileDepGraph fg; + llvm::yaml::Input yamlReader(llvm::MemoryBufferRef(buffer), nullptr); + yamlReader >> fg; + if (yamlReader.error()) + return None; + // return fg; compiles for Mac but not Linux, because it cannot be copied. + return Optional(std::move(fg)); +} + //============================================================================== // MARK: SourceFileDepGraph access //============================================================================== @@ -229,6 +256,23 @@ raw_ostream &fine_grained_dependencies::operator<<(raw_ostream &out, bool DependencyKey::verify() const { assert((getKind() != NodeKind::externalDepend || isInterface()) && "All external dependencies must be interfaces."); + switch (getKind()) { + case NodeKind::topLevel: + case NodeKind::dynamicLookup: + case NodeKind::externalDepend: + case NodeKind::sourceFileProvide: + assert(context.empty() && !name.empty() && "Must only have a name"); + break; + case NodeKind::nominal: + case NodeKind::potentialMember: + assert(!context.empty() && name.empty() && "Must only have a context"); + break; + case NodeKind::member: + assert(!context.empty() && !name.empty() && "Must have both"); + break; + case NodeKind::kindCount: + llvm_unreachable("impossible"); + } return true; } @@ -300,6 +344,15 @@ void SourceFileDepGraph::verifySame(const SourceFileDepGraph &other) const { #endif } +void SourceFileDepGraph::emitDotFile(StringRef outputPath, + DiagnosticEngine &diags) { + std::string dotFileName = outputPath.str() + ".dot"; + withOutputFile(diags, dotFileName, [&](llvm::raw_pwrite_stream &out) { + DotFileEmitter(out, *this, false, false).emit(); + return false; + }); +} + //============================================================================== // MARK: SourceFileDepGraph YAML reading & writing //============================================================================== diff --git a/lib/AST/SourceFileDepGraphConstructor.cpp b/lib/AST/SourceFileDepGraphConstructor.cpp index eba5fe33192bb..fc87e2cb59b21 100644 --- a/lib/AST/SourceFileDepGraphConstructor.cpp +++ b/lib/AST/SourceFileDepGraphConstructor.cpp @@ -46,32 +46,6 @@ using namespace swift; using namespace fine_grained_dependencies; -//============================================================================== -// MARK: Emitting and reading SourceFileDepGraph -//============================================================================== - -Optional SourceFileDepGraph::loadFromPath(StringRef path) { - auto bufferOrError = llvm::MemoryBuffer::getFile(path); - if (!bufferOrError) - return None; - return loadFromBuffer(*bufferOrError.get()); -} - -Optional -SourceFileDepGraph::loadFromBuffer(llvm::MemoryBuffer &buffer) { - SourceFileDepGraph fg; - llvm::yaml::Input yamlReader(llvm::MemoryBufferRef(buffer), nullptr); - yamlReader >> fg; - if (yamlReader.error()) - return None; - // return fg; compiles for Mac but not Linux, because it cannot be copied. - return Optional(std::move(fg)); -} - -//============================================================================== -// MARK: Start of SourceFileDepGraph building, specific to status quo -//============================================================================== - //============================================================================== // MARK: Helpers for key construction that must be in frontend //============================================================================== @@ -501,7 +475,7 @@ class SourceFileDepGraphConstructor { /// a flag indicating if the member is private to its enclosing file, and /// a flag indicating if the dependency cascades. const std::vector, bool>> - memberDepends; + dependsWithContexts; /// The base name of a class member depended-upon for dynamic lookup, and a /// cascades flag. @@ -534,7 +508,8 @@ class SourceFileDepGraphConstructor { bool hadCompilationError, const std::string &interfaceHash, ArrayRef> topLevelDepends, - ArrayRef, bool>> memberDepends, + ArrayRef, bool>> + dependsWithContexts, ArrayRef> dynamicLookupDepends, ArrayRef externalDependencies, @@ -554,7 +529,7 @@ class SourceFileDepGraphConstructor { interfaceHash(interfaceHash), topLevelDepends(topLevelDepends), - memberDepends(memberDepends), + dependsWithContexts(dependsWithContexts), dynamicLookupDepends(dynamicLookupDepends), externalDependencies(externalDependencies), @@ -569,11 +544,15 @@ class SourceFileDepGraphConstructor { classMembers(classMembers) {} - SourceFileDepGraphConstructor static forSourceFile(SourceFile *SF, - const DependencyTracker &depTracker, - StringRef swiftDeps, - const bool includePrivateDeps, - const bool hadCompilationError) { +// clang-format off +static SourceFileDepGraphConstructor +forSourceFile( + SourceFile *SF, + const DependencyTracker &depTracker, + StringRef swiftDeps, + const bool includePrivateDeps, + const bool hadCompilationError) { +// clang-format on SourceFileDeclFinder declFinder(SF, includePrivateDeps); std::vector> topLevelDepends; @@ -584,11 +563,11 @@ class SourceFileDepGraphConstructor { for (const auto &p: SF->getReferencedNameTracker()->getDynamicLookupNames()) dynamicLookupDepends.push_back(std::make_pair(p.getFirst().userFacingName(), p.getSecond())); - std::vector, bool>> memberDepends; + std::vector, bool>> dependsWithContexts; for (const auto &p: SF->getReferencedNameTracker()->getUsedMembers()) { const auto &member = p.getFirst().second; StringRef emptyOrUserFacingName = member.empty() ? "" : member.userFacingName(); - memberDepends.push_back( + dependsWithContexts.push_back( std::make_pair( std::make_tuple( mangleTypeAsContext(p.getFirst().first), @@ -604,7 +583,7 @@ class SourceFileDepGraphConstructor { getInterfaceHash(SF), topLevelDepends, - memberDepends, + dependsWithContexts, dynamicLookupDepends, depTracker.getDependencies(), @@ -782,7 +761,7 @@ void SourceFileDepGraphConstructor::addDependencyArcsToGraph() { // TODO: express the multiple provides and depends streams with variadic // templates addAllDependenciesFrom(topLevelDepends); - addAllDependenciesFrom(memberDepends); + addAllDependenciesFrom(dependsWithContexts); addAllDependenciesFrom(dynamicLookupDepends); addAllDependenciesFrom(externalDependencies); } @@ -798,7 +777,7 @@ void SourceFileDepGraphConstructor::recordThatThisWholeFileDependsOn( // Entry point from the Frontend to this whole system //============================================================================== -bool swift::fine_grained_dependencies::emitReferenceDependencies( +bool fine_grained_dependencies::emitReferenceDependencies( DiagnosticEngine &diags, SourceFile *const SF, const DependencyTracker &depTracker, StringRef outputPath, const bool alsoEmitDotFile) { @@ -814,8 +793,9 @@ bool swift::fine_grained_dependencies::emitReferenceDependencies( // we force the inclusion of private declarations when fingerprints // are enabled. const bool includeIntrafileDeps = - SF->getASTContext().LangOpts.FineGrainedDependenciesIncludeIntrafileOnes || - SF->getASTContext().LangOpts.EnableTypeFingerprints; + SF->getASTContext() + .LangOpts.FineGrainedDependenciesIncludeIntrafileOnes || + SF->getASTContext().LangOpts.EnableTypeFingerprints; const bool hadCompilationError = SF->getASTContext().hadError(); auto gc = SourceFileDepGraphConstructor::forSourceFile( SF, depTracker, outputPath, includeIntrafileDeps, hadCompilationError); @@ -832,13 +812,9 @@ bool swift::fine_grained_dependencies::emitReferenceDependencies( // If path is stdout, cannot read it back, so check for "-" assert(outputPath == "-" || g.verifyReadsWhatIsWritten(outputPath)); - if (alsoEmitDotFile) { - std::string dotFileName = outputPath.str() + ".dot"; - withOutputFile(diags, dotFileName, [&](llvm::raw_pwrite_stream &out) { - DotFileEmitter(out, g, false, false).emit(); - return false; - }); - } + if (alsoEmitDotFile) + g.emitDotFile(outputPath, diags); + return hadError; } @@ -952,7 +928,10 @@ SourceFileDepGraph SourceFileDepGraph::simulateLoad( // clang-format off SourceFileDepGraphConstructor c( - swiftDepsFilename, includePrivateDeps, hadCompilationError, interfaceHash, + swiftDepsFilename, + includePrivateDeps, + hadCompilationError, + interfaceHash, getSimpleDepends(simpleNamesByRDK[dependsTopLevel]), getCompoundDepends(simpleNamesByRDK[dependsNominal], compoundNamesByRDK[dependsMember]), @@ -961,7 +940,7 @@ SourceFileDepGraph SourceFileDepGraph::simulateLoad( {}, // precedence groups {}, // memberOperatorDecls {}, // operators - getMangledHolderProvides(simpleNamesByRDK[providesNominal]), // topNominals + {}, // topNominals getBaseNameProvides(simpleNamesByRDK[providesTopLevel]), // topValues getMangledHolderProvides(simpleNamesByRDK[providesNominal]), // allNominals getMangledHolderProvides(simpleNamesByRDK[providesNominal]), // potentialMemberHolders diff --git a/lib/Driver/FineGrainedDependencyDriverGraph.cpp b/lib/Driver/FineGrainedDependencyDriverGraph.cpp index 5bda1956f9b4c..95ce4e6c33fa6 100644 --- a/lib/Driver/FineGrainedDependencyDriverGraph.cpp +++ b/lib/Driver/FineGrainedDependencyDriverGraph.cpp @@ -12,9 +12,11 @@ #include "swift/Driver/FineGrainedDependencyDriverGraph.h" // Next two includes needed for reporting errors opening dot file for writing. +#include "swift/AST/DiagnosticEngine.h" #include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/FileSystem.h" #include "swift/Basic/ReferenceDependencyKeys.h" +#include "swift/Basic/SourceManager.h" #include "swift/Basic/Statistic.h" #include "swift/Demangling/Demangle.h" #include "swift/Driver/Job.h" @@ -57,7 +59,14 @@ ModuleDepGraph::simulateLoad(const Job *cmd, swiftDeps, includePrivateDeps, hadCompilationError, interfaceHash, simpleNames, compoundNames); - return loadFromSourceFileDepGraph(cmd, sfdg); + SourceManager sm; + DiagnosticEngine diags(sm); + // help for debugging: emit imported file, too + if (emitFineGrainedDependencyDotFileAfterEveryImport) { + sfdg.emitDotFile(swiftDeps, diags); + } + + return loadFromSourceFileDepGraph(cmd, sfdg, diags); } std::string SourceFileDepGraph::noncascading(std::string name) { @@ -104,31 +113,35 @@ ModuleDepGraph::Changes ModuleDepGraph::loadFromPath(const Job *Cmd, auto buffer = llvm::MemoryBuffer::getFile(path); if (!buffer) return None; - auto r = loadFromBuffer(Cmd, *buffer.get()); + auto r = loadFromBuffer(Cmd, *buffer.get(), diags); assert(path == getSwiftDeps(Cmd) && "Should be reading the job's swiftdeps"); assert(!r || !nodeMap[path].empty() && "Must have a node for the whole file"); - if (emitFineGrainedDependencyDotFileAfterEveryImport) - emitDotFileForJob(diags, Cmd); - if (verifyFineGrainedDependencyGraphAfterEveryImport) - verify(); return r; } /// Returns None for error or a set of changed keys ModuleDepGraph::Changes -ModuleDepGraph::loadFromBuffer(const Job *job, llvm::MemoryBuffer &buffer) { +ModuleDepGraph::loadFromBuffer(const Job *job, llvm::MemoryBuffer &buffer, + DiagnosticEngine &diags) { Optional sourceFileDepGraph = SourceFileDepGraph::loadFromBuffer(buffer); if (!sourceFileDepGraph) return None; - return loadFromSourceFileDepGraph(job, sourceFileDepGraph.getValue()); + return loadFromSourceFileDepGraph(job, sourceFileDepGraph.getValue(), diags); } ModuleDepGraph::Changes ModuleDepGraph::loadFromSourceFileDepGraph( - const Job *job, const SourceFileDepGraph &sourceFileDepGraph) { + const Job *job, const SourceFileDepGraph &sourceFileDepGraph, + DiagnosticEngine &diags) { registerJob(job); - return integrate(sourceFileDepGraph, getSwiftDeps(job)); + auto changes = integrate(sourceFileDepGraph, getSwiftDeps(job)); + + if (verifyFineGrainedDependencyGraphAfterEveryImport) + verify(); + if (emitFineGrainedDependencyDotFileAfterEveryImport) + emitDotFileForJob(diags, job); + return changes; } bool ModuleDepGraph::haveAnyNodesBeenTraversedIn(const Job *cmd) const { @@ -254,7 +267,9 @@ ModuleDepGraph::Changes ModuleDepGraph::integrate(const SourceFileDepGraph &g, auto disappearedNodes = nodeMap[swiftDepsOfJob]; // When done, changeDependencyKeys contains a list of keys that changed // as a result of this integration. - auto changedNodes = std::unordered_set(); + // Or if the integration failed, None. + Optional> changedNodes = + std::unordered_set(); g.forEachNode([&](const SourceFileDepGraphNode *integrand) { const auto &key = integrand->getKey(); @@ -264,24 +279,30 @@ ModuleDepGraph::Changes ModuleDepGraph::integrate(const SourceFileDepGraph &g, preexistingMatch.getValue().first == LocationOfPreexistingNode::here) disappearedNodes.erase(key); // Node was and still is. Do not erase it. - NullablePtr newNodeOrChangedNode = + Optional> newNodeOrChangedNode = integrateSourceFileDepGraphNode(g, integrand, preexistingMatch, swiftDepsOfJob); - if (auto *n = newNodeOrChangedNode.getPtrOrNull()) - changedNodes.insert(n); + if (!newNodeOrChangedNode) + changedNodes = None; + else if (!changedNodes) + ; + else if (auto *n = newNodeOrChangedNode.getValue().getPtrOrNull()) + changedNodes.getValue().insert(n); }); + if (!changedNodes) + return None; for (auto &p : disappearedNodes) { - changedNodes.insert(p.second); + changedNodes.getValue().insert(p.second); eraseNodeFromJob(p.second); } // Make sure the changes can be retraced: - for (auto *n : changedNodes) + for (auto *n : changedNodes.getValue()) n->clearHasBeenTraced(); - return changedNodes; + return changedNodes.getValue(); } ModuleDepGraph::PreexistingNodeIfAny ModuleDepGraph::findPreexistingMatch( @@ -310,13 +331,20 @@ ModuleDepGraph::PreexistingNodeIfAny ModuleDepGraph::findPreexistingMatch( return None; } -NullablePtr ModuleDepGraph::integrateSourceFileDepGraphNode( +Optional> +ModuleDepGraph::integrateSourceFileDepGraphNode( const SourceFileDepGraph &g, const SourceFileDepGraphNode *integrand, const PreexistingNodeIfAny preexistingMatch, const StringRef swiftDepsOfJob) { + if (!EnableTypeFingerprints && + integrand->getKey().getKind() != NodeKind::sourceFileProvide && + integrand->getFingerprint()) + return None; + if (!integrand->getIsProvides()) - return nullptr; // depends are captured by recordWhatUseDependsUpon below + return NullablePtr(); // depends are captured by + // recordWhatUseDependsUpon below auto changedAndIntegrationResultNode = integrateSourceFileDeclNode(integrand, swiftDepsOfJob, preexistingMatch); @@ -550,6 +578,14 @@ void ModuleDepGraph::emitDotFile(llvm::raw_ostream &out) { // MARK: ModuleDepGraph debugging //============================================================================== +void ModuleDepGraphNode::dump(llvm::raw_ostream &out) const { + DepGraphNode::dump(out); + if (getIsProvides()) + out << " swiftDeps: <" << getSwiftDepsOfProvides() << ">\n"; + else + out << " no swiftDeps\n"; +} + void ModuleDepGraphNode::dump() const { DepGraphNode::dump(); if (getIsProvides())