diff options
Diffstat (limited to 'clangd/index')
32 files changed, 947 insertions, 713 deletions
diff --git a/clangd/index/Background.cpp b/clangd/index/Background.cpp index 1d678197..ddb0ec43 100644 --- a/clangd/index/Background.cpp +++ b/clangd/index/Background.cpp @@ -1,9 +1,8 @@ //===-- Background.cpp - Build an index in a background thread ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -35,7 +34,6 @@ #include <string> #include <thread> -using namespace llvm; namespace clang { namespace clangd { namespace { @@ -92,46 +90,48 @@ IncludeGraph getSubGraph(const URI &U, const IncludeGraph &FullGraph) { // Creates a filter to not collect index results from files with unchanged // digests. -// \p FileDigests contains file digests for the current indexed files, and all -// changed files will be added to \p FilesToUpdate. +// \p FileDigests contains file digests for the current indexed files. decltype(SymbolCollector::Options::FileFilter) -createFileFilter(const llvm::StringMap<FileDigest> &FileDigests, - llvm::StringMap<FileDigest> &FilesToUpdate) { - return [&FileDigests, &FilesToUpdate](const SourceManager &SM, FileID FID) { - StringRef Path; - if (const auto *F = SM.getFileEntryForID(FID)) - Path = F->getName(); - if (Path.empty()) +createFileFilter(const llvm::StringMap<FileDigest> &FileDigests) { + return [&FileDigests](const SourceManager &SM, FileID FID) { + const auto *F = SM.getFileEntryForID(FID); + if (!F) return false; // Skip invalid files. - SmallString<128> AbsPath(Path); - if (std::error_code EC = - SM.getFileManager().getVirtualFileSystem()->makeAbsolute(AbsPath)) { - elog("Warning: could not make absolute file: {0}", EC.message()); + auto AbsPath = getCanonicalPath(F, SM); + if (!AbsPath) return false; // Skip files without absolute path. - } - sys::path::remove_dots(AbsPath, /*remove_dot_dot=*/true); auto Digest = digestFile(SM, FID); if (!Digest) return false; - auto D = FileDigests.find(AbsPath); + auto D = FileDigests.find(*AbsPath); if (D != FileDigests.end() && D->second == Digest) return false; // Skip files that haven't changed. - - FilesToUpdate[AbsPath] = *Digest; return true; }; } +// We cannot use vfs->makeAbsolute because Cmd.FileName is either absolute or +// relative to Cmd.Directory, which might not be the same as current working +// directory. +llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) { + llvm::SmallString<128> AbsolutePath; + if (llvm::sys::path::is_absolute(Cmd.Filename)) { + AbsolutePath = Cmd.Filename; + } else { + AbsolutePath = Cmd.Directory; + llvm::sys::path::append(AbsolutePath, Cmd.Filename); + } + return AbsolutePath; +} } // namespace BackgroundIndex::BackgroundIndex( - Context BackgroundContext, StringRef ResourceDir, - const FileSystemProvider &FSProvider, const GlobalCompilationDatabase &CDB, + Context BackgroundContext, const FileSystemProvider &FSProvider, + const GlobalCompilationDatabase &CDB, BackgroundIndexStorage::Factory IndexStorageFactory, size_t BuildIndexPeriodMs, size_t ThreadPoolSize) - : SwapIndex(make_unique<MemIndex>()), ResourceDir(ResourceDir), - FSProvider(FSProvider), CDB(CDB), - BackgroundContext(std::move(BackgroundContext)), + : SwapIndex(llvm::make_unique<MemIndex>()), FSProvider(FSProvider), + CDB(CDB), BackgroundContext(std::move(BackgroundContext)), BuildIndexPeriodMs(BuildIndexPeriodMs), SymbolsUpdatedSinceLastIndex(false), IndexStorageFactory(std::move(IndexStorageFactory)), @@ -169,7 +169,7 @@ void BackgroundIndex::stop() { void BackgroundIndex::run() { WithContext Background(BackgroundContext.clone()); while (true) { - Optional<Task> Task; + llvm::Optional<Task> Task; ThreadPriority Priority; { std::unique_lock<std::mutex> Lock(QueueMu); @@ -211,40 +211,32 @@ void BackgroundIndex::enqueue(const std::vector<std::string> &ChangedFiles) { [this, ChangedFiles] { trace::Span Tracer("BackgroundIndexEnqueue"); // We're doing this asynchronously, because we'll read shards here too. - // FIXME: read shards here too. - log("Enqueueing {0} commands for indexing", ChangedFiles.size()); SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size())); - // We shuffle the files because processing them in a random order should - // quickly give us good coverage of headers in the project. - std::vector<unsigned> Permutation(ChangedFiles.size()); - std::iota(Permutation.begin(), Permutation.end(), 0); - std::mt19937 Generator(std::random_device{}()); - std::shuffle(Permutation.begin(), Permutation.end(), Generator); - - for (const unsigned I : Permutation) - enqueue(ChangedFiles[I]); + auto NeedsReIndexing = loadShards(std::move(ChangedFiles)); + // Run indexing for files that need to be updated. + std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(), + std::mt19937(std::random_device{}())); + for (auto &Elem : NeedsReIndexing) + enqueue(std::move(Elem.first), Elem.second); }, ThreadPriority::Normal); } -void BackgroundIndex::enqueue(const std::string &File) { - ProjectInfo Project; - if (auto Cmd = CDB.getCompileCommand(File, &Project)) { - auto *Storage = IndexStorageFactory(Project.SourceRoot); - // Set priority to low, since background indexing is a long running - // task we do not want to eat up cpu when there are any other high - // priority threads. - enqueueTask(Bind( - [this, File, Storage](tooling::CompileCommand Cmd) { - Cmd.CommandLine.push_back("-resource-dir=" + ResourceDir); - if (auto Error = index(std::move(Cmd), Storage)) - log("Indexing {0} failed: {1}", File, std::move(Error)); - }, - std::move(*Cmd)), - ThreadPriority::Low); - } +void BackgroundIndex::enqueue(tooling::CompileCommand Cmd, + BackgroundIndexStorage *Storage) { + enqueueTask(Bind( + [this, Storage](tooling::CompileCommand Cmd) { + // We can't use llvm::StringRef here since we are going to + // move from Cmd during the call below. + const std::string FileName = Cmd.Filename; + if (auto Error = index(std::move(Cmd), Storage)) + elog("Indexing {0} failed: {1}", FileName, + std::move(Error)); + }, + std::move(Cmd)), + ThreadPriority::Low); } void BackgroundIndex::enqueueTask(Task T, ThreadPriority Priority) { @@ -265,22 +257,34 @@ void BackgroundIndex::enqueueTask(Task T, ThreadPriority Priority) { QueueCV.notify_all(); } -/// Given index results from a TU, only update files in \p FilesToUpdate. -void BackgroundIndex::update(StringRef MainFile, IndexFileIn Index, - const StringMap<FileDigest> &FilesToUpdate, +/// Given index results from a TU, only update symbols coming from files that +/// are different or missing from than \p DigestsSnapshot. Also stores new index +/// information on IndexStorage. +void BackgroundIndex::update(llvm::StringRef MainFile, IndexFileIn Index, + const llvm::StringMap<FileDigest> &DigestsSnapshot, BackgroundIndexStorage *IndexStorage) { // Partition symbols/references into files. struct File { - DenseSet<const Symbol *> Symbols; - DenseSet<const Ref *> Refs; + llvm::DenseSet<const Symbol *> Symbols; + llvm::DenseSet<const Ref *> Refs; + FileDigest Digest; }; - StringMap<File> Files; + llvm::StringMap<File> Files; URIToFileCache URICache(MainFile); + for (const auto &IndexIt : *Index.Sources) { + const auto &IGN = IndexIt.getValue(); + const auto AbsPath = URICache.resolve(IGN.URI); + const auto DigestIt = DigestsSnapshot.find(AbsPath); + // File has different contents. + if (DigestIt == DigestsSnapshot.end() || DigestIt->getValue() != IGN.Digest) + Files.try_emplace(AbsPath).first->getValue().Digest = IGN.Digest; + } for (const auto &Sym : *Index.Symbols) { if (Sym.CanonicalDeclaration) { auto DeclPath = URICache.resolve(Sym.CanonicalDeclaration.FileURI); - if (FilesToUpdate.count(DeclPath) != 0) - Files[DeclPath].Symbols.insert(&Sym); + const auto FileIt = Files.find(DeclPath); + if (FileIt != Files.end()) + FileIt->second.Symbols.insert(&Sym); } // For symbols with different declaration and definition locations, we store // the full symbol in both the header file and the implementation file, so @@ -289,16 +293,18 @@ void BackgroundIndex::update(StringRef MainFile, IndexFileIn Index, if (Sym.Definition && Sym.Definition.FileURI != Sym.CanonicalDeclaration.FileURI) { auto DefPath = URICache.resolve(Sym.Definition.FileURI); - if (FilesToUpdate.count(DefPath) != 0) - Files[DefPath].Symbols.insert(&Sym); + const auto FileIt = Files.find(DefPath); + if (FileIt != Files.end()) + FileIt->second.Symbols.insert(&Sym); } } - DenseMap<const Ref *, SymbolID> RefToIDs; + llvm::DenseMap<const Ref *, SymbolID> RefToIDs; for (const auto &SymRefs : *Index.Refs) { for (const auto &R : SymRefs.second) { auto Path = URICache.resolve(R.Location.FileURI); - if (FilesToUpdate.count(Path) != 0) { - auto &F = Files[Path]; + const auto FileIt = Files.find(Path); + if (FileIt != Files.end()) { + auto &F = FileIt->getValue(); RefToIDs[&R] = SymRefs.first; F.Refs.insert(&R); } @@ -306,22 +312,18 @@ void BackgroundIndex::update(StringRef MainFile, IndexFileIn Index, } // Build and store new slabs for each updated file. - for (const auto &F : Files) { - StringRef Path = F.first(); - vlog("Update symbols in {0}", Path); + for (const auto &FileIt : Files) { + llvm::StringRef Path = FileIt.getKey(); SymbolSlab::Builder Syms; RefSlab::Builder Refs; - for (const auto *S : F.second.Symbols) + for (const auto *S : FileIt.second.Symbols) Syms.insert(*S); - for (const auto *R : F.second.Refs) + for (const auto *R : FileIt.second.Refs) Refs.insert(RefToIDs[R], *R); - auto SS = llvm::make_unique<SymbolSlab>(std::move(Syms).build()); auto RS = llvm::make_unique<RefSlab>(std::move(Refs).build()); auto IG = llvm::make_unique<IncludeGraph>( getSubGraph(URI::create(Path), Index.Sources.getValue())); - - auto Hash = FilesToUpdate.lookup(Path); // We need to store shards before updating the index, since the latter // consumes slabs. if (IndexStorage) { @@ -334,13 +336,19 @@ void BackgroundIndex::update(StringRef MainFile, IndexFileIn Index, elog("Failed to write background-index shard for file {0}: {1}", Path, std::move(Error)); } - - std::lock_guard<std::mutex> Lock(DigestsMu); - // This can override a newer version that is added in another thread, - // if this thread sees the older version but finishes later. This should be - // rare in practice. - IndexedFileDigests[Path] = Hash; - IndexedSymbols.update(Path, std::move(SS), std::move(RS)); + { + std::lock_guard<std::mutex> Lock(DigestsMu); + auto Hash = FileIt.second.Digest; + // Skip if file is already up to date. + auto DigestIt = IndexedFileDigests.try_emplace(Path); + if (!DigestIt.second && DigestIt.first->second == Hash) + continue; + DigestIt.first->second = Hash; + // This can override a newer version that is added in another thread, if + // this thread sees the older version but finishes later. This should be + // rare in practice. + IndexedSymbols.update(Path, std::move(SS), std::move(RS)); + } } } @@ -349,11 +357,11 @@ void BackgroundIndex::buildIndex() { while (true) { { std::unique_lock<std::mutex> Lock(IndexMu); - if (ShouldStop) // Avoid waiting if stopped. + if (ShouldStop) // Avoid waiting if stopped. break; // Wait until this is notified to stop or `BuildIndexPeriodMs` has past. IndexCV.wait_for(Lock, std::chrono::milliseconds(BuildIndexPeriodMs)); - if (ShouldStop) // Avoid rebuilding index if stopped. + if (ShouldStop) // Avoid rebuilding index if stopped. break; } if (!SymbolsUpdatedSinceLastIndex.exchange(false)) @@ -368,56 +376,44 @@ void BackgroundIndex::buildIndex() { } } -Error BackgroundIndex::index(tooling::CompileCommand Cmd, - BackgroundIndexStorage *IndexStorage) { +llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd, + BackgroundIndexStorage *IndexStorage) { trace::Span Tracer("BackgroundIndex"); SPAN_ATTACH(Tracer, "file", Cmd.Filename); - SmallString<128> AbsolutePath; - if (sys::path::is_absolute(Cmd.Filename)) { - AbsolutePath = Cmd.Filename; - } else { - AbsolutePath = Cmd.Directory; - sys::path::append(AbsolutePath, Cmd.Filename); - } + auto AbsolutePath = getAbsolutePath(Cmd); auto FS = FSProvider.getFileSystem(); auto Buf = FS->getBufferForFile(AbsolutePath); if (!Buf) - return errorCodeToError(Buf.getError()); + return llvm::errorCodeToError(Buf.getError()); auto Hash = digest(Buf->get()->getBuffer()); // Take a snapshot of the digests to avoid locking for each file in the TU. llvm::StringMap<FileDigest> DigestsSnapshot; { std::lock_guard<std::mutex> Lock(DigestsMu); - if (IndexedFileDigests.lookup(AbsolutePath) == Hash) { - vlog("No need to index {0}, already up to date", AbsolutePath); - return Error::success(); - } - DigestsSnapshot = IndexedFileDigests; } - log("Indexing {0} (digest:={1})", Cmd.Filename, toHex(Hash)); + vlog("Indexing {0} (digest:={1})", Cmd.Filename, llvm::toHex(Hash)); ParseInputs Inputs; Inputs.FS = std::move(FS); Inputs.FS->setCurrentWorkingDirectory(Cmd.Directory); Inputs.CompileCommand = std::move(Cmd); auto CI = buildCompilerInvocation(Inputs); if (!CI) - return createStringError(inconvertibleErrorCode(), - "Couldn't build compiler invocation"); + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "Couldn't build compiler invocation"); IgnoreDiagnostics IgnoreDiags; auto Clang = prepareCompilerInstance( std::move(CI), /*Preamble=*/nullptr, std::move(*Buf), std::make_shared<PCHContainerOperations>(), Inputs.FS, IgnoreDiags); if (!Clang) - return createStringError(inconvertibleErrorCode(), - "Couldn't build compiler instance"); + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "Couldn't build compiler instance"); SymbolCollector::Options IndexOpts; - StringMap<FileDigest> FilesToUpdate; - IndexOpts.FileFilter = createFileFilter(DigestsSnapshot, FilesToUpdate); + IndexOpts.FileFilter = createFileFilter(DigestsSnapshot); IndexFileIn Index; auto Action = createStaticIndexingAction( IndexOpts, [&](SymbolSlab S) { Index.Symbols = std::move(S); }, @@ -431,19 +427,21 @@ Error BackgroundIndex::index(tooling::CompileCommand Cmd, const FrontendInputFile &Input = Clang->getFrontendOpts().Inputs.front(); if (!Action->BeginSourceFile(*Clang, Input)) - return createStringError(inconvertibleErrorCode(), - "BeginSourceFile() failed"); + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "BeginSourceFile() failed"); if (!Action->Execute()) - return createStringError(inconvertibleErrorCode(), "Execute() failed"); + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "Execute() failed"); Action->EndSourceFile(); if (Clang->hasDiagnostics() && Clang->getDiagnostics().hasUncompilableErrorOccurred()) { - return createStringError(inconvertibleErrorCode(), - "IndexingAction failed: has uncompilable errors"); + return llvm::createStringError( + llvm::inconvertibleErrorCode(), + "IndexingAction failed: has uncompilable errors"); } - assert(Index.Symbols && Index.Refs && Index.Sources - && "Symbols, Refs and Sources must be set."); + assert(Index.Symbols && Index.Refs && Index.Sources && + "Symbols, Refs and Sources must be set."); log("Indexed {0} ({1} symbols, {2} refs, {3} files)", Inputs.CompileCommand.Filename, Index.Symbols->size(), @@ -452,13 +450,7 @@ Error BackgroundIndex::index(tooling::CompileCommand Cmd, SPAN_ATTACH(Tracer, "refs", int(Index.Refs->numRefs())); SPAN_ATTACH(Tracer, "sources", int(Index.Sources->size())); - update(AbsolutePath, std::move(Index), FilesToUpdate, IndexStorage); - { - // Make sure hash for the main file is always updated even if there is no - // index data in it. - std::lock_guard<std::mutex> Lock(DigestsMu); - IndexedFileDigests[AbsolutePath] = Hash; - } + update(AbsolutePath, std::move(Index), DigestsSnapshot, IndexStorage); if (BuildIndexPeriodMs > 0) SymbolsUpdatedSinceLastIndex = true; @@ -466,7 +458,152 @@ Error BackgroundIndex::index(tooling::CompileCommand Cmd, reset( IndexedSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge)); - return Error::success(); + return llvm::Error::success(); +} + +std::vector<BackgroundIndex::Source> +BackgroundIndex::loadShard(const tooling::CompileCommand &Cmd, + BackgroundIndexStorage *IndexStorage, + llvm::StringSet<> &LoadedShards) { + struct ShardInfo { + std::string AbsolutePath; + std::unique_ptr<IndexFileIn> Shard; + FileDigest Digest; + }; + std::vector<ShardInfo> IntermediateSymbols; + // Make sure we don't have duplicate elements in the queue. Keys are absolute + // paths. + llvm::StringSet<> InQueue; + auto FS = FSProvider.getFileSystem(); + // Dependencies of this TU, paired with the information about whether they + // need to be re-indexed or not. + std::vector<Source> Dependencies; + std::queue<Source> ToVisit; + std::string AbsolutePath = getAbsolutePath(Cmd).str(); + // Up until we load the shard related to a dependency it needs to be + // re-indexed. + ToVisit.emplace(AbsolutePath, true); + InQueue.insert(AbsolutePath); + // Goes over each dependency. + while (!ToVisit.empty()) { + Dependencies.push_back(std::move(ToVisit.front())); + // Dependencies is not modified during the rest of the loop, so it is safe + // to keep the reference. + auto &CurDependency = Dependencies.back(); + ToVisit.pop(); + // If we have already seen this shard before(either loaded or failed) don't + // re-try again. Since the information in the shard won't change from one TU + // to another. + if (!LoadedShards.try_emplace(CurDependency.Path).second) { + // If the dependency needs to be re-indexed, first occurence would already + // have detected that, so we don't need to issue it again. + CurDependency.NeedsReIndexing = false; + continue; + } + + auto Shard = IndexStorage->loadShard(CurDependency.Path); + if (!Shard || !Shard->Sources) { + // File will be returned as requiring re-indexing to caller. + vlog("Failed to load shard: {0}", CurDependency.Path); + continue; + } + // These are the edges in the include graph for current dependency. + for (const auto &I : *Shard->Sources) { + auto U = URI::parse(I.getKey()); + if (!U) + continue; + auto AbsolutePath = URI::resolve(*U, CurDependency.Path); + if (!AbsolutePath) + continue; + // Add file as dependency if haven't seen before. + if (InQueue.try_emplace(*AbsolutePath).second) + ToVisit.emplace(*AbsolutePath, true); + // The node contains symbol information only for current file, the rest is + // just edges. + if (*AbsolutePath != CurDependency.Path) + continue; + + // We found source file info for current dependency. + assert(I.getValue().Digest != FileDigest{{0}} && "Digest is empty?"); + ShardInfo SI; + SI.AbsolutePath = CurDependency.Path; + SI.Shard = std::move(Shard); + SI.Digest = I.getValue().Digest; + IntermediateSymbols.push_back(std::move(SI)); + // Check if the source needs re-indexing. + // Get the digest, skip it if file doesn't exist. + auto Buf = FS->getBufferForFile(CurDependency.Path); + if (!Buf) { + elog("Couldn't get buffer for file: {0}: {1}", CurDependency.Path, + Buf.getError().message()); + continue; + } + // If digests match then dependency doesn't need re-indexing. + CurDependency.NeedsReIndexing = + digest(Buf->get()->getBuffer()) != I.getValue().Digest; + } + } + // Load shard information into background-index. + { + std::lock_guard<std::mutex> Lock(DigestsMu); + // This can override a newer version that is added in another thread, + // if this thread sees the older version but finishes later. This + // should be rare in practice. + for (const ShardInfo &SI : IntermediateSymbols) { + auto SS = + SI.Shard->Symbols + ? llvm::make_unique<SymbolSlab>(std::move(*SI.Shard->Symbols)) + : nullptr; + auto RS = SI.Shard->Refs + ? llvm::make_unique<RefSlab>(std::move(*SI.Shard->Refs)) + : nullptr; + IndexedFileDigests[SI.AbsolutePath] = SI.Digest; + IndexedSymbols.update(SI.AbsolutePath, std::move(SS), std::move(RS)); + } + } + + return Dependencies; +} + +// Goes over each changed file and loads them from index. Returns the list of +// TUs that had out-of-date/no shards. +std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>> +BackgroundIndex::loadShards(std::vector<std::string> ChangedFiles) { + std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>> + NeedsReIndexing; + // Keeps track of the files that will be reindexed, to make sure we won't + // re-index same dependencies more than once. Keys are AbsolutePaths. + llvm::StringSet<> FilesToIndex; + // Keeps track of the loaded shards to make sure we don't perform redundant + // disk IO. Keys are absolute paths. + llvm::StringSet<> LoadedShards; + for (const auto &File : ChangedFiles) { + ProjectInfo PI; + auto Cmd = CDB.getCompileCommand(File, &PI); + if (!Cmd) + continue; + BackgroundIndexStorage *IndexStorage = IndexStorageFactory(PI.SourceRoot); + auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards); + for (const auto &Dependency : Dependencies) { + if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path)) + continue; + // FIXME: Currently, we simply schedule indexing on a TU whenever any of + // its dependencies needs re-indexing. We might do it smarter by figuring + // out a minimal set of TUs that will cover all the stale dependencies. + vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.", + Cmd->Filename, Dependency.Path); + NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage}); + // Mark all of this TU's dependencies as to-be-indexed so that we won't + // try to re-index those. + for (const auto &Dependency : Dependencies) + FilesToIndex.insert(Dependency.Path); + break; + } + } + vlog("Loaded all shards"); + reset(IndexedSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge)); + + return NeedsReIndexing; } } // namespace clangd diff --git a/clangd/index/Background.h b/clangd/index/Background.h index 3318f4fc..808c03a3 100644 --- a/clangd/index/Background.h +++ b/clangd/index/Background.h @@ -1,9 +1,8 @@ //===--- Background.h - Build an index in a background thread ----*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -68,20 +67,18 @@ public: /// If BuildIndexPeriodMs is greater than 0, the symbol index will only be /// rebuilt periodically (one per \p BuildIndexPeriodMs); otherwise, index is /// rebuilt for each indexed file. - // FIXME: resource-dir injection should be hoisted somewhere common. - BackgroundIndex(Context BackgroundContext, llvm::StringRef ResourceDir, - const FileSystemProvider &, - const GlobalCompilationDatabase &CDB, - BackgroundIndexStorage::Factory IndexStorageFactory, - size_t BuildIndexPeriodMs = 0, - size_t ThreadPoolSize = llvm::hardware_concurrency()); + BackgroundIndex( + Context BackgroundContext, const FileSystemProvider &, + const GlobalCompilationDatabase &CDB, + BackgroundIndexStorage::Factory IndexStorageFactory, + size_t BuildIndexPeriodMs = 0, + size_t ThreadPoolSize = llvm::heavyweight_hardware_concurrency()); ~BackgroundIndex(); // Blocks while the current task finishes. // Enqueue translation units for indexing. // The indexing happens in a background thread, so the symbols will be // available sometime later. void enqueue(const std::vector<std::string> &ChangedFiles); - void enqueue(const std::string &File); // Cause background threads to stop after ther current task, any remaining // tasks will be discarded. @@ -92,14 +89,14 @@ public: blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds = 10); private: - /// Given index results from a TU, only update files in \p FilesToUpdate. - /// Also stores new index information on IndexStorage. + /// Given index results from a TU, only update symbols coming from files with + /// different digests than \p DigestsSnapshot. Also stores new index + /// information on IndexStorage. void update(llvm::StringRef MainFile, IndexFileIn Index, - const llvm::StringMap<FileDigest> &FilesToUpdate, + const llvm::StringMap<FileDigest> &DigestsSnapshot, BackgroundIndexStorage *IndexStorage); // configuration - std::string ResourceDir; const FileSystemProvider &FSProvider; const GlobalCompilationDatabase &CDB; Context BackgroundContext; @@ -118,6 +115,21 @@ private: std::mutex DigestsMu; BackgroundIndexStorage::Factory IndexStorageFactory; + struct Source { + std::string Path; + bool NeedsReIndexing; + Source(llvm::StringRef Path, bool NeedsReIndexing) + : Path(Path), NeedsReIndexing(NeedsReIndexing) {} + }; + // Loads the shards for a single TU and all of its dependencies. Returns the + // list of sources and whether they need to be re-indexed. + std::vector<Source> loadShard(const tooling::CompileCommand &Cmd, + BackgroundIndexStorage *IndexStorage, + llvm::StringSet<> &LoadedShards); + // Tries to load shards for the ChangedFiles. + std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>> + loadShards(std::vector<std::string> ChangedFiles); + void enqueue(tooling::CompileCommand Cmd, BackgroundIndexStorage *Storage); // queue management using Task = std::function<void()>; diff --git a/clangd/index/BackgroundIndexStorage.cpp b/clangd/index/BackgroundIndexStorage.cpp index a83bec6f..266b18ce 100644 --- a/clangd/index/BackgroundIndexStorage.cpp +++ b/clangd/index/BackgroundIndexStorage.cpp @@ -1,9 +1,8 @@ //== BackgroundIndexStorage.cpp - Provide caching support to BackgroundIndex ==/ // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/clangd/index/CanonicalIncludes.cpp b/clangd/index/CanonicalIncludes.cpp index 0cc44feb..6fa366d4 100644 --- a/clangd/index/CanonicalIncludes.cpp +++ b/clangd/index/CanonicalIncludes.cpp @@ -1,9 +1,8 @@ //===-- CanonicalIncludes.h - remap #inclue headers--------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,49 +12,50 @@ #include "llvm/Support/Path.h" #include <algorithm> -using namespace llvm; namespace clang { namespace clangd { namespace { const char IWYUPragma[] = "// IWYU pragma: private, include "; } // namespace -void CanonicalIncludes::addPathSuffixMapping(StringRef Suffix, - StringRef CanonicalPath) { - int Components = - std::distance(sys::path::begin(Suffix), sys::path::end(Suffix)); +void CanonicalIncludes::addPathSuffixMapping(llvm::StringRef Suffix, + llvm::StringRef CanonicalPath) { + int Components = std::distance(llvm::sys::path::begin(Suffix), + llvm::sys::path::end(Suffix)); MaxSuffixComponents = std::max(MaxSuffixComponents, Components); SuffixHeaderMapping[Suffix] = CanonicalPath; } -void CanonicalIncludes::addMapping(StringRef Path, StringRef CanonicalPath) { +void CanonicalIncludes::addMapping(llvm::StringRef Path, + llvm::StringRef CanonicalPath) { FullPathMapping[Path] = CanonicalPath; } -void CanonicalIncludes::addSymbolMapping(StringRef QualifiedName, - StringRef CanonicalPath) { +void CanonicalIncludes::addSymbolMapping(llvm::StringRef QualifiedName, + llvm::StringRef CanonicalPath) { this->SymbolMapping[QualifiedName] = CanonicalPath; } -StringRef CanonicalIncludes::mapHeader(ArrayRef<std::string> Headers, - StringRef QualifiedName) const { +llvm::StringRef +CanonicalIncludes::mapHeader(llvm::ArrayRef<std::string> Headers, + llvm::StringRef QualifiedName) const { assert(!Headers.empty()); auto SE = SymbolMapping.find(QualifiedName); if (SE != SymbolMapping.end()) return SE->second; // Find the first header such that the extension is not '.inc', and isn't a // recognized non-header file - auto I = llvm::find_if(Headers, [](StringRef Include) { + auto I = llvm::find_if(Headers, [](llvm::StringRef Include) { // Skip .inc file whose including header file should // be #included instead. return !Include.endswith(".inc"); }); if (I == Headers.end()) return Headers[0]; // Fallback to the declaring header. - StringRef Header = *I; + llvm::StringRef Header = *I; // If Header is not expected be included (e.g. .cc file), we fall back to // the declaring header. - StringRef Ext = sys::path::extension(Header).trim('.'); + llvm::StringRef Ext = llvm::sys::path::extension(Header).trim('.'); // Include-able headers must have precompile type. Treat files with // non-recognized extenstions (TY_INVALID) as headers. auto ExtType = driver::types::lookupTypeForExtension(Ext); @@ -68,7 +68,8 @@ StringRef CanonicalIncludes::mapHeader(ArrayRef<std::string> Headers, return MapIt->second; int Components = 1; - for (auto It = sys::path::rbegin(Header), End = sys::path::rend(Header); + for (auto It = llvm::sys::path::rbegin(Header), + End = llvm::sys::path::rend(Header); It != End && Components <= MaxSuffixComponents; ++It, ++Components) { auto SubPath = Header.substr(It->data() - Header.begin()); auto MappingIt = SuffixHeaderMapping.find(SubPath); @@ -85,7 +86,7 @@ collectIWYUHeaderMaps(CanonicalIncludes *Includes) { PragmaCommentHandler(CanonicalIncludes *Includes) : Includes(Includes) {} bool HandleComment(Preprocessor &PP, SourceRange Range) override { - StringRef Text = + llvm::StringRef Text = Lexer::getSourceText(CharSourceRange::getCharRange(Range), PP.getSourceManager(), PP.getLangOpts()); if (!Text.consume_front(IWYUPragma)) diff --git a/clangd/index/CanonicalIncludes.h b/clangd/index/CanonicalIncludes.h index 3751b000..c9baf0ad 100644 --- a/clangd/index/CanonicalIncludes.h +++ b/clangd/index/CanonicalIncludes.h @@ -1,9 +1,8 @@ //===-- CanonicalIncludes.h - remap #include header -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/clangd/index/FileIndex.cpp b/clangd/index/FileIndex.cpp index b944c72d..bc102d29 100644 --- a/clangd/index/FileIndex.cpp +++ b/clangd/index/FileIndex.cpp @@ -1,9 +1,8 @@ //===--- FileIndex.cpp - Indexes for files. ------------------------ C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -11,6 +10,7 @@ #include "ClangdUnit.h" #include "Logger.h" #include "SymbolCollector.h" +#include "index/CanonicalIncludes.h" #include "index/Index.h" #include "index/MemIndex.h" #include "index/Merge.h" @@ -24,20 +24,16 @@ #include "llvm/ADT/StringRef.h" #include <memory> -using namespace llvm; namespace clang { namespace clangd { static std::pair<SymbolSlab, RefSlab> indexSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP, - ArrayRef<Decl *> DeclsToIndex, bool IsIndexMainAST) { + llvm::ArrayRef<Decl *> DeclsToIndex, + const CanonicalIncludes &Includes, bool IsIndexMainAST) { SymbolCollector::Options CollectorOpts; - // FIXME(ioeric): we might also want to collect include headers. We would need - // to make sure all includes are canonicalized (with CanonicalIncludes), which - // is not trivial given the current way of collecting symbols: we only have - // AST at this point, but we also need preprocessor callbacks (e.g. - // CommentHandler for IWYU pragma) to canonicalize includes. - CollectorOpts.CollectIncludePath = false; + CollectorOpts.CollectIncludePath = true; + CollectorOpts.Includes = &Includes; CollectorOpts.CountReferences = false; CollectorOpts.Origin = SymbolOrigin::Dynamic; @@ -49,7 +45,7 @@ indexSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP, if (IsIndexMainAST) { // We only collect refs when indexing main AST. CollectorOpts.RefFilter = RefKind::All; - }else { + } else { IndexOpts.IndexMacrosInPreprocessor = true; CollectorOpts.CollectMacro = true; } @@ -74,16 +70,16 @@ indexSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP, std::pair<SymbolSlab, RefSlab> indexMainDecls(ParsedAST &AST) { return indexSymbols(AST.getASTContext(), AST.getPreprocessorPtr(), - AST.getLocalTopLevelDecls(), + AST.getLocalTopLevelDecls(), AST.getCanonicalIncludes(), /*IsIndexMainAST=*/true); } -SymbolSlab indexHeaderSymbols(ASTContext &AST, - std::shared_ptr<Preprocessor> PP) { +SymbolSlab indexHeaderSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP, + const CanonicalIncludes &Includes) { std::vector<Decl *> DeclsToIndex( AST.getTranslationUnitDecl()->decls().begin(), AST.getTranslationUnitDecl()->decls().end()); - return indexSymbols(AST, std::move(PP), DeclsToIndex, + return indexSymbols(AST, std::move(PP), DeclsToIndex, Includes, /*IsIndexMainAST=*/false) .first; } @@ -116,7 +112,7 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle) { std::vector<Symbol> SymsStorage; switch (DuplicateHandle) { case DuplicateHandling::Merge: { - DenseMap<SymbolID, Symbol> Merged; + llvm::DenseMap<SymbolID, Symbol> Merged; for (const auto &Slab : SymbolSlabs) { for (const auto &Sym : *Slab) { auto I = Merged.try_emplace(Sym.ID, Sym); @@ -143,9 +139,9 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle) { } std::vector<Ref> RefsStorage; // Contiguous ranges for each SymbolID. - DenseMap<SymbolID, ArrayRef<Ref>> AllRefs; + llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> AllRefs; { - DenseMap<SymbolID, SmallVector<Ref, 4>> MergedRefs; + llvm::DenseMap<SymbolID, llvm::SmallVector<Ref, 4>> MergedRefs; size_t Count = 0; for (const auto &RefSlab : RefSlabs) for (const auto &Sym : *RefSlab) { @@ -161,8 +157,8 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle) { llvm::copy(SymRefs, back_inserter(RefsStorage)); AllRefs.try_emplace( Sym.first, - ArrayRef<Ref>(&RefsStorage[RefsStorage.size() - SymRefs.size()], - SymRefs.size())); + llvm::ArrayRef<Ref>(&RefsStorage[RefsStorage.size() - SymRefs.size()], + SymRefs.size())); } } @@ -177,13 +173,13 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle) { switch (Type) { case IndexType::Light: return llvm::make_unique<MemIndex>( - make_pointee_range(AllSymbols), std::move(AllRefs), + llvm::make_pointee_range(AllSymbols), std::move(AllRefs), std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs), std::move(RefsStorage), std::move(SymsStorage)), StorageSize); case IndexType::Heavy: return llvm::make_unique<dex::Dex>( - make_pointee_range(AllSymbols), std::move(AllRefs), + llvm::make_pointee_range(AllSymbols), std::move(AllRefs), std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs), std::move(RefsStorage), std::move(SymsStorage)), StorageSize); @@ -197,8 +193,9 @@ FileIndex::FileIndex(bool UseDex) MainFileIndex(llvm::make_unique<MemIndex>()) {} void FileIndex::updatePreamble(PathRef Path, ASTContext &AST, - std::shared_ptr<Preprocessor> PP) { - auto Symbols = indexHeaderSymbols(AST, std::move(PP)); + std::shared_ptr<Preprocessor> PP, + const CanonicalIncludes &Includes) { + auto Symbols = indexHeaderSymbols(AST, std::move(PP), Includes); PreambleSymbols.update(Path, llvm::make_unique<SymbolSlab>(std::move(Symbols)), llvm::make_unique<RefSlab>()); diff --git a/clangd/index/FileIndex.h b/clangd/index/FileIndex.h index 92e3b2b6..f1e37c33 100644 --- a/clangd/index/FileIndex.h +++ b/clangd/index/FileIndex.h @@ -1,9 +1,8 @@ //===--- FileIndex.h - Index for files. ---------------------------- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,6 +19,7 @@ #include "Index.h" #include "MemIndex.h" #include "Merge.h" +#include "index/CanonicalIncludes.h" #include "clang/Lex/Preprocessor.h" #include <memory> @@ -85,7 +85,8 @@ public: /// Update preamble symbols of file \p Path with all declarations in \p AST /// and macros in \p PP. void updatePreamble(PathRef Path, ASTContext &AST, - std::shared_ptr<Preprocessor> PP); + std::shared_ptr<Preprocessor> PP, + const CanonicalIncludes &Includes); /// Update symbols and references from main file \p Path with /// `indexMainDecls`. @@ -125,8 +126,8 @@ std::pair<SymbolSlab, RefSlab> indexMainDecls(ParsedAST &AST); /// Idex declarations from \p AST and macros from \p PP that are declared in /// included headers. -SymbolSlab indexHeaderSymbols(ASTContext &AST, - std::shared_ptr<Preprocessor> PP); +SymbolSlab indexHeaderSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP, + const CanonicalIncludes &Includes); } // namespace clangd } // namespace clang diff --git a/clangd/index/Index.cpp b/clangd/index/Index.cpp index dd4b5eb3..36c591d9 100644 --- a/clangd/index/Index.cpp +++ b/clangd/index/Index.cpp @@ -1,9 +1,8 @@ //===--- Index.cpp -----------------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -14,7 +13,6 @@ #include "llvm/Support/Error.h" #include "llvm/Support/raw_ostream.h" -using namespace llvm; namespace clang { namespace clangd { @@ -35,14 +33,14 @@ void SymbolLocation::Position::setColumn(uint32_t Col) { Column = Col; } -raw_ostream &operator<<(raw_ostream &OS, const SymbolLocation &L) { +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const SymbolLocation &L) { if (!L) return OS << "(none)"; return OS << L.FileURI << "[" << L.Start.line() << ":" << L.Start.column() << "-" << L.End.line() << ":" << L.End.column() << ")"; } -raw_ostream &operator<<(raw_ostream &OS, SymbolOrigin O) { +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, SymbolOrigin O) { if (O == SymbolOrigin::Unknown) return OS << "unknown"; constexpr static char Sigils[] = "ADSM4567"; @@ -52,18 +50,18 @@ raw_ostream &operator<<(raw_ostream &OS, SymbolOrigin O) { return OS; } -raw_ostream &operator<<(raw_ostream &OS, Symbol::SymbolFlag F) { +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, Symbol::SymbolFlag F) { if (F == Symbol::None) return OS << "None"; - std::string s; + std::string S; if (F & Symbol::Deprecated) - s += "deprecated|"; + S += "deprecated|"; if (F & Symbol::IndexedForCodeCompletion) - s += "completion|"; - return OS << StringRef(s).rtrim('|'); + S += "completion|"; + return OS << llvm::StringRef(S).rtrim('|'); } -raw_ostream &operator<<(raw_ostream &OS, const Symbol &S) { +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Symbol &S) { return OS << S.Scope << S.Name; } @@ -85,8 +83,8 @@ SymbolSlab::const_iterator SymbolSlab::find(const SymbolID &ID) const { } // Copy the underlying data of the symbol into the owned arena. -static void own(Symbol &S, UniqueStringSaver &Strings) { - visitStrings(S, [&](StringRef &V) { V = Strings.save(V); }); +static void own(Symbol &S, llvm::UniqueStringSaver &Strings) { + visitStrings(S, [&](llvm::StringRef &V) { V = Strings.save(V); }); } void SymbolSlab::Builder::insert(const Symbol &S) { @@ -106,14 +104,14 @@ SymbolSlab SymbolSlab::Builder::build() && { llvm::sort(Symbols, [](const Symbol &L, const Symbol &R) { return L.ID < R.ID; }); // We may have unused strings from overwritten symbols. Build a new arena. - BumpPtrAllocator NewArena; - UniqueStringSaver Strings(NewArena); + llvm::BumpPtrAllocator NewArena; + llvm::UniqueStringSaver Strings(NewArena); for (auto &S : Symbols) own(S, Strings); return SymbolSlab(std::move(NewArena), std::move(Symbols)); } -raw_ostream &operator<<(raw_ostream &OS, RefKind K) { +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, RefKind K) { if (K == RefKind::Unknown) return OS << "Unknown"; static const std::vector<const char *> Messages = {"Decl", "Def", "Ref"}; @@ -129,7 +127,7 @@ raw_ostream &operator<<(raw_ostream &OS, RefKind K) { return OS; } -raw_ostream &operator<<(raw_ostream &OS, const Ref &R) { +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Ref &R) { return OS << R.Location << ":" << R.Kind; } @@ -143,7 +141,7 @@ void RefSlab::Builder::insert(const SymbolID &ID, const Ref &S) { RefSlab RefSlab::Builder::build() && { // We can reuse the arena, as it only has unique strings and we need them all. // Reallocate refs on the arena to reduce waste and indirections when reading. - std::vector<std::pair<SymbolID, ArrayRef<Ref>>> Result; + std::vector<std::pair<SymbolID, llvm::ArrayRef<Ref>>> Result; Result.reserve(Refs.size()); size_t NumRefs = 0; for (auto &Sym : Refs) { @@ -155,7 +153,7 @@ RefSlab RefSlab::Builder::build() && { NumRefs += SymRefs.size(); auto *Array = Arena.Allocate<Ref>(SymRefs.size()); std::uninitialized_copy(SymRefs.begin(), SymRefs.end(), Array); - Result.emplace_back(Sym.first, ArrayRef<Ref>(Array, SymRefs.size())); + Result.emplace_back(Sym.first, llvm::ArrayRef<Ref>(Array, SymRefs.size())); } return RefSlab(std::move(Result), std::move(Arena), NumRefs); } @@ -174,40 +172,42 @@ std::shared_ptr<SymbolIndex> SwapIndex::snapshot() const { return Index; } -bool fromJSON(const json::Value &Parameters, FuzzyFindRequest &Request) { - json::ObjectMapper O(Parameters); +bool fromJSON(const llvm::json::Value &Parameters, FuzzyFindRequest &Request) { + llvm::json::ObjectMapper O(Parameters); int64_t Limit; bool OK = O && O.map("Query", Request.Query) && O.map("Scopes", Request.Scopes) && O.map("AnyScope", Request.AnyScope) && O.map("Limit", Limit) && O.map("RestrictForCodeCompletion", Request.RestrictForCodeCompletion) && - O.map("ProximityPaths", Request.ProximityPaths); + O.map("ProximityPaths", Request.ProximityPaths) && + O.map("PreferredTypes", Request.PreferredTypes); if (OK && Limit <= std::numeric_limits<uint32_t>::max()) Request.Limit = Limit; return OK; } -json::Value toJSON(const FuzzyFindRequest &Request) { - return json::Object{ +llvm::json::Value toJSON(const FuzzyFindRequest &Request) { + return llvm::json::Object{ {"Query", Request.Query}, - {"Scopes", json::Array{Request.Scopes}}, + {"Scopes", llvm::json::Array{Request.Scopes}}, {"AnyScope", Request.AnyScope}, {"Limit", Request.Limit}, {"RestrictForCodeCompletion", Request.RestrictForCodeCompletion}, - {"ProximityPaths", json::Array{Request.ProximityPaths}}, + {"ProximityPaths", llvm::json::Array{Request.ProximityPaths}}, + {"PreferredTypes", llvm::json::Array{Request.PreferredTypes}}, }; } bool SwapIndex::fuzzyFind(const FuzzyFindRequest &R, - function_ref<void(const Symbol &)> CB) const { + llvm::function_ref<void(const Symbol &)> CB) const { return snapshot()->fuzzyFind(R, CB); } void SwapIndex::lookup(const LookupRequest &R, - function_ref<void(const Symbol &)> CB) const { + llvm::function_ref<void(const Symbol &)> CB) const { return snapshot()->lookup(R, CB); } void SwapIndex::refs(const RefsRequest &R, - function_ref<void(const Ref &)> CB) const { + llvm::function_ref<void(const Ref &)> CB) const { return snapshot()->refs(R, CB); } size_t SwapIndex::estimateMemoryUsage() const { diff --git a/clangd/index/Index.h b/clangd/index/Index.h index a5552d4f..4f6feb93 100644 --- a/clangd/index/Index.h +++ b/clangd/index/Index.h @@ -1,9 +1,8 @@ //===--- Index.h -------------------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -185,19 +184,23 @@ struct Symbol { SymbolOrigin Origin = SymbolOrigin::Unknown; /// A brief description of the symbol that can be appended in the completion /// candidate list. For example, "(X x, Y y) const" is a function signature. + /// Only set when the symbol is indexed for completion. llvm::StringRef Signature; /// What to insert when completing this symbol, after the symbol name. /// This is in LSP snippet syntax (e.g. "({$0})" for a no-args function). /// (When snippets are disabled, the symbol name alone is used). + /// Only set when the symbol is indexed for completion. llvm::StringRef CompletionSnippetSuffix; /// Documentation including comment for the symbol declaration. llvm::StringRef Documentation; /// Type when this symbol is used in an expression. (Short display form). /// e.g. return type of a function, or type of a variable. + /// Only set when the symbol is indexed for completion. llvm::StringRef ReturnType; /// Raw representation of the OpaqueType of the symbol, used for scoring /// purposes. + /// Only set when the symbol is indexed for completion. llvm::StringRef Type; struct IncludeHeaderWithReferences { @@ -223,17 +226,22 @@ struct Symbol { /// - If we haven't seen a definition, this covers all declarations. /// - If we have seen a definition, this covers declarations visible from /// any definition. + /// Only set when the symbol is indexed for completion. llvm::SmallVector<IncludeHeaderWithReferences, 1> IncludeHeaders; enum SymbolFlag : uint8_t { None = 0, /// Whether or not this symbol is meant to be used for the code completion. /// See also isIndexedForCodeCompletion(). + /// Note that we don't store completion information (signature, snippet, + /// type, inclues) if the symbol is not indexed for code completion. IndexedForCodeCompletion = 1 << 0, /// Indicates if the symbol is deprecated. Deprecated = 1 << 1, // Symbol is an implementation detail. ImplementationDetail = 1 << 2, + // Symbol is visible to other files (not e.g. a static helper function). + VisibleOutsideFile = 1 << 3, }; SymbolFlag Flags = SymbolFlag::None; @@ -446,14 +454,15 @@ struct FuzzyFindRequest { /// Contextually relevant files (e.g. the file we're code-completing in). /// Paths should be absolute. std::vector<std::string> ProximityPaths; - - // FIXME(ibiryukov): add expected type to the request. + /// Preferred types of symbols. These are raw representation of `OpaqueType`. + std::vector<std::string> PreferredTypes; bool operator==(const FuzzyFindRequest &Req) const { return std::tie(Query, Scopes, Limit, RestrictForCodeCompletion, - ProximityPaths) == + ProximityPaths, PreferredTypes) == std::tie(Req.Query, Req.Scopes, Req.Limit, - Req.RestrictForCodeCompletion, Req.ProximityPaths); + Req.RestrictForCodeCompletion, Req.ProximityPaths, + Req.PreferredTypes); } bool operator!=(const FuzzyFindRequest &Req) const { return !(*this == Req); } }; @@ -467,6 +476,10 @@ struct LookupRequest { struct RefsRequest { llvm::DenseSet<SymbolID> IDs; RefKind Filter = RefKind::All; + /// If set, limit the number of refers returned from the index. The index may + /// choose to return less than this, e.g. it tries to avoid returning stale + /// results. + llvm::Optional<uint32_t> Limit; }; /// Interface for symbol indexes that can be used for searching or diff --git a/clangd/index/IndexAction.cpp b/clangd/index/IndexAction.cpp index 63e1155f..a6df64b7 100644 --- a/clangd/index/IndexAction.cpp +++ b/clangd/index/IndexAction.cpp @@ -4,7 +4,6 @@ #include "clang/Index/IndexingAction.h" #include "clang/Tooling/Tooling.h" -using namespace llvm; namespace clang { namespace clangd { namespace { @@ -63,10 +62,10 @@ public: // Add edges from including files to includes. void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok, - StringRef FileName, bool IsAngled, + llvm::StringRef FileName, bool IsAngled, CharSourceRange FilenameRange, const FileEntry *File, - StringRef SearchPath, StringRef RelativePath, - const Module *Imported, + llvm::StringRef SearchPath, + llvm::StringRef RelativePath, const Module *Imported, SrcMgr::CharacteristicKind FileType) override { auto IncludeURI = toURI(File); if (!IncludeURI) @@ -116,8 +115,8 @@ public: Includes(std::move(Includes)), PragmaHandler(collectIWYUHeaderMaps(this->Includes.get())) {} - std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI, - StringRef InFile) override { + std::unique_ptr<ASTConsumer> + CreateASTConsumer(CompilerInstance &CI, llvm::StringRef InFile) override { CI.getPreprocessor().addCommentHandler(PragmaHandler.get()); if (IncludeGraphCallback != nullptr) CI.getPreprocessor().addPPCallbacks( @@ -137,7 +136,7 @@ public: const auto &CI = getCompilerInstance(); if (CI.hasDiagnostics() && CI.getDiagnostics().hasUncompilableErrorOccurred()) { - errs() << "Skipping TU due to uncompilable errors\n"; + llvm::errs() << "Skipping TU due to uncompilable errors\n"; return; } SymbolsCallback(Collector->takeSymbols()); diff --git a/clangd/index/IndexAction.h b/clangd/index/IndexAction.h index f2c5298f..748b26e1 100644 --- a/clangd/index/IndexAction.h +++ b/clangd/index/IndexAction.h @@ -1,9 +1,8 @@ //===--- IndexAction.h - Run the indexer as a frontend action ----*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/clangd/index/MemIndex.cpp b/clangd/index/MemIndex.cpp index 42340e8e..531d1f6d 100644 --- a/clangd/index/MemIndex.cpp +++ b/clangd/index/MemIndex.cpp @@ -1,9 +1,8 @@ //===--- MemIndex.cpp - Dynamic in-memory symbol index. ----------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===-------------------------------------------------------------------===// @@ -13,7 +12,6 @@ #include "Quality.h" #include "Trace.h" -using namespace llvm; namespace clang { namespace clangd { @@ -25,8 +23,9 @@ std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab, RefSlab Refs) { BackingDataSize); } -bool MemIndex::fuzzyFind(const FuzzyFindRequest &Req, - function_ref<void(const Symbol &)> Callback) const { +bool MemIndex::fuzzyFind( + const FuzzyFindRequest &Req, + llvm::function_ref<void(const Symbol &)> Callback) const { assert(!StringRef(Req.Query).contains("::") && "There must be no :: in query."); trace::Span Tracer("MemIndex fuzzyFind"); @@ -39,7 +38,7 @@ bool MemIndex::fuzzyFind(const FuzzyFindRequest &Req, const Symbol *Sym = Pair.second; // Exact match against all possible scopes. - if (!Req.AnyScope && !is_contained(Req.Scopes, Sym->Scope)) + if (!Req.AnyScope && !llvm::is_contained(Req.Scopes, Sym->Scope)) continue; if (Req.RestrictForCodeCompletion && !(Sym->Flags & Symbol::IndexedForCodeCompletion)) @@ -57,7 +56,7 @@ bool MemIndex::fuzzyFind(const FuzzyFindRequest &Req, } void MemIndex::lookup(const LookupRequest &Req, - function_ref<void(const Symbol &)> Callback) const { + llvm::function_ref<void(const Symbol &)> Callback) const { trace::Span Tracer("MemIndex lookup"); for (const auto &ID : Req.IDs) { auto I = Index.find(ID); @@ -67,15 +66,20 @@ void MemIndex::lookup(const LookupRequest &Req, } void MemIndex::refs(const RefsRequest &Req, - function_ref<void(const Ref &)> Callback) const { + llvm::function_ref<void(const Ref &)> Callback) const { trace::Span Tracer("MemIndex refs"); + uint32_t Remaining = + Req.Limit.getValueOr(std::numeric_limits<uint32_t>::max()); for (const auto &ReqID : Req.IDs) { auto SymRefs = Refs.find(ReqID); if (SymRefs == Refs.end()) continue; - for (const auto &O : SymRefs->second) - if (static_cast<int>(Req.Filter & O.Kind)) + for (const auto &O : SymRefs->second) { + if (Remaining > 0 && static_cast<int>(Req.Filter & O.Kind)) { + --Remaining; Callback(O); + } + } } } diff --git a/clangd/index/MemIndex.h b/clangd/index/MemIndex.h index 24f2ba19..47227fd7 100644 --- a/clangd/index/MemIndex.h +++ b/clangd/index/MemIndex.h @@ -1,9 +1,8 @@ //===--- MemIndex.h - Dynamic in-memory symbol index. -------------- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/clangd/index/Merge.cpp b/clangd/index/Merge.cpp index 42616f05..65e9b86d 100644 --- a/clangd/index/Merge.cpp +++ b/clangd/index/Merge.cpp @@ -1,20 +1,22 @@ //===--- Merge.cpp -----------------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "Merge.h" #include "Logger.h" #include "Trace.h" +#include "index/Index.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <iterator> -using namespace llvm; namespace clang { namespace clangd { @@ -23,8 +25,9 @@ namespace clangd { // - find the generating file from each Symbol which is Static-only // - ask Dynamic if it has that file (needs new SymbolIndex method) // - if so, drop the Symbol. -bool MergedIndex::fuzzyFind(const FuzzyFindRequest &Req, - function_ref<void(const Symbol &)> Callback) const { +bool MergedIndex::fuzzyFind( + const FuzzyFindRequest &Req, + llvm::function_ref<void(const Symbol &)> Callback) const { // We can't step through both sources in parallel. So: // 1) query all dynamic symbols, slurping results into a slab // 2) query the static symbols, for each one: @@ -43,7 +46,7 @@ bool MergedIndex::fuzzyFind(const FuzzyFindRequest &Req, }); SymbolSlab Dyn = std::move(DynB).build(); - DenseSet<SymbolID> SeenDynamicSymbols; + llvm::DenseSet<SymbolID> SeenDynamicSymbols; More |= Static->fuzzyFind(Req, [&](const Symbol &S) { auto DynS = Dyn.find(S.ID); ++StaticCount; @@ -62,8 +65,9 @@ bool MergedIndex::fuzzyFind(const FuzzyFindRequest &Req, return More; } -void MergedIndex::lookup(const LookupRequest &Req, - function_ref<void(const Symbol &)> Callback) const { +void MergedIndex::lookup( + const LookupRequest &Req, + llvm::function_ref<void(const Symbol &)> Callback) const { trace::Span Tracer("MergedIndex lookup"); SymbolSlab::Builder B; @@ -84,8 +88,10 @@ void MergedIndex::lookup(const LookupRequest &Req, } void MergedIndex::refs(const RefsRequest &Req, - function_ref<void(const Ref &)> Callback) const { + llvm::function_ref<void(const Ref &)> Callback) const { trace::Span Tracer("MergedIndex refs"); + uint32_t Remaining = + Req.Limit.getValueOr(std::numeric_limits<uint32_t>::max()); // We don't want duplicated refs from the static/dynamic indexes, // and we can't reliably duplicate them because offsets may differ slightly. // We consider the dynamic index authoritative and report all its refs, @@ -94,17 +100,41 @@ void MergedIndex::refs(const RefsRequest &Req, // FIXME: The heuristic fails if the dynamic index contains a file, but all // refs were removed (we will report stale ones from the static index). // Ultimately we should explicit check which index has the file instead. - StringSet<> DynamicIndexFileURIs; + llvm::StringSet<> DynamicIndexFileURIs; Dynamic->refs(Req, [&](const Ref &O) { DynamicIndexFileURIs.insert(O.Location.FileURI); Callback(O); + --Remaining; }); + if (Remaining == 0) + return; + // We return less than Req.Limit if static index returns more refs for dirty + // files. Static->refs(Req, [&](const Ref &O) { - if (!DynamicIndexFileURIs.count(O.Location.FileURI)) + if (Remaining > 0 && !DynamicIndexFileURIs.count(O.Location.FileURI)) { + --Remaining; Callback(O); + } }); } +// Returns true if \p L is (strictly) preferred to \p R (e.g. by file paths). If +// neither is preferred, this returns false. +bool prefer(const SymbolLocation &L, const SymbolLocation &R) { + if (!L) + return false; + if (!R) + return true; + auto HasCodeGenSuffix = [](const SymbolLocation &Loc) { + constexpr static const char *CodegenSuffixes[] = {".proto"}; + return std::any_of(std::begin(CodegenSuffixes), std::end(CodegenSuffixes), + [&](llvm::StringRef Suffix) { + return llvm::StringRef(Loc.FileURI).endswith(Suffix); + }); + }; + return HasCodeGenSuffix(L) && !HasCodeGenSuffix(R); +} + Symbol mergeSymbol(const Symbol &L, const Symbol &R) { assert(L.ID == R.ID); // We prefer information from TUs that saw the definition. @@ -119,12 +149,11 @@ Symbol mergeSymbol(const Symbol &L, const Symbol &R) { Symbol S = PreferR ? R : L; // The target symbol we're merging into. const Symbol &O = PreferR ? L : R; // The "other" less-preferred symbol. - // For each optional field, fill it from O if missing in S. - // (It might be missing in O too, but that's a no-op). - if (!S.Definition) - S.Definition = O.Definition; - if (!S.CanonicalDeclaration) + // Only use locations in \p O if it's (strictly) preferred. + if (prefer(O.CanonicalDeclaration, S.CanonicalDeclaration)) S.CanonicalDeclaration = O.CanonicalDeclaration; + if (prefer(O.Definition, S.Definition)) + S.Definition = O.Definition; S.References += O.References; if (S.Signature == "") S.Signature = O.Signature; @@ -134,6 +163,8 @@ Symbol mergeSymbol(const Symbol &L, const Symbol &R) { S.Documentation = O.Documentation; if (S.ReturnType == "") S.ReturnType = O.ReturnType; + if (S.Type == "") + S.Type = O.Type; for (const auto &OI : O.IncludeHeaders) { bool Found = false; for (auto &SI : S.IncludeHeaders) { diff --git a/clangd/index/Merge.h b/clangd/index/Merge.h index 7569c7a4..5954b6bc 100644 --- a/clangd/index/Merge.h +++ b/clangd/index/Merge.h @@ -1,9 +1,8 @@ //===--- Merge.h -------------------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/clangd/index/Serialization.cpp b/clangd/index/Serialization.cpp index 1d17a42c..20c43ab0 100644 --- a/clangd/index/Serialization.cpp +++ b/clangd/index/Serialization.cpp @@ -1,9 +1,8 @@ //===-- Serialization.cpp - Binary serialization of index data ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -17,12 +16,12 @@ #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" -using namespace llvm; namespace clang { namespace clangd { namespace { -Error makeError(const Twine &Msg) { - return make_error<StringError>(Msg, inconvertibleErrorCode()); +llvm::Error makeError(const llvm::Twine &Msg) { + return llvm::make_error<llvm::StringError>(Msg, + llvm::inconvertibleErrorCode()); } // IO PRIMITIVES @@ -40,14 +39,14 @@ class Reader { bool Err = false; public: - Reader(StringRef Data) : Begin(Data.begin()), End(Data.end()) {} + Reader(llvm::StringRef Data) : Begin(Data.begin()), End(Data.end()) {} // The "error" bit is set by reading past EOF or reading invalid data. // When in an error state, reads may return zero values: callers should check. bool err() const { return Err; } // Did we read all the data, or encounter an error? bool eof() const { return Begin == End || Err; } // All the data we didn't read yet. - StringRef rest() const { return StringRef(Begin, End - Begin); } + llvm::StringRef rest() const { return llvm::StringRef(Begin, End - Begin); } uint8_t consume8() { if (LLVM_UNLIKELY(Begin == End)) { @@ -62,17 +61,17 @@ public: Err = true; return 0; } - auto Ret = support::endian::read32le(Begin); + auto Ret = llvm::support::endian::read32le(Begin); Begin += 4; return Ret; } - StringRef consume(int N) { + llvm::StringRef consume(int N) { if (LLVM_UNLIKELY(Begin + N > End)) { Err = true; - return StringRef(); + return llvm::StringRef(); } - StringRef Ret(Begin, N); + llvm::StringRef Ret(Begin, N); Begin += N; return Ret; } @@ -90,28 +89,28 @@ public: return Val; } - StringRef consumeString(ArrayRef<StringRef> Strings) { + llvm::StringRef consumeString(llvm::ArrayRef<llvm::StringRef> Strings) { auto StringIndex = consumeVar(); if (LLVM_UNLIKELY(StringIndex >= Strings.size())) { Err = true; - return StringRef(); + return llvm::StringRef(); } return Strings[StringIndex]; } SymbolID consumeID() { - StringRef Raw = consume(SymbolID::RawSize); // short if truncated. + llvm::StringRef Raw = consume(SymbolID::RawSize); // short if truncated. return LLVM_UNLIKELY(err()) ? SymbolID() : SymbolID::fromRaw(Raw); } }; -void write32(uint32_t I, raw_ostream &OS) { - char buf[4]; - support::endian::write32le(buf, I); - OS.write(buf, sizeof(buf)); +void write32(uint32_t I, llvm::raw_ostream &OS) { + char Buf[4]; + llvm::support::endian::write32le(Buf, I); + OS.write(Buf, sizeof(Buf)); } -void writeVar(uint32_t I, raw_ostream &OS) { +void writeVar(uint32_t I, llvm::raw_ostream &OS) { constexpr static uint8_t More = 1 << 7; if (LLVM_LIKELY(I < 1 << 7)) { OS.write(I); @@ -142,10 +141,10 @@ void writeVar(uint32_t I, raw_ostream &OS) { // Maps each string to a canonical representation. // Strings remain owned externally (e.g. by SymbolSlab). class StringTableOut { - DenseSet<StringRef> Unique; - std::vector<StringRef> Sorted; + llvm::DenseSet<llvm::StringRef> Unique; + std::vector<llvm::StringRef> Sorted; // Since strings are interned, look up can be by pointer. - DenseMap<std::pair<const char *, size_t>, unsigned> Index; + llvm::DenseMap<std::pair<const char *, size_t>, unsigned> Index; public: StringTableOut() { @@ -154,22 +153,22 @@ public: Unique.insert(""); } // Add a string to the table. Overwrites S if an identical string exists. - void intern(StringRef &S) { S = *Unique.insert(S).first; }; + void intern(llvm::StringRef &S) { S = *Unique.insert(S).first; }; // Finalize the table and write it to OS. No more strings may be added. - void finalize(raw_ostream &OS) { + void finalize(llvm::raw_ostream &OS) { Sorted = {Unique.begin(), Unique.end()}; llvm::sort(Sorted); for (unsigned I = 0; I < Sorted.size(); ++I) Index.try_emplace({Sorted[I].data(), Sorted[I].size()}, I); std::string RawTable; - for (StringRef S : Sorted) { + for (llvm::StringRef S : Sorted) { RawTable.append(S); RawTable.push_back(0); } - if (zlib::isAvailable()) { - SmallString<1> Compressed; - cantFail(zlib::compress(RawTable, Compressed)); + if (llvm::zlib::isAvailable()) { + llvm::SmallString<1> Compressed; + llvm::cantFail(llvm::zlib::compress(RawTable, Compressed)); write32(RawTable.size(), OS); OS << Compressed; } else { @@ -178,7 +177,7 @@ public: } } // Get the ID of an string, which must be interned. Table must be finalized. - unsigned index(StringRef S) const { + unsigned index(llvm::StringRef S) const { assert(!Sorted.empty() && "table not finalized"); assert(Index.count({S.data(), S.size()}) && "string not interned"); return Index.find({S.data(), S.size()})->second; @@ -186,33 +185,33 @@ public: }; struct StringTableIn { - BumpPtrAllocator Arena; - std::vector<StringRef> Strings; + llvm::BumpPtrAllocator Arena; + std::vector<llvm::StringRef> Strings; }; -Expected<StringTableIn> readStringTable(StringRef Data) { +llvm::Expected<StringTableIn> readStringTable(llvm::StringRef Data) { Reader R(Data); size_t UncompressedSize = R.consume32(); if (R.err()) return makeError("Truncated string table"); - StringRef Uncompressed; - SmallString<1> UncompressedStorage; + llvm::StringRef Uncompressed; + llvm::SmallString<1> UncompressedStorage; if (UncompressedSize == 0) // No compression Uncompressed = R.rest(); else { - if (Error E = llvm::zlib::uncompress(R.rest(), UncompressedStorage, - UncompressedSize)) + if (llvm::Error E = llvm::zlib::uncompress(R.rest(), UncompressedStorage, + UncompressedSize)) return std::move(E); Uncompressed = UncompressedStorage; } StringTableIn Table; - StringSaver Saver(Table.Arena); + llvm::StringSaver Saver(Table.Arena); R = Reader(Uncompressed); for (Reader R(Uncompressed); !R.eof();) { auto Len = R.rest().find(0); - if (Len == StringRef::npos) + if (Len == llvm::StringRef::npos) return makeError("Bad string table: not null terminated"); Table.Strings.push_back(Saver.save(R.consume(Len))); R.consume8(); @@ -229,7 +228,7 @@ Expected<StringTableIn> readStringTable(StringRef Data) { // - most numbers encode as varint void writeLocation(const SymbolLocation &Loc, const StringTableOut &Strings, - raw_ostream &OS) { + llvm::raw_ostream &OS) { writeVar(Strings.index(Loc.FileURI), OS); for (const auto &Endpoint : {Loc.Start, Loc.End}) { writeVar(Endpoint.line(), OS); @@ -237,7 +236,8 @@ void writeLocation(const SymbolLocation &Loc, const StringTableOut &Strings, } } -SymbolLocation readLocation(Reader &Data, ArrayRef<StringRef> Strings) { +SymbolLocation readLocation(Reader &Data, + llvm::ArrayRef<llvm::StringRef> Strings) { SymbolLocation Loc; Loc.FileURI = Data.consumeString(Strings).data(); for (auto *Endpoint : {&Loc.Start, &Loc.End}) { @@ -261,7 +261,8 @@ IncludeGraphNode readIncludeGraphNode(Reader &Data, } void writeIncludeGraphNode(const IncludeGraphNode &IGN, - const StringTableOut &Strings, raw_ostream &OS) { + const StringTableOut &Strings, + llvm::raw_ostream &OS) { OS.write(IGN.IsTU); writeVar(Strings.index(IGN.URI), OS); llvm::StringRef Hash(reinterpret_cast<const char *>(IGN.Digest.data()), @@ -273,7 +274,7 @@ void writeIncludeGraphNode(const IncludeGraphNode &IGN, } void writeSymbol(const Symbol &Sym, const StringTableOut &Strings, - raw_ostream &OS) { + llvm::raw_ostream &OS) { OS << Sym.ID.raw(); // TODO: once we start writing xrefs and posting lists, // symbol IDs should probably be in a string table. OS.write(static_cast<uint8_t>(Sym.SymInfo.Kind)); @@ -300,7 +301,7 @@ void writeSymbol(const Symbol &Sym, const StringTableOut &Strings, WriteInclude(Include); } -Symbol readSymbol(Reader &Data, ArrayRef<StringRef> Strings) { +Symbol readSymbol(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) { Symbol Sym; Sym.ID = Data.consumeID(); Sym.SymInfo.Kind = static_cast<index::SymbolKind>(Data.consume8()); @@ -332,8 +333,8 @@ Symbol readSymbol(Reader &Data, ArrayRef<StringRef> Strings) { // - Ref[NumRefs] // Fields of Ref are encoded in turn, see implementation. -void writeRefs(const SymbolID &ID, ArrayRef<Ref> Refs, - const StringTableOut &Strings, raw_ostream &OS) { +void writeRefs(const SymbolID &ID, llvm::ArrayRef<Ref> Refs, + const StringTableOut &Strings, llvm::raw_ostream &OS) { OS << ID.raw(); writeVar(Refs.size(), OS); for (const auto &Ref : Refs) { @@ -342,8 +343,8 @@ void writeRefs(const SymbolID &ID, ArrayRef<Ref> Refs, } } -std::pair<SymbolID, std::vector<Ref>> readRefs(Reader &Data, - ArrayRef<StringRef> Strings) { +std::pair<SymbolID, std::vector<Ref>> +readRefs(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) { std::pair<SymbolID, std::vector<Ref>> Result; Result.first = Data.consumeID(); Result.second.resize(Data.consumeVar()); @@ -368,17 +369,18 @@ std::pair<SymbolID, std::vector<Ref>> readRefs(Reader &Data, // data. Later we may want to support some backward compatibility. constexpr static uint32_t Version = 8; -Expected<IndexFileIn> readRIFF(StringRef Data) { +llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data) { auto RIFF = riff::readFile(Data); if (!RIFF) return RIFF.takeError(); if (RIFF->Type != riff::fourCC("CdIx")) return makeError("wrong RIFF type"); - StringMap<StringRef> Chunks; + llvm::StringMap<llvm::StringRef> Chunks; for (const auto &Chunk : RIFF->Chunks) - Chunks.try_emplace(StringRef(Chunk.ID.data(), Chunk.ID.size()), Chunk.Data); + Chunks.try_emplace(llvm::StringRef(Chunk.ID.data(), Chunk.ID.size()), + Chunk.Data); - for (StringRef RequiredChunk : {"meta", "stri"}) + for (llvm::StringRef RequiredChunk : {"meta", "stri"}) if (!Chunks.count(RequiredChunk)) return makeError("missing required chunk " + RequiredChunk); @@ -439,14 +441,14 @@ void visitStrings(IncludeGraphNode &IGN, const Callback &CB) { CB(Include); } -void writeRIFF(const IndexFileOut &Data, raw_ostream &OS) { +void writeRIFF(const IndexFileOut &Data, llvm::raw_ostream &OS) { assert(Data.Symbols && "An index file without symbols makes no sense!"); riff::File RIFF; RIFF.Type = riff::fourCC("CdIx"); - SmallString<4> Meta; + llvm::SmallString<4> Meta; { - raw_svector_ostream MetaOS(Meta); + llvm::raw_svector_ostream MetaOS(Meta); write32(Version, MetaOS); } RIFF.Chunks.push_back({riff::fourCC("meta"), Meta}); @@ -455,13 +457,15 @@ void writeRIFF(const IndexFileOut &Data, raw_ostream &OS) { std::vector<Symbol> Symbols; for (const auto &Sym : *Data.Symbols) { Symbols.emplace_back(Sym); - visitStrings(Symbols.back(), [&](StringRef &S) { Strings.intern(S); }); + visitStrings(Symbols.back(), + [&](llvm::StringRef &S) { Strings.intern(S); }); } std::vector<IncludeGraphNode> Sources; if (Data.Sources) for (const auto &Source : *Data.Sources) { Sources.push_back(Source.getValue()); - visitStrings(Sources.back(), [&](StringRef &S) { Strings.intern(S); }); + visitStrings(Sources.back(), + [&](llvm::StringRef &S) { Strings.intern(S); }); } std::vector<std::pair<SymbolID, std::vector<Ref>>> Refs; @@ -469,7 +473,7 @@ void writeRIFF(const IndexFileOut &Data, raw_ostream &OS) { for (const auto &Sym : *Data.Refs) { Refs.emplace_back(Sym); for (auto &Ref : Refs.back().second) { - StringRef File = Ref.Location.FileURI; + llvm::StringRef File = Ref.Location.FileURI; Strings.intern(File); Ref.Location.FileURI = File.data(); } @@ -478,14 +482,14 @@ void writeRIFF(const IndexFileOut &Data, raw_ostream &OS) { std::string StringSection; { - raw_string_ostream StringOS(StringSection); + llvm::raw_string_ostream StringOS(StringSection); Strings.finalize(StringOS); } RIFF.Chunks.push_back({riff::fourCC("stri"), StringSection}); std::string SymbolSection; { - raw_string_ostream SymbolOS(SymbolSection); + llvm::raw_string_ostream SymbolOS(SymbolSection); for (const auto &Sym : Symbols) writeSymbol(Sym, Strings, SymbolOS); } @@ -494,7 +498,7 @@ void writeRIFF(const IndexFileOut &Data, raw_ostream &OS) { std::string RefsSection; if (Data.Refs) { { - raw_string_ostream RefsOS(RefsSection); + llvm::raw_string_ostream RefsOS(RefsSection); for (const auto &Sym : Refs) writeRefs(Sym.first, Sym.second, Strings, RefsOS); } @@ -504,7 +508,7 @@ void writeRIFF(const IndexFileOut &Data, raw_ostream &OS) { std::string SrcsSection; { { - raw_string_ostream SrcsOS(SrcsSection); + llvm::raw_string_ostream SrcsOS(SrcsSection); for (const auto &SF : Sources) writeIncludeGraphNode(SF, Strings, SrcsOS); } @@ -517,10 +521,10 @@ void writeRIFF(const IndexFileOut &Data, raw_ostream &OS) { } // namespace // Defined in YAMLSerialization.cpp. -void writeYAML(const IndexFileOut &, raw_ostream &); -Expected<IndexFileIn> readYAML(StringRef); +void writeYAML(const IndexFileOut &, llvm::raw_ostream &); +llvm::Expected<IndexFileIn> readYAML(llvm::StringRef); -raw_ostream &operator<<(raw_ostream &OS, const IndexFileOut &O) { +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) { switch (O.Format) { case IndexFileFormat::RIFF: writeRIFF(O, OS); @@ -532,22 +536,23 @@ raw_ostream &operator<<(raw_ostream &OS, const IndexFileOut &O) { return OS; } -Expected<IndexFileIn> readIndexFile(StringRef Data) { +llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef Data) { if (Data.startswith("RIFF")) { return readRIFF(Data); } else if (auto YAMLContents = readYAML(Data)) { return std::move(*YAMLContents); } else { return makeError("Not a RIFF file and failed to parse as YAML: " + - toString(YAMLContents.takeError())); + llvm::toString(YAMLContents.takeError())); } } -std::unique_ptr<SymbolIndex> loadIndex(StringRef SymbolFilename, bool UseDex) { +std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename, + bool UseDex) { trace::Span OverallTracer("LoadIndex"); - auto Buffer = MemoryBuffer::getFile(SymbolFilename); + auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename); if (!Buffer) { - errs() << "Can't open " << SymbolFilename << "\n"; + llvm::errs() << "Can't open " << SymbolFilename << "\n"; return nullptr; } @@ -561,7 +566,7 @@ std::unique_ptr<SymbolIndex> loadIndex(StringRef SymbolFilename, bool UseDex) { if (I->Refs) Refs = std::move(*I->Refs); } else { - errs() << "Bad Index: " << toString(I.takeError()) << "\n"; + llvm::errs() << "Bad Index: " << llvm::toString(I.takeError()) << "\n"; return nullptr; } } diff --git a/clangd/index/Serialization.h b/clangd/index/Serialization.h index c403b957..d81b8968 100644 --- a/clangd/index/Serialization.h +++ b/clangd/index/Serialization.h @@ -1,9 +1,8 @@ //===--- Serialization.h - Binary serialization of index data ----*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/clangd/index/SymbolCollector.cpp b/clangd/index/SymbolCollector.cpp index b78bc5f5..cbbbd505 100644 --- a/clangd/index/SymbolCollector.cpp +++ b/clangd/index/SymbolCollector.cpp @@ -1,9 +1,8 @@ //===--- SymbolCollector.cpp -------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -29,7 +28,6 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" -using namespace llvm; namespace clang { namespace clangd { namespace { @@ -51,38 +49,18 @@ const NamedDecl &getTemplateOrThis(const NamedDecl &ND) { // // The Path can be a path relative to the build directory, or retrieved from // the SourceManager. -Optional<std::string> toURI(const SourceManager &SM, StringRef Path, - const SymbolCollector::Options &Opts) { - SmallString<128> AbsolutePath(Path); - if (std::error_code EC = - SM.getFileManager().getVirtualFileSystem()->makeAbsolute( - AbsolutePath)) - log("Warning: could not make absolute file: {0}", EC.message()); - if (sys::path::is_absolute(AbsolutePath)) { - // Handle the symbolic link path case where the current working directory - // (getCurrentWorkingDirectory) is a symlink./ We always want to the real - // file path (instead of the symlink path) for the C++ symbols. - // - // Consider the following example: - // - // src dir: /project/src/foo.h - // current working directory (symlink): /tmp/build -> /project/src/ - // - // The file path of Symbol is "/project/src/foo.h" instead of - // "/tmp/build/foo.h" - if (const DirectoryEntry *Dir = SM.getFileManager().getDirectory( - sys::path::parent_path(AbsolutePath.str()))) { - StringRef DirName = SM.getFileManager().getCanonicalName(Dir); - SmallString<128> AbsoluteFilename; - sys::path::append(AbsoluteFilename, DirName, - sys::path::filename(AbsolutePath.str())); - AbsolutePath = AbsoluteFilename; - } - } else if (!Opts.FallbackDir.empty()) { - sys::fs::make_absolute(Opts.FallbackDir, AbsolutePath); +std::string toURI(const SourceManager &SM, llvm::StringRef Path, + const SymbolCollector::Options &Opts) { + llvm::SmallString<128> AbsolutePath(Path); + if (auto CanonPath = + getCanonicalPath(SM.getFileManager().getFile(Path), SM)) { + AbsolutePath = *CanonPath; } - - sys::path::remove_dots(AbsolutePath, /*remove_dot_dot=*/true); + // We don't perform is_absolute check in an else branch because makeAbsolute + // might return a relative path on some InMemoryFileSystems. + if (!llvm::sys::path::is_absolute(AbsolutePath) && !Opts.FallbackDir.empty()) + llvm::sys::fs::make_absolute(Opts.FallbackDir, AbsolutePath); + llvm::sys::path::remove_dots(AbsolutePath, /*remove_dot_dot=*/true); return URI::create(AbsolutePath).toString(); } @@ -122,7 +100,7 @@ bool isPrivateProtoDecl(const NamedDecl &ND) { // will include OUTER_INNER and exclude some_enum_constant. // FIXME: the heuristic relies on naming style (i.e. no underscore in // user-defined names) and can be improved. - return (ND.getKind() != Decl::EnumConstant) || any_of(Name, islower); + return (ND.getKind() != Decl::EnumConstant) || llvm::any_of(Name, islower); } // We only collect #include paths for symbols that are suitable for global code @@ -150,9 +128,9 @@ bool shouldCollectIncludePath(index::SymbolKind Kind) { /// Gets a canonical include (URI of the header or <header> or "header") for /// header of \p Loc. /// Returns None if fails to get include header for \p Loc. -Optional<std::string> getIncludeHeader(StringRef QName, const SourceManager &SM, - SourceLocation Loc, - const SymbolCollector::Options &Opts) { +llvm::Optional<std::string> +getIncludeHeader(llvm::StringRef QName, const SourceManager &SM, + SourceLocation Loc, const SymbolCollector::Options &Opts) { std::vector<std::string> Headers; // Collect the #include stack. while (true) { @@ -168,7 +146,7 @@ Optional<std::string> getIncludeHeader(StringRef QName, const SourceManager &SM, } if (Headers.empty()) return None; - StringRef Header = Headers[0]; + llvm::StringRef Header = Headers[0]; if (Opts.Includes) { Header = Opts.Includes->mapHeader(Headers, QName); if (Header.startswith("<") || Header.startswith("\"")) @@ -206,22 +184,22 @@ bool shouldIndexFile(const SourceManager &SM, FileID FID, } // Return the symbol location of the token at \p TokLoc. -Optional<SymbolLocation> getTokenLocation(SourceLocation TokLoc, - const SourceManager &SM, - const SymbolCollector::Options &Opts, - const clang::LangOptions &LangOpts, - std::string &FileURIStorage) { - auto U = toURI(SM, SM.getFilename(TokLoc), Opts); - if (!U) +llvm::Optional<SymbolLocation> +getTokenLocation(SourceLocation TokLoc, const SourceManager &SM, + const SymbolCollector::Options &Opts, + const clang::LangOptions &LangOpts, + std::string &FileURIStorage) { + auto Path = SM.getFilename(TokLoc); + if (Path.empty()) return None; - FileURIStorage = std::move(*U); + FileURIStorage = toURI(SM, Path, Opts); SymbolLocation Result; Result.FileURI = FileURIStorage.c_str(); auto Range = getTokenRange(TokLoc, SM, LangOpts); Result.Start = Range.first; Result.End = Range.second; - return std::move(Result); + return Result; } // Checks whether \p ND is a definition of a TagDecl (class/struct/enum/union) @@ -261,22 +239,20 @@ void SymbolCollector::initialize(ASTContext &Ctx) { bool SymbolCollector::shouldCollectSymbol(const NamedDecl &ND, const ASTContext &ASTCtx, - const Options &Opts) { + const Options &Opts, + bool IsMainFileOnly) { if (ND.isImplicit()) return false; // Skip anonymous declarations, e.g (anonymous enum/class/struct). if (ND.getDeclName().isEmpty()) return false; - // FIXME: figure out a way to handle internal linkage symbols (e.g. static - // variables, function) defined in the .cc files. Also we skip the symbols - // in anonymous namespace as the qualifier names of these symbols are like - // `foo::<anonymous>::bar`, which need a special handling. - // In real world projects, we have a relatively large set of header files - // that define static variables (like "static const int A = 1;"), we still - // want to collect these symbols, although they cause potential ODR - // violations. - if (ND.isInAnonymousNamespace()) + // Skip main-file symbols if we are not collecting them. + if (IsMainFileOnly && !Opts.CollectMainFileSymbols) + return false; + + // Skip symbols in anonymous namespaces in header files. + if (!IsMainFileOnly && ND.isInAnonymousNamespace()) return false; // We want most things but not "local" symbols such as symbols inside @@ -306,10 +282,6 @@ bool SymbolCollector::shouldCollectSymbol(const NamedDecl &ND, explicitTemplateSpecialization<VarDecl>(ND)) return false; - const auto &SM = ASTCtx.getSourceManager(); - // Skip decls in the main file. - if (SM.isInMainFile(SM.getExpansionLoc(ND.getBeginLoc()))) - return false; // Avoid indexing internal symbols in protobuf generated headers. if (isPrivateProtoDecl(ND)) return false; @@ -319,7 +291,7 @@ bool SymbolCollector::shouldCollectSymbol(const NamedDecl &ND, // Always return true to continue indexing. bool SymbolCollector::handleDeclOccurence( const Decl *D, index::SymbolRoleSet Roles, - ArrayRef<index::SymbolRelation> Relations, SourceLocation Loc, + llvm::ArrayRef<index::SymbolRelation> Relations, SourceLocation Loc, index::IndexDataConsumer::ASTNodeInfo ASTNode) { assert(ASTCtx && PP.get() && "ASTContext and Preprocessor must be set."); assert(CompletionAllocator && CompletionTUInfo); @@ -356,9 +328,15 @@ bool SymbolCollector::handleDeclOccurence( if (IsOnlyRef && !CollectRef) return true; - if (!shouldCollectSymbol(*ND, *ASTCtx, Opts)) + + // ND is the canonical (i.e. first) declaration. If it's in the main file, + // then no public declaration was visible, so assume it's main-file only. + bool IsMainFileOnly = SM.isWrittenInMainFile(SM.getExpansionLoc( + ND->getBeginLoc())); + if (!shouldCollectSymbol(*ND, *ASTCtx, Opts, IsMainFileOnly)) return true; - if (CollectRef && !isa<NamespaceDecl>(ND) && + // Do not store references to main-file symbols. + if (CollectRef && !IsMainFileOnly && !isa<NamespaceDecl>(ND) && (Opts.RefsInHeaders || SM.getFileID(SpellingLoc) == SM.getMainFileID())) DeclRefs[ND].emplace_back(SpellingLoc, Roles); // Don't continue indexing if this is a mere reference. @@ -369,19 +347,25 @@ bool SymbolCollector::handleDeclOccurence( if (!ID) return true; - const NamedDecl &OriginalDecl = *cast<NamedDecl>(ASTNode.OrigD); + // FIXME: ObjCPropertyDecl are not properly indexed here: + // - ObjCPropertyDecl may have an OrigD of ObjCPropertyImplDecl, which is + // not a NamedDecl. + auto *OriginalDecl = dyn_cast<NamedDecl>(ASTNode.OrigD); + if (!OriginalDecl) + return true; + const Symbol *BasicSymbol = Symbols.find(*ID); if (!BasicSymbol) // Regardless of role, ND is the canonical declaration. - BasicSymbol = addDeclaration(*ND, std::move(*ID)); - else if (isPreferredDeclaration(OriginalDecl, Roles)) + BasicSymbol = addDeclaration(*ND, std::move(*ID), IsMainFileOnly); + else if (isPreferredDeclaration(*OriginalDecl, Roles)) // If OriginalDecl is preferred, replace the existing canonical // declaration (e.g. a class forward declaration). There should be at most // one duplicate as we expect to see only one preferred declaration per // TU, because in practice they are definitions. - BasicSymbol = addDeclaration(OriginalDecl, std::move(*ID)); + BasicSymbol = addDeclaration(*OriginalDecl, std::move(*ID), IsMainFileOnly); if (Roles & static_cast<unsigned>(index::SymbolRole::Definition)) - addDefinition(OriginalDecl, *BasicSymbol); + addDefinition(*OriginalDecl, *BasicSymbol); return true; } @@ -395,13 +379,21 @@ bool SymbolCollector::handleMacroOccurence(const IdentifierInfo *Name, const auto &SM = PP->getSourceManager(); auto DefLoc = MI->getDefinitionLoc(); - if (SM.isInMainFile(SM.getExpansionLoc(DefLoc))) - return true; + // Header guards are not interesting in index. Builtin macros don't have // useful locations and are not needed for code completions. if (MI->isUsedForHeaderGuard() || MI->isBuiltinMacro()) return true; + // Skip main-file symbols if we are not collecting them. + bool IsMainFileSymbol = SM.isInMainFile(SM.getExpansionLoc(DefLoc)); + if (IsMainFileSymbol && !Opts.CollectMainFileSymbols) + return false; + + // Also avoid storing predefined macros like __DBL_MIN__. + if (SM.isWrittenInBuiltinFile(DefLoc)) + return true; + // Mark the macro as referenced if this is a reference coming from the main // file. The macro may not be an interesting symbol, but it's cheaper to check // at the end. @@ -426,7 +418,10 @@ bool SymbolCollector::handleMacroOccurence(const IdentifierInfo *Name, Symbol S; S.ID = std::move(*ID); S.Name = Name->getName(); - S.Flags |= Symbol::IndexedForCodeCompletion; + if (!IsMainFileSymbol) { + S.Flags |= Symbol::IndexedForCodeCompletion; + S.Flags |= Symbol::VisibleOutsideFile; + } S.SymInfo = index::getSymbolInfoForMacro(*MI); std::string FileURI; // FIXME: use the result to filter out symbols. @@ -481,17 +476,13 @@ void SymbolCollector::finish() { } const auto &SM = ASTCtx->getSourceManager(); - DenseMap<FileID, std::string> URICache; - auto GetURI = [&](FileID FID) -> Optional<std::string> { + llvm::DenseMap<FileID, std::string> URICache; + auto GetURI = [&](FileID FID) -> llvm::Optional<std::string> { auto Found = URICache.find(FID); if (Found == URICache.end()) { if (auto *FileEntry = SM.getFileEntryForID(FID)) { auto FileURI = toURI(SM, FileEntry->getName(), Opts); - if (!FileURI) { - log("Failed to create URI for file: {0}\n", FileEntry); - FileURI = ""; // reset to empty as we also want to cache this case. - } - Found = URICache.insert({FID, *FileURI}).first; + Found = URICache.insert({FID, FileURI}).first; } else { // Ignore cases where we can not find a corresponding file entry // for the loc, thoses are not interesting, e.g. symbols formed @@ -531,7 +522,8 @@ void SymbolCollector::finish() { } const Symbol *SymbolCollector::addDeclaration(const NamedDecl &ND, - SymbolID ID) { + SymbolID ID, + bool IsMainFileOnly) { auto &Ctx = ND.getASTContext(); auto &SM = Ctx.getSourceManager(); @@ -542,10 +534,13 @@ const Symbol *SymbolCollector::addDeclaration(const NamedDecl &ND, // FIXME: this returns foo:bar: for objective-C methods, we prefer only foo: // for consistency with CodeCompletionString and a clean name/signature split. - if (isIndexedForCodeCompletion(ND, Ctx)) + // We collect main-file symbols, but do not use them for code completion. + if (!IsMainFileOnly && isIndexedForCodeCompletion(ND, Ctx)) S.Flags |= Symbol::IndexedForCodeCompletion; if (isImplementationDetail(&ND)) S.Flags |= Symbol::ImplementationDetail; + if (!IsMainFileOnly) + S.Flags |= Symbol::VisibleOutsideFile; S.SymInfo = index::getSymbolInfo(&ND); std::string FileURI; auto Loc = findNameLoc(&ND); @@ -555,6 +550,10 @@ const Symbol *SymbolCollector::addDeclaration(const NamedDecl &ND, getTokenLocation(Loc, SM, Opts, ASTCtx->getLangOpts(), FileURI)) S.CanonicalDeclaration = *DeclLoc; + S.Origin = Opts.Origin; + if (ND.getAvailability() == AR_Deprecated) + S.Flags |= Symbol::Deprecated; + // Add completion info. // FIXME: we may want to choose a different redecl, or combine from several. assert(ASTCtx && PP.get() && "ASTContext and Preprocessor must be set."); @@ -564,13 +563,28 @@ const Symbol *SymbolCollector::addDeclaration(const NamedDecl &ND, *ASTCtx, *PP, CodeCompletionContext::CCC_Symbol, *CompletionAllocator, *CompletionTUInfo, /*IncludeBriefComments*/ false); - std::string Signature; - std::string SnippetSuffix; - getSignature(*CCS, &Signature, &SnippetSuffix); std::string Documentation = formatDocumentation(*CCS, getDocComment(Ctx, SymbolCompletion, /*CommentsFromHeaders=*/true)); + // For symbols not indexed for completion (class members), we also store their + // docs in the index, because Sema doesn't load the docs from the preamble, we + // rely on the index to get the docs. + // FIXME: this can be optimized by only storing the docs in dynamic index -- + // dynamic index should index these symbols when Sema completes a member + // completion. + S.Documentation = Documentation; + if (!(S.Flags & Symbol::IndexedForCodeCompletion)) { + Symbols.insert(S); + return Symbols.find(S.ID); + } + + std::string Signature; + std::string SnippetSuffix; + getSignature(*CCS, &Signature, &SnippetSuffix); + S.Signature = Signature; + S.CompletionSnippetSuffix = SnippetSuffix; std::string ReturnType = getReturnType(*CCS); + S.ReturnType = ReturnType; std::string Include; if (Opts.CollectIncludePath && shouldCollectIncludePath(S.SymInfo.Kind)) { @@ -580,10 +594,6 @@ const Symbol *SymbolCollector::addDeclaration(const NamedDecl &ND, QName, SM, SM.getExpansionLoc(ND.getLocation()), Opts)) Include = std::move(*Header); } - S.Signature = Signature; - S.CompletionSnippetSuffix = SnippetSuffix; - S.Documentation = Documentation; - S.ReturnType = ReturnType; if (!Include.empty()) S.IncludeHeaders.emplace_back(Include, 1); @@ -594,9 +604,6 @@ const Symbol *SymbolCollector::addDeclaration(const NamedDecl &ND, S.Type = TypeStorage->raw(); } - S.Origin = Opts.Origin; - if (ND.getAvailability() == AR_Deprecated) - S.Flags |= Symbol::Deprecated; Symbols.insert(S); return Symbols.find(S.ID); } diff --git a/clangd/index/SymbolCollector.h b/clangd/index/SymbolCollector.h index 01f2b0ad..1b10df4c 100644 --- a/clangd/index/SymbolCollector.h +++ b/clangd/index/SymbolCollector.h @@ -1,9 +1,8 @@ //===--- SymbolCollector.h ---------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_COLLECTOR_H @@ -28,13 +27,14 @@ namespace clangd { /// It collects most declarations except: /// - Implicit declarations /// - Anonymous declarations (anonymous enum/class/struct, etc) -/// - Declarations in anonymous namespaces +/// - Declarations in anonymous namespaces in headers /// - Local declarations (in function bodies, blocks, etc) -/// - Declarations in main files /// - Template specializations /// - Library-specific private declarations (e.g. private declaration generated /// by protobuf compiler) /// +/// References to main-file symbols are not collected. +/// /// See also shouldCollectSymbol(...). /// /// Clients (e.g. clangd) can use SymbolCollector together with @@ -72,6 +72,9 @@ public: /// collect macros. For example, `indexTopLevelDecls` will not index any /// macro even if this is true. bool CollectMacro = false; + /// Collect symbols local to main-files, such as static functions + /// and symbols inside an anonymous namespace. + bool CollectMainFileSymbols = true; /// If this is set, only collect symbols/references from a file if /// `FileFilter(SM, FID)` is true. If not set, all files are indexed. std::function<bool(const SourceManager &, FileID)> FileFilter = nullptr; @@ -81,7 +84,7 @@ public: /// Returns true is \p ND should be collected. static bool shouldCollectSymbol(const NamedDecl &ND, const ASTContext &ASTCtx, - const Options &Opts); + const Options &Opts, bool IsMainFileSymbol); void initialize(ASTContext &Ctx) override; @@ -105,7 +108,7 @@ public: void finish() override; private: - const Symbol *addDeclaration(const NamedDecl &, SymbolID); + const Symbol *addDeclaration(const NamedDecl &, SymbolID, bool IsMainFileSymbol); void addDefinition(const NamedDecl &, const Symbol &DeclSymbol); // All Symbols collected from the AST. diff --git a/clangd/index/SymbolID.cpp b/clangd/index/SymbolID.cpp index 0ab74810..b97103d3 100644 --- a/clangd/index/SymbolID.cpp +++ b/clangd/index/SymbolID.cpp @@ -1,54 +1,56 @@ //===--- SymbolID.cpp --------------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "SymbolID.h" #include "llvm/Support/SHA1.h" -using namespace llvm; namespace clang { namespace clangd { -SymbolID::SymbolID(StringRef USR) { - auto Hash = llvm::SHA1::hash(arrayRefFromStringRef(USR)); +SymbolID::SymbolID(llvm::StringRef USR) { + auto Hash = llvm::SHA1::hash(llvm::arrayRefFromStringRef(USR)); static_assert(sizeof(Hash) >= RawSize, "RawSize larger than SHA1"); memcpy(HashValue.data(), Hash.data(), RawSize); } llvm::StringRef SymbolID::raw() const { - return StringRef(reinterpret_cast<const char *>(HashValue.data()), RawSize); + return llvm::StringRef(reinterpret_cast<const char *>(HashValue.data()), + RawSize); } -SymbolID SymbolID::fromRaw(StringRef Raw) { +SymbolID SymbolID::fromRaw(llvm::StringRef Raw) { SymbolID ID; assert(Raw.size() == RawSize); memcpy(ID.HashValue.data(), Raw.data(), RawSize); return ID; } -std::string SymbolID::str() const { return toHex(raw()); } +std::string SymbolID::str() const { return llvm::toHex(raw()); } -Expected<SymbolID> SymbolID::fromStr(StringRef Str) { +llvm::Expected<SymbolID> SymbolID::fromStr(llvm::StringRef Str) { if (Str.size() != RawSize * 2) - return createStringError(inconvertibleErrorCode(), "Bad ID length"); + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "Bad ID length"); for (char C : Str) - if (!isHexDigit(C)) - return createStringError(inconvertibleErrorCode(), "Bad hex ID"); - return fromRaw(fromHex(Str)); + if (!llvm::isHexDigit(C)) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "Bad hex ID"); + return fromRaw(llvm::fromHex(Str)); } -raw_ostream &operator<<(raw_ostream &OS, const SymbolID &ID) { - return OS << toHex(ID.raw()); +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const SymbolID &ID) { + return OS << llvm::toHex(ID.raw()); } llvm::hash_code hash_value(const SymbolID &ID) { // We already have a good hash, just return the first bytes. - assert(sizeof(size_t) <= SymbolID::RawSize && "size_t longer than SHA1!"); + static_assert(sizeof(size_t) <= SymbolID::RawSize, + "size_t longer than SHA1!"); size_t Result; memcpy(&Result, ID.raw().data(), sizeof(size_t)); return llvm::hash_code(Result); diff --git a/clangd/index/SymbolID.h b/clangd/index/SymbolID.h index aa8208c1..0e4fc663 100644 --- a/clangd/index/SymbolID.h +++ b/clangd/index/SymbolID.h @@ -1,9 +1,8 @@ //===--- SymbolID.h ----------------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/clangd/index/YAMLSerialization.cpp b/clangd/index/YAMLSerialization.cpp index d3ffdddc..7aab39f4 100644 --- a/clangd/index/YAMLSerialization.cpp +++ b/clangd/index/YAMLSerialization.cpp @@ -1,9 +1,8 @@ //===--- SymbolYAML.cpp ------------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,8 +27,6 @@ #include "llvm/Support/raw_ostream.h" #include <cstdint> -using namespace llvm; - LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences) LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref) @@ -38,8 +35,8 @@ using RefBundle = std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>; // This is a pale imitation of std::variant<Symbol, RefBundle> struct VariantEntry { - Optional<clang::clangd::Symbol> Symbol; - Optional<RefBundle> Refs; + llvm::Optional<clang::clangd::Symbol> Symbol; + llvm::Optional<RefBundle> Refs; }; // A class helps YAML to serialize the 32-bit encoded position (Line&Column), // as YAMLIO can't directly map bitfields. @@ -66,14 +63,14 @@ using clang::index::SymbolLanguage; struct NormalizedSymbolID { NormalizedSymbolID(IO &) {} NormalizedSymbolID(IO &, const SymbolID &ID) { - raw_string_ostream OS(HexString); + llvm::raw_string_ostream OS(HexString); OS << ID; } SymbolID denormalize(IO &I) { auto ID = SymbolID::fromStr(HexString); if (!ID) { - I.setError(toString(ID.takeError())); + I.setError(llvm::toString(ID.takeError())); return SymbolID(); } return *ID; @@ -294,8 +291,8 @@ template <> struct MappingTraits<VariantEntry> { namespace clang { namespace clangd { -void writeYAML(const IndexFileOut &O, raw_ostream &OS) { - yaml::Output Yout(OS); +void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) { + llvm::yaml::Output Yout(OS); for (const auto &Sym : *O.Symbols) { VariantEntry Entry; Entry.Symbol = Sym; @@ -309,23 +306,27 @@ void writeYAML(const IndexFileOut &O, raw_ostream &OS) { } } -Expected<IndexFileIn> readYAML(StringRef Data) { +llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data) { SymbolSlab::Builder Symbols; RefSlab::Builder Refs; - BumpPtrAllocator Arena; // store the underlying data of Position::FileURI. - UniqueStringSaver Strings(Arena); - yaml::Input Yin(Data, &Strings); - do { + llvm::BumpPtrAllocator + Arena; // store the underlying data of Position::FileURI. + llvm::UniqueStringSaver Strings(Arena); + llvm::yaml::Input Yin(Data, &Strings); + while (Yin.setCurrentDocument()) { + llvm::yaml::EmptyContext Ctx; VariantEntry Variant; - Yin >> Variant; + yamlize(Yin, Variant, true, Ctx); if (Yin.error()) - return errorCodeToError(Yin.error()); + return llvm::errorCodeToError(Yin.error()); + if (Variant.Symbol) Symbols.insert(*Variant.Symbol); if (Variant.Refs) for (const auto &Ref : Variant.Refs->second) Refs.insert(Variant.Refs->first, Ref); - } while (Yin.nextDocument()); + Yin.nextDocument(); + } IndexFileIn Result; Result.Symbols.emplace(std::move(Symbols).build()); @@ -336,20 +337,20 @@ Expected<IndexFileIn> readYAML(StringRef Data) { std::string toYAML(const Symbol &S) { std::string Buf; { - raw_string_ostream OS(Buf); - yaml::Output Yout(OS); + llvm::raw_string_ostream OS(Buf); + llvm::yaml::Output Yout(OS); Symbol Sym = S; // copy: Yout<< requires mutability. Yout << Sym; } return Buf; } -std::string toYAML(const std::pair<SymbolID, ArrayRef<Ref>> &Data) { +std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) { RefBundle Refs = {Data.first, Data.second}; std::string Buf; { - raw_string_ostream OS(Buf); - yaml::Output Yout(OS); + llvm::raw_string_ostream OS(Buf); + llvm::yaml::Output Yout(OS); Yout << Refs; } return Buf; diff --git a/clangd/index/dex/Dex.cpp b/clangd/index/dex/Dex.cpp index 8d64e1b4..d767bb51 100644 --- a/clangd/index/dex/Dex.cpp +++ b/clangd/index/dex/Dex.cpp @@ -1,9 +1,8 @@ //===--- Dex.cpp - Dex Symbol Index Implementation --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -20,7 +19,6 @@ #include <algorithm> #include <queue> -using namespace llvm; namespace clang { namespace clangd { namespace dex { @@ -44,61 +42,22 @@ const Token RestrictedForCodeCompletion = // Returns the tokens which are given symbols's characteristics. For example, // trigrams and scopes. // FIXME(kbobyrev): Support more token types: -// * Types // * Namespace proximity std::vector<Token> generateSearchTokens(const Symbol &Sym) { std::vector<Token> Result = generateIdentifierTrigrams(Sym.Name); Result.emplace_back(Token::Kind::Scope, Sym.Scope); // Skip token generation for symbols with unknown declaration location. - if (!StringRef(Sym.CanonicalDeclaration.FileURI).empty()) + if (!llvm::StringRef(Sym.CanonicalDeclaration.FileURI).empty()) for (const auto &ProximityURI : generateProximityURIs(Sym.CanonicalDeclaration.FileURI)) Result.emplace_back(Token::Kind::ProximityURI, ProximityURI); if (Sym.Flags & Symbol::IndexedForCodeCompletion) Result.emplace_back(RestrictedForCodeCompletion); + if (!Sym.Type.empty()) + Result.emplace_back(Token::Kind::Type, Sym.Type); return Result; } -// Constructs BOOST iterators for Path Proximities. -std::unique_ptr<Iterator> -createFileProximityIterator(ArrayRef<std::string> ProximityPaths, - const DenseMap<Token, PostingList> &InvertedIndex, - const Corpus &Corpus) { - std::vector<std::unique_ptr<Iterator>> BoostingIterators; - // Deduplicate parent URIs extracted from the ProximityPaths. - StringSet<> ParentURIs; - StringMap<SourceParams> Sources; - for (const auto &Path : ProximityPaths) { - Sources[Path] = SourceParams(); - auto PathURI = URI::create(Path); - const auto PathProximityURIs = generateProximityURIs(PathURI.toString()); - for (const auto &ProximityURI : PathProximityURIs) - ParentURIs.insert(ProximityURI); - } - // Use SymbolRelevanceSignals for symbol relevance evaluation: use defaults - // for all parameters except for Proximity Path distance signal. - SymbolRelevanceSignals PathProximitySignals; - // DistanceCalculator will find the shortest distance from ProximityPaths to - // any URI extracted from the ProximityPaths. - URIDistance DistanceCalculator(Sources); - PathProximitySignals.FileProximityMatch = &DistanceCalculator; - // Try to build BOOST iterator for each Proximity Path provided by - // ProximityPaths. Boosting factor should depend on the distance to the - // Proximity Path: the closer processed path is, the higher boosting factor. - for (const auto &ParentURI : ParentURIs.keys()) { - Token Tok(Token::Kind::ProximityURI, ParentURI); - const auto It = InvertedIndex.find(Tok); - if (It != InvertedIndex.end()) { - // FIXME(kbobyrev): Append LIMIT on top of every BOOST iterator. - PathProximitySignals.SymbolURI = ParentURI; - BoostingIterators.push_back(Corpus.boost( - It->second.iterator(&It->first), PathProximitySignals.evaluate())); - } - } - BoostingIterators.push_back(Corpus.all()); - return Corpus.unionOf(std::move(BoostingIterators)); -} - } // namespace void Dex::buildIndex() { @@ -124,7 +83,7 @@ void Dex::buildIndex() { } // Populate TempInvertedIndex with lists for index symbols. - DenseMap<Token, std::vector<DocID>> TempInvertedIndex; + llvm::DenseMap<Token, std::vector<DocID>> TempInvertedIndex; for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank) { const auto *Sym = Symbols[SymbolRank]; for (const auto &Token : generateSearchTokens(*Sym)) @@ -143,11 +102,62 @@ std::unique_ptr<Iterator> Dex::iterator(const Token &Tok) const { : It->second.iterator(&It->first); } +// Constructs BOOST iterators for Path Proximities. +std::unique_ptr<Iterator> Dex::createFileProximityIterator( + llvm::ArrayRef<std::string> ProximityPaths) const { + std::vector<std::unique_ptr<Iterator>> BoostingIterators; + // Deduplicate parent URIs extracted from the ProximityPaths. + llvm::StringSet<> ParentURIs; + llvm::StringMap<SourceParams> Sources; + for (const auto &Path : ProximityPaths) { + Sources[Path] = SourceParams(); + auto PathURI = URI::create(Path); + const auto PathProximityURIs = generateProximityURIs(PathURI.toString()); + for (const auto &ProximityURI : PathProximityURIs) + ParentURIs.insert(ProximityURI); + } + // Use SymbolRelevanceSignals for symbol relevance evaluation: use defaults + // for all parameters except for Proximity Path distance signal. + SymbolRelevanceSignals PathProximitySignals; + // DistanceCalculator will find the shortest distance from ProximityPaths to + // any URI extracted from the ProximityPaths. + URIDistance DistanceCalculator(Sources); + PathProximitySignals.FileProximityMatch = &DistanceCalculator; + // Try to build BOOST iterator for each Proximity Path provided by + // ProximityPaths. Boosting factor should depend on the distance to the + // Proximity Path: the closer processed path is, the higher boosting factor. + for (const auto &ParentURI : ParentURIs.keys()) { + // FIXME(kbobyrev): Append LIMIT on top of every BOOST iterator. + auto It = iterator(Token(Token::Kind::ProximityURI, ParentURI)); + if (It->kind() != Iterator::Kind::False) { + PathProximitySignals.SymbolURI = ParentURI; + BoostingIterators.push_back( + Corpus.boost(std::move(It), PathProximitySignals.evaluate())); + } + } + BoostingIterators.push_back(Corpus.all()); + return Corpus.unionOf(std::move(BoostingIterators)); +} + +// Constructs BOOST iterators for preferred types. +std::unique_ptr<Iterator> +Dex::createTypeBoostingIterator(llvm::ArrayRef<std::string> Types) const { + std::vector<std::unique_ptr<Iterator>> BoostingIterators; + SymbolRelevanceSignals PreferredTypeSignals; + PreferredTypeSignals.TypeMatchesPreferred = true; + auto Boost = PreferredTypeSignals.evaluate(); + for (const auto &T : Types) + BoostingIterators.push_back( + Corpus.boost(iterator(Token(Token::Kind::Type, T)), Boost)); + BoostingIterators.push_back(Corpus.all()); + return Corpus.unionOf(std::move(BoostingIterators)); +} + /// Constructs iterators over tokens extracted from the query and exhausts it /// while applying Callback to each symbol in the order of decreasing quality /// of the matched symbols. bool Dex::fuzzyFind(const FuzzyFindRequest &Req, - function_ref<void(const Symbol &)> Callback) const { + llvm::function_ref<void(const Symbol &)> Callback) const { assert(!StringRef(Req.Query).contains("::") && "There must be no :: in query."); trace::Span Tracer("Dex fuzzyFind"); @@ -176,8 +186,9 @@ bool Dex::fuzzyFind(const FuzzyFindRequest &Req, Criteria.push_back(Corpus.unionOf(move(ScopeIterators))); // Add proximity paths boosting (all symbols, some boosted). - Criteria.push_back( - createFileProximityIterator(Req.ProximityPaths, InvertedIndex, Corpus)); + Criteria.push_back(createFileProximityIterator(Req.ProximityPaths)); + // Add boosting for preferred types. + Criteria.push_back(createTypeBoostingIterator(Req.PreferredTypes)); if (Req.RestrictForCodeCompletion) Criteria.push_back(iterator(RestrictedForCodeCompletion)); @@ -190,7 +201,7 @@ bool Dex::fuzzyFind(const FuzzyFindRequest &Req, // FIXME(kbobyrev): Tune this ratio. if (Req.Limit) Root = Corpus.limit(move(Root), *Req.Limit * 100); - SPAN_ATTACH(Tracer, "query", to_string(*Root)); + SPAN_ATTACH(Tracer, "query", llvm::to_string(*Root)); vlog("Dex query tree: {0}", *Root); using IDAndScore = std::pair<DocID, float>; @@ -204,7 +215,7 @@ bool Dex::fuzzyFind(const FuzzyFindRequest &Req, for (const auto &IDAndScore : IDAndScores) { const DocID SymbolDocID = IDAndScore.first; const auto *Sym = Symbols[SymbolDocID]; - const Optional<float> Score = Filter.match(Sym->Name); + const llvm::Optional<float> Score = Filter.match(Sym->Name); if (!Score) continue; // Combine Fuzzy Matching score, precomputed symbol quality and boosting @@ -225,7 +236,7 @@ bool Dex::fuzzyFind(const FuzzyFindRequest &Req, } void Dex::lookup(const LookupRequest &Req, - function_ref<void(const Symbol &)> Callback) const { + llvm::function_ref<void(const Symbol &)> Callback) const { trace::Span Tracer("Dex lookup"); for (const auto &ID : Req.IDs) { auto I = LookupTable.find(ID); @@ -235,12 +246,17 @@ void Dex::lookup(const LookupRequest &Req, } void Dex::refs(const RefsRequest &Req, - function_ref<void(const Ref &)> Callback) const { + llvm::function_ref<void(const Ref &)> Callback) const { trace::Span Tracer("Dex refs"); + uint32_t Remaining = + Req.Limit.getValueOr(std::numeric_limits<uint32_t>::max()); for (const auto &ID : Req.IDs) - for (const auto &Ref : Refs.lookup(ID)) - if (static_cast<int>(Req.Filter & Ref.Kind)) + for (const auto &Ref : Refs.lookup(ID)) { + if (Remaining > 0 && static_cast<int>(Req.Filter & Ref.Kind)) { + --Remaining; Callback(Ref); + } + } } size_t Dex::estimateMemoryUsage() const { @@ -254,13 +270,13 @@ size_t Dex::estimateMemoryUsage() const { return Bytes + BackingDataSize; } -std::vector<std::string> generateProximityURIs(StringRef URIPath) { +std::vector<std::string> generateProximityURIs(llvm::StringRef URIPath) { std::vector<std::string> Result; auto ParsedURI = URI::parse(URIPath); assert(ParsedURI && "Non-empty argument of generateProximityURIs() should be a valid " "URI."); - StringRef Body = ParsedURI->body(); + llvm::StringRef Body = ParsedURI->body(); // FIXME(kbobyrev): Currently, this is a heuristic which defines the maximum // size of resulting vector. Some projects might want to have higher limit if // the file hierarchy is deeper. For the generic case, it would be useful to @@ -273,7 +289,7 @@ std::vector<std::string> generateProximityURIs(StringRef URIPath) { while (!Body.empty() && --Limit > 0) { // FIXME(kbobyrev): Parsing and encoding path to URIs is not necessary and // could be optimized. - Body = sys::path::parent_path(Body, sys::path::Style::posix); + Body = llvm::sys::path::parent_path(Body, llvm::sys::path::Style::posix); URI TokenURI(ParsedURI->scheme(), ParsedURI->authority(), Body); if (!Body.empty()) Result.emplace_back(TokenURI.toString()); diff --git a/clangd/index/dex/Dex.h b/clangd/index/dex/Dex.h index c790e417..fb80ca03 100644 --- a/clangd/index/dex/Dex.h +++ b/clangd/index/dex/Dex.h @@ -1,9 +1,8 @@ //===--- Dex.h - Dex Symbol Index Implementation ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -78,6 +77,10 @@ public: private: void buildIndex(); std::unique_ptr<Iterator> iterator(const Token &Tok) const; + std::unique_ptr<Iterator> + createFileProximityIterator(llvm::ArrayRef<std::string> ProximityPaths) const; + std::unique_ptr<Iterator> + createTypeBoostingIterator(llvm::ArrayRef<std::string> Types) const; /// Stores symbols sorted in the descending order of symbol quality.. std::vector<const Symbol *> Symbols; diff --git a/clangd/index/dex/Iterator.cpp b/clangd/index/dex/Iterator.cpp index 9e62c155..cb294a33 100644 --- a/clangd/index/dex/Iterator.cpp +++ b/clangd/index/dex/Iterator.cpp @@ -1,9 +1,8 @@ //===--- Iterator.cpp - Query Symbol Retrieval ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,6 @@ #include <cassert> #include <numeric> -using namespace llvm; namespace clang { namespace clangd { namespace dex { @@ -77,7 +75,7 @@ public: } private: - raw_ostream &dump(raw_ostream &OS) const override { + llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override { OS << "(& "; auto Separator = ""; for (const auto &Child : Children) { @@ -198,7 +196,7 @@ public: } private: - raw_ostream &dump(raw_ostream &OS) const override { + llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override { OS << "(| "; auto Separator = ""; for (const auto &Child : Children) { @@ -246,7 +244,9 @@ public: size_t estimateSize() const override { return Size; } private: - raw_ostream &dump(raw_ostream &OS) const override { return OS << "true"; } + llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override { + return OS << "true"; + } DocID Index = 0; /// Size of the underlying virtual PostingList. @@ -271,7 +271,9 @@ public: size_t estimateSize() const override { return 0; } private: - raw_ostream &dump(raw_ostream &OS) const override { return OS << "false"; } + llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override { + return OS << "false"; + } }; /// Boost iterator is a wrapper around its child which multiplies scores of @@ -294,7 +296,7 @@ public: size_t estimateSize() const override { return Child->estimateSize(); } private: - raw_ostream &dump(raw_ostream &OS) const override { + llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override { return OS << "(* " << Factor << ' ' << *Child << ')'; } @@ -334,7 +336,7 @@ public: } private: - raw_ostream &dump(raw_ostream &OS) const override { + llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override { return OS << "(LIMIT " << Limit << " " << *Child << ')'; } diff --git a/clangd/index/dex/Iterator.h b/clangd/index/dex/Iterator.h index 149fd43a..34b42c32 100644 --- a/clangd/index/dex/Iterator.h +++ b/clangd/index/dex/Iterator.h @@ -1,9 +1,8 @@ //===--- Iterator.h - Query Symbol Retrieval --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/clangd/index/dex/PostingList.cpp b/clangd/index/dex/PostingList.cpp index a0c8afdf..44a668bb 100644 --- a/clangd/index/dex/PostingList.cpp +++ b/clangd/index/dex/PostingList.cpp @@ -1,9 +1,8 @@ //===--- PostingList.cpp - Symbol identifiers storage interface -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,7 +12,6 @@ #include "llvm/Support/Error.h" #include "llvm/Support/MathExtras.h" -using namespace llvm; namespace clang { namespace clangd { namespace dex { @@ -24,7 +22,7 @@ namespace { /// them on-the-fly when the contents of chunk are to be seen. class ChunkIterator : public Iterator { public: - explicit ChunkIterator(const Token *Tok, ArrayRef<Chunk> Chunks) + explicit ChunkIterator(const Token *Tok, llvm::ArrayRef<Chunk> Chunks) : Tok(Tok), Chunks(Chunks), CurrentChunk(Chunks.begin()) { if (!Chunks.empty()) { DecompressedChunk = CurrentChunk->decompress(); @@ -71,7 +69,7 @@ public: } private: - raw_ostream &dump(raw_ostream &OS) const override { + llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override { if (Tok != nullptr) return OS << *Tok; OS << '['; @@ -113,13 +111,13 @@ private: } const Token *Tok; - ArrayRef<Chunk> Chunks; + llvm::ArrayRef<Chunk> Chunks; /// Iterator over chunks. /// If CurrentChunk is valid, then DecompressedChunk is /// CurrentChunk->decompress() and CurrentID is a valid (non-end) iterator /// into it. decltype(Chunks)::const_iterator CurrentChunk; - SmallVector<DocID, Chunk::PayloadSize + 1> DecompressedChunk; + llvm::SmallVector<DocID, Chunk::PayloadSize + 1> DecompressedChunk; /// Iterator over DecompressedChunk. decltype(DecompressedChunk)::iterator CurrentID; @@ -130,11 +128,11 @@ static constexpr size_t BitsPerEncodingByte = 7; /// Writes a variable length DocID into the buffer and updates the buffer size. /// If it doesn't fit, returns false and doesn't write to the buffer. -bool encodeVByte(DocID Delta, MutableArrayRef<uint8_t> &Payload) { +bool encodeVByte(DocID Delta, llvm::MutableArrayRef<uint8_t> &Payload) { assert(Delta != 0 && "0 is not a valid PostingList delta."); // Calculate number of bytes Delta encoding would take by examining the // meaningful bits. - unsigned Width = 1 + findLastSet(Delta) / BitsPerEncodingByte; + unsigned Width = 1 + llvm::findLastSet(Delta) / BitsPerEncodingByte; if (Width > Payload.size()) return false; @@ -166,12 +164,12 @@ bool encodeVByte(DocID Delta, MutableArrayRef<uint8_t> &Payload) { /// DocIDs 42 47 7000 /// gaps 5 6958 /// Encoding (raw number) 00000101 10110110 00101110 -std::vector<Chunk> encodeStream(ArrayRef<DocID> Documents) { +std::vector<Chunk> encodeStream(llvm::ArrayRef<DocID> Documents) { assert(!Documents.empty() && "Can't encode empty sequence."); std::vector<Chunk> Result; Result.emplace_back(); DocID Last = Result.back().Head = Documents.front(); - MutableArrayRef<uint8_t> RemainingPayload = Result.back().Payload; + llvm::MutableArrayRef<uint8_t> RemainingPayload = Result.back().Payload; for (DocID Doc : Documents.drop_front()) { if (!encodeVByte(Doc - Last, RemainingPayload)) { // didn't fit, flush chunk Result.emplace_back(); @@ -185,7 +183,7 @@ std::vector<Chunk> encodeStream(ArrayRef<DocID> Documents) { /// Reads variable length DocID from the buffer and updates the buffer size. If /// the stream is terminated, return None. -Optional<DocID> readVByte(ArrayRef<uint8_t> &Bytes) { +llvm::Optional<DocID> readVByte(llvm::ArrayRef<uint8_t> &Bytes) { if (Bytes.front() == 0 || Bytes.empty()) return None; DocID Result = 0; @@ -203,9 +201,9 @@ Optional<DocID> readVByte(ArrayRef<uint8_t> &Bytes) { } // namespace -SmallVector<DocID, Chunk::PayloadSize + 1> Chunk::decompress() const { - SmallVector<DocID, Chunk::PayloadSize + 1> Result{Head}; - ArrayRef<uint8_t> Bytes(Payload); +llvm::SmallVector<DocID, Chunk::PayloadSize + 1> Chunk::decompress() const { + llvm::SmallVector<DocID, Chunk::PayloadSize + 1> Result{Head}; + llvm::ArrayRef<uint8_t> Bytes(Payload); DocID Delta; for (DocID Current = Head; !Bytes.empty(); Current += Delta) { auto MaybeDelta = readVByte(Bytes); @@ -214,10 +212,10 @@ SmallVector<DocID, Chunk::PayloadSize + 1> Chunk::decompress() const { Delta = *MaybeDelta; Result.push_back(Current + Delta); } - return SmallVector<DocID, Chunk::PayloadSize + 1>{Result}; + return llvm::SmallVector<DocID, Chunk::PayloadSize + 1>{Result}; } -PostingList::PostingList(ArrayRef<DocID> Documents) +PostingList::PostingList(llvm::ArrayRef<DocID> Documents) : Chunks(encodeStream(Documents)) {} std::unique_ptr<Iterator> PostingList::iterator(const Token *Tok) const { diff --git a/clangd/index/dex/PostingList.h b/clangd/index/dex/PostingList.h index 81ba64c1..418e4c72 100644 --- a/clangd/index/dex/PostingList.h +++ b/clangd/index/dex/PostingList.h @@ -1,9 +1,8 @@ //===--- PostingList.h - Symbol identifiers storage interface --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/clangd/index/dex/Token.h b/clangd/index/dex/Token.h index f80d9254..37859bcf 100644 --- a/clangd/index/dex/Token.h +++ b/clangd/index/dex/Token.h @@ -1,9 +1,8 @@ //===--- Token.h - Symbol Search primitive ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -63,11 +62,11 @@ struct Token { /// Example: "file:///path/to/clang-tools-extra/clangd/index/SymbolIndex.h" /// and some amount of its parents. ProximityURI, + /// Type of symbol (see `Symbol::Type`). + Type, /// Internal Token type for invalid/special tokens, e.g. empty tokens for /// llvm::DenseMap. Sentinel, - /// FIXME(kbobyrev): Add other Token Kinds - /// * Type with qualified type name or its USR }; Token(Kind TokenKind, llvm::StringRef Data) @@ -92,6 +91,9 @@ struct Token { case Kind::ProximityURI: OS << "U="; break; + case Kind::Type: + OS << "Ty="; + break; case Kind::Sentinel: OS << "?="; break; diff --git a/clangd/index/dex/Trigram.cpp b/clangd/index/dex/Trigram.cpp index 7779e8b8..24ae72bf 100644 --- a/clangd/index/dex/Trigram.cpp +++ b/clangd/index/dex/Trigram.cpp @@ -1,9 +1,8 @@ //===--- Trigram.cpp - Trigram generation for Fuzzy Matching ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -17,16 +16,15 @@ #include <queue> #include <string> -using namespace llvm; namespace clang { namespace clangd { namespace dex { -std::vector<Token> generateIdentifierTrigrams(StringRef Identifier) { +std::vector<Token> generateIdentifierTrigrams(llvm::StringRef Identifier) { // Apply fuzzy matching text segmentation. std::vector<CharRole> Roles(Identifier.size()); calculateRoles(Identifier, - makeMutableArrayRef(Roles.data(), Identifier.size())); + llvm::makeMutableArrayRef(Roles.data(), Identifier.size())); std::string LowercaseIdentifier = Identifier.lower(); @@ -48,7 +46,7 @@ std::vector<Token> generateIdentifierTrigrams(StringRef Identifier) { } } - DenseSet<Token> UniqueTrigrams; + llvm::DenseSet<Token> UniqueTrigrams; auto Add = [&](std::string Chars) { UniqueTrigrams.insert(Token(Token::Kind::Trigram, Chars)); @@ -85,7 +83,7 @@ std::vector<Token> generateIdentifierTrigrams(StringRef Identifier) { return {UniqueTrigrams.begin(), UniqueTrigrams.end()}; } -std::vector<Token> generateQueryTrigrams(StringRef Query) { +std::vector<Token> generateQueryTrigrams(llvm::StringRef Query) { if (Query.empty()) return {}; std::string LowercaseQuery = Query.lower(); @@ -94,9 +92,9 @@ std::vector<Token> generateQueryTrigrams(StringRef Query) { // Apply fuzzy matching text segmentation. std::vector<CharRole> Roles(Query.size()); - calculateRoles(Query, makeMutableArrayRef(Roles.data(), Query.size())); + calculateRoles(Query, llvm::makeMutableArrayRef(Roles.data(), Query.size())); - DenseSet<Token> UniqueTrigrams; + llvm::DenseSet<Token> UniqueTrigrams; std::string Chars; for (unsigned I = 0; I < Query.size(); ++I) { if (Roles[I] != Head && Roles[I] != Tail) diff --git a/clangd/index/dex/Trigram.h b/clangd/index/dex/Trigram.h index adce9f42..bf1e5e80 100644 --- a/clangd/index/dex/Trigram.h +++ b/clangd/index/dex/Trigram.h @@ -1,9 +1,8 @@ //===--- Trigram.h - Trigram generation for Fuzzy Matching ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/clangd/index/dex/dexp/Dexp.cpp b/clangd/index/dex/dexp/Dexp.cpp index 795173c7..820dc66b 100644 --- a/clangd/index/dex/dexp/Dexp.cpp +++ b/clangd/index/dex/dexp/Dexp.cpp @@ -1,9 +1,8 @@ //===--- Dexp.cpp - Dex EXPloration tool ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,14 +21,13 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Signals.h" -using namespace llvm; -using namespace clang; -using namespace clangd; - +namespace clang { +namespace clangd { namespace { -cl::opt<std::string> IndexPath("index-path", cl::desc("Path to the index"), - cl::Positional, cl::Required); +llvm::cl::opt<std::string> IndexPath("index-path", + llvm::cl::desc("Path to the index"), + llvm::cl::Positional, llvm::cl::Required); static const std::string Overview = R"( This is an **experimental** interactive tool to process user-provided search @@ -40,16 +38,16 @@ and manually construct non-trivial test cases. Type use "help" request to get information about the details. )"; -void reportTime(StringRef Name, function_ref<void()> F) { +void reportTime(llvm::StringRef Name, llvm::function_ref<void()> F) { const auto TimerStart = std::chrono::high_resolution_clock::now(); F(); const auto TimerStop = std::chrono::high_resolution_clock::now(); const auto Duration = std::chrono::duration_cast<std::chrono::milliseconds>( TimerStop - TimerStart); - outs() << formatv("{0} took {1:ms+n}.\n", Name, Duration); + llvm::outs() << llvm::formatv("{0} took {1:ms+n}.\n", Name, Duration); } -std::vector<SymbolID> getSymbolIDsFromIndex(StringRef QualifiedName, +std::vector<SymbolID> getSymbolIDsFromIndex(llvm::StringRef QualifiedName, const SymbolIndex *Index) { FuzzyFindRequest Request; // Remove leading "::" qualifier as FuzzyFind doesn't need leading "::" @@ -77,9 +75,9 @@ std::vector<SymbolID> getSymbolIDsFromIndex(StringRef QualifiedName, // Creating a Command populates parser options, parseAndRun() resets them. class Command { // By resetting the parser options, we lost the standard -help flag. - cl::opt<bool, false, cl::parser<bool>> Help{ - "help", cl::desc("Display available options"), cl::ValueDisallowed, - cl::cat(cl::GeneralCategory)}; + llvm::cl::opt<bool, false, llvm::cl::parser<bool>> Help{ + "help", llvm::cl::desc("Display available options"), + llvm::cl::ValueDisallowed, llvm::cl::cat(llvm::cl::GeneralCategory)}; virtual void run() = 0; protected: @@ -87,24 +85,25 @@ protected: public: virtual ~Command() = default; - virtual void parseAndRun(ArrayRef<const char *> Argv, const char *Overview, - const SymbolIndex &Index) { + virtual void parseAndRun(llvm::ArrayRef<const char *> Argv, + const char *Overview, const SymbolIndex &Index) { std::string ParseErrs; - raw_string_ostream OS(ParseErrs); - bool Ok = - cl::ParseCommandLineOptions(Argv.size(), Argv.data(), Overview, &OS); + llvm::raw_string_ostream OS(ParseErrs); + bool Ok = llvm::cl::ParseCommandLineOptions(Argv.size(), Argv.data(), + Overview, &OS); if (Help.getNumOccurrences() > 0) { // Avoid printing parse errors in this case. // (Well, in theory. A bunch get printed to llvm::errs() regardless!) - cl::PrintHelpMessage(); + llvm::cl::PrintHelpMessage(); } else { - outs() << OS.str(); + llvm::outs() << OS.str(); if (Ok) { this->Index = &Index; reportTime(Argv[0], [&] { run(); }); } } - cl::ResetCommandLineParser(); // must do this before opts are destroyed. + llvm::cl::ResetCommandLineParser(); // must do this before opts are + // destroyed. } }; @@ -118,20 +117,20 @@ public: // * print out tokens with least dense posting lists class FuzzyFind : public Command { - cl::opt<std::string> Query{ + llvm::cl::opt<std::string> Query{ "query", - cl::Positional, - cl::Required, - cl::desc("Query string to be fuzzy-matched"), + llvm::cl::Positional, + llvm::cl::Required, + llvm::cl::desc("Query string to be fuzzy-matched"), }; - cl::opt<std::string> Scopes{ + llvm::cl::opt<std::string> Scopes{ "scopes", - cl::desc("Allowed symbol scopes (comma-separated list)"), + llvm::cl::desc("Allowed symbol scopes (comma-separated list)"), }; - cl::opt<unsigned> Limit{ + llvm::cl::opt<unsigned> Limit{ "limit", - cl::init(10), - cl::desc("Max results to display"), + llvm::cl::init(10), + llvm::cl::desc("Max results to display"), }; void run() override { @@ -139,42 +138,45 @@ class FuzzyFind : public Command { Request.Limit = Limit; Request.Query = Query; if (Scopes.getNumOccurrences() > 0) { - SmallVector<StringRef, 8> Scopes; - StringRef(this->Scopes).split(Scopes, ','); + llvm::SmallVector<llvm::StringRef, 8> Scopes; + llvm::StringRef(this->Scopes).split(Scopes, ','); Request.Scopes = {Scopes.begin(), Scopes.end()}; } Request.AnyScope = Request.Scopes.empty(); // FIXME(kbobyrev): Print symbol final scores to see the distribution. static const auto OutputFormat = "{0,-4} | {1,-40} | {2,-25}\n"; - outs() << formatv(OutputFormat, "Rank", "Symbol ID", "Symbol Name"); + llvm::outs() << llvm::formatv(OutputFormat, "Rank", "Symbol ID", + "Symbol Name"); size_t Rank = 0; Index->fuzzyFind(Request, [&](const Symbol &Sym) { - outs() << formatv(OutputFormat, Rank++, Sym.ID.str(), - Sym.Scope + Sym.Name); + llvm::outs() << llvm::formatv(OutputFormat, Rank++, Sym.ID.str(), + Sym.Scope + Sym.Name); }); } }; class Lookup : public Command { - cl::opt<std::string> ID{ + llvm::cl::opt<std::string> ID{ "id", - cl::Positional, - cl::desc("Symbol ID to look up (hex)"), + llvm::cl::Positional, + llvm::cl::desc("Symbol ID to look up (hex)"), }; - cl::opt<std::string> Name{ - "name", cl::desc("Qualified name to look up."), + llvm::cl::opt<std::string> Name{ + "name", + llvm::cl::desc("Qualified name to look up."), }; void run() override { if (ID.getNumOccurrences() == 0 && Name.getNumOccurrences() == 0) { - outs() << "Missing required argument: please provide id or -name.\n"; + llvm::outs() + << "Missing required argument: please provide id or -name.\n"; return; } std::vector<SymbolID> IDs; if (ID.getNumOccurrences()) { auto SID = SymbolID::fromStr(ID); if (!SID) { - outs() << toString(SID.takeError()) << "\n"; + llvm::outs() << llvm::toString(SID.takeError()) << "\n"; return; } IDs.push_back(*SID); @@ -187,60 +189,65 @@ class Lookup : public Command { bool FoundSymbol = false; Index->lookup(Request, [&](const Symbol &Sym) { FoundSymbol = true; - outs() << toYAML(Sym); + llvm::outs() << toYAML(Sym); }); if (!FoundSymbol) - outs() << "not found\n"; + llvm::outs() << "not found\n"; } }; class Refs : public Command { - cl::opt<std::string> ID{ - "id", cl::Positional, - cl::desc("Symbol ID of the symbol being queried (hex)."), + llvm::cl::opt<std::string> ID{ + "id", + llvm::cl::Positional, + llvm::cl::desc("Symbol ID of the symbol being queried (hex)."), }; - cl::opt<std::string> Name{ - "name", cl::desc("Qualified name of the symbol being queried."), + llvm::cl::opt<std::string> Name{ + "name", + llvm::cl::desc("Qualified name of the symbol being queried."), }; - cl::opt<std::string> Filter{ - "filter", cl::init(".*"), - cl::desc( + llvm::cl::opt<std::string> Filter{ + "filter", + llvm::cl::init(".*"), + llvm::cl::desc( "Print all results from files matching this regular expression."), }; void run() override { if (ID.getNumOccurrences() == 0 && Name.getNumOccurrences() == 0) { - outs() << "Missing required argument: please provide id or -name.\n"; + llvm::outs() + << "Missing required argument: please provide id or -name.\n"; return; } std::vector<SymbolID> IDs; if (ID.getNumOccurrences()) { auto SID = SymbolID::fromStr(ID); if (!SID) { - outs() << toString(SID.takeError()) << "\n"; + llvm::outs() << llvm::toString(SID.takeError()) << "\n"; return; } IDs.push_back(*SID); } else { IDs = getSymbolIDsFromIndex(Name, Index); if (IDs.size() > 1) { - outs() << formatv("The name {0} is ambiguous, found {1} different " - "symbols. Please use id flag to disambiguate.\n", - Name, IDs.size()); + llvm::outs() << llvm::formatv( + "The name {0} is ambiguous, found {1} different " + "symbols. Please use id flag to disambiguate.\n", + Name, IDs.size()); return; } } RefsRequest RefRequest; RefRequest.IDs.insert(IDs.begin(), IDs.end()); - Regex RegexFilter(Filter); + llvm::Regex RegexFilter(Filter); Index->refs(RefRequest, [&RegexFilter](const Ref &R) { auto U = URI::parse(R.Location.FileURI); if (!U) { - outs() << U.takeError(); + llvm::outs() << U.takeError(); return; } if (RegexFilter.match(U->body())) - outs() << R << "\n"; + llvm::outs() << R << "\n"; }); } }; @@ -257,16 +264,20 @@ struct { llvm::make_unique<Refs>}, }; -std::unique_ptr<SymbolIndex> openIndex(StringRef Index) { +std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) { return loadIndex(Index, /*UseDex=*/true); } } // namespace +} // namespace clangd +} // namespace clang int main(int argc, const char *argv[]) { - cl::ParseCommandLineOptions(argc, argv, Overview); - cl::ResetCommandLineParser(); // We reuse it for REPL commands. - sys::PrintStackTraceOnErrorSignal(argv[0]); + using namespace clang::clangd; + + llvm::cl::ParseCommandLineOptions(argc, argv, Overview); + llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands. + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); std::unique_ptr<SymbolIndex> Index; reportTime("Dex build", [&]() { @@ -274,28 +285,29 @@ int main(int argc, const char *argv[]) { }); if (!Index) { - outs() << "Failed to open the index.\n"; + llvm::outs() << "Failed to open the index.\n"; return -1; } - LineEditor LE("dexp"); + llvm::LineEditor LE("dexp"); - while (Optional<std::string> Request = LE.readLine()) { + while (llvm::Optional<std::string> Request = LE.readLine()) { // Split on spaces and add required null-termination. std::replace(Request->begin(), Request->end(), ' ', '\0'); - SmallVector<StringRef, 8> Args; - StringRef(*Request).split(Args, '\0', /*MaxSplit=*/-1, /*KeepEmpty=*/false); + llvm::SmallVector<llvm::StringRef, 8> Args; + llvm::StringRef(*Request).split(Args, '\0', /*MaxSplit=*/-1, + /*KeepEmpty=*/false); if (Args.empty()) continue; if (Args.front() == "help") { - outs() << "dexp - Index explorer\nCommands:\n"; + llvm::outs() << "dexp - Index explorer\nCommands:\n"; for (const auto &C : CommandInfo) - outs() << formatv("{0,16} - {1}\n", C.Name, C.Description); - outs() << "Get detailed command help with e.g. `find -help`.\n"; + llvm::outs() << llvm::formatv("{0,16} - {1}\n", C.Name, C.Description); + llvm::outs() << "Get detailed command help with e.g. `find -help`.\n"; continue; } - SmallVector<const char *, 8> FakeArgv; - for (StringRef S : Args) + llvm::SmallVector<const char *, 8> FakeArgv; + for (llvm::StringRef S : Args) FakeArgv.push_back(S.data()); // Terminated by separator or end of string. bool Recognized = false; @@ -307,7 +319,7 @@ int main(int argc, const char *argv[]) { } } if (!Recognized) - outs() << "Unknown command. Try 'help'.\n"; + llvm::outs() << "Unknown command. Try 'help'.\n"; } return 0; |