From 4b4f8a7cc9c4cc198f9a94750e1d5b979f767c8a Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Tue, 18 Jul 2023 12:57:18 -0500
Subject: [PATCH 01/50] Add gotoDefinition other file tests

---
 test/functional/Definition.hs     | 33 +++++++++++++++++++++++++++----
 test/testdata/definition/Bar.hs   |  6 ++++++
 test/testdata/definition/Foo.hs   |  3 +++
 test/testdata/definition/hie.yaml |  5 +++++
 4 files changed, 43 insertions(+), 4 deletions(-)
 create mode 100644 test/testdata/definition/hie.yaml

diff --git a/test/functional/Definition.hs b/test/functional/Definition.hs
index 24ce49297d..3c32f2cf72 100644
--- a/test/functional/Definition.hs
+++ b/test/functional/Definition.hs
@@ -7,18 +7,43 @@ import           Test.Hls
 import           Test.Hls.Command
 
 tests :: TestTree
-tests = testGroup "definitions" [
+tests = testGroup "definitions" [symbolTests, moduleTests]
 
-    ignoreTestBecause "Broken: file:///Users/jwindsor/src/haskell-language-server/test/testdata/References.hs" $
-       testCase "goto's symbols" $ runSession hlsCommand fullCaps "test/testdata" $ do
+symbolTests :: TestTree
+symbolTests = testGroup "gotoDefinition on symbols"
+       -- gotoDefinition where the definition is in the same file
+  [    testCase "gotoDefinition in this file" $ runSession hlsCommand fullCaps "test/testdata" $ do
         doc <- openDoc "References.hs" "haskell"
         defs <- getDefinitions doc (Position 7 8)
         let expRange = Range (Position 4 0) (Position 4 3)
         liftIO $ defs @?= InL (Definition (InR [Location (doc ^. uri) expRange]))
 
+        -- gotoDefinition where the definition is in a different file
+  ,    testCase "gotoDefinition in other file" $ runSession hlsCommand fullCaps "test/testdata/definition" $ do
+        doc <- openDoc "Foo.hs" "haskell"
+        defs <- getDefinitions doc (Position 4 11)
+        let expRange = Range (Position 2 0) (Position 2 1)
+        liftIO $ do
+            fp <- canonicalizePath "test/testdata/definition/Bar.hs"
+            defs @?= InL (Definition (InR [Location (filePathToUri fp) expRange]))
+
+        -- gotoDefinition where the definition is in a different file and the
+        -- definition in the other file is on a line number that is greater
+        -- than the number of lines in the file we are requesting from
+  ,    testCase "gotoDefinition in other file past lines in this file" $ runSession hlsCommand fullCaps "test/testdata/definition" $ do
+        doc <- openDoc "Foo.hs" "haskell"
+        defs <- getDefinitions doc (Position 5 13)
+        let expRange = Range (Position 8 0) (Position 8 1)
+        liftIO $ do
+            fp <- canonicalizePath "test/testdata/definition/Bar.hs"
+            defs @?= InL (Definition (InR [Location (filePathToUri fp) expRange]))
+  ]
+
   -- -----------------------------------
 
-  , ignoreTestBecause "Broken: file:///Users/jwindsor/src/haskell-language-server/test/testdata/Bar.hs" $
+moduleTests :: TestTree
+moduleTests = testGroup "gotoDefinition on modules"
+  [ ignoreTestBecause "Broken: file:///Users/jwindsor/src/haskell-language-server/test/testdata/Bar.hs" $
     testCase "goto's imported modules" $ runSession hlsCommand fullCaps "test/testdata/definition" $ do
         doc <- openDoc "Foo.hs" "haskell"
         defs <- getDefinitions doc (Position 2 8)
diff --git a/test/testdata/definition/Bar.hs b/test/testdata/definition/Bar.hs
index 02a244cd4d..9ae116114e 100644
--- a/test/testdata/definition/Bar.hs
+++ b/test/testdata/definition/Bar.hs
@@ -1,3 +1,9 @@
 module Bar where
 
 a = 42
+
+-- These blank lines are here
+-- to ensure that b is defined
+-- on a line number larger than
+-- the number of lines in Foo.hs.
+b = 43
diff --git a/test/testdata/definition/Foo.hs b/test/testdata/definition/Foo.hs
index 6dfb3ba2e6..ca73e2d375 100644
--- a/test/testdata/definition/Foo.hs
+++ b/test/testdata/definition/Foo.hs
@@ -1,3 +1,6 @@
 module Foo (module Bar) where
 
 import Bar
+
+fortyTwo = a
+fortyThree = b
diff --git a/test/testdata/definition/hie.yaml b/test/testdata/definition/hie.yaml
new file mode 100644
index 0000000000..9adb47d0f3
--- /dev/null
+++ b/test/testdata/definition/hie.yaml
@@ -0,0 +1,5 @@
+cradle:
+  direct:
+    arguments:
+      - "Foo"
+      - "Bar"

From 3b64fbb234dd56fbb7a4c9c115b810b0321b663b Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 20 Jul 2023 11:25:09 -0500
Subject: [PATCH 02/50] Use correct position mapping in getDefinition

---
 ghcide/src/Development/IDE/Core/Actions.hs | 43 +++++++++++++++++++---
 1 file changed, 37 insertions(+), 6 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Actions.hs b/ghcide/src/Development/IDE/Core/Actions.hs
index c8e384c1b5..7c0ea1a07e 100644
--- a/ghcide/src/Development/IDE/Core/Actions.hs
+++ b/ghcide/src/Development/IDE/Core/Actions.hs
@@ -13,6 +13,7 @@ module Development.IDE.Core.Actions
 , lookupMod
 ) where
 
+import           Control.Monad.Extra                  (mapMaybeM)
 import           Control.Monad.Reader
 import           Control.Monad.Trans.Maybe
 import qualified Data.HashMap.Strict                  as HM
@@ -31,7 +32,9 @@ import           Development.IDE.Types.HscEnvEq       (hscEnv)
 import           Development.IDE.Types.Location
 import qualified HieDb
 import           Language.LSP.Protocol.Types          (DocumentHighlight (..),
-                                                       SymbolInformation (..))
+                                                       SymbolInformation (..),
+                                                       normalizedFilePathToUri,
+                                                       uriToNormalizedFilePath)
 
 
 -- | Eventually this will lookup/generate URIs for files in dependencies, but not in the
@@ -66,10 +69,36 @@ getAtPoint file pos = runMaybeT $ do
   !pos' <- MaybeT (return $ fromCurrentPosition mapping pos)
   MaybeT $ pure $ first (toCurrentRange mapping =<<) <$> AtPoint.atPoint opts hf dkMap env pos'
 
-toCurrentLocations :: PositionMapping -> [Location] -> [Location]
-toCurrentLocations mapping = mapMaybe go
+-- | For each Loacation, determine if we have the PositionMapping
+-- for the correct file. If not, get the correct position mapping
+-- and then apply the position mapping to the location.
+toCurrentLocations
+  :: PositionMapping
+  -> NormalizedFilePath
+  -> [Location]
+  -> IdeAction [Location]
+toCurrentLocations mapping file = mapMaybeM go
   where
-    go (Location uri range) = Location uri <$> toCurrentRange mapping range
+    go :: Location -> IdeAction (Maybe Location)
+    go (Location uri range) =
+      -- The Location we are going to might be in a different
+      -- file than the one we are calling gotoDefinition from.
+      -- So we check that the location file matches the file
+      -- we are in.
+      if nUri == normalizedFilePathToUri file
+      -- The Location matches the file, so use the PositionMapping
+      -- we have.
+      then pure $ Location uri <$> toCurrentRange mapping range
+      -- The Location does not match the file, so get the correct
+      -- PositionMapping and use that instead.
+      else do
+        otherLocationMapping <- fmap (fmap snd) $ runMaybeT $ do
+          otherLocationFile <- MaybeT $ pure $ uriToNormalizedFilePath nUri
+          useE GetHieAst otherLocationFile
+        pure $ Location uri <$> (flip toCurrentRange range =<< otherLocationMapping)
+      where
+        nUri :: NormalizedUri
+        nUri = toNormalizedUri uri
 
 -- | useE is useful to implement functions that aren’t rules but need shortcircuiting
 -- e.g. getDefinition.
@@ -90,7 +119,8 @@ getDefinition file pos = runMaybeT $ do
     (HAR _ hf _ _ _, mapping) <- useE GetHieAst file
     (ImportMap imports, _) <- useE GetImportMap file
     !pos' <- MaybeT (pure $ fromCurrentPosition mapping pos)
-    toCurrentLocations mapping <$> AtPoint.gotoDefinition withHieDb (lookupMod hiedbWriter) opts imports hf pos'
+    locations <- AtPoint.gotoDefinition withHieDb (lookupMod hiedbWriter) opts imports hf pos'
+    MaybeT $ Just <$> toCurrentLocations mapping file locations
 
 getTypeDefinition :: NormalizedFilePath -> Position -> IdeAction (Maybe [Location])
 getTypeDefinition file pos = runMaybeT $ do
@@ -98,7 +128,8 @@ getTypeDefinition file pos = runMaybeT $ do
     opts <- liftIO $ getIdeOptionsIO ide
     (hf, mapping) <- useE GetHieAst file
     !pos' <- MaybeT (return $ fromCurrentPosition mapping pos)
-    toCurrentLocations mapping <$> AtPoint.gotoTypeDefinition withHieDb (lookupMod hiedbWriter) opts hf pos'
+    locations <- AtPoint.gotoTypeDefinition withHieDb (lookupMod hiedbWriter) opts hf pos'
+    MaybeT $ Just <$> toCurrentLocations mapping file locations
 
 highlightAtPoint :: NormalizedFilePath -> Position -> IdeAction (Maybe [DocumentHighlight])
 highlightAtPoint file pos = runMaybeT $ do

From e23922f44d65b2d84fd934ce2aedfe1a034d97d2 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 15 Jun 2023 12:15:10 -0500
Subject: [PATCH 03/50] Implement lookupMod function

---
 ghcide/src/Development/IDE/Core/Actions.hs  | 46 +++++++++++++++++++--
 ghcide/src/Development/IDE/Spans/AtPoint.hs |  1 +
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Actions.hs b/ghcide/src/Development/IDE/Core/Actions.hs
index 7c0ea1a07e..259499de08 100644
--- a/ghcide/src/Development/IDE/Core/Actions.hs
+++ b/ghcide/src/Development/IDE/Core/Actions.hs
@@ -16,10 +16,12 @@ module Development.IDE.Core.Actions
 import           Control.Monad.Extra                  (mapMaybeM)
 import           Control.Monad.Reader
 import           Control.Monad.Trans.Maybe
+import qualified Data.ByteString                      as BS
 import qualified Data.HashMap.Strict                  as HM
 import           Data.Maybe
 import qualified Data.Text                            as T
 import           Data.Tuple.Extra
+import           Development.IDE.Core.Compile         (loadHieFile)
 import           Development.IDE.Core.OfInterest
 import           Development.IDE.Core.PositionMapping
 import           Development.IDE.Core.RuleTypes
@@ -35,10 +37,13 @@ import           Language.LSP.Protocol.Types          (DocumentHighlight (..),
                                                        SymbolInformation (..),
                                                        normalizedFilePathToUri,
                                                        uriToNormalizedFilePath)
+import           Language.LSP.Server                  (resRootPath)
+import           System.Directory                     (doesFileExist)
+import           System.FilePath                      ((</>))
 
 
--- | Eventually this will lookup/generate URIs for files in dependencies, but not in the
--- project. Right now, this is just a stub.
+-- | Generates URIs for files in dependencies, but not in the
+-- project.
 lookupMod
   :: HieDbWriter -- ^ access the database
   -> FilePath -- ^ The `.hie` file we got from the database
@@ -46,7 +51,42 @@ lookupMod
   -> Unit
   -> Bool -- ^ Is this file a boot file?
   -> MaybeT IdeAction Uri
-lookupMod _dbchan _hie_f _mod _uid _boot = MaybeT $ pure Nothing
+lookupMod _dbchan hieFile moduleName _uid _boot = MaybeT $ do
+  mProjectRoot <- (resRootPath =<<) <$> asks lspEnv
+  case mProjectRoot of
+    Nothing -> pure Nothing
+    Just projectRoot -> do
+      let toFilePath :: ModuleName -> FilePath
+          toFilePath = separateDirectories . show
+            where
+              separateDirectories :: FilePath -> FilePath
+              separateDirectories moduleNameString =
+                case breakOnDot moduleNameString of
+                  [] -> ""
+                  ms -> foldr1 (</>) ms
+              breakOnDot :: FilePath -> [FilePath]
+              breakOnDot = words . map replaceDotWithSpace
+              replaceDotWithSpace :: Char -> Char
+              replaceDotWithSpace '.' = ' '
+              replaceDotWithSpace c = c
+          writeOutDir :: FilePath
+          writeOutDir = projectRoot </> ".hls" </> "dependencies"
+          writeOutFile :: FilePath
+          writeOutFile = toFilePath moduleName ++ ".hs"
+          writeOutPath :: FilePath
+          writeOutPath = writeOutDir </> writeOutFile
+          moduleUri :: Uri
+          moduleUri = AtPoint.toUri writeOutPath
+      fileExists <- liftIO $ doesFileExist writeOutPath
+      if fileExists
+      then pure $ Just moduleUri
+      else do
+        nc <- asks ideNc
+        liftIO $ do
+          moduleSource <- hie_hs_src <$> loadHieFile (mkUpdater nc) hieFile
+          BS.writeFile writeOutPath moduleSource
+        pure $ Just moduleUri
+
 
 
 -- IMPORTANT NOTE : make sure all rules `useE`d by these have a "Persistent Stale" rule defined,
diff --git a/ghcide/src/Development/IDE/Spans/AtPoint.hs b/ghcide/src/Development/IDE/Spans/AtPoint.hs
index 37b0fbcc17..90b20c8646 100644
--- a/ghcide/src/Development/IDE/Spans/AtPoint.hs
+++ b/ghcide/src/Development/IDE/Spans/AtPoint.hs
@@ -19,6 +19,7 @@ module Development.IDE.Spans.AtPoint (
   , defRowToSymbolInfo
   , getNamesAtPoint
   , toCurrentLocation
+  , toUri
   , rowToLoc
   , nameToLocation
   , LookupModule

From fdfffb7e3501e60a0bad29e47b3d019e6dcea6ac Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Tue, 20 Jun 2023 12:11:39 -0500
Subject: [PATCH 04/50] Create .hls directory in lookupMod

---
 ghcide/src/Development/IDE/Core/Actions.hs | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Actions.hs b/ghcide/src/Development/IDE/Core/Actions.hs
index 259499de08..ce5a31996f 100644
--- a/ghcide/src/Development/IDE/Core/Actions.hs
+++ b/ghcide/src/Development/IDE/Core/Actions.hs
@@ -38,8 +38,8 @@ import           Language.LSP.Protocol.Types          (DocumentHighlight (..),
                                                        normalizedFilePathToUri,
                                                        uriToNormalizedFilePath)
 import           Language.LSP.Server                  (resRootPath)
-import           System.Directory                     (doesFileExist)
-import           System.FilePath                      ((</>))
+import           System.Directory                     (createDirectoryIfMissing, doesFileExist)
+import           System.FilePath                      ((</>), takeDirectory)
 
 
 -- | Generates URIs for files in dependencies, but not in the
@@ -51,13 +51,13 @@ lookupMod
   -> Unit
   -> Bool -- ^ Is this file a boot file?
   -> MaybeT IdeAction Uri
-lookupMod _dbchan hieFile moduleName _uid _boot = MaybeT $ do
+lookupMod _dbchan hieFile moduleName uid _boot = MaybeT $ do
   mProjectRoot <- (resRootPath =<<) <$> asks lspEnv
   case mProjectRoot of
     Nothing -> pure Nothing
     Just projectRoot -> do
       let toFilePath :: ModuleName -> FilePath
-          toFilePath = separateDirectories . show
+          toFilePath = separateDirectories . prettyModuleName
             where
               separateDirectories :: FilePath -> FilePath
               separateDirectories moduleNameString =
@@ -69,8 +69,14 @@ lookupMod _dbchan hieFile moduleName _uid _boot = MaybeT $ do
               replaceDotWithSpace :: Char -> Char
               replaceDotWithSpace '.' = ' '
               replaceDotWithSpace c = c
+              prettyModuleName :: ModuleName -> String
+              prettyModuleName = filter (/= '"')
+                . concat
+                . drop 1
+                . words
+                . show
           writeOutDir :: FilePath
-          writeOutDir = projectRoot </> ".hls" </> "dependencies"
+          writeOutDir = projectRoot </> ".hls" </> "dependencies" </> show uid
           writeOutFile :: FilePath
           writeOutFile = toFilePath moduleName ++ ".hs"
           writeOutPath :: FilePath
@@ -83,6 +89,7 @@ lookupMod _dbchan hieFile moduleName _uid _boot = MaybeT $ do
       else do
         nc <- asks ideNc
         liftIO $ do
+          createDirectoryIfMissing True $ takeDirectory writeOutPath
           moduleSource <- hie_hs_src <$> loadHieFile (mkUpdater nc) hieFile
           BS.writeFile writeOutPath moduleSource
         pure $ Just moduleUri

From 838ce20d7ad720a1c07ce622782898e2c04d59e8 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Fri, 23 Jun 2023 17:43:53 -0500
Subject: [PATCH 05/50] Add ShakeExtras arg to newHscEnvEq

---
 .../session-loader/Development/IDE/Session.hs |  7 ++---
 ghcide/src/Development/IDE/Core/RuleTypes.hs  |  2 +-
 ghcide/src/Development/IDE/Types/HscEnvEq.hs  | 13 +++++-----
 .../Development/IDE/Types/HscEnvEq.hs-boot    | 26 +++++++++++++++++++
 4 files changed, 38 insertions(+), 10 deletions(-)
 create mode 100644 ghcide/src/Development/IDE/Types/HscEnvEq.hs-boot

diff --git a/ghcide/session-loader/Development/IDE/Session.hs b/ghcide/session-loader/Development/IDE/Session.hs
index cfc9796c33..1ea2ca880f 100644
--- a/ghcide/session-loader/Development/IDE/Session.hs
+++ b/ghcide/session-loader/Development/IDE/Session.hs
@@ -585,7 +585,7 @@ loadSessionWithOptions recorder SessionLoadingOptions{..} dir = do
 
           -- New HscEnv for the component in question, returns the new HscEnvEq and
           -- a mapping from FilePath to the newly created HscEnvEq.
-          let new_cache = newComponentCache recorder optExtensions hieYaml _cfp hscEnv uids
+          let new_cache = newComponentCache recorder extras optExtensions hieYaml _cfp hscEnv uids
           (cs, res) <- new_cache new
           -- Modified cache targets for everything else in the hie.yaml file
           -- which now uses the same EPS and so on
@@ -793,6 +793,7 @@ setNameCache nc hsc = hsc { hsc_NC = nc }
 -- | Create a mapping from FilePaths to HscEnvEqs
 newComponentCache
          :: Recorder (WithPriority Log)
+         -> ShakeExtras
          -> [String]       -- File extensions to consider
          -> Maybe FilePath -- Path to cradle
          -> NormalizedFilePath -- Path to file that caused the creation of this component
@@ -800,7 +801,7 @@ newComponentCache
          -> [(UnitId, DynFlags)]
          -> ComponentInfo
          -> IO ( [TargetDetails], (IdeResult HscEnvEq, DependencyInfo))
-newComponentCache recorder exts cradlePath cfp hsc_env uids ci = do
+newComponentCache recorder extras exts cradlePath cfp hsc_env uids ci = do
     let df = componentDynFlags ci
     hscEnv' <-
 #if MIN_VERSION_ghc(9,3,0)
@@ -823,7 +824,7 @@ newComponentCache recorder exts cradlePath cfp hsc_env uids ci = do
 #endif
 
     let newFunc = maybe newHscEnvEqPreserveImportPaths newHscEnvEq cradlePath
-    henv <- newFunc hscEnv' uids
+    henv <- newFunc extras hscEnv' uids
     let targetEnv = ([], Just henv)
         targetDepends = componentDependencyInfo ci
         res = (targetEnv, targetDepends)
diff --git a/ghcide/src/Development/IDE/Core/RuleTypes.hs b/ghcide/src/Development/IDE/Core/RuleTypes.hs
index 491f4d4e0c..a5b63463ee 100644
--- a/ghcide/src/Development/IDE/Core/RuleTypes.hs
+++ b/ghcide/src/Development/IDE/Core/RuleTypes.hs
@@ -31,7 +31,7 @@ import           Development.IDE.GHC.CoreFile
 import           Development.IDE.GHC.Util
 import           Development.IDE.Graph
 import           Development.IDE.Import.DependencyInformation
-import           Development.IDE.Types.HscEnvEq               (HscEnvEq)
+import {-# SOURCE #-} Development.IDE.Types.HscEnvEq          (HscEnvEq)
 import           Development.IDE.Types.KnownTargets
 import           GHC.Generics                                 (Generic)
 
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 623e1da691..3c3ac614ca 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -22,6 +22,7 @@ import           Data.Set                        (Set)
 import qualified Data.Set                        as Set
 import           Data.Unique                     (Unique)
 import qualified Data.Unique                     as Unique
+import           Development.IDE.Core.Shake      (ShakeExtras)
 import           Development.IDE.GHC.Compat
 import qualified Development.IDE.GHC.Compat.Util as Maybes
 import           Development.IDE.GHC.Error       (catchSrcErrors)
@@ -59,8 +60,8 @@ updateHscEnvEq oldHscEnvEq newHscEnv = do
   update <$> Unique.newUnique
 
 -- | Wrap an 'HscEnv' into an 'HscEnvEq'.
-newHscEnvEq :: FilePath -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
-newHscEnvEq cradlePath hscEnv0 deps = do
+newHscEnvEq :: FilePath -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
+newHscEnvEq cradlePath se hscEnv0 deps = do
     let relativeToCradle = (takeDirectory cradlePath </>)
         hscEnv = removeImportPaths hscEnv0
 
@@ -68,10 +69,10 @@ newHscEnvEq cradlePath hscEnv0 deps = do
     importPathsCanon <-
       mapM makeAbsolute $ relativeToCradle <$> importPaths (hsc_dflags hscEnv0)
 
-    newHscEnvEqWithImportPaths (Just $ Set.fromList importPathsCanon) hscEnv deps
+    newHscEnvEqWithImportPaths (Just $ Set.fromList importPathsCanon) se hscEnv deps
 
-newHscEnvEqWithImportPaths :: Maybe (Set FilePath) -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
-newHscEnvEqWithImportPaths envImportPaths hscEnv deps = do
+newHscEnvEqWithImportPaths :: Maybe (Set FilePath) -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
+newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
 
     let dflags = hsc_dflags hscEnv
 
@@ -115,7 +116,7 @@ newHscEnvEqWithImportPaths envImportPaths hscEnv deps = do
 
 -- | Wrap an 'HscEnv' into an 'HscEnvEq'.
 newHscEnvEqPreserveImportPaths
-    :: HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
+    :: ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
 newHscEnvEqPreserveImportPaths = newHscEnvEqWithImportPaths Nothing
 
 -- | Unwrap the 'HscEnv' with the original import paths.
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs-boot b/ghcide/src/Development/IDE/Types/HscEnvEq.hs-boot
new file mode 100644
index 0000000000..6ff6390e18
--- /dev/null
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs-boot
@@ -0,0 +1,26 @@
+module Development.IDE.Types.HscEnvEq (HscEnvEq) where
+
+import           Data.Set                        (Set)
+import           Data.Unique                     (Unique)
+import           Development.IDE.GHC.Compat
+import           Development.IDE.Types.Exports   (ExportsMap)
+
+-- | An 'HscEnv' with equality. Two values are considered equal
+--   if they are created with the same call to 'newHscEnvEq'.
+data HscEnvEq = HscEnvEq
+    { envUnique             :: !Unique
+    , hscEnv                :: !HscEnv
+    , deps                  :: [(UnitId, DynFlags)]
+               -- ^ In memory components for this HscEnv
+               -- This is only used at the moment for the import dirs in
+               -- the DynFlags
+    , envImportPaths        :: Maybe (Set FilePath)
+        -- ^ If Just, import dirs originally configured in this env
+        --   If Nothing, the env import dirs are unaltered
+    , envPackageExports     :: IO ExportsMap
+    , envVisibleModuleNames :: IO (Maybe [ModuleName])
+        -- ^ 'listVisibleModuleNames' is a pure function,
+        -- but it could panic due to a ghc bug: https://github.com/haskell/haskell-language-server/issues/1365
+        -- So it's wrapped in IO here for error handling
+        -- If Nothing, 'listVisibleModuleNames' panic
+    }

From 429bd9a6bcf0e4bf5c43dc6da83472d8f80cd229 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Mon, 26 Jun 2023 12:31:54 -0500
Subject: [PATCH 06/50] Call indexHieFile in newHscEnvEq

---
 ghcide/src/Development/IDE/Core/Compile.hs    | 20 ++--
 ghcide/src/Development/IDE/Core/Rules.hs      |  2 +-
 .../src/Development/IDE/GHC/Compat/Units.hs   |  5 +
 ghcide/src/Development/IDE/Types/HscEnvEq.hs  | 91 +++++++++++++++----
 4 files changed, 88 insertions(+), 30 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Compile.hs b/ghcide/src/Development/IDE/Core/Compile.hs
index 3b8ee793a1..08294c4b5f 100644
--- a/ghcide/src/Development/IDE/Core/Compile.hs
+++ b/ghcide/src/Development/IDE/Core/Compile.hs
@@ -887,23 +887,23 @@ spliceExpressions Splices{..} =
 -- TVar to 0 in order to set it up for a fresh indexing session. Otherwise, we
 -- can just increment the 'indexCompleted' TVar and exit.
 --
-indexHieFile :: ShakeExtras -> ModSummary -> NormalizedFilePath -> Util.Fingerprint -> Compat.HieFile -> IO ()
-indexHieFile se mod_summary srcPath !hash hf = do
+indexHieFile :: ShakeExtras -> NormalizedFilePath -> HieDb.SourceFile -> Util.Fingerprint -> Compat.HieFile -> IO ()
+indexHieFile se hiePath sourceFile !hash hf = do
  IdeOptions{optProgressStyle} <- getIdeOptionsIO se
  atomically $ do
   pending <- readTVar indexPending
-  case HashMap.lookup srcPath pending of
+  case HashMap.lookup hiePath pending of
     Just pendingHash | pendingHash == hash -> pure () -- An index is already scheduled
     _ -> do
       -- hiedb doesn't use the Haskell src, so we clear it to avoid unnecessarily keeping it around
       let !hf' = hf{hie_hs_src = mempty}
-      modifyTVar' indexPending $ HashMap.insert srcPath hash
+      modifyTVar' indexPending $ HashMap.insert hiePath hash
       writeTQueue indexQueue $ \withHieDb -> do
         -- We are now in the worker thread
         -- Check if a newer index of this file has been scheduled, and if so skip this one
         newerScheduled <- atomically $ do
           pending <- readTVar indexPending
-          pure $ case HashMap.lookup srcPath pending of
+          pure $ case HashMap.lookup hiePath pending of
             Nothing          -> False
             -- If the hash in the pending list doesn't match the current hash, then skip
             Just pendingHash -> pendingHash /= hash
@@ -911,10 +911,8 @@ indexHieFile se mod_summary srcPath !hash hf = do
           -- Using bracket, so even if an exception happen during withHieDb call,
           -- the `post` (which clean the progress indicator) will still be called.
           bracket_ (pre optProgressStyle) post $
-            withHieDb (\db -> HieDb.addRefsFromLoaded db targetPath (HieDb.RealFile $ fromNormalizedFilePath srcPath) hash hf')
+            withHieDb (\db -> HieDb.addRefsFromLoaded db (fromNormalizedFilePath hiePath) sourceFile hash hf')
   where
-    mod_location    = ms_location mod_summary
-    targetPath      = Compat.ml_hie_file mod_location
     HieDbWriter{..} = hiedbWriter se
 
     -- Get a progress token to report progress and update it for the current file
@@ -978,7 +976,7 @@ indexHieFile se mod_summary srcPath !hash hf = do
       mdone <- atomically $ do
         -- Remove current element from pending
         pending <- stateTVar indexPending $
-          dupe . HashMap.update (\pendingHash -> guard (pendingHash /= hash) $> pendingHash) srcPath
+          dupe . HashMap.update (\pendingHash -> guard (pendingHash /= hash) $> pendingHash) hiePath
         modifyTVar' indexCompleted (+1)
         -- If we are done, report and reset completed
         whenMaybe (HashMap.null pending) $
@@ -986,7 +984,7 @@ indexHieFile se mod_summary srcPath !hash hf = do
       whenJust (lspEnv se) $ \env -> LSP.runLspT env $
         when (coerce $ ideTesting se) $
           LSP.sendNotification (LSP.SMethod_CustomMethod (Proxy @"ghcide/reference/ready")) $
-            toJSON $ fromNormalizedFilePath srcPath
+            toJSON $ fromNormalizedFilePath hiePath
       whenJust mdone $ \done ->
         modifyVar_ indexProgressToken $ \tok -> do
           whenJust (lspEnv se) $ \env -> LSP.runLspT env $
@@ -1007,7 +1005,7 @@ writeAndIndexHieFile hscEnv se mod_summary srcPath exports ast source =
       GHC.mkHieFile' mod_summary exports ast source
     atomicFileWrite se targetPath $ flip GHC.writeHieFile hf
     hash <- Util.getFileHash targetPath
-    indexHieFile se mod_summary srcPath hash hf
+    indexHieFile se (toNormalizedFilePath' targetPath) (HieDb.RealFile $ fromNormalizedFilePath srcPath) hash hf
   where
     dflags       = hsc_dflags hscEnv
     mod_location = ms_location mod_summary
diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs
index 109259df7b..652053daa8 100644
--- a/ghcide/src/Development/IDE/Core/Rules.hs
+++ b/ghcide/src/Development/IDE/Core/Rules.hs
@@ -876,7 +876,7 @@ getModIfaceFromDiskAndIndexRule recorder =
         -- can just re-index the file we read from disk
         Right hf -> liftIO $ do
           logWith recorder Logger.Debug $ LogReindexingHieFile f
-          indexHieFile se ms f hash hf
+          indexHieFile se (toNormalizedFilePath' hie_loc) (HieDb.RealFile $ fromNormalizedFilePath f) hash hf
 
   return (Just x)
 
diff --git a/ghcide/src/Development/IDE/GHC/Compat/Units.hs b/ghcide/src/Development/IDE/GHC/Compat/Units.hs
index 4bf7454ab5..5a9e5c0f32 100644
--- a/ghcide/src/Development/IDE/GHC/Compat/Units.hs
+++ b/ghcide/src/Development/IDE/GHC/Compat/Units.hs
@@ -24,6 +24,8 @@ module Development.IDE.GHC.Compat.Units (
     -- * UnitInfo
     UnitInfo,
     unitExposedModules,
+    unitLibraryDirs,
+    UnitInfo.unitId,
     unitDepends,
     unitHaddockInterfaces,
     unitInfoId,
@@ -273,6 +275,9 @@ preloadClosureUs = State.preloadClosure . unitState
 preloadClosureUs = const ()
 #endif
 
+unitLibraryDirs :: UnitInfo -> [FilePath]
+unitLibraryDirs = fmap ST.unpack . UnitInfo.unitLibraryDirs
+
 unitExposedModules :: UnitInfo -> [(ModuleName, Maybe Module)]
 unitExposedModules ue =
 #if MIN_VERSION_ghc(9,0,0)
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 3c3ac614ca..ca0b0d2045 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -18,17 +18,23 @@ import           Control.DeepSeq                 (force)
 import           Control.Exception               (evaluate, mask, throwIO)
 import           Control.Monad.Extra             (eitherM, join, mapMaybeM)
 import           Data.Either                     (fromRight)
+import qualified Data.Map                        as Map
 import           Data.Set                        (Set)
 import qualified Data.Set                        as Set
+import qualified Data.Text                       as T
 import           Data.Unique                     (Unique)
 import qualified Data.Unique                     as Unique
-import           Development.IDE.Core.Shake      (ShakeExtras)
+import           Development.IDE.Core.Compile    (indexHieFile, loadHieFile)
+import           Development.IDE.Core.Shake      (ShakeExtras(ideNc, logger), mkUpdater)
 import           Development.IDE.GHC.Compat
 import qualified Development.IDE.GHC.Compat.Util as Maybes
 import           Development.IDE.GHC.Error       (catchSrcErrors)
 import           Development.IDE.GHC.Util        (lookupPackageConfig)
 import           Development.IDE.Graph.Classes
 import           Development.IDE.Types.Exports   (ExportsMap, createExportsMap)
+import           Development.IDE.Types.Location  (toNormalizedFilePath')
+import qualified Development.IDE.Types.Logger    as Logger
+import           HieDb                           (SourceFile(FakeFile))
 import           OpenTelemetry.Eventlog          (withSpan)
 import           System.Directory                (makeAbsolute)
 import           System.FilePath
@@ -71,6 +77,10 @@ newHscEnvEq cradlePath se hscEnv0 deps = do
 
     newHscEnvEqWithImportPaths (Just $ Set.fromList importPathsCanon) se hscEnv deps
 
+newtype UnitInfoOrd = UnitInfoOrd UnitInfo deriving Eq
+instance Ord UnitInfoOrd where
+  compare (UnitInfoOrd u1) (UnitInfoOrd u2) = compare (unitId u1) (unitId u2)
+
 newHscEnvEqWithImportPaths :: Maybe (Set FilePath) -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
 newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
 
@@ -83,25 +93,70 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
         -- compute the package imports
         let pkgst   = unitState hscEnv
             depends = explicitUnits pkgst
-            modules =
-                [ m
-                | d        <- depends
-                , Just pkg <- [lookupPackageConfig d hscEnv]
-                , (modName, maybeOtherPkgMod) <- unitExposedModules pkg
-                , let m = case maybeOtherPkgMod of
-                        -- When module is re-exported from another package,
-                        -- the origin module is represented by value in Just
-                        Just otherPkgMod -> otherPkgMod
-                        Nothing          -> mkModule (unitInfoId pkg) modName
-                ]
-
-            doOne m = do
+            packages = [ pkg
+                       | d <- depends
+                       , Just pkg <- [lookupPackageConfig d hscEnv]
+                       ]
+            modules = Map.fromSet
+                (\(UnitInfoOrd pkg) ->
+                  [ m
+                  | (modName, maybeOtherPkgMod) <- unitExposedModules pkg
+                  , let m = case maybeOtherPkgMod of
+                          -- When module is re-exported from another package,
+                          -- the origin module is represented by value in Just
+                          Just otherPkgMod -> otherPkgMod
+                          Nothing          -> mkModule (unitInfoId pkg) modName
+                  ]
+                )
+                (Set.fromList $ map UnitInfoOrd packages)
+
+            logPackage :: UnitInfo -> IO ()
+            logPackage pkg = Logger.logDebug (logger se) $ "\n\n\n!!!!!!!!!!!! hscEnvEq :\n"
+              <> T.pack (concatMap show $ unitLibraryDirs pkg)
+              <> "\n!!!!!!!!!!!!!!!!!!!!!!\n\n\n"
+            doOnePackage :: UnitInfoOrd -> [Module] -> IO [ModIface]
+            doOnePackage (UnitInfoOrd pkg) ms = do
+              let pkgLibDir :: FilePath
+                  pkgLibDir = case unitLibraryDirs pkg of
+                    [] -> ""
+                    (libraryDir : _) -> libraryDir
+                  hieDir :: FilePath
+                  hieDir = pkgLibDir </> "extra-compliation-artifacts"
+              logPackage pkg
+              mapMaybeM (doOne hieDir) ms
+
+            doOne :: FilePath -> Module -> IO (Maybe ModIface)
+            doOne hieDir m = do
+                let toFilePath :: ModuleName -> FilePath
+                    toFilePath = separateDirectories . prettyModuleName
+                      where
+                        separateDirectories :: FilePath -> FilePath
+                        separateDirectories moduleNameString =
+                          case breakOnDot moduleNameString of
+                            [] -> ""
+                            ms -> foldr1 (</>) ms
+                        breakOnDot :: FilePath -> [FilePath]
+                        breakOnDot = words . map replaceDotWithSpace
+                        replaceDotWithSpace :: Char -> Char
+                        replaceDotWithSpace '.' = ' '
+                        replaceDotWithSpace c = c
+                        prettyModuleName :: ModuleName -> String
+                        prettyModuleName = filter (/= '"')
+                          . concat
+                          . drop 1
+                          . words
+                          . show
+                    hiePath :: FilePath
+                    hiePath = hieDir </> toFilePath (moduleName m) ++ ".hie"
                 modIface <- initIfaceLoad hscEnv $
                     loadInterface "" m (ImportByUser NotBoot)
-                return $ case modIface of
-                    Maybes.Failed    _r -> Nothing
-                    Maybes.Succeeded mi -> Just mi
-        modIfaces <- mapMaybeM doOne modules
+                case modIface of
+                    Maybes.Failed    _r -> return Nothing
+                    Maybes.Succeeded mi -> do
+                      hie <- loadHieFile (mkUpdater $ ideNc se) hiePath
+                      indexHieFile se (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash mi) hie
+                      return $ Just mi
+        modIfaces <- concat . Map.elems <$> Map.traverseWithKey doOnePackage modules
         return $ createExportsMap modIfaces
 
     -- similar to envPackageExports, evaluated lazily

From fd7f95ed63e1c67c02c701c802403438025a4189 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Wed, 28 Jun 2023 12:20:24 -0500
Subject: [PATCH 07/50] Factor out loading ModIfaces

---
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 164 +++++++++++--------
 1 file changed, 92 insertions(+), 72 deletions(-)

diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index ca0b0d2045..a427c67830 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -16,8 +16,9 @@ import           Control.Concurrent.Async        (Async, async, waitCatch)
 import           Control.Concurrent.Strict       (modifyVar, newVar)
 import           Control.DeepSeq                 (force)
 import           Control.Exception               (evaluate, mask, throwIO)
-import           Control.Monad.Extra             (eitherM, join, mapMaybeM)
+import           Control.Monad.Extra             (eitherM, join, mapMaybeM, void)
 import           Data.Either                     (fromRight)
+import           Data.Foldable                   (traverse_)
 import qualified Data.Map                        as Map
 import           Data.Set                        (Set)
 import qualified Data.Set                        as Set
@@ -77,88 +78,20 @@ newHscEnvEq cradlePath se hscEnv0 deps = do
 
     newHscEnvEqWithImportPaths (Just $ Set.fromList importPathsCanon) se hscEnv deps
 
-newtype UnitInfoOrd = UnitInfoOrd UnitInfo deriving Eq
-instance Ord UnitInfoOrd where
-  compare (UnitInfoOrd u1) (UnitInfoOrd u2) = compare (unitId u1) (unitId u2)
-
 newHscEnvEqWithImportPaths :: Maybe (Set FilePath) -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
 newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
 
-    let dflags = hsc_dflags hscEnv
+    indexDependencyHieFiles
 
     envUnique <- Unique.newUnique
 
     -- it's very important to delay the package exports computation
     envPackageExports <- onceAsync $ withSpan "Package Exports" $ \_sp -> do
         -- compute the package imports
-        let pkgst   = unitState hscEnv
-            depends = explicitUnits pkgst
-            packages = [ pkg
-                       | d <- depends
-                       , Just pkg <- [lookupPackageConfig d hscEnv]
-                       ]
-            modules = Map.fromSet
-                (\(UnitInfoOrd pkg) ->
-                  [ m
-                  | (modName, maybeOtherPkgMod) <- unitExposedModules pkg
-                  , let m = case maybeOtherPkgMod of
-                          -- When module is re-exported from another package,
-                          -- the origin module is represented by value in Just
-                          Just otherPkgMod -> otherPkgMod
-                          Nothing          -> mkModule (unitInfoId pkg) modName
-                  ]
-                )
-                (Set.fromList $ map UnitInfoOrd packages)
-
-            logPackage :: UnitInfo -> IO ()
-            logPackage pkg = Logger.logDebug (logger se) $ "\n\n\n!!!!!!!!!!!! hscEnvEq :\n"
-              <> T.pack (concatMap show $ unitLibraryDirs pkg)
-              <> "\n!!!!!!!!!!!!!!!!!!!!!!\n\n\n"
-            doOnePackage :: UnitInfoOrd -> [Module] -> IO [ModIface]
-            doOnePackage (UnitInfoOrd pkg) ms = do
-              let pkgLibDir :: FilePath
-                  pkgLibDir = case unitLibraryDirs pkg of
-                    [] -> ""
-                    (libraryDir : _) -> libraryDir
-                  hieDir :: FilePath
-                  hieDir = pkgLibDir </> "extra-compliation-artifacts"
-              logPackage pkg
-              mapMaybeM (doOne hieDir) ms
-
-            doOne :: FilePath -> Module -> IO (Maybe ModIface)
-            doOne hieDir m = do
-                let toFilePath :: ModuleName -> FilePath
-                    toFilePath = separateDirectories . prettyModuleName
-                      where
-                        separateDirectories :: FilePath -> FilePath
-                        separateDirectories moduleNameString =
-                          case breakOnDot moduleNameString of
-                            [] -> ""
-                            ms -> foldr1 (</>) ms
-                        breakOnDot :: FilePath -> [FilePath]
-                        breakOnDot = words . map replaceDotWithSpace
-                        replaceDotWithSpace :: Char -> Char
-                        replaceDotWithSpace '.' = ' '
-                        replaceDotWithSpace c = c
-                        prettyModuleName :: ModuleName -> String
-                        prettyModuleName = filter (/= '"')
-                          . concat
-                          . drop 1
-                          . words
-                          . show
-                    hiePath :: FilePath
-                    hiePath = hieDir </> toFilePath (moduleName m) ++ ".hie"
-                modIface <- initIfaceLoad hscEnv $
-                    loadInterface "" m (ImportByUser NotBoot)
-                case modIface of
-                    Maybes.Failed    _r -> return Nothing
-                    Maybes.Succeeded mi -> do
-                      hie <- loadHieFile (mkUpdater $ ideNc se) hiePath
-                      indexHieFile se (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash mi) hie
-                      return $ Just mi
-        modIfaces <- concat . Map.elems <$> Map.traverseWithKey doOnePackage modules
+        modIfaces <- concat . Map.elems <$> loadPackagesWithModIFaces
         return $ createExportsMap modIfaces
 
+    let dflags = hsc_dflags hscEnv
     -- similar to envPackageExports, evaluated lazily
     envVisibleModuleNames <- onceAsync $
       fromRight Nothing
@@ -168,6 +101,93 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
           (evaluate . force . Just $ listVisibleModuleNames hscEnv)
 
     return HscEnvEq{..}
+    where
+        indexDependencyHieFiles :: IO ()
+        indexDependencyHieFiles = do
+            packagesWithModIfaces <- loadPackagesWithModIFaces
+            void $ Map.traverseWithKey indexPackageHieFiles packagesWithModIfaces
+        logPackage :: UnitInfo -> IO ()
+        logPackage package = Logger.logDebug (logger se) $ "\n\n\n!!!!!!!!!!!! hscEnvEq :\n"
+          <> T.pack (concatMap show $ unitLibraryDirs package)
+          <> "\n!!!!!!!!!!!!!!!!!!!!!!\n\n\n"
+        indexPackageHieFiles :: Package -> [ModIface] -> IO ()
+        indexPackageHieFiles (Package package) modIfaces = do
+            let pkgLibDir :: FilePath
+                pkgLibDir = case unitLibraryDirs package of
+                  [] -> ""
+                  (libraryDir : _) -> libraryDir
+                hieDir :: FilePath
+                hieDir = pkgLibDir </> "extra-compilation-artifacts"
+            logPackage package
+            traverse_ (indexModuleHieFile hieDir) modIfaces
+        logModule :: FilePath -> IO ()
+        logModule hiePath = Logger.logDebug (logger se) $ "\n\n\n!!!!!!!!!!!! hscEnvEq :\n"
+          <> T.pack hiePath
+          <> "\n!!!!!!!!!!!!!!!!!!!!!!\n\n\n"
+        indexModuleHieFile :: FilePath -> ModIface -> IO ()
+        indexModuleHieFile hieDir modIface = do
+            let hiePath :: FilePath
+                hiePath = hieDir </> toFilePath (moduleName $ mi_module modIface) ++ ".hie"
+            logModule hiePath
+            hie <- loadHieFile (mkUpdater $ ideNc se) hiePath
+            indexHieFile se (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash modIface) hie
+        toFilePath :: ModuleName -> FilePath
+        toFilePath = separateDirectories . prettyModuleName
+            where
+                separateDirectories :: FilePath -> FilePath
+                separateDirectories moduleNameString =
+                    case breakOnDot moduleNameString of
+                        [] -> ""
+                        ms -> foldr1 (</>) ms
+                breakOnDot :: FilePath -> [FilePath]
+                breakOnDot = words . map replaceDotWithSpace
+                replaceDotWithSpace :: Char -> Char
+                replaceDotWithSpace '.' = ' '
+                replaceDotWithSpace c = c
+                prettyModuleName :: ModuleName -> String
+                prettyModuleName = filter (/= '"')
+                    . concat
+                    . drop 1
+                    . words
+                    . show
+        loadModIFace :: Module -> IO (Maybe ModIface)
+        loadModIFace m = do
+            modIface <- initIfaceLoad hscEnv $
+                loadInterface "" m (ImportByUser NotBoot)
+            return $ case modIface of
+                Maybes.Failed    _r -> Nothing
+                Maybes.Succeeded mi -> Just mi
+        loadPackagesWithModIFaces :: IO (Map.Map Package [ModIface])
+        loadPackagesWithModIFaces = Map.traverseWithKey
+            (const $ mapMaybeM loadModIFace) packagesWithModules
+        packagesWithModules :: Map.Map Package [Module]
+        packagesWithModules = Map.fromSet getModulesForPackage packages
+        packageState :: UnitState
+        packageState = unitState hscEnv
+        dependencies :: [Unit]
+        dependencies = explicitUnits packageState
+        packages :: Set Package
+        packages = Set.fromList
+            $ map Package
+            [ package
+            | d <- dependencies
+            , Just package <- [lookupPackageConfig d hscEnv]
+            ]
+        getModulesForPackage :: Package -> [Module]
+        getModulesForPackage (Package package) =
+            [ m
+            | (modName, maybeOtherPkgMod) <- unitExposedModules package
+            , let m = case maybeOtherPkgMod of
+                    -- When module is re-exported from another package,
+                    -- the origin module is represented by value in Just
+                    Just otherPkgMod -> otherPkgMod
+                    Nothing          -> mkModule (unitInfoId package) modName
+            ]
+
+newtype Package = Package UnitInfo deriving Eq
+instance Ord Package where
+  compare (Package u1) (Package u2) = compare (unitId u1) (unitId u2)
+
 
 -- | Wrap an 'HscEnv' into an 'HscEnvEq'.
 newHscEnvEqPreserveImportPaths

From 354bcb69b294d4494af4c404934ca41d9c969083 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 29 Jun 2023 07:32:58 -0500
Subject: [PATCH 08/50] Index hidden module hie files

---
 .../src/Development/IDE/GHC/Compat/Units.hs   |  4 ++
 ghcide/src/Development/IDE/Types/HscEnvEq.hs  | 42 +++++++++++++++----
 2 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/ghcide/src/Development/IDE/GHC/Compat/Units.hs b/ghcide/src/Development/IDE/GHC/Compat/Units.hs
index 5a9e5c0f32..327f344517 100644
--- a/ghcide/src/Development/IDE/GHC/Compat/Units.hs
+++ b/ghcide/src/Development/IDE/GHC/Compat/Units.hs
@@ -24,6 +24,7 @@ module Development.IDE.GHC.Compat.Units (
     -- * UnitInfo
     UnitInfo,
     unitExposedModules,
+    unitHiddenModules,
     unitLibraryDirs,
     UnitInfo.unitId,
     unitDepends,
@@ -286,6 +287,9 @@ unitExposedModules ue =
   Packages.exposedModules ue
 #endif
 
+unitHiddenModules :: UnitInfo -> [ModuleName]
+unitHiddenModules = UnitInfo.unitHiddenModules
+
 unitDepends :: UnitInfo -> [UnitId]
 #if MIN_VERSION_ghc(9,0,0)
 unitDepends = State.unitDepends
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index a427c67830..71bbf0bb51 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -88,7 +88,27 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
     -- it's very important to delay the package exports computation
     envPackageExports <- onceAsync $ withSpan "Package Exports" $ \_sp -> do
         -- compute the package imports
-        modIfaces <- concat . Map.elems <$> loadPackagesWithModIFaces
+        let pkgst   = unitState hscEnv
+            depends = explicitUnits pkgst
+            modules =
+                [ m
+                | d        <- depends
+                , Just pkg <- [lookupPackageConfig d hscEnv]
+                , (modName, maybeOtherPkgMod) <- unitExposedModules pkg
+                , let m = case maybeOtherPkgMod of
+                        -- When module is re-exported from another package,
+                        -- the origin module is represented by value in Just
+                        Just otherPkgMod -> otherPkgMod
+                        Nothing          -> mkModule (unitInfoId pkg) modName
+                ]
+
+            doOne m = do
+                modIface <- initIfaceLoad hscEnv $
+                    loadInterface "" m (ImportByUser NotBoot)
+                return $ case modIface of
+                    Maybes.Failed    _r -> Nothing
+                    Maybes.Succeeded mi -> Just mi
+        modIfaces <- mapMaybeM doOne modules
         return $ createExportsMap modIfaces
 
     let dflags = hsc_dflags hscEnv
@@ -175,14 +195,18 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
             ]
         getModulesForPackage :: Package -> [Module]
         getModulesForPackage (Package package) =
-            [ m
-            | (modName, maybeOtherPkgMod) <- unitExposedModules package
-            , let m = case maybeOtherPkgMod of
-                    -- When module is re-exported from another package,
-                    -- the origin module is represented by value in Just
-                    Just otherPkgMod -> otherPkgMod
-                    Nothing          -> mkModule (unitInfoId package) modName
-            ]
+            map makeModule allModules
+            where
+                allModules :: [(ModuleName, Maybe Module)]
+                allModules = unitExposedModules package
+                    ++ zip (unitHiddenModules package) (repeat Nothing)
+                makeModule :: (ModuleName, Maybe Module)
+                           -> Module
+                makeModule (moduleName, Nothing) =
+                    mkModule (unitInfoId package) moduleName
+                -- When module is re-exported from another package,
+                -- the origin module is represented by value in Just
+                makeModule (_, Just otherPackageMod) = otherPackageMod
 
 newtype Package = Package UnitInfo deriving Eq
 instance Ord Package where

From 599c191a721b0b684b199d2c62672ec4367b9aa4 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 29 Jun 2023 19:47:40 -0500
Subject: [PATCH 09/50] Handle loadHieFile error

---
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 52 ++++++++++----------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 71bbf0bb51..f3d2839efe 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -15,7 +15,8 @@ module Development.IDE.Types.HscEnvEq
 import           Control.Concurrent.Async        (Async, async, waitCatch)
 import           Control.Concurrent.Strict       (modifyVar, newVar)
 import           Control.DeepSeq                 (force)
-import           Control.Exception               (evaluate, mask, throwIO)
+import           Control.Exception               (SomeException, evaluate, mask, throwIO)
+import           Control.Exception.Safe          (tryAny)
 import           Control.Monad.Extra             (eitherM, join, mapMaybeM, void)
 import           Data.Either                     (fromRight)
 import           Data.Foldable                   (traverse_)
@@ -123,15 +124,15 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
     return HscEnvEq{..}
     where
         indexDependencyHieFiles :: IO ()
-        indexDependencyHieFiles = do
-            packagesWithModIfaces <- loadPackagesWithModIFaces
-            void $ Map.traverseWithKey indexPackageHieFiles packagesWithModIfaces
+        indexDependencyHieFiles = void
+            $ Map.traverseWithKey indexPackageHieFiles packagesWithModules
         logPackage :: UnitInfo -> IO ()
-        logPackage package = Logger.logDebug (logger se) $ "\n\n\n!!!!!!!!!!!! hscEnvEq :\n"
-          <> T.pack (concatMap show $ unitLibraryDirs package)
-          <> "\n!!!!!!!!!!!!!!!!!!!!!!\n\n\n"
-        indexPackageHieFiles :: Package -> [ModIface] -> IO ()
-        indexPackageHieFiles (Package package) modIfaces = do
+        logPackage package = Logger.logDebug (logger se) $ "!!!!!!!!!!!! hscEnvEq :\n"
+          <> T.pack (concatMap show $ unitLibraryDirs package) <> "\n"
+          <> T.pack (show $ unitId package)
+          <> "\n!!!!!!!!!!!!!!!!!!!!!!"
+        indexPackageHieFiles :: Package -> [Module] -> IO ()
+        indexPackageHieFiles (Package package) modules = do
             let pkgLibDir :: FilePath
                 pkgLibDir = case unitLibraryDirs package of
                   [] -> ""
@@ -139,18 +140,26 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
                 hieDir :: FilePath
                 hieDir = pkgLibDir </> "extra-compilation-artifacts"
             logPackage package
+            modIfaces <- mapMaybeM loadModIFace modules
             traverse_ (indexModuleHieFile hieDir) modIfaces
-        logModule :: FilePath -> IO ()
-        logModule hiePath = Logger.logDebug (logger se) $ "\n\n\n!!!!!!!!!!!! hscEnvEq :\n"
+        logModule :: FilePath -> Either SomeException HieFile -> IO ()
+        logModule hiePath hieResults = Logger.logDebug (logger se) $ "!!!!!!!!!!!! hscEnvEq :\n"
           <> T.pack hiePath
-          <> "\n!!!!!!!!!!!!!!!!!!!!!!\n\n\n"
+          <> (case hieResults of
+                 Left e -> "\n" <> T.pack (show e)
+                 Right _ -> ""
+             )
+          <> "\n!!!!!!!!!!!!!!!!!!!!!!"
         indexModuleHieFile :: FilePath -> ModIface -> IO ()
         indexModuleHieFile hieDir modIface = do
             let hiePath :: FilePath
                 hiePath = hieDir </> toFilePath (moduleName $ mi_module modIface) ++ ".hie"
-            logModule hiePath
-            hie <- loadHieFile (mkUpdater $ ideNc se) hiePath
-            indexHieFile se (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash modIface) hie
+            hieResults <- tryAny $ loadHieFile (mkUpdater $ ideNc se) hiePath
+            logModule hiePath hieResults
+            case hieResults of
+              Left _ -> return ()
+              Right hie ->
+                  indexHieFile se (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash modIface) hie
         toFilePath :: ModuleName -> FilePath
         toFilePath = separateDirectories . prettyModuleName
             where
@@ -177,22 +186,13 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
             return $ case modIface of
                 Maybes.Failed    _r -> Nothing
                 Maybes.Succeeded mi -> Just mi
-        loadPackagesWithModIFaces :: IO (Map.Map Package [ModIface])
-        loadPackagesWithModIFaces = Map.traverseWithKey
-            (const $ mapMaybeM loadModIFace) packagesWithModules
         packagesWithModules :: Map.Map Package [Module]
         packagesWithModules = Map.fromSet getModulesForPackage packages
-        packageState :: UnitState
-        packageState = unitState hscEnv
-        dependencies :: [Unit]
-        dependencies = explicitUnits packageState
         packages :: Set Package
         packages = Set.fromList
             $ map Package
-            [ package
-            | d <- dependencies
-            , Just package <- [lookupPackageConfig d hscEnv]
-            ]
+            $ Map.elems
+            $ getUnitInfoMap hscEnv
         getModulesForPackage :: Package -> [Module]
         getModulesForPackage (Package package) =
             map makeModule allModules

From f8726274858986d55dc3bb8d68ae46d392e4c795 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Fri, 30 Jun 2023 06:58:21 -0500
Subject: [PATCH 10/50] Log only on loadHieFile failure

---
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 21 ++++----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index f3d2839efe..5ab865a71e 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -15,7 +15,7 @@ module Development.IDE.Types.HscEnvEq
 import           Control.Concurrent.Async        (Async, async, waitCatch)
 import           Control.Concurrent.Strict       (modifyVar, newVar)
 import           Control.DeepSeq                 (force)
-import           Control.Exception               (SomeException, evaluate, mask, throwIO)
+import           Control.Exception               (evaluate, mask, throwIO)
 import           Control.Exception.Safe          (tryAny)
 import           Control.Monad.Extra             (eitherM, join, mapMaybeM, void)
 import           Data.Either                     (fromRight)
@@ -126,11 +126,6 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
         indexDependencyHieFiles :: IO ()
         indexDependencyHieFiles = void
             $ Map.traverseWithKey indexPackageHieFiles packagesWithModules
-        logPackage :: UnitInfo -> IO ()
-        logPackage package = Logger.logDebug (logger se) $ "!!!!!!!!!!!! hscEnvEq :\n"
-          <> T.pack (concatMap show $ unitLibraryDirs package) <> "\n"
-          <> T.pack (show $ unitId package)
-          <> "\n!!!!!!!!!!!!!!!!!!!!!!"
         indexPackageHieFiles :: Package -> [Module] -> IO ()
         indexPackageHieFiles (Package package) modules = do
             let pkgLibDir :: FilePath
@@ -139,25 +134,17 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
                   (libraryDir : _) -> libraryDir
                 hieDir :: FilePath
                 hieDir = pkgLibDir </> "extra-compilation-artifacts"
-            logPackage package
             modIfaces <- mapMaybeM loadModIFace modules
             traverse_ (indexModuleHieFile hieDir) modIfaces
-        logModule :: FilePath -> Either SomeException HieFile -> IO ()
-        logModule hiePath hieResults = Logger.logDebug (logger se) $ "!!!!!!!!!!!! hscEnvEq :\n"
-          <> T.pack hiePath
-          <> (case hieResults of
-                 Left e -> "\n" <> T.pack (show e)
-                 Right _ -> ""
-             )
-          <> "\n!!!!!!!!!!!!!!!!!!!!!!"
         indexModuleHieFile :: FilePath -> ModIface -> IO ()
         indexModuleHieFile hieDir modIface = do
             let hiePath :: FilePath
                 hiePath = hieDir </> toFilePath (moduleName $ mi_module modIface) ++ ".hie"
             hieResults <- tryAny $ loadHieFile (mkUpdater $ ideNc se) hiePath
-            logModule hiePath hieResults
             case hieResults of
-              Left _ -> return ()
+              Left e -> Logger.logDebug (logger se) $
+                  "Failed to index dependency HIE file:\n"
+                  <> T.pack (show e)
               Right hie ->
                   indexHieFile se (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash modIface) hie
         toFilePath :: ModuleName -> FilePath

From df0e93eb355f24298dafed75840e0bb8af2e0298 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Fri, 30 Jun 2023 07:32:57 -0500
Subject: [PATCH 11/50] Use loadModIface in ExportsMap creation

---
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 5ab865a71e..e7d6b779c5 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -102,14 +102,7 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
                         Just otherPkgMod -> otherPkgMod
                         Nothing          -> mkModule (unitInfoId pkg) modName
                 ]
-
-            doOne m = do
-                modIface <- initIfaceLoad hscEnv $
-                    loadInterface "" m (ImportByUser NotBoot)
-                return $ case modIface of
-                    Maybes.Failed    _r -> Nothing
-                    Maybes.Succeeded mi -> Just mi
-        modIfaces <- mapMaybeM doOne modules
+        modIfaces <- mapMaybeM loadModIface modules
         return $ createExportsMap modIfaces
 
     let dflags = hsc_dflags hscEnv
@@ -134,7 +127,7 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
                   (libraryDir : _) -> libraryDir
                 hieDir :: FilePath
                 hieDir = pkgLibDir </> "extra-compilation-artifacts"
-            modIfaces <- mapMaybeM loadModIFace modules
+            modIfaces <- mapMaybeM loadModIface modules
             traverse_ (indexModuleHieFile hieDir) modIfaces
         indexModuleHieFile :: FilePath -> ModIface -> IO ()
         indexModuleHieFile hieDir modIface = do
@@ -166,8 +159,8 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
                     . drop 1
                     . words
                     . show
-        loadModIFace :: Module -> IO (Maybe ModIface)
-        loadModIFace m = do
+        loadModIface :: Module -> IO (Maybe ModIface)
+        loadModIface m = do
             modIface <- initIfaceLoad hscEnv $
                 loadInterface "" m (ImportByUser NotBoot)
             return $ case modIface of

From d2b3eecb0c58ed1684b2060fa80e1b2409575d75 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Fri, 30 Jun 2023 09:07:01 -0500
Subject: [PATCH 12/50] Check if dependency HIE files already indexed

---
 ghcide/src/Development/IDE/Core/Compile.hs    | 31 +++++++++++++++----
 ghcide/src/Development/IDE/Core/Rules.hs      |  2 +-
 .../src/Development/IDE/GHC/Compat/Units.hs   |  2 +-
 ghcide/src/Development/IDE/Types/HscEnvEq.hs  | 20 +++++++-----
 4 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Compile.hs b/ghcide/src/Development/IDE/Core/Compile.hs
index 08294c4b5f..b111816679 100644
--- a/ghcide/src/Development/IDE/Core/Compile.hs
+++ b/ghcide/src/Development/IDE/Core/Compile.hs
@@ -21,6 +21,7 @@ module Development.IDE.Core.Compile
   , generateByteCode
   , generateHieAsts
   , writeAndIndexHieFile
+  , HieDbModuleQuery(..)
   , indexHieFile
   , writeHiFile
   , getModSummaryFromImports
@@ -859,6 +860,10 @@ spliceExpressions Splices{..} =
         , DL.fromList $ map fst awSplices
         ]
 
+data HieDbModuleQuery
+  = HieDbModuleQuery ModuleName Unit
+  | DontCheckForModule
+
 -- | In addition to indexing the `.hie` file, this function is responsible for
 -- maintaining the 'IndexQueue' state and notifying the user about indexing
 -- progress.
@@ -887,16 +892,14 @@ spliceExpressions Splices{..} =
 -- TVar to 0 in order to set it up for a fresh indexing session. Otherwise, we
 -- can just increment the 'indexCompleted' TVar and exit.
 --
-indexHieFile :: ShakeExtras -> NormalizedFilePath -> HieDb.SourceFile -> Util.Fingerprint -> Compat.HieFile -> IO ()
-indexHieFile se hiePath sourceFile !hash hf = do
+indexHieFile :: ShakeExtras -> HieDbModuleQuery -> NormalizedFilePath -> HieDb.SourceFile -> Util.Fingerprint -> Compat.HieFile -> IO ()
+indexHieFile se query hiePath sourceFile !hash hf = do
  IdeOptions{optProgressStyle} <- getIdeOptionsIO se
  atomically $ do
   pending <- readTVar indexPending
   case HashMap.lookup hiePath pending of
     Just pendingHash | pendingHash == hash -> pure () -- An index is already scheduled
     _ -> do
-      -- hiedb doesn't use the Haskell src, so we clear it to avoid unnecessarily keeping it around
-      let !hf' = hf{hie_hs_src = mempty}
       modifyTVar' indexPending $ HashMap.insert hiePath hash
       writeTQueue indexQueue $ \withHieDb -> do
         -- We are now in the worker thread
@@ -911,8 +914,24 @@ indexHieFile se hiePath sourceFile !hash hf = do
           -- Using bracket, so even if an exception happen during withHieDb call,
           -- the `post` (which clean the progress indicator) will still be called.
           bracket_ (pre optProgressStyle) post $
-            withHieDb (\db -> HieDb.addRefsFromLoaded db (fromNormalizedFilePath hiePath) sourceFile hash hf')
+            withHieDb indexIfNotAlready
   where
+    -- hiedb doesn't use the Haskell src, so we clear it to avoid unnecessarily keeping it around
+    hf' :: Compat.HieFile
+    !hf' = hf{hie_hs_src = mempty}
+    indexIfNotAlready :: HieDb -> IO ()
+    indexIfNotAlready db = case query of
+      DontCheckForModule -> doIndexing
+      HieDbModuleQuery moduleName unit -> do
+        mRow <- HieDb.lookupHieFile db moduleName unit
+        case mRow of
+          Nothing -> doIndexing
+          Just _row -> return ()
+      where
+        doIndexing :: IO ()
+        doIndexing =
+          HieDb.addRefsFromLoaded db (fromNormalizedFilePath hiePath) sourceFile hash hf'
+
     HieDbWriter{..} = hiedbWriter se
 
     -- Get a progress token to report progress and update it for the current file
@@ -1005,7 +1024,7 @@ writeAndIndexHieFile hscEnv se mod_summary srcPath exports ast source =
       GHC.mkHieFile' mod_summary exports ast source
     atomicFileWrite se targetPath $ flip GHC.writeHieFile hf
     hash <- Util.getFileHash targetPath
-    indexHieFile se (toNormalizedFilePath' targetPath) (HieDb.RealFile $ fromNormalizedFilePath srcPath) hash hf
+    indexHieFile se DontCheckForModule (toNormalizedFilePath' targetPath) (HieDb.RealFile $ fromNormalizedFilePath srcPath) hash hf
   where
     dflags       = hsc_dflags hscEnv
     mod_location = ms_location mod_summary
diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs
index 652053daa8..3a9a26f3d6 100644
--- a/ghcide/src/Development/IDE/Core/Rules.hs
+++ b/ghcide/src/Development/IDE/Core/Rules.hs
@@ -876,7 +876,7 @@ getModIfaceFromDiskAndIndexRule recorder =
         -- can just re-index the file we read from disk
         Right hf -> liftIO $ do
           logWith recorder Logger.Debug $ LogReindexingHieFile f
-          indexHieFile se (toNormalizedFilePath' hie_loc) (HieDb.RealFile $ fromNormalizedFilePath f) hash hf
+          indexHieFile se DontCheckForModule (toNormalizedFilePath' hie_loc) (HieDb.RealFile $ fromNormalizedFilePath f) hash hf
 
   return (Just x)
 
diff --git a/ghcide/src/Development/IDE/GHC/Compat/Units.hs b/ghcide/src/Development/IDE/GHC/Compat/Units.hs
index 327f344517..23af072063 100644
--- a/ghcide/src/Development/IDE/GHC/Compat/Units.hs
+++ b/ghcide/src/Development/IDE/GHC/Compat/Units.hs
@@ -27,6 +27,7 @@ module Development.IDE.GHC.Compat.Units (
     unitHiddenModules,
     unitLibraryDirs,
     UnitInfo.unitId,
+    UnitInfo.mkUnit,
     unitDepends,
     unitHaddockInterfaces,
     unitInfoId,
@@ -125,7 +126,6 @@ type PreloadUnitClosure = ()
 type Unit = UnitId
 #endif
 
-
 #if !MIN_VERSION_ghc(9,0,0)
 unitString :: Unit -> String
 unitString = Module.unitIdString
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index e7d6b779c5..f9ec469429 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -26,7 +26,7 @@ import qualified Data.Set                        as Set
 import qualified Data.Text                       as T
 import           Data.Unique                     (Unique)
 import qualified Data.Unique                     as Unique
-import           Development.IDE.Core.Compile    (indexHieFile, loadHieFile)
+import           Development.IDE.Core.Compile    (HieDbModuleQuery(HieDbModuleQuery), indexHieFile, loadHieFile)
 import           Development.IDE.Core.Shake      (ShakeExtras(ideNc, logger), mkUpdater)
 import           Development.IDE.GHC.Compat
 import qualified Development.IDE.GHC.Compat.Util as Maybes
@@ -127,19 +127,25 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
                   (libraryDir : _) -> libraryDir
                 hieDir :: FilePath
                 hieDir = pkgLibDir </> "extra-compilation-artifacts"
+                packageUnit :: Unit
+                packageUnit = mkUnit package
             modIfaces <- mapMaybeM loadModIface modules
-            traverse_ (indexModuleHieFile hieDir) modIfaces
-        indexModuleHieFile :: FilePath -> ModIface -> IO ()
-        indexModuleHieFile hieDir modIface = do
-            let hiePath :: FilePath
-                hiePath = hieDir </> toFilePath (moduleName $ mi_module modIface) ++ ".hie"
+            traverse_ (indexModuleHieFile hieDir packageUnit) modIfaces
+        indexModuleHieFile :: FilePath -> Unit -> ModIface -> IO ()
+        indexModuleHieFile hieDir packageUnit modIface = do
+            let modName :: ModuleName
+                modName = moduleName $ mi_module modIface
+                hiePath :: FilePath
+                hiePath = hieDir </> toFilePath modName ++ ".hie"
+                query :: HieDbModuleQuery
+                query = HieDbModuleQuery modName packageUnit
             hieResults <- tryAny $ loadHieFile (mkUpdater $ ideNc se) hiePath
             case hieResults of
               Left e -> Logger.logDebug (logger se) $
                   "Failed to index dependency HIE file:\n"
                   <> T.pack (show e)
               Right hie ->
-                  indexHieFile se (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash modIface) hie
+                  indexHieFile se query (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash modIface) hie
         toFilePath :: ModuleName -> FilePath
         toFilePath = separateDirectories . prettyModuleName
             where

From 352f3c23c9d9f130ec058557f03ae35bd2fb13c8 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Mon, 3 Jul 2023 10:16:46 -0500
Subject: [PATCH 13/50] Check SourceFileOrigin in GetHieAst

---
 ghcide/src/Development/IDE/Core/Rules.hs | 26 ++++++++++++++++++++----
 ghcide/src/Development/IDE/Core/Shake.hs | 11 ++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs
index 3a9a26f3d6..f9b128c8e4 100644
--- a/ghcide/src/Development/IDE/Core/Rules.hs
+++ b/ghcide/src/Development/IDE/Core/Rules.hs
@@ -569,10 +569,28 @@ reportImportCyclesRule recorder =
 
 getHieAstsRule :: Recorder (WithPriority Log) -> Rules ()
 getHieAstsRule recorder =
-    define (cmapWithPrio LogShake recorder) $ \GetHieAst f -> do
-      tmr <- use_ TypeCheck f
-      hsc <- hscEnv <$> use_ GhcSessionDeps f
-      getHieAstRuleDefinition f hsc tmr
+    define (cmapWithPrio LogShake recorder) $ \GetHieAst f ->
+        case getSourceFileOrigin f of
+            FromProject -> do
+                tmr <- use_ TypeCheck f
+                hsc <- hscEnv <$> use_ GhcSessionDeps f
+                getHieAstRuleDefinition f hsc tmr
+            FromDependency -> do
+                se <- getShakeExtras
+                mHieFile <- liftIO
+                    $ runIdeAction "GetHieAst" se
+                    $ runMaybeT
+                    $ readHieFileForSrcFromDisk recorder f
+                pure ([], makeHieAstResult <$> mHieFile)
+    where
+        makeHieAstResult :: Compat.HieFile -> HieAstResult
+        makeHieAstResult hieFile =
+            HAR
+                (Compat.hie_module hieFile)
+                (Compat.hie_asts hieFile)
+                mempty
+                mempty
+                (HieFromDisk hieFile)
 
 persistentHieFileRule :: Recorder (WithPriority Log) -> Rules ()
 persistentHieFileRule recorder = addPersistentRule GetHieAst $ \file -> runMaybeT $ do
diff --git a/ghcide/src/Development/IDE/Core/Shake.hs b/ghcide/src/Development/IDE/Core/Shake.hs
index 4ba1090087..7d6a9f57fd 100644
--- a/ghcide/src/Development/IDE/Core/Shake.hs
+++ b/ghcide/src/Development/IDE/Core/Shake.hs
@@ -44,6 +44,8 @@ module Development.IDE.Core.Shake(
     define, defineNoDiagnostics,
     defineEarlyCutoff,
     defineNoFile, defineEarlyCutOffNoFile,
+    getSourceFileOrigin,
+    SourceFileOrigin(..),
     getDiagnostics,
     mRunLspT, mRunLspTCallback,
     getHiddenDiagnostics,
@@ -107,6 +109,7 @@ import qualified Data.HashMap.Strict                    as HMap
 import           Data.HashSet                           (HashSet)
 import qualified Data.HashSet                           as HSet
 import           Data.IORef
+import           Data.List                              (isInfixOf)
 import           Data.List.Extra                        (foldl', partition,
                                                          takeEnd)
 import qualified Data.Map.Strict                        as Map
@@ -1125,6 +1128,14 @@ defineEarlyCutOffNoFile recorder f = defineEarlyCutoff recorder $ RuleNoDiagnost
     if file == emptyFilePath then do (hash, res) <- f k; return (Just hash, Just res) else
         fail $ "Rule " ++ show k ++ " should always be called with the empty string for a file"
 
+data SourceFileOrigin = FromProject | FromDependency
+
+getSourceFileOrigin :: NormalizedFilePath -> SourceFileOrigin
+getSourceFileOrigin f =
+    case isInfixOf ".hls/dependencies" (show f) of
+        True -> FromDependency
+        False -> FromProject
+
 defineEarlyCutoff'
     :: forall k v. IdeRule k v
     => (Maybe Int32 -> [FileDiagnostic] -> Action ()) -- ^ update diagnostics

From 50df9cf921a3d2ca9303b57cc520fc786894dde7 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Tue, 4 Jul 2023 12:04:54 -0500
Subject: [PATCH 14/50] Index .hls/dependencies files in lookupMod

---
 ghcide/src/Development/IDE/Core/Actions.hs | 78 +++++++++++++---------
 1 file changed, 46 insertions(+), 32 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Actions.hs b/ghcide/src/Development/IDE/Core/Actions.hs
index ce5a31996f..bd9ae8bc46 100644
--- a/ghcide/src/Development/IDE/Core/Actions.hs
+++ b/ghcide/src/Development/IDE/Core/Actions.hs
@@ -13,6 +13,10 @@ module Development.IDE.Core.Actions
 , lookupMod
 ) where
 
+import           Control.Concurrent.MVar              (MVar, newEmptyMVar, putMVar, readMVar)
+import           Control.Concurrent.STM               (atomically)
+import           Control.Concurrent.STM.TQueue        (unGetTQueue)
+import           Control.Monad                        (unless)
 import           Control.Monad.Extra                  (mapMaybeM)
 import           Control.Monad.Reader
 import           Control.Monad.Trans.Maybe
@@ -51,48 +55,58 @@ lookupMod
   -> Unit
   -> Bool -- ^ Is this file a boot file?
   -> MaybeT IdeAction Uri
-lookupMod _dbchan hieFile moduleName uid _boot = MaybeT $ do
+lookupMod HieDbWriter{indexQueue} hieFile moduleName uid _boot = MaybeT $ do
   mProjectRoot <- (resRootPath =<<) <$> asks lspEnv
   case mProjectRoot of
     Nothing -> pure Nothing
     Just projectRoot -> do
-      let toFilePath :: ModuleName -> FilePath
-          toFilePath = separateDirectories . prettyModuleName
-            where
-              separateDirectories :: FilePath -> FilePath
-              separateDirectories moduleNameString =
-                case breakOnDot moduleNameString of
-                  [] -> ""
-                  ms -> foldr1 (</>) ms
-              breakOnDot :: FilePath -> [FilePath]
-              breakOnDot = words . map replaceDotWithSpace
-              replaceDotWithSpace :: Char -> Char
-              replaceDotWithSpace '.' = ' '
-              replaceDotWithSpace c = c
-              prettyModuleName :: ModuleName -> String
-              prettyModuleName = filter (/= '"')
-                . concat
-                . drop 1
-                . words
-                . show
-          writeOutDir :: FilePath
-          writeOutDir = projectRoot </> ".hls" </> "dependencies" </> show uid
-          writeOutFile :: FilePath
-          writeOutFile = toFilePath moduleName ++ ".hs"
-          writeOutPath :: FilePath
-          writeOutPath = writeOutDir </> writeOutFile
-          moduleUri :: Uri
-          moduleUri = AtPoint.toUri writeOutPath
+      completionToken <- liftIO $ newEmptyMVar
+      moduleUri <- writeAndIndexSource projectRoot completionToken
+      liftIO $ readMVar completionToken
+      pure $ Just moduleUri
+  where
+    writeAndIndexSource :: FilePath -> MVar () -> IdeAction Uri
+    writeAndIndexSource projectRoot completionToken = do
       fileExists <- liftIO $ doesFileExist writeOutPath
-      if fileExists
-      then pure $ Just moduleUri
-      else do
+      unless fileExists $ do
         nc <- asks ideNc
         liftIO $ do
           createDirectoryIfMissing True $ takeDirectory writeOutPath
           moduleSource <- hie_hs_src <$> loadHieFile (mkUpdater nc) hieFile
           BS.writeFile writeOutPath moduleSource
-        pure $ Just moduleUri
+      liftIO $ atomically $
+        unGetTQueue indexQueue $ \withHieDb -> withHieDb $ \db -> do
+          HieDb.addSrcFile db hieFile writeOutPath False
+          putMVar completionToken ()
+      pure $ moduleUri
+      where
+        writeOutDir :: FilePath
+        writeOutDir = projectRoot </> ".hls" </> "dependencies" </> show uid
+        writeOutFile :: FilePath
+        writeOutFile = toFilePath moduleName ++ ".hs"
+        writeOutPath :: FilePath
+        writeOutPath = writeOutDir </> writeOutFile
+        moduleUri :: Uri
+        moduleUri = AtPoint.toUri writeOutPath
+        toFilePath :: ModuleName -> FilePath
+        toFilePath = separateDirectories . prettyModuleName
+          where
+            separateDirectories :: FilePath -> FilePath
+            separateDirectories moduleNameString =
+              case breakOnDot moduleNameString of
+                [] -> ""
+                ms -> foldr1 (</>) ms
+            breakOnDot :: FilePath -> [FilePath]
+            breakOnDot = words . map replaceDotWithSpace
+            replaceDotWithSpace :: Char -> Char
+            replaceDotWithSpace '.' = ' '
+            replaceDotWithSpace c = c
+            prettyModuleName :: ModuleName -> String
+            prettyModuleName = filter (/= '"')
+              . concat
+              . drop 1
+              . words
+              . show
 
 
 

From 5fa24f0800c0628bcca3497b09abccc33659d15d Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Tue, 4 Jul 2023 19:19:54 -0500
Subject: [PATCH 15/50] WIP only use GetHieAst on dependencies

---
 ghcide/src/Development/IDE/Core/OfInterest.hs | 10 ++++++++--
 ghcide/src/Development/IDE/Core/Shake.hs      | 18 +++++++++++++-----
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/OfInterest.hs b/ghcide/src/Development/IDE/Core/OfInterest.hs
index ddb919a424..b23f122262 100644
--- a/ghcide/src/Development/IDE/Core/OfInterest.hs
+++ b/ghcide/src/Development/IDE/Core/OfInterest.hs
@@ -137,16 +137,22 @@ kick = do
             mRunLspT lspEnv $
                 LSP.sendNotification (LSP.SMethod_CustomMethod msg) $
                 toJSON $ map fromNormalizedFilePath files
+        isProjectFile :: NormalizedFilePath -> Bool
+        isProjectFile file = case getSourceFileOrigin file of
+            FromProject -> True
+            FromDependency -> False
+        projectFiles :: [NormalizedFilePath]
+        projectFiles = filter isProjectFile files
 
     signal (Proxy @"kick/start")
     liftIO $ progressUpdate progress KickStarted
 
     -- Update the exports map
-    results <- uses GenerateCore files
+    results <- uses GenerateCore projectFiles
             <* uses GetHieAst files
             -- needed to have non local completions on the first edit
             -- when the first edit breaks the module header
-            <* uses NonLocalCompletions files
+            <* uses NonLocalCompletions projectFiles
     let mguts = catMaybes results
     void $ liftIO $ atomically $ modifyTVar' exportsMap (updateExportsMapMg mguts)
 
diff --git a/ghcide/src/Development/IDE/Core/Shake.hs b/ghcide/src/Development/IDE/Core/Shake.hs
index 7d6a9f57fd..1a465d8ca5 100644
--- a/ghcide/src/Development/IDE/Core/Shake.hs
+++ b/ghcide/src/Development/IDE/Core/Shake.hs
@@ -1174,11 +1174,19 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
                     Just (Succeeded ver v, _) -> Stale Nothing ver v
                     Just (Stale d ver v, _)   -> Stale d ver v
                     Just (Failed b, _)        -> Failed b
-                (bs, (diags, res)) <- actionCatch
-                    (do v <- action staleV; liftIO $ evaluate $ force v) $
-                    \(e :: SomeException) -> do
-                        pure (Nothing, ([ideErrorText file $ T.pack $ show e | not $ isBadDependency e],Nothing))
-
+                (bs, (diags, res)) <- do
+                    let doAction = actionCatch
+                            (do v <- action staleV; liftIO $ evaluate $ force v) $
+                            \(e :: SomeException) -> do
+                                pure (Nothing, ([ideErrorText file $ T.pack $ show e | not $ isBadDependency e],Nothing))
+                    case getSourceFileOrigin file of
+                        FromProject -> doAction
+                        FromDependency -> case eqT @k @GetHieAst of
+                            Just Refl -> doAction
+                            Nothing -> error $
+                                "defineEarlyCutoff': Undefined action for dependency source files\n"
+                                ++ show file ++ "\n"
+                                ++ show key
                 ver <- estimateFileVersionUnsafely key res file
                 (bs, res) <- case res of
                     Nothing -> do

From ac6a36638b0d0057988e9ca1f442b59ae0d1468c Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Wed, 5 Jul 2023 10:50:49 -0500
Subject: [PATCH 16/50] Add ReadOnly to FileOfInterestStatus

---
 ghcide/src/Development/IDE/Core/OfInterest.hs | 11 +++++-----
 ghcide/src/Development/IDE/Core/RuleTypes.hs  |  1 +
 ghcide/src/Development/IDE/Core/Rules.hs      | 20 +++++++++----------
 ghcide/src/Development/IDE/Core/Shake.hs      | 15 +++++++++++---
 .../src/Development/IDE/LSP/Notifications.hs  |  8 ++++++--
 5 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/OfInterest.hs b/ghcide/src/Development/IDE/Core/OfInterest.hs
index b23f122262..3b208a26a8 100644
--- a/ghcide/src/Development/IDE/Core/OfInterest.hs
+++ b/ghcide/src/Development/IDE/Core/OfInterest.hs
@@ -130,19 +130,18 @@ scheduleGarbageCollection state = do
 --   Could be improved
 kick :: Action ()
 kick = do
-    files <- HashMap.keys <$> getFilesOfInterestUntracked
+    filesOfInterestMap <- getFilesOfInterestUntracked
     ShakeExtras{exportsMap, ideTesting = IdeTesting testing, lspEnv, progress} <- getShakeExtras
     let signal :: KnownSymbol s => Proxy s -> Action ()
         signal msg = when testing $ liftIO $
             mRunLspT lspEnv $
                 LSP.sendNotification (LSP.SMethod_CustomMethod msg) $
                 toJSON $ map fromNormalizedFilePath files
-        isProjectFile :: NormalizedFilePath -> Bool
-        isProjectFile file = case getSourceFileOrigin file of
-            FromProject -> True
-            FromDependency -> False
+        files :: [NormalizedFilePath]
+        files = HashMap.keys filesOfInterestMap
         projectFiles :: [NormalizedFilePath]
-        projectFiles = filter isProjectFile files
+        projectFiles = HashMap.keys
+            $ HashMap.filter (/= ReadOnly) filesOfInterestMap
 
     signal (Proxy @"kick/start")
     liftIO $ progressUpdate progress KickStarted
diff --git a/ghcide/src/Development/IDE/Core/RuleTypes.hs b/ghcide/src/Development/IDE/Core/RuleTypes.hs
index a5b63463ee..529db88ed3 100644
--- a/ghcide/src/Development/IDE/Core/RuleTypes.hs
+++ b/ghcide/src/Development/IDE/Core/RuleTypes.hs
@@ -340,6 +340,7 @@ instance Hashable GetFileExists
 
 data FileOfInterestStatus
   = OnDisk
+  | ReadOnly
   | Modified { firstOpen :: !Bool -- ^ was this file just opened
              }
   deriving (Eq, Show, Typeable, Generic)
diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs
index f9b128c8e4..fcef81ebe8 100644
--- a/ghcide/src/Development/IDE/Core/Rules.hs
+++ b/ghcide/src/Development/IDE/Core/Rules.hs
@@ -569,19 +569,20 @@ reportImportCyclesRule recorder =
 
 getHieAstsRule :: Recorder (WithPriority Log) -> Rules ()
 getHieAstsRule recorder =
-    define (cmapWithPrio LogShake recorder) $ \GetHieAst f ->
-        case getSourceFileOrigin f of
-            FromProject -> do
-                tmr <- use_ TypeCheck f
-                hsc <- hscEnv <$> use_ GhcSessionDeps f
-                getHieAstRuleDefinition f hsc tmr
-            FromDependency -> do
+    define (cmapWithPrio LogShake recorder) $ \GetHieAst f -> do
+        isFoi <- use_ IsFileOfInterest f
+        case isFoi of
+            IsFOI ReadOnly -> do
                 se <- getShakeExtras
                 mHieFile <- liftIO
                     $ runIdeAction "GetHieAst" se
                     $ runMaybeT
                     $ readHieFileForSrcFromDisk recorder f
                 pure ([], makeHieAstResult <$> mHieFile)
+            _ -> do
+                tmr <- use_ TypeCheck f
+                hsc <- hscEnv <$> use_ GhcSessionDeps f
+                getHieAstRuleDefinition f isFoi hsc tmr
     where
         makeHieAstResult :: Compat.HieFile -> HieAstResult
         makeHieAstResult hieFile =
@@ -604,12 +605,11 @@ persistentHieFileRule recorder = addPersistentRule GetHieAst $ \file -> runMaybe
       del = deltaFromDiff (T.decodeUtf8 $ Compat.hie_hs_src res) currentSource
   pure (HAR (Compat.hie_module res) (Compat.hie_asts res) refmap mempty (HieFromDisk res),del,ver)
 
-getHieAstRuleDefinition :: NormalizedFilePath -> HscEnv -> TcModuleResult -> Action (IdeResult HieAstResult)
-getHieAstRuleDefinition f hsc tmr = do
+getHieAstRuleDefinition :: NormalizedFilePath -> IsFileOfInterestResult -> HscEnv -> TcModuleResult -> Action (IdeResult HieAstResult)
+getHieAstRuleDefinition f isFoi hsc tmr = do
   (diags, masts) <- liftIO $ generateHieAsts hsc tmr
   se <- getShakeExtras
 
-  isFoi <- use_ IsFileOfInterest f
   diagsWrite <- case isFoi of
     IsFOI Modified{firstOpen = False} -> do
       when (coerce $ ideTesting se) $ liftIO $ mRunLspT (lspEnv se) $
diff --git a/ghcide/src/Development/IDE/Core/Shake.hs b/ghcide/src/Development/IDE/Core/Shake.hs
index 1a465d8ca5..396d15ae96 100644
--- a/ghcide/src/Development/IDE/Core/Shake.hs
+++ b/ghcide/src/Development/IDE/Core/Shake.hs
@@ -1181,9 +1181,9 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
                                 pure (Nothing, ([ideErrorText file $ T.pack $ show e | not $ isBadDependency e],Nothing))
                     case getSourceFileOrigin file of
                         FromProject -> doAction
-                        FromDependency -> case eqT @k @GetHieAst of
-                            Just Refl -> doAction
-                            Nothing -> error $
+                        FromDependency -> if isSafeDependencyRule key
+                            then doAction
+                            else error $
                                 "defineEarlyCutoff': Undefined action for dependency source files\n"
                                 ++ show file ++ "\n"
                                 ++ show key
@@ -1229,6 +1229,15 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
         --  * creating a dependency: If everything depends on GetModificationTime, we lose early cutoff
         --  * creating bogus "file does not exists" diagnostics
         | otherwise = useWithoutDependency (GetModificationTime_ False) fp
+    isSafeDependencyRule
+        :: forall k v
+         . IdeRule k v
+        => k
+        -> Bool
+    isSafeDependencyRule _k
+        | Just Refl <- eqT @k @GetHieAst = True
+        | Just Refl <- eqT @k @IsFileOfInterest = True
+        | otherwise = False
 
 traceA :: A v -> String
 traceA (A Failed{})    = "Failed"
diff --git a/ghcide/src/Development/IDE/LSP/Notifications.hs b/ghcide/src/Development/IDE/LSP/Notifications.hs
index 80b956904d..fdb99762a6 100644
--- a/ghcide/src/Development/IDE/LSP/Notifications.hs
+++ b/ghcide/src/Development/IDE/LSP/Notifications.hs
@@ -61,10 +61,14 @@ descriptor recorder plId = (defaultPluginDescriptor plId) { pluginNotificationHa
       \ide vfs _ (DidOpenTextDocumentParams TextDocumentItem{_uri,_version}) -> liftIO $ do
       atomically $ updatePositionMapping ide (VersionedTextDocumentIdentifier _uri _version) []
       whenUriFile _uri $ \file -> do
+          let foiStatus = case getSourceFileOrigin file of
+                  FromProject -> Modified{firstOpen=True}
+                  FromDependency -> ReadOnly
           -- We don't know if the file actually exists, or if the contents match those on disk
           -- For example, vscode restores previously unsaved contents on open
-          addFileOfInterest ide file Modified{firstOpen=True}
-          setFileModified (cmapWithPrio LogFileStore recorder) (VFSModified vfs) ide False file
+          addFileOfInterest ide file foiStatus
+          unless (foiStatus == ReadOnly)
+              $ setFileModified (cmapWithPrio LogFileStore recorder) (VFSModified vfs) ide False file
           logDebug (ideLogger ide) $ "Opened text document: " <> getUri _uri
 
   , mkPluginNotificationHandler LSP.SMethod_TextDocumentDidChange $

From 4e84b99c07951eaecc39c4860d80190f3e3f6d10 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Wed, 5 Jul 2023 16:00:27 -0500
Subject: [PATCH 17/50] Prevent GetParsedModule call for dependencies

---
 ghcide/src/Development/IDE/LSP/Outline.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ghcide/src/Development/IDE/LSP/Outline.hs b/ghcide/src/Development/IDE/LSP/Outline.hs
index 64c7e14bd9..c93c9f475b 100644
--- a/ghcide/src/Development/IDE/LSP/Outline.hs
+++ b/ghcide/src/Development/IDE/LSP/Outline.hs
@@ -39,7 +39,9 @@ moduleOutline
 moduleOutline ideState DocumentSymbolParams{ _textDocument = TextDocumentIdentifier uri }
   = liftIO $ case uriToFilePath uri of
     Just (toNormalizedFilePath' -> fp) -> do
-      mb_decls <- fmap fst <$> runAction "Outline" ideState (useWithStale GetParsedModule fp)
+      mb_decls <- case getSourceFileOrigin fp of
+        FromDependency -> pure Nothing
+        FromProject -> fmap fst <$> runAction "Outline" ideState (useWithStale GetParsedModule fp)
       pure $ Right $ case mb_decls of
         Nothing -> InL []
         Just ParsedModule { pm_parsed_source = L _ltop HsModule { hsmodName, hsmodDecls, hsmodImports } }

From b0af06f79292dbbccac7d0887ca4bd6ba5eeb209 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Wed, 5 Jul 2023 16:26:22 -0500
Subject: [PATCH 18/50] Prevent GhcSession call on open dependency

---
 .../src/Development/IDE/Plugin/TypeLenses.hs  | 93 ++++++++++---------
 1 file changed, 48 insertions(+), 45 deletions(-)

diff --git a/ghcide/src/Development/IDE/Plugin/TypeLenses.hs b/ghcide/src/Development/IDE/Plugin/TypeLenses.hs
index 0056fb0f7b..71ff810e4b 100644
--- a/ghcide/src/Development/IDE/Plugin/TypeLenses.hs
+++ b/ghcide/src/Development/IDE/Plugin/TypeLenses.hs
@@ -107,52 +107,55 @@ codeLensProvider :: PluginMethodHandler IdeState Method_TextDocumentCodeLens
 codeLensProvider ideState pId CodeLensParams{_textDocument = TextDocumentIdentifier uri} = pluginResponse $ do
     mode <- liftIO $ runAction "codeLens.config" ideState $ usePropertyAction #mode pId properties
     nfp <- getNormalizedFilePath uri
-    env <- hscEnv . fst
-            <$> (handleMaybeM "Unable to get GhcSession"
-                $ liftIO
-                $ runAction "codeLens.GhcSession" ideState (useWithStale GhcSession nfp)
-                )
-    tmr <- fst <$> (
-                handleMaybeM "Unable to TypeCheck"
+    case Shake.getSourceFileOrigin nfp of
+        Shake.FromDependency -> pure $ InL []
+        Shake.FromProject -> do
+            env <- hscEnv . fst
+                    <$> (handleMaybeM "Unable to get GhcSession"
+                        $ liftIO
+                        $ runAction "codeLens.GhcSession" ideState (useWithStale GhcSession nfp)
+                        )
+            tmr <- fst <$> (
+                        handleMaybeM "Unable to TypeCheck"
+                      $ liftIO
+                      $ runAction "codeLens.TypeCheck" ideState (useWithStale TypeCheck nfp)
+                      )
+            bindings <- fst <$> (
+                        handleMaybeM "Unable to GetBindings"
+                        $ liftIO
+                        $ runAction "codeLens.GetBindings" ideState (useWithStale GetBindings nfp)
+                        )
+            (gblSigs@(GlobalBindingTypeSigsResult gblSigs'), gblSigsMp) <-
+              handleMaybeM "Unable to GetGlobalBindingTypeSigs"
               $ liftIO
-              $ runAction "codeLens.TypeCheck" ideState (useWithStale TypeCheck nfp)
-              )
-    bindings <- fst <$> (
-                handleMaybeM "Unable to GetBindings"
-                $ liftIO
-                $ runAction "codeLens.GetBindings" ideState (useWithStale GetBindings nfp)
-                )
-    (gblSigs@(GlobalBindingTypeSigsResult gblSigs'), gblSigsMp) <-
-      handleMaybeM "Unable to GetGlobalBindingTypeSigs"
-      $ liftIO
-      $ runAction "codeLens.GetGlobalBindingTypeSigs" ideState (useWithStale GetGlobalBindingTypeSigs nfp)
-
-    diag <- liftIO $ atomically $ getDiagnostics ideState
-    hDiag <- liftIO $ atomically $ getHiddenDiagnostics ideState
-
-    let toWorkSpaceEdit tedit = WorkspaceEdit (Just $ Map.singleton uri $ tedit) Nothing Nothing
-        generateLensForGlobal mp sig@GlobalBindingTypeSig{gbRendered} = do
-            range <- toCurrentRange mp =<< srcSpanToRange (gbSrcSpan sig)
-            tedit <- gblBindingTypeSigToEdit sig (Just gblSigsMp)
-            let wedit = toWorkSpaceEdit [tedit]
-            pure $ generateLens pId range (T.pack gbRendered) wedit
-        generateLensFromDiags f =
-              [ generateLens pId _range title edit
-              | (dFile, _, dDiag@Diagnostic{_range = _range}) <- diag ++ hDiag
-              , dFile == nfp
-              , (title, tedit) <- f dDiag
-              , let edit = toWorkSpaceEdit tedit
-              ]
-    -- `suggestLocalSignature` relies on diagnostic, if diagnostics don't have the local signature warning,
-    -- the `bindings` is useless, and if diagnostic has, that means we parsed success, and the `bindings` is fresh.
-    pure $ InL $ case mode of
-        Always ->
-          mapMaybe (generateLensForGlobal gblSigsMp) gblSigs'
-            <> generateLensFromDiags
-                (suggestLocalSignature False (Just env) (Just tmr) (Just bindings)) -- we still need diagnostics for local bindings
-        Exported -> mapMaybe (generateLensForGlobal gblSigsMp) (filter gbExported gblSigs')
-        Diagnostics -> generateLensFromDiags
-            $ suggestSignature False (Just env) (Just gblSigs) (Just tmr) (Just bindings)
+              $ runAction "codeLens.GetGlobalBindingTypeSigs" ideState (useWithStale GetGlobalBindingTypeSigs nfp)
+
+            diag <- liftIO $ atomically $ getDiagnostics ideState
+            hDiag <- liftIO $ atomically $ getHiddenDiagnostics ideState
+
+            let toWorkSpaceEdit tedit = WorkspaceEdit (Just $ Map.singleton uri $ tedit) Nothing Nothing
+                generateLensForGlobal mp sig@GlobalBindingTypeSig{gbRendered} = do
+                    range <- toCurrentRange mp =<< srcSpanToRange (gbSrcSpan sig)
+                    tedit <- gblBindingTypeSigToEdit sig (Just gblSigsMp)
+                    let wedit = toWorkSpaceEdit [tedit]
+                    pure $ generateLens pId range (T.pack gbRendered) wedit
+                generateLensFromDiags f =
+                      [ generateLens pId _range title edit
+                      | (dFile, _, dDiag@Diagnostic{_range = _range}) <- diag ++ hDiag
+                      , dFile == nfp
+                      , (title, tedit) <- f dDiag
+                      , let edit = toWorkSpaceEdit tedit
+                      ]
+            -- `suggestLocalSignature` relies on diagnostic, if diagnostics don't have the local signature warning,
+            -- the `bindings` is useless, and if diagnostic has, that means we parsed success, and the `bindings` is fresh.
+            pure $ InL $ case mode of
+                Always ->
+                  mapMaybe (generateLensForGlobal gblSigsMp) gblSigs'
+                    <> generateLensFromDiags
+                        (suggestLocalSignature False (Just env) (Just tmr) (Just bindings)) -- we still need diagnostics for local bindings
+                Exported -> mapMaybe (generateLensForGlobal gblSigsMp) (filter gbExported gblSigs')
+                Diagnostics -> generateLensFromDiags
+                    $ suggestSignature False (Just env) (Just gblSigs) (Just tmr) (Just bindings)
 
 generateLens :: PluginId -> Range -> T.Text -> WorkspaceEdit -> CodeLens
 generateLens pId _range title edit =

From 33e03d2ceb171ba19624746a07ffb984b5cb4d23 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 6 Jul 2023 07:10:33 -0500
Subject: [PATCH 19/50] Make hover work in dependency files

---
 ghcide/src/Development/IDE/Core/Actions.hs  | 10 ++++--
 ghcide/src/Development/IDE/Core/Rules.hs    | 14 ++++-----
 ghcide/src/Development/IDE/Spans/AtPoint.hs | 35 ++++++++++++++-------
 3 files changed, 37 insertions(+), 22 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Actions.hs b/ghcide/src/Development/IDE/Core/Actions.hs
index bd9ae8bc46..f4a2983aa5 100644
--- a/ghcide/src/Development/IDE/Core/Actions.hs
+++ b/ghcide/src/Development/IDE/Core/Actions.hs
@@ -124,11 +124,15 @@ getAtPoint file pos = runMaybeT $ do
   opts <- liftIO $ getIdeOptionsIO ide
 
   (hf, mapping) <- useE GetHieAst file
-  env <- hscEnv . fst <$> useE GhcSession file
-  dkMap <- lift $ maybe (DKMap mempty mempty) fst <$> runMaybeT (useE GetDocMap file)
+  (mEnv, mDkMap) <- case getSourceFileOrigin file of
+    FromDependency -> pure (Nothing, Nothing)
+    FromProject -> do
+      env <- hscEnv . fst <$> useE GhcSession file
+      dkMap <- lift $ maybe (DKMap mempty mempty) fst <$> runMaybeT (useE GetDocMap file)
+      pure (Just env, Just dkMap)
 
   !pos' <- MaybeT (return $ fromCurrentPosition mapping pos)
-  MaybeT $ pure $ first (toCurrentRange mapping =<<) <$> AtPoint.atPoint opts hf dkMap env pos'
+  MaybeT $ pure $ first (toCurrentRange mapping =<<) <$> AtPoint.atPoint opts hf mDkMap mEnv pos'
 
 -- | For each Loacation, determine if we have the PositionMapping
 -- for the correct file. If not, get the correct position mapping
diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs
index fcef81ebe8..d4460e92c3 100644
--- a/ghcide/src/Development/IDE/Core/Rules.hs
+++ b/ghcide/src/Development/IDE/Core/Rules.hs
@@ -570,19 +570,18 @@ reportImportCyclesRule recorder =
 getHieAstsRule :: Recorder (WithPriority Log) -> Rules ()
 getHieAstsRule recorder =
     define (cmapWithPrio LogShake recorder) $ \GetHieAst f -> do
-        isFoi <- use_ IsFileOfInterest f
-        case isFoi of
-            IsFOI ReadOnly -> do
+        case getSourceFileOrigin f of
+            FromDependency -> do
                 se <- getShakeExtras
                 mHieFile <- liftIO
                     $ runIdeAction "GetHieAst" se
                     $ runMaybeT
                     $ readHieFileForSrcFromDisk recorder f
                 pure ([], makeHieAstResult <$> mHieFile)
-            _ -> do
+            FromProject -> do
                 tmr <- use_ TypeCheck f
                 hsc <- hscEnv <$> use_ GhcSessionDeps f
-                getHieAstRuleDefinition f isFoi hsc tmr
+                getHieAstRuleDefinition f hsc tmr
     where
         makeHieAstResult :: Compat.HieFile -> HieAstResult
         makeHieAstResult hieFile =
@@ -605,11 +604,12 @@ persistentHieFileRule recorder = addPersistentRule GetHieAst $ \file -> runMaybe
       del = deltaFromDiff (T.decodeUtf8 $ Compat.hie_hs_src res) currentSource
   pure (HAR (Compat.hie_module res) (Compat.hie_asts res) refmap mempty (HieFromDisk res),del,ver)
 
-getHieAstRuleDefinition :: NormalizedFilePath -> IsFileOfInterestResult -> HscEnv -> TcModuleResult -> Action (IdeResult HieAstResult)
-getHieAstRuleDefinition f isFoi hsc tmr = do
+getHieAstRuleDefinition :: NormalizedFilePath -> HscEnv -> TcModuleResult -> Action (IdeResult HieAstResult)
+getHieAstRuleDefinition f hsc tmr = do
   (diags, masts) <- liftIO $ generateHieAsts hsc tmr
   se <- getShakeExtras
 
+  isFoi <- use_ IsFileOfInterest f
   diagsWrite <- case isFoi of
     IsFOI Modified{firstOpen = False} -> do
       when (coerce $ ideTesting se) $ liftIO $ mRunLspT (lspEnv se) $
diff --git a/ghcide/src/Development/IDE/Spans/AtPoint.hs b/ghcide/src/Development/IDE/Spans/AtPoint.hs
index 90b20c8646..9ebb0112bf 100644
--- a/ghcide/src/Development/IDE/Spans/AtPoint.hs
+++ b/ghcide/src/Development/IDE/Spans/AtPoint.hs
@@ -211,11 +211,11 @@ gotoDefinition withHieDb getHieFile ideOpts imports srcSpans pos
 atPoint
   :: IdeOptions
   -> HieAstResult
-  -> DocAndKindMap
-  -> HscEnv
+  -> Maybe DocAndKindMap
+  -> Maybe HscEnv
   -> Position
   -> Maybe (Maybe Range, [T.Text])
-atPoint IdeOptions{} (HAR _ hf _ _ kind) (DKMap dm km) env pos = listToMaybe $ pointCommand hf pos hoverInfo
+atPoint IdeOptions{} (HAR _ hf _ _ kind) mDkMap mEnv pos = listToMaybe $ pointCommand hf pos hoverInfo
   where
     -- Hover info for values/data
     hoverInfo ast = (Just range, prettyNames ++ pTypes)
@@ -245,22 +245,33 @@ atPoint IdeOptions{} (HAR _ hf _ _ kind) (DKMap dm km) env pos = listToMaybe $ p
         prettyName (Right n, dets) = T.unlines $
           wrapHaskell (printOutputable n <> maybe "" (" :: " <>) ((prettyType <$> identType dets) <|> maybeKind))
           : maybeToList (pretty (definedAt n) (prettyPackageName n))
-          ++ catMaybes [ T.unlines . spanDocToMarkdown <$> lookupNameEnv dm n
+          ++ catMaybes [ T.unlines . spanDocToMarkdown <$> maybeDoc
                        ]
-          where maybeKind = fmap printOutputable $ safeTyThingType =<< lookupNameEnv km n
+          where maybeKind = do
+                  (DKMap _ km) <- mDkMap
+                  nameEnv <- lookupNameEnv km n
+                  printOutputable <$> safeTyThingType nameEnv
+                maybeDoc = do
+                  (DKMap dm _) <- mDkMap
+                  lookupNameEnv dm n
                 pretty Nothing Nothing = Nothing
                 pretty (Just define) Nothing = Just $ define <> "\n"
                 pretty Nothing (Just pkgName) = Just $ pkgName <> "\n"
                 pretty (Just define) (Just pkgName) = Just $ define <> " " <> pkgName <> "\n"
         prettyName (Left m,_) = printOutputable m
 
-        prettyPackageName n = do
-          m <- nameModule_maybe n
-          let pid = moduleUnit m
-          conf <- lookupUnit env pid
-          let pkgName = T.pack $ unitPackageNameString conf
-              version = T.pack $ showVersion (unitPackageVersion conf)
-          pure $ "*(" <> pkgName <> "-" <> version <> ")*"
+        prettyPackageName n = case mEnv of
+          Just env -> do
+            pid <- getUnit n
+            conf <- lookupUnit env pid
+            let pkgName = T.pack $ unitPackageNameString conf
+                version = T.pack $ showVersion (unitPackageVersion conf)
+            pure $ "*(" <> pkgName <> "-" <> version <> ")*"
+          Nothing -> do
+            u <- getUnit n
+            let pkgStr = takeWhile (/= ':') $ show $ toUnitId u
+            pure $ "*(" <> T.pack pkgStr <> ")*"
+        getUnit n = moduleUnit <$> nameModule_maybe n
 
         prettyTypes = map (("_ :: "<>) . prettyType) types
         prettyType t = case kind of

From e896d7713be7c9786525ee84bdabc61ded942c38 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 6 Jul 2023 08:03:29 -0500
Subject: [PATCH 20/50] Remove gotoDefinition polymorphism

---
 ghcide/src/Development/IDE/Spans/AtPoint.hs | 35 +++++++++------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/ghcide/src/Development/IDE/Spans/AtPoint.hs b/ghcide/src/Development/IDE/Spans/AtPoint.hs
index 9ebb0112bf..c36150a91a 100644
--- a/ghcide/src/Development/IDE/Spans/AtPoint.hs
+++ b/ghcide/src/Development/IDE/Spans/AtPoint.hs
@@ -34,6 +34,7 @@ import           Language.LSP.Protocol.Types          hiding
 -- compiler and infrastructure
 import           Development.IDE.Core.PositionMapping
 import           Development.IDE.Core.RuleTypes
+import           Development.IDE.Core.Shake           (IdeAction)
 import           Development.IDE.GHC.Compat
 import qualified Development.IDE.GHC.Compat.Util      as Util
 import           Development.IDE.GHC.Util             (printOutputable)
@@ -184,26 +185,24 @@ documentHighlight hf rf pos = pure highlights
         else DocumentHighlightKind_Read
 
 gotoTypeDefinition
-  :: MonadIO m
-  => WithHieDb
-  -> LookupModule m
+  :: WithHieDb
+  -> LookupModule IdeAction
   -> IdeOptions
   -> HieAstResult
   -> Position
-  -> MaybeT m [Location]
+  -> MaybeT IdeAction [Location]
 gotoTypeDefinition withHieDb lookupModule ideOpts srcSpans pos
   = lift $ typeLocationsAtPoint withHieDb lookupModule ideOpts pos srcSpans
 
 -- | Locate the definition of the name at a given position.
 gotoDefinition
-  :: MonadIO m
-  => WithHieDb
-  -> LookupModule m
+  :: WithHieDb
+  -> LookupModule IdeAction
   -> IdeOptions
   -> M.Map ModuleName NormalizedFilePath
   -> HieASTs a
   -> Position
-  -> MaybeT m [Location]
+  -> MaybeT IdeAction [Location]
 gotoDefinition withHieDb getHieFile ideOpts imports srcSpans pos
   = lift $ locationsAtPoint withHieDb getHieFile ideOpts imports pos srcSpans
 
@@ -286,14 +285,12 @@ atPoint IdeOptions{} (HAR _ hf _ _ kind) mDkMap mEnv pos = listToMaybe $ pointCo
             _ -> Just $ "*Defined " <> printOutputable (pprNameDefnLoc name) <> "*"
 
 typeLocationsAtPoint
-  :: forall m
-   . MonadIO m
-  => WithHieDb
-  -> LookupModule m
+  :: WithHieDb
+  -> LookupModule IdeAction
   -> IdeOptions
   -> Position
   -> HieAstResult
-  -> m [Location]
+  -> IdeAction [Location]
 typeLocationsAtPoint withHieDb lookupModule _ideOptions pos (HAR _ ast _ _ hieKind) =
   case hieKind of
     HieFromDisk hf ->
@@ -336,15 +333,13 @@ getTypes :: [Type] -> [Name]
 getTypes ts = concatMap namesInType ts
 
 locationsAtPoint
-  :: forall m a
-   . MonadIO m
-  => WithHieDb
-  -> LookupModule m
+  :: WithHieDb
+  -> LookupModule IdeAction
   -> IdeOptions
   -> M.Map ModuleName NormalizedFilePath
   -> Position
   -> HieASTs a
-  -> m [Location]
+  -> IdeAction [Location]
 locationsAtPoint withHieDb lookupModule _ideOptions imports pos ast =
   let ns = concat $ pointCommand ast pos (M.keys . getNodeIds)
       zeroPos = Position 0 0
@@ -353,7 +348,7 @@ locationsAtPoint withHieDb lookupModule _ideOptions imports pos ast =
     in fmap (nubOrd . concat) $ mapMaybeM (either (pure . modToLocation) $ nameToLocation withHieDb lookupModule) ns
 
 -- | Given a 'Name' attempt to find the location where it is defined.
-nameToLocation :: MonadIO m => WithHieDb -> LookupModule m -> Name -> m (Maybe [Location])
+nameToLocation :: WithHieDb -> LookupModule IdeAction -> Name -> IdeAction (Maybe [Location])
 nameToLocation withHieDb lookupModule name = runMaybeT $
   case nameSrcSpan name of
     sp@(RealSrcSpan rsp _)
@@ -389,7 +384,7 @@ nameToLocation withHieDb lookupModule name = runMaybeT $
             xs -> lift $ mapMaybeM (runMaybeT . defRowToLocation lookupModule) xs
         xs -> lift $ mapMaybeM (runMaybeT . defRowToLocation lookupModule) xs
 
-defRowToLocation :: Monad m => LookupModule m -> Res DefRow -> MaybeT m Location
+defRowToLocation :: LookupModule IdeAction -> Res DefRow -> MaybeT IdeAction Location
 defRowToLocation lookupModule (row:.info) = do
   let start = Position (fromIntegral $ defSLine row - 1) (fromIntegral $ defSCol row - 1)
       end   = Position (fromIntegral $ defELine row - 1) (fromIntegral $ defECol row - 1)

From 3c5aaf26b1ad53d98f92b15d6e4fab9e4509af81 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Fri, 7 Jul 2023 11:25:56 -0500
Subject: [PATCH 21/50] Always add dependency files of interest ReadOnly

---
 ghcide/src/Development/IDE/Core/OfInterest.hs   |  1 +
 ghcide/src/Development/IDE/LSP/Notifications.hs | 16 ++++++++++++----
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/OfInterest.hs b/ghcide/src/Development/IDE/Core/OfInterest.hs
index 3b208a26a8..4af0d41108 100644
--- a/ghcide/src/Development/IDE/Core/OfInterest.hs
+++ b/ghcide/src/Development/IDE/Core/OfInterest.hs
@@ -78,6 +78,7 @@ ofInterestRules recorder = do
     summarize (IsFOI OnDisk)           = BS.singleton 1
     summarize (IsFOI (Modified False)) = BS.singleton 2
     summarize (IsFOI (Modified True))  = BS.singleton 3
+    summarize (IsFOI ReadOnly)         = BS.singleton 4
 
 ------------------------------------------------------------
 newtype GarbageCollectVar = GarbageCollectVar (Var Bool)
diff --git a/ghcide/src/Development/IDE/LSP/Notifications.hs b/ghcide/src/Development/IDE/LSP/Notifications.hs
index fdb99762a6..db6b909735 100644
--- a/ghcide/src/Development/IDE/LSP/Notifications.hs
+++ b/ghcide/src/Development/IDE/LSP/Notifications.hs
@@ -75,15 +75,23 @@ descriptor recorder plId = (defaultPluginDescriptor plId) { pluginNotificationHa
       \ide vfs _ (DidChangeTextDocumentParams identifier@VersionedTextDocumentIdentifier{_uri} changes) -> liftIO $ do
         atomically $ updatePositionMapping ide identifier changes
         whenUriFile _uri $ \file -> do
-          addFileOfInterest ide file Modified{firstOpen=False}
-          setFileModified (cmapWithPrio LogFileStore recorder) (VFSModified vfs) ide False file
+          let foiStatus = case getSourceFileOrigin file of
+                  FromProject -> Modified{firstOpen=True}
+                  FromDependency -> ReadOnly
+          addFileOfInterest ide file foiStatus
+          unless (foiStatus == ReadOnly)
+              $ setFileModified (cmapWithPrio LogFileStore recorder) (VFSModified vfs) ide False file
         logDebug (ideLogger ide) $ "Modified text document: " <> getUri _uri
 
   , mkPluginNotificationHandler LSP.SMethod_TextDocumentDidSave $
       \ide vfs _ (DidSaveTextDocumentParams TextDocumentIdentifier{_uri} _) -> liftIO $ do
         whenUriFile _uri $ \file -> do
-            addFileOfInterest ide file OnDisk
-            setFileModified (cmapWithPrio LogFileStore recorder) (VFSModified vfs) ide True file
+            let foiStatus = case getSourceFileOrigin file of
+                    FromProject -> OnDisk
+                    FromDependency -> ReadOnly
+            addFileOfInterest ide file foiStatus
+            unless (foiStatus == ReadOnly)
+                $ setFileModified (cmapWithPrio LogFileStore recorder) (VFSModified vfs) ide True file
         logDebug (ideLogger ide) $ "Saved text document: " <> getUri _uri
 
   , mkPluginNotificationHandler LSP.SMethod_TextDocumentDidClose $

From ef19f2cb6ad01c363c9d86451defeea83227d1ba Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Fri, 7 Jul 2023 21:52:47 -0500
Subject: [PATCH 22/50] Prevent GetModificationTime in dependency file

---
 ghcide/src/Development/IDE/Core/Shake.hs | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Shake.hs b/ghcide/src/Development/IDE/Core/Shake.hs
index 396d15ae96..7930ac6e28 100644
--- a/ghcide/src/Development/IDE/Core/Shake.hs
+++ b/ghcide/src/Development/IDE/Core/Shake.hs
@@ -1158,7 +1158,7 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
                     -- No changes in the dependencies and we have
                     -- an existing successful result.
                     Just (v@(Succeeded _ x), diags) -> do
-                        ver <- estimateFileVersionUnsafely key (Just x) file
+                        ver <- estimateFileVersionUnsafely key FromProject (Just x) file
                         doDiagnostics (vfsVersion =<< ver) $ Vector.toList diags
                         return $ Just $ RunResult ChangedNothing old $ A v
                     _ -> return Nothing
@@ -1174,20 +1174,20 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
                     Just (Succeeded ver v, _) -> Stale Nothing ver v
                     Just (Stale d ver v, _)   -> Stale d ver v
                     Just (Failed b, _)        -> Failed b
-                (bs, (diags, res)) <- do
+                (fileOrigin, (bs, (diags, res))) <- do
                     let doAction = actionCatch
                             (do v <- action staleV; liftIO $ evaluate $ force v) $
                             \(e :: SomeException) -> do
                                 pure (Nothing, ([ideErrorText file $ T.pack $ show e | not $ isBadDependency e],Nothing))
                     case getSourceFileOrigin file of
-                        FromProject -> doAction
+                        FromProject -> (\r -> (FromProject, r)) <$> doAction
                         FromDependency -> if isSafeDependencyRule key
-                            then doAction
+                            then (\r -> (FromDependency, r)) <$> doAction
                             else error $
                                 "defineEarlyCutoff': Undefined action for dependency source files\n"
                                 ++ show file ++ "\n"
                                 ++ show key
-                ver <- estimateFileVersionUnsafely key res file
+                ver <- estimateFileVersionUnsafely key fileOrigin res file
                 (bs, res) <- case res of
                     Nothing -> do
                         pure (toShakeValue ShakeStale bs, staleV)
@@ -1214,10 +1214,11 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
         :: forall k v
          . IdeRule k v
         => k
+        -> SourceFileOrigin
         -> Maybe v
         -> NormalizedFilePath
         -> Action (Maybe FileVersion)
-    estimateFileVersionUnsafely _k v fp
+    estimateFileVersionUnsafely _k fileOrigin v fp
         | fp == emptyFilePath = pure Nothing
         | Just Refl <- eqT @k @GetModificationTime = pure v
         -- GetModificationTime depends on these rules, so avoid creating a cycle
@@ -1228,7 +1229,10 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
         -- For all other rules - compute the version properly without:
         --  * creating a dependency: If everything depends on GetModificationTime, we lose early cutoff
         --  * creating bogus "file does not exists" diagnostics
-        | otherwise = useWithoutDependency (GetModificationTime_ False) fp
+        | otherwise =
+             case fileOrigin of
+                 FromDependency -> pure Nothing
+                 FromProject -> useWithoutDependency (GetModificationTime_ False) fp
     isSafeDependencyRule
         :: forall k v
          . IdeRule k v

From 102722903188cad248383fce623f75e22dc6a264 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Mon, 10 Jul 2023 06:10:58 -0500
Subject: [PATCH 23/50] Check that hiedb source files exist

---
 ghcide/src/Development/IDE/Spans/AtPoint.hs | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/ghcide/src/Development/IDE/Spans/AtPoint.hs b/ghcide/src/Development/IDE/Spans/AtPoint.hs
index c36150a91a..b07626227a 100644
--- a/ghcide/src/Development/IDE/Spans/AtPoint.hs
+++ b/ghcide/src/Development/IDE/Spans/AtPoint.hs
@@ -389,9 +389,19 @@ defRowToLocation lookupModule (row:.info) = do
   let start = Position (fromIntegral $ defSLine row - 1) (fromIntegral $ defSCol row - 1)
       end   = Position (fromIntegral $ defELine row - 1) (fromIntegral $ defECol row - 1)
       range = Range start end
+      lookupMod = lookupModule (defSrc row) (modInfoName info) (modInfoUnit info) (modInfoIsBoot info)
   file <- case modInfoSrcFile info of
-    Just src -> pure $ toUri src
-    Nothing -> lookupModule (defSrc row) (modInfoName info) (modInfoUnit info) (modInfoIsBoot info)
+    Just src -> do
+      -- Checking that the file exists covers the case where a
+      -- dependency file in .hls is in the database but got deleted
+      -- for any reason.
+      -- See the function `lookupMod` in Development.IDE.Core.Actions
+      -- for where dependency files get created and indexed in hiedb.
+      fileExists <- liftIO $ doesFileExist src
+      if fileExists
+      then pure $ toUri src
+      else lookupMod
+    Nothing -> lookupMod
   pure $ Location file range
 
 toUri :: FilePath -> Uri

From 9866f19166fa6930264a160932e965c454fed05e Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Mon, 10 Jul 2023 09:45:54 -0500
Subject: [PATCH 24/50] Revert "Prevent GetModificationTime in dependency file"

This reverts commit e4fd5f7dd916aa0b706d8fdd62a3d21ea1785eda.
---
 ghcide/src/Development/IDE/Core/Shake.hs | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Shake.hs b/ghcide/src/Development/IDE/Core/Shake.hs
index 7930ac6e28..396d15ae96 100644
--- a/ghcide/src/Development/IDE/Core/Shake.hs
+++ b/ghcide/src/Development/IDE/Core/Shake.hs
@@ -1158,7 +1158,7 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
                     -- No changes in the dependencies and we have
                     -- an existing successful result.
                     Just (v@(Succeeded _ x), diags) -> do
-                        ver <- estimateFileVersionUnsafely key FromProject (Just x) file
+                        ver <- estimateFileVersionUnsafely key (Just x) file
                         doDiagnostics (vfsVersion =<< ver) $ Vector.toList diags
                         return $ Just $ RunResult ChangedNothing old $ A v
                     _ -> return Nothing
@@ -1174,20 +1174,20 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
                     Just (Succeeded ver v, _) -> Stale Nothing ver v
                     Just (Stale d ver v, _)   -> Stale d ver v
                     Just (Failed b, _)        -> Failed b
-                (fileOrigin, (bs, (diags, res))) <- do
+                (bs, (diags, res)) <- do
                     let doAction = actionCatch
                             (do v <- action staleV; liftIO $ evaluate $ force v) $
                             \(e :: SomeException) -> do
                                 pure (Nothing, ([ideErrorText file $ T.pack $ show e | not $ isBadDependency e],Nothing))
                     case getSourceFileOrigin file of
-                        FromProject -> (\r -> (FromProject, r)) <$> doAction
+                        FromProject -> doAction
                         FromDependency -> if isSafeDependencyRule key
-                            then (\r -> (FromDependency, r)) <$> doAction
+                            then doAction
                             else error $
                                 "defineEarlyCutoff': Undefined action for dependency source files\n"
                                 ++ show file ++ "\n"
                                 ++ show key
-                ver <- estimateFileVersionUnsafely key fileOrigin res file
+                ver <- estimateFileVersionUnsafely key res file
                 (bs, res) <- case res of
                     Nothing -> do
                         pure (toShakeValue ShakeStale bs, staleV)
@@ -1214,11 +1214,10 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
         :: forall k v
          . IdeRule k v
         => k
-        -> SourceFileOrigin
         -> Maybe v
         -> NormalizedFilePath
         -> Action (Maybe FileVersion)
-    estimateFileVersionUnsafely _k fileOrigin v fp
+    estimateFileVersionUnsafely _k v fp
         | fp == emptyFilePath = pure Nothing
         | Just Refl <- eqT @k @GetModificationTime = pure v
         -- GetModificationTime depends on these rules, so avoid creating a cycle
@@ -1229,10 +1228,7 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
         -- For all other rules - compute the version properly without:
         --  * creating a dependency: If everything depends on GetModificationTime, we lose early cutoff
         --  * creating bogus "file does not exists" diagnostics
-        | otherwise =
-             case fileOrigin of
-                 FromDependency -> pure Nothing
-                 FromProject -> useWithoutDependency (GetModificationTime_ False) fp
+        | otherwise = useWithoutDependency (GetModificationTime_ False) fp
     isSafeDependencyRule
         :: forall k v
          . IdeRule k v

From 556982bead912840e180f6be0dd3625f13109faa Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Mon, 10 Jul 2023 09:53:13 -0500
Subject: [PATCH 25/50] Whitelist GetModificationTime for dependencies

---
 ghcide/src/Development/IDE/Core/Shake.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ghcide/src/Development/IDE/Core/Shake.hs b/ghcide/src/Development/IDE/Core/Shake.hs
index 396d15ae96..9a90df4eb3 100644
--- a/ghcide/src/Development/IDE/Core/Shake.hs
+++ b/ghcide/src/Development/IDE/Core/Shake.hs
@@ -1237,6 +1237,7 @@ defineEarlyCutoff' doDiagnostics cmp key file old mode action = do
     isSafeDependencyRule _k
         | Just Refl <- eqT @k @GetHieAst = True
         | Just Refl <- eqT @k @IsFileOfInterest = True
+        | Just Refl <- eqT @k @GetModificationTime = True
         | otherwise = False
 
 traceA :: A v -> String

From 13b19643db91546dd46330188a5f64ba498c5a73 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Tue, 11 Jul 2023 11:46:49 -0500
Subject: [PATCH 26/50] Unindex dependency srcs if .hls is missing

---
 cabal.project                                |  5 ++++
 ghcide/ghcide.cabal                          |  2 +-
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 29 ++++++++++++++++----
 3 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/cabal.project b/cabal.project
index 2dc12eb573..2c750a4edd 100644
--- a/cabal.project
+++ b/cabal.project
@@ -38,6 +38,11 @@ packages:
          ./plugins/hls-refactor-plugin
          ./plugins/hls-overloaded-record-dot-plugin
 
+source-repository-package
+  type:git
+  location: https://github.com/nlander/HieDb.git
+  tag: 038bc785c80f13615f4ac1ec5066345f2eb999c2
+
 -- Standard location for temporary packages needed for particular environments
 -- For example it is used in the project gitlab mirror to help in the MAcOS M1 build script
 -- See https://github.com/haskell/haskell-language-server/blob/master/.gitlab-ci.yml
diff --git a/ghcide/ghcide.cabal b/ghcide/ghcide.cabal
index 9ba17e756a..253187fa0e 100644
--- a/ghcide/ghcide.cabal
+++ b/ghcide/ghcide.cabal
@@ -68,7 +68,7 @@ library
         hls-plugin-api == 2.1.0.0,
         lens,
         list-t,
-        hiedb == 0.4.3.*,
+        hiedb,
         lsp-types ^>= 2.0.0.1,
         lsp ^>= 2.0.0.0 ,
         mtl,
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index f9ec469429..d293b7ab11 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -13,10 +13,14 @@ module Development.IDE.Types.HscEnvEq
 
 
 import           Control.Concurrent.Async        (Async, async, waitCatch)
+import           Control.Concurrent.MVar         (newEmptyMVar, putMVar, readMVar)
+import           Control.Concurrent.STM          (atomically)
+import           Control.Concurrent.STM.TQueue   (unGetTQueue)
 import           Control.Concurrent.Strict       (modifyVar, newVar)
 import           Control.DeepSeq                 (force)
 import           Control.Exception               (evaluate, mask, throwIO)
 import           Control.Exception.Safe          (tryAny)
+import           Control.Monad                   (unless)
 import           Control.Monad.Extra             (eitherM, join, mapMaybeM, void)
 import           Data.Either                     (fromRight)
 import           Data.Foldable                   (traverse_)
@@ -27,7 +31,7 @@ import qualified Data.Text                       as T
 import           Data.Unique                     (Unique)
 import qualified Data.Unique                     as Unique
 import           Development.IDE.Core.Compile    (HieDbModuleQuery(HieDbModuleQuery), indexHieFile, loadHieFile)
-import           Development.IDE.Core.Shake      (ShakeExtras(ideNc, logger), mkUpdater)
+import           Development.IDE.Core.Shake      (HieDbWriter(indexQueue), ShakeExtras(hiedbWriter, ideNc, logger, lspEnv), mkUpdater)
 import           Development.IDE.GHC.Compat
 import qualified Development.IDE.GHC.Compat.Util as Maybes
 import           Development.IDE.GHC.Error       (catchSrcErrors)
@@ -36,9 +40,10 @@ import           Development.IDE.Graph.Classes
 import           Development.IDE.Types.Exports   (ExportsMap, createExportsMap)
 import           Development.IDE.Types.Location  (toNormalizedFilePath')
 import qualified Development.IDE.Types.Logger    as Logger
-import           HieDb                           (SourceFile(FakeFile))
+import           HieDb                           (SourceFile(FakeFile), removeDependencySrcFiles)
+import           Language.LSP.Server             (resRootPath)
 import           OpenTelemetry.Eventlog          (withSpan)
-import           System.Directory                (makeAbsolute)
+import           System.Directory                (doesDirectoryExist, makeAbsolute)
 import           System.FilePath
 
 -- | An 'HscEnv' with equality. Two values are considered equal
@@ -117,8 +122,22 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
     return HscEnvEq{..}
     where
         indexDependencyHieFiles :: IO ()
-        indexDependencyHieFiles = void
-            $ Map.traverseWithKey indexPackageHieFiles packagesWithModules
+        indexDependencyHieFiles = do
+            dotHlsDirExists <- maybe (pure False) doesDirectoryExist mHlsDir
+            unless dotHlsDirExists deleteMissingDependencySources
+            void $ Map.traverseWithKey indexPackageHieFiles packagesWithModules
+        mHlsDir :: Maybe FilePath
+        mHlsDir = do
+            projectDir <- resRootPath =<< lspEnv se
+            pure $ projectDir </> ".hls"
+        deleteMissingDependencySources :: IO ()
+        deleteMissingDependencySources = do
+            completionToken <- newEmptyMVar
+            atomically $ unGetTQueue (indexQueue $ hiedbWriter se) $
+                \withHieDb -> withHieDb $ \db -> do
+                    removeDependencySrcFiles db
+                    putMVar completionToken ()
+            readMVar completionToken
         indexPackageHieFiles :: Package -> [Module] -> IO ()
         indexPackageHieFiles (Package package) modules = do
             let pkgLibDir :: FilePath

From deebf4039c79bf656a3cea4e668b55667ae92023 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Wed, 12 Jul 2023 07:39:49 -0500
Subject: [PATCH 27/50] Revert "Check if dependency HIE files already indexed"

This reverts commit 096c52b80633a91c83e1239917c3326bbd7fafbe.
---
 ghcide/src/Development/IDE/Core/Compile.hs    | 31 ++++---------------
 ghcide/src/Development/IDE/Core/Rules.hs      |  2 +-
 .../src/Development/IDE/GHC/Compat/Units.hs   |  2 +-
 ghcide/src/Development/IDE/Types/HscEnvEq.hs  | 20 +++++-------
 4 files changed, 15 insertions(+), 40 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Compile.hs b/ghcide/src/Development/IDE/Core/Compile.hs
index b111816679..08294c4b5f 100644
--- a/ghcide/src/Development/IDE/Core/Compile.hs
+++ b/ghcide/src/Development/IDE/Core/Compile.hs
@@ -21,7 +21,6 @@ module Development.IDE.Core.Compile
   , generateByteCode
   , generateHieAsts
   , writeAndIndexHieFile
-  , HieDbModuleQuery(..)
   , indexHieFile
   , writeHiFile
   , getModSummaryFromImports
@@ -860,10 +859,6 @@ spliceExpressions Splices{..} =
         , DL.fromList $ map fst awSplices
         ]
 
-data HieDbModuleQuery
-  = HieDbModuleQuery ModuleName Unit
-  | DontCheckForModule
-
 -- | In addition to indexing the `.hie` file, this function is responsible for
 -- maintaining the 'IndexQueue' state and notifying the user about indexing
 -- progress.
@@ -892,14 +887,16 @@ data HieDbModuleQuery
 -- TVar to 0 in order to set it up for a fresh indexing session. Otherwise, we
 -- can just increment the 'indexCompleted' TVar and exit.
 --
-indexHieFile :: ShakeExtras -> HieDbModuleQuery -> NormalizedFilePath -> HieDb.SourceFile -> Util.Fingerprint -> Compat.HieFile -> IO ()
-indexHieFile se query hiePath sourceFile !hash hf = do
+indexHieFile :: ShakeExtras -> NormalizedFilePath -> HieDb.SourceFile -> Util.Fingerprint -> Compat.HieFile -> IO ()
+indexHieFile se hiePath sourceFile !hash hf = do
  IdeOptions{optProgressStyle} <- getIdeOptionsIO se
  atomically $ do
   pending <- readTVar indexPending
   case HashMap.lookup hiePath pending of
     Just pendingHash | pendingHash == hash -> pure () -- An index is already scheduled
     _ -> do
+      -- hiedb doesn't use the Haskell src, so we clear it to avoid unnecessarily keeping it around
+      let !hf' = hf{hie_hs_src = mempty}
       modifyTVar' indexPending $ HashMap.insert hiePath hash
       writeTQueue indexQueue $ \withHieDb -> do
         -- We are now in the worker thread
@@ -914,24 +911,8 @@ indexHieFile se query hiePath sourceFile !hash hf = do
           -- Using bracket, so even if an exception happen during withHieDb call,
           -- the `post` (which clean the progress indicator) will still be called.
           bracket_ (pre optProgressStyle) post $
-            withHieDb indexIfNotAlready
+            withHieDb (\db -> HieDb.addRefsFromLoaded db (fromNormalizedFilePath hiePath) sourceFile hash hf')
   where
-    -- hiedb doesn't use the Haskell src, so we clear it to avoid unnecessarily keeping it around
-    hf' :: Compat.HieFile
-    !hf' = hf{hie_hs_src = mempty}
-    indexIfNotAlready :: HieDb -> IO ()
-    indexIfNotAlready db = case query of
-      DontCheckForModule -> doIndexing
-      HieDbModuleQuery moduleName unit -> do
-        mRow <- HieDb.lookupHieFile db moduleName unit
-        case mRow of
-          Nothing -> doIndexing
-          Just _row -> return ()
-      where
-        doIndexing :: IO ()
-        doIndexing =
-          HieDb.addRefsFromLoaded db (fromNormalizedFilePath hiePath) sourceFile hash hf'
-
     HieDbWriter{..} = hiedbWriter se
 
     -- Get a progress token to report progress and update it for the current file
@@ -1024,7 +1005,7 @@ writeAndIndexHieFile hscEnv se mod_summary srcPath exports ast source =
       GHC.mkHieFile' mod_summary exports ast source
     atomicFileWrite se targetPath $ flip GHC.writeHieFile hf
     hash <- Util.getFileHash targetPath
-    indexHieFile se DontCheckForModule (toNormalizedFilePath' targetPath) (HieDb.RealFile $ fromNormalizedFilePath srcPath) hash hf
+    indexHieFile se (toNormalizedFilePath' targetPath) (HieDb.RealFile $ fromNormalizedFilePath srcPath) hash hf
   where
     dflags       = hsc_dflags hscEnv
     mod_location = ms_location mod_summary
diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs
index d4460e92c3..6456e44fb4 100644
--- a/ghcide/src/Development/IDE/Core/Rules.hs
+++ b/ghcide/src/Development/IDE/Core/Rules.hs
@@ -894,7 +894,7 @@ getModIfaceFromDiskAndIndexRule recorder =
         -- can just re-index the file we read from disk
         Right hf -> liftIO $ do
           logWith recorder Logger.Debug $ LogReindexingHieFile f
-          indexHieFile se DontCheckForModule (toNormalizedFilePath' hie_loc) (HieDb.RealFile $ fromNormalizedFilePath f) hash hf
+          indexHieFile se (toNormalizedFilePath' hie_loc) (HieDb.RealFile $ fromNormalizedFilePath f) hash hf
 
   return (Just x)
 
diff --git a/ghcide/src/Development/IDE/GHC/Compat/Units.hs b/ghcide/src/Development/IDE/GHC/Compat/Units.hs
index 23af072063..327f344517 100644
--- a/ghcide/src/Development/IDE/GHC/Compat/Units.hs
+++ b/ghcide/src/Development/IDE/GHC/Compat/Units.hs
@@ -27,7 +27,6 @@ module Development.IDE.GHC.Compat.Units (
     unitHiddenModules,
     unitLibraryDirs,
     UnitInfo.unitId,
-    UnitInfo.mkUnit,
     unitDepends,
     unitHaddockInterfaces,
     unitInfoId,
@@ -126,6 +125,7 @@ type PreloadUnitClosure = ()
 type Unit = UnitId
 #endif
 
+
 #if !MIN_VERSION_ghc(9,0,0)
 unitString :: Unit -> String
 unitString = Module.unitIdString
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index d293b7ab11..2ac6dd2430 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -30,7 +30,7 @@ import qualified Data.Set                        as Set
 import qualified Data.Text                       as T
 import           Data.Unique                     (Unique)
 import qualified Data.Unique                     as Unique
-import           Development.IDE.Core.Compile    (HieDbModuleQuery(HieDbModuleQuery), indexHieFile, loadHieFile)
+import           Development.IDE.Core.Compile    (indexHieFile, loadHieFile)
 import           Development.IDE.Core.Shake      (HieDbWriter(indexQueue), ShakeExtras(hiedbWriter, ideNc, logger, lspEnv), mkUpdater)
 import           Development.IDE.GHC.Compat
 import qualified Development.IDE.GHC.Compat.Util as Maybes
@@ -146,25 +146,19 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
                   (libraryDir : _) -> libraryDir
                 hieDir :: FilePath
                 hieDir = pkgLibDir </> "extra-compilation-artifacts"
-                packageUnit :: Unit
-                packageUnit = mkUnit package
             modIfaces <- mapMaybeM loadModIface modules
-            traverse_ (indexModuleHieFile hieDir packageUnit) modIfaces
-        indexModuleHieFile :: FilePath -> Unit -> ModIface -> IO ()
-        indexModuleHieFile hieDir packageUnit modIface = do
-            let modName :: ModuleName
-                modName = moduleName $ mi_module modIface
-                hiePath :: FilePath
-                hiePath = hieDir </> toFilePath modName ++ ".hie"
-                query :: HieDbModuleQuery
-                query = HieDbModuleQuery modName packageUnit
+            traverse_ (indexModuleHieFile hieDir) modIfaces
+        indexModuleHieFile :: FilePath -> ModIface -> IO ()
+        indexModuleHieFile hieDir modIface = do
+            let hiePath :: FilePath
+                hiePath = hieDir </> toFilePath (moduleName $ mi_module modIface) ++ ".hie"
             hieResults <- tryAny $ loadHieFile (mkUpdater $ ideNc se) hiePath
             case hieResults of
               Left e -> Logger.logDebug (logger se) $
                   "Failed to index dependency HIE file:\n"
                   <> T.pack (show e)
               Right hie ->
-                  indexHieFile se query (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash modIface) hie
+                  indexHieFile se (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash modIface) hie
         toFilePath :: ModuleName -> FilePath
         toFilePath = separateDirectories . prettyModuleName
             where

From 6ed5c1988a6350f9d3cde7e12ac0bdb20ad9f143 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Wed, 12 Jul 2023 11:30:37 -0500
Subject: [PATCH 28/50] Factor out common hie file load checks

---
 .../session-loader/Development/IDE/Session.hs |  5 +-
 ghcide/src/Development/IDE/Core/Rules.hs      | 98 ++++++++++++-------
 ghcide/src/Development/IDE/Types/HscEnvEq.hs  | 42 ++++----
 .../Development/IDE/Types/HscEnvEq.hs-boot    | 18 +++-
 4 files changed, 107 insertions(+), 56 deletions(-)

diff --git a/ghcide/session-loader/Development/IDE/Session.hs b/ghcide/session-loader/Development/IDE/Session.hs
index 1ea2ca880f..40831564e7 100644
--- a/ghcide/session-loader/Development/IDE/Session.hs
+++ b/ghcide/session-loader/Development/IDE/Session.hs
@@ -48,6 +48,7 @@ import qualified Data.Text                            as T
 import           Data.Time.Clock
 import           Data.Version
 import           Development.IDE.Core.RuleTypes
+import qualified Development.IDE.Core.Rules           as Rules
 import           Development.IDE.Core.Shake           hiding (Log, Priority,
                                                        withHieDb)
 import qualified Development.IDE.GHC.Compat           as Compat
@@ -127,6 +128,7 @@ data Log
   | LogNoneCradleFound FilePath
   | LogNewComponentCache !(([FileDiagnostic], Maybe HscEnvEq), DependencyInfo)
   | LogHieBios HieBios.Log
+  | LogRules Rules.Log
 deriving instance Show Log
 
 instance Pretty Log where
@@ -197,6 +199,7 @@ instance Pretty Log where
     LogNewComponentCache componentCache ->
       "New component cache HscEnvEq:" <+> viaShow componentCache
     LogHieBios log -> pretty log
+    LogRules log -> pretty log
 
 -- | Bump this version number when making changes to the format of the data stored in hiedb
 hiedbDataVersion :: String
@@ -824,7 +827,7 @@ newComponentCache recorder extras exts cradlePath cfp hsc_env uids ci = do
 #endif
 
     let newFunc = maybe newHscEnvEqPreserveImportPaths newHscEnvEq cradlePath
-    henv <- newFunc extras hscEnv' uids
+    henv <- newFunc (cmapWithPrio LogRules recorder) extras hscEnv' uids
     let targetEnv = ([], Just henv)
         targetDepends = componentDependencyInfo ci
         res = (targetEnv, targetDepends)
diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs
index 6456e44fb4..9842b29221 100644
--- a/ghcide/src/Development/IDE/Core/Rules.hs
+++ b/ghcide/src/Development/IDE/Core/Rules.hs
@@ -13,7 +13,9 @@ module Development.IDE.Core.Rules(
     -- * Types
     IdeState, GetParsedModule(..), TransitiveDependencies(..),
     Priority(..), GhcSessionIO(..), GetClientSettings(..),
+    HieFileCheck(..),
     -- * Functions
+    checkHieFile,
     priorityTypeCheck,
     priorityGenerateCore,
     priorityFilesOfInterest,
@@ -78,6 +80,7 @@ import           Data.Aeson                                   (Result (Success),
                                                                toJSON)
 import qualified Data.Aeson.Types                             as A
 import qualified Data.Binary                                  as B
+import           Data.Bool                                    (bool)
 import qualified Data.ByteString                              as BS
 import qualified Data.ByteString.Lazy                         as LBS
 import           Data.Coerce
@@ -129,7 +132,7 @@ import qualified Development.IDE.Spans.AtPoint                as AtPoint
 import           Development.IDE.Spans.Documentation
 import           Development.IDE.Spans.LocalBindings
 import           Development.IDE.Types.Diagnostics            as Diag
-import           Development.IDE.Types.HscEnvEq
+import {-# SOURCE #-} Development.IDE.Types.HscEnvEq
 import           Development.IDE.Types.Location
 import           Development.IDE.Types.Options
 import qualified GHC.LanguageExtensions                       as LangExt
@@ -172,8 +175,9 @@ data Log
   = LogShake Shake.Log
   | LogReindexingHieFile !NormalizedFilePath
   | LogLoadingHieFile !NormalizedFilePath
-  | LogLoadingHieFileFail !FilePath !SomeException
-  | LogLoadingHieFileSuccess !FilePath
+  | LogMissingHieFile !NormalizedFilePath
+  | LogLoadingHieFileFail !NormalizedFilePath !SomeException
+  | LogLoadingHieFileSuccess !NormalizedFilePath
   | LogTypecheckedFOI !NormalizedFilePath
   deriving Show
 
@@ -184,13 +188,15 @@ instance Pretty Log where
       "Re-indexing hie file for" <+> pretty (fromNormalizedFilePath path)
     LogLoadingHieFile path ->
       "LOADING HIE FILE FOR" <+> pretty (fromNormalizedFilePath path)
+    LogMissingHieFile path ->
+      "MISSING HIE FILE" <+> pretty (fromNormalizedFilePath path)
     LogLoadingHieFileFail path e ->
       nest 2 $
         vcat
-          [ "FAILED LOADING HIE FILE FOR" <+> pretty path
+          [ "FAILED LOADING HIE FILE" <+> pretty (fromNormalizedFilePath path)
           , pretty (displayException e) ]
     LogLoadingHieFileSuccess path ->
-      "SUCCEEDED LOADING HIE FILE FOR" <+> pretty path
+      "SUCCEEDED LOADING HIE FILE" <+> pretty (fromNormalizedFilePath path)
     LogTypecheckedFOI path -> vcat
       [ "Typechecked a file which is not currently open in the editor:" <+> pretty (fromNormalizedFilePath path)
       , "This can indicate a bug which results in excessive memory usage."
@@ -665,14 +671,14 @@ readHieFileForSrcFromDisk :: Recorder (WithPriority Log) -> NormalizedFilePath -
 readHieFileForSrcFromDisk recorder file = do
   ShakeExtras{withHieDb} <- ask
   row <- MaybeT $ liftIO $ withHieDb (\hieDb -> HieDb.lookupHieFileFromSource hieDb $ fromNormalizedFilePath file)
-  let hie_loc = HieDb.hieModuleHieFile row
+  let hie_loc = toNormalizedFilePath' $ HieDb.hieModuleHieFile row
   liftIO $ logWith recorder Logger.Debug $ LogLoadingHieFile file
   exceptToMaybeT $ readHieFileFromDisk recorder hie_loc
 
-readHieFileFromDisk :: Recorder (WithPriority Log) -> FilePath -> ExceptT SomeException IdeAction Compat.HieFile
+readHieFileFromDisk :: Recorder (WithPriority Log) -> NormalizedFilePath -> ExceptT SomeException IdeAction Compat.HieFile
 readHieFileFromDisk recorder hie_loc = do
   nc <- asks ideNc
-  res <- liftIO $ tryAny $ loadHieFile (mkUpdater nc) hie_loc
+  res <- liftIO $ tryAny $ loadHieFile (mkUpdater nc) (fromNormalizedFilePath hie_loc)
   let log = (liftIO .) . logWith recorder
   case res of
     Left e -> log Logger.Debug $ LogLoadingHieFileFail hie_loc e
@@ -854,6 +860,43 @@ getModIfaceFromDiskRule recorder = defineEarlyCutoff (cmapWithPrio LogShake reco
           let !fp = Just $! hiFileFingerPrint x
           return (fp, (diags, Just x))
 
+data HieFileCheck
+  = HieFileMissing
+  | HieAlreadyIndexed
+  | CouldNotLoadHie SomeException
+  | DoIndexing Util.Fingerprint HieFile
+
+checkHieFile
+  :: Recorder (WithPriority Log)
+  -> ShakeExtras
+  -> String
+  -> NormalizedFilePath
+  -> IO HieFileCheck
+checkHieFile recorder se@ShakeExtras{withHieDb} tag hieFileLocation = do
+  hieFileExists <- doesFileExist $ fromNormalizedFilePath hieFileLocation
+  bool logHieFileMissing checkExistingHieFile hieFileExists
+  where
+    logHieFileMissing :: IO HieFileCheck
+    logHieFileMissing = do
+      let log :: Log
+          log = LogMissingHieFile hieFileLocation
+      logWith recorder Logger.Debug log
+      pure HieFileMissing
+    checkExistingHieFile :: IO HieFileCheck
+    checkExistingHieFile = do
+      hash <- Util.getFileHash $ fromNormalizedFilePath hieFileLocation
+      mrow <- withHieDb (\hieDb -> HieDb.lookupHieFileFromHash hieDb hash)
+      dbHieFileLocation <- traverse (makeAbsolute . HieDb.hieModuleHieFile) mrow
+      bool (tryLoadingHieFile hash) (pure HieAlreadyIndexed) $
+        Just hieFileLocation == fmap toNormalizedFilePath' dbHieFileLocation
+    tryLoadingHieFile :: Util.Fingerprint -> IO HieFileCheck
+    tryLoadingHieFile hash = do
+      ehf <- runIdeAction tag se $ runExceptT $
+        readHieFileFromDisk recorder hieFileLocation
+      pure $ case ehf of
+        Left err -> CouldNotLoadHie err
+        Right hf -> DoIndexing hash hf
+
 -- | Check state of hiedb after loading an iface from disk - have we indexed the corresponding `.hie` file?
 -- This function is responsible for ensuring database consistency
 -- Whenever we read a `.hi` file, we must check to ensure we have also
@@ -871,31 +914,20 @@ getModIfaceFromDiskAndIndexRule recorder =
 
   -- GetModIfaceFromDisk should have written a `.hie` file, must check if it matches version in db
   let ms = hirModSummary x
-      hie_loc = Compat.ml_hie_file $ ms_location ms
-  hash <- liftIO $ Util.getFileHash hie_loc
-  mrow <- liftIO $ withHieDb (\hieDb -> HieDb.lookupHieFileFromSource hieDb (fromNormalizedFilePath f))
-  hie_loc' <- liftIO $ traverse (makeAbsolute . HieDb.hieModuleHieFile) mrow
-  case mrow of
-    Just row
-      | hash == HieDb.modInfoHash (HieDb.hieModInfo row)
-      && Just hie_loc == hie_loc'
-      -> do
-      -- All good, the db has indexed the file
-      when (coerce $ ideTesting se) $ liftIO $ mRunLspT (lspEnv se) $
-        LSP.sendNotification (SMethod_CustomMethod (Proxy @"ghcide/reference/ready")) $
-          toJSON $ fromNormalizedFilePath f
-    -- Not in db, must re-index
-    _ -> do
-      ehf <- liftIO $ runIdeAction "GetModIfaceFromDiskAndIndex" se $ runExceptT $
-        readHieFileFromDisk recorder hie_loc
-      case ehf of
-        -- Uh oh, we failed to read the file for some reason, need to regenerate it
-        Left err -> fail $ "failed to read .hie file " ++ show hie_loc ++ ": " ++ displayException err
-        -- can just re-index the file we read from disk
-        Right hf -> liftIO $ do
-          logWith recorder Logger.Debug $ LogReindexingHieFile f
-          indexHieFile se (toNormalizedFilePath' hie_loc) (HieDb.RealFile $ fromNormalizedFilePath f) hash hf
-
+      hie_loc = toNormalizedFilePath' $ Compat.ml_hie_file $ ms_location ms
+      hieFailure :: Maybe SomeException -> Action ()
+      hieFailure mErr = fail $ "failed to read .hie file " ++ show hie_loc ++ ": " ++
+        maybe "Does not exist" displayException mErr
+  hieCheck <- liftIO $ checkHieFile recorder se "GetModIfaceFromDiskAndIndex" hie_loc
+  case hieCheck of
+    HieFileMissing -> hieFailure Nothing
+    HieAlreadyIndexed -> when (coerce $ ideTesting se) $ liftIO $ mRunLspT (lspEnv se) $
+      LSP.sendNotification (SMethod_CustomMethod (Proxy @"ghcide/reference/ready")) $
+        toJSON $ fromNormalizedFilePath f
+    CouldNotLoadHie err -> hieFailure $ Just err
+    DoIndexing hash hf -> liftIO $ do
+      logWith recorder Logger.Debug $ LogReindexingHieFile f
+      indexHieFile se hie_loc (HieDb.RealFile $ fromNormalizedFilePath f) hash hf
   return (Just x)
 
 newtype DisplayTHWarning = DisplayTHWarning (IO())
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 2ac6dd2430..fd148465bd 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -19,7 +19,6 @@ import           Control.Concurrent.STM.TQueue   (unGetTQueue)
 import           Control.Concurrent.Strict       (modifyVar, newVar)
 import           Control.DeepSeq                 (force)
 import           Control.Exception               (evaluate, mask, throwIO)
-import           Control.Exception.Safe          (tryAny)
 import           Control.Monad                   (unless)
 import           Control.Monad.Extra             (eitherM, join, mapMaybeM, void)
 import           Data.Either                     (fromRight)
@@ -27,19 +26,19 @@ import           Data.Foldable                   (traverse_)
 import qualified Data.Map                        as Map
 import           Data.Set                        (Set)
 import qualified Data.Set                        as Set
-import qualified Data.Text                       as T
 import           Data.Unique                     (Unique)
 import qualified Data.Unique                     as Unique
-import           Development.IDE.Core.Compile    (indexHieFile, loadHieFile)
-import           Development.IDE.Core.Shake      (HieDbWriter(indexQueue), ShakeExtras(hiedbWriter, ideNc, logger, lspEnv), mkUpdater)
+import           Development.IDE.Core.Compile    (indexHieFile)
+import           Development.IDE.Core.Rules      (HieFileCheck(..), Log, checkHieFile)
+import           Development.IDE.Core.Shake      (HieDbWriter(indexQueue), ShakeExtras(hiedbWriter, lspEnv))
 import           Development.IDE.GHC.Compat
 import qualified Development.IDE.GHC.Compat.Util as Maybes
 import           Development.IDE.GHC.Error       (catchSrcErrors)
 import           Development.IDE.GHC.Util        (lookupPackageConfig)
 import           Development.IDE.Graph.Classes
 import           Development.IDE.Types.Exports   (ExportsMap, createExportsMap)
-import           Development.IDE.Types.Location  (toNormalizedFilePath')
-import qualified Development.IDE.Types.Logger    as Logger
+import           Development.IDE.Types.Location  (NormalizedFilePath, toNormalizedFilePath')
+import           Development.IDE.Types.Logger    (Recorder, WithPriority)
 import           HieDb                           (SourceFile(FakeFile), removeDependencySrcFiles)
 import           Language.LSP.Server             (resRootPath)
 import           OpenTelemetry.Eventlog          (withSpan)
@@ -73,8 +72,8 @@ updateHscEnvEq oldHscEnvEq newHscEnv = do
   update <$> Unique.newUnique
 
 -- | Wrap an 'HscEnv' into an 'HscEnvEq'.
-newHscEnvEq :: FilePath -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
-newHscEnvEq cradlePath se hscEnv0 deps = do
+newHscEnvEq :: FilePath -> Recorder (WithPriority Log) -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
+newHscEnvEq cradlePath recorder se hscEnv0 deps = do
     let relativeToCradle = (takeDirectory cradlePath </>)
         hscEnv = removeImportPaths hscEnv0
 
@@ -82,10 +81,10 @@ newHscEnvEq cradlePath se hscEnv0 deps = do
     importPathsCanon <-
       mapM makeAbsolute $ relativeToCradle <$> importPaths (hsc_dflags hscEnv0)
 
-    newHscEnvEqWithImportPaths (Just $ Set.fromList importPathsCanon) se hscEnv deps
+    newHscEnvEqWithImportPaths (Just $ Set.fromList importPathsCanon) recorder se hscEnv deps
 
-newHscEnvEqWithImportPaths :: Maybe (Set FilePath) -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
-newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
+newHscEnvEqWithImportPaths :: Maybe (Set FilePath) -> Recorder (WithPriority Log) -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
+newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
 
     indexDependencyHieFiles
 
@@ -150,15 +149,16 @@ newHscEnvEqWithImportPaths envImportPaths se hscEnv deps = do
             traverse_ (indexModuleHieFile hieDir) modIfaces
         indexModuleHieFile :: FilePath -> ModIface -> IO ()
         indexModuleHieFile hieDir modIface = do
-            let hiePath :: FilePath
-                hiePath = hieDir </> toFilePath (moduleName $ mi_module modIface) ++ ".hie"
-            hieResults <- tryAny $ loadHieFile (mkUpdater $ ideNc se) hiePath
-            case hieResults of
-              Left e -> Logger.logDebug (logger se) $
-                  "Failed to index dependency HIE file:\n"
-                  <> T.pack (show e)
-              Right hie ->
-                  indexHieFile se (toNormalizedFilePath' hiePath) (FakeFile Nothing) (mi_src_hash modIface) hie
+            let hiePath :: NormalizedFilePath
+                hiePath = toNormalizedFilePath' $
+                  hieDir </> toFilePath (moduleName $ mi_module modIface) ++ ".hie"
+            hieCheck <- checkHieFile recorder se "newHscEnvEqWithImportPaths" hiePath
+            case hieCheck of
+                HieFileMissing -> return ()
+                HieAlreadyIndexed -> return ()
+                CouldNotLoadHie _e -> return ()
+                DoIndexing hash hie ->
+                    indexHieFile se hiePath (FakeFile Nothing) hash hie
         toFilePath :: ModuleName -> FilePath
         toFilePath = separateDirectories . prettyModuleName
             where
@@ -214,7 +214,7 @@ instance Ord Package where
 
 -- | Wrap an 'HscEnv' into an 'HscEnvEq'.
 newHscEnvEqPreserveImportPaths
-    :: ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
+    :: Recorder (WithPriority Log) -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
 newHscEnvEqPreserveImportPaths = newHscEnvEqWithImportPaths Nothing
 
 -- | Unwrap the 'HscEnv' with the original import paths.
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs-boot b/ghcide/src/Development/IDE/Types/HscEnvEq.hs-boot
index 6ff6390e18..681d6331a5 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs-boot
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs-boot
@@ -1,8 +1,16 @@
-module Development.IDE.Types.HscEnvEq (HscEnvEq) where
+module Development.IDE.Types.HscEnvEq
+(   HscEnvEq,
+    hscEnv,
+    hscEnvWithImportPaths,
+    updateHscEnvEq,
+    envImportPaths,
+    deps
+) where
 
 import           Data.Set                        (Set)
 import           Data.Unique                     (Unique)
 import           Development.IDE.GHC.Compat
+import           Development.IDE.Graph.Classes
 import           Development.IDE.Types.Exports   (ExportsMap)
 
 -- | An 'HscEnv' with equality. Two values are considered equal
@@ -24,3 +32,11 @@ data HscEnvEq = HscEnvEq
         -- So it's wrapped in IO here for error handling
         -- If Nothing, 'listVisibleModuleNames' panic
     }
+
+instance Show HscEnvEq
+instance Hashable HscEnvEq
+instance NFData HscEnvEq
+
+updateHscEnvEq :: HscEnvEq -> HscEnv -> IO HscEnvEq
+
+hscEnvWithImportPaths :: HscEnvEq -> HscEnv

From 48f7dce82b4eb5b979051083b9e90504b8a09ff9 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 13 Jul 2023 18:30:35 -0500
Subject: [PATCH 29/50] Use moduleNameSlashes

---
 ghcide/src/Development/IDE/Core/Actions.hs   | 21 +-------------------
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 21 +-------------------
 2 files changed, 2 insertions(+), 40 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Actions.hs b/ghcide/src/Development/IDE/Core/Actions.hs
index f4a2983aa5..4fef6d26d6 100644
--- a/ghcide/src/Development/IDE/Core/Actions.hs
+++ b/ghcide/src/Development/IDE/Core/Actions.hs
@@ -83,30 +83,11 @@ lookupMod HieDbWriter{indexQueue} hieFile moduleName uid _boot = MaybeT $ do
         writeOutDir :: FilePath
         writeOutDir = projectRoot </> ".hls" </> "dependencies" </> show uid
         writeOutFile :: FilePath
-        writeOutFile = toFilePath moduleName ++ ".hs"
+        writeOutFile = moduleNameSlashes moduleName ++ ".hs"
         writeOutPath :: FilePath
         writeOutPath = writeOutDir </> writeOutFile
         moduleUri :: Uri
         moduleUri = AtPoint.toUri writeOutPath
-        toFilePath :: ModuleName -> FilePath
-        toFilePath = separateDirectories . prettyModuleName
-          where
-            separateDirectories :: FilePath -> FilePath
-            separateDirectories moduleNameString =
-              case breakOnDot moduleNameString of
-                [] -> ""
-                ms -> foldr1 (</>) ms
-            breakOnDot :: FilePath -> [FilePath]
-            breakOnDot = words . map replaceDotWithSpace
-            replaceDotWithSpace :: Char -> Char
-            replaceDotWithSpace '.' = ' '
-            replaceDotWithSpace c = c
-            prettyModuleName :: ModuleName -> String
-            prettyModuleName = filter (/= '"')
-              . concat
-              . drop 1
-              . words
-              . show
 
 
 
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index fd148465bd..589f4ba5a5 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -151,7 +151,7 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
         indexModuleHieFile hieDir modIface = do
             let hiePath :: NormalizedFilePath
                 hiePath = toNormalizedFilePath' $
-                  hieDir </> toFilePath (moduleName $ mi_module modIface) ++ ".hie"
+                  hieDir </> moduleNameSlashes (moduleName $ mi_module modIface) ++ ".hie"
             hieCheck <- checkHieFile recorder se "newHscEnvEqWithImportPaths" hiePath
             case hieCheck of
                 HieFileMissing -> return ()
@@ -159,25 +159,6 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
                 CouldNotLoadHie _e -> return ()
                 DoIndexing hash hie ->
                     indexHieFile se hiePath (FakeFile Nothing) hash hie
-        toFilePath :: ModuleName -> FilePath
-        toFilePath = separateDirectories . prettyModuleName
-            where
-                separateDirectories :: FilePath -> FilePath
-                separateDirectories moduleNameString =
-                    case breakOnDot moduleNameString of
-                        [] -> ""
-                        ms -> foldr1 (</>) ms
-                breakOnDot :: FilePath -> [FilePath]
-                breakOnDot = words . map replaceDotWithSpace
-                replaceDotWithSpace :: Char -> Char
-                replaceDotWithSpace '.' = ' '
-                replaceDotWithSpace c = c
-                prettyModuleName :: ModuleName -> String
-                prettyModuleName = filter (/= '"')
-                    . concat
-                    . drop 1
-                    . words
-                    . show
         loadModIface :: Module -> IO (Maybe ModIface)
         loadModIface m = do
             modIface <- initIfaceLoad hscEnv $

From a1d88b8fe28575de2aa111e599459821b5fbb91a Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 13 Jul 2023 18:57:33 -0500
Subject: [PATCH 30/50] Remove unecessary getModIface

---
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 589f4ba5a5..bb039e7653 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -145,13 +145,12 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
                   (libraryDir : _) -> libraryDir
                 hieDir :: FilePath
                 hieDir = pkgLibDir </> "extra-compilation-artifacts"
-            modIfaces <- mapMaybeM loadModIface modules
-            traverse_ (indexModuleHieFile hieDir) modIfaces
-        indexModuleHieFile :: FilePath -> ModIface -> IO ()
-        indexModuleHieFile hieDir modIface = do
+            traverse_ (indexModuleHieFile hieDir) modules
+        indexModuleHieFile :: FilePath -> Module -> IO ()
+        indexModuleHieFile hieDir m = do
             let hiePath :: NormalizedFilePath
                 hiePath = toNormalizedFilePath' $
-                  hieDir </> moduleNameSlashes (moduleName $ mi_module modIface) ++ ".hie"
+                  hieDir </> moduleNameSlashes (moduleName m) ++ ".hie"
             hieCheck <- checkHieFile recorder se "newHscEnvEqWithImportPaths" hiePath
             case hieCheck of
                 HieFileMissing -> return ()

From 93aeceae5fb6512d65d97cc466bbacdd297f3389 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 13 Jul 2023 19:59:00 -0500
Subject: [PATCH 31/50] Generate RefMap from HieFile

---
 ghcide/src/Development/IDE/Core/Rules.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs
index 9842b29221..257c81ceeb 100644
--- a/ghcide/src/Development/IDE/Core/Rules.hs
+++ b/ghcide/src/Development/IDE/Core/Rules.hs
@@ -593,10 +593,13 @@ getHieAstsRule recorder =
         makeHieAstResult hieFile =
             HAR
                 (Compat.hie_module hieFile)
-                (Compat.hie_asts hieFile)
-                mempty
+                hieAsts
+                (Compat.generateReferencesMap $ M.elems $ getAsts hieAsts)
                 mempty
                 (HieFromDisk hieFile)
+            where
+                hieAsts :: HieASTs TypeIndex
+                hieAsts = Compat.hie_asts hieFile
 
 persistentHieFileRule :: Recorder (WithPriority Log) -> Rules ()
 persistentHieFileRule recorder = addPersistentRule GetHieAst $ \file -> runMaybeT $ do

From a807ae85edf358f34fecd4cac5871311b76528b4 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 13 Jul 2023 20:36:50 -0500
Subject: [PATCH 32/50] Avoid indexing redundant modules

---
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index bb039e7653..c4d5ac7560 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -24,6 +24,7 @@ import           Control.Monad.Extra             (eitherM, join, mapMaybeM, void
 import           Data.Either                     (fromRight)
 import           Data.Foldable                   (traverse_)
 import qualified Data.Map                        as Map
+import           Data.Maybe                      (isNothing)
 import           Data.Set                        (Set)
 import qualified Data.Set                        as Set
 import           Data.Unique                     (Unique)
@@ -176,16 +177,15 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
         getModulesForPackage (Package package) =
             map makeModule allModules
             where
-                allModules :: [(ModuleName, Maybe Module)]
-                allModules = unitExposedModules package
-                    ++ zip (unitHiddenModules package) (repeat Nothing)
-                makeModule :: (ModuleName, Maybe Module)
+                allModules :: [ModuleName]
+                allModules = map fst
+                    ( filter (isNothing . snd)
+                    $ unitExposedModules package
+                    )
+                    ++ unitHiddenModules package
+                makeModule :: ModuleName
                            -> Module
-                makeModule (moduleName, Nothing) =
-                    mkModule (unitInfoId package) moduleName
-                -- When module is re-exported from another package,
-                -- the origin module is represented by value in Just
-                makeModule (_, Just otherPackageMod) = otherPackageMod
+                makeModule = mkModule (unitInfoId package)
 
 newtype Package = Package UnitInfo deriving Eq
 instance Ord Package where

From 095dadd400f9da7b6b5762a2439914f580d35f55 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Sat, 15 Jul 2023 06:07:30 -0500
Subject: [PATCH 33/50] Use more System.FilePath functions

---
 ghcide/src/Development/IDE/Core/Actions.hs   | 4 ++--
 ghcide/src/Development/IDE/Core/Shake.hs     | 5 ++++-
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Actions.hs b/ghcide/src/Development/IDE/Core/Actions.hs
index 4fef6d26d6..efdeb382e6 100644
--- a/ghcide/src/Development/IDE/Core/Actions.hs
+++ b/ghcide/src/Development/IDE/Core/Actions.hs
@@ -43,7 +43,7 @@ import           Language.LSP.Protocol.Types          (DocumentHighlight (..),
                                                        uriToNormalizedFilePath)
 import           Language.LSP.Server                  (resRootPath)
 import           System.Directory                     (createDirectoryIfMissing, doesFileExist)
-import           System.FilePath                      ((</>), takeDirectory)
+import           System.FilePath                      ((</>), (<.>), takeDirectory)
 
 
 -- | Generates URIs for files in dependencies, but not in the
@@ -83,7 +83,7 @@ lookupMod HieDbWriter{indexQueue} hieFile moduleName uid _boot = MaybeT $ do
         writeOutDir :: FilePath
         writeOutDir = projectRoot </> ".hls" </> "dependencies" </> show uid
         writeOutFile :: FilePath
-        writeOutFile = moduleNameSlashes moduleName ++ ".hs"
+        writeOutFile = moduleNameSlashes moduleName <.> "hs"
         writeOutPath :: FilePath
         writeOutPath = writeOutDir </> writeOutFile
         moduleUri :: Uri
diff --git a/ghcide/src/Development/IDE/Core/Shake.hs b/ghcide/src/Development/IDE/Core/Shake.hs
index 9a90df4eb3..9852181ed2 100644
--- a/ghcide/src/Development/IDE/Core/Shake.hs
+++ b/ghcide/src/Development/IDE/Core/Shake.hs
@@ -1132,9 +1132,12 @@ data SourceFileOrigin = FromProject | FromDependency
 
 getSourceFileOrigin :: NormalizedFilePath -> SourceFileOrigin
 getSourceFileOrigin f =
-    case isInfixOf ".hls/dependencies" (show f) of
+    case [".hls", "dependencies"] `isInfixOf` (splitDirectories file) of
         True -> FromDependency
         False -> FromProject
+    where
+        file :: FilePath
+        file = fromNormalizedFilePath f
 
 defineEarlyCutoff'
     :: forall k v. IdeRule k v
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index c4d5ac7560..9befb13fe0 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -151,7 +151,7 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
         indexModuleHieFile hieDir m = do
             let hiePath :: NormalizedFilePath
                 hiePath = toNormalizedFilePath' $
-                  hieDir </> moduleNameSlashes (moduleName m) ++ ".hie"
+                  hieDir </> moduleNameSlashes (moduleName m) <.> "hie"
             hieCheck <- checkHieFile recorder se "newHscEnvEqWithImportPaths" hiePath
             case hieCheck of
                 HieFileMissing -> return ()

From af1141c6dec952cefebc26cec1fa6a387c8899df Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Sat, 15 Jul 2023 06:23:54 -0500
Subject: [PATCH 34/50] Correct completion token placement

---
 ghcide/src/Development/IDE/Core/Actions.hs   | 5 +++--
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 9 +++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Actions.hs b/ghcide/src/Development/IDE/Core/Actions.hs
index efdeb382e6..4d44a88240 100644
--- a/ghcide/src/Development/IDE/Core/Actions.hs
+++ b/ghcide/src/Development/IDE/Core/Actions.hs
@@ -75,8 +75,9 @@ lookupMod HieDbWriter{indexQueue} hieFile moduleName uid _boot = MaybeT $ do
           moduleSource <- hie_hs_src <$> loadHieFile (mkUpdater nc) hieFile
           BS.writeFile writeOutPath moduleSource
       liftIO $ atomically $
-        unGetTQueue indexQueue $ \withHieDb -> withHieDb $ \db -> do
-          HieDb.addSrcFile db hieFile writeOutPath False
+        unGetTQueue indexQueue $ \withHieDb -> do
+          withHieDb $ \db ->
+            HieDb.addSrcFile db hieFile writeOutPath False
           putMVar completionToken ()
       pure $ moduleUri
       where
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 9befb13fe0..1e858209ea 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -15,7 +15,7 @@ module Development.IDE.Types.HscEnvEq
 import           Control.Concurrent.Async        (Async, async, waitCatch)
 import           Control.Concurrent.MVar         (newEmptyMVar, putMVar, readMVar)
 import           Control.Concurrent.STM          (atomically)
-import           Control.Concurrent.STM.TQueue   (unGetTQueue)
+import           Control.Concurrent.STM.TQueue   (writeTQueue)
 import           Control.Concurrent.Strict       (modifyVar, newVar)
 import           Control.DeepSeq                 (force)
 import           Control.Exception               (evaluate, mask, throwIO)
@@ -133,9 +133,10 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
         deleteMissingDependencySources :: IO ()
         deleteMissingDependencySources = do
             completionToken <- newEmptyMVar
-            atomically $ unGetTQueue (indexQueue $ hiedbWriter se) $
-                \withHieDb -> withHieDb $ \db -> do
-                    removeDependencySrcFiles db
+            atomically $ writeTQueue (indexQueue $ hiedbWriter se) $
+                \withHieDb -> do
+                    withHieDb $ \db ->
+                        removeDependencySrcFiles db
                     putMVar completionToken ()
             readMVar completionToken
         indexPackageHieFiles :: Package -> [Module] -> IO ()

From 54b253af70d470cde4faef3fa72e089698d28478 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Sat, 15 Jul 2023 08:42:40 -0500
Subject: [PATCH 35/50] Add PluginFileType

---
 ghcide/src/Development/IDE/Core/Shake.hs      | 14 +------
 .../src/Development/IDE/LSP/Notifications.hs  |  1 +
 .../src/Development/IDE/Plugin/HLS/GhcIde.hs  |  5 ++-
 hls-plugin-api/src/Ide/Types.hs               | 40 ++++++++++++++++---
 4 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/ghcide/src/Development/IDE/Core/Shake.hs b/ghcide/src/Development/IDE/Core/Shake.hs
index 9852181ed2..18576be31c 100644
--- a/ghcide/src/Development/IDE/Core/Shake.hs
+++ b/ghcide/src/Development/IDE/Core/Shake.hs
@@ -168,7 +168,8 @@ import           Ide.Plugin.Config
 import qualified Ide.PluginUtils                        as HLS
 import           Ide.Types                              (IdePlugins (IdePlugins),
                                                          PluginDescriptor (pluginId),
-                                                         PluginId)
+                                                         PluginId, SourceFileOrigin(..),
+                                                         getSourceFileOrigin)
 import           Language.LSP.Diagnostics
 import           Language.LSP.Protocol.Message
 import           Language.LSP.Protocol.Types
@@ -1128,17 +1129,6 @@ defineEarlyCutOffNoFile recorder f = defineEarlyCutoff recorder $ RuleNoDiagnost
     if file == emptyFilePath then do (hash, res) <- f k; return (Just hash, Just res) else
         fail $ "Rule " ++ show k ++ " should always be called with the empty string for a file"
 
-data SourceFileOrigin = FromProject | FromDependency
-
-getSourceFileOrigin :: NormalizedFilePath -> SourceFileOrigin
-getSourceFileOrigin f =
-    case [".hls", "dependencies"] `isInfixOf` (splitDirectories file) of
-        True -> FromDependency
-        False -> FromProject
-    where
-        file :: FilePath
-        file = fromNormalizedFilePath f
-
 defineEarlyCutoff'
     :: forall k v. IdeRule k v
     => (Maybe Int32 -> [FileDiagnostic] -> Action ()) -- ^ update diagnostics
diff --git a/ghcide/src/Development/IDE/LSP/Notifications.hs b/ghcide/src/Development/IDE/LSP/Notifications.hs
index db6b909735..65fa1f69eb 100644
--- a/ghcide/src/Development/IDE/LSP/Notifications.hs
+++ b/ghcide/src/Development/IDE/LSP/Notifications.hs
@@ -158,6 +158,7 @@ descriptor recorder plId = (defaultPluginDescriptor plId) { pluginNotificationHa
     -- The ghcide descriptors should come last'ish so that the notification handlers
     -- (which restart the Shake build) run after everything else
         pluginPriority = ghcideNotificationsPluginPriority
+    ,   pluginFileType = PluginFileType [FromProject, FromDependency] defaultPluginFileExtensions
     }
 
 ghcideNotificationsPluginPriority :: Natural
diff --git a/ghcide/src/Development/IDE/Plugin/HLS/GhcIde.hs b/ghcide/src/Development/IDE/Plugin/HLS/GhcIde.hs
index d419710d51..c2a2814cd5 100644
--- a/ghcide/src/Development/IDE/Plugin/HLS/GhcIde.hs
+++ b/ghcide/src/Development/IDE/Plugin/HLS/GhcIde.hs
@@ -53,9 +53,10 @@ descriptor plId = (defaultPluginDescriptor plId)
                   <> mkPluginHandler SMethod_TextDocumentDocumentHighlight (\ide _ DocumentHighlightParams{..} ->
                       documentHighlight ide TextDocumentPositionParams{..})
                   <> mkPluginHandler SMethod_TextDocumentReferences (\ide _ params -> references ide params)
-                  <> mkPluginHandler SMethod_WorkspaceSymbol (\ide _ params -> fmap InL <$> wsSymbols ide params),
+                  <> mkPluginHandler SMethod_WorkspaceSymbol (\ide _ params -> fmap InL <$> wsSymbols ide params)
 
-    pluginConfigDescriptor = defaultConfigDescriptor
+  , pluginConfigDescriptor = defaultConfigDescriptor
+  , pluginFileType = PluginFileType [FromProject, FromDependency] defaultPluginFileExtensions
   }
 
 -- ---------------------------------------------------------------------
diff --git a/hls-plugin-api/src/Ide/Types.hs b/hls-plugin-api/src/Ide/Types.hs
index bd35a3312d..79d7bf3323 100644
--- a/hls-plugin-api/src/Ide/Types.hs
+++ b/hls-plugin-api/src/Ide/Types.hs
@@ -24,7 +24,7 @@
 {-# LANGUAGE ViewPatterns               #-}
 module Ide.Types
 ( PluginDescriptor(..), defaultPluginDescriptor, defaultCabalPluginDescriptor
-, defaultPluginPriority
+, defaultPluginPriority, defaultPluginFileExtensions
 , IdeCommand(..)
 , IdeMethod(..)
 , IdeNotification(..)
@@ -37,6 +37,7 @@ module Ide.Types
 , FormattingType(..), FormattingMethod, FormattingHandler, mkFormattingHandlers
 , HasTracing(..)
 , PluginCommand(..), CommandId(..), CommandFunction, mkLspCommand, mkLspCmdId
+, PluginFileType(..)
 , PluginId(..)
 , PluginHandler(..), mkPluginHandler
 , PluginHandlers(..)
@@ -45,6 +46,8 @@ module Ide.Types
 , PluginNotificationHandler(..), mkPluginNotificationHandler
 , PluginNotificationHandlers(..)
 , PluginRequestMethod(..)
+, SourceFileOrigin(..)
+, getSourceFileOrigin
 , getProcessID, getPid
 , installSigUsr1Handler
 , responseError
@@ -73,6 +76,7 @@ import           Data.GADT.Compare
 import           Data.Hashable                 (Hashable)
 import           Data.HashMap.Strict           (HashMap)
 import qualified Data.HashMap.Strict           as HashMap
+import           Data.List                     (isInfixOf)
 import           Data.List.Extra               (find, sortOn)
 import           Data.List.NonEmpty            (NonEmpty (..), toList)
 import qualified Data.Map                      as Map
@@ -268,23 +272,44 @@ data PluginDescriptor (ideState :: *) =
                    , pluginNotificationHandlers :: PluginNotificationHandlers ideState
                    , pluginModifyDynflags :: DynFlagsModifications
                    , pluginCli            :: Maybe (ParserInfo (IdeCommand ideState))
-                   , pluginFileType       :: [T.Text]
+                   , pluginFileType       :: PluginFileType
                    -- ^ File extension of the files the plugin is responsible for.
                    --   The plugin is only allowed to handle files with these extensions.
                    --   When writing handlers, etc. for this plugin it can be assumed that all handled files are of this type.
                    --   The file extension must have a leading '.'.
                    }
 
+data PluginFileType = PluginFileType [SourceFileOrigin] [T.Text]
+
+data SourceFileOrigin = FromProject | FromDependency deriving Eq
+
+getSourceFileOrigin :: NormalizedFilePath -> SourceFileOrigin
+getSourceFileOrigin f =
+    case [".hls", "dependencies"] `isInfixOf` (splitDirectories file) of
+        True -> FromDependency
+        False -> FromProject
+    where
+        file :: FilePath
+        file = fromNormalizedFilePath f
+
 -- | Check whether the given plugin descriptor is responsible for the file with the given path.
 --   Compares the file extension of the file at the given path with the file extension
 --   the plugin is responsible for.
 pluginResponsible :: Uri -> PluginDescriptor c -> Bool
 pluginResponsible uri pluginDesc
     | Just fp <- mfp
-    , T.pack (takeExtension fp) `elem` pluginFileType pluginDesc = True
+    , checkFile (pluginFileType pluginDesc) fp = True
     | otherwise = False
     where
-      mfp = uriToFilePath uri
+        checkFile :: PluginFileType -> NormalizedFilePath -> Bool
+        checkFile (PluginFileType validOrigins validExtensions) fp =
+            getSourceFileOrigin fp `elem` validOrigins
+            &&
+            getExtension fp `elem` validExtensions
+        getExtension :: NormalizedFilePath -> T.Text
+        getExtension = T.pack . takeExtension . fromNormalizedFilePath
+        mfp :: Maybe NormalizedFilePath
+        mfp = uriToNormalizedFilePath $ toNormalizedUri uri
 
 -- | An existential wrapper of 'Properties'
 data CustomConfig = forall r. CustomConfig (Properties r)
@@ -852,7 +877,10 @@ defaultPluginDescriptor plId =
     mempty
     mempty
     Nothing
-    [".hs", ".lhs", ".hs-boot"]
+    (PluginFileType [FromProject] defaultPluginFileExtensions)
+
+defaultPluginFileExtensions :: [T.Text]
+defaultPluginFileExtensions = [".hs", ".lhs", ".hs-boot"]
 
 -- | Set up a plugin descriptor, initialized with default values.
 -- This plugin descriptor is prepared for @.cabal@ files and as such,
@@ -872,7 +900,7 @@ defaultCabalPluginDescriptor plId =
     mempty
     mempty
     Nothing
-    [".cabal"]
+    (PluginFileType [FromProject] [".cabal"])
 
 newtype CommandId = CommandId T.Text
   deriving (Show, Read, Eq, Ord)

From f4ba2b9784c3912ff58a3e2c2ae0cb31ed31da33 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Sat, 15 Jul 2023 08:53:16 -0500
Subject: [PATCH 36/50] Revert "Prevent GhcSession call on open dependency"

This reverts commit 80b9a9630446b7619fde5d234d57cab3e9cb6ab3.
---
 .../src/Development/IDE/Plugin/TypeLenses.hs  | 93 +++++++++----------
 1 file changed, 45 insertions(+), 48 deletions(-)

diff --git a/ghcide/src/Development/IDE/Plugin/TypeLenses.hs b/ghcide/src/Development/IDE/Plugin/TypeLenses.hs
index 71ff810e4b..0056fb0f7b 100644
--- a/ghcide/src/Development/IDE/Plugin/TypeLenses.hs
+++ b/ghcide/src/Development/IDE/Plugin/TypeLenses.hs
@@ -107,55 +107,52 @@ codeLensProvider :: PluginMethodHandler IdeState Method_TextDocumentCodeLens
 codeLensProvider ideState pId CodeLensParams{_textDocument = TextDocumentIdentifier uri} = pluginResponse $ do
     mode <- liftIO $ runAction "codeLens.config" ideState $ usePropertyAction #mode pId properties
     nfp <- getNormalizedFilePath uri
-    case Shake.getSourceFileOrigin nfp of
-        Shake.FromDependency -> pure $ InL []
-        Shake.FromProject -> do
-            env <- hscEnv . fst
-                    <$> (handleMaybeM "Unable to get GhcSession"
-                        $ liftIO
-                        $ runAction "codeLens.GhcSession" ideState (useWithStale GhcSession nfp)
-                        )
-            tmr <- fst <$> (
-                        handleMaybeM "Unable to TypeCheck"
-                      $ liftIO
-                      $ runAction "codeLens.TypeCheck" ideState (useWithStale TypeCheck nfp)
-                      )
-            bindings <- fst <$> (
-                        handleMaybeM "Unable to GetBindings"
-                        $ liftIO
-                        $ runAction "codeLens.GetBindings" ideState (useWithStale GetBindings nfp)
-                        )
-            (gblSigs@(GlobalBindingTypeSigsResult gblSigs'), gblSigsMp) <-
-              handleMaybeM "Unable to GetGlobalBindingTypeSigs"
+    env <- hscEnv . fst
+            <$> (handleMaybeM "Unable to get GhcSession"
+                $ liftIO
+                $ runAction "codeLens.GhcSession" ideState (useWithStale GhcSession nfp)
+                )
+    tmr <- fst <$> (
+                handleMaybeM "Unable to TypeCheck"
               $ liftIO
-              $ runAction "codeLens.GetGlobalBindingTypeSigs" ideState (useWithStale GetGlobalBindingTypeSigs nfp)
-
-            diag <- liftIO $ atomically $ getDiagnostics ideState
-            hDiag <- liftIO $ atomically $ getHiddenDiagnostics ideState
-
-            let toWorkSpaceEdit tedit = WorkspaceEdit (Just $ Map.singleton uri $ tedit) Nothing Nothing
-                generateLensForGlobal mp sig@GlobalBindingTypeSig{gbRendered} = do
-                    range <- toCurrentRange mp =<< srcSpanToRange (gbSrcSpan sig)
-                    tedit <- gblBindingTypeSigToEdit sig (Just gblSigsMp)
-                    let wedit = toWorkSpaceEdit [tedit]
-                    pure $ generateLens pId range (T.pack gbRendered) wedit
-                generateLensFromDiags f =
-                      [ generateLens pId _range title edit
-                      | (dFile, _, dDiag@Diagnostic{_range = _range}) <- diag ++ hDiag
-                      , dFile == nfp
-                      , (title, tedit) <- f dDiag
-                      , let edit = toWorkSpaceEdit tedit
-                      ]
-            -- `suggestLocalSignature` relies on diagnostic, if diagnostics don't have the local signature warning,
-            -- the `bindings` is useless, and if diagnostic has, that means we parsed success, and the `bindings` is fresh.
-            pure $ InL $ case mode of
-                Always ->
-                  mapMaybe (generateLensForGlobal gblSigsMp) gblSigs'
-                    <> generateLensFromDiags
-                        (suggestLocalSignature False (Just env) (Just tmr) (Just bindings)) -- we still need diagnostics for local bindings
-                Exported -> mapMaybe (generateLensForGlobal gblSigsMp) (filter gbExported gblSigs')
-                Diagnostics -> generateLensFromDiags
-                    $ suggestSignature False (Just env) (Just gblSigs) (Just tmr) (Just bindings)
+              $ runAction "codeLens.TypeCheck" ideState (useWithStale TypeCheck nfp)
+              )
+    bindings <- fst <$> (
+                handleMaybeM "Unable to GetBindings"
+                $ liftIO
+                $ runAction "codeLens.GetBindings" ideState (useWithStale GetBindings nfp)
+                )
+    (gblSigs@(GlobalBindingTypeSigsResult gblSigs'), gblSigsMp) <-
+      handleMaybeM "Unable to GetGlobalBindingTypeSigs"
+      $ liftIO
+      $ runAction "codeLens.GetGlobalBindingTypeSigs" ideState (useWithStale GetGlobalBindingTypeSigs nfp)
+
+    diag <- liftIO $ atomically $ getDiagnostics ideState
+    hDiag <- liftIO $ atomically $ getHiddenDiagnostics ideState
+
+    let toWorkSpaceEdit tedit = WorkspaceEdit (Just $ Map.singleton uri $ tedit) Nothing Nothing
+        generateLensForGlobal mp sig@GlobalBindingTypeSig{gbRendered} = do
+            range <- toCurrentRange mp =<< srcSpanToRange (gbSrcSpan sig)
+            tedit <- gblBindingTypeSigToEdit sig (Just gblSigsMp)
+            let wedit = toWorkSpaceEdit [tedit]
+            pure $ generateLens pId range (T.pack gbRendered) wedit
+        generateLensFromDiags f =
+              [ generateLens pId _range title edit
+              | (dFile, _, dDiag@Diagnostic{_range = _range}) <- diag ++ hDiag
+              , dFile == nfp
+              , (title, tedit) <- f dDiag
+              , let edit = toWorkSpaceEdit tedit
+              ]
+    -- `suggestLocalSignature` relies on diagnostic, if diagnostics don't have the local signature warning,
+    -- the `bindings` is useless, and if diagnostic has, that means we parsed success, and the `bindings` is fresh.
+    pure $ InL $ case mode of
+        Always ->
+          mapMaybe (generateLensForGlobal gblSigsMp) gblSigs'
+            <> generateLensFromDiags
+                (suggestLocalSignature False (Just env) (Just tmr) (Just bindings)) -- we still need diagnostics for local bindings
+        Exported -> mapMaybe (generateLensForGlobal gblSigsMp) (filter gbExported gblSigs')
+        Diagnostics -> generateLensFromDiags
+            $ suggestSignature False (Just env) (Just gblSigs) (Just tmr) (Just bindings)
 
 generateLens :: PluginId -> Range -> T.Text -> WorkspaceEdit -> CodeLens
 generateLens pId _range title edit =

From 9a4b009c79a2087c5a3e154a18b556676d34969d Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Mon, 17 Jul 2023 08:00:50 -0500
Subject: [PATCH 37/50] Check package before indexing

---
 cabal.project                                |  2 +-
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 12 +++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/cabal.project b/cabal.project
index 2c750a4edd..65753a4e2d 100644
--- a/cabal.project
+++ b/cabal.project
@@ -41,7 +41,7 @@ packages:
 source-repository-package
   type:git
   location: https://github.com/nlander/HieDb.git
-  tag: 038bc785c80f13615f4ac1ec5066345f2eb999c2
+  tag: f10051a6dc1b809d5f40a45beab92205d1829736
 
 -- Standard location for temporary packages needed for particular environments
 -- For example it is used in the project gitlab mirror to help in the MAcOS M1 build script
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 1e858209ea..132f385c91 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -31,7 +31,7 @@ import           Data.Unique                     (Unique)
 import qualified Data.Unique                     as Unique
 import           Development.IDE.Core.Compile    (indexHieFile)
 import           Development.IDE.Core.Rules      (HieFileCheck(..), Log, checkHieFile)
-import           Development.IDE.Core.Shake      (HieDbWriter(indexQueue), ShakeExtras(hiedbWriter, lspEnv))
+import           Development.IDE.Core.Shake      (HieDbWriter(indexQueue), ShakeExtras(hiedbWriter, lspEnv, withHieDb))
 import           Development.IDE.GHC.Compat
 import qualified Development.IDE.GHC.Compat.Util as Maybes
 import           Development.IDE.GHC.Error       (catchSrcErrors)
@@ -40,7 +40,7 @@ import           Development.IDE.Graph.Classes
 import           Development.IDE.Types.Exports   (ExportsMap, createExportsMap)
 import           Development.IDE.Types.Location  (NormalizedFilePath, toNormalizedFilePath')
 import           Development.IDE.Types.Logger    (Recorder, WithPriority)
-import           HieDb                           (SourceFile(FakeFile), removeDependencySrcFiles)
+import           HieDb                           (SourceFile(FakeFile), lookupPackage, removeDependencySrcFiles)
 import           Language.LSP.Server             (resRootPath)
 import           OpenTelemetry.Eventlog          (withSpan)
 import           System.Directory                (doesDirectoryExist, makeAbsolute)
@@ -147,7 +147,13 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
                   (libraryDir : _) -> libraryDir
                 hieDir :: FilePath
                 hieDir = pkgLibDir </> "extra-compilation-artifacts"
-            traverse_ (indexModuleHieFile hieDir) modules
+                unit :: Unit
+                unit = RealUnit $ Definite $ unitId package
+            moduleRows <- withHieDb se $ \db ->
+                lookupPackage db unit
+            case moduleRows of
+                [] -> traverse_ (indexModuleHieFile hieDir) modules
+                _  -> return ()
         indexModuleHieFile :: FilePath -> Module -> IO ()
         indexModuleHieFile hieDir m = do
             let hiePath :: NormalizedFilePath

From 94d3b073d450328eb9213121cd74c76707ed148b Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Fri, 21 Jul 2023 08:46:30 -0500
Subject: [PATCH 38/50] Disable write and execute permissions for dependency
 sources

---
 ghcide/src/Development/IDE/Core/Actions.hs | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/ghcide/src/Development/IDE/Core/Actions.hs b/ghcide/src/Development/IDE/Core/Actions.hs
index 4d44a88240..1b232a4070 100644
--- a/ghcide/src/Development/IDE/Core/Actions.hs
+++ b/ghcide/src/Development/IDE/Core/Actions.hs
@@ -21,6 +21,7 @@ import           Control.Monad.Extra                  (mapMaybeM)
 import           Control.Monad.Reader
 import           Control.Monad.Trans.Maybe
 import qualified Data.ByteString                      as BS
+import           Data.Function                        ((&))
 import qualified Data.HashMap.Strict                  as HM
 import           Data.Maybe
 import qualified Data.Text                            as T
@@ -42,7 +43,12 @@ import           Language.LSP.Protocol.Types          (DocumentHighlight (..),
                                                        normalizedFilePathToUri,
                                                        uriToNormalizedFilePath)
 import           Language.LSP.Server                  (resRootPath)
-import           System.Directory                     (createDirectoryIfMissing, doesFileExist)
+import           System.Directory                     (createDirectoryIfMissing,
+                                                       doesFileExist,
+                                                       getPermissions,
+                                                       setOwnerExecutable,
+                                                       setOwnerWritable,
+                                                       setPermissions)
 import           System.FilePath                      ((</>), (<.>), takeDirectory)
 
 
@@ -74,6 +80,11 @@ lookupMod HieDbWriter{indexQueue} hieFile moduleName uid _boot = MaybeT $ do
           createDirectoryIfMissing True $ takeDirectory writeOutPath
           moduleSource <- hie_hs_src <$> loadHieFile (mkUpdater nc) hieFile
           BS.writeFile writeOutPath moduleSource
+          fileDefaultPermissions <- getPermissions writeOutPath
+          let filePermissions = fileDefaultPermissions
+                              & setOwnerWritable False
+                              & setOwnerExecutable False
+          setPermissions writeOutPath filePermissions
       liftIO $ atomically $
         unGetTQueue indexQueue $ \withHieDb -> do
           withHieDb $ \db ->

From a5628113187db549078ed4523757e321cc294496 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Wed, 26 Jul 2023 16:50:52 -0500
Subject: [PATCH 39/50] Add gotoDefinition dependency test

---
 test/functional/Definition.hs              | 24 ++++++++++++++++++++++
 test/testdata/definition/Bar.hs            |  5 +++++
 test/testdata/definition/cabal.project     |  5 +++++
 test/testdata/definition/definitions.cabal |  2 ++
 4 files changed, 36 insertions(+)
 create mode 100644 test/testdata/definition/cabal.project

diff --git a/test/functional/Definition.hs b/test/functional/Definition.hs
index 3c32f2cf72..61601f9a1d 100644
--- a/test/functional/Definition.hs
+++ b/test/functional/Definition.hs
@@ -1,8 +1,10 @@
 module Definition (tests) where
 
 import           Control.Lens
+import           Data.List                  (isSuffixOf)
 import           Language.LSP.Protocol.Lens
 import           System.Directory
+import           System.FilePath            (splitDirectories)
 import           Test.Hls
 import           Test.Hls.Command
 
@@ -37,6 +39,28 @@ symbolTests = testGroup "gotoDefinition on symbols"
         liftIO $ do
             fp <- canonicalizePath "test/testdata/definition/Bar.hs"
             defs @?= InL (Definition (InR [Location (filePathToUri fp) expRange]))
+
+        -- gotoDefinition where the definition is in an external
+        -- dependency.
+  ,    testCase "gotoDefinition in dependency" $ runSession hlsCommand fullCaps "test/testdata/definition" $ do
+        doc <- openDoc "Bar.hs" "haskell"
+        defs <- getDefinitions doc (Position 13 12)
+        let expRange = Range (Position 513 0) (Position 513 4)
+        case defs of
+            InL (Definition (InR [Location fp actualRange])) ->
+                liftIO $ do
+                    let locationDirectories :: [String]
+                        locationDirectories =
+                            maybe [] splitDirectories $
+                                uriToFilePath fp
+                    assertBool "empty not found in Data.Set.Internal"
+                        $ ["Data", "Set", "Internal.hs"]
+                            `isSuffixOf` locationDirectories
+                    actualRange @?= expRange
+            wrongLocation ->
+                liftIO $
+                    assertFailure $ "Wrong location for Set.empty: "
+                        ++ show wrongLocation
   ]
 
   -- -----------------------------------
diff --git a/test/testdata/definition/Bar.hs b/test/testdata/definition/Bar.hs
index 9ae116114e..018d8007d2 100644
--- a/test/testdata/definition/Bar.hs
+++ b/test/testdata/definition/Bar.hs
@@ -1,5 +1,7 @@
 module Bar where
 
+import Data.Set (Set, empty)
+
 a = 42
 
 -- These blank lines are here
@@ -7,3 +9,6 @@ a = 42
 -- on a line number larger than
 -- the number of lines in Foo.hs.
 b = 43
+
+emptySet :: Set Integer
+emptySet = empty
diff --git a/test/testdata/definition/cabal.project b/test/testdata/definition/cabal.project
new file mode 100644
index 0000000000..26c41767bf
--- /dev/null
+++ b/test/testdata/definition/cabal.project
@@ -0,0 +1,5 @@
+packages: .
+source-repository-package
+  type:git
+  location: https://github.com/haskell/containers.git
+  tag: cde5e58b12e744ca4742db71443bee6584dfd1e9
diff --git a/test/testdata/definition/definitions.cabal b/test/testdata/definition/definitions.cabal
index cde0040a7e..48f3738747 100644
--- a/test/testdata/definition/definitions.cabal
+++ b/test/testdata/definition/definitions.cabal
@@ -8,3 +8,5 @@ library
   other-modules:    Bar
   default-language: Haskell2010
   build-depends:    base
+                  , containers
+  ghc-options: -fwrite-ide-info

From f0b73609dd84822978c1e14c6957ff376e7c3456 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Fri, 28 Jul 2023 18:21:39 -0500
Subject: [PATCH 40/50] Add check for indexing message

---
 haskell-language-server.cabal              |  1 +
 test/functional/Definition.hs              | 57 +++++++++++++++++++---
 test/testdata/definition/Bar.hs            |  6 +--
 test/testdata/definition/cabal.project     |  4 +-
 test/testdata/definition/definitions.cabal |  2 +-
 5 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/haskell-language-server.cabal b/haskell-language-server.cabal
index 3afbe687fd..2a13374bf5 100644
--- a/haskell-language-server.cabal
+++ b/haskell-language-server.cabal
@@ -568,6 +568,7 @@ test-suite func-test
     , containers
     , unordered-containers
     , row-types
+    , process
 
   hs-source-dirs:     test/functional test/utils
 
diff --git a/test/functional/Definition.hs b/test/functional/Definition.hs
index 61601f9a1d..62e02b7457 100644
--- a/test/functional/Definition.hs
+++ b/test/functional/Definition.hs
@@ -1,10 +1,24 @@
+{-# LANGUAGE DataKinds                #-}
+{-# LANGUAGE DisambiguateRecordFields #-}
+{-# LANGUAGE GADTs                    #-}
+{-# LANGUAGE LambdaCase               #-}
+{-# LANGUAGE NamedFieldPuns           #-}
+{-# LANGUAGE TypeApplications         #-}
+{-# LANGUAGE ViewPatterns             #-}
+
 module Definition (tests) where
 
 import           Control.Lens
+import           Data.Aeson                 (Result (Success), fromJSON)
+import           Data.Bool                  (bool)
 import           Data.List                  (isSuffixOf)
-import           Language.LSP.Protocol.Lens
+import           Data.Proxy                 (Proxy (Proxy))
+import qualified Data.Text                  as T
+import           Language.LSP.Protocol.Lens (uri)
 import           System.Directory
+import           System.Exit                (ExitCode(ExitSuccess))
 import           System.FilePath            (splitDirectories)
+import           System.Process             (readCreateProcessWithExitCode, shell)
 import           Test.Hls
 import           Test.Hls.Command
 
@@ -43,9 +57,13 @@ symbolTests = testGroup "gotoDefinition on symbols"
         -- gotoDefinition where the definition is in an external
         -- dependency.
   ,    testCase "gotoDefinition in dependency" $ runSession hlsCommand fullCaps "test/testdata/definition" $ do
+        liftIO $ do
+            (exitCode, _out, _err) <- readCreateProcessWithExitCode (shell "cabal build") ""
+            exitCode @?= ExitSuccess
         doc <- openDoc "Bar.hs" "haskell"
-        defs <- getDefinitions doc (Position 13 12)
-        let expRange = Range (Position 513 0) (Position 513 4)
+        _mHieFile <- fileDoneIndexing ["Data", "Aeson", "Types", "Internal.hie"]
+        defs <- getDefinitions doc (Position 13 13)
+        let expRange = Range (Position 370 13) (Position 370 16)
         case defs of
             InL (Definition (InR [Location fp actualRange])) ->
                 liftIO $ do
@@ -53,16 +71,43 @@ symbolTests = testGroup "gotoDefinition on symbols"
                         locationDirectories =
                             maybe [] splitDirectories $
                                 uriToFilePath fp
-                    assertBool "empty not found in Data.Set.Internal"
-                        $ ["Data", "Set", "Internal.hs"]
+                    assertBool "empty not found in Data.Aeson.Types.Internal"
+                        $ ["Data", "Aeson", "Types", "Internal.hs"]
                             `isSuffixOf` locationDirectories
                     actualRange @?= expRange
             wrongLocation ->
                 liftIO $
-                    assertFailure $ "Wrong location for Set.empty: "
+                    assertFailure $ "Wrong location for Null: "
                         ++ show wrongLocation
   ]
 
+fileDoneIndexing :: [String] -> Session (Maybe FilePath)
+fileDoneIndexing fpSuffix =
+    skipManyTill anyMessage (indexedFile <|> doneIndexing)
+    where
+        indexedFile :: Session (Maybe FilePath)
+        indexedFile = do
+            NotMess TNotificationMessage{_params} <-
+                customNotification (Proxy @"ghcide/reference/ready")
+            case fromJSON _params of
+                Success fp -> do
+                    let fpDirs :: [String]
+                        fpDirs = splitDirectories fp
+                    bool empty (pure (Just fp)) $
+                        fpSuffix `isSuffixOf` fpDirs
+                other -> error $ "Failed to parse ghcide/reference/ready file: " <> show other
+        doneIndexing :: Session (Maybe FilePath)
+        doneIndexing = satisfyMaybe $ \case
+            FromServerMess SMethod_Progress (TNotificationMessage _ _ (ProgressParams t (preview _workDoneProgressEnd -> Just params))) ->
+                case params of
+                    (WorkDoneProgressEnd _ m) ->
+                        case m of
+                            Just message -> bool Nothing (Just Nothing) $
+                                "Finished indexing" `T.isPrefixOf` message
+                            _ -> Nothing
+                    _ -> Nothing
+            _ -> Nothing
+
   -- -----------------------------------
 
 moduleTests :: TestTree
diff --git a/test/testdata/definition/Bar.hs b/test/testdata/definition/Bar.hs
index 018d8007d2..d6eb6bcc6e 100644
--- a/test/testdata/definition/Bar.hs
+++ b/test/testdata/definition/Bar.hs
@@ -1,6 +1,6 @@
 module Bar where
 
-import Data.Set (Set, empty)
+import Data.Aeson (Value(Null))
 
 a = 42
 
@@ -10,5 +10,5 @@ a = 42
 -- the number of lines in Foo.hs.
 b = 43
 
-emptySet :: Set Integer
-emptySet = empty
+nullValue :: Value
+nullValue = Null
diff --git a/test/testdata/definition/cabal.project b/test/testdata/definition/cabal.project
index 26c41767bf..e6b3b832f5 100644
--- a/test/testdata/definition/cabal.project
+++ b/test/testdata/definition/cabal.project
@@ -1,5 +1,5 @@
 packages: .
 source-repository-package
   type:git
-  location: https://github.com/haskell/containers.git
-  tag: cde5e58b12e744ca4742db71443bee6584dfd1e9
+  location: https://github.com/haskell/aeson.git
+  tag: fc5f5bb067613a273de358f09760b635d6f78c82
diff --git a/test/testdata/definition/definitions.cabal b/test/testdata/definition/definitions.cabal
index 48f3738747..21eb31c28f 100644
--- a/test/testdata/definition/definitions.cabal
+++ b/test/testdata/definition/definitions.cabal
@@ -8,5 +8,5 @@ library
   other-modules:    Bar
   default-language: Haskell2010
   build-depends:    base
-                  , containers
+                  , aeson
   ghc-options: -fwrite-ide-info

From b7a097e14a6b4ee950e67f1df33dc425adf2dcde Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Mon, 31 Jul 2023 20:29:16 -0500
Subject: [PATCH 41/50] Make goto dependency test in ghcide

---
 ghcide/test/data/dependency/Dependency.hs    |  7 +++
 ghcide/test/data/dependency/cabal.project    |  5 ++
 ghcide/test/data/dependency/dependency.cabal | 11 ++++
 ghcide/test/data/dependency/hie.yaml         |  2 +
 ghcide/test/exe/Main.hs                      | 59 ++++++++++++++++++++
 5 files changed, 84 insertions(+)
 create mode 100644 ghcide/test/data/dependency/Dependency.hs
 create mode 100644 ghcide/test/data/dependency/cabal.project
 create mode 100644 ghcide/test/data/dependency/dependency.cabal
 create mode 100644 ghcide/test/data/dependency/hie.yaml

diff --git a/ghcide/test/data/dependency/Dependency.hs b/ghcide/test/data/dependency/Dependency.hs
new file mode 100644
index 0000000000..c65e061761
--- /dev/null
+++ b/ghcide/test/data/dependency/Dependency.hs
@@ -0,0 +1,7 @@
+{-# LANGUAGE OverloadedStrings #-}
+module Dependency where
+
+import Lucid (Attributes, width_)
+
+width4em :: Attributes
+width4em = width_ "4em"
diff --git a/ghcide/test/data/dependency/cabal.project b/ghcide/test/data/dependency/cabal.project
new file mode 100644
index 0000000000..af9b7be88d
--- /dev/null
+++ b/ghcide/test/data/dependency/cabal.project
@@ -0,0 +1,5 @@
+packages: .
+source-repository-package
+  type:git
+  location: https://github.com/chrisdone/lucid.git
+  tag: 3dd3fbf79d58e377e38df27cc67f78ad9fe8e031
diff --git a/ghcide/test/data/dependency/dependency.cabal b/ghcide/test/data/dependency/dependency.cabal
new file mode 100644
index 0000000000..7246e9433d
--- /dev/null
+++ b/ghcide/test/data/dependency/dependency.cabal
@@ -0,0 +1,11 @@
+name:          dependency
+version:       0.1.0.0
+cabal-version: 2.0
+build-type:    Simple
+
+library
+  exposed-modules:  Dependency
+  default-language: Haskell2010
+  build-depends:    base
+                  , lucid2
+  ghc-options: -fwrite-ide-info
diff --git a/ghcide/test/data/dependency/hie.yaml b/ghcide/test/data/dependency/hie.yaml
new file mode 100644
index 0000000000..04cd24395e
--- /dev/null
+++ b/ghcide/test/data/dependency/hie.yaml
@@ -0,0 +1,2 @@
+cradle:
+  cabal:
diff --git a/ghcide/test/exe/Main.hs b/ghcide/test/exe/Main.hs
index 208871a933..93fbdb8cff 100644
--- a/ghcide/test/exe/Main.hs
+++ b/ghcide/test/exe/Main.hs
@@ -41,8 +41,10 @@
 
 module Main (main) where
 
+import           Data.Bool                                (bool)
 import Data.Row
 import           Control.Applicative.Combinators
+import           Control.Applicative.Combinators          as Applicative
 import           Control.Concurrent
 import           Control.Exception                        (bracket_, catch,
                                                            finally)
@@ -224,6 +226,7 @@ main = do
     , codeLensesTests
     , outlineTests
     , highlightTests
+    , gotoDependencyDefinitionTests
     , findDefinitionAndHoverTests
     , pluginSimpleTests
     , pluginParsedResultTests
@@ -1008,6 +1011,62 @@ checkDefs (defToLocation -> defs) mkExpectations = traverse_ check =<< mkExpecta
 canonicalizeLocation :: Location -> IO Location
 canonicalizeLocation (Location uri range) = Location <$> canonicalizeUri uri <*> pure range
 
+gotoDependencyDefinitionTests :: TestTree
+gotoDependencyDefinitionTests =
+    testGroup "gotoDefinition for dependencies"
+        [ dependencyTest
+        ]
+    where
+        dependencyTest :: TestTree
+        dependencyTest = testSessionWithExtraFiles "dependency" "gotoDefinition in lucid" $
+            \dir -> do
+                doc <- openTestDataDoc (dir </> "Dependency" <.> "hs")
+                _mHieFile <- fileDoneIndexing ["Lucid", "Html5.hie"]
+                defs <- getDefinitions doc (Position 6 12)
+                let expRange = Range (Position 1125 0) (Position 1125 6)
+                case defs of
+                    InL (Definition (InR [Location fp actualRange])) ->
+                        liftIO $ do
+                            let locationDirectories :: [String]
+                                locationDirectories =
+                                    maybe [] splitDirectories $
+                                        uriToFilePath fp
+                            assertBool "width_ found in a module that is not Lucid.Html5"
+                                $ ["Lucid", "Html5.hs"]
+                                    `isSuffixOf` locationDirectories
+                            actualRange @?= expRange
+                    wrongLocation ->
+                        liftIO $
+                            assertFailure $ "Wrong location for width_: "
+                                ++ show wrongLocation
+        fileDoneIndexing :: [String] -> Session (Maybe FilePath)
+        fileDoneIndexing fpSuffix =
+            skipManyTill anyMessage (indexedFile <|> doneIndexing)
+            where
+                indexedFile :: Session (Maybe FilePath)
+                indexedFile = do
+                    NotMess TNotificationMessage{_params} <-
+                        customNotification (Proxy @"ghcide/reference/ready")
+                    case A.fromJSON _params of
+                        A.Success fp -> do
+                            let fpDirs :: [String]
+                                fpDirs = splitDirectories fp
+                            bool Applicative.empty (pure (Just fp)) $
+                                fpSuffix `isSuffixOf` fpDirs
+                        other -> error $ "Failed to parse ghcide/reference/ready file: " <> show other
+                doneIndexing :: Session (Maybe FilePath)
+                doneIndexing = satisfyMaybe $ \case
+                    FromServerMess SMethod_Progress (TNotificationMessage _ _ (ProgressParams t (Lens.preview _workDoneProgressEnd -> Just params))) ->
+                        case params of
+                            (WorkDoneProgressEnd _ m) ->
+                                case m of
+                                    Just message -> bool Nothing (Just Nothing) $
+                                        "Finished indexing" `T.isPrefixOf` message
+                                    _ -> Nothing
+                            _ -> Nothing
+                    _ -> Nothing
+
+
 findDefinitionAndHoverTests :: TestTree
 findDefinitionAndHoverTests = let
 

From 11518ce83e0b2b87d57d51770181f5624ad6674d Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Tue, 1 Aug 2023 07:09:54 -0500
Subject: [PATCH 42/50] Use async for dependency test

---
 ghcide/test/data/dependency/Dependency.hs    |  7 ++--
 ghcide/test/data/dependency/cabal.project    |  5 ---
 ghcide/test/data/dependency/dependency.cabal |  2 +-
 ghcide/test/exe/Main.hs                      | 34 +++++++-------------
 4 files changed, 15 insertions(+), 33 deletions(-)
 delete mode 100644 ghcide/test/data/dependency/cabal.project

diff --git a/ghcide/test/data/dependency/Dependency.hs b/ghcide/test/data/dependency/Dependency.hs
index c65e061761..3544ca928c 100644
--- a/ghcide/test/data/dependency/Dependency.hs
+++ b/ghcide/test/data/dependency/Dependency.hs
@@ -1,7 +1,6 @@
-{-# LANGUAGE OverloadedStrings #-}
 module Dependency where
 
-import Lucid (Attributes, width_)
+import Control.Concurrent.Async (AsyncCancelled(..))
 
-width4em :: Attributes
-width4em = width_ "4em"
+asyncCancelled :: AsyncCancelled
+asyncCancelled = AsyncCancelled
diff --git a/ghcide/test/data/dependency/cabal.project b/ghcide/test/data/dependency/cabal.project
deleted file mode 100644
index af9b7be88d..0000000000
--- a/ghcide/test/data/dependency/cabal.project
+++ /dev/null
@@ -1,5 +0,0 @@
-packages: .
-source-repository-package
-  type:git
-  location: https://github.com/chrisdone/lucid.git
-  tag: 3dd3fbf79d58e377e38df27cc67f78ad9fe8e031
diff --git a/ghcide/test/data/dependency/dependency.cabal b/ghcide/test/data/dependency/dependency.cabal
index 7246e9433d..154f6d4f88 100644
--- a/ghcide/test/data/dependency/dependency.cabal
+++ b/ghcide/test/data/dependency/dependency.cabal
@@ -7,5 +7,5 @@ library
   exposed-modules:  Dependency
   default-language: Haskell2010
   build-depends:    base
-                  , lucid2
+                  , async == 2.2.4
   ghc-options: -fwrite-ide-info
diff --git a/ghcide/test/exe/Main.hs b/ghcide/test/exe/Main.hs
index 93fbdb8cff..78bacec7a1 100644
--- a/ghcide/test/exe/Main.hs
+++ b/ghcide/test/exe/Main.hs
@@ -1018,12 +1018,12 @@ gotoDependencyDefinitionTests =
         ]
     where
         dependencyTest :: TestTree
-        dependencyTest = testSessionWithExtraFiles "dependency" "gotoDefinition in lucid" $
+        dependencyTest = testSessionWithExtraFiles "dependency" "gotoDefinition in async" $
             \dir -> do
                 doc <- openTestDataDoc (dir </> "Dependency" <.> "hs")
-                _mHieFile <- fileDoneIndexing ["Lucid", "Html5.hie"]
-                defs <- getDefinitions doc (Position 6 12)
-                let expRange = Range (Position 1125 0) (Position 1125 6)
+                _hieFile <- fileDoneIndexing ["Control", "Concurrent", "Async.hie"]
+                defs <- getDefinitions doc (Position 5 20)
+                let expRange = Range (Position 430 22) (Position 430 36)
                 case defs of
                     InL (Definition (InR [Location fp actualRange])) ->
                         liftIO $ do
@@ -1031,19 +1031,19 @@ gotoDependencyDefinitionTests =
                                 locationDirectories =
                                     maybe [] splitDirectories $
                                         uriToFilePath fp
-                            assertBool "width_ found in a module that is not Lucid.Html5"
-                                $ ["Lucid", "Html5.hs"]
+                            assertBool "AsyncCancelled found in a module that is not Control.Concurrent.Async"
+                                $ ["Control", "Concurrent", "Async.hs"]
                                     `isSuffixOf` locationDirectories
                             actualRange @?= expRange
                     wrongLocation ->
                         liftIO $
-                            assertFailure $ "Wrong location for width_: "
+                            assertFailure $ "Wrong location for AsyncCancelled: "
                                 ++ show wrongLocation
-        fileDoneIndexing :: [String] -> Session (Maybe FilePath)
+        fileDoneIndexing :: [String] -> Session FilePath
         fileDoneIndexing fpSuffix =
-            skipManyTill anyMessage (indexedFile <|> doneIndexing)
+            skipManyTill anyMessage indexedFile
             where
-                indexedFile :: Session (Maybe FilePath)
+                indexedFile :: Session FilePath
                 indexedFile = do
                     NotMess TNotificationMessage{_params} <-
                         customNotification (Proxy @"ghcide/reference/ready")
@@ -1051,21 +1051,9 @@ gotoDependencyDefinitionTests =
                         A.Success fp -> do
                             let fpDirs :: [String]
                                 fpDirs = splitDirectories fp
-                            bool Applicative.empty (pure (Just fp)) $
+                            bool Applicative.empty (pure fp) $
                                 fpSuffix `isSuffixOf` fpDirs
                         other -> error $ "Failed to parse ghcide/reference/ready file: " <> show other
-                doneIndexing :: Session (Maybe FilePath)
-                doneIndexing = satisfyMaybe $ \case
-                    FromServerMess SMethod_Progress (TNotificationMessage _ _ (ProgressParams t (Lens.preview _workDoneProgressEnd -> Just params))) ->
-                        case params of
-                            (WorkDoneProgressEnd _ m) ->
-                                case m of
-                                    Just message -> bool Nothing (Just Nothing) $
-                                        "Finished indexing" `T.isPrefixOf` message
-                                    _ -> Nothing
-                            _ -> Nothing
-                    _ -> Nothing
-
 
 findDefinitionAndHoverTests :: TestTree
 findDefinitionAndHoverTests = let

From 462f81136be342897d9d5684fef973ef67b3bac7 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Tue, 1 Aug 2023 07:13:19 -0500
Subject: [PATCH 43/50] Revert "Add check for indexing message"

This reverts commit f0b73609dd84822978c1e14c6957ff376e7c3456.
---
 haskell-language-server.cabal              |  1 -
 test/functional/Definition.hs              | 57 +++-------------------
 test/testdata/definition/Bar.hs            |  6 +--
 test/testdata/definition/cabal.project     |  4 +-
 test/testdata/definition/definitions.cabal |  2 +-
 5 files changed, 12 insertions(+), 58 deletions(-)

diff --git a/haskell-language-server.cabal b/haskell-language-server.cabal
index 2a13374bf5..3afbe687fd 100644
--- a/haskell-language-server.cabal
+++ b/haskell-language-server.cabal
@@ -568,7 +568,6 @@ test-suite func-test
     , containers
     , unordered-containers
     , row-types
-    , process
 
   hs-source-dirs:     test/functional test/utils
 
diff --git a/test/functional/Definition.hs b/test/functional/Definition.hs
index 62e02b7457..61601f9a1d 100644
--- a/test/functional/Definition.hs
+++ b/test/functional/Definition.hs
@@ -1,24 +1,10 @@
-{-# LANGUAGE DataKinds                #-}
-{-# LANGUAGE DisambiguateRecordFields #-}
-{-# LANGUAGE GADTs                    #-}
-{-# LANGUAGE LambdaCase               #-}
-{-# LANGUAGE NamedFieldPuns           #-}
-{-# LANGUAGE TypeApplications         #-}
-{-# LANGUAGE ViewPatterns             #-}
-
 module Definition (tests) where
 
 import           Control.Lens
-import           Data.Aeson                 (Result (Success), fromJSON)
-import           Data.Bool                  (bool)
 import           Data.List                  (isSuffixOf)
-import           Data.Proxy                 (Proxy (Proxy))
-import qualified Data.Text                  as T
-import           Language.LSP.Protocol.Lens (uri)
+import           Language.LSP.Protocol.Lens
 import           System.Directory
-import           System.Exit                (ExitCode(ExitSuccess))
 import           System.FilePath            (splitDirectories)
-import           System.Process             (readCreateProcessWithExitCode, shell)
 import           Test.Hls
 import           Test.Hls.Command
 
@@ -57,13 +43,9 @@ symbolTests = testGroup "gotoDefinition on symbols"
         -- gotoDefinition where the definition is in an external
         -- dependency.
   ,    testCase "gotoDefinition in dependency" $ runSession hlsCommand fullCaps "test/testdata/definition" $ do
-        liftIO $ do
-            (exitCode, _out, _err) <- readCreateProcessWithExitCode (shell "cabal build") ""
-            exitCode @?= ExitSuccess
         doc <- openDoc "Bar.hs" "haskell"
-        _mHieFile <- fileDoneIndexing ["Data", "Aeson", "Types", "Internal.hie"]
-        defs <- getDefinitions doc (Position 13 13)
-        let expRange = Range (Position 370 13) (Position 370 16)
+        defs <- getDefinitions doc (Position 13 12)
+        let expRange = Range (Position 513 0) (Position 513 4)
         case defs of
             InL (Definition (InR [Location fp actualRange])) ->
                 liftIO $ do
@@ -71,43 +53,16 @@ symbolTests = testGroup "gotoDefinition on symbols"
                         locationDirectories =
                             maybe [] splitDirectories $
                                 uriToFilePath fp
-                    assertBool "empty not found in Data.Aeson.Types.Internal"
-                        $ ["Data", "Aeson", "Types", "Internal.hs"]
+                    assertBool "empty not found in Data.Set.Internal"
+                        $ ["Data", "Set", "Internal.hs"]
                             `isSuffixOf` locationDirectories
                     actualRange @?= expRange
             wrongLocation ->
                 liftIO $
-                    assertFailure $ "Wrong location for Null: "
+                    assertFailure $ "Wrong location for Set.empty: "
                         ++ show wrongLocation
   ]
 
-fileDoneIndexing :: [String] -> Session (Maybe FilePath)
-fileDoneIndexing fpSuffix =
-    skipManyTill anyMessage (indexedFile <|> doneIndexing)
-    where
-        indexedFile :: Session (Maybe FilePath)
-        indexedFile = do
-            NotMess TNotificationMessage{_params} <-
-                customNotification (Proxy @"ghcide/reference/ready")
-            case fromJSON _params of
-                Success fp -> do
-                    let fpDirs :: [String]
-                        fpDirs = splitDirectories fp
-                    bool empty (pure (Just fp)) $
-                        fpSuffix `isSuffixOf` fpDirs
-                other -> error $ "Failed to parse ghcide/reference/ready file: " <> show other
-        doneIndexing :: Session (Maybe FilePath)
-        doneIndexing = satisfyMaybe $ \case
-            FromServerMess SMethod_Progress (TNotificationMessage _ _ (ProgressParams t (preview _workDoneProgressEnd -> Just params))) ->
-                case params of
-                    (WorkDoneProgressEnd _ m) ->
-                        case m of
-                            Just message -> bool Nothing (Just Nothing) $
-                                "Finished indexing" `T.isPrefixOf` message
-                            _ -> Nothing
-                    _ -> Nothing
-            _ -> Nothing
-
   -- -----------------------------------
 
 moduleTests :: TestTree
diff --git a/test/testdata/definition/Bar.hs b/test/testdata/definition/Bar.hs
index d6eb6bcc6e..018d8007d2 100644
--- a/test/testdata/definition/Bar.hs
+++ b/test/testdata/definition/Bar.hs
@@ -1,6 +1,6 @@
 module Bar where
 
-import Data.Aeson (Value(Null))
+import Data.Set (Set, empty)
 
 a = 42
 
@@ -10,5 +10,5 @@ a = 42
 -- the number of lines in Foo.hs.
 b = 43
 
-nullValue :: Value
-nullValue = Null
+emptySet :: Set Integer
+emptySet = empty
diff --git a/test/testdata/definition/cabal.project b/test/testdata/definition/cabal.project
index e6b3b832f5..26c41767bf 100644
--- a/test/testdata/definition/cabal.project
+++ b/test/testdata/definition/cabal.project
@@ -1,5 +1,5 @@
 packages: .
 source-repository-package
   type:git
-  location: https://github.com/haskell/aeson.git
-  tag: fc5f5bb067613a273de358f09760b635d6f78c82
+  location: https://github.com/haskell/containers.git
+  tag: cde5e58b12e744ca4742db71443bee6584dfd1e9
diff --git a/test/testdata/definition/definitions.cabal b/test/testdata/definition/definitions.cabal
index 21eb31c28f..48f3738747 100644
--- a/test/testdata/definition/definitions.cabal
+++ b/test/testdata/definition/definitions.cabal
@@ -8,5 +8,5 @@ library
   other-modules:    Bar
   default-language: Haskell2010
   build-depends:    base
-                  , aeson
+                  , containers
   ghc-options: -fwrite-ide-info

From 38e6cf540f9d8fc7a6e279a449d53fd4f900a760 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Tue, 1 Aug 2023 07:13:45 -0500
Subject: [PATCH 44/50] Revert "Add gotoDefinition dependency test"

This reverts commit a5628113187db549078ed4523757e321cc294496.
---
 test/functional/Definition.hs              | 24 ----------------------
 test/testdata/definition/Bar.hs            |  5 -----
 test/testdata/definition/cabal.project     |  5 -----
 test/testdata/definition/definitions.cabal |  2 --
 4 files changed, 36 deletions(-)
 delete mode 100644 test/testdata/definition/cabal.project

diff --git a/test/functional/Definition.hs b/test/functional/Definition.hs
index 61601f9a1d..3c32f2cf72 100644
--- a/test/functional/Definition.hs
+++ b/test/functional/Definition.hs
@@ -1,10 +1,8 @@
 module Definition (tests) where
 
 import           Control.Lens
-import           Data.List                  (isSuffixOf)
 import           Language.LSP.Protocol.Lens
 import           System.Directory
-import           System.FilePath            (splitDirectories)
 import           Test.Hls
 import           Test.Hls.Command
 
@@ -39,28 +37,6 @@ symbolTests = testGroup "gotoDefinition on symbols"
         liftIO $ do
             fp <- canonicalizePath "test/testdata/definition/Bar.hs"
             defs @?= InL (Definition (InR [Location (filePathToUri fp) expRange]))
-
-        -- gotoDefinition where the definition is in an external
-        -- dependency.
-  ,    testCase "gotoDefinition in dependency" $ runSession hlsCommand fullCaps "test/testdata/definition" $ do
-        doc <- openDoc "Bar.hs" "haskell"
-        defs <- getDefinitions doc (Position 13 12)
-        let expRange = Range (Position 513 0) (Position 513 4)
-        case defs of
-            InL (Definition (InR [Location fp actualRange])) ->
-                liftIO $ do
-                    let locationDirectories :: [String]
-                        locationDirectories =
-                            maybe [] splitDirectories $
-                                uriToFilePath fp
-                    assertBool "empty not found in Data.Set.Internal"
-                        $ ["Data", "Set", "Internal.hs"]
-                            `isSuffixOf` locationDirectories
-                    actualRange @?= expRange
-            wrongLocation ->
-                liftIO $
-                    assertFailure $ "Wrong location for Set.empty: "
-                        ++ show wrongLocation
   ]
 
   -- -----------------------------------
diff --git a/test/testdata/definition/Bar.hs b/test/testdata/definition/Bar.hs
index 018d8007d2..9ae116114e 100644
--- a/test/testdata/definition/Bar.hs
+++ b/test/testdata/definition/Bar.hs
@@ -1,7 +1,5 @@
 module Bar where
 
-import Data.Set (Set, empty)
-
 a = 42
 
 -- These blank lines are here
@@ -9,6 +7,3 @@ a = 42
 -- on a line number larger than
 -- the number of lines in Foo.hs.
 b = 43
-
-emptySet :: Set Integer
-emptySet = empty
diff --git a/test/testdata/definition/cabal.project b/test/testdata/definition/cabal.project
deleted file mode 100644
index 26c41767bf..0000000000
--- a/test/testdata/definition/cabal.project
+++ /dev/null
@@ -1,5 +0,0 @@
-packages: .
-source-repository-package
-  type:git
-  location: https://github.com/haskell/containers.git
-  tag: cde5e58b12e744ca4742db71443bee6584dfd1e9
diff --git a/test/testdata/definition/definitions.cabal b/test/testdata/definition/definitions.cabal
index 48f3738747..cde0040a7e 100644
--- a/test/testdata/definition/definitions.cabal
+++ b/test/testdata/definition/definitions.cabal
@@ -8,5 +8,3 @@ library
   other-modules:    Bar
   default-language: Haskell2010
   build-depends:    base
-                  , containers
-  ghc-options: -fwrite-ide-info

From 44cfd6924f1bd56b37199bf49ecb9a697585db2a Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Tue, 1 Aug 2023 16:24:33 -0500
Subject: [PATCH 45/50] Calculate transitive dependencies

---
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 23 +++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 132f385c91..769a2b24d6 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -179,7 +179,28 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
         packages = Set.fromList
             $ map Package
             $ Map.elems
-            $ getUnitInfoMap hscEnv
+            $ Map.filterWithKey (\uid _ -> uid `Set.member` dependencyIds) unitInfoMap
+            where
+                unitInfoMap :: UnitInfoMap
+                unitInfoMap = getUnitInfoMap hscEnv
+                dependencyIds :: Set UnitId
+                dependencyIds =
+                    calculateTransitiveDependencies unitInfoMap directDependencyIds directDependencyIds
+                directDependencyIds :: Set UnitId
+                directDependencyIds = Set.fromList $ map toUnitId $ explicitUnits $ unitState hscEnv
+        calculateTransitiveDependencies :: UnitInfoMap -> Set UnitId -> Set UnitId -> Set UnitId
+        calculateTransitiveDependencies unitInfoMap allDependencies newDepencencies
+            | Set.null newDepencencies = allDependencies
+            | otherwise = calculateTransitiveDependencies unitInfoMap nextAll nextNew
+            where
+                nextAll :: Set UnitId
+                nextAll = Set.union allDependencies nextNew
+                nextNew :: Set UnitId
+                nextNew = flip Set.difference allDependencies
+                    $ Set.unions
+                    $ map (Set.fromList . unitDepends)
+                    $ Map.elems
+                    $ Map.filterWithKey (\uid _ -> uid `Set.member` newDepencencies) unitInfoMap
         getModulesForPackage :: Package -> [Module]
         getModulesForPackage (Package package) =
             map makeModule allModules

From 39ba758057f96122c8bfa3c9ae69a0415c9bd32a Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 3 Aug 2023 17:17:31 -0500
Subject: [PATCH 46/50] Fix indexHiefile ready message

---
 ghcide/src/Development/IDE/Core/Compile.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ghcide/src/Development/IDE/Core/Compile.hs b/ghcide/src/Development/IDE/Core/Compile.hs
index 08294c4b5f..c995432482 100644
--- a/ghcide/src/Development/IDE/Core/Compile.hs
+++ b/ghcide/src/Development/IDE/Core/Compile.hs
@@ -984,7 +984,9 @@ indexHieFile se hiePath sourceFile !hash hf = do
       whenJust (lspEnv se) $ \env -> LSP.runLspT env $
         when (coerce $ ideTesting se) $
           LSP.sendNotification (LSP.SMethod_CustomMethod (Proxy @"ghcide/reference/ready")) $
-            toJSON $ fromNormalizedFilePath hiePath
+            toJSON $ case sourceFile of
+              HieDb.RealFile sourceFilePath -> sourceFilePath
+              HieDb.FakeFile _ -> fromNormalizedFilePath hiePath
       whenJust mdone $ \done ->
         modifyVar_ indexProgressToken $ \tok -> do
           whenJust (lspEnv se) $ \env -> LSP.runLspT env $

From 3e9d307dfce23213c58833093f07dff931d9048e Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 3 Aug 2023 20:53:17 -0500
Subject: [PATCH 47/50] Index dependencies asynchronously

---
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 769a2b24d6..b7d1d1222e 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -87,7 +87,7 @@ newHscEnvEq cradlePath recorder se hscEnv0 deps = do
 newHscEnvEqWithImportPaths :: Maybe (Set FilePath) -> Recorder (WithPriority Log) -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
 newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
 
-    indexDependencyHieFiles
+    _ <- async indexDependencyHieFiles
 
     envUnique <- Unique.newUnique
 

From 4cae6403bae438f148a32d9310f84ed0d47b04e2 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Thu, 3 Aug 2023 21:34:40 -0500
Subject: [PATCH 48/50] Don't wait for dependency file unindexing

---
 ghcide/src/Development/IDE/Types/HscEnvEq.hs | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index b7d1d1222e..844f021ee9 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -13,7 +13,6 @@ module Development.IDE.Types.HscEnvEq
 
 
 import           Control.Concurrent.Async        (Async, async, waitCatch)
-import           Control.Concurrent.MVar         (newEmptyMVar, putMVar, readMVar)
 import           Control.Concurrent.STM          (atomically)
 import           Control.Concurrent.STM.TQueue   (writeTQueue)
 import           Control.Concurrent.Strict       (modifyVar, newVar)
@@ -131,14 +130,11 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
             projectDir <- resRootPath =<< lspEnv se
             pure $ projectDir </> ".hls"
         deleteMissingDependencySources :: IO ()
-        deleteMissingDependencySources = do
-            completionToken <- newEmptyMVar
+        deleteMissingDependencySources =
             atomically $ writeTQueue (indexQueue $ hiedbWriter se) $
-                \withHieDb -> do
+                \withHieDb ->
                     withHieDb $ \db ->
                         removeDependencySrcFiles db
-                    putMVar completionToken ()
-            readMVar completionToken
         indexPackageHieFiles :: Package -> [Module] -> IO ()
         indexPackageHieFiles (Package package) modules = do
             let pkgLibDir :: FilePath

From 7f8038aadf69379de27f0cba3e5cab725c4a5d92 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Fri, 4 Aug 2023 07:10:56 -0500
Subject: [PATCH 49/50] Move dependency indexing to its own module

---
 ghcide/ghcide.cabal                           |   1 +
 .../src/Development/IDE/Core/Dependencies.hs  | 116 ++++++++++++++++++
 ghcide/src/Development/IDE/Types/HscEnvEq.hs  | 108 +---------------
 3 files changed, 123 insertions(+), 102 deletions(-)
 create mode 100644 ghcide/src/Development/IDE/Core/Dependencies.hs

diff --git a/ghcide/ghcide.cabal b/ghcide/ghcide.cabal
index 253187fa0e..045ca7b512 100644
--- a/ghcide/ghcide.cabal
+++ b/ghcide/ghcide.cabal
@@ -150,6 +150,7 @@ library
         Development.IDE.Core.Actions
         Development.IDE.Main.HeapStats
         Development.IDE.Core.Debouncer
+        Development.IDE.Core.Dependencies
         Development.IDE.Core.FileStore
         Development.IDE.Core.FileUtils
         Development.IDE.Core.IdeConfiguration
diff --git a/ghcide/src/Development/IDE/Core/Dependencies.hs b/ghcide/src/Development/IDE/Core/Dependencies.hs
new file mode 100644
index 0000000000..0cbbe168a1
--- /dev/null
+++ b/ghcide/src/Development/IDE/Core/Dependencies.hs
@@ -0,0 +1,116 @@
+module Development.IDE.Core.Dependencies
+  ( indexDependencyHieFiles
+  ) where
+
+import           Control.Concurrent.STM          (atomically)
+import           Control.Concurrent.STM.TQueue   (writeTQueue)
+import           Control.Monad                   (unless, void)
+import           Data.Foldable                   (traverse_)
+import qualified Data.Map                        as Map
+import           Data.Maybe                      (isNothing)
+import           Data.Set                        (Set)
+import qualified Data.Set                        as Set
+import           Development.IDE.Core.Compile    (indexHieFile)
+import           Development.IDE.Core.Rules      (HieFileCheck(..), Log, checkHieFile)
+import           Development.IDE.Core.Shake      (HieDbWriter(indexQueue), ShakeExtras(hiedbWriter, lspEnv, withHieDb))
+import qualified Development.IDE.GHC.Compat      as GHC
+import           Development.IDE.Types.Location  (NormalizedFilePath, toNormalizedFilePath')
+import           Development.IDE.Types.Logger    (Recorder, WithPriority)
+import           HieDb                           (SourceFile(FakeFile), lookupPackage, removeDependencySrcFiles)
+import           Language.LSP.Server             (resRootPath)
+import           System.Directory                (doesDirectoryExist)
+import           System.FilePath                 ((</>), (<.>))
+
+newtype Package = Package GHC.UnitInfo deriving Eq
+instance Ord Package where
+  compare (Package u1) (Package u2) = compare (GHC.unitId u1) (GHC.unitId u2)
+
+indexDependencyHieFiles :: Recorder (WithPriority Log) -> ShakeExtras -> GHC.HscEnv -> IO ()
+indexDependencyHieFiles recorder se hscEnv = do
+    dotHlsDirExists <- maybe (pure False) doesDirectoryExist mHlsDir
+    unless dotHlsDirExists deleteMissingDependencySources
+    void $ Map.traverseWithKey indexPackageHieFiles packagesWithModules
+    where
+        mHlsDir :: Maybe FilePath
+        mHlsDir = do
+            projectDir <- resRootPath =<< lspEnv se
+            pure $ projectDir </> ".hls"
+        deleteMissingDependencySources :: IO ()
+        deleteMissingDependencySources =
+            atomically $ writeTQueue (indexQueue $ hiedbWriter se) $
+                \withHieDb ->
+                    withHieDb $ \db ->
+                        removeDependencySrcFiles db
+        indexPackageHieFiles :: Package -> [GHC.Module] -> IO ()
+        indexPackageHieFiles (Package package) modules = do
+            let pkgLibDir :: FilePath
+                pkgLibDir = case GHC.unitLibraryDirs package of
+                  [] -> ""
+                  (libraryDir : _) -> libraryDir
+                hieDir :: FilePath
+                hieDir = pkgLibDir </> "extra-compilation-artifacts"
+                unit :: GHC.Unit
+                unit = GHC.RealUnit $ GHC.Definite $ GHC.unitId package
+            moduleRows <- withHieDb se $ \db ->
+                lookupPackage db unit
+            case moduleRows of
+                [] -> traverse_ (indexModuleHieFile hieDir) modules
+                _  -> return ()
+        indexModuleHieFile :: FilePath -> GHC.Module -> IO ()
+        indexModuleHieFile hieDir m = do
+            let hiePath :: NormalizedFilePath
+                hiePath = toNormalizedFilePath' $
+                  hieDir </> GHC.moduleNameSlashes (GHC.moduleName m) <.> "hie"
+            hieCheck <- checkHieFile recorder se "newHscEnvEqWithImportPaths" hiePath
+            case hieCheck of
+                HieFileMissing -> return ()
+                HieAlreadyIndexed -> return ()
+                CouldNotLoadHie _e -> return ()
+                DoIndexing hash hie ->
+                    indexHieFile se hiePath (FakeFile Nothing) hash hie
+        packagesWithModules :: Map.Map Package [GHC.Module]
+        packagesWithModules = Map.fromSet getModulesForPackage packages
+        packages :: Set Package
+        packages = Set.fromList
+            $ map Package
+            $ Map.elems
+            $ Map.filterWithKey (\uid _ -> uid `Set.member` dependencyIds) unitInfoMap
+            where
+                unitInfoMap :: GHC.UnitInfoMap
+                unitInfoMap = GHC.getUnitInfoMap hscEnv
+                dependencyIds :: Set GHC.UnitId
+                dependencyIds =
+                    calculateTransitiveDependencies unitInfoMap directDependencyIds directDependencyIds
+                directDependencyIds :: Set GHC.UnitId
+                directDependencyIds = Set.fromList
+                    $ map GHC.toUnitId
+                    $ GHC.explicitUnits
+                    $ GHC.unitState hscEnv
+
+calculateTransitiveDependencies :: GHC.UnitInfoMap -> Set GHC.UnitId -> Set GHC.UnitId -> Set GHC.UnitId
+calculateTransitiveDependencies unitInfoMap allDependencies newDepencencies
+    | Set.null newDepencencies = allDependencies
+    | otherwise = calculateTransitiveDependencies unitInfoMap nextAll nextNew
+    where
+        nextAll :: Set GHC.UnitId
+        nextAll = Set.union allDependencies nextNew
+        nextNew :: Set GHC.UnitId
+        nextNew = flip Set.difference allDependencies
+            $ Set.unions
+            $ map (Set.fromList . GHC.unitDepends)
+            $ Map.elems
+            $ Map.filterWithKey (\uid _ -> uid `Set.member` newDepencencies) unitInfoMap
+
+getModulesForPackage :: Package -> [GHC.Module]
+getModulesForPackage (Package package) =
+    map makeModule allModules
+    where
+        allModules :: [GHC.ModuleName]
+        allModules = map fst
+            ( filter (isNothing . snd)
+            $ GHC.unitExposedModules package
+            )
+            ++ GHC.unitHiddenModules package
+        makeModule :: GHC.ModuleName
+                   -> GHC.Module
+        makeModule = GHC.mkModule (GHC.unitInfoId package)
diff --git a/ghcide/src/Development/IDE/Types/HscEnvEq.hs b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
index 844f021ee9..2800cd2be8 100644
--- a/ghcide/src/Development/IDE/Types/HscEnvEq.hs
+++ b/ghcide/src/Development/IDE/Types/HscEnvEq.hs
@@ -13,36 +13,27 @@ module Development.IDE.Types.HscEnvEq
 
 
 import           Control.Concurrent.Async        (Async, async, waitCatch)
-import           Control.Concurrent.STM          (atomically)
-import           Control.Concurrent.STM.TQueue   (writeTQueue)
 import           Control.Concurrent.Strict       (modifyVar, newVar)
 import           Control.DeepSeq                 (force)
 import           Control.Exception               (evaluate, mask, throwIO)
-import           Control.Monad                   (unless)
-import           Control.Monad.Extra             (eitherM, join, mapMaybeM, void)
+import           Control.Monad.Extra             (eitherM, join, mapMaybeM)
 import           Data.Either                     (fromRight)
-import           Data.Foldable                   (traverse_)
-import qualified Data.Map                        as Map
-import           Data.Maybe                      (isNothing)
 import           Data.Set                        (Set)
 import qualified Data.Set                        as Set
 import           Data.Unique                     (Unique)
 import qualified Data.Unique                     as Unique
-import           Development.IDE.Core.Compile    (indexHieFile)
-import           Development.IDE.Core.Rules      (HieFileCheck(..), Log, checkHieFile)
-import           Development.IDE.Core.Shake      (HieDbWriter(indexQueue), ShakeExtras(hiedbWriter, lspEnv, withHieDb))
+import           Development.IDE.Core.Dependencies (indexDependencyHieFiles)
+import           Development.IDE.Core.Rules      (Log)
+import           Development.IDE.Core.Shake      (ShakeExtras)
 import           Development.IDE.GHC.Compat
 import qualified Development.IDE.GHC.Compat.Util as Maybes
 import           Development.IDE.GHC.Error       (catchSrcErrors)
 import           Development.IDE.GHC.Util        (lookupPackageConfig)
 import           Development.IDE.Graph.Classes
 import           Development.IDE.Types.Exports   (ExportsMap, createExportsMap)
-import           Development.IDE.Types.Location  (NormalizedFilePath, toNormalizedFilePath')
 import           Development.IDE.Types.Logger    (Recorder, WithPriority)
-import           HieDb                           (SourceFile(FakeFile), lookupPackage, removeDependencySrcFiles)
-import           Language.LSP.Server             (resRootPath)
 import           OpenTelemetry.Eventlog          (withSpan)
-import           System.Directory                (doesDirectoryExist, makeAbsolute)
+import           System.Directory                (makeAbsolute)
 import           System.FilePath
 
 -- | An 'HscEnv' with equality. Two values are considered equal
@@ -86,7 +77,7 @@ newHscEnvEq cradlePath recorder se hscEnv0 deps = do
 newHscEnvEqWithImportPaths :: Maybe (Set FilePath) -> Recorder (WithPriority Log) -> ShakeExtras -> HscEnv -> [(UnitId, DynFlags)] -> IO HscEnvEq
 newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
 
-    _ <- async indexDependencyHieFiles
+    _ <- async $ indexDependencyHieFiles recorder se hscEnv
 
     envUnique <- Unique.newUnique
 
@@ -120,48 +111,6 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
 
     return HscEnvEq{..}
     where
-        indexDependencyHieFiles :: IO ()
-        indexDependencyHieFiles = do
-            dotHlsDirExists <- maybe (pure False) doesDirectoryExist mHlsDir
-            unless dotHlsDirExists deleteMissingDependencySources
-            void $ Map.traverseWithKey indexPackageHieFiles packagesWithModules
-        mHlsDir :: Maybe FilePath
-        mHlsDir = do
-            projectDir <- resRootPath =<< lspEnv se
-            pure $ projectDir </> ".hls"
-        deleteMissingDependencySources :: IO ()
-        deleteMissingDependencySources =
-            atomically $ writeTQueue (indexQueue $ hiedbWriter se) $
-                \withHieDb ->
-                    withHieDb $ \db ->
-                        removeDependencySrcFiles db
-        indexPackageHieFiles :: Package -> [Module] -> IO ()
-        indexPackageHieFiles (Package package) modules = do
-            let pkgLibDir :: FilePath
-                pkgLibDir = case unitLibraryDirs package of
-                  [] -> ""
-                  (libraryDir : _) -> libraryDir
-                hieDir :: FilePath
-                hieDir = pkgLibDir </> "extra-compilation-artifacts"
-                unit :: Unit
-                unit = RealUnit $ Definite $ unitId package
-            moduleRows <- withHieDb se $ \db ->
-                lookupPackage db unit
-            case moduleRows of
-                [] -> traverse_ (indexModuleHieFile hieDir) modules
-                _  -> return ()
-        indexModuleHieFile :: FilePath -> Module -> IO ()
-        indexModuleHieFile hieDir m = do
-            let hiePath :: NormalizedFilePath
-                hiePath = toNormalizedFilePath' $
-                  hieDir </> moduleNameSlashes (moduleName m) <.> "hie"
-            hieCheck <- checkHieFile recorder se "newHscEnvEqWithImportPaths" hiePath
-            case hieCheck of
-                HieFileMissing -> return ()
-                HieAlreadyIndexed -> return ()
-                CouldNotLoadHie _e -> return ()
-                DoIndexing hash hie ->
-                    indexHieFile se hiePath (FakeFile Nothing) hash hie
         loadModIface :: Module -> IO (Maybe ModIface)
         loadModIface m = do
             modIface <- initIfaceLoad hscEnv $
@@ -169,51 +118,6 @@ newHscEnvEqWithImportPaths envImportPaths recorder se hscEnv deps = do
             return $ case modIface of
                 Maybes.Failed    _r -> Nothing
                 Maybes.Succeeded mi -> Just mi
-        packagesWithModules :: Map.Map Package [Module]
-        packagesWithModules = Map.fromSet getModulesForPackage packages
-        packages :: Set Package
-        packages = Set.fromList
-            $ map Package
-            $ Map.elems
-            $ Map.filterWithKey (\uid _ -> uid `Set.member` dependencyIds) unitInfoMap
-            where
-                unitInfoMap :: UnitInfoMap
-                unitInfoMap = getUnitInfoMap hscEnv
-                dependencyIds :: Set UnitId
-                dependencyIds =
-                    calculateTransitiveDependencies unitInfoMap directDependencyIds directDependencyIds
-                directDependencyIds :: Set UnitId
-                directDependencyIds = Set.fromList $ map toUnitId $ explicitUnits $ unitState hscEnv
-        calculateTransitiveDependencies :: UnitInfoMap -> Set UnitId -> Set UnitId -> Set UnitId
-        calculateTransitiveDependencies unitInfoMap allDependencies newDepencencies
-            | Set.null newDepencencies = allDependencies
-            | otherwise = calculateTransitiveDependencies unitInfoMap nextAll nextNew
-            where
-                nextAll :: Set UnitId
-                nextAll = Set.union allDependencies nextNew
-                nextNew :: Set UnitId
-                nextNew = flip Set.difference allDependencies
-                    $ Set.unions
-                    $ map (Set.fromList . unitDepends)
-                    $ Map.elems
-                    $ Map.filterWithKey (\uid _ -> uid `Set.member` newDepencencies) unitInfoMap
-        getModulesForPackage :: Package -> [Module]
-        getModulesForPackage (Package package) =
-            map makeModule allModules
-            where
-                allModules :: [ModuleName]
-                allModules = map fst
-                    ( filter (isNothing . snd)
-                    $ unitExposedModules package
-                    )
-                    ++ unitHiddenModules package
-                makeModule :: ModuleName
-                           -> Module
-                makeModule = mkModule (unitInfoId package)
-
-newtype Package = Package UnitInfo deriving Eq
-instance Ord Package where
-  compare (Package u1) (Package u2) = compare (unitId u1) (unitId u2)
 
 
 -- | Wrap an 'HscEnv' into an 'HscEnvEq'.

From 68b06f277a31661e878e520d7381bfd8cbc8e401 Mon Sep 17 00:00:00 2001
From: Elodie Lander <lander89@gmail.com>
Date: Fri, 4 Aug 2023 07:21:56 -0500
Subject: [PATCH 50/50] Revert "Check that hiedb source files exist"

This reverts commit 102722903188cad248383fce623f75e22dc6a264.
---
 ghcide/src/Development/IDE/Spans/AtPoint.hs | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/ghcide/src/Development/IDE/Spans/AtPoint.hs b/ghcide/src/Development/IDE/Spans/AtPoint.hs
index b07626227a..c36150a91a 100644
--- a/ghcide/src/Development/IDE/Spans/AtPoint.hs
+++ b/ghcide/src/Development/IDE/Spans/AtPoint.hs
@@ -389,19 +389,9 @@ defRowToLocation lookupModule (row:.info) = do
   let start = Position (fromIntegral $ defSLine row - 1) (fromIntegral $ defSCol row - 1)
       end   = Position (fromIntegral $ defELine row - 1) (fromIntegral $ defECol row - 1)
       range = Range start end
-      lookupMod = lookupModule (defSrc row) (modInfoName info) (modInfoUnit info) (modInfoIsBoot info)
   file <- case modInfoSrcFile info of
-    Just src -> do
-      -- Checking that the file exists covers the case where a
-      -- dependency file in .hls is in the database but got deleted
-      -- for any reason.
-      -- See the function `lookupMod` in Development.IDE.Core.Actions
-      -- for where dependency files get created and indexed in hiedb.
-      fileExists <- liftIO $ doesFileExist src
-      if fileExists
-      then pure $ toUri src
-      else lookupMod
-    Nothing -> lookupMod
+    Just src -> pure $ toUri src
+    Nothing -> lookupModule (defSrc row) (modInfoName info) (modInfoUnit info) (modInfoIsBoot info)
   pure $ Location file range
 
 toUri :: FilePath -> Uri