From 5c8e9997a77d1794cd8d78687871cde8a6cd86f5 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Sat, 30 Oct 2021 19:28:44 +0100 Subject: [PATCH 01/17] Improve the performance of GetModIfaceFromDisk in large repos There are three benefits: 1. GetModIfaceFromDisk and GhcSessionDeps no longer depend on the transitive module summaries. This means fewer edges in the build graph = smaller build graph = faster builds 2. Avoid duplicate computations in setting up the GHC session with the dependencies of the module. Previously the total work done was O(NlogN) in the number of transitive dependencies, now it is O(N). 3. Increased sharing of HPT and FinderCache. Ideally we should also share the module graphs, but the datatype is abstract, doesn't have a monoid instance, and cannot be coerced to something that has. We will need to add the Monoid instance in GHC first. On the Sigma repo: - the startup metric goes down by ~34%. - The edit metric also goes down by 15%. - Max residency is down by 30% in the edit benchmark. --- ghcide/src/Development/IDE/Core/Rules.hs | 73 +++++++++++++++++------- 1 file changed, 53 insertions(+), 20 deletions(-) diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index 23fa70d3df..35a019aa26 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -150,6 +150,12 @@ import Ide.PluginUtils (configForPlugin) import Ide.Types (DynFlagsModifications (dynFlagsModifyGlobal, dynFlagsModifyParser), PluginId) import qualified Data.HashSet as HS +import Unsafe.Coerce (unsafeCoerce) +import Data.Map (Map) +import GhcPlugins (FinderCache, mgModSummaries) +import qualified Development.IDE.GHC.Compat as GHC +import Development.IDE.GHC.Compat (installedModule) +import Data.List.Extra (nubOrdOn) -- | This is useful for rules to convert rules that can only produce errors or -- a result into the more general IdeResult type that supports producing @@ -691,49 +697,76 @@ loadGhcSession = do Nothing -> LBS.toStrict $ B.encode (hash (snd val)) return (Just cutoffHash, val) - define $ \GhcSessionDeps file -> ghcSessionDepsDefinition file + defineNoDiagnostics $ \GhcSessionDeps file -> Just <$> ghcSessionDepsDefinition file -ghcSessionDepsDefinition :: NormalizedFilePath -> Action (IdeResult HscEnvEq) +ghcSessionDepsDefinition :: NormalizedFilePath -> Action HscEnvEq ghcSessionDepsDefinition file = do env <- use_ GhcSession file let hsc = hscEnv env ms <- msrModSummary <$> use_ GetModSummaryWithoutTimestamps file - deps <- use_ GetDependencies file - let tdeps = transitiveModuleDeps deps - uses_th_qq = + deps <- mapMaybe (fmap artifactFilePath . snd) <$> use_ GetLocatedImports file + mss <- map msrModSummary <$> uses_ GetModSummaryWithoutTimestamps deps + + depSessions <- uses_ GhcSessionDeps deps + session' <- liftIO $ mergeEnvs hsc mss $ map hscEnv depSessions + let uses_th_qq = xopt LangExt.TemplateHaskell dflags || xopt LangExt.QuasiQuotes dflags dflags = ms_hspp_opts ms ifaces <- if uses_th_qq - then uses_ GetModIface tdeps - else uses_ GetModIfaceWithoutLinkable tdeps - - -- Currently GetDependencies returns things in topological order so A comes before B if A imports B. - -- We need to reverse this as GHC gets very unhappy otherwise and complains about broken interfaces. - -- Long-term we might just want to change the order returned by GetDependencies - let inLoadOrder = reverse (map hirHomeMod ifaces) + then uses_ GetModIface deps + else uses_ GetModIfaceWithoutLinkable deps - session' <- liftIO $ loadModulesHome inLoadOrder <$> setupFinderCache (map hirModSummary ifaces) hsc - - res <- liftIO $ newHscEnvEqWithImportPaths (envImportPaths env) session' [] - return ([], Just res) + let session'' = loadModulesHome inLoadOrder $ session'{ + hsc_HPT = foldMap (hsc_HPT . hscEnv) depSessions + } + -- Currently GetDependencies returns things in topological order so A comes before B if A imports B. + -- We need to reverse this as GHC gets very unhappy otherwise and complains about broken interfaces. + -- Long-term we might just want to change the order returned by GetDependencies + inLoadOrder = reverse $ map hirHomeMod ifaces + + liftIO $ newHscEnvEqWithImportPaths (envImportPaths env) session'' [] + +-- Merge the HPTs, module graphs and FinderCaches +mergeEnvs :: HscEnv -> [ModSummary] -> [HscEnv] -> IO HscEnv +mergeEnvs env mss envs = do + prevFinderCache <- concatFC <$> mapM (readIORef . hsc_FC) envs + let ims = map (installedModule (homeUnitId_ $ hsc_dflags env) . moduleName . ms_mod) mss + ifrs = zipWith (\ms -> InstalledFound (ms_location ms)) mss ims + newFinderCache <- newIORef $ + foldl' + (\fc (im, ifr) -> GHC.extendInstalledModuleEnv fc im ifr) prevFinderCache + $ zip ims ifrs + return env{ + hsc_HPT = foldMap hsc_HPT envs, + hsc_FC = newFinderCache, + hsc_mod_graph = mkModuleGraph $ mss ++ nubOrdOn ms_mod (concatMap (mgModSummaries . hsc_mod_graph) envs) + } + where + -- required because 'FinderCache': + -- 1) doesn't have a 'Monoid' instance, + -- 2) is abstract and doesn't export constructors + -- To work around this, we coerce to the underlying type + -- To remove this, I plan to upstream the missing Monoid instance + concatFC :: [FinderCache] -> FinderCache + concatFC = unsafeCoerce (mconcat @(Map InstalledModule InstalledFindResult)) -- | Load a iface from disk, or generate it if there isn't one or it is out of date -- This rule also ensures that the `.hie` and `.o` (if needed) files are written out. getModIfaceFromDiskRule :: Rules () getModIfaceFromDiskRule = defineEarlyCutoff $ Rule $ \GetModIfaceFromDisk f -> do ms <- msrModSummary <$> use_ GetModSummary f - (diags_session, mb_session) <- ghcSessionDepsDefinition f + mb_session <- use GhcSessionDeps f case mb_session of - Nothing -> return (Nothing, (diags_session, Nothing)) + Nothing -> return (Nothing, ([], Nothing)) Just session -> do sourceModified <- use_ IsHiFileStable f linkableType <- getLinkableType f r <- loadInterface (hscEnv session) ms sourceModified linkableType (regenerateHiFile session f ms) case r of - (diags, Nothing) -> return (Nothing, (diags ++ diags_session, Nothing)) + (diags, Nothing) -> return (Nothing, (diags, Nothing)) (diags, Just x) -> do let !fp = Just $! hiFileFingerPrint x - return (fp, (diags <> diags_session, Just x)) + return (fp, (diags, Just x)) -- | Check state of hiedb after loading an iface from disk - have we indexed the corresponding `.hie` file? -- This function is responsible for ensuring database consistency From 91e544a0b873af0c1027a025920974d25beb2801 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 10:19:35 +0000 Subject: [PATCH 02/17] format importes --- ghcide/src/Development/IDE/Core/Rules.hs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index 35a019aa26..8687307268 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -87,6 +87,8 @@ import Data.IORef import Data.IntMap.Strict (IntMap) import qualified Data.IntMap.Strict as IntMap import Data.List +import Data.List.Extra (nubOrdOn) +import Data.Map (Map) import qualified Data.Map as M import Data.Maybe import qualified Data.Rope.UTF16 as Rope @@ -133,13 +135,13 @@ import Development.IDE.Types.Options import GHC.Generics (Generic) import GHC.IO.Encoding import qualified GHC.LanguageExtensions as LangExt +import GhcPlugins (FinderCache, mgModSummaries) import qualified HieDb import Ide.Plugin.Config import qualified Language.LSP.Server as LSP import Language.LSP.Types (SMethod (SCustomMethod)) import Language.LSP.VFS import System.Directory (canonicalizePath, makeAbsolute) - import Data.Default (def) import Ide.Plugin.Properties (HasProperty, KeyNameProxy, @@ -149,13 +151,7 @@ import Ide.Plugin.Properties (HasProperty, import Ide.PluginUtils (configForPlugin) import Ide.Types (DynFlagsModifications (dynFlagsModifyGlobal, dynFlagsModifyParser), PluginId) -import qualified Data.HashSet as HS -import Unsafe.Coerce (unsafeCoerce) -import Data.Map (Map) -import GhcPlugins (FinderCache, mgModSummaries) -import qualified Development.IDE.GHC.Compat as GHC -import Development.IDE.GHC.Compat (installedModule) -import Data.List.Extra (nubOrdOn) +import Unsafe.Coerce (unsafeCoerce) -- | This is useful for rules to convert rules that can only produce errors or -- a result into the more general IdeResult type that supports producing @@ -340,7 +336,7 @@ getLocatedImportsRule = return $ if itExists then Just nfp' else Nothing | Just tt <- HM.lookup (TargetModule modName) targets = do -- reuse the existing NormalizedFilePath in order to maximize sharing - let ttmap = HM.mapWithKey const (HS.toMap tt) + let ttmap = HM.mapWithKey const (HashSet.toMap tt) nfp' = HM.lookupDefault nfp nfp ttmap itExists <- getFileExists nfp' return $ if itExists then Just nfp' else Nothing @@ -730,11 +726,11 @@ ghcSessionDepsDefinition file = do mergeEnvs :: HscEnv -> [ModSummary] -> [HscEnv] -> IO HscEnv mergeEnvs env mss envs = do prevFinderCache <- concatFC <$> mapM (readIORef . hsc_FC) envs - let ims = map (installedModule (homeUnitId_ $ hsc_dflags env) . moduleName . ms_mod) mss + let ims = map (Compat.installedModule (homeUnitId_ $ hsc_dflags env) . moduleName . ms_mod) mss ifrs = zipWith (\ms -> InstalledFound (ms_location ms)) mss ims newFinderCache <- newIORef $ foldl' - (\fc (im, ifr) -> GHC.extendInstalledModuleEnv fc im ifr) prevFinderCache + (\fc (im, ifr) -> Compat.extendInstalledModuleEnv fc im ifr) prevFinderCache $ zip ims ifrs return env{ hsc_HPT = foldMap hsc_HPT envs, From 1bcf14d8dd8c4cce252f303ecd9c73aa43bd7884 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 10:25:18 +0000 Subject: [PATCH 03/17] clean up --- ghcide/.hlint.yaml | 2 +- ghcide/src/Development/IDE/Core/Compile.hs | 31 +++++++++++++++- ghcide/src/Development/IDE/Core/Rules.hs | 41 +++------------------- 3 files changed, 36 insertions(+), 38 deletions(-) diff --git a/ghcide/.hlint.yaml b/ghcide/.hlint.yaml index 01f035184a..84cd0879a8 100644 --- a/ghcide/.hlint.yaml +++ b/ghcide/.hlint.yaml @@ -133,7 +133,7 @@ # Things that are unsafe in Haskell base library - {name: unsafeInterleaveIO, within: [Development.IDE.LSP.LanguageServer]} - {name: unsafeDupablePerformIO, within: []} - - {name: unsafeCoerce, within: [Ide.Plugin.Eval.Code, Development.IDE.Types.Shake]} + - {name: unsafeCoerce, within: [Ide.Plugin.Eval.Code, Development.IDE.Core.Compile, Development.IDE.Types.Shake]} # Things that are a bit dangerous in the GHC API - {name: nameModule, within: []} diff --git a/ghcide/src/Development/IDE/Core/Compile.hs b/ghcide/src/Development/IDE/Core/Compile.hs index 13de867963..8c5a8db193 100644 --- a/ghcide/src/Development/IDE/Core/Compile.hs +++ b/ghcide/src/Development/IDE/Core/Compile.hs @@ -30,7 +30,7 @@ module Development.IDE.Core.Compile , setupFinderCache , getDocsBatch , lookupName - ) where + ,mergeEnvs) where import Development.IDE.Core.Preprocessor import Development.IDE.Core.RuleTypes @@ -89,8 +89,10 @@ import System.Directory import System.FilePath import System.IO.Extra (fixIO, newTempFileWithin) +-- GHC API imports -- GHC API imports import GHC (GetDocsFailure (..), + mgModSummaries, parsedSource) import Control.Concurrent.Extra @@ -100,11 +102,14 @@ import Data.Binary import Data.Coerce import Data.Functor import qualified Data.HashMap.Strict as HashMap +import Data.Map (Map) import Data.Tuple.Extra (dupe) import Data.Unique as Unique import Development.IDE.Core.Tracing (withTrace) +import GhcPlugins (FinderCache) import qualified Language.LSP.Server as LSP import qualified Language.LSP.Types as LSP +import Unsafe.Coerce -- | Given a string buffer, return the string (after preprocessing) and the 'ParsedModule'. parseModule @@ -686,6 +691,30 @@ loadModulesHome mod_infos e = where mod_name = moduleName . mi_module . hm_iface +-- Merge the HPTs, module graphs and FinderCaches +mergeEnvs :: HscEnv -> [ModSummary] -> [HomeModInfo] -> [HscEnv] -> IO HscEnv +mergeEnvs env extraModSummaries extraMods envs = do + prevFinderCache <- concatFC <$> mapM (readIORef . hsc_FC) envs + let ims = map (Compat.installedModule (homeUnitId_ $ hsc_dflags env) . moduleName . ms_mod) extraModSummaries + ifrs = zipWith (\ms -> InstalledFound (ms_location ms)) extraModSummaries ims + newFinderCache <- newIORef $ + foldl' + (\fc (im, ifr) -> Compat.extendInstalledModuleEnv fc im ifr) prevFinderCache + $ zip ims ifrs + return $ loadModulesHome extraMods $ env{ + hsc_HPT = foldMap hsc_HPT envs, + hsc_FC = newFinderCache, + hsc_mod_graph = mkModuleGraph $ extraModSummaries ++ nubOrdOn ms_mod (concatMap (mgModSummaries . hsc_mod_graph) envs) + } + where + -- required because 'FinderCache': + -- 1) doesn't have a 'Monoid' instance, + -- 2) is abstract and doesn't export constructors + -- To work around this, we coerce to the underlying type + -- To remove this, I plan to upstream the missing Monoid instance + concatFC :: [FinderCache] -> FinderCache + concatFC = unsafeCoerce (mconcat @(Map InstalledModule InstalledFindResult)) + withBootSuffix :: HscSource -> ModLocation -> ModLocation withBootSuffix HsBootFile = addBootSuffixLocnOut withBootSuffix _ = id diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index 8687307268..5ddb8f5ac2 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -87,8 +87,6 @@ import Data.IORef import Data.IntMap.Strict (IntMap) import qualified Data.IntMap.Strict as IntMap import Data.List -import Data.List.Extra (nubOrdOn) -import Data.Map (Map) import qualified Data.Map as M import Data.Maybe import qualified Data.Rope.UTF16 as Rope @@ -135,7 +133,6 @@ import Development.IDE.Types.Options import GHC.Generics (Generic) import GHC.IO.Encoding import qualified GHC.LanguageExtensions as LangExt -import GhcPlugins (FinderCache, mgModSummaries) import qualified HieDb import Ide.Plugin.Config import qualified Language.LSP.Server as LSP @@ -151,7 +148,6 @@ import Ide.Plugin.Properties (HasProperty, import Ide.PluginUtils (configForPlugin) import Ide.Types (DynFlagsModifications (dynFlagsModifyGlobal, dynFlagsModifyParser), PluginId) -import Unsafe.Coerce (unsafeCoerce) -- | This is useful for rules to convert rules that can only produce errors or -- a result into the more general IdeResult type that supports producing @@ -703,8 +699,7 @@ ghcSessionDepsDefinition file = do deps <- mapMaybe (fmap artifactFilePath . snd) <$> use_ GetLocatedImports file mss <- map msrModSummary <$> uses_ GetModSummaryWithoutTimestamps deps - depSessions <- uses_ GhcSessionDeps deps - session' <- liftIO $ mergeEnvs hsc mss $ map hscEnv depSessions + depSessions <- map hscEnv <$> uses_ GhcSessionDeps deps let uses_th_qq = xopt LangExt.TemplateHaskell dflags || xopt LangExt.QuasiQuotes dflags dflags = ms_hspp_opts ms @@ -712,39 +707,13 @@ ghcSessionDepsDefinition file = do then uses_ GetModIface deps else uses_ GetModIfaceWithoutLinkable deps - let session'' = loadModulesHome inLoadOrder $ session'{ - hsc_HPT = foldMap (hsc_HPT . hscEnv) depSessions - } + let inLoadOrder = reverse $ map hirHomeMod ifaces + session' <- liftIO $ mergeEnvs hsc mss inLoadOrder depSessions -- Currently GetDependencies returns things in topological order so A comes before B if A imports B. -- We need to reverse this as GHC gets very unhappy otherwise and complains about broken interfaces. -- Long-term we might just want to change the order returned by GetDependencies - inLoadOrder = reverse $ map hirHomeMod ifaces - - liftIO $ newHscEnvEqWithImportPaths (envImportPaths env) session'' [] - --- Merge the HPTs, module graphs and FinderCaches -mergeEnvs :: HscEnv -> [ModSummary] -> [HscEnv] -> IO HscEnv -mergeEnvs env mss envs = do - prevFinderCache <- concatFC <$> mapM (readIORef . hsc_FC) envs - let ims = map (Compat.installedModule (homeUnitId_ $ hsc_dflags env) . moduleName . ms_mod) mss - ifrs = zipWith (\ms -> InstalledFound (ms_location ms)) mss ims - newFinderCache <- newIORef $ - foldl' - (\fc (im, ifr) -> Compat.extendInstalledModuleEnv fc im ifr) prevFinderCache - $ zip ims ifrs - return env{ - hsc_HPT = foldMap hsc_HPT envs, - hsc_FC = newFinderCache, - hsc_mod_graph = mkModuleGraph $ mss ++ nubOrdOn ms_mod (concatMap (mgModSummaries . hsc_mod_graph) envs) - } - where - -- required because 'FinderCache': - -- 1) doesn't have a 'Monoid' instance, - -- 2) is abstract and doesn't export constructors - -- To work around this, we coerce to the underlying type - -- To remove this, I plan to upstream the missing Monoid instance - concatFC :: [FinderCache] -> FinderCache - concatFC = unsafeCoerce (mconcat @(Map InstalledModule InstalledFindResult)) + + liftIO $ newHscEnvEqWithImportPaths (envImportPaths env) session' [] -- | Load a iface from disk, or generate it if there isn't one or it is out of date -- This rule also ensures that the `.hie` and `.o` (if needed) files are written out. From 8555d612459d79cba5319dad2a98d34dc94f86eb Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 10:58:57 +0000 Subject: [PATCH 04/17] remove stale comment --- ghcide/src/Development/IDE/Core/Rules.hs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index 5ddb8f5ac2..ea082eeea1 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -707,11 +707,8 @@ ghcSessionDepsDefinition file = do then uses_ GetModIface deps else uses_ GetModIfaceWithoutLinkable deps - let inLoadOrder = reverse $ map hirHomeMod ifaces + let inLoadOrder = map hirHomeMod ifaces session' <- liftIO $ mergeEnvs hsc mss inLoadOrder depSessions - -- Currently GetDependencies returns things in topological order so A comes before B if A imports B. - -- We need to reverse this as GHC gets very unhappy otherwise and complains about broken interfaces. - -- Long-term we might just want to change the order returned by GetDependencies liftIO $ newHscEnvEqWithImportPaths (envImportPaths env) session' [] From 680ff1f213e7b722ee12579d8da1bce6c0964e02 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 11:02:03 +0000 Subject: [PATCH 05/17] fix build in GHC 9 --- ghcide/src/Development/IDE/Core/Compile.hs | 1 - ghcide/src/Development/IDE/GHC/Compat/Units.hs | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ghcide/src/Development/IDE/Core/Compile.hs b/ghcide/src/Development/IDE/Core/Compile.hs index 8c5a8db193..de727ca4ad 100644 --- a/ghcide/src/Development/IDE/Core/Compile.hs +++ b/ghcide/src/Development/IDE/Core/Compile.hs @@ -106,7 +106,6 @@ import Data.Map (Map) import Data.Tuple.Extra (dupe) import Data.Unique as Unique import Development.IDE.Core.Tracing (withTrace) -import GhcPlugins (FinderCache) import qualified Language.LSP.Server as LSP import qualified Language.LSP.Types as LSP import Unsafe.Coerce diff --git a/ghcide/src/Development/IDE/GHC/Compat/Units.hs b/ghcide/src/Development/IDE/GHC/Compat/Units.hs index d36ad94bc8..59583cffd2 100644 --- a/ghcide/src/Development/IDE/GHC/Compat/Units.hs +++ b/ghcide/src/Development/IDE/GHC/Compat/Units.hs @@ -45,6 +45,7 @@ module Development.IDE.GHC.Compat.Units ( ExternalPackageState(..), -- * Utils filterInplaceUnits, + FinderCache, ) where #if MIN_VERSION_ghc(9,0,0) @@ -59,6 +60,7 @@ import GHC.Driver.Types import GHC.Data.FastString import qualified GHC.Driver.Session as DynFlags import GHC.Types.Unique.Set +import GHC.Unit.Finder (finderCache) import qualified GHC.Unit.Info as UnitInfo import GHC.Unit.State (LookupResult, UnitInfo, UnitState (unitInfoMap)) From 0019fa9a8e4fc187c59725c6243074223653480f Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 15:11:48 +0000 Subject: [PATCH 06/17] clean up --- ghcide/src/Development/IDE/Core/Rules.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index ea082eeea1..adc953a674 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -695,9 +695,8 @@ ghcSessionDepsDefinition :: NormalizedFilePath -> Action HscEnvEq ghcSessionDepsDefinition file = do env <- use_ GhcSession file let hsc = hscEnv env - ms <- msrModSummary <$> use_ GetModSummaryWithoutTimestamps file deps <- mapMaybe (fmap artifactFilePath . snd) <$> use_ GetLocatedImports file - mss <- map msrModSummary <$> uses_ GetModSummaryWithoutTimestamps deps + ms:mss <- map msrModSummary <$> uses_ GetModSummaryWithoutTimestamps (file:deps) depSessions <- map hscEnv <$> uses_ GhcSessionDeps deps let uses_th_qq = From 1739927003b882aefdd638a8a9cda1545012e639 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 18:33:28 +0000 Subject: [PATCH 07/17] Unify defintions of ghcSessionDeps --- ghcide/src/Development/IDE/Core/Rules.hs | 42 ++++++++++--------- .../src/Ide/Plugin/Eval/CodeLens.hs | 26 ++++-------- 2 files changed, 29 insertions(+), 39 deletions(-) diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index adc953a674..0ea66f3c23 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -689,27 +689,29 @@ loadGhcSession = do Nothing -> LBS.toStrict $ B.encode (hash (snd val)) return (Just cutoffHash, val) - defineNoDiagnostics $ \GhcSessionDeps file -> Just <$> ghcSessionDepsDefinition file - -ghcSessionDepsDefinition :: NormalizedFilePath -> Action HscEnvEq -ghcSessionDepsDefinition file = do + defineNoDiagnostics $ \GhcSessionDeps file -> do env <- use_ GhcSession file - let hsc = hscEnv env - deps <- mapMaybe (fmap artifactFilePath . snd) <$> use_ GetLocatedImports file - ms:mss <- map msrModSummary <$> uses_ GetModSummaryWithoutTimestamps (file:deps) - - depSessions <- map hscEnv <$> uses_ GhcSessionDeps deps - let uses_th_qq = - xopt LangExt.TemplateHaskell dflags || xopt LangExt.QuasiQuotes dflags - dflags = ms_hspp_opts ms - ifaces <- if uses_th_qq - then uses_ GetModIface deps - else uses_ GetModIfaceWithoutLinkable deps - - let inLoadOrder = map hirHomeMod ifaces - session' <- liftIO $ mergeEnvs hsc mss inLoadOrder depSessions - - liftIO $ newHscEnvEqWithImportPaths (envImportPaths env) session' [] + Just <$> ghcSessionDepsDefinition False env file + +ghcSessionDepsDefinition :: Bool -> HscEnvEq -> NormalizedFilePath -> Action HscEnvEq +ghcSessionDepsDefinition forceLinkable env file = do + let hsc = hscEnv env + deps <- mapMaybe (fmap artifactFilePath . snd) <$> use_ GetLocatedImports file + _ <- uses_ ReportImportCycles deps + ms:mss <- map msrModSummary <$> uses_ GetModSummaryWithoutTimestamps (file:deps) + + depSessions <- map hscEnv <$> uses_ GhcSessionDeps deps + let uses_th_qq = + xopt LangExt.TemplateHaskell dflags || xopt LangExt.QuasiQuotes dflags + dflags = ms_hspp_opts ms + ifaces <- if uses_th_qq || forceLinkable + then uses_ GetModIface deps + else uses_ GetModIfaceWithoutLinkable deps + + let inLoadOrder = map hirHomeMod ifaces + session' <- liftIO $ mergeEnvs hsc mss inLoadOrder depSessions + + liftIO $ newHscEnvEqWithImportPaths (envImportPaths env) session' [] -- | Load a iface from disk, or generate it if there isn't one or it is out of date -- This rule also ensures that the `.hie` and `.o` (if needed) files are written out. diff --git a/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs b/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs index 41dea1bd48..4b2c4a43bd 100644 --- a/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs +++ b/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs @@ -52,7 +52,7 @@ import Development.IDE (Action, GetDependencies (..), HscEnvEq, IdeState, ModSummaryResult (..), NeedsCompilation (NeedsCompilation), - evalGhcEnv, + evalGhcEnv, hscEnv, hscEnvWithImportPaths, prettyPrint, runAction, textToStringBuffer, @@ -61,7 +61,8 @@ import Development.IDE (Action, GetDependencies (..), useWithStale_, use_, uses_) import Development.IDE.Core.Compile (loadModulesHome, setupFinderCache) -import Development.IDE.Core.Rules (TransitiveDependencies (transitiveModuleDeps)) +import Development.IDE.Core.Rules (TransitiveDependencies (transitiveModuleDeps), + ghcSessionDepsDefinition) import Development.IDE.GHC.Compat hiding (typeKind, unitState) import qualified Development.IDE.GHC.Compat as Compat import qualified Development.IDE.GHC.Compat as SrcLoc @@ -533,30 +534,17 @@ prettyWarn Warn{..} = prettyPrint (SrcLoc.getLoc warnMsg) <> ": warning:\n" <> " " <> SrcLoc.unLoc warnMsg -ghcSessionDepsDefinition :: HscEnvEq -> NormalizedFilePath -> Action HscEnv -ghcSessionDepsDefinition env file = do - let hsc = hscEnvWithImportPaths env - deps <- use_ GetDependencies file - let tdeps = transitiveModuleDeps deps - ifaces <- uses_ GetModIface tdeps - liftIO $ assert (all (isJust . hm_linkable . hirHomeMod) ifaces) $ pure () - - -- Currently GetDependencies returns things in topological order so A comes before B if A imports B. - -- We need to reverse this as GHC gets very unhappy otherwise and complains about broken interfaces. - -- Long-term we might just want to change the order returned by GetDependencies - let inLoadOrder = reverse (map hirHomeMod ifaces) - - liftIO $ loadModulesHome inLoadOrder <$> setupFinderCache (map hirModSummary ifaces) hsc - runGetSession :: MonadIO m => IdeState -> NormalizedFilePath -> m HscEnv runGetSession st nfp = liftIO $ runAction "eval" st $ do -- Create a new GHC Session rather than reusing an existing one -- to avoid interfering with ghcide + -- UPDATE: I suspect that this doesn't really work, we always get the same Session + -- we probably cache hscEnvs in the Session state IdeGhcSession{loadSessionFun} <- useNoFile_ GhcSessionIO let fp = fromNormalizedFilePath nfp ((_, res),_) <- liftIO $ loadSessionFun fp - let hscEnv = fromMaybe (error $ "Unknown file: " <> fp) res - ghcSessionDepsDefinition hscEnv nfp + let env = fromMaybe (error $ "Unknown file: " <> fp) res + hscEnv <$> ghcSessionDepsDefinition False env nfp needsQuickCheck :: [(Section, Test)] -> Bool needsQuickCheck = any (isProperty . snd) From fdcb77f071cd8ff38f73a2d65a285dbbf0ed83dd Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 18:44:43 +0000 Subject: [PATCH 08/17] mark test as no longer failing --- ghcide/test/exe/Main.hs | 3 --- 1 file changed, 3 deletions(-) diff --git a/ghcide/test/exe/Main.hs b/ghcide/test/exe/Main.hs index 14fa2f6a8a..19b23b8157 100644 --- a/ghcide/test/exe/Main.hs +++ b/ghcide/test/exe/Main.hs @@ -3843,9 +3843,6 @@ checkFileCompiles fp diag = pluginSimpleTests :: TestTree pluginSimpleTests = ignoreInWindowsForGHC88And810 $ -#if __GLASGOW_HASKELL__ == 810 && __GLASGOW_HASKELL_PATCHLEVEL1__ == 5 - expectFailBecause "known broken for ghc 8.10.5 (see GHC #19763)" $ -#endif testSessionWithExtraFiles "plugin-knownnat" "simple plugin" $ \dir -> do _ <- openDoc (dir "KnownNat.hs") "haskell" liftIO $ writeFile (dir"hie.yaml") From 05f67b6db48b798d939b0605b7f3d8796f0bd60a Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 18:49:58 +0000 Subject: [PATCH 09/17] Prevent duplicate missing module diagnostics --- ghcide/src/Development/IDE/Core/Rules.hs | 40 ++++++++++--------- .../src/Ide/Plugin/Eval/CodeLens.hs | 23 ++++------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index 0ea66f3c23..fc0c8ae7f8 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -691,27 +691,31 @@ loadGhcSession = do defineNoDiagnostics $ \GhcSessionDeps file -> do env <- use_ GhcSession file - Just <$> ghcSessionDepsDefinition False env file + ghcSessionDepsDefinition False env file -ghcSessionDepsDefinition :: Bool -> HscEnvEq -> NormalizedFilePath -> Action HscEnvEq +ghcSessionDepsDefinition :: Bool -> HscEnvEq -> NormalizedFilePath -> Action (Maybe HscEnvEq) ghcSessionDepsDefinition forceLinkable env file = do let hsc = hscEnv env - deps <- mapMaybe (fmap artifactFilePath . snd) <$> use_ GetLocatedImports file - _ <- uses_ ReportImportCycles deps - ms:mss <- map msrModSummary <$> uses_ GetModSummaryWithoutTimestamps (file:deps) - - depSessions <- map hscEnv <$> uses_ GhcSessionDeps deps - let uses_th_qq = - xopt LangExt.TemplateHaskell dflags || xopt LangExt.QuasiQuotes dflags - dflags = ms_hspp_opts ms - ifaces <- if uses_th_qq || forceLinkable - then uses_ GetModIface deps - else uses_ GetModIfaceWithoutLinkable deps - - let inLoadOrder = map hirHomeMod ifaces - session' <- liftIO $ mergeEnvs hsc mss inLoadOrder depSessions - - liftIO $ newHscEnvEqWithImportPaths (envImportPaths env) session' [] + + mbdeps <- mapM(fmap artifactFilePath . snd) <$> use_ GetLocatedImports file + case mbdeps of + Nothing -> return Nothing + Just deps -> do + _ <- uses_ ReportImportCycles deps + ms:mss <- map msrModSummary <$> uses_ GetModSummaryWithoutTimestamps (file:deps) + + depSessions <- map hscEnv <$> uses_ GhcSessionDeps deps + let uses_th_qq = + xopt LangExt.TemplateHaskell dflags || xopt LangExt.QuasiQuotes dflags + dflags = ms_hspp_opts ms + ifaces <- if uses_th_qq || forceLinkable + then uses_ GetModIface deps + else uses_ GetModIfaceWithoutLinkable deps + + let inLoadOrder = map hirHomeMod ifaces + session' <- liftIO $ mergeEnvs hsc mss inLoadOrder depSessions + + Just <$> liftIO (newHscEnvEqWithImportPaths (envImportPaths env) session' []) -- | Load a iface from disk, or generate it if there isn't one or it is out of date -- This rule also ensures that the `.hie` and `.o` (if needed) files are written out. diff --git a/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs b/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs index 4b2c4a43bd..db2fa7d230 100644 --- a/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs +++ b/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs @@ -27,7 +27,7 @@ module Ide.Plugin.Eval.CodeLens ( import Control.Applicative (Alternative ((<|>))) import Control.Arrow (second, (>>>)) -import Control.Exception (assert, try) +import Control.Exception (try) import qualified Control.Exception as E import Control.Lens (_1, _3, (%~), (<&>), (^.)) import Control.Monad (guard, join, void, when) @@ -38,31 +38,23 @@ import Data.Char (isSpace) import qualified Data.HashMap.Strict as HashMap import Data.List (dropWhileEnd, find, intercalate, intersperse) -import Data.Maybe (catMaybes, fromMaybe, isJust) +import Data.Maybe (catMaybes, fromMaybe) import Data.String (IsString) import Data.Text (Text) import qualified Data.Text as T import Data.Time (getCurrentTime) import Data.Typeable (Typeable) -import Development.IDE (Action, GetDependencies (..), - GetModIface (..), - GetModSummary (..), - GhcSessionIO (..), - HiFileResult (hirHomeMod, hirModSummary), - HscEnvEq, IdeState, +import Development.IDE (GetModSummary (..), + GhcSessionIO (..), IdeState, ModSummaryResult (..), NeedsCompilation (NeedsCompilation), evalGhcEnv, hscEnv, - hscEnvWithImportPaths, prettyPrint, runAction, textToStringBuffer, toNormalizedFilePath', uriToFilePath', useNoFile_, - useWithStale_, use_, uses_) -import Development.IDE.Core.Compile (loadModulesHome, - setupFinderCache) -import Development.IDE.Core.Rules (TransitiveDependencies (transitiveModuleDeps), - ghcSessionDepsDefinition) + useWithStale_, use_) +import Development.IDE.Core.Rules (ghcSessionDepsDefinition) import Development.IDE.GHC.Compat hiding (typeKind, unitState) import qualified Development.IDE.GHC.Compat as Compat import qualified Development.IDE.GHC.Compat as SrcLoc @@ -544,7 +536,8 @@ runGetSession st nfp = liftIO $ runAction "eval" st $ do let fp = fromNormalizedFilePath nfp ((_, res),_) <- liftIO $ loadSessionFun fp let env = fromMaybe (error $ "Unknown file: " <> fp) res - hscEnv <$> ghcSessionDepsDefinition False env nfp + res <- fmap hscEnv <$> ghcSessionDepsDefinition False env nfp + return $ fromMaybe (error $ "Unable to load file: " <> fp) res needsQuickCheck :: [(Section, Test)] -> Bool needsQuickCheck = any (isProperty . snd) From fbdda1054822e96baa0a878916f389a0ce0d90d6 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 18:54:52 +0000 Subject: [PATCH 10/17] delete GetDependencies --- ghcide/src/Development/IDE/Core/RuleTypes.hs | 10 +--------- ghcide/src/Development/IDE/Core/Rules.hs | 19 +++---------------- .../IDE/Import/DependencyInformation.hs | 1 + ghcide/src/Development/IDE/Plugin/Test.hs | 1 - ghcide/test/exe/Main.hs | 13 +++++-------- 5 files changed, 10 insertions(+), 34 deletions(-) diff --git a/ghcide/src/Development/IDE/Core/RuleTypes.hs b/ghcide/src/Development/IDE/Core/RuleTypes.hs index 6f8900b54e..5d98fd873b 100644 --- a/ghcide/src/Development/IDE/Core/RuleTypes.hs +++ b/ghcide/src/Development/IDE/Core/RuleTypes.hs @@ -73,10 +73,6 @@ type instance RuleResult GetParsedModuleWithComments = ParsedModule -- a module could not be parsed or an import cycle. type instance RuleResult GetDependencyInformation = DependencyInformation --- | Transitive module and pkg dependencies based on the information produced by GetDependencyInformation. --- This rule is also responsible for calling ReportImportCycles for each file in the transitive closure. -type instance RuleResult GetDependencies = TransitiveDependencies - type instance RuleResult GetModuleGraph = DependencyInformation data GetKnownTargets = GetKnownTargets @@ -234,6 +230,7 @@ type instance RuleResult GetDocMap = DocAndKindMap type instance RuleResult GhcSession = HscEnvEq -- | A GHC session preloaded with all the dependencies +-- This rule is also responsible for calling ReportImportCycles for the direct dependencies type instance RuleResult GhcSessionDeps = HscEnvEq -- | Resolve the imports in a module to the file path of a module in the same package @@ -389,11 +386,6 @@ data ReportImportCycles = ReportImportCycles instance Hashable ReportImportCycles instance NFData ReportImportCycles -data GetDependencies = GetDependencies - deriving (Eq, Show, Typeable, Generic) -instance Hashable GetDependencies -instance NFData GetDependencies - data TypeCheck = TypeCheck deriving (Eq, Show, Typeable, Generic) instance Hashable TypeCheck diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index fc0c8ae7f8..162355d10a 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -11,7 +11,7 @@ -- module Development.IDE.Core.Rules( -- * Types - IdeState, GetDependencies(..), GetParsedModule(..), TransitiveDependencies(..), + IdeState, GetParsedModule(..), TransitiveDependencies(..), Priority(..), GhcSessionIO(..), GetClientSettings(..), -- * Functions priorityTypeCheck, @@ -35,7 +35,6 @@ module Development.IDE.Core.Rules( getLocatedImportsRule, getDependencyInformationRule, reportImportCyclesRule, - getDependenciesRule, typeCheckRule, getDocMapRule, loadGhcSession, @@ -161,7 +160,8 @@ toIdeResult = either (, Nothing) (([],) . Just) -- | Get all transitive file dependencies of a given module. -- Does not include the file itself. getDependencies :: NormalizedFilePath -> Action (Maybe [NormalizedFilePath]) -getDependencies file = fmap transitiveModuleDeps <$> use GetDependencies file +getDependencies file = + fmap transitiveModuleDeps . (`transitiveDeps` file) <$> use_ GetDependencyInformation file getSourceFileSource :: NormalizedFilePath -> Action BS.ByteString getSourceFileSource nfp = do @@ -490,18 +490,6 @@ reportImportCyclesRule = pure (moduleNameString . moduleName . ms_mod $ ms) showCycle mods = T.intercalate ", " (map T.pack mods) --- returns all transitive dependencies in topological order. --- NOTE: result does not include the argument file. -getDependenciesRule :: Rules () -getDependenciesRule = - defineEarlyCutoff $ RuleNoDiagnostics $ \GetDependencies file -> do - depInfo <- use_ GetDependencyInformation file - let allFiles = reachableModules depInfo - _ <- uses_ ReportImportCycles allFiles - opts <- getIdeOptions - let mbFingerprints = map (Util.fingerprintString . fromNormalizedFilePath) allFiles <$ optShakeFiles opts - return (fingerprintToBS . Util.fingerprintFingerprints <$> mbFingerprints, transitiveDeps depInfo file) - getHieAstsRule :: Rules () getHieAstsRule = define $ \GetHieAst f -> do @@ -1065,7 +1053,6 @@ mainRule = do getLocatedImportsRule getDependencyInformationRule reportImportCyclesRule - getDependenciesRule typeCheckRule getDocMapRule loadGhcSession diff --git a/ghcide/src/Development/IDE/Import/DependencyInformation.hs b/ghcide/src/Development/IDE/Import/DependencyInformation.hs index 9e58cbd0f6..6f85af7678 100644 --- a/ghcide/src/Development/IDE/Import/DependencyInformation.hs +++ b/ghcide/src/Development/IDE/Import/DependencyInformation.hs @@ -327,6 +327,7 @@ immediateReverseDependencies file DependencyInformation{..} = do FilePathId cur_id <- lookupPathToId depPathIdMap file return $ map (idToPath depPathIdMap . FilePathId) (maybe mempty IntSet.toList (IntMap.lookup cur_id depReverseModuleDeps)) +-- | returns all transitive dependencies in topological order. transitiveDeps :: DependencyInformation -> NormalizedFilePath -> Maybe TransitiveDependencies transitiveDeps DependencyInformation{..} file = do let !fileId = pathToId depPathIdMap file diff --git a/ghcide/src/Development/IDE/Plugin/Test.hs b/ghcide/src/Development/IDE/Plugin/Test.hs index 881aed4406..14e861f38b 100644 --- a/ghcide/src/Development/IDE/Plugin/Test.hs +++ b/ghcide/src/Development/IDE/Plugin/Test.hs @@ -150,7 +150,6 @@ parseAction "getparsedmodule" fp = Right . isJust <$> use GetParsedModule fp parseAction "ghcsession" fp = Right . isJust <$> use GhcSession fp parseAction "ghcsessiondeps" fp = Right . isJust <$> use GhcSessionDeps fp parseAction "gethieast" fp = Right . isJust <$> use GetHieAst fp -parseAction "getDependencies" fp = Right . isJust <$> use GetDependencies fp parseAction "getFileContents" fp = Right . isJust <$> use GetFileContents fp parseAction other _ = return $ Left $ "Cannot parse ide rule: " <> pack (original other) diff --git a/ghcide/test/exe/Main.hs b/ghcide/test/exe/Main.hs index 19b23b8157..5ae5709e94 100644 --- a/ghcide/test/exe/Main.hs +++ b/ghcide/test/exe/Main.hs @@ -669,9 +669,9 @@ diagnosticTests = testGroup "diagnostics" expectDiagnostics [("Foo.hs", [(DsError, (1,7), "Could not find module 'MissingModule'")])] , testGroup "Cancellation" - [ cancellationTestGroup "edit header" editHeader yesDepends yesSession noParse noTc - , cancellationTestGroup "edit import" editImport noDepends noSession yesParse noTc - , cancellationTestGroup "edit body" editBody yesDepends yesSession yesParse yesTc + [ cancellationTestGroup "edit header" editHeader yesSession noParse noTc + , cancellationTestGroup "edit import" editImport noSession yesParse noTc + , cancellationTestGroup "edit body" editBody yesSession yesParse yesTc ] ] where @@ -685,17 +685,14 @@ diagnosticTests = testGroup "diagnostics" noParse = False yesParse = True - noDepends = False - yesDepends = True - noSession = False yesSession = True noTc = False yesTc = True -cancellationTestGroup :: TestName -> (TextDocumentContentChangeEvent, TextDocumentContentChangeEvent) -> Bool -> Bool -> Bool -> Bool -> TestTree -cancellationTestGroup name edits dependsOutcome sessionDepsOutcome parseOutcome tcOutcome = testGroup name +cancellationTestGroup :: TestName -> (TextDocumentContentChangeEvent, TextDocumentContentChangeEvent) -> Bool -> Bool -> Bool -> TestTree +cancellationTestGroup name edits sessionDepsOutcome parseOutcome tcOutcome = testGroup name [ cancellationTemplate edits Nothing , cancellationTemplate edits $ Just ("GetFileContents", True) , cancellationTemplate edits $ Just ("GhcSession", True) From 1bf410f7a3d2a21da03d2dc93e80649cfb96934a Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 19:21:46 +0000 Subject: [PATCH 11/17] add a test for deeply nested import cycles --- ghcide/test/exe/Main.hs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ghcide/test/exe/Main.hs b/ghcide/test/exe/Main.hs index 5ae5709e94..39ecd961dd 100644 --- a/ghcide/test/exe/Main.hs +++ b/ghcide/test/exe/Main.hs @@ -410,6 +410,30 @@ diagnosticTests = testGroup "diagnostics" , [(DsError, (1, 7), "Cyclic module dependency between ModuleA, ModuleB")] ) ] + , testSession' "deeply nested cyclic module dependency" $ \path -> do + let contentA = unlines + [ "module ModuleA where" , "import ModuleB" ] + let contentB = unlines + [ "module ModuleB where" , "import ModuleA" ] + let contentC = unlines + [ "module ModuleC where" , "import ModuleB" ] + let contentD = T.unlines + [ "module ModuleD where" , "import ModuleC" ] + cradle = + "cradle: {direct: {arguments: [ModuleA, ModuleB, ModuleC, ModuleD]}}" + liftIO $ writeFile (path "ModuleA.hs") contentA + liftIO $ writeFile (path "ModuleB.hs") contentB + liftIO $ writeFile (path "ModuleC.hs") contentC + liftIO $ writeFile (path "hie.yaml") cradle + _ <- createDoc "ModuleD.hs" "haskell" contentD + expectDiagnostics + [ ( "ModuleA.hs" + , [(DsError, (1, 7), "Cyclic module dependency between ModuleA, ModuleB")] + ) + , ( "ModuleB.hs" + , [(DsError, (1, 7), "Cyclic module dependency between ModuleA, ModuleB")] + ) + ] , testSessionWait "cyclic module dependency with hs-boot" $ do let contentA = T.unlines [ "module ModuleA where" From f2137c990f775eab3013c97aaab1077a9362a6f2 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 20:12:04 +0000 Subject: [PATCH 12/17] Fix build in GHC 9.0 --- ghcide/src/Development/IDE/GHC/Compat/Units.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ghcide/src/Development/IDE/GHC/Compat/Units.hs b/ghcide/src/Development/IDE/GHC/Compat/Units.hs index 59583cffd2..61f8d82644 100644 --- a/ghcide/src/Development/IDE/GHC/Compat/Units.hs +++ b/ghcide/src/Development/IDE/GHC/Compat/Units.hs @@ -54,13 +54,13 @@ import qualified GHC.Data.ShortText as ST import GHC.Driver.Env (hsc_unit_dbs) import GHC.Unit.Env import GHC.Unit.External +import GHC.Unit.Finder #else import GHC.Driver.Types #endif import GHC.Data.FastString import qualified GHC.Driver.Session as DynFlags import GHC.Types.Unique.Set -import GHC.Unit.Finder (finderCache) import qualified GHC.Unit.Info as UnitInfo import GHC.Unit.State (LookupResult, UnitInfo, UnitState (unitInfoMap)) From 3e03be5c8fb7062531b6edfa35a8fccc4ce576dc Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 2 Nov 2021 20:54:13 +0000 Subject: [PATCH 13/17] bump ghcide version --- ghcide/ghcide.cabal | 2 +- plugins/hls-eval-plugin/hls-eval-plugin.cabal | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ghcide/ghcide.cabal b/ghcide/ghcide.cabal index a6da170c14..aa5a0e1b9f 100644 --- a/ghcide/ghcide.cabal +++ b/ghcide/ghcide.cabal @@ -2,7 +2,7 @@ cabal-version: 2.4 build-type: Simple category: Development name: ghcide -version: 1.4.2.3 +version: 1.4.2.4 license: Apache-2.0 license-file: LICENSE author: Digital Asset and Ghcide contributors diff --git a/plugins/hls-eval-plugin/hls-eval-plugin.cabal b/plugins/hls-eval-plugin/hls-eval-plugin.cabal index e37acde140..d6f0f508fe 100644 --- a/plugins/hls-eval-plugin/hls-eval-plugin.cabal +++ b/plugins/hls-eval-plugin/hls-eval-plugin.cabal @@ -64,7 +64,7 @@ library , ghc , ghc-boot-th , ghc-paths - , ghcide >=1.2 && <1.5 + , ghcide >=1.4.2.4 && <1.5 , hashable , hls-graph , hls-plugin-api ^>=1.2 From 9f95dbb26023cc9d9d3ee97927838eeb9bc06254 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Fri, 5 Nov 2021 19:04:27 +0000 Subject: [PATCH 14/17] Introduce config options for the main rules Surfacing the performance tradeoffs in the core build rules --- ghcide/exe/Main.hs | 3 +- ghcide/src/Development/IDE/Core/Rules.hs | 51 ++++++++++++++----- ghcide/src/Development/IDE/Main.hs | 2 +- ghcide/test/exe/Main.hs | 1 - plugins/hls-eval-plugin/hls-eval-plugin.cabal | 1 + .../src/Ide/Plugin/Eval/CodeLens.hs | 7 ++- 6 files changed, 46 insertions(+), 19 deletions(-) diff --git a/ghcide/exe/Main.hs b/ghcide/exe/Main.hs index 78af32e8ba..1e4f367140 100644 --- a/ghcide/exe/Main.hs +++ b/ghcide/exe/Main.hs @@ -8,6 +8,7 @@ module Main(main) where import Arguments (Arguments (..), getArguments) import Control.Monad.Extra (unless, whenJust) +import Data.Default (def) import Data.Version (showVersion) import Development.GitRev (gitHash) import Development.IDE (Priority (Debug, Info), @@ -60,7 +61,7 @@ main = withTelemetryLogger $ \telemetryLogger -> do ,Main.argsRules = do -- install the main and ghcide-plugin rules - mainRule + mainRule def -- install the kick action, which triggers a typecheck on every -- Shake database restart, i.e. on every user edit. unless argsDisableKick $ diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index 162355d10a..4ea001a432 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -22,6 +22,7 @@ module Development.IDE.Core.Rules( defineNoFile, defineEarlyCutOffNoFile, mainRule, + RulesConfig(..), getDependencies, getParsedModule, getParsedModuleWithComments, @@ -56,6 +57,7 @@ module Development.IDE.Core.Rules( ghcSessionDepsDefinition, getParsedModuleDefinition, typeCheckRuleDefinition, + GhcSessionDepsConfig(..), ) where #if !MIN_VERSION_ghc(8,8,0) @@ -138,7 +140,7 @@ import qualified Language.LSP.Server as LSP import Language.LSP.Types (SMethod (SCustomMethod)) import Language.LSP.VFS import System.Directory (canonicalizePath, makeAbsolute) -import Data.Default (def) +import Data.Default (def, Default) import Ide.Plugin.Properties (HasProperty, KeyNameProxy, Properties, @@ -640,8 +642,8 @@ currentLinkables = do where go (mod, time) = LM time mod [] -loadGhcSession :: Rules () -loadGhcSession = do +loadGhcSession :: GhcSessionDepsConfig -> Rules () +loadGhcSession ghcSessionDepsConfig = do -- This function should always be rerun because it tracks changes -- to the version of the collection of HscEnv's. defineEarlyCutOffNoFile $ \GhcSessionIO -> do @@ -679,24 +681,34 @@ loadGhcSession = do defineNoDiagnostics $ \GhcSessionDeps file -> do env <- use_ GhcSession file - ghcSessionDepsDefinition False env file - -ghcSessionDepsDefinition :: Bool -> HscEnvEq -> NormalizedFilePath -> Action (Maybe HscEnvEq) -ghcSessionDepsDefinition forceLinkable env file = do + ghcSessionDepsDefinition ghcSessionDepsConfig env file + +data GhcSessionDepsConfig = GhcSessionDepsConfig + { checkForImportCycles :: Bool + , forceLinkables :: Bool + } +instance Default GhcSessionDepsConfig where + def = GhcSessionDepsConfig + { checkForImportCycles = True + , forceLinkables = False + } + +ghcSessionDepsDefinition :: GhcSessionDepsConfig -> HscEnvEq -> NormalizedFilePath -> Action (Maybe HscEnvEq) +ghcSessionDepsDefinition GhcSessionDepsConfig{..} env file = do let hsc = hscEnv env mbdeps <- mapM(fmap artifactFilePath . snd) <$> use_ GetLocatedImports file case mbdeps of Nothing -> return Nothing Just deps -> do - _ <- uses_ ReportImportCycles deps + when checkForImportCycles $ void $ uses_ ReportImportCycles deps ms:mss <- map msrModSummary <$> uses_ GetModSummaryWithoutTimestamps (file:deps) depSessions <- map hscEnv <$> uses_ GhcSessionDeps deps let uses_th_qq = xopt LangExt.TemplateHaskell dflags || xopt LangExt.QuasiQuotes dflags dflags = ms_hspp_opts ms - ifaces <- if uses_th_qq || forceLinkable + ifaces <- if uses_th_qq || forceLinkables then uses_ GetModIface deps else uses_ GetModIfaceWithoutLinkable deps @@ -1043,9 +1055,18 @@ writeHiFileAction hsc hiFile = do resetInterfaceStore extras $ toNormalizedFilePath' targetPath writeHiFile hsc hiFile +data RulesConfig = RulesConfig + { -- | Disable import cycle checking for improved performance in large codebases + checkForImportCycles :: Bool + -- | Disable TH for improved performance in large codebases + , enableTemplateHaskell :: Bool + } + +instance Default RulesConfig where def = RulesConfig True True + -- | A rule that wires per-file rules together -mainRule :: Rules () -mainRule = do +mainRule :: RulesConfig -> Rules () +mainRule RulesConfig{..} = do linkables <- liftIO $ newVar emptyModuleEnv addIdeGlobal $ CompiledLinkables linkables getParsedModuleRule @@ -1055,7 +1076,7 @@ mainRule = do reportImportCyclesRule typeCheckRule getDocMapRule - loadGhcSession + loadGhcSession def{checkForImportCycles} getModIfaceFromDiskRule getModIfaceFromDiskAndIndexRule getModIfaceRule @@ -1073,8 +1094,10 @@ mainRule = do -- * ObjectLinkable -> BCOLinkable : the prev linkable can be reused, signal "no change" -- * Object/BCO -> NoLinkable : the prev linkable can be ignored, signal "no change" -- * otherwise : the prev linkable cannot be reused, signal "value has changed" - defineEarlyCutoff $ RuleWithCustomNewnessCheck (<=) $ \NeedsCompilation file -> - needsCompilationRule file + if enableTemplateHaskell + then defineEarlyCutoff $ RuleWithCustomNewnessCheck (<=) $ \NeedsCompilation file -> + needsCompilationRule file + else defineNoDiagnostics $ \NeedsCompilation _ -> return $ Just Nothing generateCoreRule getImportMapRule getAnnotatedParsedSourceRule diff --git a/ghcide/src/Development/IDE/Main.hs b/ghcide/src/Development/IDE/Main.hs index a732fcd6fb..f35c45d526 100644 --- a/ghcide/src/Development/IDE/Main.hs +++ b/ghcide/src/Development/IDE/Main.hs @@ -187,7 +187,7 @@ defaultArguments priority = Arguments { argsOTMemoryProfiling = False , argCommand = LSP , argsLogger = stderrLogger priority - , argsRules = mainRule >> action kick + , argsRules = mainRule def >> action kick , argsGhcidePlugin = mempty , argsHlsPlugins = pluginDescToIdePlugins Ghcide.descriptors , argsSessionLoadingOptions = def diff --git a/ghcide/test/exe/Main.hs b/ghcide/test/exe/Main.hs index 39ecd961dd..d021feea49 100644 --- a/ghcide/test/exe/Main.hs +++ b/ghcide/test/exe/Main.hs @@ -725,7 +725,6 @@ cancellationTestGroup name edits sessionDepsOutcome parseOutcome tcOutcome = tes , cancellationTemplate edits $ Just ("GetModSummaryWithoutTimestamps", True) -- getLocatedImports never fails , cancellationTemplate edits $ Just ("GetLocatedImports", True) - , cancellationTemplate edits $ Just ("GetDependencies", dependsOutcome) , cancellationTemplate edits $ Just ("GhcSessionDeps", sessionDepsOutcome) , cancellationTemplate edits $ Just ("GetParsedModule", parseOutcome) , cancellationTemplate edits $ Just ("TypeCheck", tcOutcome) diff --git a/plugins/hls-eval-plugin/hls-eval-plugin.cabal b/plugins/hls-eval-plugin/hls-eval-plugin.cabal index d6f0f508fe..e3114e1656 100644 --- a/plugins/hls-eval-plugin/hls-eval-plugin.cabal +++ b/plugins/hls-eval-plugin/hls-eval-plugin.cabal @@ -55,6 +55,7 @@ library , aeson , base >=4.12 && <5 , containers + , data-default , deepseq , Diff ^>=0.4.0 , directory diff --git a/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs b/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs index db2fa7d230..46ab009470 100644 --- a/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs +++ b/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs @@ -35,6 +35,7 @@ import Control.Monad.IO.Class (MonadIO (liftIO)) import Control.Monad.Trans.Except (ExceptT (..)) import Data.Aeson (toJSON) import Data.Char (isSpace) +import Data.Default import qualified Data.HashMap.Strict as HashMap import Data.List (dropWhileEnd, find, intercalate, intersperse) @@ -54,7 +55,8 @@ import Development.IDE (GetModSummary (..), toNormalizedFilePath', uriToFilePath', useNoFile_, useWithStale_, use_) -import Development.IDE.Core.Rules (ghcSessionDepsDefinition) +import Development.IDE.Core.Rules (GhcSessionDepsConfig (..), + ghcSessionDepsDefinition) import Development.IDE.GHC.Compat hiding (typeKind, unitState) import qualified Development.IDE.GHC.Compat as Compat import qualified Development.IDE.GHC.Compat as SrcLoc @@ -536,7 +538,8 @@ runGetSession st nfp = liftIO $ runAction "eval" st $ do let fp = fromNormalizedFilePath nfp ((_, res),_) <- liftIO $ loadSessionFun fp let env = fromMaybe (error $ "Unknown file: " <> fp) res - res <- fmap hscEnv <$> ghcSessionDepsDefinition False env nfp + ghcSessionDepsConfig = def{forceLinkables = True, checkForImportCycles = False} + res <- fmap hscEnv <$> ghcSessionDepsDefinition ghcSessionDepsConfig env nfp return $ fromMaybe (error $ "Unable to load file: " <> fp) res needsQuickCheck :: [(Section, Test)] -> Bool From bf2dbfadf4fbd8dba96b76c08960d17389fe848a Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Sat, 6 Nov 2021 08:10:21 +0000 Subject: [PATCH 15/17] Avoid using the Monoid instance (removed in 9.4 ?????) --- ghcide/src/Development/IDE/Core/Compile.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ghcide/src/Development/IDE/Core/Compile.hs b/ghcide/src/Development/IDE/Core/Compile.hs index de727ca4ad..bf027aacbb 100644 --- a/ghcide/src/Development/IDE/Core/Compile.hs +++ b/ghcide/src/Development/IDE/Core/Compile.hs @@ -108,6 +108,7 @@ import Data.Unique as Unique import Development.IDE.Core.Tracing (withTrace) import qualified Language.LSP.Server as LSP import qualified Language.LSP.Types as LSP +import UniqDFM (emptyUDFM, plusUDFM) import Unsafe.Coerce -- | Given a string buffer, return the string (after preprocessing) and the 'ParsedModule'. @@ -701,7 +702,7 @@ mergeEnvs env extraModSummaries extraMods envs = do (\fc (im, ifr) -> Compat.extendInstalledModuleEnv fc im ifr) prevFinderCache $ zip ims ifrs return $ loadModulesHome extraMods $ env{ - hsc_HPT = foldMap hsc_HPT envs, + hsc_HPT = foldMapBy plusUDFM emptyUDFM hsc_HPT envs, hsc_FC = newFinderCache, hsc_mod_graph = mkModuleGraph $ extraModSummaries ++ nubOrdOn ms_mod (concatMap (mgModSummaries . hsc_mod_graph) envs) } From 9d6ba9e6536bd0090529ee015ec37c97efc9dfac Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Sat, 6 Nov 2021 08:22:58 +0000 Subject: [PATCH 16/17] Fix build with GHC 9 --- ghcide/src/Development/IDE/Core/Compile.hs | 2 +- ghcide/src/Development/IDE/GHC/Compat/Util.hs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ghcide/src/Development/IDE/Core/Compile.hs b/ghcide/src/Development/IDE/Core/Compile.hs index bf027aacbb..75d5870e24 100644 --- a/ghcide/src/Development/IDE/Core/Compile.hs +++ b/ghcide/src/Development/IDE/Core/Compile.hs @@ -106,9 +106,9 @@ import Data.Map (Map) import Data.Tuple.Extra (dupe) import Data.Unique as Unique import Development.IDE.Core.Tracing (withTrace) +import Development.IDE.GHC.Compat.Util (emptyUDFM, plusUDFM) import qualified Language.LSP.Server as LSP import qualified Language.LSP.Types as LSP -import UniqDFM (emptyUDFM, plusUDFM) import Unsafe.Coerce -- | Given a string buffer, return the string (after preprocessing) and the 'ParsedModule'. diff --git a/ghcide/src/Development/IDE/GHC/Compat/Util.hs b/ghcide/src/Development/IDE/GHC/Compat/Util.hs index 198a94c03b..0b1ff0f6c0 100644 --- a/ghcide/src/Development/IDE/GHC/Compat/Util.hs +++ b/ghcide/src/Development/IDE/GHC/Compat/Util.hs @@ -59,6 +59,9 @@ module Development.IDE.GHC.Compat.Util ( Unique, mkUnique, newTagUnique, + -- * UniqDFM + emptyUDFM, + plusUDFM, -- * String Buffer StringBuffer(..), hGetStringBuffer, @@ -76,6 +79,7 @@ import GHC.Data.Maybe import GHC.Data.Pair import GHC.Data.StringBuffer import GHC.Types.Unique +import GHC.Types.Unique.DFM import GHC.Utils.Fingerprint import GHC.Utils.Misc import GHC.Utils.Outputable (pprHsString) @@ -94,6 +98,7 @@ import Pair import Outputable (pprHsString) import Panic hiding (try) import StringBuffer +import UniqDFM import Unique import Util #endif From ef370d52de08f229a2f956bcde5127fe17a77ce8 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Sat, 6 Nov 2021 15:22:04 +0000 Subject: [PATCH 17/17] Fix Eval plugin --- ghcide/src/Development/IDE/Core/Rules.hs | 6 +++++- plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ghcide/src/Development/IDE/Core/Rules.hs b/ghcide/src/Development/IDE/Core/Rules.hs index 4ea001a432..1dc0c0f2a2 100644 --- a/ghcide/src/Development/IDE/Core/Rules.hs +++ b/ghcide/src/Development/IDE/Core/Rules.hs @@ -686,11 +686,13 @@ loadGhcSession ghcSessionDepsConfig = do data GhcSessionDepsConfig = GhcSessionDepsConfig { checkForImportCycles :: Bool , forceLinkables :: Bool + , fullModSummary :: Bool } instance Default GhcSessionDepsConfig where def = GhcSessionDepsConfig { checkForImportCycles = True , forceLinkables = False + , fullModSummary = False } ghcSessionDepsDefinition :: GhcSessionDepsConfig -> HscEnvEq -> NormalizedFilePath -> Action (Maybe HscEnvEq) @@ -702,7 +704,9 @@ ghcSessionDepsDefinition GhcSessionDepsConfig{..} env file = do Nothing -> return Nothing Just deps -> do when checkForImportCycles $ void $ uses_ ReportImportCycles deps - ms:mss <- map msrModSummary <$> uses_ GetModSummaryWithoutTimestamps (file:deps) + ms:mss <- map msrModSummary <$> if fullModSummary + then uses_ GetModSummary (file:deps) + else uses_ GetModSummaryWithoutTimestamps (file:deps) depSessions <- map hscEnv <$> uses_ GhcSessionDeps deps let uses_th_qq = diff --git a/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs b/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs index 46ab009470..ef9541eba0 100644 --- a/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs +++ b/plugins/hls-eval-plugin/src/Ide/Plugin/Eval/CodeLens.hs @@ -50,6 +50,7 @@ import Development.IDE (GetModSummary (..), ModSummaryResult (..), NeedsCompilation (NeedsCompilation), evalGhcEnv, hscEnv, + hscEnvWithImportPaths, prettyPrint, runAction, textToStringBuffer, toNormalizedFilePath', @@ -538,8 +539,12 @@ runGetSession st nfp = liftIO $ runAction "eval" st $ do let fp = fromNormalizedFilePath nfp ((_, res),_) <- liftIO $ loadSessionFun fp let env = fromMaybe (error $ "Unknown file: " <> fp) res - ghcSessionDepsConfig = def{forceLinkables = True, checkForImportCycles = False} - res <- fmap hscEnv <$> ghcSessionDepsDefinition ghcSessionDepsConfig env nfp + ghcSessionDepsConfig = def + { forceLinkables = True + , checkForImportCycles = False + , fullModSummary = True + } + res <- fmap hscEnvWithImportPaths <$> ghcSessionDepsDefinition ghcSessionDepsConfig env nfp return $ fromMaybe (error $ "Unable to load file: " <> fp) res needsQuickCheck :: [(Section, Test)] -> Bool