diff --git a/model/Model.md b/model/Model.md index dba54a1c6..cb29de335 100644 --- a/model/Model.md +++ b/model/Model.md @@ -92,6 +92,7 @@ For those who like concrete details, which might change at any point in the futu ,built :: Step -- when it was actually run ,changed :: Step -- when the result last changed ,depends :: [[Id]] -- dependencies + ,rdepends :: [Id] -- reverse dependencies ,execution :: Float -- duration of last run ,traces :: [Trace] -- a trace of the expensive operations } deriving Show @@ -180,13 +181,13 @@ isn't the case, and "output" would still be clean. > > In you rule `File -(ModTime, [(File, ModTime)]`. Is the time stored for a dependency > -> 1 - the time the dependency has been last used +> 1 - the time the dependency has been last used > > 2 - the dependency last modification when the dependency has been used? > > For example. Let's say B depends on A and A has been modified yesterday. > -> If I'm building B today: scenario (1) would be store +> If I'm building B today: scenario (1) would be store > > database B = (Today, [(A, Today)]) > diff --git a/src/Development/Shake.hs b/src/Development/Shake.hs index ba30db9d4..bac813c2d 100644 --- a/src/Development/Shake.hs +++ b/src/Development/Shake.hs @@ -123,6 +123,7 @@ import Development.Shake.Internal.Value import Development.Shake.Internal.Options import Development.Shake.Internal.Core.Types import Development.Shake.Internal.Core.Action +import Development.Shake.Internal.Core.Build import Development.Shake.Internal.Core.Rules import Development.Shake.Internal.Resource import Development.Shake.Internal.Derived @@ -137,7 +138,6 @@ import Development.Shake.Internal.Rules.File import Development.Shake.Internal.Rules.Files import Development.Shake.Internal.Rules.Oracle import Development.Shake.Internal.Rules.OrderOnly -import Development.Shake.Internal.Rules.Rerun -- $writing -- diff --git a/src/Development/Shake/Database.hs b/src/Development/Shake/Database.hs index b9f3aadb5..a59fb378b 100644 --- a/src/Development/Shake/Database.hs +++ b/src/Development/Shake/Database.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE RankNTypes #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE RecordWildCards #-} @@ -20,6 +21,8 @@ module Development.Shake.Database( shakeWithDatabase, shakeOneShotDatabase, shakeRunDatabase, + shakeRunDatabaseForKeys, + SomeShakeValue(..), shakeLiveFilesDatabase, shakeProfileDatabase, shakeErrorsDatabase, @@ -29,8 +32,11 @@ module Development.Shake.Database( import Control.Concurrent.Extra import Control.Exception import Control.Monad +import Control.Monad.Extra (whenJust) import Control.Monad.IO.Class +import qualified Data.HashSet as HashSet import Data.IORef +import Data.Maybe import General.Cleanup import Development.Shake.Internal.Errors import Development.Shake.Internal.Options @@ -38,6 +44,8 @@ import Development.Shake.Internal.Core.Rules import Development.Shake.Internal.Core.Run import Development.Shake.Internal.Core.Types import Development.Shake.Internal.Rules.Default +import Development.Shake.Internal.Value (SomeShakeValue(..), newKey) +import Development.Shake.Internal.Core.Database (flushDirty, markDirty, getIdFromKey, runLocked) data UseState @@ -135,17 +143,46 @@ shakeErrorsDatabase (ShakeDatabase use s) = -- actions along with a list of actions to run after the database was closed, as added with -- 'Development.Shake.runAfter' and 'Development.Shake.removeFilesAfter'. shakeRunDatabase :: ShakeDatabase -> [Action a] -> IO ([a], [IO ()]) -shakeRunDatabase (ShakeDatabase use s) as = +shakeRunDatabase = shakeRunDatabaseForKeys Nothing + +-- | Given an open 'ShakeDatabase', run both whatever actions were added to the 'Rules', +-- plus the list of 'Action' given here. +-- +-- Requires 'shakeReverseDependencies', otherwise it falls back to 'shakeRunDatabase'. +-- +-- If a set of dirty keys is given, only the reverse dependencies of these keys +-- will be considered potentially changed; all other keys will be assumed unchanged. +-- This includes the 'AlwaysRerunQ' key which is by default always dirty, but +-- will not here, unless it is included in the input. +-- +-- Returns the results from the explicitly passed actions along with a list +-- of actions to run after the database was closed, as added with +-- 'Development.Shake.runAfter' and 'Development.Shake.removeFilesAfter'. +shakeRunDatabaseForKeys + :: Maybe [SomeShakeValue] -- ^ Set of keys changed since last run + -> ShakeDatabase + -> [Action a] + -> IO ([a], [IO ()]) +shakeRunDatabaseForKeys keysChanged (ShakeDatabase use s) as = uninterruptibleMask $ \continue -> withOpen use "shakeRunDatabase" (\o -> o{openRequiresReset=True}) $ \Open{..} -> do when openRequiresReset $ do when openOneShot $ throwM $ errorStructured "Error when calling shakeRunDatabase twice, after calling shakeOneShotDatabase" [] "" reset s - (refs, as) <- fmap unzip $ forM as $ \a -> do - ref <- newIORef Nothing - pure (ref, liftIO . writeIORef ref . Just =<< a) - after <- run s openOneShot $ map void as - results <- mapM readIORef refs - case sequence results of - Just result -> pure (result, after) - Nothing -> throwM $ errorInternal "Expected all results were written, but some where not" + runLocked (database s) $ flushDirty (database s) + + -- record the keys changed and continue + whenJust keysChanged $ \kk -> do + getId <- getIdFromKey (database s) + let ids = mapMaybe (\(SomeShakeValue x) -> getId $ newKey x) kk + markDirty (database s) $ HashSet.fromList ids + + continue $ do + (refs, as) <- fmap unzip $ forM as $ \a -> do + ref <- newIORef Nothing + pure (ref, liftIO . writeIORef ref . Just =<< a) + after <- run s openOneShot (isJust keysChanged) $ map void as + results <- mapM readIORef refs + case sequence results of + Just result -> pure (result, after) + Nothing -> throwM $ errorInternal "Expected all results were written, but some where not" \ No newline at end of file diff --git a/src/Development/Shake/Internal/Core/Build.hs b/src/Development/Shake/Internal/Core/Build.hs index cab866b3a..e65a1add1 100644 --- a/src/Development/Shake/Internal/Core/Build.hs +++ b/src/Development/Shake/Internal/Core/Build.hs @@ -7,6 +7,7 @@ module Development.Shake.Internal.Core.Build( historyIsEnabled, historySave, historyLoad, applyKeyValue, apply, apply1, + alwaysRerun ) where import Development.Shake.Classes @@ -19,6 +20,7 @@ import Development.Shake.Internal.Core.Action import Development.Shake.Internal.History.Shared import Development.Shake.Internal.History.Cloud import Development.Shake.Internal.Options +import Development.Shake.Internal.Rules.Rerun import Development.Shake.Internal.Core.Monad import General.Wait import qualified Data.ByteString.Char8 as BS @@ -30,6 +32,7 @@ import Control.Exception import Control.Monad.Extra import Numeric.Extra import qualified Data.HashMap.Strict as Map +import qualified Data.HashSet as HashSet import Development.Shake.Internal.Core.Rules import Data.Typeable import Data.Maybe @@ -101,41 +104,91 @@ buildOne global@Global{..} stack database i k r = case addStack i k stack of pure $ Left e Right stack -> Later $ \continue -> do setIdKeyStatus global database i k (Running (NoShow continue) r) - let go = buildRunMode global stack database r + let go = buildRunMode global stack database i r fromLater go $ \mode -> liftIO $ addPool PoolStart globalPool $ - runKey global stack k r mode $ \res -> do + runKey global stack k r mode $ \result -> do runLocked database $ do - let val = fmap runValue res + let val = fmap runValue result res <- liftIO $ getKeyValueFromId database i w <- case res of Just (_, Running (NoShow w) _) -> pure w -- We used to be able to hit here, but we fixed it by ensuring the thread pool workers are all -- dead _before_ any exception bubbles up _ -> throwM $ errorInternal $ "expected Waiting but got " ++ maybe "nothing" (statusType . snd) res ++ ", key " ++ show k - setIdKeyStatus global database i k $ either mkError Ready val + + -- Make sure that the reverse dependencies are marked to avoid unsoundness + maskLocked $ do + setIdKeyStatus global database i k $ either mkError Ready val + liftIO $ unmarkDirty database i + + -- update reverse dependencies efficiently - have they changed since last time? + case result of + Right RunResult{..} + | shakeReverseDependencies globalOptions && + runChanged `elem` [ChangedRecomputeDiff, ChangedRecomputeSame ] -> + updateReverseDeps i database (depends <$> r) (depends runValue) + _ -> pure () + w val - case res of + case result of Right RunResult{..} | runChanged /= ChangedNothing -> setDisk database i k $ Loaded runValue{result=runStore} _ -> pure () where mkError e = Failed e $ if globalOneShot then Nothing else r + +-- | Refresh all the reverse dependencies of an id +updateReverseDeps :: Id -> Database -> Maybe [Depends] -> [Depends] -> Locked () +updateReverseDeps myId db prev new = {-# SCC "updateReverseDeps" #-} do + let added = foldMap fromDepends new + deleted = [] -- an efficient impl. is expensive in space, so we overestimate for now + forM_ added $ doOne (HashSet.insert myId) + forM_ deleted $ doOne (HashSet.delete myId) + where + doOne f id = do + rdeps <- liftIO $ getReverseDependencies db id + setReverseDependencies db id (f $ fromMaybe mempty rdeps) + -- | Compute the value for a given RunMode and a restore function to run -buildRunMode :: Global -> Stack -> Database -> Maybe (Result a) -> Wait Locked RunMode -buildRunMode global stack database me = do - changed <- case me of +buildRunMode :: Global -> Stack -> Database -> Id -> Maybe (Result a) -> Wait Locked RunMode +buildRunMode global stack database i r = do + changed <- case r of Nothing -> pure True - Just me -> buildRunDependenciesChanged global stack database me + Just me -> do + isDirty <- liftIO $ if globalUseDirtySet global then isDirty database i else pure True + if isDirty + -- Event if I am dirty, it is still possible that all my dependencies are unchanged + -- thanks to early cutoff, and therefore we must check to avoid redundant work + then buildRunDependenciesChanged global stack database i me + -- If I am not dirty then none of my dependencies are, so they must be unchanged + else do + -- The only exception is rules with a direct dependency on alwaysRerun + lookup <- liftIO $ getIdFromKey database + let alwaysRerunId = lookup $ newKey $ AlwaysRerunQ () + pure $ case alwaysRerunId of + Nothing -> False + Just id -> any (\(Depends x) -> id `elem` x) (depends me) pure $ if changed then RunDependenciesChanged else RunDependenciesSame +isDirtyOrAlwaysRerun :: MonadIO m => DatabasePoly Key v -> m (Id -> Bool) +isDirtyOrAlwaysRerun database = do + lookup <- liftIO $ getIdFromKey database + dirtySet <- liftIO $ getDirtySet database + let alwaysRerunId = lookup $ newKey $ AlwaysRerunQ () + pure $ \id -> Just id == alwaysRerunId || id `HashSet.member` dirtySet -- | Have the dependencies changed -buildRunDependenciesChanged :: Global -> Stack -> Database -> Result a -> Wait Locked Bool -buildRunDependenciesChanged global stack database me = isJust <$> firstJustM id - [firstJustWaitUnordered (fmap test . lookupOne global stack database) x | Depends x <- depends me] +buildRunDependenciesChanged :: Global -> Stack -> Database -> Id -> Result a -> Wait Locked Bool +buildRunDependenciesChanged global stack database i r = do + isDirty <- isDirtyOrAlwaysRerun database + isJust <$> firstJustM id + [firstJustWaitUnordered (fmap test . lookupOne global stack database) x' + | Depends x <- depends r + , let x' = if globalUseDirtySet global then filter isDirty x else x + ] where - test (Right dep) | changed dep <= built me = Nothing + test (Right dep) | changed dep <= built r = Nothing test _ = Just () @@ -341,3 +394,25 @@ runIdentify :: Map.HashMap TypeRep BuiltinRule -> Key -> Value -> Maybe BS.ByteS runIdentify mp k v | Just BuiltinRule{..} <- Map.lookup (typeKey k) mp = builtinIdentity k v | otherwise = throwImpure $ errorInternal "runIdentify can't find rule" + +------------------------------------------------------------------------------- +-- SPECIAL RULES + +-- | Always rerun the associated action. Useful for defining rules that query +-- the environment. For example: +-- +-- @ +-- \"ghcVersion.txt\" 'Development.Shake.%>' \\out -> do +-- 'alwaysRerun' +-- 'Development.Shake.Stdout' stdout <- 'Development.Shake.cmd' \"ghc --numeric-version\" +-- 'Development.Shake.writeFileChanged' out stdout +-- @ +-- +-- In @make@, the @.PHONY@ attribute on file-producing rules has a similar effect. +-- +-- Note that 'alwaysRerun' is applied when a rule is executed. Modifying an existing rule +-- to insert 'alwaysRerun' will /not/ cause that rule to rerun next time. +alwaysRerun :: Action () +alwaysRerun = do + historyDisable + apply1 $ AlwaysRerunQ () \ No newline at end of file diff --git a/src/Development/Shake/Internal/Core/Database.hs b/src/Development/Shake/Internal/Core/Database.hs index cff4d50b2..e3ad365f8 100644 --- a/src/Development/Shake/Internal/Core/Database.hs +++ b/src/Development/Shake/Internal/Core/Database.hs @@ -1,12 +1,16 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE GeneralizedNewtypeDeriving, RecordWildCards #-} +{-# LANGUAGE TupleSections #-} module Development.Shake.Internal.Core.Database( Locked, runLocked, + maskLocked, DatabasePoly, createDatabase, mkId, getValueFromKey, getIdFromKey, getKeyValues, getKeyValueFromId, getKeyValuesFromId, - setMem, setDisk, modifyAllMem + setMem, setDisk, modifyAllMem, + isDirty, getDirtySet, markDirty, unmarkDirty, flushDirty, + getReverseDependencies, setReverseDependencies ) where import Data.Tuple.Extra @@ -20,6 +24,9 @@ import Control.Monad.IO.Class import qualified General.Ids as Ids import Control.Monad.Fail import Prelude +import Control.Exception (mask_) +import Data.HashSet (HashSet) +import qualified Data.HashSet as HSet newtype Locked a = Locked (IO a) @@ -28,6 +35,8 @@ newtype Locked a = Locked (IO a) runLocked :: DatabasePoly k v -> Locked b -> IO b runLocked db (Locked act) = withLock (lock db) act +maskLocked :: Locked a -> Locked a +maskLocked (Locked act) = Locked $ mask_ act -- | Invariant: The database does not have any cycles where a Key depends on itself. -- Everything is mutable. intern and status must form a bijecttion. @@ -37,21 +46,27 @@ data DatabasePoly k v = Database {lock :: Lock ,intern :: IORef (Intern k) -- ^ Key |-> Id mapping ,status :: Ids.Ids (k, v) -- ^ Id |-> (Key, Status) mapping + ,rdeps :: Ids.Ids (HashSet Id) -- ^ Id |-> reverse dependencies ,journal :: Id -> k -> v -> IO () -- ^ Record all changes to status ,vDefault :: v + ,clean,dirty :: IORef (HashSet Id) + -- ^ An approximation of the dirty set across runs of 'shakeRunDatabaseForKeys' } createDatabase :: (Eq k, Hashable k) => Ids.Ids (k, v) + -> Ids.Ids (HashSet Id) -> (Id -> k -> v -> IO ()) -> v -> IO (DatabasePoly k v) -createDatabase status journal vDefault = do +createDatabase status rdeps journal vDefault = do xs <- Ids.toList status intern <- newIORef $ Intern.fromList [(k, i) | (i, (k,_)) <- xs] lock <- newLock + dirty <- newIORef mempty + clean <- newIORef mempty pure Database{..} @@ -80,6 +95,14 @@ getIdFromKey Database{..} = do is <- readIORef intern pure $ flip Intern.lookup is +isDirty :: DatabasePoly k v -> Id -> IO Bool +isDirty Database{..} i = HSet.member i <$> readIORef dirty + +getDirtySet :: DatabasePoly k v -> IO (HashSet Id) +getDirtySet Database{..} = readIORef dirty + +getReverseDependencies :: DatabasePoly k v -> Id -> IO (Maybe (HashSet Id)) +getReverseDependencies Database{..} = Ids.lookup rdeps --------------------------------------------------------------------- -- MUTATING @@ -101,6 +124,9 @@ mkId Database{..} k = liftIO $ do setMem :: DatabasePoly k v -> Id -> k -> v -> Locked () setMem Database{..} i k v = liftIO $ Ids.insert status i (k,v) +setReverseDependencies :: DatabasePoly k v -> Id -> HashSet Id -> Locked () +setReverseDependencies Database{..} = (liftIO.) . Ids.insert rdeps + modifyAllMem :: DatabasePoly k v -> (v -> v) -> Locked () modifyAllMem Database{..} f = liftIO $ Ids.forMutate status $ \(k,v) -> let !v' = f v @@ -108,3 +134,15 @@ modifyAllMem Database{..} f = liftIO $ Ids.forMutate status $ \(k,v) -> setDisk :: DatabasePoly k v -> Id -> k -> v -> IO () setDisk = journal + +markDirty :: DatabasePoly k v -> HashSet Id -> IO () +markDirty Database{..} ids = atomicModifyIORef'_ dirty $ HSet.union ids + +unmarkDirty :: DatabasePoly k v -> Id -> IO () +unmarkDirty Database{..} i = do + atomicModifyIORef'_ clean (HSet.insert i) + +flushDirty :: DatabasePoly k v -> Locked () +flushDirty Database{..} = liftIO $ do + cleanIds <- atomicModifyIORef' clean (mempty,) + atomicModifyIORef'_ dirty (`HSet.difference` cleanIds) \ No newline at end of file diff --git a/src/Development/Shake/Internal/Core/Run.hs b/src/Development/Shake/Internal/Core/Run.hs index 680a9378b..68e95e8f3 100644 --- a/src/Development/Shake/Internal/Core/Run.hs +++ b/src/Development/Shake/Internal/Core/Run.hs @@ -3,8 +3,10 @@ {-# LANGUAGE ConstraintKinds, TupleSections, ViewPatterns #-} {-# LANGUAGE TypeFamilies, NamedFieldPuns #-} +{-# LANGUAGE RankNTypes #-} module Development.Shake.Internal.Core.Run( RunState, + database, open, reset, run, @@ -19,6 +21,7 @@ import Data.Tuple.Extra import Control.Concurrent.Extra hiding (withNumCapabilities) import Development.Shake.Internal.Core.Database import Control.Monad.IO.Class +import qualified Control.Monad.Trans.State.Strict as State import General.Binary import Development.Shake.Classes import Development.Shake.Internal.Core.Storage @@ -52,8 +55,11 @@ import General.Timing import General.Thread import General.Extra import General.Cleanup +import qualified General.Ids as Ids +import Data.Foldable (traverse_) import Data.Monoid import Prelude +import Text.Printf --------------------------------------------------------------------- @@ -100,8 +106,13 @@ reset RunState{..} = runLocked database $ f x = x -run :: RunState -> Bool -> [Action ()] -> IO [IO ()] -run RunState{..} oneshot actions2 = +run + :: RunState + -> Bool -- ^ oneshot + -> Bool -- ^ use dirty set + -> [Action ()] + -> IO [IO ()] +run RunState{..} oneshot useDirtySet actions2 = withInit opts $ \opts@ShakeOptions{..} diagnostic output -> do -- timings are a bit delicate, we want to make sure we clear them before we leave (so each run is fresh) @@ -141,8 +152,12 @@ run RunState{..} oneshot actions2 = addTiming "Running rules" locals <- newIORef [] + + let reallyUseDirtySet = useDirtySet && shakeReverseDependencies + when reallyUseDirtySet $ updateDirtySet diagnostic database + runPool (shakeThreads == 1) shakeThreads $ \pool -> do - let global = Global applyKeyValue database pool cleanup start builtinRules output opts diagnostic ruleFinished after absent getProgress userRules shared cloud step oneshot + let global = Global applyKeyValue database pool cleanup start builtinRules output opts diagnostic ruleFinished after absent getProgress userRules shared cloud step oneshot reallyUseDirtySet -- give each action a stack to start with! forM_ (actions ++ map (emptyStack,) actions2) $ \(stack, act) -> do let local = newLocal stack shakeVerbosity @@ -158,7 +173,7 @@ run RunState{..} oneshot actions2 = end <- start if null actions && null actions2 then putWhen Info "Warning: No want/action statements, nothing to do" - else + else recordRoot step locals end database when (isJust shakeLint) $ do @@ -187,6 +202,28 @@ run RunState{..} oneshot actions2 = putStr . unlines pure res +-- | Transitively expand the dirty set +updateDirtySet :: (IO String -> IO()) -> Database -> IO () +updateDirtySet diag database = {-# SCC updateDirtySet #-} do + let loop x = do + seen <- State.get + if x `Set.member` seen then pure () else do + State.put (Set.insert x seen) + next <- liftIO $ getReverseDependencies database x + traverse_ loop (fromMaybe mempty next) + ids <- getDirtySet database + transitive <- flip State.execStateT Set.empty $ traverse_ loop ids + + markDirty database transitive + + diag $ do + dirtySet <- getDirtySet database + let st = Set.size dirtySet + res = take 500 $ Set.toList dirtySet + ellipsis = if st > 500 then "..." else "" + keys <- unlines . map (show . fst) . catMaybes <$> mapM (getKeyValueFromId database) res + pure $ printf "%d dirty set: \n%s\n%s" st keys ellipsis + -- | Run a set of IO actions, treated as \"after\" actions, typically returned from -- 'Development.Shake.Database.shakeRunDatabase'. The actions will be run with diagnostics @@ -323,8 +360,9 @@ usingDatabase cleanup opts diagnostic owitness = do [ (QTypeRep t, (version, BinaryOp (putDatabase putOp) (getDatabase getOp))) | (t,(version, BinaryOp{..})) <- step : root : Map.toList (Map.map (\BuiltinRule{..} -> (builtinVersion, builtinKey)) owitness)] (status, journal) <- usingStorage cleanup opts diagnostic witness + rdeps <- Ids.empty journal<- pure $ \i k v -> journal (QTypeRep $ typeKey k) i (k, v) - createDatabase status journal Missing + createDatabase status rdeps journal Missing incrementStep :: Database -> IO Step @@ -380,8 +418,8 @@ loadSharedCloud var opts owitness = do putDatabase :: (Key -> Builder) -> ((Key, Status) -> Builder) -putDatabase putKey (key, Loaded (Result x1 x2 x3 x4 x5 x6)) = - putExN (putKey key) <> putExN (putEx x1) <> putEx x2 <> putEx x3 <> putEx x5 <> putExN (putEx x4) <> putEx x6 +putDatabase putKey (key, Loaded (Result x1 x2 x3 x4 x6 x7)) = + putExN (putKey key) <> putExN (putEx x1) <> putEx x2 <> putEx x3 <> putEx x6 <> putExN (putEx x4) <> putEx x7 putDatabase _ (_, x) = throwImpure $ errorInternal $ "putWith, Cannot write Status with constructor " ++ statusType x @@ -389,6 +427,6 @@ getDatabase :: (BS.ByteString -> Key) -> BS.ByteString -> (Key, Status) getDatabase getKey bs | (key, bs) <- getExN bs , (x1, bs) <- getExN bs - , (x2, x3, x5, bs) <- binarySplit3 bs - , (x4, x6) <- getExN bs - = (getKey key, Loaded (Result x1 x2 x3 (getEx x4) x5 (getEx x6))) + , (x2, x3, x6, bs) <- binarySplit3 bs + , (x4, x7) <- getExN bs + = (getKey key, Loaded (Result x1 x2 x3 (getEx x4) x6 (getEx x7))) diff --git a/src/Development/Shake/Internal/Core/Types.hs b/src/Development/Shake/Internal/Core/Types.hs index 51652a52d..8b335162e 100755 --- a/src/Development/Shake/Internal/Core/Types.hs +++ b/src/Development/Shake/Internal/Core/Types.hs @@ -239,7 +239,10 @@ data Result a = Result ,depends :: ![Depends] -- ^ dependencies (don't run them early) ,execution :: {-# UNPACK #-} !Float -- ^ how long it took when it was last run (seconds) ,traces :: ![Trace] -- ^ a trace of the expensive operations (start/end in seconds since beginning of run) - } deriving (Show,Functor) + } deriving (Functor) + +instance Show (Result a) where + show _ = "" instance NFData a => NFData (Result a) where -- ignore unpacked fields @@ -453,6 +456,7 @@ data Global = Global ,globalCloud :: Maybe Cloud ,globalStep :: {-# UNPACK #-} !Step ,globalOneShot :: Bool -- ^ I am running in one-shot mode so don't need to store BS's for Result/Failed + ,globalUseDirtySet :: Bool -- ^ Use the dirty set approximation when running rules } -- local variables of Action diff --git a/src/Development/Shake/Internal/Options.hs b/src/Development/Shake/Internal/Options.hs index 1a15bcda6..8cd9178a4 100644 --- a/src/Development/Shake/Internal/Options.hs +++ b/src/Development/Shake/Internal/Options.hs @@ -216,6 +216,9 @@ data ShakeOptions = ShakeOptions -- undefined results. Provided for compatibility with @ninja@. ,shakeAllowRedefineRules :: Bool -- ^ Whether to allow calling addBuiltinRule for the same key more than once + ,shakeReverseDependencies :: Bool + -- ^ Enables the tracking of reverse dependencies, + -- used by `shakeRunDatabaseForKeys' to reduce internal overheads when checking for rule freshness. ,shakeProgress :: IO Progress -> IO () -- ^ Defaults to no action. A function called when the build starts, allowing progress to be reported. -- The function is called on a separate thread, and that thread is killed when the build completes. @@ -240,7 +243,7 @@ data ShakeOptions = ShakeOptions shakeOptions :: ShakeOptions shakeOptions = ShakeOptions ".shake" 1 "1" Info False [] Nothing [] [] [] [] (Just 10) [] [] False True False - True ChangeModtime True [] False False Nothing [] False False False + True ChangeModtime True [] False False Nothing [] False False False False (const $ pure ()) (const $ BS.putStrLn . UTF8.fromString) -- try and output atomically using BS (\_ _ _ -> pure ()) @@ -252,20 +255,20 @@ fieldsShakeOptions = ,"shakeFlush", "shakeRebuild", "shakeAbbreviations", "shakeStorageLog" ,"shakeLineBuffering", "shakeTimings", "shakeRunCommands", "shakeChange", "shakeCreationCheck" ,"shakeLiveFiles", "shakeVersionIgnore", "shakeColor", "shakeShare", "shakeCloud", "shakeSymlink" - ,"shakeNeedDirectory", "shakeCanRedefineRules" + ,"shakeNeedDirectory", "shakeCanRedefineRules", "shakeReverseDependencies" ,"shakeProgress", "shakeOutput", "shakeTrace", "shakeExtra"] tyShakeOptions = mkDataType "Development.Shake.Types.ShakeOptions" [conShakeOptions] conShakeOptions = mkConstr tyShakeOptions "ShakeOptions" fieldsShakeOptions Prefix -unhide x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 x22 x23 x24 x25 x26 x27 x28 y1 y2 y3 y4 = - ShakeOptions x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 x22 x23 x24 x25 x26 x27 x28 +unhide x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 x22 x23 x24 x25 x26 x27 x28 x29 y1 y2 y3 y4 = + ShakeOptions x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 x22 x23 x24 x25 x26 x27 x28 x29 (fromHidden y1) (fromHidden y2) (fromHidden y3) (fromHidden y4) instance Data ShakeOptions where - gfoldl k z (ShakeOptions x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 x22 x23 x24 x25 x26 x27 x28 y1 y2 y3 y4) = + gfoldl k z (ShakeOptions x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 x16 x17 x18 x19 x20 x21 x22 x23 x24 x25 x26 x27 x28 x29 y1 y2 y3 y4) = z unhide `k` x1 `k` x2 `k` x3 `k` x4 `k` x5 `k` x6 `k` x7 `k` x8 `k` x9 `k` x10 `k` x11 `k` - x12 `k` x13 `k` x14 `k` x15 `k` x16 `k` x17 `k` x18 `k` x19 `k` x20 `k` x21 `k` x22 `k` x23 `k` x24 `k` x25 `k` x26 `k` x27 `k` x28 `k` + x12 `k` x13 `k` x14 `k` x15 `k` x16 `k` x17 `k` x18 `k` x19 `k` x20 `k` x21 `k` x22 `k` x23 `k` x24 `k` x25 `k` x26 `k` x27 `k` x28 `k` x29 `k` Hidden y1 `k` Hidden y2 `k` Hidden y3 `k` Hidden y4 - gunfold k z _ = k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ z unhide + gunfold k z _ = k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ k $ z unhide toConstr ShakeOptions{} = conShakeOptions dataTypeOf _ = tyShakeOptions diff --git a/src/Development/Shake/Internal/Rules/File.hs b/src/Development/Shake/Internal/Rules/File.hs index 3946ee2f5..857db5cdc 100644 --- a/src/Development/Shake/Internal/Rules/File.hs +++ b/src/Development/Shake/Internal/Rules/File.hs @@ -29,7 +29,6 @@ import Development.Shake.Internal.Core.Rules import Development.Shake.Internal.Core.Build import Development.Shake.Internal.Core.Action import Development.Shake.Internal.FileName -import Development.Shake.Internal.Rules.Rerun import Development.Shake.Classes import Development.Shake.FilePath(toStandard) import Development.Shake.Internal.FilePattern diff --git a/src/Development/Shake/Internal/Rules/Files.hs b/src/Development/Shake/Internal/Rules/Files.hs index a490578d8..05e89cea1 100644 --- a/src/Development/Shake/Internal/Rules/Files.hs +++ b/src/Development/Shake/Internal/Rules/Files.hs @@ -20,7 +20,6 @@ import Development.Shake.Internal.Errors import General.Extra import Development.Shake.Internal.FileName import Development.Shake.Classes -import Development.Shake.Internal.Rules.Rerun import Development.Shake.Internal.Rules.File import Development.Shake.Internal.FilePattern import Development.Shake.FilePath diff --git a/src/Development/Shake/Internal/Rules/Rerun.hs b/src/Development/Shake/Internal/Rules/Rerun.hs index a9a13b6ba..c82ffbac5 100644 --- a/src/Development/Shake/Internal/Rules/Rerun.hs +++ b/src/Development/Shake/Internal/Rules/Rerun.hs @@ -2,13 +2,11 @@ {-# LANGUAGE TypeFamilies #-} module Development.Shake.Internal.Rules.Rerun( - defaultRuleRerun, alwaysRerun + defaultRuleRerun, AlwaysRerunQ(..) ) where import Development.Shake.Internal.Core.Rules import Development.Shake.Internal.Core.Types -import Development.Shake.Internal.Core.Build -import Development.Shake.Internal.Core.Action import Development.Shake.Classes import qualified Data.ByteString as BS import General.Binary @@ -21,24 +19,6 @@ instance Show AlwaysRerunQ where show _ = "alwaysRerun" type instance RuleResult AlwaysRerunQ = () --- | Always rerun the associated action. Useful for defining rules that query --- the environment. For example: --- --- @ --- \"ghcVersion.txt\" 'Development.Shake.%>' \\out -> do --- 'alwaysRerun' --- 'Development.Shake.Stdout' stdout <- 'Development.Shake.cmd' \"ghc --numeric-version\" --- 'Development.Shake.writeFileChanged' out stdout --- @ --- --- In @make@, the @.PHONY@ attribute on file-producing rules has a similar effect. --- --- Note that 'alwaysRerun' is applied when a rule is executed. Modifying an existing rule --- to insert 'alwaysRerun' will /not/ cause that rule to rerun next time. -alwaysRerun :: Action () -alwaysRerun = do - historyDisable - apply1 $ AlwaysRerunQ () defaultRuleRerun :: Rules () defaultRuleRerun = diff --git a/src/Development/Shake/Internal/Value.hs b/src/Development/Shake/Internal/Value.hs index 536449cb7..89de03146 100644 --- a/src/Development/Shake/Internal/Value.hs +++ b/src/Development/Shake/Internal/Value.hs @@ -5,7 +5,7 @@ module Development.Shake.Internal.Value( Value, newValue, fromValue, Key, newKey, fromKey, typeKey, - ShakeValue + ShakeValue, SomeShakeValue(..) ) where import Development.Shake.Classes @@ -49,6 +49,12 @@ import Unsafe.Coerce -- when Shake is parallelized. type ShakeValue a = (Show a, Typeable a, Eq a, Hashable a, Binary a, NFData a) +data SomeShakeValue = forall k . ShakeValue k => SomeShakeValue k + +instance Eq SomeShakeValue where SomeShakeValue a == SomeShakeValue b = cast a == Just b +instance Hashable SomeShakeValue where hashWithSalt s (SomeShakeValue x) = hashWithSalt s x +instance Show SomeShakeValue where show (SomeShakeValue x) = show x + -- We deliberately avoid Typeable instances on Key/Value to stop them accidentally -- being used inside themselves data Key = forall a . Key