Merge pull request #5849 from unisonweb/improve-update-perf

improve `update` performance

Author: aryairani

codebase2/codebase-sqlite/U/Codebase/Sqlite/Operations.hs

@@ -214,7 +214,7 @@ import Unison.NameSegment.Internal qualified as NameSegment
 import Unison.Prelude
 import Unison.ShortHash (ShortCausalHash (..), ShortNamespaceHash (..))
 import Unison.Sqlite
-import Unison.Util.Defns (DefnsF)
+import Unison.Util.Defns (Defns (..), DefnsF)
 import Unison.Util.List qualified as List
 import Unison.Util.Map qualified as Map
 import Unison.Util.Monoid (foldMapM)
@@ -1155,21 +1155,23 @@ directDependentsWithinScope scope0 query0 = do
 -- | `transitiveDependentsWithinScope scope query` returns all transitive dependents of `query` that are in `scope` (not
 -- including `query` itself).
 transitiveDependentsWithinScope ::
-  Set C.Reference.Id ->
+  DefnsF Set C.TermReferenceId C.TypeReferenceId ->
   Set C.Reference ->
   Transaction (DefnsF Set C.TermReferenceId C.TypeReferenceId)
-transitiveDependentsWithinScope scope0 query0 = do
-  -- Convert C -> S
-  scope1 <- Set.traverse c2sReferenceId scope0
-  query1 <- Set.traverse c2sReference query0
+transitiveDependentsWithinScope scope0 query0
+  | (Set.null scope0.terms && Set.null scope0.types) || Set.null query0 = pure (Defns Set.empty Set.empty)
+  | otherwise = do
+      -- Convert C -> S
+      scope1 <- bitraverse (Set.traverse c2sReferenceId) (Set.traverse c2sReferenceId) scope0
+      query1 <- Set.traverse c2sReference query0
 
-  -- Do the query
-  dependents0 <- Q.getTransitiveDependentsWithinScope scope1 query1
+      -- Do the query
+      dependents0 <- Q.getTransitiveDependentsWithinScope scope1 query1
 
-  -- Convert S -> C
-  dependents1 <- bitraverse (Set.traverse s2cReferenceId) (Set.traverse s2cReferenceId) dependents0
+      -- Convert S -> C
+      dependents1 <- bitraverse (Set.traverse s2cReferenceId) (Set.traverse s2cReferenceId) dependents0
 
-  pure dependents1
+      pure dependents1
 
 -- | returns a list of known definitions referencing `h`
 dependentsOfComponent :: H.Hash -> Transaction (Set C.Reference.Id)

codebase2/codebase-sqlite/U/Codebase/Sqlite/Queries.hs

@@ -273,6 +273,7 @@ module U.Codebase.Sqlite.Queries
     addProjectBranchLastAccessedColumn,
     addMergeBranchTables,
     addUpdateBranchTable,
+    addDerivedDependentsByDependencyIndex,
 
     -- ** schema version
     currentSchemaVersion,
@@ -318,6 +319,7 @@ import Control.Monad.Writer (MonadWriter, runWriterT)
 import Control.Monad.Writer qualified as Writer
 import Data.Aeson qualified as Aeson
 import Data.Aeson.Text qualified as Aeson
+import Data.Bifoldable (bifold)
 import Data.Bitraversable (bitraverse)
 import Data.ByteString.Lazy (LazyByteString)
 import Data.Bytes.Put (runPutS)
@@ -441,7 +443,7 @@ type TextPathSegments = [Text]
 -- * main squeeze
 
 currentSchemaVersion :: SchemaVersion
-currentSchemaVersion = 20
+currentSchemaVersion = 21
 
 runCreateSql :: Transaction ()
 runCreateSql =
@@ -519,6 +521,10 @@ addUpdateBranchTable :: Transaction ()
 addUpdateBranchTable =
   executeStatements $(embedProjectStringFile "sql/017-add-update-branch-table.sql")
 
+addDerivedDependentsByDependencyIndex :: Transaction ()
+addDerivedDependentsByDependencyIndex =
+  executeStatements $(embedProjectStringFile "sql/018-add-derived-dependents-by-dependency-index.sql")
+
 schemaVersion :: Transaction SchemaVersion
 schemaVersion =
   queryOneCol
@@ -1969,13 +1975,13 @@ getDirectDependentsWithinScope scope query = do
 -- | `getTransitiveDependentsWithinScope scope query` returns all transitive dependents of `query` that are in `scope`
 -- (not including `query` itself).
 getTransitiveDependentsWithinScope ::
-  Set S.Reference.Id ->
+  DefnsF Set S.TermReferenceId S.TypeReferenceId ->
   Set S.Reference ->
   Transaction (DefnsF Set S.TermReferenceId S.TypeReferenceId)
 getTransitiveDependentsWithinScope scope query = do
   -- Populate a temporary table with all of the references in `scope`
   let scopeTableName = [sql| dependents_search_scope |]
-  createTemporaryTableOfReferenceIds scopeTableName scope
+  createTemporaryTableOfReferenceIds scopeTableName (bifold scope)
 
   -- Populate a temporary table with all of the references in `query`
   let queryTableName = [sql| dependencies_query |]
@@ -2002,29 +2008,33 @@ getTransitiveDependentsWithinScope scope query = do
   result0 :: [S.Reference.Id :. Only ObjectType] <-
     queryListRow
       [sql|
-        WITH RECURSIVE transitive_dependents (dependent_object_id, dependent_component_index, type_id) AS (
-          SELECT d.dependent_object_id, d.dependent_component_index, object.type_id
-          FROM dependents_index d
-          JOIN object ON d.dependent_object_id = object.id
-          JOIN $queryTableName q
-            ON q.builtin IS d.dependency_builtin
-            AND q.object_id IS d.dependency_object_id
-            AND q.component_index IS d.dependency_component_index
-          JOIN $scopeTableName s
-            ON s.object_id = d.dependent_object_id
-            AND s.component_index = d.dependent_component_index
-
-          UNION SELECT d.dependent_object_id, d.dependent_component_index, object.type_id
-          FROM dependents_index d
-          JOIN object ON d.dependent_object_id = object.id
-          JOIN transitive_dependents t
-            ON t.dependent_object_id = d.dependency_object_id
-            AND t.dependent_component_index = d.dependency_component_index
-          JOIN $scopeTableName s
-            ON s.object_id = d.dependent_object_id
-            AND s.component_index = d.dependent_component_index
+        WITH RECURSIVE
+        dependents_index_in_scope AS (
+          SELECT *
+          FROM dependents_index
+          WHERE (dependent_object_id, dependent_component_index) IN (
+            SELECT object_id, component_index
+            FROM $scopeTableName
+          )
+        ),
+        transitive_dependents (object_id, component_index, type_id) AS (
+          SELECT d.dependent_object_id, d.dependent_component_index, o.type_id
+          FROM $queryTableName q
+            JOIN dependents_index_in_scope d
+              ON d.dependency_builtin IS q.builtin
+                AND d.dependency_object_id IS q.object_id
+                AND d.dependency_component_index IS q.component_index
+            JOIN object o ON d.dependent_object_id = o.id
+          UNION
+          SELECT d.dependent_object_id, d.dependent_component_index, o.type_id
+          FROM transitive_dependents t
+            JOIN dependents_index_in_scope d
+              ON t.object_id = d.dependency_object_id
+              AND t.component_index = d.dependency_component_index
+            JOIN object o ON d.dependent_object_id = o.id
         )
-        SELECT * FROM transitive_dependents
+        SELECT *
+        FROM transitive_dependents t
       |]
 
   execute [sql| DROP TABLE $scopeTableName |]
@@ -2034,8 +2044,8 @@ getTransitiveDependentsWithinScope scope query = do
   let result1 =
         List.foldl'
           ( \deps -> \case
-              dep :. Only TermComponent -> Defns (Set.insert dep deps.terms) deps.types
-              dep :. Only DeclComponent -> Defns deps.terms (Set.insert dep deps.types)
+              dep :. Only TermComponent -> let !terms = Set.insert dep deps.terms in Defns terms deps.types
+              dep :. Only DeclComponent -> let !types = Set.insert dep deps.types in Defns deps.terms types
               _ -> deps -- impossible; could error here
           )
           (Defns Set.empty Set.empty)

codebase2/codebase-sqlite/sql/018-add-derived-dependents-by-dependency-index.sql

@@ -0,0 +1,2 @@
+CREATE INDEX derived_dependents_by_dependency
+  ON dependents_index (dependency_object_id, dependency_component_index);

codebase2/codebase-sqlite/unison-codebase-sqlite.cabal

@@ -27,6 +27,7 @@ extra-source-files:
     sql/015-add-project-branch-last-accessed.sql
     sql/016-add-merge-branch-tables.sql
     sql/017-add-update-branch-table.sql
+    sql/018-add-derived-dependents-by-dependency-index.sql
     sql/create.sql
 
 source-repository head

lib/unison-prelude/src/Unison/Util/Timing.hs

@@ -28,7 +28,7 @@ startTiming = (,) <$> getMonotonicTimeNSec <*> getCPUTime
 stopTiming :: String -> (Word64, Integer) -> IO ()
 stopTiming label (systemTimeStart, cpuTimeStart) = do
   (systemTimeEnd, cpuTimeEnd) <- startTiming
-  let systemDiff = realToFrac @Word64 @Double (systemTimeEnd - systemTimeStart) / 1_000
+  let systemDiff = realToFrac @Word64 @Double (systemTimeEnd - systemTimeStart)
   let cpuDiff = realToFrac @Integer @Double (cpuTimeEnd - cpuTimeStart) / 1_000
   printf "%s: %s (cpu), %s (system)\n" label (renderNanos cpuDiff) (renderNanos systemDiff)
   where

parser-typechecker/src/Unison/Codebase/SqliteCodebase/Migrations.hs

@@ -87,7 +87,8 @@ migrations regionVar getDeclType termBuffer declBuffer rootCodebasePath =
       (17 {- This migration takes a raw sqlite connection -}, \conn -> migrateSchema16To17 conn),
       sqlMigration 18 Q.addProjectBranchLastAccessedColumn,
       sqlMigration 19 Q.addMergeBranchTables,
-      sqlMigration 20 Q.addUpdateBranchTable
+      sqlMigration 20 Q.addUpdateBranchTable,
+      sqlMigration 21 Q.addDerivedDependentsByDependencyIndex
     ]
   where
     runT :: Sqlite.Transaction () -> Sqlite.Connection -> IO ()

parser-typechecker/src/Unison/Codebase/SqliteCodebase/Operations.hs

@@ -101,6 +101,7 @@ createSchema = do
   Q.addProjectBranchLastAccessedColumn
   Q.addMergeBranchTables
   Q.addUpdateBranchTable
+  Q.addDerivedDependentsByDependencyIndex
   (_, emptyCausalHashId) <- emptyCausalHash
   (_, ProjectBranch {projectId, branchId}) <- insertProjectAndBranch scratchProjectName scratchBranchName emptyCausalHashId
   Q.setCurrentProjectPath projectId branchId []

unison-cli/src/Unison/Cli/UpdateUtils.hs

@@ -20,7 +20,7 @@ module Unison.Cli.UpdateUtils
 where
 
 import Control.Monad.Reader (ask)
-import Data.Bifoldable (bifold, bifoldMap)
+import Data.Bifoldable (bifold)
 import Data.Bitraversable (bitraverse)
 import Data.Foldable qualified as Foldable
 import Data.List qualified as List
@@ -157,7 +157,7 @@ getNamespaceDependentsOf2 ::
 getNamespaceDependentsOf2 defns dependencies = do
   let toTermScope = Set.mapMaybe Referent.toReferenceId . BiMultimap.dom
   let toTypeScope = Set.mapMaybe Reference.toId . BiMultimap.dom
-  let scope = bifoldMap toTermScope toTypeScope defns
+  let scope = bimap toTermScope toTypeScope defns
   Operations.transitiveDependentsWithinScope scope dependencies
     <&> bimap (Set.foldl' addTerms Map.empty) (Set.foldl' addTypes Map.empty)
   where
@@ -180,7 +180,7 @@ getNamespaceDependentsOf3 ::
 getNamespaceDependentsOf3 defns dependencies = do
   let toTermScope = Set.mapMaybe Referent.toReferenceId . BiMultimap.dom
   let toTypeScope = Set.mapMaybe Reference.toId . BiMultimap.dom
-  let scope = bifoldMap toTermScope toTypeScope defns
+  let scope = bimap toTermScope toTypeScope defns
   Operations.transitiveDependentsWithinScope scope (bifold dependencies)
 
 ------------------------------------------------------------------------------------------------------------------------

unison-cli/src/Unison/Codebase/Editor/HandleInput/EditDependents.hs

@@ -62,7 +62,10 @@ handleEditDependents name = do
       dependents <-
         Cli.runTransaction do
           Operations.transitiveDependentsWithinScope
-            (Branch.deepTermReferenceIds branchWithoutLibdeps <> Branch.deepTypeReferenceIds branchWithoutLibdeps)
+            Defns
+              { terms = Branch.deepTermReferenceIds branchWithoutLibdeps,
+                types = Branch.deepTypeReferenceIds branchWithoutLibdeps
+              }
             (bifold refs)
 
       let refsAndDependents =

unison-core/src/Unison/Names.hs

@@ -70,6 +70,7 @@ import Data.These (These (..))
 import Text.EditDistance
 import Text.FuzzyFind qualified as FZF
 import Unison.ConstructorReference (GConstructorReference (..))
+import Unison.ConstructorReference qualified as ConstructorReference
 import Unison.ConstructorType qualified as CT
 import Unison.HashQualified qualified as HQ
 import Unison.HashQualifiedPrime qualified as HQ'
@@ -87,6 +88,7 @@ import Unison.Referent qualified as Referent
 import Unison.ShortHash (ShortHash)
 import Unison.ShortHash qualified as SH
 import Unison.Util.Defns (Defns (..), DefnsF)
+import Unison.Util.Defns qualified as Defns
 import Unison.Util.Nametree (Nametree, unflattenNametree)
 import Unison.Util.Relation (Relation)
 import Unison.Util.Relation qualified as R
@@ -236,13 +238,29 @@ queryEditDistances' nameToText query names = do
 editDistance :: String -> String -> Int
 editDistance = restrictedDamerauLevenshteinDistance defaultEditCosts
 
--- | Get all (untagged) term/type references ids in a @Names@.
-referenceIds :: Names -> Set Reference.Id
+-- | Get all term/type references ids in a @Names@.
+referenceIds :: Names -> DefnsF Set TermReferenceId TypeReferenceId
 referenceIds Names {terms, types} =
-  fromTerms <> fromTypes
+  foldMap fromTerms (Relation.domain terms) <> foldMap fromTypes (Relation.domain types)
   where
-    fromTerms = Set.mapMaybe Referent.toReferenceId (Relation.ran terms)
-    fromTypes = Set.mapMaybe Reference.toId (Relation.ran types)
+    fromTerms :: Set Referent -> DefnsF Set TermReferenceId TypeReferenceId
+    fromTerms =
+      foldMap \case
+        Referent.Con ref _ ->
+          ref
+            & view ConstructorReference.reference_
+            & Reference.toId
+            & maybe Set.empty Set.singleton
+            & Defns.fromTypes
+        Referent.Ref ref ->
+          ref
+            & Reference.toId
+            & maybe Set.empty Set.singleton
+            & Defns.fromTerms
+
+    fromTypes :: (Ord terms) => Set TypeReference -> DefnsF Set terms TypeReferenceId
+    fromTypes =
+      Defns.fromTypes . Set.mapMaybe Reference.toId
 
 -- | Returns all constructor term references. Constructors are omitted.
 termReferences :: Names -> Set TermReference