Snapshot tiles trunk

contrib/test/run_fd_shred_cap.sh

@@ -90,16 +90,16 @@ echo "
     idx_max = 8192
     alloc_max = 1073741824
     file = \"$DATA_DIR/shredcap_testnet.blockstore\"
+[funk]
+    max_account_records = 150000000
+    heap_size_gib = 100
+    max_database_transactions = 2000
 [tiles]
     [tiles.shred]
         max_pending_shred_sets = 16384
     [tiles.replay]
         snapshot = \"$SNAPSHOT\"
         incremental = \"$INCREMENTAL\"
-        funk_sz_gb = 100
-        funk_rec_max = 150000000
-        funk_txn_max = 2000
-        funk_file = \"$DATA_DIR/shredcap_testnet.funk\"
     [tiles.store_int]
         shred_cap_replay = \"$SHREDCAP\"
         shred_cap_end_slot = 317018450

contrib/test/test_firedancer_leader.sh

@@ -50,10 +50,6 @@ echo "
     [tiles.replay]
         capture = \"firedancer-dev.solcap\"
         snapshot = \"$FULL_SNAPSHOT\"
-        funk_sz_gb = 32
-        funk_rec_max = 10000000
-        funk_txn_max = 1024
-        funk_file = \"/tmp/localnet.funk\"
         cluster_version = \"2.0.14\"
     [tiles.gui]
         enabled = false
@@ -72,6 +68,10 @@ echo "
     txn_max = 1024
     alloc_max = 10737418240
     file = \"/tmp/localnet.blockstore\"
+[funk]
+    max_account_records = 10000000
+    heap_size_gib = 32
+    max_database_transactions = 1024
 [log]
     path = \"firedancer-dev.log\"
     level_stderr = \"INFO\"

snapload.toml

@@ -0,0 +1,11 @@
+[hugetlbfs]
+max_page_size = "huge"
+
+[funk]
+max_account_records = 100_000_000
+heap_size_gib = 64
+
+[log]
+    level_stderr = "INFO"
+    level_logfile = "INFO"
+    path = "-"

src/app/firedancer-dev/Local.mk

@@ -13,6 +13,7 @@ $(call add-objs,commands/gossip,fd_firedancer_dev)
 $(call add-objs,commands/bench,fd_firedancer_dev)
 $(call add-objs,commands/dev,fd_firedancer_dev)
 $(call add-objs,commands/sim,fd_firedancer_dev)
+$(call add-objs,commands/snapshot_load,fd_firedancer_dev)
 $(call add-objs,commands/backtest,fd_firedancer_dev)
 
 $(call make-bin,firedancer-dev,main,fd_firedancer_dev fd_firedancer fddev_shared fdctl_shared fd_discof fd_disco fd_choreo fd_flamenco fd_funk fd_quic fd_tls fd_reedsol fd_ballet fd_waltz fd_tango fd_util firedancer_version, $(SECP256K1_LIBS) $(ROCKSDB_LIBS))

src/app/firedancer-dev/commands/backtest.c

@@ -15,132 +15,20 @@
 
 */
 
-#include "../../shared/commands/configure/configure.h"
+#include "../../firedancer/topology.h"
 #include "../../shared/commands/run/run.h" /* initialize_workspaces */
 #include "../../shared/fd_config.h" /* config_t */
 #include "../../../disco/tiles.h"
 #include "../../../disco/topo/fd_cpu_topo.h" /* fd_topo_cpus */
 #include "../../../disco/topo/fd_topob.h"
 #include "../../../disco/topo/fd_pod_format.h"
 #include "../../../discof/geyser/fd_replay_notif.h"
-#include "../../../flamenco/runtime/fd_runtime.h"
 #include "../../../flamenco/runtime/fd_txncache.h"
-#include "../../../flamenco/snapshot/fd_snapshot_base.h"
 
 #include <unistd.h> /* pause */
 extern fd_topo_obj_callbacks_t * CALLBACKS[];
 fd_topo_run_tile_t fdctl_tile_run( fd_topo_tile_t const * tile );
 
-static fd_topo_obj_t *
-setup_topo_runtime_pub( fd_topo_t *  topo,
-                        char const * wksp_name,
-                        ulong        mem_max ) {
-  fd_topo_obj_t * obj = fd_topob_obj( topo, "runtime_pub", wksp_name );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, mem_max, "obj.%lu.mem_max",  obj->id ) );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, 12UL,    "obj.%lu.wksp_tag", obj->id ) );
-  return obj;
-}
-
-static fd_topo_obj_t *
-setup_topo_txncache( fd_topo_t *  topo,
-                    char const * wksp_name,
-                    ulong        max_rooted_slots,
-                    ulong        max_live_slots,
-                    ulong        max_txn_per_slot,
-                    ulong        max_constipated_slots ) {
-  fd_topo_obj_t * obj = fd_topob_obj( topo, "txncache", wksp_name );
-
-  FD_TEST( fd_pod_insertf_ulong( topo->props, max_rooted_slots, "obj.%lu.max_rooted_slots", obj->id ) );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, max_live_slots,   "obj.%lu.max_live_slots",   obj->id ) );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, max_txn_per_slot, "obj.%lu.max_txn_per_slot", obj->id ) );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, max_constipated_slots, "obj.%lu.max_constipated_slots", obj->id ) );
-
-  return obj;
-}
-
-#include <sys/random.h>
-#include "../../../flamenco/runtime/fd_blockstore.h"
-static fd_topo_obj_t *
-setup_topo_blockstore( fd_topo_t *  topo,
-                       char const * wksp_name,
-                       ulong        shred_max,
-                       ulong        block_max,
-                       ulong        idx_max,
-                       ulong        txn_max,
-                       ulong        alloc_max ) {
-  fd_topo_obj_t * obj = fd_topob_obj( topo, "blockstore", wksp_name );
-
-  ulong seed;
-  FD_TEST( sizeof(ulong) == getrandom( &seed, sizeof(ulong), 0 ) );
-
-  FD_TEST( fd_pod_insertf_ulong( topo->props, 1UL,        "obj.%lu.wksp_tag",   obj->id ) );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, seed,       "obj.%lu.seed",       obj->id ) );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, shred_max,  "obj.%lu.shred_max",  obj->id ) );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, block_max,  "obj.%lu.block_max",  obj->id ) );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, idx_max,    "obj.%lu.idx_max",    obj->id ) );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, txn_max,    "obj.%lu.txn_max",    obj->id ) );
-  FD_TEST( fd_pod_insertf_ulong( topo->props, alloc_max,  "obj.%lu.alloc_max",  obj->id ) );
-
-  /* DO NOT MODIFY LOOSE WITHOUT CHANGING HOW BLOCKSTORE ALLOCATES INTERNAL STRUCTURES */
-
-  ulong blockstore_footprint = fd_blockstore_footprint( shred_max, block_max, idx_max, txn_max ) + alloc_max;
-  FD_TEST( fd_pod_insertf_ulong( topo->props, blockstore_footprint,  "obj.%lu.loose", obj->id ) );
-
-  return obj;
-}
-
-static void
-setup_snapshots( config_t *       config,
-                 fd_topo_tile_t * tile ) {
-  uchar incremental_is_file, incremental_is_url;
-  if( strnlen( config->tiles.replay.incremental, PATH_MAX )>0UL ) {
-    incremental_is_file = 1U;
-  } else {
-    incremental_is_file = 0U;
-  }
-  if( strnlen( config->tiles.replay.incremental_url, PATH_MAX )>0UL ) {
-    incremental_is_url = 1U;
-  } else {
-    incremental_is_url = 0U;
-  }
-  if( FD_UNLIKELY( incremental_is_file && incremental_is_url ) ) {
-    FD_LOG_ERR(( "At most one of the incremental snapshot source strings in the configuration file under [tiles.replay.incremental] and [tiles.replay.incremental_url] may be set." ));
-  }
-  tile->replay.incremental_src_type = INT_MAX;
-  if( FD_LIKELY( incremental_is_url ) ) {
-    strncpy( tile->replay.incremental, config->tiles.replay.incremental_url, sizeof(tile->replay.incremental) );
-    tile->replay.incremental_src_type = FD_SNAPSHOT_SRC_HTTP;
-  }
-  if( FD_UNLIKELY( incremental_is_file ) ) {
-    strncpy( tile->replay.incremental, config->tiles.replay.incremental, sizeof(tile->replay.incremental) );
-    tile->replay.incremental_src_type = FD_SNAPSHOT_SRC_FILE;
-  }
-
-  uchar snapshot_is_file, snapshot_is_url;
-  if( strnlen( config->tiles.replay.snapshot, PATH_MAX )>0UL ) {
-    snapshot_is_file = 1U;
-  } else {
-    snapshot_is_file = 0U;
-  }
-  if( strnlen( config->tiles.replay.snapshot_url, PATH_MAX )>0UL ) {
-    snapshot_is_url = 1U;
-  } else {
-    snapshot_is_url = 0U;
-  }
-  if( FD_UNLIKELY( snapshot_is_file && snapshot_is_url ) ) {
-    FD_LOG_ERR(( "At most one of the full snapshot source strings in the configuration file under [tiles.replay.snapshot] and [tiles.replay.snapshot_url] may be set." ));
-  }
-  tile->replay.snapshot_src_type = INT_MAX;
-  if( FD_LIKELY( snapshot_is_url ) ) {
-    strncpy( tile->replay.snapshot, config->tiles.replay.snapshot_url, sizeof(tile->replay.snapshot) );
-    tile->replay.snapshot_src_type = FD_SNAPSHOT_SRC_HTTP;
-  }
-  if( FD_UNLIKELY( snapshot_is_file ) ) {
-    strncpy( tile->replay.snapshot, config->tiles.replay.snapshot, sizeof(tile->replay.snapshot) );
-    tile->replay.snapshot_src_type = FD_SNAPSHOT_SRC_FILE;
-  }
-}
-
 static void
 backtest_topo( config_t * config ) {
   fd_topo_cpus_t cpus[1];
@@ -164,10 +52,7 @@ backtest_topo( config_t * config ) {
   /**********************************************************************/
   fd_topob_wksp( topo, "metric" );
   fd_topob_wksp( topo, "metric_in" );
-  fd_topo_tile_t * metric_tile = fd_topob_tile( topo, "metric", "metric", "metric_in", metric_cpu_idx, 0, 0 );
-  if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->tiles.metric.prometheus_listen_address, &metric_tile->metric.prometheus_listen_addr ) ) )
-    FD_LOG_ERR(( "failed to parse prometheus listen address `%s`", config->tiles.metric.prometheus_listen_address ));
-  metric_tile->metric.prometheus_listen_port = config->tiles.metric.prometheus_listen_port;
+  fd_topob_tile( topo, "metric", "metric", "metric_in", metric_cpu_idx, 0, 0 );
 
   /**********************************************************************/
   /* Add the backtest tile to topo                                      */
@@ -187,52 +72,16 @@ backtest_topo( config_t * config ) {
   /**********************************************************************/
   fd_topob_wksp( topo, "replay" );
   fd_topo_tile_t * replay_tile = fd_topob_tile( topo, "replay", "replay", "metric_in", replay_cpu_idx, 0, 0 );
-  replay_tile->replay.fec_max = config->tiles.shred.max_pending_shred_sets;
-  replay_tile->replay.max_vote_accounts = config->firedancer.runtime.limits.max_vote_accounts;
 
   /* specified by [tiles.replay] */
 
-  strncpy( replay_tile->replay.blockstore_file,    config->firedancer.blockstore.file,    sizeof(replay_tile->replay.blockstore_file) );
-  strncpy( replay_tile->replay.blockstore_checkpt, config->firedancer.blockstore.checkpt, sizeof(replay_tile->replay.blockstore_checkpt) );
-
-  replay_tile->replay.tx_metadata_storage = config->rpc.extended_tx_metadata_storage;
-  strncpy( replay_tile->replay.capture, config->tiles.replay.capture, sizeof(replay_tile->replay.capture) );
-  strncpy( replay_tile->replay.funk_checkpt, config->tiles.replay.funk_checkpt, sizeof(replay_tile->replay.funk_checkpt) );
-  replay_tile->replay.funk_rec_max = config->tiles.replay.funk_rec_max;
-  replay_tile->replay.funk_sz_gb   = config->tiles.replay.funk_sz_gb;
-  replay_tile->replay.funk_txn_max = config->tiles.replay.funk_txn_max;
-  strncpy( replay_tile->replay.funk_file, config->tiles.replay.funk_file, sizeof(replay_tile->replay.funk_file) );
-  replay_tile->replay.plugins_enabled = config->tiles.gui.enabled;
-
-  if( FD_UNLIKELY( !strncmp( config->tiles.replay.genesis,  "", 1 )
-                   && !strncmp( config->tiles.replay.snapshot, "", 1 ) ) ) {
-    fd_cstr_printf_check(  config->tiles.replay.genesis, PATH_MAX, NULL, "%s/genesis.bin", config->paths.ledger );
-  }
-  strncpy( replay_tile->replay.genesis, config->tiles.replay.genesis, sizeof(replay_tile->replay.genesis) );
-
-  setup_snapshots( config, replay_tile );
+  fd_topob_wksp( topo, "funk" );
+  fd_topo_obj_t * funk_obj = setup_topo_funk( topo, "funk",
+      config->firedancer.funk.max_account_records,
+      config->firedancer.funk.max_database_transactions,
+      config->firedancer.funk.heap_size_gib );
 
-  strncpy( replay_tile->replay.slots_replayed, config->tiles.replay.slots_replayed, sizeof(replay_tile->replay.slots_replayed) );
-  strncpy( replay_tile->replay.status_cache, config->tiles.replay.status_cache, sizeof(replay_tile->replay.status_cache) );
-  strncpy( replay_tile->replay.cluster_version, config->tiles.replay.cluster_version, sizeof(replay_tile->replay.cluster_version) );
-  replay_tile->replay.bank_tile_count = config->layout.bank_tile_count;
-  replay_tile->replay.exec_tile_count   = config->firedancer.layout.exec_tile_count;
-  replay_tile->replay.writer_tile_cuont = config->firedancer.layout.writer_tile_count;
-  strncpy( replay_tile->replay.tower_checkpt, config->tiles.replay.tower_checkpt, sizeof(replay_tile->replay.tower_checkpt) );
-
-  replay_tile->replay.enable_features_cnt = config->tiles.replay.enable_features_cnt;
-  for( ulong i = 0; i < replay_tile->replay.enable_features_cnt; i++ ) {
-    strncpy( replay_tile->replay.enable_features[i], config->tiles.replay.enable_features[i], sizeof(replay_tile->replay.enable_features[i]) );
-  }
-
-  /* not specified by [tiles.replay] */
-
-  strncpy( replay_tile->replay.identity_key_path, config->paths.identity_key, sizeof(replay_tile->replay.identity_key_path) );
-  replay_tile->replay.ip_addr = config->net.ip_addr;
-  replay_tile->replay.vote = config->firedancer.consensus.vote;
-  strncpy( replay_tile->replay.vote_account_path, config->paths.vote_account, sizeof(replay_tile->replay.vote_account_path) );
-  replay_tile->replay.full_interval        = config->tiles.batch.full_interval;
-  replay_tile->replay.incremental_interval = config->tiles.batch.incremental_interval;
+  fd_topob_tile_uses( topo, replay_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
 
   /**********************************************************************/
   /* Add the executor tiles to topo                                     */
@@ -427,6 +276,29 @@ backtest_topo( config_t * config ) {
   fd_topob_tile_uses( topo, replay_tile, constipated_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
   FD_TEST( fd_pod_insertf_ulong( topo->props, constipated_obj->id, "constipate" ) );
 
+  for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
+    fd_topo_tile_t * tile = &topo->tiles[ i ];
+    if( !strcmp( tile->name, "rocksdb" ) ) {
+      tile->archiver.end_slot = config->tiles.archiver.end_slot;
+      strncpy( tile->archiver.archiver_path, config->tiles.archiver.archiver_path, PATH_MAX );
+      if( FD_UNLIKELY( 0==strlen( tile->archiver.archiver_path ) ) ) {
+        FD_LOG_ERR(( "Rocksdb not found, check `archiver.archiver_path` in toml" ));
+      } else {
+        FD_LOG_NOTICE(( "Found rocksdb path from config: %s", tile->archiver.archiver_path ));
+      }
+    } else if( !fd_topo_configure_tile( tile, config ) ) {
+      FD_LOG_ERR(( "unknown tile name %lu `%s`", i, tile->name ));
+    }
+
+    /* Override */
+    if( !strcmp( tile->name, "replay" ) ) {
+      tile->replay.enable_features_cnt = config->tiles.replay.enable_features_cnt;
+      for( ulong i = 0; i < tile->replay.enable_features_cnt; i++ ) {
+        strncpy( tile->replay.enable_features[i], config->tiles.replay.enable_features[i], sizeof(tile->replay.enable_features[i]) );
+      }
+    }
+  }
+
   /**********************************************************************/
   /* Finish and print out the topo information                          */
   /**********************************************************************/
@@ -436,7 +308,7 @@ backtest_topo( config_t * config ) {
 
 static void
 backtest_cmd_fn( args_t *   args FD_PARAM_UNUSED,
-                config_t * config ) {
+                 config_t * config ) {
   FD_LOG_NOTICE(( "Start to run the backtest cmd" ));
   backtest_topo( config );
 
@@ -459,13 +331,13 @@ backtest_cmd_fn( args_t *   args FD_PARAM_UNUSED,
 
 static void
 backtest_cmd_perm( args_t *         args   FD_PARAM_UNUSED,
-                  fd_cap_chk_t *   chk    FD_PARAM_UNUSED,
-                  config_t const * config FD_PARAM_UNUSED ) {}
+                   fd_cap_chk_t *   chk    FD_PARAM_UNUSED,
+                   config_t const * config FD_PARAM_UNUSED ) {}
 
 static void
 backtest_cmd_args( int *    pargc FD_PARAM_UNUSED,
-                  char *** pargv FD_PARAM_UNUSED,
-                  args_t * args  FD_PARAM_UNUSED ) {}
+                   char *** pargv FD_PARAM_UNUSED,
+                   args_t * args  FD_PARAM_UNUSED ) {}
 
 action_t fd_action_backtest = {
   .name = "backtest",