Skip to content

Gossip V2 #5154

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ravyu-jump merged 10 commits into from
Aug 23, 2025
Merged

Gossip V2 #5154

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
a2bb683
gossip: implement sketch of gossip protocol
mmcgee-jump May 9, 2025
d9869f9
gossip: implement details of gossip protocol
ravyu-jump May 16, 2025
18106a7
gossip: split tile in two
mmcgee-jump Aug 13, 2025
582bb4a
gossip: contact info sidetable eviction
ravyu-jump Aug 18, 2025
218d7d6
gossip: multiple gossvf tiles
ravyu-jump Aug 18, 2025
2ccb841
gossip: stake maps for fd_gossip and gossvf
ravyu-jump Aug 21, 2025
ce66a32
gossip: update stem burst
ravyu-jump Aug 21, 2025
1fc19f2
gossip: integration with full client topology
ravyu-jump Aug 20, 2025
da3d2fa
gossip: publish update messages for crds originating from us
ravyu-jump Aug 22, 2025
b38e52e
gossip: compiler and ubsan fixes
ravyu-jump Aug 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
376 changes: 203 additions & 173 deletions book/api/metrics-generated.md
View file
Open in desktop

Large diffs are not rendered by default.

1,060 changes: 994 additions & 66 deletions src/app/firedancer-dev/commands/gossip.c
View file
Open in desktop

Large diffs are not rendered by default.

31 changes: 7 additions & 24 deletions src/app/firedancer-dev/commands/repair.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,16 +85,12 @@ repair_topo( config_t * config ) {
fd_topob_wksp( topo, "gossip_sign" );
fd_topob_wksp( topo, "sign_gossip" );

fd_topob_wksp( topo, "crds_shred" );
fd_topob_wksp( topo, "gossip_repai" );
fd_topob_wksp( topo, "gossip_verif" );
fd_topob_wksp( topo, "gossip_tower" );
fd_topob_wksp( topo, "gossip_out" );

fd_topob_wksp( topo, "repair_sign" );
fd_topob_wksp( topo, "sign_repair" );

fd_topob_wksp( topo, "repair_repla" );
fd_topob_wksp( topo, "gossip_send" );
fd_topob_wksp( topo, "send_txns" );

fd_topob_wksp( topo, "shred" );
Expand Down Expand Up @@ -123,12 +119,7 @@ repair_topo( config_t * config ) {
/**/ fd_topob_link( topo, "gossip_sign", "gossip_sign", 128UL, 2048UL, 1UL );
/**/ fd_topob_link( topo, "sign_gossip", "sign_gossip", 128UL, 64UL, 1UL );

/**/ fd_topob_link( topo, "gossip_verif", "gossip_verif", config->tiles.verify.receive_buffer_size, FD_TPU_MTU, 1UL );
/**/ fd_topob_link( topo, "gossip_tower", "gossip_tower", 128UL, FD_TPU_MTU, 1UL );

/**/ fd_topob_link( topo, "crds_shred", "crds_shred", 128UL, 8UL + 40200UL * 38UL, 1UL );
/**/ fd_topob_link( topo, "gossip_repai", "gossip_repai", 128UL, 40200UL * 38UL, 1UL );
/**/ fd_topob_link( topo, "gossip_send", "gossip_send", 128UL, 40200UL * 38UL, 1UL );
/**/ fd_topob_link( topo, "gossip_out", "gossip_out", 65536UL, 2048UL, 1UL );

/**/ fd_topob_link( topo, "gossip_net", "net_gossip", config->net.ingress_buffer_size, FD_NET_MTU, 1UL );

Expand Down Expand Up @@ -188,7 +179,7 @@ repair_topo( config_t * config ) {
FOR(shred_tile_cnt) fd_topob_tile( topo, "shred", "shred", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 1 );
FOR(sign_tile_cnt) fd_topob_tile( topo, "sign", "sign", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 1 );
/**/ fd_topob_tile( topo, "metric", "metric", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
/**/ fd_topob_tile( topo, "gossip", "gossip", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
/**/ fd_topob_tile( topo, "gossip", "gossip", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 1 );
fd_topo_tile_t * repair_tile = fd_topob_tile( topo, "repair", "repair", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );

/* Setup a shared wksp object for fec sets. */
Expand Down Expand Up @@ -249,7 +240,7 @@ repair_topo( config_t * config ) {
fd_topob_tile_in( topo, "shred", i, "metric_in", "net_shred", j, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
FOR(shred_tile_cnt) fd_topob_tile_in( topo, "shred", i, "metric_in", "poh_shred", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
FOR(shred_tile_cnt) fd_topob_tile_in( topo, "shred", i, "metric_in", "stake_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
FOR(shred_tile_cnt) fd_topob_tile_in( topo, "shred", i, "metric_in", "crds_shred", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
FOR(shred_tile_cnt) fd_topob_tile_in( topo, "shred", i, "metric_in", "gossip_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
FOR(shred_tile_cnt) fd_topob_tile_out( topo, "shred", i, "shred_repair", i );
FOR(shred_tile_cnt) fd_topob_tile_out( topo, "shred", i, "shred_net", i );

Expand All @@ -270,18 +261,14 @@ repair_topo( config_t * config ) {

FOR(net_tile_cnt) fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "net_gossip", i, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_net", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "crds_shred", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_repai", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_verif", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_out", 0UL );
/**/ fd_topob_tile_in( topo, "sign", 0UL, "metric_in", "gossip_sign", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_sign", 0UL );
/**/ fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "sign_gossip", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
/**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_gossip", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_send", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_tower", 0UL );

FOR(net_tile_cnt) fd_topob_tile_in( topo, "repair", 0UL, "metric_in", "net_repair", i, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
/**/ fd_topob_tile_in( topo, "repair", 0UL, "metric_in", "gossip_repai", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "repair", 0UL, "metric_in", "gossip_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "repair", 0UL, "metric_in", "stake_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
fd_topob_tile_in( topo, "repair", 0UL, "metric_in", "snap_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
FOR(shred_tile_cnt) fd_topob_tile_in( topo, "repair", 0UL, "metric_in", "shred_repair", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
Expand Down Expand Up @@ -316,8 +303,7 @@ repair_topo( config_t * config ) {
for( ulong j=0UL; j<shred_tile_cnt; j++ ) {
fd_topob_tile_in( topo, "scap", 0UL, "metric_in", "shred_repair", j, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
}
fd_topob_tile_in( topo, "scap", 0UL, "metric_in", "crds_shred", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
fd_topob_tile_in( topo, "scap", 0UL, "metric_in", "gossip_repai", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
fd_topob_tile_in( topo, "scap", 0UL, "metric_in", "gossip_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );

fd_topob_tile_in( topo, "scap", 0UL, "metric_in", "repair_scap", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
fd_topob_tile_in( topo, "scap", 0UL, "metric_in", "replay_scap", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
Expand All @@ -334,9 +320,6 @@ repair_topo( config_t * config ) {
FD_TEST( link_permit_no_producers( topo, "replay_scap" ) == 1UL );

FD_TEST( link_permit_no_consumers( topo, "net_quic" ) == quic_tile_cnt );
FD_TEST( link_permit_no_consumers( topo, "gossip_verif" ) == 1UL );
FD_TEST( link_permit_no_consumers( topo, "gossip_tower" ) == 1UL );
FD_TEST( link_permit_no_consumers( topo, "gossip_send" ) == 1UL );
FD_TEST( link_permit_no_consumers( topo, "repair_repla" ) == 1UL );

FOR(net_tile_cnt) fd_topos_net_tile_finish( topo, i );
Expand Down
2 changes: 2 additions & 0 deletions src/app/firedancer-dev/config/testnet.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@
"entrypoint2.testnet.solana.com:8001",
"entrypoint3.testnet.solana.com:8001",
]
[layout]
gossvf_tile_count = 2
2 changes: 2 additions & 0 deletions src/app/firedancer-dev/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ extern fd_topo_run_tile_t fd_tile_pktgen;
extern fd_topo_run_tile_t fd_tile_udpecho;
extern fd_topo_run_tile_t fd_tile_ipecho;

extern fd_topo_run_tile_t fd_tile_gossvf;
extern fd_topo_run_tile_t fd_tile_gossip;
extern fd_topo_run_tile_t fd_tile_repair;
extern fd_topo_run_tile_t fd_tile_replay;
Expand Down Expand Up @@ -126,6 +127,7 @@ fd_topo_run_tile_t * TILES[] = {
&fd_tile_benchg,
&fd_tile_benchs,
&fd_tile_bundle,
&fd_tile_gossvf,
&fd_tile_gossip,
&fd_tile_repair,
&fd_tile_replay,
Expand Down
122 changes: 78 additions & 44 deletions src/app/firedancer/config/default.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,10 @@ user = ""
# +-- vote-account.json
# +-- genesis.bin
#
# Two substitutions will be performed on this string. If "{user}" is
# present it will be replaced with the user running Firedancer, as
# above, and "{name}" will be replaced with the name of the Firedancer
# instance.
# Two substitutions will be performed on this string. If "{user}"
# is present it will be replaced with the user running Firedancer,
# as above, and "{name}" will be replaced with the name of the
# Firedancer instance.
base = "/home/{user}/.firedancer/{name}"

# Absolute directory path to place the ledger. If no ledger path is
Expand Down Expand Up @@ -642,6 +642,21 @@ user = ""
# denial of service or spam attack.
verify_tile_count = 6

# How many gossip verify tiles to run. Gossip verify tiles are
# architecturally similar to verify tiles in that inbound Gossip
# messages are routed through the gossip verify tiles first before
# arriving at the gossip tile (if verified successfully).

# Likewise, they are designed to be scaled linearly and should be
# increased until a favorable drop rate is achieved (~10%) once
# the table is nearly fully populated. Large drop rates are expected
# during the boot sequence due to the influx of pull responses as a
# result of making a pull request from an empty Gossip table. This
# can be mitigated by adding more cores, but it is difficult to
# avoid completely on mainnet and testnet loads.

gossvf_tile_count = 1

# How many bank tiles to run. Should be set to 4 for perf and
# balanced scheduling modes. Bank tiles execute transactions, so
# the validator can include the results of the transaction into a
Expand Down Expand Up @@ -783,21 +798,22 @@ user = ""
# Firedancer. Possible values for the option are:
#
# "xdp" (default)
# Use Linux express data path APIs for high performance networking.
# These APIs map memory from the network device directly into
# Firedancer to bypass parts of the kernel when reading and writing
# data.
# Use Linux express data path APIs for high performance
# networking. These APIs map memory from the network device
# directly into Firedancer to bypass parts of the kernel when
# reading and writing data.
#
# "socket"
# Use UDP sockets and system calls like `send(2)` for networking.
# Sockets are slower and socket support is provided on a best-effort
# basis. The socket option is provided as a fallback for hosts which
# do not support XDP, and to support experimental features that are
# not yet implemented in the XDP stack. Currently, DoubleZero is only
# supported by the socket provider.
#
# Using the XDP networking stack is strongly preferred where possible,
# as it is faster and better tested by the development team.
# Sockets are slower and socket support is provided on a
# best-effort basis. The socket option is provided as a fallback
# for hosts which do not support XDP, and to support experimental
# features that are not yet implemented in the XDP stack.
# Currently, DoubleZero is only supported by the socket provider.
#
# Using the XDP networking stack is strongly preferred where
# possible, as it is faster and better tested by the development
# team.
provider = "xdp"

# Which interface to bind to for network traffic. Currently,
Expand Down Expand Up @@ -877,10 +893,10 @@ user = ""
xdp_rx_queue_size = 32768
xdp_tx_queue_size = 32768

# When the oldest packet in a batch has been queued for more than
# the specified duration, flushes the batch out to the network
# device. Smaller values improve latency, larger values may
# improve throughput.
# When the oldest packet in a batch has been queued for more
# than the specified duration, flushes the batch out to the
# network device. Smaller values improve latency, larger values
# may improve throughput.
flush_timeout_micros = 20

[net.socket]
Expand All @@ -897,17 +913,17 @@ user = ""
# which CPU core to create a functioning system, below is the individual
# settings that can change behavior of the tiles.
[tiles]
# The netlink tile forwards Linux network configuration to net tiles.
# This config section contains advanced options that typically do not
# need to be changed.
# For further info, see https://docs.firedancer.io/guide/netlink.html
# The netlink tile forwards Linux network configuration to net
# tiles. This config section contains advanced options that
# typically do not need to be changed. For further info, see
# https://docs.firedancer.io/guide/netlink.html
[tiles.netlink]
# The maximum number of routes per route table.
#
# The netlink tile imports two route tables from Linux, namely
# `local` and `main`. You can view them by running
# `ip route show table main`. Decreasing this option can result
# in connectivity issues. Increasing this option can drastically
# `local` and `main`. You can view them by running `ip route
# show table main`. Decreasing this option can result in
# connectivity issues. Increasing this option can drastically
# decrease performance.
#
# For virtually all cloud and bare-metal server providers, the
Expand All @@ -923,11 +939,21 @@ user = ""

# The maximum number of Ethernet neighbors.
#
# This should be roughly as large as the size your Ethernet subnet.
# E.g. if your IP address is 198.51.100.3/24, then your subnet has
# up to 256 neighbors (2^(32-24)).
# This should be roughly as large as the size your Ethernet
# subnet. E.g. if your IP address is 198.51.100.3/24, then your
# subnet has up to 256 neighbors (2^(32-24)).
max_neighbors = 4096

[tiles.gossip]
# The maximum number of entries that can be stored in the gossip
# table before entries from low stake validators are evicted.
#
# The gossip protocol periodically expires old entries from the
# table, and this number should be set so that the table never
# gets full, and all entries expire before they would need to be
# evicted.
max_entries = 2_097_152

# QUIC tiles are responsible for serving network traffic, including
# parsing and responding to packets and managing connection timeouts
# and state machines. These tiles implement the QUIC protocol,
Expand Down Expand Up @@ -1412,6 +1438,13 @@ user = ""
# IP address of the second network namespace.
interface1_addr = "198.18.0.2"

[development.gossip]
# Under normal operating conditions, a validator should always
# reach out to a host located on the public internet. If this
# value is true, it allows the validator to gossip with nodes
# located on a private internet (rfc1918).
allow_private_address = false

[development.genesis]
# When creating a new chain from genesis during development,
# this option can be used to specify the number of hashes in
Expand Down Expand Up @@ -1467,12 +1500,12 @@ user = ""
# one that is used by the bootstrap validator.
vote_account_stake_lamports = 500000000

# Setting warmup epochs to true will allow shorter epochs towards
# the beginning of the cluster. This allows for faster stake
# activation. The first epoch will be 32 slots long and the
# duration of each subsequent epoch will be double that of the
# one before it until it reaches the desired epoch duration of the
# cluster.
# Setting warmup epochs to true will allow shorter epochs
# towards the beginning of the cluster. This allows for faster
# stake activation. The first epoch will be 32 slots long and
# the duration of each subsequent epoch will be double that of
# the one before it until it reaches the desired epoch duration
# of the cluster.
warmup_epochs = false

[development.bench]
Expand Down Expand Up @@ -1566,20 +1599,21 @@ user = ""
# This setting overrides [layout.affinity] above.
affinity = "auto"

# fake_dst_ip should be set to the IP address of a neighbor on the
# Ethernet network. (For example the router the system is directly
# connected to.)
# fake_dst_ip should be set to the IP address of a neighbor on
# the Ethernet network. (For example the router the system is
# directly connected to.)
#
# The net tile queries the route and neighbor tables as part of the
# benchmark. In short, these resolve the dst MAC address of the
# outgoing Ethernet packets. Since pktgen does not produce IPv4
# packets, no traffic actually gets sent to this IP.
# The net tile queries the route and neighbor tables as part of
# the benchmark. In short, these resolve the dst MAC address of
# the outgoing Ethernet packets. Since pktgen does not produce
# IPv4 packets, no traffic actually gets sent to this IP.
#
# The IP address should fulfill these requirements:
# - 'ip route get <IP>' resolves to the XDP interface
# (see [net.interface])
# - 'ip neigh get <NEXTHOP> dev <INTERFACE>' returns REACHABLE
# If these requirements are not met, no packets will be sent out.
# If these requirements are not met, no packets will be sent
# out.
fake_dst_ip = "" # e.g. "192.0.2.64"

# The following options relate to the 'fddev udpecho' tool only.
Expand Down
5 changes: 2 additions & 3 deletions src/app/firedancer/config/testnet.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,9 @@
entrypoints = [
"entrypoint.testnet.solana.com:8001",
"entrypoint2.testnet.solana.com:8001",
"entrypoint3.testnet.solana.com:8001",
]
[funk]
heap_size_gib = 40
max_account_records = 200_000_000
[consensus]
expected_shred_version = 9065
[layout]
gossvf_tile_count = 4
2 changes: 2 additions & 0 deletions src/app/firedancer/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ extern fd_topo_run_tile_t fd_tile_gui;
extern fd_topo_run_tile_t fd_tile_plugin;
extern fd_topo_run_tile_t fd_tile_bundle;

extern fd_topo_run_tile_t fd_tile_gossvf;
extern fd_topo_run_tile_t fd_tile_gossip;
extern fd_topo_run_tile_t fd_tile_repair;
extern fd_topo_run_tile_t fd_tile_replay;
Expand Down Expand Up @@ -102,6 +103,7 @@ fd_topo_run_tile_t * TILES[] = {
&fd_tile_gui,
&fd_tile_plugin,
&fd_tile_bundle,
&fd_tile_gossvf,
&fd_tile_gossip,
&fd_tile_repair,
&fd_tile_replay,
Expand Down
Loading
Loading