merge

Author: ibhatt-jumptrading

.github/workflows/on_nightly.yml

@@ -2,22 +2,15 @@ name: On Nightly
 on:
   workflow_dispatch:
   schedule:
-    - cron: 30 11 * * *
-    - cron: 0 3 * * *
+    - cron: 0 4 * * *
 jobs:
-  tests:
-    uses: ./.github/workflows/coverage_report_clusterfuzz.yml
-    secrets: inherit
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.schedule == '30 11 * * *' }}
   coverage-report:
     uses: ./.github/workflows/coverage_report.yml
     secrets: inherit
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.schedule == '30 11 * * *' }}
   codeql:
     uses: ./.github/workflows/codeql.yml
     permissions:
       security-events: write
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.schedule == '30 11 * * *' }}
   builds:
     uses: ./.github/workflows/builds.yml
     with:
@@ -34,4 +27,3 @@ jobs:
         ALL,linux_gcc_riscv,ALL
       verbose: false
       build_arm: true
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.schedule == '0 3 * * *' }}

.github/workflows/tests.yml

@@ -24,28 +24,25 @@ jobs:
           - linux_clang_x86_64
           - linux_clang_icelake
           - native
+          - native-no-deps
         # Attach additional params to machine types
         include:
           - test-case: linux_gcc_noarch64
             machine: linux_gcc_noarch64
             label: X64
-            extras: "rpath handholding"
             deps-extras: "+dev"
-            targets: "all"
+            targets: "check"
             compiler: gcc
-            compiler-version: 8.5.0
-            run-unit-tests: false
-            run-integration-tests: false
+            compiler-version: 11.4.0
           - test-case: linux_gcc_x86_64
             machine: linux_gcc_x86_64
             label: X64
             extras: "rpath handholding"
             deps-extras: "+dev"
             targets: "all integration-test fdctl firedancer"
             compiler: gcc
-            compiler-version: 8.5.0
+            compiler-version: 11.4.0
             run-unit-tests: true
-            run-integration-tests: false
           - test-case: linux_gcc_icelake
             machine: linux_gcc_icelake
             label: icelake
@@ -55,7 +52,6 @@ jobs:
             compiler: gcc
             compiler-version: 12.4.0
             run-unit-tests: true
-            run-integration-tests: false
           - test-case: linux_clang_x86_64
             machine: linux_clang_x86_64
             label: X64
@@ -65,7 +61,6 @@ jobs:
             compiler: clang
             compiler-version: 15.0.6
             run-unit-tests: true
-            run-integration-tests: false
           - test-case: linux_clang_icelake
             machine: linux_clang_icelake
             label: icelake
@@ -75,7 +70,6 @@ jobs:
             compiler: clang
             compiler-version: 15.0.6
             run-unit-tests: true
-            run-integration-tests: false
           - test-case: native
             machine: native
             label: 512G
@@ -86,6 +80,13 @@ jobs:
             compiler-version: 15.0.6
             run-unit-tests: true
             run-integration-tests: true
+          - test-case: native-no-deps
+            machine: native
+            label: X64
+            extras: no-deps
+            targets: check
+            compiler: clang
+            compiler-version: 15.0.6
     runs-on: ${{ matrix.label }}
     env:
       MACHINE: ${{ matrix.machine }}
@@ -102,9 +103,12 @@ jobs:
           compiler: ${{ matrix.compiler }}
           compiler-version: ${{ matrix.compiler-version }}
           extras: ${{ matrix.deps-extras || '' }}
+        if: ${{ !contains(matrix.extras, 'no-deps') }}
       - uses: ./.github/actions/hugepages
+        if: ${{ matrix.targets != 'check' }}
       - uses: ./.github/actions/cpusonline
       - uses: dtolnay/[email protected]
+        if: ${{ contains(matrix.targets, 'fdctl') }}
 
       - name: clean targets
         run: |

.gitignore

@@ -10,6 +10,7 @@ genhtml/
 target/
 *.lcov
 *.profraw
+*.profdata
 /report
 flamegraph.html
 perf.data
@@ -20,7 +21,7 @@ perf.data.old
 
 # Clangd
 .cache
-.clangd
+/.clangd
 compile_commands.json
 
 # Rust

NOTICE

@@ -279,7 +279,7 @@ with this software. If not, see
 
 =======================================================================
 
-'src/flamenco/nanopb' is currently a copy of the Nanopb library at
+'src/ballet/nanopb' is currently a copy of the Nanopb library at
 https://jpa.kapsi.fi/nanopb/ (imported ca 2023-Nov).
 
 Copyright (c) 2011 Petteri Aimonen <jpa at nanopb.mail.kapsi.fi>
@@ -466,3 +466,68 @@ ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+=======================================================================
+
+src/waltz/resolv is currently a modified implementation of the network
+code from musl.  musl can be found at
+https://musl.libc.org/releases/musl-1.2.5.tar.gz
+
+The list of modifications is documented in src/waltz/resolv/README.md
+
+Copyright © 2005-2020 Rich Felker, et al.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+=======================================================================
+
+The fd_h2 library (src/waltz/h2) includes a modified version of the
+HPACK Huffman decoder from nghttp2 (circa 2025-Mar).  Original sources:
+- https://github.com/nghttp2/nghttp2/blob/master/lib/nghttp2_hd.h
+- https://github.com/nghttp2/nghttp2/blob/master/lib/nghttp2_hd_huffman.h
+- https://github.com/nghttp2/nghttp2/blob/master/lib/nghttp2_hd_huffman.c
+- https://github.com/nghttp2/nghttp2/blob/master/lib/nghttp2_hd_huffman_data.c
+
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2013 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+=======================================================================

agave

@@ -149,6 +149,16 @@
 | bundle_​transaction_​received | `counter` | Total count of transactions received, including transactions within bundles |
 | bundle_​packet_​received | `counter` | Total count of packets received |
 | bundle_​bundle_​received | `counter` | Total count of bundles received |
+| bundle_​errors_​protobuf | `counter` | Number of gRPC errors encountered (Protobuf decode/encode error) |
+| bundle_​errors_​transport | `counter` | Number of gRPC errors encountered (Transport error) |
+| bundle_​errors_​timeout | `counter` | Number of gRPC errors encountered (I/O timeout) |
+| bundle_​errors_​no_​fee_​info | `counter` | Number of gRPC errors encountered (Bundle dropped due to missing fee info) |
+| bundle_​errors_​ssl_​alloc | `counter` | Number of gRPC errors encountered (OpenSSL alloc fail) |
+| bundle_​heap_​size | `gauge` | Workspace heap size |
+| bundle_​heap_​free_​bytes | `gauge` | Approx free space in workspace |
+| bundle_​shredstream_​heartbeats | `counter` | Number of ShredStream heartbeats successfully sent |
+| bundle_​keepalives | `counter` | Number of HTTP/2 PINGs acknowledged by server |
+| bundle_​connected | `gauge` | 1 if connected to the bundle server, 0 if not |
 
 ## Verify Tile
 | Metric | Type | Description |

book/api/metrics-generated.md

@@ -1,7 +1,7 @@
 MAKEFLAGS += --no-builtin-rules
 MAKEFLAGS += --no-builtin-variables
 .SUFFIXES:
-.PHONY: all info bin rust include lib unit-test integration-test fuzz-test help clean distclean asm ppp show-deps
+.PHONY: all info check bin rust include lib unit-test integration-test fuzz-test help clean distclean asm ppp show-deps
 .PHONY: run-unit-test run-integration-test run-script-test run-fuzz-test
 .PHONY: seccomp-policies cov-report dist-cov-report frontend
 .SECONDARY:
@@ -13,7 +13,10 @@ CPPFLAGS+=-DFD_BUILD_INFO=\"$(OBJDIR)/info\"
 CPPFLAGS+=$(EXTRA_CPPFLAGS)
 
 # Auxiliary rules that should not set up dependencies
-AUX_RULES:=clean distclean help show-deps run-unit-test run-integration-test cov-report dist-cov-report seccomp-policies frontend
+AUX_RULES:=clean distclean help run-unit-test run-integration-test cov-report dist-cov-report seccomp-policies frontend
+
+# Dry rules that should set up dependency targets, but not generate them
+DRY_RULES:=check show-deps
 
 all: info bin include lib unit-test fuzz-test
 
@@ -48,6 +51,7 @@ help:
 	# Explicit goals are: all bin include lib unit-test integration-test help clean distclean asm ppp
 	# "make all" is equivalent to "make bin include lib unit-test fuzz-test"
 	# "make info" makes build info $(OBJDIR)/info for the current platform (if not already made)
+	# "make check" quickly checks for obvious compile errors
 	# "make bin" makes all binaries for the current platform (except those requiring the Rust toolchain)
 	# "make include" makes all include files for the current platform
 	# "make lib" makes all libraries for the current platform
@@ -356,6 +360,12 @@ $(OBJDIR)/obj/%.i : src/%.cxx $(OBJDIR)/info
 	$(MKDIR) $(dir $@) && \
 $(CXX) $(CPPFLAGS) $(CXXFLAGS) -E $< -o $@
 
+$(OBJDIR)/obj/%.check : src/%.c
+	@$(CC) $(CPPFLAGS) $(CFLAGS) -fsyntax-only $<
+
+$(OBJDIR)/obj/%.check : src/%.cxx
+	@$(CXX) $(CPPFLAGS) $(CXXFLAGS) -fsyntax-only $<
+
 $(OBJDIR)/lib/%.a :
 	#######################################################################
 	# Creating library $@ from $^
@@ -402,7 +412,15 @@ $(foreach mk,$(shell $(FIND) -L src -type f -name Local.mk),$(eval $(call _inclu
 show-deps:
 	@for d in $(DEPFILES); do echo $$d; done
 
+# Define the check target.  Must be after the make fragments include so that
+# DEPFILES is fully populated
+
+check: $(DEPFILES:.d=.check)
+
+ifeq ($(filter $(MAKECMDGOALS),$(AUX_RULES) $(DRY_RULES)),)
+# Generate dependency files
 include $(DEPFILES)
+endif
 
 # Define the asm target.  Must be after the make fragments include so that
 # DEPFILES is fully populated

config/everything.mk

@@ -0,0 +1,2 @@
+# Set opt a path that does not exist
+OPT:=/nonexistent

config/extra/with-no-deps-pre.mk

@@ -0,0 +1 @@
+# See with-no-deps-pre.mk

config/extra/with-no-deps.mk

@@ -4,7 +4,7 @@ OPENSSL_LIBS=$(OPT)/lib/libssl.a $(OPT)/lib/libcrypto.a
 FD_HAS_OPENSSL:=1
 CPPFLAGS+=-DFD_HAS_OPENSSL=1
 
-CPPFLAGS+=-DOPENSSL_API_COMPAT=30000 -DOPENSSL_NO_DEPRECATED
+CPPFLAGS+=-DOPENSSL_API_COMPAT=30500 -DOPENSSL_NO_DEPRECATED
 else
 $(info "openssl not installed, skipping")
 endif

config/extra/with-openssl.mk

@@ -55,7 +55,6 @@ FD_HAS_DOUBLE:=1
 CPPFLAGS+=-DFD_HAS_DOUBLE=1
 $(call map-define,FD_HAS_ALLOCA, __linux__)
 $(call map-define,FD_HAS_THREADS, __linux__)
-$(call map-define,FD_HAS_OPENSSL, __linux__)
 $(call map-define,FD_HAS_X86, __x86_64__)
 $(call map-define,FD_HAS_SSE, __SSE4_2__)
 $(call map-define,FD_HAS_AVX, __AVX2__)

config/machine/native.mk

@@ -3,9 +3,6 @@ name = "bundle-test-server"
 version = "0.1.0"
 edition = "2021"
 
-[[bin]]
-name = "bundlesvr"
-
 [dependencies]
 tonic = { version = "0.12.2", features = ["tls-roots", "tls", "tls-webpki-roots"] }
 prost = "0.13.3"

plugin/bundle/test-server/Cargo.toml renamed to contrib/bundle-test-server/Cargo.toml

@@ -24,8 +24,8 @@ fn main() -> Result<(), std::io::Error> {
     }
 
     configure()
-        .build_client(true)
-        .build_server(false)
+        .build_client(false)
+        .build_server(true)
         .type_attribute(
             "TransactionErrorType",
             "#[cfg_attr(test, derive(enum_iterator::Sequence))]",
@@ -34,5 +34,5 @@ fn main() -> Result<(), std::io::Error> {
             "InstructionErrorType",
             "#[cfg_attr(test, derive(enum_iterator::Sequence))]",
         )
-        .compile_protos(&protos, &[proto_base_path])
+        .compile_protos(&protos, &[std::path::PathBuf::from("protos")])
 }