Merge branch 'develop' into feat/use-burnchain-timeout

Author: obycode

.github/workflows/bitcoin-tests.yml

@@ -45,7 +45,8 @@ jobs:
           - tests::neon_integrations::liquid_ustx_integration
           - tests::neon_integrations::microblock_fork_poison_integration_test
           - tests::neon_integrations::microblock_integration_test
-          - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY
+          # Disable this flaky test. Microblocks are no longer supported anyways.
+          # - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY
           - tests::neon_integrations::microblock_limit_hit_integration_test
           - tests::neon_integrations::miner_submit_twice
           - tests::neon_integrations::mining_events_integration_test
@@ -86,7 +87,6 @@ jobs:
           - tests::nakamoto_integrations::nakamoto_attempt_time
           - tests::signer::v0::block_proposal_rejection
           - tests::signer::v0::miner_gather_signatures
-          - tests::signer::v0::mine_2_nakamoto_reward_cycles
           - tests::signer::v0::end_of_tenure
           - tests::signer::v0::forked_tenure_okay
           - tests::signer::v0::forked_tenure_invalid
@@ -95,12 +95,16 @@ jobs:
           - tests::signer::v0::multiple_miners
           - tests::signer::v0::mock_sign_epoch_25
           - tests::signer::v0::multiple_miners_mock_sign_epoch_25
-          - tests::signer::v0::signer_set_rollover
           - tests::signer::v0::miner_forking
           - tests::signer::v0::reloads_signer_set_in
           - tests::signer::v0::signers_broadcast_signed_blocks
           - tests::signer::v0::min_gap_between_blocks
           - tests::signer::v0::duplicate_signers
+          - tests::signer::v0::retry_on_rejection
+          - tests::signer::v0::locally_accepted_blocks_overriden_by_global_rejection
+          - tests::signer::v0::locally_rejected_blocks_overriden_by_global_acceptance
+          - tests::signer::v0::reorg_locally_accepted_blocks_across_tenures_succeeds
+          - tests::signer::v0::miner_recovers_when_broadcast_block_delay_across_tenures_occurs
           - tests::nakamoto_integrations::stack_stx_burn_op_integration_test
           - tests::nakamoto_integrations::check_block_heights
           - tests::nakamoto_integrations::clarity_burn_state
@@ -115,6 +119,8 @@ jobs:
           - tests::nakamoto_integrations::utxo_check_on_startup_recover
           - tests::signer::v0::multiple_miners_with_nakamoto_blocks
           - tests::signer::v0::partial_tenure_fork
+          - tests::signer::v0::mine_2_nakamoto_reward_cycles
+          - tests::signer::v0::signer_set_rollover
           # Do not run this one until we figure out why it fails in CI
           # - tests::neon_integrations::bitcoin_reorg_flap
           # - tests::neon_integrations::bitcoin_reorg_flap_with_follower

docs/rpc/openapi.yaml

@@ -568,7 +568,7 @@ paths:
             example:
               $ref: ./api/core-node/post-block-proposal-req.example.json
 
-  /v2/stacker_set/{cycle_number}:
+  /v3/stacker_set/{cycle_number}:
     get:
       summary: Fetch the stacker and signer set information for a given cycle.
       tags:

libsigner/src/events.rs

@@ -314,6 +314,9 @@ impl<T: SignerEventTrait> EventReceiver<T> for SignerEventReceiver<T> {
                 process_proposal_response(request)
             } else if request.url() == "/new_burn_block" {
                 process_new_burn_block_event(request)
+            } else if request.url() == "/shutdown" {
+                event_receiver.stop_signal.store(true, Ordering::SeqCst);
+                return Err(EventError::Terminated);
             } else {
                 let url = request.url().to_string();
                 // `/new_block` is expected, but not specifically handled. do not log.

libsigner/src/v0/messages.rs

@@ -42,6 +42,7 @@ use blockstack_lib::util_lib::boot::boot_code_id;
 use blockstack_lib::util_lib::signed_structured_data::{
     make_structured_data_domain, structured_data_message_hash,
 };
+use clarity::consts::{CHAIN_ID_MAINNET, CHAIN_ID_TESTNET};
 use clarity::types::chainstate::{
     BlockHeaderHash, ConsensusHash, StacksPrivateKey, StacksPublicKey,
 };
@@ -525,7 +526,9 @@ RejectCodeTypePrefix {
     /// The block was rejected due to no sortition view
     NoSortitionView = 3,
     /// The block was rejected due to a mismatch with expected sortition view
-    SortitionViewMismatch = 4
+    SortitionViewMismatch = 4,
+    /// The block was rejected due to a testing directive
+    TestingDirective = 5
 });
 
 impl TryFrom<u8> for RejectCodeTypePrefix {
@@ -545,6 +548,7 @@ impl From<&RejectCode> for RejectCodeTypePrefix {
             RejectCode::RejectedInPriorRound => RejectCodeTypePrefix::RejectedInPriorRound,
             RejectCode::NoSortitionView => RejectCodeTypePrefix::NoSortitionView,
             RejectCode::SortitionViewMismatch => RejectCodeTypePrefix::SortitionViewMismatch,
+            RejectCode::TestingDirective => RejectCodeTypePrefix::TestingDirective,
         }
     }
 }
@@ -562,6 +566,8 @@ pub enum RejectCode {
     RejectedInPriorRound,
     /// The block was rejected due to a mismatch with expected sortition view
     SortitionViewMismatch,
+    /// The block was rejected due to a testing directive
+    TestingDirective,
 }
 
 define_u8_enum!(
@@ -615,8 +621,8 @@ impl std::fmt::Display for BlockResponse {
             BlockResponse::Rejected(r) => {
                 write!(
                     f,
-                    "BlockRejected: signer_sighash = {}, code = {}, reason = {}",
-                    r.reason_code, r.reason, r.signer_signature_hash
+                    "BlockRejected: signer_sighash = {}, code = {}, reason = {}, signature = {}",
+                    r.reason_code, r.reason, r.signer_signature_hash, r.signature
                 )
             }
         }
@@ -629,9 +635,14 @@ impl BlockResponse {
         Self::Accepted((hash, sig))
     }
 
-    /// Create a new rejected BlockResponse for the provided block signer signature hash and rejection code
-    pub fn rejected(hash: Sha512Trunc256Sum, reject_code: RejectCode) -> Self {
-        Self::Rejected(BlockRejection::new(hash, reject_code))
+    /// Create a new rejected BlockResponse for the provided block signer signature hash and rejection code and sign it with the provided private key
+    pub fn rejected(
+        hash: Sha512Trunc256Sum,
+        reject_code: RejectCode,
+        private_key: &StacksPrivateKey,
+        mainnet: bool,
+    ) -> Self {
+        Self::Rejected(BlockRejection::new(hash, reject_code, private_key, mainnet))
     }
 }
 
@@ -677,16 +688,94 @@ pub struct BlockRejection {
     pub reason_code: RejectCode,
     /// The signer signature hash of the block that was rejected
     pub signer_signature_hash: Sha512Trunc256Sum,
+    /// The signer's signature across the rejection
+    pub signature: MessageSignature,
+    /// The chain id
+    pub chain_id: u32,
 }
 
 impl BlockRejection {
     /// Create a new BlockRejection for the provided block and reason code
-    pub fn new(signer_signature_hash: Sha512Trunc256Sum, reason_code: RejectCode) -> Self {
-        Self {
+    pub fn new(
+        signer_signature_hash: Sha512Trunc256Sum,
+        reason_code: RejectCode,
+        private_key: &StacksPrivateKey,
+        mainnet: bool,
+    ) -> Self {
+        let chain_id = if mainnet {
+            CHAIN_ID_MAINNET
+        } else {
+            CHAIN_ID_TESTNET
+        };
+        let mut rejection = Self {
             reason: reason_code.to_string(),
             reason_code,
             signer_signature_hash,
+            signature: MessageSignature::empty(),
+            chain_id,
+        };
+        rejection
+            .sign(private_key)
+            .expect("Failed to sign BlockRejection");
+        rejection
+    }
+
+    /// Create a new BlockRejection from a BlockValidateRejection
+    pub fn from_validate_rejection(
+        reject: BlockValidateReject,
+        private_key: &StacksPrivateKey,
+        mainnet: bool,
+    ) -> Self {
+        let chain_id = if mainnet {
+            CHAIN_ID_MAINNET
+        } else {
+            CHAIN_ID_TESTNET
+        };
+        let mut rejection = Self {
+            reason: reject.reason,
+            reason_code: RejectCode::ValidationFailed(reject.reason_code),
+            signer_signature_hash: reject.signer_signature_hash,
+            chain_id,
+            signature: MessageSignature::empty(),
+        };
+        rejection
+            .sign(private_key)
+            .expect("Failed to sign BlockRejection");
+        rejection
+    }
+
+    /// The signature hash for the block rejection
+    pub fn hash(&self) -> Sha256Sum {
+        let domain_tuple = make_structured_data_domain("block-rejection", "1.0.0", self.chain_id);
+        let data = Value::buff_from(self.signer_signature_hash.as_bytes().into()).unwrap();
+        structured_data_message_hash(data, domain_tuple)
+    }
+
+    /// Sign the block rejection and set the internal signature field
+    fn sign(&mut self, private_key: &StacksPrivateKey) -> Result<(), String> {
+        let signature_hash = self.hash();
+        self.signature = private_key.sign(signature_hash.as_bytes())?;
+        Ok(())
+    }
+
+    /// Verify the rejection's signature against the provided signer public key
+    pub fn verify(&self, public_key: &StacksPublicKey) -> Result<bool, String> {
+        if self.signature == MessageSignature::empty() {
+            return Ok(false);
+        }
+        let signature_hash = self.hash();
+        public_key
+            .verify(&signature_hash.0, &self.signature)
+            .map_err(|e| e.to_string())
+    }
+
+    /// Recover the public key from the rejection signature
+    pub fn recover_public_key(&self) -> Result<StacksPublicKey, &'static str> {
+        if self.signature == MessageSignature::empty() {
+            return Err("No signature to recover public key from");
         }
+        let signature_hash = self.hash();
+        StacksPublicKey::recover_to_pubkey(signature_hash.as_bytes(), &self.signature)
     }
 }
 
@@ -695,6 +784,8 @@ impl StacksMessageCodec for BlockRejection {
         write_next(fd, &self.reason.as_bytes().to_vec())?;
         write_next(fd, &self.reason_code)?;
         write_next(fd, &self.signer_signature_hash)?;
+        write_next(fd, &self.chain_id)?;
+        write_next(fd, &self.signature)?;
         Ok(())
     }
 
@@ -705,24 +796,18 @@ impl StacksMessageCodec for BlockRejection {
         })?;
         let reason_code = read_next::<RejectCode, _>(fd)?;
         let signer_signature_hash = read_next::<Sha512Trunc256Sum, _>(fd)?;
+        let chain_id = read_next::<u32, _>(fd)?;
+        let signature = read_next::<MessageSignature, _>(fd)?;
         Ok(Self {
             reason,
             reason_code,
             signer_signature_hash,
+            chain_id,
+            signature,
         })
     }
 }
 
-impl From<BlockValidateReject> for BlockRejection {
-    fn from(reject: BlockValidateReject) -> Self {
-        Self {
-            reason: reject.reason,
-            reason_code: RejectCode::ValidationFailed(reject.reason_code),
-            signer_signature_hash: reject.signer_signature_hash,
-        }
-    }
-}
-
 impl StacksMessageCodec for RejectCode {
     fn consensus_serialize<W: Write>(&self, fd: &mut W) -> Result<(), CodecError> {
         write_next(fd, &(RejectCodeTypePrefix::from(self) as u8))?;
@@ -732,7 +817,8 @@ impl StacksMessageCodec for RejectCode {
             RejectCode::ConnectivityIssues
             | RejectCode::RejectedInPriorRound
             | RejectCode::NoSortitionView
-            | RejectCode::SortitionViewMismatch => {
+            | RejectCode::SortitionViewMismatch
+            | RejectCode::TestingDirective => {
                 // No additional data to serialize / deserialize
             }
         };
@@ -755,6 +841,7 @@ impl StacksMessageCodec for RejectCode {
             RejectCodeTypePrefix::RejectedInPriorRound => RejectCode::RejectedInPriorRound,
             RejectCodeTypePrefix::NoSortitionView => RejectCode::NoSortitionView,
             RejectCodeTypePrefix::SortitionViewMismatch => RejectCode::SortitionViewMismatch,
+            RejectCodeTypePrefix::TestingDirective => RejectCode::TestingDirective,
         };
         Ok(code)
     }
@@ -782,6 +869,9 @@ impl std::fmt::Display for RejectCode {
                     "The block was rejected due to a mismatch with expected sortition view."
                 )
             }
+            RejectCode::TestingDirective => {
+                write!(f, "The block was rejected due to a testing directive.")
+            }
         }
     }
 }
@@ -792,12 +882,6 @@ impl From<BlockResponse> for SignerMessage {
     }
 }
 
-impl From<BlockValidateReject> for BlockResponse {
-    fn from(rejection: BlockValidateReject) -> Self {
-        Self::Rejected(rejection.into())
-    }
-}
-
 #[cfg(test)]
 mod test {
     use blockstack_lib::chainstate::nakamoto::NakamotoBlockHeader;
@@ -851,14 +935,20 @@ mod test {
         let rejection = BlockRejection::new(
             Sha512Trunc256Sum([0u8; 32]),
             RejectCode::ValidationFailed(ValidateRejectCode::InvalidBlock),
+            &StacksPrivateKey::new(),
+            thread_rng().gen_bool(0.5),
         );
         let serialized_rejection = rejection.serialize_to_vec();
         let deserialized_rejection = read_next::<BlockRejection, _>(&mut &serialized_rejection[..])
             .expect("Failed to deserialize BlockRejection");
         assert_eq!(rejection, deserialized_rejection);
 
-        let rejection =
-            BlockRejection::new(Sha512Trunc256Sum([1u8; 32]), RejectCode::ConnectivityIssues);
+        let rejection = BlockRejection::new(
+            Sha512Trunc256Sum([1u8; 32]),
+            RejectCode::ConnectivityIssues,
+            &StacksPrivateKey::new(),
+            thread_rng().gen_bool(0.5),
+        );
         let serialized_rejection = rejection.serialize_to_vec();
         let deserialized_rejection = read_next::<BlockRejection, _>(&mut &serialized_rejection[..])
             .expect("Failed to deserialize BlockRejection");
@@ -877,6 +967,8 @@ mod test {
         let response = BlockResponse::Rejected(BlockRejection::new(
             Sha512Trunc256Sum([1u8; 32]),
             RejectCode::ValidationFailed(ValidateRejectCode::InvalidBlock),
+            &StacksPrivateKey::new(),
+            thread_rng().gen_bool(0.5),
         ));
         let serialized_response = response.serialize_to_vec();
         let deserialized_response = read_next::<BlockResponse, _>(&mut &serialized_response[..])

stacks-common/src/bitvec.rs

@@ -100,8 +100,8 @@ impl<const MAX_SIZE: u16> Serialize for BitVec<MAX_SIZE> {
 
 impl<'de, const MAX_SIZE: u16> Deserialize<'de> for BitVec<MAX_SIZE> {
     fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
-        let hex: &str = Deserialize::deserialize(deserializer)?;
-        let bytes = hex_bytes(hex).map_err(serde::de::Error::custom)?;
+        let hex: String = Deserialize::deserialize(deserializer)?;
+        let bytes = hex_bytes(hex.as_str()).map_err(serde::de::Error::custom)?;
         Self::consensus_deserialize(&mut bytes.as_slice()).map_err(serde::de::Error::custom)
     }
 }
@@ -412,4 +412,21 @@ mod test {
             check_ok_vector(i.as_slice());
         }
     }
+
+    #[test]
+    fn test_serde() {
+        let mut bitvec_zero_10 = BitVec::<10>::zeros(10).unwrap();
+        bitvec_zero_10.set(0, true).unwrap();
+        bitvec_zero_10.set(5, true).unwrap();
+        bitvec_zero_10.set(3, true).unwrap();
+        assert_eq!(
+            bitvec_zero_10.binary_str(),
+            "1001010000",
+            "Binary string should be 1001010000"
+        );
+
+        let serde_bitvec_json = serde_json::to_string(&bitvec_zero_10).unwrap();
+        let serde_bitvec: BitVec<10> = serde_json::from_str(&serde_bitvec_json).unwrap();
+        assert_eq!(serde_bitvec, bitvec_zero_10);
+    }
 }