Merge pull request #899 from HathorNetwork/release-candidate

Release v0.58.0

.github/workflows/main.yml

@@ -8,6 +8,9 @@ on: # yamllint disable-line rule:truthy
     tags:
       - v*
   pull_request:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 jobs:
   matrix:
     runs-on: ubuntu-latest

extras/custom_checks.sh

@@ -72,19 +72,55 @@ function check_do_not_use_builtin_random_in_tests() {
 }
 
 function check_deprecated_typing() {
-	if grep -R '\<typing .*\<import .*\<\(Tuple\|List\|Dict\|Set\|FrozenSet\|AbstractSet\|DefaultDict\|OrderedDict\)\>' "${SOURCE_DIRS[@]}"; then
+	if grep -RI '\<typing .*\<import .*\<\(Tuple\|List\|Dict\|Set\|FrozenSet\|AbstractSet\|DefaultDict\|OrderedDict\)\>' "${SOURCE_DIRS[@]}"; then
 		echo 'do not use typing.List/Tuple/Dict/... for type annotations use builtin list/tuple/dict/... instead'
 		echo 'for more info check the PEP 585 doc: https://peps.python.org/pep-0585/'
 		return 1
 	fi
 	return 0
 }
 
+function check_do_not_import_tests_in_hathor() {
+	if grep -R '\<.*import .*tests.*\>\|\<.*from .*tests.* import\>' "hathor"; then
+		echo 'do not import test definitions in the hathor module'
+		echo 'move them from tests to hathor instead'
+		return 1
+	fi
+	return 0
+}
+
+function check_do_not_import_from_hathor_in_entrypoints() {
+    PATTERN='^import .*hathor.*\|^from .*hathor.* import'
+
+    if grep -R "$PATTERN" "hathor/cli" | grep -v 'from hathor.cli.run_node import RunNode' | grep -v '# skip-cli-import-custom-check'; then
+        echo 'do not import from `hathor` in the module-level of a CLI entrypoint.'
+        echo 'instead, import locally inside the function that uses the import.'
+        echo 'alternatively, comment `# skip-cli-import-custom-check` to exclude a line.'
+        return 1
+    fi
+    return 0
+}
+
+function check_do_not_import_twisted_reactor_directly() {
+    EXCLUDES="--exclude=reactor.py --exclude=conftest.py"
+    PATTERN='\<.*from .*twisted.internet import .*reactor\>'
+
+    if grep -R $EXCLUDES "$PATTERN" "${SOURCE_DIRS[@]}"; then
+        echo 'do not use `from twisted.internet import reactor` directly.'
+        echo 'instead, use `hathor.reactor.get_global_reactor()`.'
+        return 1
+    fi
+    return 0
+}
+
 # List of functions to be executed
 checks=(
 	check_version_match
 	check_do_not_use_builtin_random_in_tests
 	check_deprecated_typing
+	check_do_not_import_tests_in_hathor
+	check_do_not_import_from_hathor_in_entrypoints
+	check_do_not_import_twisted_reactor_directly
 )
 
 # Initialize a variable to track if any check fails

hathor/builder/builder.py

@@ -13,14 +13,15 @@
 # limitations under the License.
 
 from enum import Enum
-from typing import Any, NamedTuple, Optional
+from typing import Any, Callable, NamedTuple, Optional, TypeAlias
 
 from structlog import get_logger
 
 from hathor.checkpoint import Checkpoint
 from hathor.conf.get_settings import get_settings
 from hathor.conf.settings import HathorSettings as HathorSettingsType
 from hathor.consensus import ConsensusAlgorithm
+from hathor.daa import DifficultyAdjustmentAlgorithm
 from hathor.event import EventManager
 from hathor.event.storage import EventMemoryStorage, EventRocksDBStorage, EventStorage
 from hathor.event.websocket import EventWebsocketFactory
@@ -29,9 +30,11 @@
 from hathor.feature_activation.feature_service import FeatureService
 from hathor.indexes import IndexesManager, MemoryIndexesManager, RocksDBIndexesManager
 from hathor.manager import HathorManager
+from hathor.mining.cpu_mining_service import CpuMiningService
 from hathor.p2p.manager import ConnectionsManager
 from hathor.p2p.peer_id import PeerId
 from hathor.pubsub import PubSubManager
+from hathor.reactor import ReactorProtocol as Reactor
 from hathor.storage import RocksDBStorage
 from hathor.stratum import StratumFactory
 from hathor.transaction.storage import (
@@ -40,7 +43,7 @@
     TransactionRocksDBStorage,
     TransactionStorage,
 )
-from hathor.util import Random, Reactor, get_environment_info
+from hathor.util import Random, get_environment_info, not_none
 from hathor.verification.verification_service import VerificationService, VertexVerifiers
 from hathor.wallet import BaseWallet, Wallet
 
@@ -63,12 +66,19 @@ class BuildArtifacts(NamedTuple):
     pubsub: PubSubManager
     consensus: ConsensusAlgorithm
     tx_storage: TransactionStorage
+    feature_service: FeatureService
     indexes: Optional[IndexesManager]
     wallet: Optional[BaseWallet]
     rocksdb_storage: Optional[RocksDBStorage]
     stratum_factory: Optional[StratumFactory]
 
 
+_VertexVerifiersBuilder: TypeAlias = Callable[
+    [HathorSettingsType, DifficultyAdjustmentAlgorithm, FeatureService],
+    VertexVerifiers
+]
+
+
 class Builder:
     """Builder builds the core objects to run a full node.
 
@@ -102,7 +112,11 @@ def __init__(self) -> None:
         self._feature_service: Optional[FeatureService] = None
         self._bit_signaling_service: Optional[BitSignalingService] = None
 
+        self._daa: Optional[DifficultyAdjustmentAlgorithm] = None
+        self._cpu_mining_service: Optional[CpuMiningService] = None
+
         self._vertex_verifiers: Optional[VertexVerifiers] = None
+        self._vertex_verifiers_builder: _VertexVerifiersBuilder | None = None
         self._verification_service: Optional[VerificationService] = None
 
         self._rocksdb_path: Optional[str] = None
@@ -127,8 +141,7 @@ def __init__(self) -> None:
         self._enable_tokens_index: bool = False
         self._enable_utxo_index: bool = False
 
-        self._enable_sync_v1: bool = False
-        self._enable_sync_v1_1: bool = True
+        self._enable_sync_v1: bool = True
         self._enable_sync_v2: bool = False
 
         self._enable_stratum_server: Optional[bool] = None
@@ -158,10 +171,12 @@ def build(self) -> BuildArtifacts:
         wallet = self._get_or_create_wallet()
         event_manager = self._get_or_create_event_manager()
         indexes = self._get_or_create_indexes_manager()
-        tx_storage = self._get_or_create_tx_storage(indexes)
-        feature_service = self._get_or_create_feature_service(tx_storage)
-        bit_signaling_service = self._get_or_create_bit_signaling_service(tx_storage)
+        tx_storage = self._get_or_create_tx_storage()
+        feature_service = self._get_or_create_feature_service()
+        bit_signaling_service = self._get_or_create_bit_signaling_service()
         verification_service = self._get_or_create_verification_service()
+        daa = self._get_or_create_daa()
+        cpu_mining_service = self._get_or_create_cpu_mining_service()
 
         if self._enable_address_index:
             indexes.enable_address_index(pubsub)
@@ -186,6 +201,7 @@ def build(self) -> BuildArtifacts:
             network=self._network,
             pubsub=pubsub,
             consensus_algorithm=consensus_algorithm,
+            daa=daa,
             peer_id=peer_id,
             tx_storage=tx_storage,
             p2p_manager=p2p_manager,
@@ -198,6 +214,7 @@ def build(self) -> BuildArtifacts:
             feature_service=feature_service,
             bit_signaling_service=bit_signaling_service,
             verification_service=verification_service,
+            cpu_mining_service=cpu_mining_service,
             **kwargs
         )
 
@@ -221,6 +238,7 @@ def build(self) -> BuildArtifacts:
             wallet=wallet,
             rocksdb_storage=self._rocksdb_storage,
             stratum_factory=stratum_factory,
+            feature_service=feature_service,
         )
 
         return self.artifacts
@@ -265,6 +283,7 @@ def set_peer_id(self, peer_id: PeerId) -> 'Builder':
         return self
 
     def _get_or_create_settings(self) -> HathorSettingsType:
+        """Return the HathorSettings instance set on this builder, or a new one if not set."""
         if self._settings is None:
             self._settings = get_settings()
         return self._settings
@@ -293,7 +312,7 @@ def _get_or_create_pubsub(self) -> PubSubManager:
         return self._pubsub
 
     def _create_stratum_server(self, manager: HathorManager) -> StratumFactory:
-        stratum_factory = StratumFactory(manager=manager)
+        stratum_factory = StratumFactory(manager=manager, reactor=self._get_reactor())
         manager.stratum_factory = stratum_factory
         manager.metrics.stratum_factory = stratum_factory
         return stratum_factory
@@ -316,6 +335,10 @@ def _get_or_create_rocksdb_storage(self) -> RocksDBStorage:
         return self._rocksdb_storage
 
     def _get_p2p_manager(self) -> ConnectionsManager:
+        from hathor.p2p.sync_v1.factory import SyncV11Factory
+        from hathor.p2p.sync_v2.factory import SyncV2Factory
+        from hathor.p2p.sync_version import SyncVersion
+
         enable_ssl = True
         reactor = self._get_reactor()
         my_peer = self._get_peer_id()
@@ -330,10 +353,13 @@ def _get_p2p_manager(self) -> ConnectionsManager:
             ssl=enable_ssl,
             whitelist_only=False,
             rng=self._rng,
-            enable_sync_v1=self._enable_sync_v1,
-            enable_sync_v1_1=self._enable_sync_v1_1,
-            enable_sync_v2=self._enable_sync_v2,
         )
+        p2p_manager.add_sync_factory(SyncVersion.V1_1, SyncV11Factory(p2p_manager))
+        p2p_manager.add_sync_factory(SyncVersion.V2, SyncV2Factory(p2p_manager))
+        if self._enable_sync_v1:
+            p2p_manager.enable_sync_version(SyncVersion.V1_1)
+        if self._enable_sync_v2:
+            p2p_manager.enable_sync_version(SyncVersion.V2)
         return p2p_manager
 
     def _get_or_create_indexes_manager(self) -> IndexesManager:
@@ -352,7 +378,9 @@ def _get_or_create_indexes_manager(self) -> IndexesManager:
 
         return self._indexes_manager
 
-    def _get_or_create_tx_storage(self, indexes: IndexesManager) -> TransactionStorage:
+    def _get_or_create_tx_storage(self) -> TransactionStorage:
+        indexes = self._get_or_create_indexes_manager()
+
         if self._tx_storage is not None:
             # If a tx storage is provided, set the indexes manager to it.
             self._tx_storage.indexes = indexes
@@ -396,9 +424,16 @@ def _get_or_create_event_storage(self) -> EventStorage:
 
     def _get_or_create_event_manager(self) -> EventManager:
         if self._event_manager is None:
+            peer_id = self._get_peer_id()
+            settings = self._get_or_create_settings()
             reactor = self._get_reactor()
             storage = self._get_or_create_event_storage()
-            factory = EventWebsocketFactory(reactor, storage)
+            factory = EventWebsocketFactory(
+                peer_id=not_none(peer_id.id),
+                network=settings.NETWORK_NAME,
+                reactor=reactor,
+                event_storage=storage,
+            )
             self._event_manager = EventManager(
                 reactor=reactor,
                 pubsub=self._get_or_create_pubsub(),
@@ -408,22 +443,26 @@ def _get_or_create_event_manager(self) -> EventManager:
 
         return self._event_manager
 
-    def _get_or_create_feature_service(self, tx_storage: TransactionStorage) -> FeatureService:
+    def _get_or_create_feature_service(self) -> FeatureService:
+        """Return the FeatureService instance set on this builder, or a new one if not set."""
         if self._feature_service is None:
             settings = self._get_or_create_settings()
+            tx_storage = self._get_or_create_tx_storage()
             self._feature_service = FeatureService(
                 feature_settings=settings.FEATURE_ACTIVATION,
                 tx_storage=tx_storage
             )
 
         return self._feature_service
 
-    def _get_or_create_bit_signaling_service(self, tx_storage: TransactionStorage) -> BitSignalingService:
+    def _get_or_create_bit_signaling_service(self) -> BitSignalingService:
         if self._bit_signaling_service is None:
             settings = self._get_or_create_settings()
+            tx_storage = self._get_or_create_tx_storage()
+            feature_service = self._get_or_create_feature_service()
             self._bit_signaling_service = BitSignalingService(
                 feature_settings=settings.FEATURE_ACTIVATION,
-                feature_service=self._get_or_create_feature_service(tx_storage),
+                feature_service=feature_service,
                 tx_storage=tx_storage,
                 support_features=self._support_features,
                 not_support_features=self._not_support_features,
@@ -441,10 +480,33 @@ def _get_or_create_verification_service(self) -> VerificationService:
     def _get_or_create_vertex_verifiers(self) -> VertexVerifiers:
         if self._vertex_verifiers is None:
             settings = self._get_or_create_settings()
-            self._vertex_verifiers = VertexVerifiers.create_defaults(settings=settings)
+            feature_service = self._get_or_create_feature_service()
+            daa = self._get_or_create_daa()
+
+            if self._vertex_verifiers_builder:
+                self._vertex_verifiers = self._vertex_verifiers_builder(settings, daa, feature_service)
+            else:
+                self._vertex_verifiers = VertexVerifiers.create_defaults(
+                    settings=settings,
+                    daa=daa,
+                    feature_service=feature_service,
+                )
 
         return self._vertex_verifiers
 
+    def _get_or_create_daa(self) -> DifficultyAdjustmentAlgorithm:
+        if self._daa is None:
+            settings = self._get_or_create_settings()
+            self._daa = DifficultyAdjustmentAlgorithm(settings=settings)
+
+        return self._daa
+
+    def _get_or_create_cpu_mining_service(self) -> CpuMiningService:
+        if self._cpu_mining_service is None:
+            self._cpu_mining_service = CpuMiningService()
+
+        return self._cpu_mining_service
+
     def use_memory(self) -> 'Builder':
         self.check_if_can_modify()
         self._storage_type = StorageType.MEMORY
@@ -547,6 +609,21 @@ def set_vertex_verifiers(self, vertex_verifiers: VertexVerifiers) -> 'Builder':
         self._vertex_verifiers = vertex_verifiers
         return self
 
+    def set_vertex_verifiers_builder(self, builder: _VertexVerifiersBuilder) -> 'Builder':
+        self.check_if_can_modify()
+        self._vertex_verifiers_builder = builder
+        return self
+
+    def set_daa(self, daa: DifficultyAdjustmentAlgorithm) -> 'Builder':
+        self.check_if_can_modify()
+        self._daa = daa
+        return self
+
+    def set_cpu_mining_service(self, cpu_mining_service: CpuMiningService) -> 'Builder':
+        self.check_if_can_modify()
+        self._cpu_mining_service = cpu_mining_service
+        return self
+
     def set_reactor(self, reactor: Reactor) -> 'Builder':
         self.check_if_can_modify()
         self._reactor = reactor
@@ -567,11 +644,6 @@ def set_enable_sync_v1(self, enable_sync_v1: bool) -> 'Builder':
         self._enable_sync_v1 = enable_sync_v1
         return self
 
-    def set_enable_sync_v1_1(self, enable_sync_v1_1: bool) -> 'Builder':
-        self.check_if_can_modify()
-        self._enable_sync_v1_1 = enable_sync_v1_1
-        return self
-
     def set_enable_sync_v2(self, enable_sync_v2: bool) -> 'Builder':
         self.check_if_can_modify()
         self._enable_sync_v2 = enable_sync_v2
@@ -587,16 +659,6 @@ def disable_sync_v1(self) -> 'Builder':
         self._enable_sync_v1 = False
         return self
 
-    def enable_sync_v1_1(self) -> 'Builder':
-        self.check_if_can_modify()
-        self._enable_sync_v1_1 = True
-        return self
-
-    def disable_sync_v1_1(self) -> 'Builder':
-        self.check_if_can_modify()
-        self._enable_sync_v1_1 = False
-        return self
-
     def enable_sync_v2(self) -> 'Builder':
         self.check_if_can_modify()
         self._enable_sync_v2 = True