Merge pull request #1217 from HathorNetwork/release-candidate

Release v0.63.1

docs/event-queue-feature.md

@@ -15,19 +15,14 @@ When the Event Queue feature is enabled, the full node will generate specific ev
 
 ## Enabling the Event Queue
 
-To enable the Event Queue feature, you must add two CLI options when running the full node:
-
-1. Add `--unsafe-mode [network_name]`
-2. Add `--x-enable-event-queue`
+To enable the Event Queue feature, you must add this CLI option when running the full node: `--enable-event-queue`.
 
 For example:
 
 ```bash
-poetry run hathor-cli run_node --memory-storage --status 8080 --testnet --unsafe-mode testnet-golf --x-enable-event-queue
+poetry run hathor-cli run_node --memory-storage --status 8080 --testnet --enable-event-queue
 ```
 
-**ATTENTION**: While the Event Queue is in beta, it's considered unsafe. You must not use it in production environments.
-
 ### First run
 
 If this is the first time your full node is running with the event queue enabled, there are 3 possibilities:
@@ -45,7 +40,7 @@ For case 2.2, an extra loading step will be performed during full node initializ
 
 After running the full node with the Event Queue enabled, if you restart your full node (that is, stop it and then run it again), there are 2 possibilities:
 
-1. You run the full node with the `--x-enable-event-queue` CLI option, that is, you keep the Event Queue enabled, or
+1. You run the full node with the `--enable-event-queue` CLI option, that is, you keep the Event Queue enabled, or
 2. You run the full node without the CLI option, that is, you don't enable it, but you **have to clear the event data in the database**.
 
 For case 1, the full node will start normally, and continue to generate new events for synced vertices from where it stopped in the previous run.

hathor/builder/builder.py

@@ -66,21 +66,14 @@ def add_factories(
         cls,
         settings: HathorSettingsType,
         p2p_manager: ConnectionsManager,
-        sync_v1_support: 'SyncSupportLevel',
         sync_v2_support: 'SyncSupportLevel',
         vertex_parser: VertexParser,
         vertex_handler: VertexHandler,
     ) -> None:
         """Adds the sync factory to the manager according to the support level."""
-        from hathor.p2p.sync_v1.factory import SyncV11Factory
         from hathor.p2p.sync_v2.factory import SyncV2Factory
         from hathor.p2p.sync_version import SyncVersion
 
-        # sync-v1 support:
-        if sync_v1_support > cls.UNAVAILABLE:
-            p2p_manager.add_sync_factory(SyncVersion.V1_1, SyncV11Factory(p2p_manager, vertex_parser=vertex_parser))
-        if sync_v1_support is cls.ENABLED:
-            p2p_manager.enable_sync_version(SyncVersion.V1_1)
         # sync-v2 support:
         if sync_v2_support > cls.UNAVAILABLE:
             sync_v2_factory = SyncV2Factory(
@@ -185,8 +178,7 @@ def __init__(self) -> None:
         self._enable_tokens_index: bool = False
         self._enable_utxo_index: bool = False
 
-        self._sync_v1_support: SyncSupportLevel = SyncSupportLevel.UNAVAILABLE
-        self._sync_v2_support: SyncSupportLevel = SyncSupportLevel.UNAVAILABLE
+        self._sync_v2_support: SyncSupportLevel = SyncSupportLevel.ENABLED
 
         self._enable_stratum_server: Optional[bool] = None
 
@@ -209,7 +201,7 @@ def build(self) -> BuildArtifacts:
         if self.artifacts is not None:
             raise ValueError('cannot call build twice')
 
-        if SyncSupportLevel.ENABLED not in {self._sync_v1_support, self._sync_v2_support}:
+        if SyncSupportLevel.ENABLED not in {self._sync_v2_support}:
             raise TypeError('you must enable at least one sync version')
 
         settings = self._get_or_create_settings()
@@ -380,8 +372,7 @@ def _get_or_create_consensus(self) -> ConsensusAlgorithm:
         if self._consensus is None:
             soft_voided_tx_ids = self._get_soft_voided_tx_ids()
             pubsub = self._get_or_create_pubsub()
-            execution_manager = self._get_or_create_execution_manager()
-            self._consensus = ConsensusAlgorithm(soft_voided_tx_ids, pubsub, execution_manager=execution_manager)
+            self._consensus = ConsensusAlgorithm(soft_voided_tx_ids, pubsub)
 
         return self._consensus
 
@@ -435,7 +426,6 @@ def _get_or_create_p2p_manager(self) -> ConnectionsManager:
         SyncSupportLevel.add_factories(
             self._get_or_create_settings(),
             self._p2p_manager,
-            self._sync_v1_support,
             self._sync_v2_support,
             self._get_or_create_vertex_parser(),
             self._get_or_create_vertex_handler(),
@@ -620,6 +610,7 @@ def _get_or_create_vertex_handler(self) -> VertexHandler:
                 verification_service=self._get_or_create_verification_service(),
                 consensus=self._get_or_create_consensus(),
                 feature_service=self._get_or_create_feature_service(),
+                execution_manager=self._get_or_create_execution_manager(),
                 pubsub=self._get_or_create_pubsub(),
                 wallet=self._get_or_create_wallet(),
             )
@@ -772,26 +763,11 @@ def set_pubsub(self, pubsub: PubSubManager) -> 'Builder':
         self._pubsub = pubsub
         return self
 
-    def set_sync_v1_support(self, support_level: SyncSupportLevel) -> 'Builder':
-        self.check_if_can_modify()
-        self._sync_v1_support = support_level
-        return self
-
     def set_sync_v2_support(self, support_level: SyncSupportLevel) -> 'Builder':
         self.check_if_can_modify()
         self._sync_v2_support = support_level
         return self
 
-    def enable_sync_v1(self) -> 'Builder':
-        self.check_if_can_modify()
-        self._sync_v1_support = SyncSupportLevel.ENABLED
-        return self
-
-    def disable_sync_v1(self) -> 'Builder':
-        self.check_if_can_modify()
-        self._sync_v1_support = SyncSupportLevel.DISABLED
-        return self
-
     def enable_sync_v2(self) -> 'Builder':
         self.check_if_can_modify()
         self._sync_v2_support = SyncSupportLevel.ENABLED

hathor/builder/cli_builder.py

@@ -16,7 +16,6 @@
 import os
 import platform
 import sys
-from enum import Enum, auto
 from typing import Any, Optional
 
 from structlog import get_logger
@@ -53,13 +52,6 @@
 DEFAULT_CACHE_SIZE: int = 100000
 
 
-class SyncChoice(Enum):
-    V1_DEFAULT = auto()  # v1 enabled, v2 disabled but can be enabled in runtime
-    V2_DEFAULT = auto()  # v2 enabled, v1 disabled but can be enabled in runtime
-    BRIDGE_DEFAULT = auto()  # both enabled, either can be disabled in runtime
-    V2_ONLY = auto()  # v1 is unavailable, it cannot be enabled in runtime
-
-
 class CliBuilder:
     """CliBuilder builds the core objects from args.
 
@@ -198,46 +190,20 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
 
         hostname = self.get_hostname()
 
-        sync_choice: SyncChoice
         if self._args.sync_bridge:
-            self.log.warn('--sync-bridge is deprecated and will be removed')
-            sync_choice = SyncChoice.BRIDGE_DEFAULT
+            raise BuilderError('--sync-bridge was removed')
         elif self._args.sync_v1_only:
-            self.log.warn('--sync-v1-only is deprecated and will be removed')
-            sync_choice = SyncChoice.V1_DEFAULT
+            raise BuilderError('--sync-v1-only was removed')
         elif self._args.sync_v2_only:
             self.log.warn('--sync-v2-only is the default, this parameter has no effect')
-            sync_choice = SyncChoice.V2_DEFAULT
         elif self._args.x_remove_sync_v1:
-            sync_choice = SyncChoice.V2_ONLY
+            self.log.warn('--x-remove-sync-v1 is deprecated and has no effect')
         elif self._args.x_sync_bridge:
-            self.log.warn('--x-sync-bridge is deprecated and will be removed')
-            sync_choice = SyncChoice.BRIDGE_DEFAULT
+            raise BuilderError('--x-sync-bridge was removed')
         elif self._args.x_sync_v1_only:
-            self.log.warn('--x-sync-v1-only is deprecated and will be removed')
-            sync_choice = SyncChoice.V1_DEFAULT
+            raise BuilderError('--x-sync-v1-only was removed')
         elif self._args.x_sync_v2_only:
             self.log.warn('--x-sync-v2-only is deprecated and will be removed')
-            sync_choice = SyncChoice.V2_DEFAULT
-        else:
-            # XXX: this is the default behavior when no parameter is given
-            sync_choice = SyncChoice.V2_DEFAULT
-
-        sync_v1_support: SyncSupportLevel
-        sync_v2_support: SyncSupportLevel
-        match sync_choice:
-            case SyncChoice.V1_DEFAULT:
-                sync_v1_support = SyncSupportLevel.ENABLED
-                sync_v2_support = SyncSupportLevel.DISABLED
-            case SyncChoice.V2_DEFAULT:
-                sync_v1_support = SyncSupportLevel.DISABLED
-                sync_v2_support = SyncSupportLevel.ENABLED
-            case SyncChoice.BRIDGE_DEFAULT:
-                sync_v1_support = SyncSupportLevel.ENABLED
-                sync_v2_support = SyncSupportLevel.ENABLED
-            case SyncChoice.V2_ONLY:
-                sync_v1_support = SyncSupportLevel.UNAVAILABLE
-                sync_v2_support = SyncSupportLevel.ENABLED
 
         pubsub = PubSubManager(reactor)
 
@@ -270,19 +236,12 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             self.log.debug('enable utxo index')
             tx_storage.indexes.enable_utxo_index()
 
-        full_verification = False
-        if self._args.x_full_verification:
-            self.check_or_raise(
-                not self._args.x_enable_event_queue and not self._args.enable_event_queue,
-                '--x-full-verification cannot be used with --enable-event-queue'
-            )
-            full_verification = True
+        self.check_or_raise(not self._args.x_full_verification, '--x-full-verification is deprecated')
 
         soft_voided_tx_ids = set(settings.SOFT_VOIDED_TX_IDS)
         consensus_algorithm = ConsensusAlgorithm(
             soft_voided_tx_ids,
             pubsub=pubsub,
-            execution_manager=execution_manager
         )
 
         if self._args.x_enable_event_queue or self._args.enable_event_queue:
@@ -341,18 +300,12 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             consensus=consensus_algorithm,
             feature_service=self.feature_service,
             pubsub=pubsub,
+            execution_manager=execution_manager,
             wallet=self.wallet,
             log_vertex_bytes=self._args.log_vertex_bytes,
         )
 
-        SyncSupportLevel.add_factories(
-            settings,
-            p2p_manager,
-            sync_v1_support,
-            sync_v2_support,
-            vertex_parser,
-            vertex_handler,
-        )
+        SyncSupportLevel.add_factories(settings, p2p_manager, SyncSupportLevel.ENABLED, vertex_parser, vertex_handler)
 
         from hathor.consensus.poa import PoaBlockProducer, PoaSignerFile
         poa_block_producer: PoaBlockProducer | None = None
@@ -380,7 +333,7 @@ def create_manager(self, reactor: Reactor) -> HathorManager:
             wallet=self.wallet,
             checkpoints=settings.CHECKPOINTS,
             environment_info=get_environment_info(args=str(self._args), peer_id=str(peer.id)),
-            full_verification=full_verification,
+            full_verification=False,
             enable_event_queue=self._args.x_enable_event_queue or self._args.enable_event_queue,
             bit_signaling_service=bit_signaling_service,
             verification_service=verification_service,

hathor/cli/openapi_files/openapi_base.json

@@ -7,7 +7,7 @@
         ],
 	"info": {
             "title": "Hathor API",
-            "version": "0.63.0"
+            "version": "0.63.1"
 	},
 	"consumes": [
             "application/json"

hathor/cli/run_node.py

@@ -125,7 +125,7 @@ def create_parser(cls) -> ArgumentParser:
         parser.add_argument('--recursion-limit', type=int, help='Set python recursion limit')
         parser.add_argument('--allow-mining-without-peers', action='store_true', help='Allow mining without peers')
         fvargs = parser.add_mutually_exclusive_group()
-        fvargs.add_argument('--x-full-verification', action='store_true', help='Fully validate the local database')
+        fvargs.add_argument('--x-full-verification', action='store_true', help=SUPPRESS)  # deprecated
         parser.add_argument('--procname-prefix', help='Add a prefix to the process name', default='')
         parser.add_argument('--allow-non-standard-script', action='store_true', help='Accept non-standard scripts on '
                             '/push-tx API')
@@ -135,14 +135,13 @@ def create_parser(cls) -> ArgumentParser:
         parser.add_argument('--enable-debug-api', action='store_true', help='Enable _debug/* endpoints')
         parser.add_argument('--enable-crash-api', action='store_true', help='Enable _crash/* endpoints')
         sync_args = parser.add_mutually_exclusive_group()
-        sync_args.add_argument('--sync-bridge', action='store_true', help=SUPPRESS)  # moved to --x-sync-bridge
-        sync_args.add_argument('--sync-v1-only', action='store_true', help=SUPPRESS)  # moved to --x-sync-v1-only
-        sync_args.add_argument('--sync-v2-only', action='store_true', help=SUPPRESS)  # already default
-        sync_args.add_argument('--x-remove-sync-v1', action='store_true', help='Make sync-v1 unavailable, thus '
-                               'impossible to be enabled in runtime.')
-        sync_args.add_argument('--x-sync-v1-only', action='store_true', help='Disable support for running sync-v2.')
-        sync_args.add_argument('--x-sync-v2-only', action='store_true', help=SUPPRESS)  # old argument
-        sync_args.add_argument('--x-sync-bridge', action='store_true', help='Enable running both sync protocols.')
+        sync_args.add_argument('--sync-bridge', action='store_true', help=SUPPRESS)  # deprecated
+        sync_args.add_argument('--sync-v1-only', action='store_true', help=SUPPRESS)  # deprecated
+        sync_args.add_argument('--sync-v2-only', action='store_true', help=SUPPRESS)  # deprecated
+        sync_args.add_argument('--x-remove-sync-v1', action='store_true', help=SUPPRESS)  # deprecated
+        sync_args.add_argument('--x-sync-v1-only', action='store_true', help=SUPPRESS)  # deprecated
+        sync_args.add_argument('--x-sync-v2-only', action='store_true', help=SUPPRESS)  # deprecated
+        sync_args.add_argument('--x-sync-bridge', action='store_true', help=SUPPRESS)  # deprecated
         parser.add_argument('--x-localhost-only', action='store_true', help='Only connect to peers on localhost')
         parser.add_argument('--x-rocksdb-indexes', action='store_true', help=SUPPRESS)
         parser.add_argument('--x-enable-event-queue', action='store_true',

hathor/cli/util.py

@@ -269,11 +269,6 @@ def setup_logging(
                     'level': 'INFO' if logging_options.debug else 'WARN',
                     'propagate': False,
                 },
-                'hathor.p2p.sync_v1': {
-                    'handlers': handlers,
-                    'level': 'DEBUG' if debug_sync else 'INFO',
-                    'propagate': False,
-                },
                 'hathor.p2p.sync_v2': {
                     'handlers': handlers,
                     'level': 'DEBUG' if debug_sync else 'INFO',

hathor/consensus/consensus.py

@@ -23,7 +23,6 @@
 from hathor.consensus.block_consensus import BlockConsensusAlgorithmFactory
 from hathor.consensus.context import ConsensusAlgorithmContext
 from hathor.consensus.transaction_consensus import TransactionConsensusAlgorithmFactory
-from hathor.execution_manager import ExecutionManager
 from hathor.profiler import get_cpu_profiler
 from hathor.pubsub import HathorEvents, PubSubManager
 from hathor.transaction import BaseTransaction
@@ -68,38 +67,31 @@ def __init__(
         self,
         soft_voided_tx_ids: set[bytes],
         pubsub: PubSubManager,
-        *,
-        execution_manager: ExecutionManager
     ) -> None:
         self._settings = get_global_settings()
         self.log = logger.new()
         self._pubsub = pubsub
         self.soft_voided_tx_ids = frozenset(soft_voided_tx_ids)
         self.block_algorithm_factory = BlockConsensusAlgorithmFactory()
         self.transaction_algorithm_factory = TransactionConsensusAlgorithmFactory()
-        self._execution_manager = execution_manager
 
     def create_context(self) -> ConsensusAlgorithmContext:
         """Handy method to create a context that can be used to access block and transaction algorithms."""
         return ConsensusAlgorithmContext(self, self._pubsub)
 
     @cpu.profiler(key=lambda self, base: 'consensus!{}'.format(base.hash.hex()))
-    def update(self, base: BaseTransaction) -> None:
+    def unsafe_update(self, base: BaseTransaction) -> None:
+        """
+        Run a consensus update with its own context, indexes will be updated accordingly.
+
+        It is considered unsafe because the caller is responsible for crashing the full node
+        if this method throws any exception.
+        """
+        from hathor.transaction import Block, Transaction
         assert base.storage is not None
         assert base.storage.is_only_valid_allowed()
         meta = base.get_metadata()
         assert meta.validation.is_valid()
-        try:
-            self._unsafe_update(base)
-        except BaseException:
-            meta.add_voided_by(self._settings.CONSENSUS_FAIL_ID)
-            assert base.storage is not None
-            base.storage.save_transaction(base, only_metadata=True)
-            self._execution_manager.crash_and_exit(reason=f'Consensus update failed for tx {base.hash_hex}')
-
-    def _unsafe_update(self, base: BaseTransaction) -> None:
-        """Run a consensus update with its own context, indexes will be updated accordingly."""
-        from hathor.transaction import Block, Transaction
 
         # XXX: first make sure we can run the consensus update on this tx:
         meta = base.get_metadata()

hathor/dag_builder/vertex_exporter.py

@@ -137,11 +137,21 @@ def get_min_timestamp(self, node: DAGNode) -> int:
         timestamp = 1 + max(self._vertices[name].timestamp for name in deps)
         return timestamp
 
-    def update_vertex_hash(self, vertex: BaseTransaction) -> None:
+    def update_vertex_hash(self, vertex: BaseTransaction, *, fix_conflict: bool = True) -> None:
         """Resolve vertex and update its hash."""
         self._vertex_resolver(vertex)
         vertex.update_hash()
 
+        if fix_conflict:
+            max_attempts = 100
+            while vertex.hash in self._vertice_per_id:
+                max_attempts -= 1
+                if max_attempts <= 0:
+                    raise ValueError('could not resolve a conflict')
+                vertex.nonce += 1
+                self._vertex_resolver(vertex)
+                vertex.update_hash()
+
     def sign_all_inputs(self, node: DAGNode, vertex: Transaction) -> None:
         """Sign all inputs of a vertex."""
         data_to_sign = vertex.get_sighash_all()
@@ -277,6 +287,8 @@ def create_vertex(self, node: DAGNode) -> BaseTransaction:
                 raise NotImplementedError(node.type)
 
         assert vertex is not None
+        assert vertex.hash not in self._vertice_per_id
+        assert node.name not in self._vertices
         self._vertice_per_id[vertex.hash] = vertex
         self._vertices[node.name] = vertex
         return vertex

hathor/indexes/manager.py

@@ -256,7 +256,7 @@ def del_tx(self, tx: BaseTransaction, *, remove_all: bool = False, relax_assert:
             self.sorted_txs.del_tx(tx)
 
         if self.tokens:
-            self.tokens.del_tx(tx)
+            self.tokens.del_tx(tx, remove_all=remove_all)
 
 
 class MemoryIndexesManager(IndexesManager):