diff --git a/common/types/db.go b/common/types/db.go index 0c5b8d8f24..a476829872 100644 --- a/common/types/db.go +++ b/common/types/db.go @@ -113,6 +113,8 @@ const ( ProverTaskFailureTypeObjectAlreadyVerified // ProverTaskFailureTypeReassignedByAdmin reassigned by admin, this value is used in admin-system and defined here for clarity ProverTaskFailureTypeReassignedByAdmin + // ProverTaskFailureTypeMaxRetriesExceeded prover exceeded maximum number of invalid proof retries + ProverTaskFailureTypeMaxRetriesExceeded ) func (r ProverTaskFailureType) String() string { @@ -131,6 +133,8 @@ func (r ProverTaskFailureType) String() string { return "prover task failure object already verified" case ProverTaskFailureTypeReassignedByAdmin: return "prover task failure reassigned by admin" + case ProverTaskFailureTypeMaxRetriesExceeded: + return "prover task failure exceeded maximum number of invalid proof retries" default: return fmt.Sprintf("illegal prover task failure type (%d)", int32(r)) } diff --git a/common/types/db_test.go b/common/types/db_test.go index df2a159a75..2e4f4fad9b 100644 --- a/common/types/db_test.go +++ b/common/types/db_test.go @@ -312,6 +312,21 @@ func TestProverTaskFailureType(t *testing.T) { ProverTaskFailureTypeServerError, "prover task failure server exception", }, + { + "ProverTaskFailureTypeObjectAlreadyVerified", + ProverTaskFailureTypeObjectAlreadyVerified, + "prover task failure object already verified", + }, + { + "ProverTaskFailureTypeReassignedByAdmin", + ProverTaskFailureTypeReassignedByAdmin, + "prover task failure reassigned by admin", + }, + { + "ProverTaskFailureTypeMaxRetriesExceeded", + ProverTaskFailureTypeMaxRetriesExceeded, + "prover task failure exceeded maximum number of invalid proof retries", + }, { "Invalid Value", ProverTaskFailureType(999), diff --git a/coordinator/conf/config.json b/coordinator/conf/config.json index 301f89e12c..2128a99e4c 100644 --- a/coordinator/conf/config.json +++ b/coordinator/conf/config.json @@ -3,6 +3,7 @@ "provers_per_session": 1, "session_attempts": 5, "external_prover_threshold": 32, + "max_invalid_proof_retries": 3, "bundle_collection_time_sec": 180, "batch_collection_time_sec": 180, "chunk_collection_time_sec": 180, diff --git a/coordinator/internal/config/config.go b/coordinator/internal/config/config.go index 110405fb00..ae693e69f5 100644 --- a/coordinator/internal/config/config.go +++ b/coordinator/internal/config/config.go @@ -18,6 +18,9 @@ type ProverManager struct { SessionAttempts uint8 `json:"session_attempts"` // Threshold for activating the external prover based on unassigned task count. ExternalProverThreshold int64 `json:"external_prover_threshold"` + // Maximum number of invalid proof submissions allowed per prover per task. + // This helps prevent DoS attacks through repeated invalid proof submissions. + MaxInvalidProofRetries uint8 `json:"max_invalid_proof_retries"` // Zk verifier config. Verifier *VerifierConfig `json:"verifier"` // BatchCollectionTimeSec batch Proof collection time (in seconds). diff --git a/coordinator/internal/logic/submitproof/proof_receiver.go b/coordinator/internal/logic/submitproof/proof_receiver.go index 444670ffe4..b8dcf583ae 100644 --- a/coordinator/internal/logic/submitproof/proof_receiver.go +++ b/coordinator/internal/logic/submitproof/proof_receiver.go @@ -39,6 +39,8 @@ var ( ErrValidatorFailureVerifiedFailed = errors.New("verification failed, verifier returns error") // ErrValidatorSuccessInvalidProof successful verified and the proof is invalid ErrValidatorSuccessInvalidProof = errors.New("verification succeeded, it's an invalid proof") + // ErrValidatorFailureMaxRetriesExceeded prover exceeded maximum number of invalid proof retries + ErrValidatorFailureMaxRetriesExceeded = errors.New("validator failure prover exceeded maximum number of invalid proof retries") // ErrGetHardForkNameFailed failed to get hard fork name ErrGetHardForkNameFailed = errors.New("failed to get hard fork name") // ErrCoordinatorInternalFailure coordinator internal db failure @@ -69,6 +71,7 @@ type ProofReceiverLogic struct { validateFailureProverTaskStatusNotOk prometheus.Counter validateFailureProverTaskTimeout prometheus.Counter validateFailureProverTaskHaveVerifier prometheus.Counter + validateFailureMaxRetriesExceeded prometheus.Counter } // NewSubmitProofReceiverLogic create a proof receiver logic @@ -127,6 +130,10 @@ func NewSubmitProofReceiverLogic(cfg *config.ProverManager, chainCfg *params.Cha Name: "coordinator_validate_failure_submit_have_been_verifier", Help: "Total number of submit proof validate failure proof have been verifier.", }), + validateFailureMaxRetriesExceeded: promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Name: "coordinator_validate_failure_max_retries_exceeded", + Help: "Total number of submit proof validate failure max retries exceeded.", + }), } } @@ -252,7 +259,7 @@ func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.Prov // In order to prevent DoS attacks, it is forbidden to repeatedly submit valid proofs. // TODO: Defend invalid proof resubmissions by one of the following two methods: // (i) slash the prover for each submission of invalid proof - // (ii) set the maximum failure retry times + // (ii) set the maximum failure retry times - IMPLEMENTED via MaxInvalidProofRetries config log.Warn( "cannot submit valid proof for a prover task twice", "taskType", proverTask.TaskType, "hash", proofParameter.TaskID, @@ -302,6 +309,32 @@ func (m *ProofReceiverLogic) validator(ctx context.Context, proverTask *orm.Prov "taskType", proverTask.TaskType, "proverName", proverTask.ProverName, "proverPublicKey", pk) return ErrValidatorFailureTaskHaveVerifiedSuccess } + + // Check if prover has exceeded maximum invalid proof retries + if m.cfg.MaxInvalidProofRetries > 0 { + failedTasks, getFailedErr := m.proverTaskOrm.GetFailedProverTasksByHash(ctx, message.ProofType(proofParameter.TaskType), proofParameter.TaskID, int(m.cfg.MaxInvalidProofRetries)+1) + if getFailedErr != nil { + log.Error("failed to get failed prover tasks", "hash", proofParameter.TaskID, "proverPublicKey", pk, "error", getFailedErr) + return getFailedErr + } + + proverFailureCount := 0 + for _, task := range failedTasks { + if task.ProverPublicKey == pk && task.ProvingStatus == int16(types.ProverProofInvalid) { + proverFailureCount++ + } + } + + if proverFailureCount >= int(m.cfg.MaxInvalidProofRetries) { + m.proofRecover(ctx, proverTask, types.ProverTaskFailureTypeMaxRetriesExceeded, proofParameter) + m.validateFailureMaxRetriesExceeded.Inc() + log.Warn("prover has exceeded maximum invalid proof retries", + "hash", proofParameter.TaskID, "proverPublicKey", pk, "failureCount", proverFailureCount, + "maxRetries", m.cfg.MaxInvalidProofRetries, "taskType", proverTask.TaskType, "proverName", proverTask.ProverName) + return ErrValidatorFailureMaxRetriesExceeded + } + } + return nil }