diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000..4b940d4e --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,93 @@ +name: Coverage + +on: + push: + branches: ["master"] + pull_request: + branches: ["master"] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + coverage: + name: Code Coverage + runs-on: [self-hosted, linux] + timeout-minutes: 30 + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + submodules: recursive + - run: sudo apt-get update + - uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: librocksdb-dev libzstd-dev libbz2-dev liblz4-dev llvm + # - name: Cache SPM + # uses: actions/cache@v4 + # with: + # path: '**/.build' + # key: ${{ runner.os }}-spm-${{ hashFiles('**/Package.resolved') }} + # restore-keys: | + # ${{ runner.os }}-spm- + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Cache bandersnatch_vrfs static lib + uses: actions/cache@v4 + with: + path: .lib/libbandersnatch_vrfs.a + key: ${{ runner.os }}-libs-libbandersnatch-${{ hashFiles('Utils/Sources/bandersnatch/**') }} + restore-keys: | + ${{ runner.os }}-libs-libbandersnatch + - name: Cache bls static lib + uses: actions/cache@v4 + with: + path: .lib/libbls.a + key: ${{ runner.os }}-libs-libbls-${{ hashFiles('Utils/Sources/bls/**') }} + restore-keys: | + ${{ runner.os }}-libs-libbls + - name: Cache erasure-coding static lib + uses: actions/cache@v4 + with: + path: .lib/libec.a + key: ${{ runner.os }}-libs-libec-${{ hashFiles('Utils/Sources/erasure-coding/**') }} + restore-keys: | + ${{ runner.os }}-libs-libec + - name: Setup Swift + uses: SwiftyLab/setup-swift@latest + - name: Setup Rust + uses: dtolnay/rust-toolchain@nightly + - name: Build deps + run: make deps + - name: Test Coverage + run: make test-coverage + - name: Merge and generate coverage report + run: | + llvm-profdata merge -sparse $(find . -type f -path '*/.build/*/debug/codecov/*.profdata') -o default.profdata + for file in **/Tests; do + BIN_PATH="$(swift build --show-bin-path --package-path "$(dirname "$file")")" + XCTEST_PATHS=$(find "${BIN_PATH}" -name '*.xctest') + for XCTEST_PATH in $XCTEST_PATHS; do + echo "Processing $XCTEST_PATH" + # Export the code coverage for the current subproject and append to coverage.lcov + llvm-cov export "${XCTEST_PATH}" \ + -instr-profile=default.profdata \ + -format lcov >> coverage.lcov \ + -ignore-filename-regex=".build/repositories/*" + done + done + - uses: codecov/codecov-action@v4 + with: + file: coverage.lcov + fail_ci_if_error: true # optional (default = false) + token: ${{ secrets.CODECOV_TOKEN }} # required + verbose: true # optional (default = false) diff --git a/Blockchain/Sources/Blockchain/Blockchain.swift b/Blockchain/Sources/Blockchain/Blockchain.swift index 49575762..8b578e33 100644 --- a/Blockchain/Sources/Blockchain/Blockchain.swift +++ b/Blockchain/Sources/Blockchain/Blockchain.swift @@ -39,9 +39,11 @@ public final class Blockchain: ServiceBase, @unchecked Sendable { let runtime = Runtime(config: config) let parent = try await dataProvider.getState(hash: block.header.parentHash) + let stateRoot = await parent.value.stateRoot let timeslot = timeProvider.getTime().timeToTimeslot(config: config) // TODO: figure out what is the best way to deal with block received a bit too early - let state = try await runtime.apply(block: block, state: parent, context: .init(timeslot: timeslot + 1)) + let context = Runtime.ApplyContext(timeslot: timeslot + 1, stateRoot: stateRoot) + let state = try await runtime.apply(block: block, state: parent, context: context) try await dataProvider.blockImported(block: block, state: state) diff --git a/Blockchain/Sources/Blockchain/Config/ProtocolConfig+Preset.swift b/Blockchain/Sources/Blockchain/Config/ProtocolConfig+Preset.swift index 58b88109..b79d1e4c 100644 --- a/Blockchain/Sources/Blockchain/Config/ProtocolConfig+Preset.swift +++ b/Blockchain/Sources/Blockchain/Config/ProtocolConfig+Preset.swift @@ -11,11 +11,13 @@ extension Ref where T == ProtocolConfig { preimagePurgePeriod: 28800, epochLength: 6, auditBiasFactor: 2, - coreAccumulationGas: Gas(10_000_000), // TODO: check this - workPackageAuthorizerGas: Gas(10_000_000), // TODO: check this - workPackageRefineGas: Gas(10_000_000), // TODO: check this + coreAccumulationGas: Gas(100_000), + workPackageAuthorizerGas: Gas(1_000_000), + workPackageRefineGas: Gas(500_000_000), + totalAccumulationGas: Gas(341_000_000), recentHistorySize: 8, maxWorkItems: 4, + maxDepsInWorkReport: 8, maxTicketsPerExtrinsic: 4, maxLookupAnchorAge: 14400, transferMemoSize: 128, @@ -49,11 +51,13 @@ extension Ref where T == ProtocolConfig { preimagePurgePeriod: 28800, epochLength: 12, auditBiasFactor: 2, - coreAccumulationGas: Gas(10_000_000), // TODO: check this - workPackageAuthorizerGas: Gas(10_000_000), // TODO: check this - workPackageRefineGas: Gas(10_000_000), // TODO: check this + coreAccumulationGas: Gas(100_000), + workPackageAuthorizerGas: Gas(1_000_000), + workPackageRefineGas: Gas(500_000_000), + totalAccumulationGas: Gas(341_000_000), recentHistorySize: 8, maxWorkItems: 4, + maxDepsInWorkReport: 8, maxTicketsPerExtrinsic: 16, maxLookupAnchorAge: 14400, transferMemoSize: 128, @@ -86,11 +90,13 @@ extension Ref where T == ProtocolConfig { preimagePurgePeriod: 28800, epochLength: 600, auditBiasFactor: 2, - coreAccumulationGas: Gas(10_000_000), // TODO: check this - workPackageAuthorizerGas: Gas(10_000_000), // TODO: check this - workPackageRefineGas: Gas(10_000_000), // TODO: check this + coreAccumulationGas: Gas(100_000), + workPackageAuthorizerGas: Gas(1_000_000), + workPackageRefineGas: Gas(500_000_000), + totalAccumulationGas: Gas(341_000_000), recentHistorySize: 8, maxWorkItems: 4, + maxDepsInWorkReport: 8, maxTicketsPerExtrinsic: 16, maxLookupAnchorAge: 14400, transferMemoSize: 128, diff --git a/Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift b/Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift index 3d41b524..193f0cdf 100644 --- a/Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift +++ b/Blockchain/Sources/Blockchain/Config/ProtocolConfig.swift @@ -37,21 +37,24 @@ public struct ProtocolConfig: Sendable, Codable, Equatable { // GR: The total gas allocated for a work-package’s Refine logic. public var workPackageRefineGas: Gas + // GT: The total gas allocated across all cores for Accumulation. + public var totalAccumulationGas: Gas + // H = 8: The size of recent history, in blocks. public var recentHistorySize: Int // I = 4: The maximum amount of work items in a package. public var maxWorkItems: Int + // J = 8: The maximum sum of dependency items in a work-report. + public var maxDepsInWorkReport: Int + // K = 16: The maximum number of tickets which may be submitted in a single extrinsic. public var maxTicketsPerExtrinsic: Int // L = 14, 400: The maximum age in timeslots of the lookup anchor. public var maxLookupAnchorAge: Int - // WT = 128: The size of a transfer memo in octets. - public var transferMemoSize: Int - // N = 2: The number of ticket entries per validator. public var ticketEntriesPerValidator: Int @@ -67,16 +70,16 @@ public struct ProtocolConfig: Sendable, Codable, Equatable { // R = 10: The rotation period of validator-core assignments, in timeslots. public var coreAssignmentRotationPeriod: Int - // S = 4,000,000: The maximum size of service code in octets. - public var maxServiceCodeSize: Int - // U = 5: The period in timeslots after which reported but unavailable work may be replaced. public var preimageReplacementPeriod: Int // V = 1023: The total number of validators. public var totalNumberOfValidators: Int - // WC = 684: The basic size of our erasure-coded pieces. + // WC = 4,000,000: The maximum size of service code in octets. + public var maxServiceCodeSize: Int + + // WE = 684: The basic size of our erasure-coded pieces. public var erasureCodedPieceSize: Int // WM = 2^11: The maximum number of entries in a work-package manifest. @@ -92,6 +95,9 @@ public struct ProtocolConfig: Sendable, Codable, Equatable { // WS = 6: The size of an exported segment in erasure-coded pieces. public var erasureCodedSegmentSize: Int + // WT = 128: The size of a transfer memo in octets. + public var transferMemoSize: Int + // Y = 500: The number of slots into an epoch at which ticket-submission ends. public var ticketSubmissionEndSlot: Int @@ -119,8 +125,10 @@ public struct ProtocolConfig: Sendable, Codable, Equatable { coreAccumulationGas: Gas, workPackageAuthorizerGas: Gas, workPackageRefineGas: Gas, + totalAccumulationGas: Gas, recentHistorySize: Int, maxWorkItems: Int, + maxDepsInWorkReport: Int, maxTicketsPerExtrinsic: Int, maxLookupAnchorAge: Int, transferMemoSize: Int, @@ -154,8 +162,10 @@ public struct ProtocolConfig: Sendable, Codable, Equatable { self.coreAccumulationGas = coreAccumulationGas self.workPackageAuthorizerGas = workPackageAuthorizerGas self.workPackageRefineGas = workPackageRefineGas + self.totalAccumulationGas = totalAccumulationGas self.recentHistorySize = recentHistorySize self.maxWorkItems = maxWorkItems + self.maxDepsInWorkReport = maxDepsInWorkReport self.maxTicketsPerExtrinsic = maxTicketsPerExtrinsic self.maxLookupAnchorAge = maxLookupAnchorAge self.transferMemoSize = transferMemoSize @@ -215,9 +225,13 @@ extension ProtocolConfig { ? other.workPackageAuthorizerGas : workPackageAuthorizerGas, workPackageRefineGas: other.workPackageRefineGas.value != 0 ? other.workPackageRefineGas : workPackageRefineGas, + totalAccumulationGas: other.totalAccumulationGas.value != 0 + ? other.totalAccumulationGas : totalAccumulationGas, recentHistorySize: other.recentHistorySize != 0 ? other.recentHistorySize : recentHistorySize, maxWorkItems: other.maxWorkItems != 0 ? other.maxWorkItems : maxWorkItems, + maxDepsInWorkReport: other.maxDepsInWorkReport != 0 + ? other.maxDepsInWorkReport : maxDepsInWorkReport, maxTicketsPerExtrinsic: other.maxTicketsPerExtrinsic != 0 ? other.maxTicketsPerExtrinsic : maxTicketsPerExtrinsic, maxLookupAnchorAge: other.maxLookupAnchorAge != 0 @@ -295,8 +309,12 @@ extension ProtocolConfig { workPackageRefineGas = try decode( .workPackageRefineGas, defaultValue: Gas(0), required: required ) + totalAccumulationGas = try decode( + .totalAccumulationGas, defaultValue: Gas(0), required: required + ) recentHistorySize = try decode(.recentHistorySize, defaultValue: 0, required: required) maxWorkItems = try decode(.maxWorkItems, defaultValue: 0, required: required) + maxDepsInWorkReport = try decode(.maxDepsInWorkReport, defaultValue: 0, required: required) maxTicketsPerExtrinsic = try decode( .maxTicketsPerExtrinsic, defaultValue: 0, required: required ) @@ -434,6 +452,14 @@ extension ProtocolConfig { } } + public enum TotalAccumulationGas: ReadGas { + public typealias TConfig = ProtocolConfigRef + public typealias TOutput = Gas + public static func read(config: ProtocolConfigRef) -> Gas { + config.value.totalAccumulationGas + } + } + public enum RecentHistorySize: ReadInt { public typealias TConfig = ProtocolConfigRef public static func read(config: ProtocolConfigRef) -> Int { @@ -448,6 +474,13 @@ extension ProtocolConfig { } } + public enum MaxDepsInWorkReport: ReadInt { + public typealias TConfig = ProtocolConfigRef + public static func read(config: ProtocolConfigRef) -> Int { + config.value.maxDepsInWorkReport + } + } + public enum MaxTicketsPerExtrinsic: ReadInt { public typealias TConfig = ProtocolConfigRef public static func read(config: ProtocolConfigRef) -> Int { diff --git a/Blockchain/Sources/Blockchain/RuntimeProtocols/AccumulateFunction.swift b/Blockchain/Sources/Blockchain/RuntimeProtocols/AccumulateFunction.swift index 8d41594c..061d1146 100644 --- a/Blockchain/Sources/Blockchain/RuntimeProtocols/AccumulateFunction.swift +++ b/Blockchain/Sources/Blockchain/RuntimeProtocols/AccumulateFunction.swift @@ -2,9 +2,13 @@ import Foundation import Utils public struct AccumulateArguments: Codable { + /// o public var result: WorkResult + /// l public var paylaodHash: Data32 + /// k public var packageHash: Data32 + /// a public var authorizationOutput: Data public init(result: WorkResult, paylaodHash: Data32, packageHash: Data32, authorizationOutput: Data) { @@ -60,7 +64,7 @@ public struct AccumulateState { /// X public struct AccumlateResultContext { /// d - public var serviceAccounts: [ServiceIndex: ServiceAccount] + public var serviceAccounts: ServiceAccounts /// s: the accumulating service account index public var serviceIndex: ServiceIndex /// u @@ -75,7 +79,7 @@ public protocol AccumulateFunction { func invoke( config: ProtocolConfigRef, // prior accounts - accounts: ServiceAccounts, + accounts: inout some ServiceAccounts, // u state: AccumulateState, // s diff --git a/Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift b/Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift index 1fcf5bba..d5190e81 100644 --- a/Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift +++ b/Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift @@ -5,20 +5,54 @@ public enum AccumulationError: Error { case duplicatedServiceIndex } +public struct AccumulationQueueItem: Sendable, Equatable, Codable { + public var workReport: WorkReport + public var dependencies: Set + + public init(workReport: WorkReport, dependencies: Set) { + self.workReport = workReport + self.dependencies = dependencies + } +} + +// accumulation output pairing +public struct Commitment: Hashable { + public var serviceIndex: ServiceIndex + public var hash: Data32 + + public init(service: ServiceIndex, hash: Data32) { + serviceIndex = service + self.hash = hash + } +} + +/// outer accumulation function ∆+ output public struct AccumulationOutput { - public var commitments: [(ServiceIndex, Data32)] - public var privilegedServices: PrivilegedServices - public var validatorQueue: ConfigFixedSizeArray< - ValidatorKey, ProtocolConfig.TotalNumberOfValidators - > - public var authorizationQueue: ConfigFixedSizeArray< - ConfigFixedSizeArray< - Data32, - ProtocolConfig.MaxAuthorizationsQueueItems - >, - ProtocolConfig.TotalNumberOfCores - > - public var serviceAccounts: [ServiceIndex: ServiceAccount] + // number of work results accumulated + public var numAccumulated: Int + public var state: AccumulateState + public var transfers: [DeferredTransfers] + public var commitments: Set +} + +/// parallelized accumulation function ∆* output +public struct ParallelAccumulationOutput { + public var gasUsed: Gas + public var state: AccumulateState + public var transfers: [DeferredTransfers] + public var commitments: Set +} + +/// single-service accumulation function ∆1 output +public struct SingleAccumulationOutput { + // o + public var state: AccumulateState + // t + public var transfers: [DeferredTransfers] + // b + public var commitment: Data32? + // u + public var gasUsed: Gas } public protocol Accumulation: ServiceAccounts { @@ -36,44 +70,31 @@ public protocol Accumulation: ServiceAccounts { var entropyPool: EntropyPool { get } var accumlateFunction: AccumulateFunction { get } var onTransferFunction: OnTransferFunction { get } + var accumulationQueue: StateKeys.AccumulationQueueKey.Value { get } + var accumulationHistory: StateKeys.AccumulationHistoryKey.Value { get } } extension Accumulation { - public func update(config: ProtocolConfigRef, block: BlockRef, workReports: [WorkReport]) async throws -> AccumulationOutput { - var servicesGasRatio: [ServiceIndex: Gas] = [:] - var servicesGas: [ServiceIndex: Gas] = [:] - - // privileged gas - for (service, gas) in privilegedServices.basicGas { - servicesGas[service] = gas - } - - let totalGasRatio = workReports.flatMap(\.results).reduce(Gas(0)) { $0 + $1.gasRatio } - var totalMinimalGas = Gas(0) - for report in workReports { - for result in report.results { - servicesGasRatio[result.serviceIndex, default: Gas(0)] += result.gasRatio - let acc = try await get(serviceAccount: result.serviceIndex).unwrap(orError: AccumulationError.invalidServiceIndex) - totalMinimalGas += acc.minAccumlateGas - servicesGas[result.serviceIndex, default: Gas(0)] += acc.minAccumlateGas - } - } - let remainingGas = config.value.coreAccumulationGas - totalMinimalGas - - for (service, gas) in servicesGas { - servicesGas[service] = gas + servicesGasRatio[service, default: Gas(0)] * remainingGas / totalGasRatio - } - - var serviceArguments: [ServiceIndex: [AccumulateArguments]] = [:] + /// single-service accumulate function ∆1 + private mutating func singleAccumulate( + config: ProtocolConfigRef, + state: AccumulateState, + workReports: [WorkReport], + service: ServiceIndex, + block: BlockRef, + privilegedGas: [ServiceIndex: Gas] + ) async throws -> SingleAccumulationOutput { + var gas = Gas(0) + var arguments: [AccumulateArguments] = [] - // ensure privileged services will be called - for service in privilegedServices.basicGas.keys { - serviceArguments[service] = [] + for basicGas in privilegedGas.values { + gas += basicGas } for report in workReports { - for result in report.results { - serviceArguments[result.serviceIndex, default: []].append(AccumulateArguments( + for result in report.results where result.serviceIndex == service { + gas += result.gasRatio + arguments.append(AccumulateArguments( result: result, paylaodHash: result.payloadHash, packageHash: report.packageSpecification.workPackageHash, @@ -82,7 +103,39 @@ extension Accumulation { } } - var commitments = [(ServiceIndex, Data32)]() + let (newState, transfers, commitment, gasUsed) = try await accumlateFunction.invoke( + config: config, + accounts: &self, + state: state, + serviceIndex: service, + gas: gas, + arguments: arguments, + initialIndex: Blake2b256.hash(service.encode(), entropyPool.t0.data, block.header.timeslot.encode()) + .data.decode(UInt32.self), + timeslot: block.header.timeslot + ) + + return SingleAccumulationOutput( + state: newState, + transfers: transfers, + commitment: commitment, + gasUsed: gasUsed + ) + } + + /// parallelized accumulate function ∆* + private mutating func parallelizedAccumulate( + config: ProtocolConfigRef, + block: BlockRef, + state: AccumulateState, + workReports: [WorkReport], + privilegedGas: [ServiceIndex: Gas] + ) async throws -> ParallelAccumulationOutput { + var services = Set() + var gasUsed = Gas(0) + var transfers: [DeferredTransfers] = [] + var commitments = Set() + var newServiceAccounts = [ServiceIndex: ServiceAccount]() var newPrivilegedServices: PrivilegedServices? var newValidatorQueue: ConfigFixedSizeArray< ValidatorKey, ProtocolConfig.TotalNumberOfValidators @@ -95,37 +148,38 @@ extension Accumulation { ProtocolConfig.TotalNumberOfCores >? - var newServiceAccounts = [ServiceIndex: ServiceAccount]() + for report in workReports { + for result in report.results { + services.insert(result.serviceIndex) + } + } - var transferReceivers = [ServiceIndex: [DeferredTransfers]]() + for service in privilegedGas.keys { + services.insert(service) + } - for (service, arguments) in serviceArguments { - guard let gas = servicesGas[service] else { - assertionFailure("unreachable: service not found") - throw AccumulationError.invalidServiceIndex - } - let (newState, transfers, commitment, _) = try await accumlateFunction.invoke( + for service in services { + let singleOutput = try await singleAccumulate( config: config, - accounts: self, - state: AccumulateState( - serviceAccounts: newServiceAccounts, - validatorQueue: validatorQueue, - authorizationQueue: authorizationQueue, - privilegedServices: privilegedServices - ), - serviceIndex: service, - gas: gas, - arguments: arguments, - initialIndex: Blake2b256.hash(service.encode(), entropyPool.t0.data, block.header.timeslot.encode()) - .data.decode(UInt32.self), - timeslot: block.header.timeslot + state: state, + workReports: workReports, + service: service, + block: block, + privilegedGas: privilegedGas ) - if let commitment { - commitments.append((service, commitment)) + gasUsed += singleOutput.gasUsed + + if let commitment = singleOutput.commitment { + commitments.insert(Commitment(service: service, hash: commitment)) + } + + for transfer in singleOutput.transfers { + transfers.append(transfer) } - for (service, account) in newState.serviceAccounts { - guard newServiceAccounts[service] == nil else { + // new service accounts + for (service, account) in singleOutput.state.serviceAccounts { + guard newServiceAccounts[service] == nil, try await get(serviceAccount: service) == nil else { throw AccumulationError.duplicatedServiceIndex } newServiceAccounts[service] = account @@ -133,41 +187,182 @@ extension Accumulation { switch service { case privilegedServices.empower: - newPrivilegedServices = newState.privilegedServices + newPrivilegedServices = singleOutput.state.privilegedServices case privilegedServices.assign: - newAuthorizationQueue = newState.authorizationQueue + newAuthorizationQueue = singleOutput.state.authorizationQueue case privilegedServices.designate: - newValidatorQueue = newState.validatorQueue + newValidatorQueue = singleOutput.state.validatorQueue default: break } + } + + return ParallelAccumulationOutput( + gasUsed: gasUsed, + state: AccumulateState( + serviceAccounts: newServiceAccounts, + validatorQueue: newValidatorQueue ?? validatorQueue, + authorizationQueue: newAuthorizationQueue ?? authorizationQueue, + privilegedServices: newPrivilegedServices ?? privilegedServices + ), + transfers: transfers, + commitments: commitments + ) + } - for transfer in transfers { - transferReceivers[transfer.sender, default: []].append(transfer) + /// outer accumulate function ∆+ + private mutating func outerAccumulate( + config: ProtocolConfigRef, + block: BlockRef, + state: AccumulateState, + workReports: [WorkReport], + privilegedGas: [ServiceIndex: Gas], + gasLimit: Gas + ) async throws -> AccumulationOutput { + var i = 0 + var sumGasRequired = Gas(0) + for report in workReports { + for result in report.results { + if result.gasRatio + sumGasRequired > gasLimit { + break + } + sumGasRequired += result.gasRatio + i += 1 } } - for (service, transfers) in transferReceivers { - let acc = try await get(serviceAccount: service).unwrap(orError: AccumulationError.invalidServiceIndex) - let code = try await get(serviceAccount: service, preimageHash: acc.codeHash) - guard let code else { - continue - } - newServiceAccounts[service] = try onTransferFunction.invoke( + if i == 0 { + return AccumulationOutput( + numAccumulated: 0, + state: state, + transfers: [], + commitments: Set() + ) + } else { + let parallelOutput = try await parallelizedAccumulate( + config: config, + block: block, + state: state, + workReports: Array(workReports[0 ..< i]), + privilegedGas: privilegedGas + ) + let outerOutput = try await outerAccumulate( + config: config, + block: block, + state: parallelOutput.state, + workReports: Array(workReports[i ..< workReports.count]), + privilegedGas: [:], + gasLimit: gasLimit - parallelOutput.gasUsed + ) + return AccumulationOutput( + numAccumulated: i + outerOutput.numAccumulated, + state: outerOutput.state, + transfers: parallelOutput.transfers + outerOutput.transfers, + commitments: parallelOutput.commitments.union(outerOutput.commitments) + ) + } + } + + // E: edit the accumulation queue, remove the dependencies of the items that are already accumulated + public func editAccumulatedItems(items: inout [AccumulationQueueItem], accumulatedPackages: Set) { + for index in items.indices + where !accumulatedPackages.contains(items[index].workReport.packageSpecification.workPackageHash) + { + items[index].dependencies.subtract(accumulatedPackages) + } + } + + // Q: find the reports that have no dependencies + private func findNoDepsReports(items: inout [AccumulationQueueItem]) -> [WorkReport] { + let noDepsReports = items.filter(\.dependencies.isEmpty).map(\.workReport) + if noDepsReports.isEmpty { + return [] + } else { + editAccumulatedItems(items: &items, accumulatedPackages: Set(noDepsReports.map(\.packageSpecification.workPackageHash))) + return noDepsReports + findNoDepsReports(items: &items) + } + } + + // newly available work-reports, W, are partitioned into two sequences based on the condition of having zero prerequisite work-reports + public func partitionWorkReports( + availableReports: [WorkReport], + history: StateKeys.AccumulationHistoryKey.Value + ) -> (zeroPrereqReports: [WorkReport], newQueueItems: [AccumulationQueueItem]) { + let zeroPrereqReports = availableReports.filter { report in + report.refinementContext.prerequisiteWorkPackages.isEmpty && report.lookup.isEmpty + } + + let queuedReports = availableReports.filter { !zeroPrereqReports.contains($0) } + + var newQueueItems: [AccumulationQueueItem] = [] + for report in queuedReports { + newQueueItems.append(.init( + workReport: report, + dependencies: report.refinementContext.prerequisiteWorkPackages.union(report.lookup.keys) + )) + } + + editAccumulatedItems( + items: &newQueueItems, + accumulatedPackages: Set(history.array.reduce(into: Set()) { $0.formUnion($1) }) + ) + + return (zeroPrereqReports, newQueueItems) + } + + public func getAccumulatableReports( + index: Int, availableReports: [WorkReport], + history: StateKeys.AccumulationHistoryKey.Value + ) -> (accumulatableReports: [WorkReport], newQueueItems: [AccumulationQueueItem]) { + let (zeroPrereqReports, newQueueItems) = partitionWorkReports(availableReports: availableReports, history: history) + + let rightQueueItems = accumulationQueue.array[index...] + let leftQueueItems = accumulationQueue.array[0 ..< index] + var allQueueItems = rightQueueItems.flatMap { $0 } + leftQueueItems.flatMap { $0 } + newQueueItems + + editAccumulatedItems(items: &allQueueItems, accumulatedPackages: Set(zeroPrereqReports.map(\.packageSpecification.workPackageHash))) + + return (zeroPrereqReports + findNoDepsReports(items: &allQueueItems), newQueueItems) + } + + public mutating func update( + config: ProtocolConfigRef, + block: BlockRef, + workReports: [WorkReport] + ) async throws -> (numAccumulated: Int, state: AccumulateState, commitments: Set) { + let sumPrevilegedGas = privilegedServices.basicGas.values.reduce(Gas(0)) { $0 + $1.value } + let minTotalGas = config.value.coreAccumulationGas * Gas(config.value.totalNumberOfCores) + sumPrevilegedGas + let gasLimit = max(config.value.totalAccumulationGas, minTotalGas) + + let res = try await outerAccumulate( + config: config, + block: block, + state: AccumulateState( + serviceAccounts: [:], + validatorQueue: validatorQueue, + authorizationQueue: authorizationQueue, + privilegedServices: privilegedServices + ), + workReports: workReports, + privilegedGas: privilegedServices.basicGas, + gasLimit: gasLimit + ) + + var transferGroups = [ServiceIndex: [DeferredTransfers]]() + + for transfer in res.transfers { + transferGroups[transfer.destination, default: []].append(transfer) + } + + for (service, transfers) in transferGroups { + try await onTransferFunction.invoke( config: config, service: service, - code: code, - serviceAccounts: newServiceAccounts, + serviceAccounts: &self, transfers: transfers ) } - return .init( - commitments: commitments, - privilegedServices: newPrivilegedServices ?? privilegedServices, - validatorQueue: newValidatorQueue ?? validatorQueue, - authorizationQueue: newAuthorizationQueue ?? authorizationQueue, - serviceAccounts: newServiceAccounts - ) + return (res.numAccumulated, res.state, res.commitments) } } diff --git a/Blockchain/Sources/Blockchain/RuntimeProtocols/Guaranteeing.swift b/Blockchain/Sources/Blockchain/RuntimeProtocols/Guaranteeing.swift index d85bbf56..867db13f 100644 --- a/Blockchain/Sources/Blockchain/RuntimeProtocols/Guaranteeing.swift +++ b/Blockchain/Sources/Blockchain/RuntimeProtocols/Guaranteeing.swift @@ -10,9 +10,11 @@ public enum GuaranteeingError: Error { case outOfGas case invalidContext case duplicatedWorkPackage - case prerequistieNotFound + case prerequisiteNotFound case invalidResultCodeHash + case invalidServiceGas case invalidPublicKey + case invalidSegmentLookup } public protocol Guaranteeing { @@ -38,6 +40,14 @@ public protocol Guaranteeing { > { get } var recentHistory: RecentHistory { get } var offenders: Set { get } + var accumulationQueue: ConfigFixedSizeArray< + [AccumulationQueueItem], + ProtocolConfig.EpochLength + > { get } + var accumulationHistory: ConfigFixedSizeArray< + Set, + ProtocolConfig.EpochLength + > { get } func serviceAccount(index: ServiceIndex) -> ServiceAccountDetails? } @@ -102,9 +112,13 @@ extension Guaranteeing { var totalMinGasRequirement = Gas(0) + var oldLookups = [Data32: Data32]() + for guarantee in extrinsic.guarantees { let report = guarantee.workReport + oldLookups[report.packageSpecification.workPackageHash] = report.packageSpecification.segmentRoot + for credential in guarantee.credential { let isCurrent = (guarantee.timeslot / coreAssignmentRotationPeriod) == (timeslot / coreAssignmentRotationPeriod) let keys = isCurrent ? currentCoreKeys : pareviousCoreKeys @@ -146,6 +160,10 @@ extension Guaranteeing { throw .invalidResultCodeHash } + guard result.gasRatio >= acc.minAccumlateGas else { + throw .invalidServiceGas + } + totalMinGasRequirement += acc.minAccumlateGas } } @@ -154,14 +172,24 @@ extension Guaranteeing { throw .outOfGas } - let allRecentWorkReportHashes = Set(recentHistory.items.flatMap(\.workReportHashes.array)) - guard allRecentWorkReportHashes.isDisjoint(with: workReportHashes) else { + let recentWorkReportHashes: Set = Set(recentHistory.items.flatMap(\.lookup.keys)) + let accumulateHistoryReports = Set(accumulationHistory.array.flatMap { $0 }) + let accumulateQueueReports = Set(accumulationQueue.array.flatMap { $0 } + .flatMap(\.workReport.refinementContext.prerequisiteWorkPackages)) + let pendingWorkReportHashes = Set(reports.array.flatMap { $0?.workReport.refinementContext.prerequisiteWorkPackages ?? [] }) + let pipelinedWorkReportHashes = recentWorkReportHashes.union(accumulateHistoryReports).union(accumulateQueueReports) + .union(pendingWorkReportHashes) + guard pipelinedWorkReportHashes.isDisjoint(with: workReportHashes) else { throw .duplicatedWorkPackage } - let contexts = Set(extrinsic.guarantees.map(\.workReport.refinementContext)) + for item in recentHistory.items { + oldLookups.merge(item.lookup, uniquingKeysWith: { _, new in new }) + } - for context in contexts { + for guarantee in extrinsic.guarantees { + let report = guarantee.workReport + let context = report.refinementContext let history = recentHistory.items.first { $0.headerHash == context.anchor.headerHash } guard let history else { throw .invalidContext @@ -172,15 +200,21 @@ extension Guaranteeing { guard context.anchor.beefyRoot == history.mmr.hash() else { throw .invalidContext } - guard context.lokupAnchor.timeslot >= timeslot - UInt32(config.value.maxLookupAnchorAge) else { + guard context.lookupAnchor.timeslot >= timeslot - UInt32(config.value.maxLookupAnchorAge) else { throw .invalidContext } - if let prerequistieWorkPackage = context.prerequistieWorkPackage { - guard allRecentWorkReportHashes.contains(prerequistieWorkPackage) || - workReportHashes.contains(prerequistieWorkPackage) + for prerequisiteWorkPackage in context.prerequisiteWorkPackages.union(report.lookup.keys) { + guard recentWorkReportHashes.contains(prerequisiteWorkPackage) || + workReportHashes.contains(prerequisiteWorkPackage) else { - throw .prerequistieNotFound + throw .prerequisiteNotFound + } + } + + for (hash, root) in report.lookup { + guard oldLookups[hash] == root else { + throw .invalidSegmentLookup } } } diff --git a/Blockchain/Sources/Blockchain/RuntimeProtocols/OnTransferFunction.swift b/Blockchain/Sources/Blockchain/RuntimeProtocols/OnTransferFunction.swift index e002c409..675b982c 100644 --- a/Blockchain/Sources/Blockchain/RuntimeProtocols/OnTransferFunction.swift +++ b/Blockchain/Sources/Blockchain/RuntimeProtocols/OnTransferFunction.swift @@ -4,8 +4,7 @@ public protocol OnTransferFunction { func invoke( config: ProtocolConfigRef, service: ServiceIndex, - code: Data, - serviceAccounts: [ServiceIndex: ServiceAccount], + serviceAccounts: inout some ServiceAccounts, transfers: [DeferredTransfers] - ) throws -> ServiceAccount + ) async throws } diff --git a/Blockchain/Sources/Blockchain/RuntimeProtocols/Runtime.swift b/Blockchain/Sources/Blockchain/RuntimeProtocols/Runtime.swift index 5f506e4f..eb50be6f 100644 --- a/Blockchain/Sources/Blockchain/RuntimeProtocols/Runtime.swift +++ b/Blockchain/Sources/Blockchain/RuntimeProtocols/Runtime.swift @@ -35,9 +35,11 @@ public final class Runtime { public struct ApplyContext { public let timeslot: TimeslotIndex + public let stateRoot: Data32 - public init(timeslot: TimeslotIndex) { + public init(timeslot: TimeslotIndex, stateRoot: Data32) { self.timeslot = timeslot + self.stateRoot = stateRoot } } @@ -54,7 +56,7 @@ public final class Runtime { throw Error.invalidParentHash } - guard block.header.priorStateRoot == state.stateRoot else { + guard block.header.priorStateRoot == context.stateRoot else { throw Error.invalidHeaderStateRoot } @@ -170,24 +172,15 @@ public final class Runtime { // depends on Safrole and Disputes let availableReports = try updateReports(block: block, state: &newState) - let res = try await newState.update(config: config, block: block, workReports: availableReports) - newState.privilegedServices = res.privilegedServices - for (service, account) in res.serviceAccounts { - newState[serviceAccount: service] = account.toDetails() - for (hash, value) in account.storage { - newState[serviceAccount: service, storageKey: hash] = value - } - for (hash, value) in account.preimages { - newState[serviceAccount: service, preimageHash: hash] = value - } - for (hashLength, value) in account.preimageInfos { - newState[serviceAccount: service, preimageHash: hashLength.hash, length: hashLength.length] = value - } - } - - newState.authorizationQueue = res.authorizationQueue - newState.validatorQueue = res.validatorQueue + // accumulation + try await accumulate( + config: config, + block: block, + availableReports: availableReports, + state: &newState, + prevTimeslot: prevState.value.timeslot + ) newState.coreAuthorizationPool = try updateAuthorizationPool( block: block, state: prevState @@ -212,13 +205,75 @@ public final class Runtime { return StateRef(newState) } + // accumulation related state updates + public func accumulate( + config: ProtocolConfigRef, + block: BlockRef, + availableReports: [WorkReport], + state: inout State, + prevTimeslot: TimeslotIndex + ) async throws { + let curIndex = Int(block.header.timeslot) % config.value.epochLength + var (accumulatableReports, newQueueItems) = state.getAccumulatableReports( + index: curIndex, + availableReports: availableReports, + history: state.accumulationHistory + ) + + // accumulate and transfers + let (numAccumulated, accumulateState, _) = try await state.update(config: config, block: block, workReports: accumulatableReports) + + state.authorizationQueue = accumulateState.authorizationQueue + state.validatorQueue = accumulateState.validatorQueue + state.privilegedServices = accumulateState.privilegedServices + for (service, account) in accumulateState.serviceAccounts { + state[serviceAccount: service] = account.toDetails() + for (hash, value) in account.storage { + state[serviceAccount: service, storageKey: hash] = value + } + for (hash, value) in account.preimages { + state[serviceAccount: service, preimageHash: hash] = value + } + for (hashLength, value) in account.preimageInfos { + state[serviceAccount: service, preimageHash: hashLength.hash, length: hashLength.length] = value + } + } + + // update accumulation history + let accumulated = accumulatableReports[0 ..< numAccumulated] + let newHistoryItem = Set(accumulated.map(\.packageSpecification.workPackageHash)) + for i in 0 ..< config.value.epochLength { + if i == config.value.epochLength - 1 { + state.accumulationHistory[i] = newHistoryItem + } else { + state.accumulationHistory[i] = state.accumulationHistory[i + 1] + } + } + + // update accumulation queue + for i in 0 ..< config.value.epochLength { + let queueIdx = (curIndex - i) %% config.value.epochLength + if i == 0 { + state.editAccumulatedItems(items: &newQueueItems, accumulatedPackages: newHistoryItem) + state.accumulationQueue[queueIdx] = newQueueItems + } else if i >= 1, i < state.timeslot - prevTimeslot { + state.accumulationQueue[queueIdx] = [] + } else { + state.editAccumulatedItems(items: &state.accumulationQueue[queueIdx], accumulatedPackages: newHistoryItem) + } + } + } + public func updateRecentHistory(block: BlockRef, state newState: inout State) throws { - let workReportHashes = block.extrinsic.reports.guarantees.map(\.workReport.packageSpecification.workPackageHash) - try newState.recentHistory.update( + let lookup: [Data32: Data32] = Dictionary(uniqueKeysWithValues: block.extrinsic.reports.guarantees.map { ( + $0.workReport.packageSpecification.workPackageHash, + $0.workReport.packageSpecification.segmentRoot + ) }) + newState.recentHistory.update( headerHash: block.hash, parentStateRoot: block.header.priorStateRoot, accumulateRoot: Data32(), // TODO: calculate accumulation result - workReportHashes: ConfigLimitedSizeArray(config: config, array: workReportHashes) + lookup: lookup ) } diff --git a/Blockchain/Sources/Blockchain/State/InMemoryBackend.swift b/Blockchain/Sources/Blockchain/State/InMemoryBackend.swift index 7716d086..af5dadce 100644 --- a/Blockchain/Sources/Blockchain/State/InMemoryBackend.swift +++ b/Blockchain/Sources/Blockchain/State/InMemoryBackend.swift @@ -1,42 +1,107 @@ import Codec import Foundation +import TracingUtils import Utils -public actor InMemoryBackend: StateBackend { - private let config: ProtocolConfigRef - private var store: [Data32: Data] +private let logger = Logger(label: "InMemoryBackend") - public init(config: ProtocolConfigRef, store: [Data32: Data] = [:]) { - self.config = config - self.store = store +public actor InMemoryBackend: StateBackendProtocol { + public struct KVPair: Comparable, Sendable { + var key: Data + var value: Data + + public static func < (lhs: KVPair, rhs: KVPair) -> Bool { + lhs.key.lexicographicallyPrecedes(rhs.key) + } } - public func readImpl(_ key: any StateKey) async throws -> (Codable & Sendable)? { - guard let value = store[key.encode()] else { - return nil + // we really should be using Heap or some other Tree based structure here + // but let's keep it simple for now + public private(set) var store: SortedArray = .init([]) + private var rawValues: [Data32: Data] = [:] + public private(set) var refCounts: [Data: Int] = [:] + private var rawValueRefCounts: [Data32: Int] = [:] + + public init() {} + + public func read(key: Data) async throws -> Data? { + let idx = store.insertIndex(KVPair(key: key, value: Data())) + let item = store.array[safe: idx] + if item?.key == key { + return item?.value } - return try JamDecoder.decode(key.decodeType(), from: value, withConfig: config) + return nil } - public func batchRead(_ keys: [any StateKey]) async throws -> [(key: any StateKey, value: Codable & Sendable)] { - try keys.map { - let data = try store[$0.encode()].unwrap() - return try ($0, JamDecoder.decode($0.decodeType(), from: data, withConfig: config)) + public func readAll(prefix: Data, startKey: Data?, limit: UInt32?) async throws -> [(key: Data, value: Data)] { + var resp = [(key: Data, value: Data)]() + let startKey = startKey ?? prefix + let startIndex = store.insertIndex(KVPair(key: startKey, value: Data())) + for i in startIndex ..< store.array.count { + let item = store.array[i] + if item.key.starts(with: prefix) { + resp.append((item.key, item.value)) + } else { + break + } + if let limit, resp.count == limit { + break + } } + return resp } - public func batchWrite(_ changes: [(key: any StateKey, value: Codable & Sendable)]) async throws { - for (key, value) in changes { - store[key.encode()] = try JamEncoder.encode(value) + public func batchUpdate(_ updates: [StateBackendOperation]) async throws { + for update in updates { + switch update { + case let .write(key, value): + let idx = store.insertIndex(KVPair(key: key, value: value)) + let item = store.array[safe: idx] + if let item, item.key == key { // found + // value is not used for ordering so this is safe + store.unsafeArrayAccess[idx].value = value + } else { // not found + store.insert(KVPair(key: key, value: value)) + } + case let .writeRawValue(key, value): + rawValues[key] = value + rawValueRefCounts[key, default: 0] += 1 + case let .refIncrement(key): + refCounts[key, default: 0] += 1 + case let .refDecrement(key): + refCounts[key, default: 0] -= 1 + } } } - public func readAll() async throws -> [Data32: Data] { - store + public func readValue(hash: Data32) async throws -> Data? { + rawValues[hash] } - public func stateRoot() async throws -> Data32 { - // TODO: store intermediate state so we can calculate the root efficiently - try stateMerklize(kv: store) + public func gc(callback: @Sendable (Data) -> Data32?) async throws { + // check ref counts and remove keys with 0 ref count + for (key, count) in refCounts where count == 0 { + let idx = store.insertIndex(KVPair(key: key, value: Data())) + let item = store.array[safe: idx] + if let item, item.key == key { + store.remove(at: idx) + if let rawValueKey = callback(item.value) { + rawValueRefCounts[rawValueKey, default: 0] -= 1 + if rawValueRefCounts[rawValueKey] == 0 { + rawValues.removeValue(forKey: rawValueKey) + rawValueRefCounts.removeValue(forKey: rawValueKey) + } + } + } + } + } + + public func debugPrint() { + for item in store.array { + let refCount = refCounts[item.key, default: 0] + logger.info("key: \(item.key.toHexString())") + logger.info("value: \(item.value.toHexString())") + logger.info("ref count: \(refCount)") + } } } diff --git a/Blockchain/Sources/Blockchain/State/ServiceAccounts.swift b/Blockchain/Sources/Blockchain/State/ServiceAccounts.swift index 879f89bd..624e4323 100644 --- a/Blockchain/Sources/Blockchain/State/ServiceAccounts.swift +++ b/Blockchain/Sources/Blockchain/State/ServiceAccounts.swift @@ -7,15 +7,15 @@ public protocol ServiceAccounts { func get(serviceAccount index: ServiceIndex, preimageHash hash: Data32) async throws -> Data? func get( serviceAccount index: ServiceIndex, preimageHash hash: Data32, length: UInt32 - ) async throws -> StateKeys.ServiceAccountPreimageInfoKey.Value.ValueType + ) async throws -> StateKeys.ServiceAccountPreimageInfoKey.Value? - mutating func set(serviceAccount index: ServiceIndex, account: ServiceAccountDetails) - mutating func set(serviceAccount index: ServiceIndex, storageKey key: Data32, value: Data) - mutating func set(serviceAccount index: ServiceIndex, preimageHash hash: Data32, value: Data) + mutating func set(serviceAccount index: ServiceIndex, account: ServiceAccountDetails?) + mutating func set(serviceAccount index: ServiceIndex, storageKey key: Data32, value: Data?) + mutating func set(serviceAccount index: ServiceIndex, preimageHash hash: Data32, value: Data?) mutating func set( serviceAccount index: ServiceIndex, preimageHash hash: Data32, length: UInt32, - value: StateKeys.ServiceAccountPreimageInfoKey.Value.ValueType + value: StateKeys.ServiceAccountPreimageInfoKey.Value? ) } diff --git a/Blockchain/Sources/Blockchain/State/State+Genesis.swift b/Blockchain/Sources/Blockchain/State/State+Genesis.swift index 2b60149b..7e142721 100644 --- a/Blockchain/Sources/Blockchain/State/State+Genesis.swift +++ b/Blockchain/Sources/Blockchain/State/State+Genesis.swift @@ -37,7 +37,7 @@ extension State { headerHash: block.hash, mmr: MMR([]), stateRoot: Data32(), - workReportHashes: ConfigLimitedSizeArray(config: config) + lookup: [Data32: Data32]() )) return (StateRef(state), block) diff --git a/Blockchain/Sources/Blockchain/State/State.swift b/Blockchain/Sources/Blockchain/State/State.swift index dab62028..6295ae5e 100644 --- a/Blockchain/Sources/Blockchain/State/State.swift +++ b/Blockchain/Sources/Blockchain/State/State.swift @@ -17,7 +17,7 @@ public struct State: Sendable { } // α: The core αuthorizations pool. - public var coreAuthorizationPool: StateKeys.CoreAuthorizationPoolKey.Value.ValueType { + public var coreAuthorizationPool: StateKeys.CoreAuthorizationPoolKey.Value { get { layer.coreAuthorizationPool } @@ -27,7 +27,7 @@ public struct State: Sendable { } // φ: The authorization queue. - public var authorizationQueue: StateKeys.AuthorizationQueueKey.Value.ValueType { + public var authorizationQueue: StateKeys.AuthorizationQueueKey.Value { get { layer.authorizationQueue } @@ -37,7 +37,7 @@ public struct State: Sendable { } // β: Information on the most recent βlocks. - public var recentHistory: StateKeys.RecentHistoryKey.Value.ValueType { + public var recentHistory: StateKeys.RecentHistoryKey.Value { get { layer.recentHistory } @@ -47,7 +47,7 @@ public struct State: Sendable { } // γ: State concerning Safrole. - public var safroleState: StateKeys.SafroleStateKey.Value.ValueType { + public var safroleState: StateKeys.SafroleStateKey.Value { get { layer.safroleState } @@ -57,7 +57,7 @@ public struct State: Sendable { } // ψ: past judgements - public var judgements: StateKeys.JudgementsKey.Value.ValueType { + public var judgements: StateKeys.JudgementsKey.Value { get { layer.judgements } @@ -67,7 +67,7 @@ public struct State: Sendable { } // η: The eηtropy accumulator and epochal raηdomness. - public var entropyPool: StateKeys.EntropyPoolKey.Value.ValueType { + public var entropyPool: StateKeys.EntropyPoolKey.Value { get { layer.entropyPool } @@ -77,7 +77,7 @@ public struct State: Sendable { } // ι: The validator keys and metadata to be drawn from next. - public var validatorQueue: StateKeys.ValidatorQueueKey.Value.ValueType { + public var validatorQueue: StateKeys.ValidatorQueueKey.Value { get { layer.validatorQueue } @@ -87,7 +87,7 @@ public struct State: Sendable { } // κ: The validator κeys and metadata currently active. - public var currentValidators: StateKeys.CurrentValidatorsKey.Value.ValueType { + public var currentValidators: StateKeys.CurrentValidatorsKey.Value { get { layer.currentValidators } @@ -97,7 +97,7 @@ public struct State: Sendable { } // λ: The validator keys and metadata which were active in the prior epoch. - public var previousValidators: StateKeys.PreviousValidatorsKey.Value.ValueType { + public var previousValidators: StateKeys.PreviousValidatorsKey.Value { get { layer.previousValidators } @@ -107,7 +107,7 @@ public struct State: Sendable { } // ρ: The ρending reports, per core, which are being made available prior to accumulation. - public var reports: StateKeys.ReportsKey.Value.ValueType { + public var reports: StateKeys.ReportsKey.Value { get { layer.reports } @@ -117,7 +117,7 @@ public struct State: Sendable { } // τ: The most recent block’s τimeslot. - public var timeslot: StateKeys.TimeslotKey.Value.ValueType { + public var timeslot: StateKeys.TimeslotKey.Value { get { layer.timeslot } @@ -127,7 +127,7 @@ public struct State: Sendable { } // χ: The privileged service indices. - public var privilegedServices: StateKeys.PrivilegedServicesKey.Value.ValueType { + public var privilegedServices: StateKeys.PrivilegedServicesKey.Value { get { layer.privilegedServices } @@ -137,7 +137,7 @@ public struct State: Sendable { } // π: The activity statistics for the validators. - public var activityStatistics: StateKeys.ActivityStatisticsKey.Value.ValueType { + public var activityStatistics: StateKeys.ActivityStatisticsKey.Value { get { layer.activityStatistics } @@ -146,8 +146,28 @@ public struct State: Sendable { } } + // ϑ: The accumulation queue. + public var accumulationQueue: StateKeys.AccumulationQueueKey.Value { + get { + layer.accumulationQueue + } + set { + layer.accumulationQueue = newValue + } + } + + // ξ: The accumulation history. + public var accumulationHistory: StateKeys.AccumulationHistoryKey.Value { + get { + layer.accumulationHistory + } + set { + layer.accumulationHistory = newValue + } + } + // δ: The (prior) state of the service accounts. - public subscript(serviceAccount index: ServiceIndex) -> StateKeys.ServiceAccountKey.Value.ValueType? { + public subscript(serviceAccount index: ServiceIndex) -> StateKeys.ServiceAccountKey.Value? { get { layer[serviceAccount: index] } @@ -157,7 +177,7 @@ public struct State: Sendable { } // s - public subscript(serviceAccount index: ServiceIndex, storageKey key: Data32) -> StateKeys.ServiceAccountStorageKey.Value.ValueType? { + public subscript(serviceAccount index: ServiceIndex, storageKey key: Data32) -> StateKeys.ServiceAccountStorageKey.Value? { get { layer[serviceAccount: index, storageKey: key] } @@ -169,7 +189,7 @@ public struct State: Sendable { // p public subscript( serviceAccount index: ServiceIndex, preimageHash hash: Data32 - ) -> StateKeys.ServiceAccountPreimagesKey.Value.ValueType? { + ) -> StateKeys.ServiceAccountPreimagesKey.Value? { get { layer[serviceAccount: index, preimageHash: hash] } @@ -181,7 +201,7 @@ public struct State: Sendable { // l public subscript( serviceAccount index: ServiceIndex, preimageHash hash: Data32, length length: UInt32 - ) -> StateKeys.ServiceAccountPreimageInfoKey.Value.ValueType? { + ) -> StateKeys.ServiceAccountPreimageInfoKey.Value? { get { layer[serviceAccount: index, preimageHash: hash, length: length] } @@ -197,9 +217,18 @@ public struct State: Sendable { } } - public func stateRoot() -> Data32 { - // TODO: incorporate layer changes and calculate state root - Data32() + // TODO: we don't really want to write to the underlying backend here + // instead, it should be writting to a in memory layer + // and when actually saving the state, save the in memory layer to the presistent store + public func save() async throws -> Data32 { + try await backend.write(layer.toKV()) + return await backend.rootHash + } + + public var stateRoot: Data32 { + get async { + await backend.rootHash + } } } @@ -208,57 +237,6 @@ extension State { recentHistory.items.last.map(\.headerHash)! } - private class KVSequence: Sequence { - typealias Element = (key: Data32, value: Data) - - let seq: any Sequence<(key: Data32, value: Data)> - let layer: [Data32: Data] - - init(state: State) async throws { - seq = try await state.backend.readAll() - var layer = [Data32: Data]() - for (key, value) in state.layer.toKV() { - layer[key.encode()] = try JamEncoder.encode(value) - } - self.layer = layer - } - - func makeIterator() -> KVSequence.Iterator { - KVSequence.Iterator(iter: seq.makeIterator(), layer: layer) - } - - struct Iterator: IteratorProtocol { - typealias Element = (key: Data32, value: Data) - - var iter: any IteratorProtocol - var layerIterator: (any IteratorProtocol)? - let layer: [Data32: Data] - - init(iter: any IteratorProtocol, layer: [Data32: Data]) { - self.iter = iter - self.layer = layer - } - - mutating func next() -> KVSequence.Iterator.Element? { - if layerIterator != nil { - return layerIterator?.next() - } - if let (key, value) = iter.next() { - if layer[key] != nil { - return next() // skip this one - } - return (key, value) - } - layerIterator = layer.makeIterator() - return layerIterator?.next() - } - } - } - - public func toKV() async throws -> some Sequence<(key: Data32, value: Data)> { - try await KVSequence(state: self) - } - public func asRef() -> StateRef { StateRef(self) } @@ -272,37 +250,45 @@ extension State: Dummy { } public static func dummy(config: Config, block: BlockRef?) -> State { - let coreAuthorizationPool: StateKeys.CoreAuthorizationPoolKey.Value.ValueType = + let coreAuthorizationPool: StateKeys.CoreAuthorizationPoolKey.Value = try! ConfigFixedSizeArray(config: config, defaultValue: ConfigLimitedSizeArray(config: config)) - var recentHistory: StateKeys.RecentHistoryKey.Value.ValueType = RecentHistory.dummy(config: config) + var recentHistory: StateKeys.RecentHistoryKey.Value = RecentHistory.dummy(config: config) if let block { recentHistory.items.safeAppend(RecentHistory.HistoryItem( headerHash: block.hash, mmr: MMR([]), stateRoot: Data32(), - workReportHashes: try! ConfigLimitedSizeArray(config: config) + lookup: [Data32: Data32]() )) } - let safroleState: StateKeys.SafroleStateKey.Value.ValueType = SafroleState.dummy(config: config) - let entropyPool: StateKeys.EntropyPoolKey.Value.ValueType = EntropyPool((Data32(), Data32(), Data32(), Data32())) - let validatorQueue: StateKeys.ValidatorQueueKey.Value.ValueType = + let safroleState: StateKeys.SafroleStateKey.Value = SafroleState.dummy(config: config) + let entropyPool: StateKeys.EntropyPoolKey.Value = EntropyPool((Data32(), Data32(), Data32(), Data32())) + let validatorQueue: StateKeys.ValidatorQueueKey.Value = try! ConfigFixedSizeArray(config: config, defaultValue: ValidatorKey.dummy(config: config)) - let currentValidators: StateKeys.CurrentValidatorsKey.Value.ValueType = + let currentValidators: StateKeys.CurrentValidatorsKey.Value = try! ConfigFixedSizeArray(config: config, defaultValue: ValidatorKey.dummy(config: config)) - let previousValidators: StateKeys.PreviousValidatorsKey.Value.ValueType = + let previousValidators: StateKeys.PreviousValidatorsKey.Value = try! ConfigFixedSizeArray(config: config, defaultValue: ValidatorKey.dummy(config: config)) - let reports: StateKeys.ReportsKey.Value.ValueType = try! ConfigFixedSizeArray(config: config, defaultValue: nil) - let timeslot: StateKeys.TimeslotKey.Value.ValueType = block?.header.timeslot ?? 0 - let authorizationQueue: StateKeys.AuthorizationQueueKey.Value.ValueType = + let reports: StateKeys.ReportsKey.Value = try! ConfigFixedSizeArray(config: config, defaultValue: nil) + let timeslot: StateKeys.TimeslotKey.Value = block?.header.timeslot ?? 0 + let authorizationQueue: StateKeys.AuthorizationQueueKey.Value = try! ConfigFixedSizeArray(config: config, defaultValue: ConfigFixedSizeArray(config: config, defaultValue: Data32())) - let privilegedServices: StateKeys.PrivilegedServicesKey.Value.ValueType = PrivilegedServices( + let privilegedServices: StateKeys.PrivilegedServicesKey.Value = PrivilegedServices( empower: ServiceIndex(), assign: ServiceIndex(), designate: ServiceIndex(), basicGas: [:] ) - let judgements: StateKeys.JudgementsKey.Value.ValueType = JudgementsState.dummy(config: config) - let activityStatistics: StateKeys.ActivityStatisticsKey.Value.ValueType = ValidatorActivityStatistics.dummy(config: config) + let judgements: StateKeys.JudgementsKey.Value = JudgementsState.dummy(config: config) + let activityStatistics: StateKeys.ActivityStatisticsKey.Value = ValidatorActivityStatistics.dummy(config: config) + let accumulationQueue: StateKeys.AccumulationQueueKey.Value = try! ConfigFixedSizeArray( + config: config, + defaultValue: [AccumulationQueueItem]() + ) + let accumulationHistory: StateKeys.AccumulationHistoryKey.Value = try! ConfigFixedSizeArray( + config: config, + defaultValue: Set() + ) let kv: [(any StateKey, Codable & Sendable)] = [ (StateKeys.CoreAuthorizationPoolKey(), coreAuthorizationPool), @@ -318,17 +304,17 @@ extension State: Dummy { (StateKeys.TimeslotKey(), timeslot), (StateKeys.PrivilegedServicesKey(), privilegedServices), (StateKeys.ActivityStatisticsKey(), activityStatistics), + (StateKeys.AccumulationQueueKey(), accumulationQueue), + (StateKeys.AccumulationHistoryKey(), accumulationHistory), ] var store: [Data32: Data] = [:] for (key, value) in kv { store[key.encode()] = try! JamEncoder.encode(value) } + let rootHash = try! stateMerklize(kv: store) - let backend = InMemoryBackend( - config: config, - store: store - ) + let backend = StateBackend(InMemoryBackend(), config: config, rootHash: rootHash) let layer = StateLayer(changes: kv) @@ -360,22 +346,22 @@ extension State: ServiceAccounts { public func get( serviceAccount index: ServiceIndex, preimageHash hash: Data32, length: UInt32 - ) async throws -> StateKeys.ServiceAccountPreimageInfoKey.Value.ValueType { + ) async throws -> StateKeys.ServiceAccountPreimageInfoKey.Value? { if let res = layer[serviceAccount: index, preimageHash: hash, length: length] { return res } return try await backend.read(StateKeys.ServiceAccountPreimageInfoKey(index: index, hash: hash, length: length)) } - public mutating func set(serviceAccount index: ServiceIndex, account: ServiceAccountDetails) { + public mutating func set(serviceAccount index: ServiceIndex, account: ServiceAccountDetails?) { layer[serviceAccount: index] = account } - public mutating func set(serviceAccount index: ServiceIndex, storageKey key: Data32, value: Data) { + public mutating func set(serviceAccount index: ServiceIndex, storageKey key: Data32, value: Data?) { layer[serviceAccount: index, storageKey: key] = value } - public mutating func set(serviceAccount index: ServiceIndex, preimageHash hash: Data32, value: Data) { + public mutating func set(serviceAccount index: ServiceIndex, preimageHash hash: Data32, value: Data?) { layer[serviceAccount: index, preimageHash: hash] = value } @@ -383,7 +369,7 @@ extension State: ServiceAccounts { serviceAccount index: ServiceIndex, preimageHash hash: Data32, length: UInt32, - value: StateKeys.ServiceAccountPreimageInfoKey.Value.ValueType + value: StateKeys.ServiceAccountPreimageInfoKey.Value? ) { layer[serviceAccount: index, preimageHash: hash, length: length] = value } @@ -446,24 +432,23 @@ extension State: Guaranteeing { struct DummyFunction: AccumulateFunction, OnTransferFunction { func invoke( config _: ProtocolConfigRef, - accounts _: ServiceAccounts, + accounts _: inout some ServiceAccounts, state _: AccumulateState, serviceIndex _: ServiceIndex, gas _: Gas, arguments _: [AccumulateArguments], initialIndex _: ServiceIndex, timeslot _: TimeslotIndex - ) throws -> (state: AccumulateState, transfers: [DeferredTransfers], result: Data32?, gas: Gas) { + ) async throws -> (state: AccumulateState, transfers: [DeferredTransfers], result: Data32?, gas: Gas) { fatalError("not implemented") } func invoke( config _: ProtocolConfigRef, service _: ServiceIndex, - code _: Data, - serviceAccounts _: [ServiceIndex: ServiceAccount], + serviceAccounts _: inout some ServiceAccounts, transfers _: [DeferredTransfers] - ) throws -> ServiceAccount { + ) async throws { fatalError("not implemented") } } @@ -482,8 +467,4 @@ public class StateRef: Ref, @unchecked Sendable { public static func dummy(config: ProtocolConfigRef, block: BlockRef?) -> StateRef { StateRef(State.dummy(config: config, block: block)) } - - public var stateRoot: Data32 { - value.stateRoot() - } } diff --git a/Blockchain/Sources/Blockchain/State/StateBackend.swift b/Blockchain/Sources/Blockchain/State/StateBackend.swift index 8c75f329..36ecf36f 100644 --- a/Blockchain/Sources/Blockchain/State/StateBackend.swift +++ b/Blockchain/Sources/Blockchain/State/StateBackend.swift @@ -1,28 +1,73 @@ +import Codec import Foundation import Utils public enum StateBackendError: Error { case missingState + case invalidData } -public protocol StateBackend: Sendable { - func readImpl(_ key: any StateKey) async throws -> (Codable & Sendable)? +public final class StateBackend: Sendable { + private let impl: StateBackendProtocol + private let config: ProtocolConfigRef + private let trie: StateTrie - func batchRead(_ keys: [any StateKey]) async throws -> [(key: any StateKey, value: Codable & Sendable)] - mutating func batchWrite(_ changes: [(key: any StateKey, value: Codable & Sendable)]) async throws - - func readAll() async throws -> [Data32: Data] + public init(_ impl: StateBackendProtocol, config: ProtocolConfigRef, rootHash: Data32) { + self.impl = impl + self.config = config + trie = StateTrie(rootHash: rootHash, backend: impl) + } - func stateRoot() async throws -> Data32 + public var rootHash: Data32 { + get async { + await trie.rootHash + } + } - // TODO: aux store for full key and intermidate merkle root -} + public func read(_ key: Key) async throws -> Key.Value? { + let encodedKey = key.encode() + if let ret = try await trie.read(key: encodedKey) { + guard let ret = try JamDecoder.decode(key.decodeType(), from: ret, withConfig: config) as? Key.Value else { + throw StateBackendError.invalidData + } + return ret + } + if Key.optional { + return nil + } + throw StateBackendError.missingState + } -extension StateBackend { - public func read(_ key: Key) async throws -> Key.Value.ValueType { - guard let ret = try await readImpl(key) as? Key.Value.ValueType else { - throw StateBackendError.missingState + public func batchRead(_ keys: [any StateKey]) async throws -> [(key: any StateKey, value: (Codable & Sendable)?)] { + var ret = [(key: any StateKey, value: (Codable & Sendable)?)]() + ret.reserveCapacity(keys.count) + for key in keys { + try await ret.append((key, read(key))) } return ret } + + public func write(_ values: any Sequence<(key: any StateKey, value: (Codable & Sendable)?)>) async throws { + try await trie.update(values.map { try (key: $0.key.encode(), value: $0.value.map { try JamEncoder.encode($0) }) }) + try await trie.save() + } + + public func writeRaw(_ values: [(key: Data32, value: Data?)]) async throws { + try await trie.update(values) + try await trie.save() + } + + public func gc() async throws { + try await impl.gc { data in + guard data.count == 64 else { + // unexpected data size + return nil + } + let isRegularLeaf = data[0] & 0b1100_0000 == 0b1100_0000 + if isRegularLeaf { + return Data32(data.suffix(from: 32))! + } + return nil + } + } } diff --git a/Blockchain/Sources/Blockchain/State/StateBackendProtocol.swift b/Blockchain/Sources/Blockchain/State/StateBackendProtocol.swift new file mode 100644 index 00000000..5c377dca --- /dev/null +++ b/Blockchain/Sources/Blockchain/State/StateBackendProtocol.swift @@ -0,0 +1,31 @@ +import Foundation +import Utils + +public enum StateBackendOperation: Sendable { + case write(key: Data, value: Data) + case writeRawValue(key: Data32, value: Data) + case refIncrement(key: Data) + case refDecrement(key: Data) +} + +/// key: trie node hash (31 bytes) +/// value: trie node data (64 bytes) +/// ref counting requirements: +/// - write do not increment ref count, only explicit ref increment do +/// - lazy prune is used. e.g. when ref count is reduced to zero, the value will only be removed +/// when gc is performed +/// - raw value have its own ref counting +/// - writeRawValue increment ref count, and write if necessary +/// - raw value ref count is only decremented when connected trie node is removed during gc +public protocol StateBackendProtocol: Sendable { + func read(key: Data) async throws -> Data? + func readAll(prefix: Data, startKey: Data?, limit: UInt32?) async throws -> [(key: Data, value: Data)] + func batchUpdate(_ ops: [StateBackendOperation]) async throws + + // hash is the blake2b256 hash of the value + func readValue(hash: Data32) async throws -> Data? + + /// remove entries with zero ref count + /// callback returns a dependent raw value key if the data is regular leaf node + func gc(callback: @Sendable (Data) -> Data32?) async throws +} diff --git a/Blockchain/Sources/Blockchain/State/StateKeys.swift b/Blockchain/Sources/Blockchain/State/StateKeys.swift index 422c474a..df80ebbe 100644 --- a/Blockchain/Sources/Blockchain/State/StateKeys.swift +++ b/Blockchain/Sources/Blockchain/State/StateKeys.swift @@ -2,34 +2,19 @@ import Foundation import Utils public protocol StateKey: Hashable, Sendable { - associatedtype Value: StateValueProtocol + associatedtype Value: Codable & Sendable func encode() -> Data32 + static var optional: Bool { get } } extension StateKey { public func decodeType() -> (Sendable & Codable).Type { - Value.DecodeType.self + Value.self } -} - -public protocol StateValueProtocol { - associatedtype ValueType: Codable & Sendable - associatedtype DecodeType: Codable & Sendable - static var optional: Bool { get } -} -public struct StateValue: StateValueProtocol { - public typealias ValueType = T - public typealias DecodeType = T public static var optional: Bool { false } } -public struct StateOptionalValue: StateValueProtocol { - public typealias ValueType = T? - public typealias DecodeType = T - public static var optional: Bool { true } -} - private func constructKey(_ idx: UInt8) -> Data32 { var data = Data(repeating: 0, count: 32) data[0] = idx @@ -68,16 +53,32 @@ private func constructKey(_ service: ServiceIndex, _ val: UInt32, _: Data) -> Da } public enum StateKeys { + public static let prefetchKeys: [any StateKey] = [ + CoreAuthorizationPoolKey(), + AuthorizationQueueKey(), + RecentHistoryKey(), + SafroleStateKey(), + JudgementsKey(), + EntropyPoolKey(), + ValidatorQueueKey(), + CurrentValidatorsKey(), + PreviousValidatorsKey(), + ReportsKey(), + TimeslotKey(), + PrivilegedServicesKey(), + ActivityStatisticsKey(), + AccumulationQueueKey(), + AccumulationHistoryKey(), + ] + public struct CoreAuthorizationPoolKey: StateKey { - public typealias Value = StateValue< - ConfigFixedSizeArray< - ConfigLimitedSizeArray< - Data32, - ProtocolConfig.Int0, - ProtocolConfig.MaxAuthorizationsPoolItems - >, - ProtocolConfig.TotalNumberOfCores - > + public typealias Value = ConfigFixedSizeArray< + ConfigLimitedSizeArray< + Data32, + ProtocolConfig.Int0, + ProtocolConfig.MaxAuthorizationsPoolItems + >, + ProtocolConfig.TotalNumberOfCores > public init() {} @@ -88,14 +89,12 @@ public enum StateKeys { } public struct AuthorizationQueueKey: StateKey { - public typealias Value = StateValue< + public typealias Value = ConfigFixedSizeArray< ConfigFixedSizeArray< - ConfigFixedSizeArray< - Data32, - ProtocolConfig.MaxAuthorizationsQueueItems - >, - ProtocolConfig.TotalNumberOfCores - > + Data32, + ProtocolConfig.MaxAuthorizationsQueueItems + >, + ProtocolConfig.TotalNumberOfCores > public init() {} @@ -106,7 +105,7 @@ public enum StateKeys { } public struct RecentHistoryKey: StateKey { - public typealias Value = StateValue + public typealias Value = RecentHistory public init() {} @@ -116,7 +115,7 @@ public enum StateKeys { } public struct SafroleStateKey: StateKey { - public typealias Value = StateValue + public typealias Value = SafroleState public init() {} @@ -126,7 +125,7 @@ public enum StateKeys { } public struct JudgementsKey: StateKey { - public typealias Value = StateValue + public typealias Value = JudgementsState public init() {} @@ -136,7 +135,7 @@ public enum StateKeys { } public struct EntropyPoolKey: StateKey { - public typealias Value = StateValue + public typealias Value = EntropyPool public init() {} @@ -146,11 +145,9 @@ public enum StateKeys { } public struct ValidatorQueueKey: StateKey { - public typealias Value = StateValue< - ConfigFixedSizeArray< - ValidatorKey, - ProtocolConfig.TotalNumberOfValidators - > + public typealias Value = ConfigFixedSizeArray< + ValidatorKey, + ProtocolConfig.TotalNumberOfValidators > public init() {} @@ -161,11 +158,9 @@ public enum StateKeys { } public struct CurrentValidatorsKey: StateKey { - public typealias Value = StateValue< - ConfigFixedSizeArray< - ValidatorKey, - ProtocolConfig.TotalNumberOfValidators - > + public typealias Value = ConfigFixedSizeArray< + ValidatorKey, + ProtocolConfig.TotalNumberOfValidators > public init() {} @@ -176,11 +171,9 @@ public enum StateKeys { } public struct PreviousValidatorsKey: StateKey { - public typealias Value = StateValue< - ConfigFixedSizeArray< - ValidatorKey, - ProtocolConfig.TotalNumberOfValidators - > + public typealias Value = ConfigFixedSizeArray< + ValidatorKey, + ProtocolConfig.TotalNumberOfValidators > public init() {} @@ -191,11 +184,9 @@ public enum StateKeys { } public struct ReportsKey: StateKey { - public typealias Value = StateValue< - ConfigFixedSizeArray< - ReportItem?, - ProtocolConfig.TotalNumberOfCores - > + public typealias Value = ConfigFixedSizeArray< + ReportItem?, + ProtocolConfig.TotalNumberOfCores > public init() {} @@ -206,7 +197,7 @@ public enum StateKeys { } public struct TimeslotKey: StateKey { - public typealias Value = StateValue + public typealias Value = TimeslotIndex public init() {} @@ -216,7 +207,7 @@ public enum StateKeys { } public struct PrivilegedServicesKey: StateKey { - public typealias Value = StateValue + public typealias Value = PrivilegedServices public init() {} @@ -226,7 +217,7 @@ public enum StateKeys { } public struct ActivityStatisticsKey: StateKey { - public typealias Value = StateValue + public typealias Value = ValidatorActivityStatistics public init() {} @@ -235,8 +226,35 @@ public enum StateKeys { } } + public struct AccumulationQueueKey: StateKey { + public typealias Value = ConfigFixedSizeArray< + [AccumulationQueueItem], + ProtocolConfig.EpochLength + > + + public init() {} + + public func encode() -> Data32 { + constructKey(14) + } + } + + public struct AccumulationHistoryKey: StateKey { + public typealias Value = ConfigFixedSizeArray< + Set, + ProtocolConfig.EpochLength + > + + public init() {} + + public func encode() -> Data32 { + constructKey(15) + } + } + public struct ServiceAccountKey: StateKey { - public typealias Value = StateOptionalValue + public typealias Value = ServiceAccountDetails + public static var optional: Bool { true } public var index: ServiceIndex @@ -250,7 +268,8 @@ public enum StateKeys { } public struct ServiceAccountStorageKey: StateKey { - public typealias Value = StateOptionalValue + public typealias Value = Data + public static var optional: Bool { true } public var index: ServiceIndex public var key: Data32 @@ -266,7 +285,8 @@ public enum StateKeys { } public struct ServiceAccountPreimagesKey: StateKey { - public typealias Value = StateOptionalValue + public typealias Value = Data + public static var optional: Bool { true } public var index: ServiceIndex public var hash: Data32 @@ -282,7 +302,8 @@ public enum StateKeys { } public struct ServiceAccountPreimageInfoKey: StateKey { - public typealias Value = StateOptionalValue> + public typealias Value = LimitedSizeArray + public static var optional: Bool { true } public var index: ServiceIndex public var hash: Data32 diff --git a/Blockchain/Sources/Blockchain/State/StateLayer.swift b/Blockchain/Sources/Blockchain/State/StateLayer.swift index e866eb1b..8bfb413e 100644 --- a/Blockchain/Sources/Blockchain/State/StateLayer.swift +++ b/Blockchain/Sources/Blockchain/State/StateLayer.swift @@ -1,238 +1,262 @@ import Foundation import Utils +private enum StateLayerValue: Sendable { + case value(Codable & Sendable) + case deleted + + init(_ value: (Codable & Sendable)?) { + if let value { + self = .value(value) + } else { + self = .deleted + } + } + + func value() -> T? { + if case let .value(value) = self { + return value as? T + } + return nil + } +} + // @unchecked because AnyHashable is not Sendable public struct StateLayer: @unchecked Sendable { - private var changes: [AnyHashable: Codable & Sendable] = [:] + private var changes: [AnyHashable: StateLayerValue] = [:] public init(backend: StateBackend) async throws { - let keys: [any StateKey] = [ - StateKeys.CoreAuthorizationPoolKey(), - StateKeys.AuthorizationQueueKey(), - StateKeys.RecentHistoryKey(), - StateKeys.SafroleStateKey(), - StateKeys.JudgementsKey(), - StateKeys.EntropyPoolKey(), - StateKeys.ValidatorQueueKey(), - StateKeys.CurrentValidatorsKey(), - StateKeys.PreviousValidatorsKey(), - StateKeys.ReportsKey(), - StateKeys.TimeslotKey(), - StateKeys.PrivilegedServicesKey(), - StateKeys.ActivityStatisticsKey(), - ] - - let results = try await backend.batchRead(keys) + let results = try await backend.batchRead(StateKeys.prefetchKeys) for (key, value) in results { - changes[AnyHashable(key)] = value + changes[AnyHashable(key)] = try .init(value.unwrap()) } } public init(changes: [(key: any StateKey, value: Codable & Sendable)]) { for (key, value) in changes { - self.changes[AnyHashable(key)] = value + self.changes[AnyHashable(key)] = .value(value) } } // α: The core αuthorizations pool. - public var coreAuthorizationPool: StateKeys.CoreAuthorizationPoolKey.Value.ValueType { + public var coreAuthorizationPool: StateKeys.CoreAuthorizationPoolKey.Value { get { - changes[StateKeys.CoreAuthorizationPoolKey()] as! StateKeys.CoreAuthorizationPoolKey.Value.ValueType + changes[StateKeys.CoreAuthorizationPoolKey()]!.value()! } set { - changes[StateKeys.CoreAuthorizationPoolKey()] = newValue + changes[StateKeys.CoreAuthorizationPoolKey()] = .init(newValue) } } // φ: The authorization queue. - public var authorizationQueue: StateKeys.AuthorizationQueueKey.Value.ValueType { + public var authorizationQueue: StateKeys.AuthorizationQueueKey.Value { get { - changes[StateKeys.AuthorizationQueueKey()] as! StateKeys.AuthorizationQueueKey.Value.ValueType + changes[StateKeys.AuthorizationQueueKey()]!.value()! } set { - changes[StateKeys.AuthorizationQueueKey()] = newValue + changes[StateKeys.AuthorizationQueueKey()] = .init(newValue) } } // β: Information on the most recent βlocks. - public var recentHistory: StateKeys.RecentHistoryKey.Value.ValueType { + public var recentHistory: StateKeys.RecentHistoryKey.Value { get { - changes[StateKeys.RecentHistoryKey()] as! StateKeys.RecentHistoryKey.Value.ValueType + changes[StateKeys.RecentHistoryKey()]!.value()! } set { - changes[StateKeys.RecentHistoryKey()] = newValue + changes[StateKeys.RecentHistoryKey()] = .init(newValue) } } // γ: State concerning Safrole. - public var safroleState: StateKeys.SafroleStateKey.Value.ValueType { + public var safroleState: StateKeys.SafroleStateKey.Value { get { - changes[StateKeys.SafroleStateKey()] as! StateKeys.SafroleStateKey.Value.ValueType + changes[StateKeys.SafroleStateKey()]!.value()! } set { - changes[StateKeys.SafroleStateKey()] = newValue + changes[StateKeys.SafroleStateKey()] = .init(newValue) } } // ψ: past judgements - public var judgements: StateKeys.JudgementsKey.Value.ValueType { + public var judgements: StateKeys.JudgementsKey.Value { get { - changes[StateKeys.JudgementsKey()] as! StateKeys.JudgementsKey.Value.ValueType + changes[StateKeys.JudgementsKey()]!.value()! } set { - changes[StateKeys.JudgementsKey()] = newValue + changes[StateKeys.JudgementsKey()] = .init(newValue) } } // η: The eηtropy accumulator and epochal raηdomness. - public var entropyPool: StateKeys.EntropyPoolKey.Value.ValueType { + public var entropyPool: StateKeys.EntropyPoolKey.Value { get { - changes[StateKeys.EntropyPoolKey()] as! StateKeys.EntropyPoolKey.Value.ValueType + changes[StateKeys.EntropyPoolKey()]!.value()! } set { - changes[StateKeys.EntropyPoolKey()] = newValue + changes[StateKeys.EntropyPoolKey()] = .init(newValue) } } // ι: The validator keys and metadata to be drawn from next. - public var validatorQueue: StateKeys.ValidatorQueueKey.Value.ValueType { + public var validatorQueue: StateKeys.ValidatorQueueKey.Value { get { - changes[StateKeys.ValidatorQueueKey()] as! StateKeys.ValidatorQueueKey.Value.ValueType + changes[StateKeys.ValidatorQueueKey()]!.value()! } set { - changes[StateKeys.ValidatorQueueKey()] = newValue + changes[StateKeys.ValidatorQueueKey()] = .init(newValue) } } // κ: The validator κeys and metadata currently active. - public var currentValidators: StateKeys.CurrentValidatorsKey.Value.ValueType { + public var currentValidators: StateKeys.CurrentValidatorsKey.Value { get { - changes[StateKeys.CurrentValidatorsKey()] as! StateKeys.CurrentValidatorsKey.Value.ValueType + changes[StateKeys.CurrentValidatorsKey()]!.value()! } set { - changes[StateKeys.CurrentValidatorsKey()] = newValue + changes[StateKeys.CurrentValidatorsKey()] = .init(newValue) } } // λ: The validator keys and metadata which were active in the prior epoch. - public var previousValidators: StateKeys.PreviousValidatorsKey.Value.ValueType { + public var previousValidators: StateKeys.PreviousValidatorsKey.Value { get { - changes[StateKeys.PreviousValidatorsKey()] as! StateKeys.PreviousValidatorsKey.Value.ValueType + changes[StateKeys.PreviousValidatorsKey()]!.value()! } set { - changes[StateKeys.PreviousValidatorsKey()] = newValue + changes[StateKeys.PreviousValidatorsKey()] = .init(newValue) } } // ρ: The ρending reports, per core, which are being made available prior to accumulation. - public var reports: StateKeys.ReportsKey.Value.ValueType { + public var reports: StateKeys.ReportsKey.Value { get { - changes[StateKeys.ReportsKey()] as! StateKeys.ReportsKey.Value.ValueType + changes[StateKeys.ReportsKey()]!.value()! } set { - changes[StateKeys.ReportsKey()] = newValue + changes[StateKeys.ReportsKey()] = .init(newValue) } } // τ: The most recent block’s τimeslot. - public var timeslot: StateKeys.TimeslotKey.Value.ValueType { + public var timeslot: StateKeys.TimeslotKey.Value { get { - changes[StateKeys.TimeslotKey()] as! StateKeys.TimeslotKey.Value.ValueType + changes[StateKeys.TimeslotKey()]!.value()! } set { - changes[StateKeys.TimeslotKey()] = newValue + changes[StateKeys.TimeslotKey()] = .init(newValue) } } // χ: The privileged service indices. - public var privilegedServices: StateKeys.PrivilegedServicesKey.Value.ValueType { + public var privilegedServices: StateKeys.PrivilegedServicesKey.Value { get { - changes[StateKeys.PrivilegedServicesKey()] as! StateKeys.PrivilegedServicesKey.Value.ValueType + changes[StateKeys.PrivilegedServicesKey()]!.value()! } set { - changes[StateKeys.PrivilegedServicesKey()] = newValue + changes[StateKeys.PrivilegedServicesKey()] = .init(newValue) } } // π: The activity statistics for the validators. - public var activityStatistics: StateKeys.ActivityStatisticsKey.Value.ValueType { + public var activityStatistics: StateKeys.ActivityStatisticsKey.Value { + get { + changes[StateKeys.ActivityStatisticsKey()]!.value()! + } + set { + changes[StateKeys.ActivityStatisticsKey()] = .init(newValue) + } + } + + // ϑ: The accumulation queue. + public var accumulationQueue: StateKeys.AccumulationQueueKey.Value { + get { + changes[StateKeys.AccumulationQueueKey()]!.value()! + } + set { + changes[StateKeys.AccumulationQueueKey()] = .init(newValue) + } + } + + // ξ: The accumulation history. + public var accumulationHistory: StateKeys.AccumulationHistoryKey.Value { get { - changes[StateKeys.ActivityStatisticsKey()] as! StateKeys.ActivityStatisticsKey.Value.ValueType + changes[StateKeys.AccumulationHistoryKey()]!.value()! } set { - changes[StateKeys.ActivityStatisticsKey()] = newValue + changes[StateKeys.AccumulationHistoryKey()] = .init(newValue) } } // δ: The (prior) state of the service accounts. - public subscript(serviceAccount index: ServiceIndex) -> StateKeys.ServiceAccountKey.Value.ValueType? { + public subscript(serviceAccount index: ServiceIndex) -> StateKeys.ServiceAccountKey.Value? { get { - changes[StateKeys.ServiceAccountKey(index: index)] as? StateKeys.ServiceAccountKey.Value.ValueType + changes[StateKeys.ServiceAccountKey(index: index)]?.value() } set { - changes[StateKeys.ServiceAccountKey(index: index)] = newValue + changes[StateKeys.ServiceAccountKey(index: index)] = .init(newValue) } } // s - public subscript(serviceAccount index: ServiceIndex, storageKey key: Data32) -> StateKeys.ServiceAccountStorageKey.Value.ValueType? { + public subscript(serviceAccount index: ServiceIndex, storageKey key: Data32) -> StateKeys.ServiceAccountStorageKey.Value? { get { - changes[StateKeys.ServiceAccountStorageKey(index: index, key: key)] as? StateKeys.ServiceAccountStorageKey.Value.ValueType + changes[StateKeys.ServiceAccountStorageKey(index: index, key: key)]?.value() } set { - changes[StateKeys.ServiceAccountStorageKey(index: index, key: key)] = newValue + changes[StateKeys.ServiceAccountStorageKey(index: index, key: key)] = .init(newValue) } } // p public subscript( serviceAccount index: ServiceIndex, preimageHash hash: Data32 - ) -> StateKeys.ServiceAccountPreimagesKey.Value.ValueType? { + ) -> StateKeys.ServiceAccountPreimagesKey.Value? { get { - changes[StateKeys.ServiceAccountPreimagesKey(index: index, hash: hash)] as? StateKeys.ServiceAccountPreimagesKey.Value.ValueType + changes[StateKeys.ServiceAccountPreimagesKey(index: index, hash: hash)]?.value() } set { - changes[StateKeys.ServiceAccountPreimagesKey(index: index, hash: hash)] = newValue + changes[StateKeys.ServiceAccountPreimagesKey(index: index, hash: hash)] = .init(newValue) } } // l public subscript( serviceAccount index: ServiceIndex, preimageHash hash: Data32, length length: UInt32 - ) -> StateKeys.ServiceAccountPreimageInfoKey.Value.ValueType? { + ) -> StateKeys.ServiceAccountPreimageInfoKey.Value? { get { changes[ StateKeys.ServiceAccountPreimageInfoKey(index: index, hash: hash, length: length) - ] as? StateKeys.ServiceAccountPreimageInfoKey.Value.ValueType + ]?.value() } set { - changes[StateKeys.ServiceAccountPreimageInfoKey(index: index, hash: hash, length: length)] = newValue + changes[StateKeys.ServiceAccountPreimageInfoKey(index: index, hash: hash, length: length)] = .init(newValue) } } } extension StateLayer { - public func toKV() -> some Sequence<(key: any StateKey, value: Codable & Sendable)> { - changes.map { (key: $0.key.base as! any StateKey, value: $0.value) } + public func toKV() -> some Sequence<(key: any StateKey, value: (Codable & Sendable)?)> { + changes.map { (key: $0.key.base as! any StateKey, value: $0.value.value()) } } } extension StateLayer { - public func read(_ key: Key) -> Key.Value.ValueType? { - changes[key] as? Key.Value.ValueType + public func read(_ key: Key) -> Key.Value? { + changes[key] as? Key.Value } - public mutating func write(_ key: Key, value: Key.Value.ValueType) { - changes[key] = value + public mutating func write(_ key: Key, value: Key.Value?) { + changes[key] = .init(value) } public subscript(key: any StateKey) -> (Codable & Sendable)? { get { - changes[AnyHashable(key)] + changes[AnyHashable(key)]?.value() } set { - changes[AnyHashable(key)] = newValue + changes[AnyHashable(key)] = .init(newValue) } } } diff --git a/Blockchain/Sources/Blockchain/State/StateTrie.swift b/Blockchain/Sources/Blockchain/State/StateTrie.swift new file mode 100644 index 00000000..40e26a47 --- /dev/null +++ b/Blockchain/Sources/Blockchain/State/StateTrie.swift @@ -0,0 +1,365 @@ +import Foundation +import TracingUtils +import Utils + +private let logger = Logger(label: "StateTrie") + +private enum TrieNodeType { + case branch + case embeddedLeaf + case regularLeaf +} + +private struct TrieNode { + let hash: Data32 + let left: Data32 + let right: Data32 + let type: TrieNodeType + let isNew: Bool + let rawValue: Data? + + init(hash: Data32, data: Data64, isNew: Bool = false) { + self.hash = hash + left = Data32(data.data.prefix(32))! + right = Data32(data.data.suffix(32))! + self.isNew = isNew + rawValue = nil + switch data.data[0] & 0b1100_0000 { + case 0b1000_0000: + type = .embeddedLeaf + case 0b1100_0000: + type = .regularLeaf + default: + type = .branch + } + } + + private init(left: Data32, right: Data32, type: TrieNodeType, isNew: Bool, rawValue: Data?) { + hash = Blake2b256.hash(left.data, right.data) + self.left = left + self.right = right + self.type = type + self.isNew = isNew + self.rawValue = rawValue + } + + var encodedData: Data64 { + Data64(left.data + right.data)! + } + + var isBranch: Bool { + type == .branch + } + + var isLeaf: Bool { + !isBranch + } + + func isLeaf(key: Data32) -> Bool { + isLeaf && left.data[relative: 1 ..< 32] == key.data.prefix(31) + } + + var value: Data? { + if let rawValue { + return rawValue + } + guard type == .embeddedLeaf else { + return nil + } + let len = left.data[0] & 0b0011_1111 + return right.data[relative: 0 ..< Int(len)] + } + + static func leaf(key: Data32, value: Data) -> TrieNode { + var newKey = Data(capacity: 32) + if value.count <= 32 { + newKey.append(0b1000_0000 | UInt8(value.count)) + newKey += key.data.prefix(31) + let newValue = value + Data(repeating: 0, count: 32 - value.count) + return .init(left: Data32(newKey)!, right: Data32(newValue)!, type: .embeddedLeaf, isNew: true, rawValue: value) + } + newKey.append(0b1100_0000) + newKey += key.data.prefix(31) + return .init(left: Data32(newKey)!, right: value.blake2b256hash(), type: .regularLeaf, isNew: true, rawValue: value) + } + + static func branch(left: Data32, right: Data32) -> TrieNode { + var left = left.data + left[0] = left[0] & 0b0111_1111 // clear the highest bit + return .init(left: Data32(left)!, right: right, type: .branch, isNew: true, rawValue: nil) + } +} + +public enum StateTrieError: Error { + case invalidData + case invalidParent +} + +public actor StateTrie: Sendable { + private let backend: StateBackendProtocol + public private(set) var rootHash: Data32 + private var nodes: [Data: TrieNode] = [:] + private var deleted: Set = [] + + public init(rootHash: Data32, backend: StateBackendProtocol) { + self.rootHash = rootHash + self.backend = backend + } + + public func read(key: Data32) async throws -> Data? { + let node = try await find(hash: rootHash, key: key, depth: 0) + guard let node else { + return nil + } + if let value = node.value { + return value + } + return try await backend.readValue(hash: node.right) + } + + private func find(hash: Data32, key: Data32, depth: UInt8) async throws -> TrieNode? { + guard let node = try await get(hash: hash) else { + return nil + } + if node.isBranch { + let bitValue = bitAt(key.data, position: depth) + if bitValue { + return try await find(hash: node.right, key: key, depth: depth + 1) + } else { + return try await find(hash: node.left, key: key, depth: depth + 1) + } + } else if node.isLeaf(key: key) { + return node + } + return nil + } + + private func get(hash: Data32) async throws -> TrieNode? { + if hash == Data32() { + return nil + } + let id = hash.data.suffix(31) + if deleted.contains(id) { + return nil + } + if let node = nodes[id] { + return node + } + guard let data = try await backend.read(key: id) else { + return nil + } + + guard let data64 = Data64(data) else { + throw StateTrieError.invalidData + } + + let node = TrieNode(hash: hash, data: data64) + saveNode(node: node) + return node + } + + public func update(_ updates: [(key: Data32, value: Data?)]) async throws { + // TODO: somehow improve the efficiency of this + for (key, value) in updates { + if let value { + rootHash = try await insert(hash: rootHash, key: key, value: value, depth: 0) + } else { + rootHash = try await delete(hash: rootHash, key: key, depth: 0) + } + } + } + + public func save() async throws { + var ops = [StateBackendOperation]() + var refChanges = [Data: Int]() + + // process deleted nodes + for id in deleted { + guard let node = nodes[id] else { + continue + } + if node.isBranch { + // assign -1 to not worry about duplicates + refChanges[node.hash.data.suffix(31)] = -1 + refChanges[node.left.data.suffix(31)] = -1 + refChanges[node.right.data.suffix(31)] = -1 + } + nodes.removeValue(forKey: id) + } + deleted.removeAll() + + for node in nodes.values where node.isNew { + ops.append(.write(key: node.hash.data.suffix(31), value: node.encodedData.data)) + if node.type == .regularLeaf { + try ops.append(.writeRawValue(key: node.right, value: node.rawValue.unwrap())) + } + if node.isBranch { + refChanges[node.left.data.suffix(31), default: 0] += 1 + refChanges[node.right.data.suffix(31), default: 0] += 1 + } + } + + // pin root node + refChanges[rootHash.data.suffix(31), default: 0] += 1 + + nodes.removeAll() + + let zeros = Data(repeating: 0, count: 32) + for (key, value) in refChanges { + if key == zeros { + continue + } + if value > 0 { + ops.append(.refIncrement(key: key.suffix(31))) + } else if value < 0 { + ops.append(.refDecrement(key: key.suffix(31))) + } + } + + try await backend.batchUpdate(ops) + } + + private func insert( + hash: Data32, key: Data32, value: Data, depth: UInt8 + ) async throws -> Data32 { + guard let parent = try await get(hash: hash) else { + let node = TrieNode.leaf(key: key, value: value) + saveNode(node: node) + return node.hash + } + + if parent.isBranch { + removeNode(hash: hash) + + let bitValue = bitAt(key.data, position: depth) + var left = parent.left + var right = parent.right + if bitValue { + right = try await insert(hash: parent.right, key: key, value: value, depth: depth + 1) + } else { + left = try await insert(hash: parent.left, key: key, value: value, depth: depth + 1) + } + let newBranch = TrieNode.branch(left: left, right: right) + saveNode(node: newBranch) + return newBranch.hash + } else { + // leaf + return try await insertLeafNode(existing: parent, newKey: key, newValue: value, depth: depth) + } + } + + private func insertLeafNode(existing: TrieNode, newKey: Data32, newValue: Data, depth: UInt8) async throws -> Data32 { + if existing.isLeaf(key: newKey) { + // update existing leaf + let newLeaf = TrieNode.leaf(key: newKey, value: newValue) + saveNode(node: newLeaf) + return newLeaf.hash + } + + let existingKeyBit = bitAt(existing.left.data[1...], position: depth) + let newKeyBit = bitAt(newKey.data, position: depth) + + if existingKeyBit == newKeyBit { + // need to go deeper + let childNodeHash = try await insertLeafNode( + existing: existing, newKey: newKey, newValue: newValue, depth: depth + 1 + ) + let newBranch = if existingKeyBit { + TrieNode.branch(left: Data32(), right: childNodeHash) + } else { + TrieNode.branch(left: childNodeHash, right: Data32()) + } + saveNode(node: newBranch) + return newBranch.hash + } else { + let newLeaf = TrieNode.leaf(key: newKey, value: newValue) + saveNode(node: newLeaf) + let newBranch = if existingKeyBit { + TrieNode.branch(left: newLeaf.hash, right: existing.hash) + } else { + TrieNode.branch(left: existing.hash, right: newLeaf.hash) + } + saveNode(node: newBranch) + return newBranch.hash + } + } + + private func delete(hash: Data32, key: Data32, depth: UInt8) async throws -> Data32 { + let node = try await get(hash: hash).unwrap(orError: StateTrieError.invalidParent) + + if node.isBranch { + removeNode(hash: hash) + + let bitValue = bitAt(key.data, position: depth) + var left = node.left + var right = node.right + + if bitValue { + right = try await delete(hash: node.right, key: key, depth: depth + 1) + } else { + left = try await delete(hash: node.left, key: key, depth: depth + 1) + } + + if left == Data32(), right == Data32() { + // this branch is empty + return Data32() + } + + let newBranch = TrieNode.branch(left: left, right: right) + saveNode(node: newBranch) + return newBranch.hash + } else { + // leaf + return Data32() + } + } + + private func removeNode(hash: Data32) { + let id = hash.data.suffix(31) + deleted.insert(id) + nodes.removeValue(forKey: id) + } + + private func saveNode(node: TrieNode) { + let id = node.hash.data.suffix(31) + nodes[id] = node + deleted.remove(id) // TODO: maybe this is not needed + } + + public func debugPrint() async throws { + func printNode(_ hash: Data32, depth: UInt8) async throws { + let prefix = String(repeating: " ", count: Int(depth)) + if hash == Data32() { + logger.info("\(prefix) nil") + return + } + let node = try await get(hash: hash) + guard let node else { + return logger.info("\(prefix) ????") + } + logger.info("\(prefix)\(node.hash.toHexString()) \(node.type)") + if node.isBranch { + logger.info("\(prefix) left:") + try await printNode(node.left, depth: depth + 1) + + logger.info("\(prefix) right:") + try await printNode(node.right, depth: depth + 1) + } else { + logger.info("\(prefix) key: \(node.left.toHexString())") + if let value = node.value { + logger.info("\(prefix) value: \(value.toHexString())") + } + } + } + + try await printNode(rootHash, depth: 0) + } +} + +/// bit at i, returns true if it is 1 +private func bitAt(_ data: Data, position: UInt8) -> Bool { + let byteIndex = position / 8 + let bitIndex = 7 - (position % 8) + let byte = data[safeRelative: Int(byteIndex)] ?? 0 + return (byte & (1 << bitIndex)) != 0 +} diff --git a/Blockchain/Sources/Blockchain/Types/RecentHistory.swift b/Blockchain/Sources/Blockchain/Types/RecentHistory.swift index b5848f84..b76389ff 100644 --- a/Blockchain/Sources/Blockchain/Types/RecentHistory.swift +++ b/Blockchain/Sources/Blockchain/Types/RecentHistory.swift @@ -1,3 +1,4 @@ +import Codec import Utils // β @@ -12,19 +13,19 @@ public struct RecentHistory: Sendable, Equatable, Codable { // s public var stateRoot: Data32 - // p - public var workReportHashes: ConfigLimitedSizeArray + // p: work package hash -> segment root lookup + @CodingAs> public var lookup: [Data32: Data32] public init( headerHash: Data32, mmr: MMR, stateRoot: Data32, - workReportHashes: ConfigLimitedSizeArray + lookup: [Data32: Data32] ) { self.headerHash = headerHash self.mmr = mmr self.stateRoot = stateRoot - self.workReportHashes = workReportHashes + self.lookup = lookup } } @@ -40,7 +41,7 @@ extension RecentHistory: Dummy { headerHash: Data32(), mmr: MMR([]), stateRoot: Data32(), - workReportHashes: ConfigLimitedSizeArray(config: config) + lookup: [Data32: Data32]() )] )) } @@ -51,7 +52,7 @@ extension RecentHistory { headerHash: Data32, parentStateRoot: Data32, accumulateRoot: Data32, - workReportHashes: ConfigLimitedSizeArray + lookup: [Data32: Data32] ) { if items.count > 0 { // if this is not block #0 // write the state root of last block @@ -65,7 +66,7 @@ extension RecentHistory { headerHash: headerHash, mmr: mmr, stateRoot: Data32(), // empty and will be updated upon next block - workReportHashes: workReportHashes + lookup: lookup ) items.safeAppend(newItem) diff --git a/Blockchain/Sources/Blockchain/Types/RefinementContext.swift b/Blockchain/Sources/Blockchain/Types/RefinementContext.swift index ef0027ba..3fc97553 100644 --- a/Blockchain/Sources/Blockchain/Types/RefinementContext.swift +++ b/Blockchain/Sources/Blockchain/Types/RefinementContext.swift @@ -3,8 +3,8 @@ import Utils // A refinement context, denoted by the set X, describes the context of the chain // at the point that the report’s corresponding work-package was evaluated. -public struct RefinementContext: Sendable, Equatable, Codable, Hashable { - public struct Anchor: Sendable, Equatable, Codable, Hashable { +public struct RefinementContext: Sendable, Equatable, Codable { + public struct Anchor: Sendable, Equatable, Codable { // a public var headerHash: Data32 // s @@ -23,7 +23,7 @@ public struct RefinementContext: Sendable, Equatable, Codable, Hashable { } } - public struct LokupAnchor: Sendable, Equatable, Codable, Hashable { + public struct LookupAnchor: Sendable, Equatable, Codable, Hashable { // l public var headerHash: Data32 // t @@ -40,15 +40,15 @@ public struct RefinementContext: Sendable, Equatable, Codable, Hashable { public var anchor: Anchor - public var lokupAnchor: LokupAnchor + public var lookupAnchor: LookupAnchor // p - public var prerequistieWorkPackage: Data32? + @CodingAs> public var prerequisiteWorkPackages: Set - public init(anchor: Anchor, lokupAnchor: LokupAnchor, prerequistieWorkPackage: Data32?) { + public init(anchor: Anchor, lookupAnchor: LookupAnchor, prerequisiteWorkPackages: Set) { self.anchor = anchor - self.lokupAnchor = lokupAnchor - self.prerequistieWorkPackage = prerequistieWorkPackage + self.lookupAnchor = lookupAnchor + self.prerequisiteWorkPackages = prerequisiteWorkPackages } } @@ -61,11 +61,11 @@ extension RefinementContext: Dummy { stateRoot: Data32(), beefyRoot: Data32() ), - lokupAnchor: LokupAnchor( + lookupAnchor: LookupAnchor( headerHash: Data32(), timeslot: 0 ), - prerequistieWorkPackage: nil + prerequisiteWorkPackages: Set() ) } } @@ -80,7 +80,7 @@ extension RefinementContext.Anchor: EncodedSize { } } -extension RefinementContext.LokupAnchor: EncodedSize { +extension RefinementContext.LookupAnchor: EncodedSize { public var encodedSize: Int { headerHash.encodedSize + timeslot.encodedSize } @@ -92,7 +92,7 @@ extension RefinementContext.LokupAnchor: EncodedSize { extension RefinementContext: EncodedSize { public var encodedSize: Int { - anchor.encodedSize + lokupAnchor.encodedSize + prerequistieWorkPackage.encodedSize + anchor.encodedSize + lookupAnchor.encodedSize + prerequisiteWorkPackages.encodedSize } public static var encodeedSizeHint: Int? { diff --git a/Blockchain/Sources/Blockchain/Types/ServiceAccount.swift b/Blockchain/Sources/Blockchain/Types/ServiceAccount.swift index b44ac61a..92d0516a 100644 --- a/Blockchain/Sources/Blockchain/Types/ServiceAccount.swift +++ b/Blockchain/Sources/Blockchain/Types/ServiceAccount.swift @@ -19,6 +19,14 @@ public struct ServiceAccountDetails: Sendable, Equatable, Codable { // i: number of items in storage public var itemsCount: UInt32 + + // t: the minimum, or threshold, balance needed for any given service account in terms of its storage footprint + public func thresholdBalance(config: ProtocolConfigRef) -> Balance { + let base = Balance(config.value.serviceMinBalance) + let items = Balance(config.value.additionalMinBalancePerStateItem) * Balance(itemsCount) + let bytes = Balance(config.value.additionalMinBalancePerStateByte) * Balance(totalByteLength) + return base + items + bytes + } } public struct ServiceAccount: Sendable, Equatable, Codable { diff --git a/Blockchain/Sources/Blockchain/Types/WorkItem.swift b/Blockchain/Sources/Blockchain/Types/WorkItem.swift index 41bfa59b..a2383ccd 100644 --- a/Blockchain/Sources/Blockchain/Types/WorkItem.swift +++ b/Blockchain/Sources/Blockchain/Types/WorkItem.swift @@ -25,7 +25,7 @@ public struct WorkItem: Sendable, Equatable, Codable { // g public var gasLimit: Gas - // i: a sequence of imported data segments i identified by the root of the segments tree and an index into it + // i: a sequence of imported data segments which identify a prior exported segment through an index public var inputs: [ImportedDataSegment] // x: a sequence of hashed of blob hashes and lengths to be introduced in this block diff --git a/Blockchain/Sources/Blockchain/Types/WorkReport.swift b/Blockchain/Sources/Blockchain/Types/WorkReport.swift index d7bbfd28..0b5d6054 100644 --- a/Blockchain/Sources/Blockchain/Types/WorkReport.swift +++ b/Blockchain/Sources/Blockchain/Types/WorkReport.swift @@ -18,6 +18,9 @@ public struct WorkReport: Sendable, Equatable, Codable { // o: authorization output public var authorizationOutput: Data + // l: segment-root lookup dictionary + @CodingAs> public var lookup: [Data32: Data32] + // r: the results of the evaluation of each of the items in the package public var results: ConfigLimitedSizeArray< WorkResult, @@ -31,6 +34,7 @@ public struct WorkReport: Sendable, Equatable, Codable { authorizationOutput: Data, refinementContext: RefinementContext, packageSpecification: AvailabilitySpecifications, + lookup: [Data32: Data32], results: ConfigLimitedSizeArray ) { self.authorizerHash = authorizerHash @@ -38,6 +42,7 @@ public struct WorkReport: Sendable, Equatable, Codable { self.authorizationOutput = authorizationOutput self.refinementContext = refinementContext self.packageSpecification = packageSpecification + self.lookup = lookup self.results = results } } @@ -51,6 +56,7 @@ extension WorkReport: Dummy { authorizationOutput: Data(), refinementContext: RefinementContext.dummy(config: config), packageSpecification: AvailabilitySpecifications.dummy(config: config), + lookup: [:], results: try! ConfigLimitedSizeArray(config: config, defaultValue: WorkResult.dummy(config: config)) ) } @@ -64,9 +70,8 @@ extension WorkReport { extension WorkReport: EncodedSize { public var encodedSize: Int { - authorizerHash.encodedSize + coreIndex.encodedSize + authorizationOutput.encodedSize + refinementContext - .encodedSize + packageSpecification - .encodedSize + results.encodedSize + authorizerHash.encodedSize + coreIndex.encodedSize + authorizationOutput.encodedSize + + refinementContext.encodedSize + packageSpecification.encodedSize + lookup.encodedSize + results.encodedSize } public static var encodeedSizeHint: Int? { @@ -78,9 +83,13 @@ extension WorkReport: Validate { public enum WorkReportError: Swift.Error { case tooBig case invalidCoreIndex + case tooManyDependencies } public func validate(config: Config) throws(WorkReportError) { + guard refinementContext.prerequisiteWorkPackages.count + lookup.count <= config.value.maxDepsInWorkReport else { + throw .tooManyDependencies + } guard encodedSize <= config.value.maxEncodedWorkReportSize else { throw .tooBig } diff --git a/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCall.swift b/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCall.swift index c9c3cffa..90f0302c 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCall.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCall.swift @@ -7,11 +7,11 @@ public protocol HostCall { static var identifier: UInt8 { get } func gasCost(state: VMState) -> Gas - func _callImpl(config: ProtocolConfigRef, state: VMState) throws + func _callImpl(config: ProtocolConfigRef, state: VMState) async throws } extension HostCall { - public func call(config: ProtocolConfigRef, state: VMState) -> ExecOutcome { + public func call(config: ProtocolConfigRef, state: VMState) async -> ExecOutcome { guard hasEnoughGas(state: state) else { logger.debug("not enough gas") return .exit(.outOfGas) @@ -20,7 +20,7 @@ extension HostCall { logger.debug("consumed \(gasCost(state: state)) gas") do { - try _callImpl(config: config, state: state) + try await _callImpl(config: config, state: state) return .continued } catch let e as Memory.Error { logger.error("memory error: \(e)") diff --git a/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift b/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift index 584a13a8..ea414bdd 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift @@ -9,7 +9,7 @@ import Utils public class GasFn: HostCall { public static var identifier: UInt8 { 0 } - public func _callImpl(config _: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config _: ProtocolConfigRef, state: VMState) async throws { state.writeRegister(Registers.Index(raw: 7), UInt32(bitPattern: Int32(state.getGas().value & 0xFFFF_FFFF))) state.writeRegister(Registers.Index(raw: 8), UInt32(bitPattern: Int32(state.getGas().value >> 32))) } @@ -19,31 +19,29 @@ public class GasFn: HostCall { public class Lookup: HostCall { public static var identifier: UInt8 { 1 } - public let serviceAccount: ServiceAccount public let serviceIndex: ServiceIndex - public let serviceAccounts: [ServiceIndex: ServiceAccount] + public let serviceAccounts: ServiceAccounts - public init(account: ServiceAccount, serviceIndex: ServiceIndex, accounts: [ServiceIndex: ServiceAccount]) { - serviceAccount = account + public init(serviceIndex: ServiceIndex, accounts: some ServiceAccounts) { self.serviceIndex = serviceIndex serviceAccounts = accounts } - public func _callImpl(config _: ProtocolConfigRef, state: VMState) throws { - var account: ServiceAccount? + public func _callImpl(config _: ProtocolConfigRef, state: VMState) async throws { + var service: ServiceIndex let reg = state.readRegister(Registers.Index(raw: 7)) if reg == serviceIndex || reg == Int32.max { - account = serviceAccount + service = serviceIndex } else { - account = serviceAccounts[reg] + service = reg } let regs = state.readRegisters(in: 8 ..< 11) let preimageHash = try? Blake2b256.hash(state.readMemory(address: regs[0], length: 32)) - let value: Data? = if let account, let preimageHash { - account.preimages[preimageHash] + let value: Data? = if let preimageHash { + try await serviceAccounts.get(serviceAccount: service, preimageHash: preimageHash) } else { nil } @@ -70,31 +68,29 @@ public class Lookup: HostCall { public class Read: HostCall { public static var identifier: UInt8 { 2 } - public let serviceAccount: ServiceAccount public let serviceIndex: ServiceIndex - public let serviceAccounts: [ServiceIndex: ServiceAccount] + public let serviceAccounts: ServiceAccounts - public init(account: ServiceAccount, serviceIndex: ServiceIndex, accounts: [ServiceIndex: ServiceAccount]) { - serviceAccount = account + public init(serviceIndex: ServiceIndex, accounts: ServiceAccounts) { self.serviceIndex = serviceIndex serviceAccounts = accounts } - public func _callImpl(config _: ProtocolConfigRef, state: VMState) throws { - var account: ServiceAccount? + public func _callImpl(config _: ProtocolConfigRef, state: VMState) async throws { + var service: ServiceIndex let reg = state.readRegister(Registers.Index(raw: 7)) if reg == serviceIndex || reg == Int32.max { - account = serviceAccount + service = serviceIndex } else { - account = serviceAccounts[reg] + service = reg } let regs = state.readRegisters(in: 8 ..< 12) let key = try? Blake2b256.hash(serviceIndex.encode(), state.readMemory(address: regs[0], length: Int(regs[1]))) - let value: Data? = if let account, let key { - account.storage[key] + let value: Data? = if let key { + try await serviceAccounts.get(serviceAccount: service, storageKey: key) } else { nil } @@ -121,41 +117,44 @@ public class Read: HostCall { public class Write: HostCall { public static var identifier: UInt8 { 3 } - public var serviceAccount: ServiceAccount public let serviceIndex: ServiceIndex + public var serviceAccounts: ServiceAccounts - public init(account: inout ServiceAccount, serviceIndex: ServiceIndex) { - serviceAccount = account + public init(serviceIndex: ServiceIndex, accounts: inout ServiceAccounts) { self.serviceIndex = serviceIndex + serviceAccounts = accounts } - public func _callImpl(config: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { let regs = state.readRegisters(in: 7 ..< 11) let key = try? Blake2b256.hash(serviceIndex.encode(), state.readMemory(address: regs[0], length: Int(regs[1]))) - var account: ServiceAccount? - if let key, state.isMemoryReadable(address: regs[2], length: Int(regs[3])) { - account = serviceAccount - if regs[3] == 0 { - account?.storage.removeValue(forKey: key) - } else { - account?.storage[key] = try state.readMemory(address: regs[2], length: Int(regs[3])) - } + let service: ServiceIndex? = if key != nil, state.isMemoryReadable(address: regs[2], length: Int(regs[3])) { + serviceIndex } else { - account = nil + nil } - let l = if let key, serviceAccount.storage.keys.contains(key) { - UInt32(serviceAccount.storage[key]!.count) + let len = if let key, let value = try await serviceAccounts.get(serviceAccount: service!, storageKey: key) { + UInt32(value.count) } else { HostCallResultCode.NONE.rawValue } - if key != nil, let account, account.thresholdBalance(config: config) <= account.balance { - state.writeRegister(Registers.Index(raw: 7), l) - serviceAccount = account - } else if let account, account.thresholdBalance(config: config) > account.balance { + let acc: ServiceAccountDetails? = (service != nil) ? try await serviceAccounts.get(serviceAccount: service!) : nil + if key != nil, let service, let acc, acc.thresholdBalance(config: config) <= acc.balance { + state.writeRegister(Registers.Index(raw: 7), len) + if regs[3] == 0 { + serviceAccounts.set(serviceAccount: service, storageKey: key!, value: nil) + } else { + try serviceAccounts.set( + serviceAccount: service, + storageKey: key!, + value: state.readMemory(address: regs[2], length: Int(regs[3])) + ) + } + } else if let acc, acc.thresholdBalance(config: config) > acc.balance { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.FULL.rawValue) } else { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue) @@ -168,28 +167,26 @@ public class Info: HostCall { public static var identifier: UInt8 { 4 } public let serviceIndex: ServiceIndex - public let serviceAccounts: [ServiceIndex: ServiceAccount] + public let serviceAccounts: ServiceAccounts - public init( - serviceIndex: ServiceIndex, - accounts: [ServiceIndex: ServiceAccount] - ) { + public init(serviceIndex: ServiceIndex, accounts: ServiceAccounts) { self.serviceIndex = serviceIndex serviceAccounts = accounts } - public func _callImpl(config: ProtocolConfigRef, state: VMState) throws { - var account: ServiceAccount? + public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { + var service: ServiceIndex let reg = state.readRegister(Registers.Index(raw: 7)) if reg == Int32.max { - account = serviceAccounts[serviceIndex] + service = serviceIndex } else { - account = serviceAccounts[reg] + service = reg } let o = state.readRegister(Registers.Index(raw: 8)) let m: Data? + let account = try await serviceAccounts.get(serviceAccount: service) if let account { // codeHash, balance, thresholdBalance, minAccumlateGas, minOnTransferGas, totalByteLength, itemsCount let capacity = 32 + 8 * 5 + 4 @@ -229,7 +226,7 @@ public class Empower: HostCall { self.x = x } - public func _callImpl(config _: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config _: ProtocolConfigRef, state: VMState) async throws { let regs = state.readRegisters(in: 7 ..< 12) var basicGas: [ServiceIndex: Gas] = [:] @@ -265,7 +262,7 @@ public class Assign: HostCall { self.x = x } - public func _callImpl(config: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { let (targetCoreIndex, startAddr) = state.readRegister(Registers.Index(raw: 7), Registers.Index(raw: 8)) var authorizationQueue: [Data32] = [] @@ -298,7 +295,7 @@ public class Designate: HostCall { self.x = x } - public func _callImpl(config: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { let startAddr = state.readRegister(Registers.Index(raw: 7)) var validatorQueue: [ValidatorKey] = [] @@ -331,7 +328,7 @@ public class Checkpoint: HostCall { self.y = y } - public func _callImpl(config _: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config _: ProtocolConfigRef, state: VMState) async throws { state.writeRegister(Registers.Index(raw: 7), UInt32(bitPattern: Int32(state.getGas().value & 0xFFFF_FFFF))) state.writeRegister(Registers.Index(raw: 8), UInt32(bitPattern: Int32(state.getGas().value >> 32))) @@ -344,20 +341,16 @@ public class New: HostCall { public static var identifier: UInt8 { 9 } public var x: AccumlateResultContext - public var account: ServiceAccount - public let accounts: [ServiceIndex: ServiceAccount] - public init(x: inout AccumlateResultContext, account: ServiceAccount, accounts: [ServiceIndex: ServiceAccount]) { + public init(x: inout AccumlateResultContext) { self.x = x - self.account = account - self.accounts = accounts } private func bump(i: ServiceIndex) -> ServiceIndex { 256 + ((i - 256 + 42) & (serviceIndexModValue - 1)) } - public func _callImpl(config: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { let regs = state.readRegisters(in: 7 ..< 13) let codeHash: Data32? = try? Data32(state.readMemory(address: regs[0], length: 32)) @@ -378,14 +371,16 @@ public class New: HostCall { newAccount!.balance = newAccount!.thresholdBalance(config: config) } - if let newAccount { - account.balance -= newAccount.balance - } + if let newAccount, + var acc = try await x.serviceAccounts.get(serviceAccount: x.serviceIndex), + acc.balance >= acc.thresholdBalance(config: config) + { + acc.balance -= newAccount.balance + x.serviceAccounts.set(serviceAccount: x.serviceIndex, account: acc) - if let newAccount, account.balance >= account.thresholdBalance(config: config) { state.writeRegister(Registers.Index(raw: 7), x.nextAccountIndex) - x.accumulateState.serviceAccounts.merge([x.nextAccountIndex: newAccount, x.serviceIndex: account]) { _, new in new } - x.nextAccountIndex = try AccumulateContext.check(i: bump(i: x.nextAccountIndex), serviceAccounts: accounts) + x.accumulateState.serviceAccounts.merge([x.nextAccountIndex: newAccount]) { _, new in new } + x.nextAccountIndex = AccumulateContext.check(i: bump(i: x.nextAccountIndex), serviceAccounts: x.accumulateState.serviceAccounts) } else if codeHash == nil { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue) } else { @@ -404,18 +399,19 @@ public class Upgrade: HostCall { self.x = x } - public func _callImpl(config _: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config _: ProtocolConfigRef, state: VMState) async throws { let regs = state.readRegisters(in: 7 ..< 12) let codeHash: Data32? = try? Data32(state.readMemory(address: regs[0], length: 32)) let minAccumlateGas = Gas(0x1_0000_0000) * Gas(regs[1]) + Gas(regs[2]) let minOnTransferGas = Gas(0x1_0000_0000) * Gas(regs[3]) + Gas(regs[4]) - if let codeHash { + if let codeHash, var acc = try await x.serviceAccounts.get(serviceAccount: x.serviceIndex) { + acc.codeHash = codeHash + acc.minAccumlateGas = minAccumlateGas + acc.minOnTransferGas = minOnTransferGas + x.serviceAccounts.set(serviceAccount: x.serviceIndex, account: acc) state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OK.rawValue) - x.accumulateState.serviceAccounts[x.serviceIndex]?.codeHash = codeHash - x.accumulateState.serviceAccounts[x.serviceIndex]?.minAccumlateGas = minAccumlateGas - x.accumulateState.serviceAccounts[x.serviceIndex]?.minOnTransferGas = minOnTransferGas } else { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue) } @@ -427,13 +423,9 @@ public class Transfer: HostCall { public static var identifier: UInt8 { 11 } public var x: AccumlateResultContext - public let account: ServiceAccount - public let accounts: [ServiceIndex: ServiceAccount] - public init(x: inout AccumlateResultContext, account: ServiceAccount, accounts: [ServiceIndex: ServiceAccount]) { + public init(x: inout AccumlateResultContext) { self.x = x - self.account = account - self.accounts = accounts } public func gasCost(state: VMState) -> Gas { @@ -441,26 +433,34 @@ public class Transfer: HostCall { return Gas(10) + Gas(reg8) + Gas(0x1_0000_0000) * Gas(reg9) } - public func _callImpl(config: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { let regs = state.readRegisters(in: 0 ..< 6) let amount = Balance(0x1_0000_0000) * Balance(regs[2]) + Balance(regs[1]) let gasLimit = Gas(0x1_0000_0000) * Gas(regs[4]) + Gas(regs[3]) let memo = try? state.readMemory(address: regs[5], length: config.value.transferMemoSize) let dest = regs[0] - let newBalance = account.balance - amount + let acc = try await x.serviceAccounts.get(serviceAccount: x.serviceIndex) + + let destAcc: ServiceAccountDetails? = if try await x.serviceAccounts.get(serviceAccount: dest) != nil { + try await x.serviceAccounts.get(serviceAccount: dest) + } else if x.accumulateState.serviceAccounts[dest] != nil { + x.accumulateState.serviceAccounts[dest]?.toDetails() + } else { + nil + } if memo == nil { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue) - } else if accounts[dest] == nil { + } else if destAcc == nil { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.WHO.rawValue) - } else if gasLimit < accounts[dest]!.minOnTransferGas { + } else if gasLimit < destAcc!.minOnTransferGas { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.LOW.rawValue) } else if Gas(state.getGas()) < gasLimit { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.HIGH.rawValue) - } else if newBalance < account.thresholdBalance(config: config) { + } else if let acc, acc.balance - amount < acc.thresholdBalance(config: config) { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.CASH.rawValue) - } else { + } else if var acc { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OK.rawValue) x.transfers.append(DeferredTransfers( sender: x.serviceIndex, @@ -469,7 +469,8 @@ public class Transfer: HostCall { memo: Data128(memo!)!, gasLimit: gasLimit )) - x.accumulateState.serviceAccounts[x.serviceIndex]!.balance = newBalance + acc.balance -= amount + x.serviceAccounts.set(serviceAccount: x.serviceIndex, account: acc) } } } @@ -479,42 +480,43 @@ public class Quit: HostCall { public static var identifier: UInt8 { 12 } public var x: AccumlateResultContext - public let account: ServiceAccount - public let accounts: [ServiceIndex: ServiceAccount] - public init(x: inout AccumlateResultContext, account: ServiceAccount, accounts: [ServiceIndex: ServiceAccount]) { + public init(x: inout AccumlateResultContext) { self.x = x - self.account = account - self.accounts = accounts } - public func gasCost(state: VMState) -> Gas { - let (reg8, reg9) = state.readRegister(Registers.Index(raw: 8), Registers.Index(raw: 9)) - return Gas(10) + Gas(reg8) + Gas(0x1_0000_0000) * Gas(reg9) - } - - public func _callImpl(config: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { let (dest, startAddr) = state.readRegister(Registers.Index(raw: 7), Registers.Index(raw: 8)) - let amount = account.balance - account.thresholdBalance(config: config) + Balance(config.value.serviceMinBalance) + let acc = try await x.serviceAccounts.get(serviceAccount: x.serviceIndex).expect("service account not found") + let amount = acc.balance - acc.thresholdBalance(config: config) + Balance(config.value.serviceMinBalance) let gasLimit = Gas(state.getGas()) let isValidDest = dest == x.serviceIndex || dest == Int32.max let memoData = try? state.readMemory(address: startAddr, length: config.value.transferMemoSize) let memo = memoData != nil ? try JamDecoder.decode(Data128.self, from: memoData!) : nil + let destAcc: ServiceAccountDetails? = if try await x.serviceAccounts.get(serviceAccount: dest) != nil { + try await x.serviceAccounts.get(serviceAccount: dest) + } else if x.accumulateState.serviceAccounts[dest] != nil { + x.accumulateState.serviceAccounts[dest]?.toDetails() + } else { + nil + } + if isValidDest { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OK.rawValue) - x.accumulateState.serviceAccounts.removeValue(forKey: x.serviceIndex) + x.serviceAccounts.set(serviceAccount: x.serviceIndex, account: nil) throw VMInvocationsError.forceHalt } else if memo == nil { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue) - } else if accounts[dest] == nil { + } else if destAcc == nil { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.WHO.rawValue) - } else if gasLimit < accounts[dest]!.minOnTransferGas { + } else if gasLimit < destAcc!.minOnTransferGas { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.LOW.rawValue) } else { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OK.rawValue) - x.accumulateState.serviceAccounts.removeValue(forKey: x.serviceIndex) + // TODO: need to remove all storage and preimages? + x.serviceAccounts.set(serviceAccount: x.serviceIndex, account: nil) x.transfers.append(DeferredTransfers( sender: x.serviceIndex, destination: dest, @@ -539,28 +541,30 @@ public class Solicit: HostCall { self.timeslot = timeslot } - public func _callImpl(config: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { let (startAddr, length) = state.readRegister(Registers.Index(raw: 7), Registers.Index(raw: 8)) let hash = try? state.readMemory(address: startAddr, length: 32) - var account: ServiceAccount? - if let hash { - let hashAndLength = HashAndLength(hash: Data32(hash)!, length: length) - account = x.accumulateState.serviceAccounts[x.serviceIndex] - if account?.preimageInfos[hashAndLength] == nil { - account?.preimageInfos[hashAndLength] = [] - } else if account?.preimageInfos[hashAndLength]!.count == 2 { - account?.preimageInfos[hashAndLength]!.append(timeslot) - } - } + + let preimageInfo = try await x.serviceAccounts.get(serviceAccount: x.serviceIndex, preimageHash: Data32(hash!)!, length: length) + let notRequestedYet = preimageInfo == nil + let isPreviouslyAvailable = preimageInfo?.count == 2 + let canSolicit = notRequestedYet || isPreviouslyAvailable + + let acc = try await x.serviceAccounts.get(serviceAccount: x.serviceIndex) if hash == nil { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue) - } else if account == nil { + } else if !canSolicit { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.HUH.rawValue) - } else if account!.balance < account!.thresholdBalance(config: config) { + } else if let acc, acc.balance < acc.thresholdBalance(config: config) { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.FULL.rawValue) } else { - x.accumulateState.serviceAccounts[x.serviceIndex] = account + if notRequestedYet { + x.serviceAccounts.set(serviceAccount: x.serviceIndex, preimageHash: Data32(hash!)!, length: length, value: []) + } else if isPreviouslyAvailable, var preimageInfo { + preimageInfo.append(timeslot) + x.serviceAccounts.set(serviceAccount: x.serviceIndex, preimageHash: Data32(hash!)!, length: length, value: preimageInfo) + } } } } @@ -577,32 +581,34 @@ public class Forget: HostCall { self.timeslot = timeslot } - public func _callImpl(config: ProtocolConfigRef, state: VMState) throws { + public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { let (startAddr, length) = state.readRegister(Registers.Index(raw: 7), Registers.Index(raw: 8)) let hash = try? state.readMemory(address: startAddr, length: 32) - var account: ServiceAccount? - if let hash { - let hashAndLength = HashAndLength(hash: Data32(hash)!, length: length) - account = x.accumulateState.serviceAccounts[x.serviceIndex] - let value = account?.preimageInfos[hashAndLength] - let minHoldPeriod = TimeslotIndex(config.value.preimagePurgePeriod) - - if value?.count == 0 || (value?.count == 2 && value![1] < timeslot - minHoldPeriod) { - account?.preimageInfos.removeValue(forKey: hashAndLength) - account?.preimages.removeValue(forKey: hashAndLength.hash) - } else if value?.count == 1 { - account?.preimageInfos[hashAndLength]!.append(timeslot) - } else if value?.count == 3, value![1] < timeslot - minHoldPeriod { - account?.preimageInfos[hashAndLength] = [value![2], timeslot] - } - } + let minHoldPeriod = TimeslotIndex(config.value.preimagePurgePeriod) + + let preimageInfo = try await x.serviceAccounts.get(serviceAccount: x.serviceIndex, preimageHash: Data32(hash!)!, length: length) + let historyCount = preimageInfo?.count + + let canExpunge = historyCount == 0 || (historyCount == 2 && preimageInfo![1] < timeslot - minHoldPeriod) + let isAvailable1 = historyCount == 1 + let isAvailable3 = historyCount == 3 && (preimageInfo![1] < timeslot - minHoldPeriod) + let canForget = canExpunge || isAvailable1 || isAvailable3 if hash == nil { state.writeRegister(Registers.Index(raw: 0), HostCallResultCode.OOB.rawValue) - } else if account == nil { + } else if !canForget { state.writeRegister(Registers.Index(raw: 0), HostCallResultCode.HUH.rawValue) } else { - x.accumulateState.serviceAccounts[x.serviceIndex] = account + if canExpunge { + x.serviceAccounts.set(serviceAccount: x.serviceIndex, preimageHash: Data32(hash!)!, length: length, value: nil) + x.serviceAccounts.set(serviceAccount: x.serviceIndex, preimageHash: Data32(hash!)!, value: nil) + } else if isAvailable1, var preimageInfo { + preimageInfo.append(timeslot) + x.serviceAccounts.set(serviceAccount: x.serviceIndex, preimageHash: Data32(hash!)!, length: length, value: preimageInfo) + } else if isAvailable3, var preimageInfo { + preimageInfo = [preimageInfo[2], timeslot] + x.serviceAccounts.set(serviceAccount: x.serviceIndex, preimageHash: Data32(hash!)!, length: length, value: preimageInfo) + } } } } diff --git a/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/AccumulateContext.swift b/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/AccumulateContext.swift index bb6d2509..1d7d4eb5 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/AccumulateContext.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/AccumulateContext.swift @@ -14,61 +14,52 @@ public class AccumulateContext: InvocationContext { public var config: ProtocolConfigRef public var context: ContextType - public init(context: ContextType, config: ProtocolConfigRef) { + public init(context: inout ContextType, config: ProtocolConfigRef) { self.config = config self.context = context } - public func dispatch(index: UInt32, state: VMState) -> ExecOutcome { + public func dispatch(index: UInt32, state: VMState) async -> ExecOutcome { logger.debug("dispatching host-call: \(index)") - // the accumulating service account - var account = context.x.accumulateState.serviceAccounts[context.x.serviceIndex]! - - var allAccounts = context.x.serviceAccounts - allAccounts.merge(context.x.accumulateState.serviceAccounts) { _, new in new } - switch UInt8(index) { case Read.identifier: - return Read(account: account, serviceIndex: context.x.serviceIndex, accounts: allAccounts) + return await Read(serviceIndex: context.x.serviceIndex, accounts: context.x.serviceAccounts) .call(config: config, state: state) case Write.identifier: - let execOutcome = Write(account: &account, serviceIndex: context.x.serviceIndex) + return await Write(serviceIndex: context.x.serviceIndex, accounts: &context.x.serviceAccounts) .call(config: config, state: state) - // G function in Gray Paper - context.x.accumulateState.serviceAccounts[context.x.serviceIndex] = account - return execOutcome case Lookup.identifier: - return Lookup(account: account, serviceIndex: context.x.serviceIndex, accounts: allAccounts) + return await Lookup(serviceIndex: context.x.serviceIndex, accounts: context.x.serviceAccounts) .call(config: config, state: state) case GasFn.identifier: - return GasFn().call(config: config, state: state) + return await GasFn().call(config: config, state: state) case Info.identifier: - return Info(serviceIndex: context.x.serviceIndex, accounts: allAccounts) + return await Info(serviceIndex: context.x.serviceIndex, accounts: context.x.serviceAccounts) .call(config: config, state: state) case Empower.identifier: - return Empower(x: &context.x).call(config: config, state: state) + return await Empower(x: &context.x).call(config: config, state: state) case Assign.identifier: - return Assign(x: &context.x).call(config: config, state: state) + return await Assign(x: &context.x).call(config: config, state: state) case Designate.identifier: - return Designate(x: &context.x).call(config: config, state: state) + return await Designate(x: &context.x).call(config: config, state: state) case Checkpoint.identifier: - return Checkpoint(x: context.x, y: &context.y).call(config: config, state: state) + return await Checkpoint(x: context.x, y: &context.y).call(config: config, state: state) case New.identifier: - return New(x: &context.x, account: account, accounts: allAccounts).call(config: config, state: state) + return await New(x: &context.x).call(config: config, state: state) case Upgrade.identifier: - return Upgrade(x: &context.x) + return await Upgrade(x: &context.x) .call(config: config, state: state) case Transfer.identifier: - return Transfer(x: &context.x, account: account, accounts: allAccounts) + return await Transfer(x: &context.x) .call(config: config, state: state) case Quit.identifier: - return Quit(x: &context.x, account: account, accounts: allAccounts) + return await Quit(x: &context.x) .call(config: config, state: state) case Solicit.identifier: - return Solicit(x: &context.x, timeslot: context.timeslot).call(config: config, state: state) + return await Solicit(x: &context.x, timeslot: context.timeslot).call(config: config, state: state) case Forget.identifier: - return Forget(x: &context.x, timeslot: context.timeslot).call(config: config, state: state) + return await Forget(x: &context.x, timeslot: context.timeslot).call(config: config, state: state) default: state.consumeGas(Gas(10)) state.writeRegister(Registers.Index(raw: 0), HostCallResultCode.WHAT.rawValue) @@ -77,23 +68,11 @@ public class AccumulateContext: InvocationContext { } // a check function to find the first such index in this sequence which does not already represent a service - public static func check(i: ServiceIndex, serviceAccounts: [ServiceIndex: ServiceAccount]) throws -> ServiceIndex { - var currentIndex = i - let maxIter = serviceIndexModValue - var iter = 0 - - guard currentIndex >= 255 else { - throw VMInvocationsError.checkIndexTooSmall + public static func check(i: ServiceIndex, serviceAccounts: [ServiceIndex: ServiceAccount]) -> ServiceIndex { + if serviceAccounts[i] == nil { + return i } - while serviceAccounts.keys.contains(currentIndex) { - currentIndex = (currentIndex - 255) & (serviceIndexModValue - 1) + 256 - iter += 1 - - if iter > maxIter { - throw VMInvocationsError.checkMaxDepthLimit - } - } - return currentIndex + return check(i: (i - 255) & (serviceIndexModValue - 1) + 256, serviceAccounts: serviceAccounts) } } diff --git a/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/IsAuthorizedContext.swift b/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/IsAuthorizedContext.swift index 66c74022..e92733fe 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/IsAuthorizedContext.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/IsAuthorizedContext.swift @@ -14,9 +14,9 @@ public class IsAuthorizedContext: InvocationContext { self.config = config } - public func dispatch(index: UInt32, state: VMState) -> ExecOutcome { + public func dispatch(index: UInt32, state: VMState) async -> ExecOutcome { if index == GasFn.identifier { - return GasFn().call(config: config, state: state) + return await GasFn().call(config: config, state: state) } else { state.consumeGas(Gas(10)) state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.WHAT.rawValue) diff --git a/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/OnTransferContext.swift b/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/OnTransferContext.swift index 3317376c..ec8f2e83 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/OnTransferContext.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/InvocationContexts/OnTransferContext.swift @@ -6,35 +6,34 @@ private let logger = Logger(label: "OnTransferContext") public class OnTransferContext: InvocationContext { public typealias ContextType = ( - account: ServiceAccount, index: ServiceIndex, - accounts: [ServiceIndex: ServiceAccount] + accounts: ServiceAccounts ) public var config: ProtocolConfigRef public var context: ContextType - public init(context: ContextType, config: ProtocolConfigRef) { + public init(context: inout ContextType, config: ProtocolConfigRef) { self.config = config self.context = context } - public func dispatch(index: UInt32, state: VMState) -> ExecOutcome { + public func dispatch(index: UInt32, state: VMState) async -> ExecOutcome { logger.debug("dispatching host-call: \(index)") switch UInt8(index) { case Lookup.identifier: - return Lookup(account: context.account, serviceIndex: context.index, accounts: context.accounts) + return await Lookup(serviceIndex: context.index, accounts: context.accounts) .call(config: config, state: state) case Read.identifier: - return Read(account: context.account, serviceIndex: context.index, accounts: context.accounts) + return await Read(serviceIndex: context.index, accounts: context.accounts) .call(config: config, state: state) case Write.identifier: - return Write(account: &context.account, serviceIndex: context.index) + return await Write(serviceIndex: context.index, accounts: &context.accounts) .call(config: config, state: state) case GasFn.identifier: - return GasFn().call(config: config, state: state) + return await GasFn().call(config: config, state: state) case Info.identifier: - return Info(serviceIndex: context.index, accounts: context.accounts) + return await Info(serviceIndex: context.index, accounts: context.accounts) .call(config: config, state: state) default: state.consumeGas(Gas(10)) diff --git a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/AccumulateInvocation.swift b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/AccumulateInvocation.swift index 23b3ea4e..d65f7596 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/AccumulateInvocation.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/AccumulateInvocation.swift @@ -6,61 +6,40 @@ import Utils extension AccumulateFunction { public func invoke( config: ProtocolConfigRef, + accounts: inout some ServiceAccounts, state: AccumulateState, serviceIndex: ServiceIndex, gas: Gas, arguments: [AccumulateArguments], initialIndex: ServiceIndex, timeslot: TimeslotIndex - ) throws -> (state: AccumulateState, transfers: [DeferredTransfers], result: Data32?, gas: Gas) { - var serviceAccounts = state.serviceAccounts - - let defaultState = AccumulateState( - serviceAccounts: [:], - validatorQueue: state.validatorQueue, - authorizationQueue: state.authorizationQueue, - privilegedServices: state.privilegedServices - ) - - if serviceAccounts[serviceIndex]?.codeHash.data == nil { - return (defaultState, [], nil, Gas(0)) + ) async throws -> (state: AccumulateState, transfers: [DeferredTransfers], result: Data32?, gas: Gas) { + guard let accumulatingAccountDetails = try await accounts.get(serviceAccount: serviceIndex) else { + return (state, [], nil, Gas(0)) } - guard let accumulatingAccount = serviceAccounts[serviceIndex] else { - throw AccumulationError.invalidServiceIndex - } - - serviceAccounts.removeValue(forKey: serviceIndex) - - let defaultCtx = try AccumlateResultContext( - serviceAccounts: serviceAccounts, + let resultCtx = AccumlateResultContext( + serviceAccounts: accounts, serviceIndex: serviceIndex, - accumulateState: AccumulateState( - serviceAccounts: [serviceIndex: accumulatingAccount], - validatorQueue: state.validatorQueue, - authorizationQueue: state.authorizationQueue, - privilegedServices: state.privilegedServices - ), + accumulateState: state, nextAccountIndex: AccumulateContext.check( i: initialIndex & (serviceIndexModValue - 1) + 256, - serviceAccounts: [serviceIndex: accumulatingAccount] + serviceAccounts: [:] ), transfers: [] ) - let ctx = AccumulateContext( - context: ( - x: defaultCtx, - y: defaultCtx, - timeslot: timeslot - ), - config: config + var contextContent = AccumulateContext.ContextType( + x: resultCtx, + y: resultCtx, + timeslot: timeslot ) + let ctx = AccumulateContext(context: &contextContent, config: config) let argument = try JamEncoder.encode(arguments) - let (exitReason, gas, output) = invokePVM( + let (exitReason, gas, output) = await invokePVM( config: config, - blob: serviceAccounts[serviceIndex]!.codeHash.data, + blob: accumulatingAccountDetails.codeHash.data, pc: 10, gas: gas, argumentData: argument, diff --git a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/IsAuthorizedInvocation.swift b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/IsAuthorizedInvocation.swift index 1446c595..cbf1dde7 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/IsAuthorizedInvocation.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/IsAuthorizedInvocation.swift @@ -11,11 +11,13 @@ public protocol IsAuthorizedFunction { } extension IsAuthorizedFunction { - public func invoke(config: ProtocolConfigRef, package: WorkPackage, coreIndex: CoreIndex) throws -> Result { + public func invoke(config: ProtocolConfigRef, package: WorkPackage, + coreIndex: CoreIndex) async throws -> Result + { let args = try JamEncoder.encode(package) + JamEncoder.encode(coreIndex) let ctx = IsAuthorizedContext(config: config) - let (exitReason, _, output) = invokePVM( + let (exitReason, _, output) = await invokePVM( config: config, blob: package.authorizationCodeHash.data, pc: 0, diff --git a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/OnTransferInvocation.swift b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/OnTransferInvocation.swift index e6db3e8d..6271b5e2 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/Invocations/OnTransferInvocation.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/Invocations/OnTransferInvocation.swift @@ -7,25 +7,25 @@ extension OnTransferFunction { public func invoke( config: ProtocolConfigRef, service: ServiceIndex, - code _: Data, - serviceAccounts: [ServiceIndex: ServiceAccount], + serviceAccounts: inout some ServiceAccounts, transfers: [DeferredTransfers] - ) throws -> ServiceAccount { - guard var account = serviceAccounts[service] else { + ) async throws { + guard var account = try await serviceAccounts.get(serviceAccount: service) else { throw VMInvocationsError.serviceAccountNotFound } account.balance += transfers.reduce(Balance(0)) { $0 + $1.amount } if account.codeHash.data.isEmpty || transfers.isEmpty { - return account + return } - let ctx = OnTransferContext(context: (account, service, serviceAccounts), config: config) + var contextContent = OnTransferContext.ContextType(service, serviceAccounts) + let ctx = OnTransferContext(context: &contextContent, config: config) let gasLimitSum = transfers.reduce(Balance(0)) { $0 + $1.gasLimit } let argument = try JamEncoder.encode(transfers) - _ = invokePVM( + _ = await invokePVM( config: config, blob: account.codeHash.data, pc: 15, @@ -33,7 +33,5 @@ extension OnTransferFunction { argumentData: argument, ctx: ctx ) - - return ctx.context.0 } } diff --git a/Blockchain/Sources/Blockchain/Validator/BlockAuthor.swift b/Blockchain/Sources/Blockchain/Validator/BlockAuthor.swift index 137ab636..785a027f 100644 --- a/Blockchain/Sources/Blockchain/Validator/BlockAuthor.swift +++ b/Blockchain/Sources/Blockchain/Validator/BlockAuthor.swift @@ -54,6 +54,7 @@ public final class BlockAuthor: ServiceBase2, @unchecked Sendable { // TODO: verify we are indeed the block author let state = try await dataProvider.getState(hash: parentHash) + let stateRoot = await state.value.stateRoot let epoch = timeslot.timeslotToEpochIndex(config: config) let pendingTickets = await extrinsicPool.getPendingTickets(epoch: epoch) @@ -120,7 +121,7 @@ public final class BlockAuthor: ServiceBase2, @unchecked Sendable { let unsignedHeader = Header.Unsigned( parentHash: parentHash, - priorStateRoot: state.stateRoot, + priorStateRoot: stateRoot, extrinsicsHash: extrinsic.hash(), timeslot: timeslot, epoch: safroleResult.epochMark, diff --git a/Blockchain/Tests/BlockchainTests/BlockAuthorTests.swift b/Blockchain/Tests/BlockchainTests/BlockAuthorTests.swift index 16b809c4..4a0b3ed5 100644 --- a/Blockchain/Tests/BlockchainTests/BlockAuthorTests.swift +++ b/Blockchain/Tests/BlockchainTests/BlockAuthorTests.swift @@ -22,6 +22,7 @@ struct BlockAuthorTests { let config = services.config let timeProvider = services.timeProvider let genesisState = services.genesisState + let stateRoot = await genesisState.value.stateRoot let timeslot = timeProvider.getTime().timeToTimeslot(config: config) @@ -42,7 +43,10 @@ struct BlockAuthorTests { let block = try await blockAuthor.createNewBlock(timeslot: timeslot, claim: .right(pubkey)) // Verify block - try _ = await runtime.apply(block: block, state: genesisState, context: .init(timeslot: timeslot + 1)) + try _ = await runtime.apply(block: block, state: genesisState, context: .init( + timeslot: timeslot + 1, + stateRoot: stateRoot + )) } @Test @@ -95,7 +99,10 @@ struct BlockAuthorTests { let block = try await blockAuthor.createNewBlock(timeslot: timeslot, claim: .left((ticket, devKey.bandersnatch))) // Verify block - try _ = await runtime.apply(block: block, state: newStateRef, context: .init(timeslot: timeslot + 1)) + try _ = await runtime.apply(block: block, state: newStateRef, context: .init( + timeslot: timeslot + 1, + stateRoot: newStateRef.value.stateRoot + )) } @Test @@ -122,7 +129,10 @@ struct BlockAuthorTests { let timeslot = timeProvider.getTime().timeToTimeslot(config: config) // Verify block - try _ = await runtime.apply(block: block.block, state: genesisState, context: .init(timeslot: timeslot + 1)) + try _ = await runtime.apply(block: block.block, state: genesisState, context: .init( + timeslot: timeslot + 1, + stateRoot: genesisState.value.stateRoot + )) } // TODO: test including extrinsic tickets from extrinsic pool diff --git a/Blockchain/Tests/BlockchainTests/BlockchainDataProviderTests.swift b/Blockchain/Tests/BlockchainTests/BlockchainDataProviderTests.swift index 27286e99..8b4320e0 100644 --- a/Blockchain/Tests/BlockchainTests/BlockchainDataProviderTests.swift +++ b/Blockchain/Tests/BlockchainTests/BlockchainDataProviderTests.swift @@ -78,11 +78,11 @@ struct BlockchainDataProviderTests { // Verify state exists #expect(try await provider.hasState(hash: block.hash)) - #expect(try await provider.getState(hash: block.hash).stateRoot == state.stateRoot) + #expect(try await provider.getState(hash: block.hash).value.stateRoot == state.value.stateRoot) // Test getting best state let bestState = try await provider.getBestState() - #expect(bestState.stateRoot == state.stateRoot) + #expect(await bestState.value.stateRoot == state.value.stateRoot) } @Test func testStateOperationsErrors() async throws { diff --git a/Blockchain/Tests/BlockchainTests/ExtrinsicPoolServiceTests.swift b/Blockchain/Tests/BlockchainTests/ExtrinsicPoolServiceTests.swift index 9822e0ae..9c993ea0 100644 --- a/Blockchain/Tests/BlockchainTests/ExtrinsicPoolServiceTests.swift +++ b/Blockchain/Tests/BlockchainTests/ExtrinsicPoolServiceTests.swift @@ -215,7 +215,7 @@ struct ExtrinsicPoolServiceTests { headerHash: newBlock.hash, mmr: MMR([]), stateRoot: Data32(), - workReportHashes: ConfigLimitedSizeArray(config: config) + lookup: [Data32: Data32]() )) } diff --git a/Blockchain/Tests/BlockchainTests/SafroleServiceTests.swift b/Blockchain/Tests/BlockchainTests/SafroleServiceTests.swift index 0f50da12..268db2d5 100644 --- a/Blockchain/Tests/BlockchainTests/SafroleServiceTests.swift +++ b/Blockchain/Tests/BlockchainTests/SafroleServiceTests.swift @@ -62,7 +62,7 @@ struct SafroleServiceTests { headerHash: newBlock.hash, mmr: MMR([]), stateRoot: Data32(), - workReportHashes: ConfigLimitedSizeArray(config: config) + lookup: [Data32: Data32]() )) } @@ -91,7 +91,7 @@ struct SafroleServiceTests { headerHash: newBlock.hash, mmr: MMR([]), stateRoot: Data32(), - workReportHashes: ConfigLimitedSizeArray(config: config) + lookup: [Data32: Data32]() )) } diff --git a/Blockchain/Tests/BlockchainTests/StateTrieTests.swift b/Blockchain/Tests/BlockchainTests/StateTrieTests.swift new file mode 100644 index 00000000..7b090ce0 --- /dev/null +++ b/Blockchain/Tests/BlockchainTests/StateTrieTests.swift @@ -0,0 +1,190 @@ +import Foundation +import Testing +import Utils + +@testable import Blockchain + +private func merklize(_ data: some Sequence<(key: Data32, value: Data)>) -> Data32 { + var dict = [Data32: Data]() + for (key, value) in data { + dict[key] = value + } + return try! stateMerklize(kv: dict) +} + +struct StateTrieTests { + let backend = InMemoryBackend() + + // MARK: - Basic Operations Tests + + @Test + func testEmptyTrie() async throws { + let trie = StateTrie(rootHash: Data32(), backend: backend) + let key = Data32.random() + let value = try await trie.read(key: key) + #expect(value == nil) + } + + @Test + func testInsertAndRetrieveSingleValue() async throws { + let trie = StateTrie(rootHash: Data32(), backend: backend) + let key = Data([1]).blake2b256hash() + let value = Data("test value".utf8) + + try await trie.update([(key: key, value: value)]) + try await trie.save() + + let retrieved = try await trie.read(key: key) + #expect(retrieved == value) + } + + @Test + func testInsertAndRetrieveSimple() async throws { + let trie = StateTrie(rootHash: Data32(), backend: backend) + let remainKey = Data(repeating: 0, count: 31) + let pairs = [ + (key: Data32(Data([0b0000_0000]) + remainKey)!, value: Data([0])), + (key: Data32(Data([0b1000_0000]) + remainKey)!, value: Data([1])), + (key: Data32(Data([0b0100_0000]) + remainKey)!, value: Data([2])), + (key: Data32(Data([0b1100_0000]) + remainKey)!, value: Data([3])), + ] + + for (i, pair) in pairs.enumerated() { + try await trie.update([(key: pair.key, value: pair.value)]) + + let expectedRoot = merklize(pairs[0 ... i]) + let trieRoot = await trie.rootHash + #expect(expectedRoot == trieRoot) + } + + for (i, (key, value)) in pairs.enumerated() { + let retrieved = try await trie.read(key: key) + #expect(retrieved == value, "Failed at index \(i)") + } + + try await trie.save() + + for (i, (key, value)) in pairs.enumerated() { + let retrieved = try await trie.read(key: key) + #expect(retrieved == value, "Failed at index \(i)") + } + } + + @Test + func testInsertAndRetrieveMultipleValues() async throws { + let trie = StateTrie(rootHash: Data32(), backend: backend) + let pairs = (0 ..< 50).map { i in + let data = Data([UInt8(i)]) + return (key: data.blake2b256hash(), value: data) + } + + for (i, pair) in pairs.enumerated() { + try await trie.update([(key: pair.key, value: pair.value)]) + + let expectedRoot = merklize(pairs[0 ... i]) + let trieRoot = await trie.rootHash + #expect(expectedRoot == trieRoot) + } + + for (i, (key, value)) in pairs.enumerated() { + let retrieved = try await trie.read(key: key) + #expect(retrieved == value, "Failed at index \(i)") + } + + try await trie.save() + + for (i, (key, value)) in pairs.enumerated() { + let retrieved = try await trie.read(key: key) + #expect(retrieved == value, "Failed at index \(i)") + } + } + + // MARK: - Update Tests + + @Test + func testUpdateExistingValue() async throws { + let trie = StateTrie(rootHash: Data32(), backend: backend) + let key = Data32.random() + let value1 = Data("value1".utf8) + let value2 = Data("value2".utf8) + + try await trie.update([(key: key, value: value1)]) + try await trie.save() + + try await trie.update([(key: key, value: value2)]) + try await trie.save() + + let retrieved = try await trie.read(key: key) + #expect(retrieved == value2) + } + + @Test + func testDeleteValue() async throws { + let trie = StateTrie(rootHash: Data32(), backend: backend) + let key = Data32.random() + let value = Data("test".utf8) + + try await trie.update([(key: key, value: value)]) + try await trie.save() + + try await trie.update([(key: key, value: nil)]) + try await trie.save() + + let retrieved = try await trie.read(key: key) + #expect(retrieved == nil) + } + + // MARK: - Large Value Tests + + @Test + func testLargeValue() async throws { + let trie = StateTrie(rootHash: Data32(), backend: backend) + let key = Data32.random() + let value = Data(repeating: 0xFF, count: 1000) // Value larger than 32 bytes + + try await trie.update([(key: key, value: value)]) + try await trie.save() + + let retrieved = try await trie.read(key: key) + #expect(retrieved == value) + } + + // MARK: - Root Hash Tests + + @Test + func testRootHashChanges() async throws { + let trie = StateTrie(rootHash: Data32(), backend: backend) + let initialRoot = await trie.rootHash + + let key = Data32.random() + let value = Data("test".utf8) + + try await trie.update([(key: key, value: value)]) + try await trie.save() + + let newRoot = await trie.rootHash + #expect(initialRoot != newRoot) + } + + @Test + func testRootHashConsistency() async throws { + let trie1 = StateTrie(rootHash: Data32(), backend: backend) + let trie2 = StateTrie(rootHash: Data32(), backend: backend) + + let pairs = (0 ..< 5).map { i in + let data = Data(String(i).utf8) + return (key: data.blake2b256hash(), value: data) + } + + // Apply same updates to both tries + try await trie1.update(pairs) + try await trie1.save() + + try await trie2.update(pairs) + try await trie2.save() + + #expect(await trie1.rootHash == trie2.rootHash) + } + + // TODO: test for gc, ref counting & pruning, raw value ref counting & cleaning +} diff --git a/Database/Sources/Database/RocksDB.swift b/Database/Sources/Database/RocksDB.swift index 51d1c9f5..dd73f824 100644 --- a/Database/Sources/Database/RocksDB.swift +++ b/Database/Sources/Database/RocksDB.swift @@ -28,7 +28,8 @@ public final class RocksDB { // Optimize rocksdb rocksdb_options_increase_parallelism(dbOptions, Int32(cpus)) - rocksdb_options_optimize_level_style_compaction(dbOptions, 0) // TODO: check this + let memtable_memory_budget: UInt64 = 512 * 1024 * 1024 // 512 MB + rocksdb_options_optimize_level_style_compaction(dbOptions, memtable_memory_budget) // create the DB if it's not already present rocksdb_options_set_create_if_missing(dbOptions, 1) diff --git a/JAMTests/Tests/JAMTests/CodecTests.swift b/JAMTests/Tests/JAMTests/CodecTests.swift index 490081e2..adf9b875 100644 --- a/JAMTests/Tests/JAMTests/CodecTests.swift +++ b/JAMTests/Tests/JAMTests/CodecTests.swift @@ -103,9 +103,9 @@ struct CodecTests { return [ "anchor": json["anchor"]!["headerHash"]!, "beefy_root": json["anchor"]!["beefyRoot"]!, - "lookup_anchor": json["lokupAnchor"]!["headerHash"]!, - "lookup_anchor_slot": json["lokupAnchor"]!["timeslot"]!, - "prerequisite": json["prerequistieWorkPackage"] ?? .null, + "lookup_anchor": json["lookupAnchor"]!["headerHash"]!, + "lookup_anchor_slot": json["lookupAnchor"]!["timeslot"]!, + "prerequisite": json["prerequisiteWorkPackages"] ?? .null, "state_root": json["anchor"]!["stateRoot"]!, ].json } @@ -120,7 +120,7 @@ struct CodecTests { if value is WorkResult { return [ "code_hash": json["codeHash"]!, - "gas_ratio": json["gasRatio"]!, + "gas": json["gasRatio"]!, "payload_hash": json["payloadHash"]!, "service": json["serviceIndex"]!, "result": json["output"]!["success"] == nil ? json["output"]! : [ @@ -169,6 +169,7 @@ struct CodecTests { "authorizer_hash": json["authorizerHash"]!, "auth_output": json["authorizationOutput"]!, "results": transform(json["results"]!, value: value.results), + "segment_root_lookup": transform(json["lookup"]!, value: value.lookup), ].json } if value is AvailabilitySpecifications { @@ -241,8 +242,10 @@ struct CodecTests { @Test func block() throws { - let (actual, expected) = try Self.test(Block.self, path: "block") - #expect(actual == expected) + withKnownIssue("waiting for refine_context.prerequisite updates", isIntermittent: true) { + let (actual, expected) = try Self.test(Block.self, path: "block") + #expect(actual == expected) + } } @Test @@ -253,14 +256,18 @@ struct CodecTests { @Test func extrinsic() throws { - let (actual, expected) = try Self.test(Extrinsic.self, path: "extrinsic") - #expect(actual == expected) + withKnownIssue("waiting for refine_context.prerequisite updates", isIntermittent: true) { + let (actual, expected) = try Self.test(Extrinsic.self, path: "extrinsic") + #expect(actual == expected) + } } @Test func guarantees_extrinsic() throws { - let (actual, expected) = try Self.test(ExtrinsicGuarantees.self, path: "guarantees_extrinsic") - #expect(actual == expected) + withKnownIssue("waiting for refine_context.prerequisite updates", isIntermittent: true) { + let (actual, expected) = try Self.test(ExtrinsicGuarantees.self, path: "guarantees_extrinsic") + #expect(actual == expected) + } } @Test @@ -283,8 +290,10 @@ struct CodecTests { @Test func refine_context() throws { - let (actual, expected) = try Self.test(RefinementContext.self, path: "refine_context") - #expect(actual == expected) + withKnownIssue("waiting for refine_context.prerequisite updates", isIntermittent: true) { + let (actual, expected) = try Self.test(RefinementContext.self, path: "refine_context") + #expect(actual == expected) + } } @Test @@ -301,14 +310,18 @@ struct CodecTests { @Test func work_package() throws { - let (actual, expected) = try Self.test(WorkPackage.self, path: "work_package") - #expect(actual == expected) + withKnownIssue("waiting for refine_context.prerequisite updates", isIntermittent: true) { + let (actual, expected) = try Self.test(WorkPackage.self, path: "work_package") + #expect(actual == expected) + } } @Test func work_report() throws { - let (actual, expected) = try Self.test(WorkReport.self, path: "work_report") - #expect(actual == expected) + withKnownIssue("waiting for refine_context.prerequisite updates", isIntermittent: true) { + let (actual, expected) = try Self.test(WorkReport.self, path: "work_report") + #expect(actual == expected) + } } @Test diff --git a/JAMTests/Tests/JAMTests/PVMTests.swift b/JAMTests/Tests/JAMTests/PVMTests.swift index 14808010..0c9af80e 100644 --- a/JAMTests/Tests/JAMTests/PVMTests.swift +++ b/JAMTests/Tests/JAMTests/PVMTests.swift @@ -74,7 +74,7 @@ struct PVMTests { } @Test(arguments: try loadTests()) - func testPVM(testCase: Testcase) throws { + func testPVM(testCase: Testcase) async throws { let decoder = JSONDecoder() let testCase = try decoder.decode(PolkaVMTestcase.self, from: testCase.data) let program = try ProgramCode(Data(testCase.program)) @@ -90,7 +90,7 @@ struct PVMTests { memory: memory ) let engine = Engine(config: DefaultPvmConfig()) - let exitReason = engine.execute(program: program, state: vmState) + let exitReason = await engine.execute(program: program, state: vmState) logger.debug("exit reason: \(exitReason)") let exitReason2: Status = switch exitReason { case .halt: diff --git a/JAMTests/Tests/JAMTests/RecentHistoryTests.swift b/JAMTests/Tests/JAMTests/RecentHistoryTests.swift index e6636ab9..ecfce568 100644 --- a/JAMTests/Tests/JAMTests/RecentHistoryTests.swift +++ b/JAMTests/Tests/JAMTests/RecentHistoryTests.swift @@ -6,11 +6,16 @@ import Utils @testable import JAMTests +struct ReportedWorkPackage: Codable { + var hash: Data32 + var exportsRoot: Data32 +} + struct RecentHistoryInput: Codable { var headerHash: Data32 var parentStateRoot: Data32 var accumulateRoot: Data32 - var workPackages: [Data32] + var workPackages: [ReportedWorkPackage] } struct RecentHisoryTestcase: Codable { @@ -30,11 +35,14 @@ struct RecentHistoryTests { let testcase = try JamDecoder.decode(RecentHisoryTestcase.self, from: testcase.data, withConfig: config) var state = testcase.preState - try state.update( + state.update( headerHash: testcase.input.headerHash, parentStateRoot: testcase.input.parentStateRoot, accumulateRoot: testcase.input.accumulateRoot, - workReportHashes: ConfigLimitedSizeArray(config: config, array: testcase.input.workPackages) + lookup: Dictionary(uniqueKeysWithValues: testcase.input.workPackages.map { ( + $0.hash, + $0.exportsRoot + ) }) ) #expect(state == testcase.postState) diff --git a/JAMTests/jamtestvectors b/JAMTests/jamtestvectors index c2f228a2..a46c3539 160000 --- a/JAMTests/jamtestvectors +++ b/JAMTests/jamtestvectors @@ -1 +1 @@ -Subproject commit c2f228a2a35c744354a591e1c5218a51501e44ad +Subproject commit a46c3539d79188a499fbdde933c50a82ac98a0f1 diff --git a/Makefile b/Makefile index 6c261c65..0d07c7d9 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,10 @@ test-cargo: .PHONY: test-all test-all: test test-cargo +.PHONY: test-coverage +test-coverage: + ./scripts/runTests.sh test --enable-code-coverage + .PHONY: build build: githooks deps ./scripts/run.sh build diff --git a/Networking/Sources/Networking/Peer.swift b/Networking/Sources/Networking/Peer.swift index 705aa10c..32829072 100644 --- a/Networking/Sources/Networking/Peer.swift +++ b/Networking/Sources/Networking/Peer.swift @@ -271,19 +271,9 @@ final class PeerImpl: Sendable { if role == .builder { let currentCount = connections.byAddr.values.filter { $0.role == role }.count if currentCount >= self.settings.maxBuilderConnections { - if let leastActiveConn = connections.byAddr.values.filter({ $0.role == .builder }) - .sorted(by: { $0.lastActiveTimeStamp < $1.lastActiveTimeStamp }).first - { - self.logger.warning( - "Replacing least active builder connection at \(leastActiveConn.remoteAddress)" - ) - leastActiveConn.close(abort: false) - } else { - self.logger.warning( - "Max builder connections reached, no eligible replacement found" - ) - return false - } + self.logger.warning("max builder connections reached") + // TODO: consider connection rotation strategy + return false } } if connections.byAddr[addr] != nil { @@ -351,9 +341,7 @@ final class PeerImpl: Sendable { } guard state.attempt < maxRetryAttempts else { - logger.warning( - "Reopen attempt for stream \(kind) on connection \(connection.id) exceeded max attempts" - ) + logger.warning("Reopen attempt for stream \(kind) on connection \(connection.id) exceeded max attempts") return } @@ -367,9 +355,7 @@ final class PeerImpl: Sendable { Task { try await Task.sleep(for: .seconds(state.delay)) do { - logger.debug( - "Attempting to reopen UP stream of kind \(kind) for connection \(connection.id)" - ) + logger.debug("Attempting to reopen UP stream of kind \(kind) for connection \(connection.id)") try connection.createPreistentStream(kind: kind) } catch { logger.error("Failed to reopen UP stream for connection \(connection.id): \(error)") @@ -602,13 +588,8 @@ private struct PeerEventHandler: QuicEventHandler { let stream = impl.streams.read { streams in streams[stream.id] } - if let stream { stream.received(data: data) - let connection = impl.connections.read { connections in - connections.byId[stream.connectionId] - } - connection?.updateLastActive() } } @@ -630,9 +611,7 @@ private struct PeerEventHandler: QuicEventHandler { do { try connection.createPreistentStream(kind: kind) } catch { - logger.error( - "Attempt to recreate the persistent stream failed: \(error)" - ) + logger.error("Attempt to recreate the persistent stream failed: \(error)") } } } @@ -650,9 +629,7 @@ private struct PeerEventHandler: QuicEventHandler { } // TODO: Add all the cases about reopen up stream - private func shouldReopenStream( - connection: Connection, stream: Stream, status: QuicStatus - ) -> Bool { + private func shouldReopenStream(connection: Connection, stream: Stream, status: QuicStatus) -> Bool { // Only reopen if the stream is a persistent UP stream and the closure was unexpected if connection.isClosed || connection.needReconnect || stream.kind == nil { return false diff --git a/Node/Package.resolved b/Node/Package.resolved index f1dd53fc..21e00dc6 100644 --- a/Node/Package.resolved +++ b/Node/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "33177d9b5bd122b13d37a99e2337556cfc5a1b7e466229e7bdb19fd9423a117a", + "originHash" : "e6fc7ac1513fbfe8e482dc8b89ef5388fcc4ffb7a67e2def60644a806029f64e", "pins" : [ { "identity" : "async-channels", @@ -105,8 +105,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "3d2dc41a01f9e49d84f0a3925fb858bed64f702d", - "version" : "1.1.2" + "revision" : "671108c96644956dddcd89dd59c203dcdb36cec7", + "version" : "1.1.4" } }, { diff --git a/Node/Sources/Node/Genesis.swift b/Node/Sources/Node/Genesis.swift index 41aa55e1..1e419ac3 100644 --- a/Node/Sources/Node/Genesis.swift +++ b/Node/Sources/Node/Genesis.swift @@ -45,8 +45,10 @@ extension Genesis { let config = preset.config let (state, block) = try State.devGenesis(config: config) var kv = [String: Data]() - for (key, value) in try await state.value.toKV() { - kv[key.toHexString()] = value + for (key, value) in state.value.layer.toKV() { + if let value { + kv[key.encode().toHexString()] = try JamEncoder.encode(value) + } } return try ChainSpec( name: preset.rawValue, diff --git a/Node/Sources/Node/Node.swift b/Node/Sources/Node/Node.swift index 503b63a5..438759d4 100644 --- a/Node/Sources/Node/Node.swift +++ b/Node/Sources/Node/Node.swift @@ -44,7 +44,8 @@ public class Node { let chainspec = try await genesis.load() let genesisBlock = try chainspec.getBlock() let genesisStateData = try chainspec.getState() - let backend = try InMemoryBackend(config: chainspec.getConfig(), store: genesisStateData) + let backend = try StateBackend(InMemoryBackend(), config: chainspec.getConfig(), rootHash: Data32()) + try await backend.writeRaw(Array(genesisStateData)) let genesisState = try await State(backend: backend) let genesisStateRef = StateRef(genesisState) let protocolConfig = try chainspec.getConfig() diff --git a/Node/Tests/NodeTests/ChainSpecTests.swift b/Node/Tests/NodeTests/ChainSpecTests.swift index 52c14bd1..bf09de32 100644 --- a/Node/Tests/NodeTests/ChainSpecTests.swift +++ b/Node/Tests/NodeTests/ChainSpecTests.swift @@ -1,6 +1,7 @@ import Blockchain import Foundation import Testing +import Utils @testable import Node @@ -17,12 +18,14 @@ struct ChainSpecTests { for preset in GenesisPreset.allCases { let genesis = Genesis.preset(preset) let chainspec = try await genesis.load() - let backend = try InMemoryBackend(config: chainspec.getConfig(), store: chainspec.getState()) + let backend = try StateBackend(InMemoryBackend(), config: chainspec.getConfig(), rootHash: Data32()) + let state = try chainspec.getState() + try await backend.writeRaw(state.map { (key: $0.key, value: $0.value) }) let block = try chainspec.getBlock() let config = try chainspec.getConfig() let recentHistory = try await backend.read(StateKeys.RecentHistoryKey()) - #expect(recentHistory.items.last?.headerHash == block.hash) + #expect(recentHistory?.items.last?.headerHash == block.hash) // Verify config matches preset #expect(config == preset.config) diff --git a/Node/Tests/NodeTests/chainfiles/devnet_allconfig_spec.json b/Node/Tests/NodeTests/chainfiles/devnet_allconfig_spec.json index 8e6e9233..2680f59f 100644 --- a/Node/Tests/NodeTests/chainfiles/devnet_allconfig_spec.json +++ b/Node/Tests/NodeTests/chainfiles/devnet_allconfig_spec.json @@ -36,7 +36,9 @@ "totalNumberOfValidators" : 3, "transferMemoSize" : 128, "workPackageAuthorizerGas" : 10000000, - "workPackageRefineGas" : 10000000 + "workPackageRefineGas" : 10000000, + "totalAccumulationGas": 341000000, + "maxDepsInWorkReport": 8 }, "state" : {}, "block":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" diff --git a/PolkaVM/Sources/PolkaVM/Engine.swift b/PolkaVM/Sources/PolkaVM/Engine.swift index 2dbd061b..be53f1f7 100644 --- a/PolkaVM/Sources/PolkaVM/Engine.swift +++ b/PolkaVM/Sources/PolkaVM/Engine.swift @@ -13,7 +13,7 @@ public class Engine { self.invocationContext = invocationContext } - public func execute(program: ProgramCode, state: VMState) -> ExitReason { + public func execute(program: ProgramCode, state: VMState) async -> ExitReason { let context = ExecutionContext(state: state, config: config) while true { guard state.getGas() > GasInt(0) else { @@ -22,7 +22,7 @@ public class Engine { if case let .exit(reason) = step(program: program, context: context) { switch reason { case let .hostCall(callIndex): - if case let .exit(hostExitReason) = hostCall(state: state, callIndex: callIndex) { + if case let .exit(hostExitReason) = await hostCall(state: state, callIndex: callIndex) { return hostExitReason } default: @@ -32,12 +32,12 @@ public class Engine { } } - func hostCall(state: VMState, callIndex: UInt32) -> ExecOutcome { + func hostCall(state: VMState, callIndex: UInt32) async -> ExecOutcome { guard let invocationContext else { return .exit(.panic(.trap)) } - let result = invocationContext.dispatch(index: callIndex, state: state) + let result = await invocationContext.dispatch(index: callIndex, state: state) switch result { case let .exit(reason): switch reason { @@ -47,7 +47,7 @@ public class Engine { let pc = state.pc let skip = state.program.skip(pc) state.increasePC(skip + 1) - return hostCall(state: state, callIndex: callIndexInner) + return await hostCall(state: state, callIndex: callIndexInner) default: return .exit(reason) } diff --git a/PolkaVM/Sources/PolkaVM/InvocationContext.swift b/PolkaVM/Sources/PolkaVM/InvocationContext.swift index 5bbb9500..277f7bad 100644 --- a/PolkaVM/Sources/PolkaVM/InvocationContext.swift +++ b/PolkaVM/Sources/PolkaVM/InvocationContext.swift @@ -5,5 +5,5 @@ public protocol InvocationContext { var context: ContextType { get set } /// host-call dispatch function - func dispatch(index: UInt32, state: VMState) -> ExecOutcome + func dispatch(index: UInt32, state: VMState) async -> ExecOutcome } diff --git a/PolkaVM/Sources/PolkaVM/invokePVM.swift b/PolkaVM/Sources/PolkaVM/invokePVM.swift index 9f2e5292..053e34c3 100644 --- a/PolkaVM/Sources/PolkaVM/invokePVM.swift +++ b/PolkaVM/Sources/PolkaVM/invokePVM.swift @@ -12,11 +12,11 @@ public func invokePVM( gas: Gas, argumentData: Data?, ctx: any InvocationContext -) -> (ExitReason, Gas, Data?) { +) async -> (ExitReason, Gas, Data?) { do { let state = try VMState(standardProgramBlob: blob, pc: pc, gas: gas, argumentData: argumentData) let engine = Engine(config: config, invocationContext: ctx) - let exitReason = engine.execute(program: state.program, state: state) + let exitReason = await engine.execute(program: state.program, state: state) switch exitReason { case .outOfGas: diff --git a/RPC/Sources/RPC/Handlers/ChainHandler.swift b/RPC/Sources/RPC/Handlers/ChainHandler.swift index 880d261a..47686eb5 100644 --- a/RPC/Sources/RPC/Handlers/ChainHandler.swift +++ b/RPC/Sources/RPC/Handlers/ChainHandler.swift @@ -38,18 +38,21 @@ struct ChainHandler { throw JSONError(code: -32602, message: "Invalid block hash") } let state = try await source.getState(hash: data32) + guard let state else { + return JSON.null + } // return state root for now - return [ - "stateRoot": state?.stateRoot.description, - "blockHash": hash.description, + return await [ + "stateRoot": state.value.stateRoot.toHexString(), + "blockHash": hash, ] } else { // return best block state by default let block = try await source.getBestBlock() let state = try await source.getState(hash: block.hash) - return [ - "stateRoot": state?.stateRoot.description, - "blockHash": block.hash.description, + return await [ + "stateRoot": state?.value.stateRoot.toHexString(), + "blockHash": block.hash.toHexString(), ] } } diff --git a/Utils/Sources/Utils/Extensions/Int+Utils.swift b/Utils/Sources/Utils/Extensions/Int+Utils.swift new file mode 100644 index 00000000..366f75cc --- /dev/null +++ b/Utils/Sources/Utils/Extensions/Int+Utils.swift @@ -0,0 +1,9 @@ +infix operator %% + +extension Int { + public static func %% (_ left: Int, _ right: Int) -> Int { + if left >= 0 { return left % right } + if left >= -right { return left + right } + return ((left % right) + right) % right + } +} diff --git a/Utils/Sources/Utils/Merklization/StateMerklization.swift b/Utils/Sources/Utils/Merklization/StateMerklization.swift index 1c852b25..b212e6d5 100644 --- a/Utils/Sources/Utils/Merklization/StateMerklization.swift +++ b/Utils/Sources/Utils/Merklization/StateMerklization.swift @@ -10,7 +10,7 @@ public enum MerklizeError: Error { public func stateMerklize(kv: [Data32: Data], i: Int = 0) throws(MerklizeError) -> Data32 { func branch(l: Data32, r: Data32) -> Data64 { var data = l.data + r.data - data[0] = l.data[0] & 0x7F + data[0] = l.data[0] & 0x7F // clear the highest bit return Data64(data)! } diff --git a/Utils/Sources/Utils/SortedContainer/SortedArray.swift b/Utils/Sources/Utils/SortedContainer/SortedArray.swift index 3b86e327..a33aa42f 100644 --- a/Utils/Sources/Utils/SortedContainer/SortedArray.swift +++ b/Utils/Sources/Utils/SortedContainer/SortedArray.swift @@ -44,6 +44,13 @@ public struct SortedArray: SortedContainer { public mutating func remove(where predicate: (T) throws -> Bool) rethrows { try array.removeAll(where: predicate) } + + // mutate access to underlying array directly + // this is unsafe and should be used with care + public var unsafeArrayAccess: [T] { + _read { yield array } + _modify { yield &array } + } } extension SortedArray: Encodable where T: Encodable { diff --git a/Utils/Tests/UtilsTests/Extensions/IntTests.swift b/Utils/Tests/UtilsTests/Extensions/IntTests.swift new file mode 100644 index 00000000..dbe6db04 --- /dev/null +++ b/Utils/Tests/UtilsTests/Extensions/IntTests.swift @@ -0,0 +1,16 @@ +import Testing + +@testable import Utils + +struct IntUtilsTests { + @Test func mod() throws { + #expect((1 %% 5) == 1) + #expect((0 %% 5) == 0) + #expect((-1 %% 5) == 4) + #expect((5 %% 3) == 2) + #expect((-5 %% 3) == 1) + #expect((-1 %% 3) == 2) + #expect((-10 %% 3) == 2) + #expect((-10 %% -3) == -1) + } +}