From 1d19460061fe7fe0291a96c6b84460bd705fd6b3 Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Fri, 27 Dec 2024 11:02:33 +0800 Subject: [PATCH 1/7] refactor pvm memory --- Blockchain/Package.resolved | 9 + .../VMInvocations/HostCall/HostCall.swift | 2 +- .../VMInvocations/HostCall/HostCalls.swift | 23 +- JAMTests/Package.resolved | 13 +- PolkaVM/Package.resolved | 13 +- PolkaVM/Package.swift | 2 + PolkaVM/Sources/PolkaVM/Instruction.swift | 2 +- PolkaVM/Sources/PolkaVM/Memory.swift | 620 ++++++++++++------ PolkaVM/Sources/PolkaVM/StandardProgram.swift | 6 +- PolkaVM/Sources/PolkaVM/VMState.swift | 12 +- .../xcshareddata/swiftpm/Package.resolved | 19 +- 11 files changed, 493 insertions(+), 228 deletions(-) diff --git a/Blockchain/Package.resolved b/Blockchain/Package.resolved index dd4094c8..fdd3bd60 100644 --- a/Blockchain/Package.resolved +++ b/Blockchain/Package.resolved @@ -10,6 +10,15 @@ "version" : "0.2.0" } }, + { + "identity" : "lrucache", + "kind" : "remoteSourceControl", + "location" : "https://github.com/nicklockwood/LRUCache.git", + "state" : { + "revision" : "542f0449556327415409ededc9c43a4bd0a397dc", + "version" : "1.0.7" + } + }, { "identity" : "swift-crypto", "kind" : "remoteSourceControl", diff --git a/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCall.swift b/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCall.swift index 90f0302c..f8fa0d36 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCall.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCall.swift @@ -22,7 +22,7 @@ extension HostCall { do { try await _callImpl(config: config, state: state) return .continued - } catch let e as Memory.Error { + } catch let e as MemoryError { logger.error("memory error: \(e)") return .exit(.pageFault(e.address)) } catch VMInvocationsError.forceHalt { diff --git a/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift b/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift index d09455a3..5cbb5c96 100644 --- a/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift +++ b/Blockchain/Sources/Blockchain/VMInvocations/HostCall/HostCalls.swift @@ -763,7 +763,7 @@ public class Machine: HostCall { let innerVmIndex = UInt64(context.pvms.count) let code = isReadable ? try state.readMemory(address: regs[0], length: Int(regs[1])) : nil let pc = UInt32(truncatingIfNeeded: regs[2]) - let mem = Memory(pageMap: [], chunks: []) + let mem = try GeneralMemory(pageMap: [], chunks: []) if code == nil { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue) @@ -852,7 +852,7 @@ public class Zero: HostCall { self.context = context } - public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { + public func _callImpl(config _: ProtocolConfigRef, state: VMState) async throws { let regs: [UInt64] = state.readRegisters(in: 7 ..< 10) if context.pvms[regs[0]] == nil { @@ -864,11 +864,7 @@ public class Zero: HostCall { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue) } else { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OK.rawValue) - // TODO: update after pvm memory is updated to have pages and able to allocate with accessiblity - try context.pvms[regs[0]]!.memory.write( - address: UInt32(truncatingIfNeeded: regs[1] * UInt64(config.value.pvmMemoryPageSize)), - values: Data(repeating: 0, count: Int(regs[2] * UInt64(config.value.pvmMemoryPageSize))) - ) + try context.pvms[regs[0]]!.memory.zero(pageIndex: UInt32(truncatingIfNeeded: regs[1]), pages: Int(regs[2])) } } } @@ -883,7 +879,7 @@ public class VoidFn: HostCall { self.context = context } - public func _callImpl(config: ProtocolConfigRef, state: VMState) async throws { + public func _callImpl(config _: ProtocolConfigRef, state: VMState) async throws { let regs: [UInt64] = state.readRegisters(in: 7 ..< 10) if context.pvms[regs[0]] == nil { @@ -891,16 +887,13 @@ public class VoidFn: HostCall { return } - // TODO: update when can check if pages are inaccessible - if (regs[1] + regs[2]) >= (1 << 32) { + if (regs[1] + regs[2]) >= (1 << 32) || + !context.pvms[regs[0]]!.memory.isReadable(pageStart: UInt32(truncatingIfNeeded: regs[1]), pages: Int(regs[2])) + { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OOB.rawValue) } else { state.writeRegister(Registers.Index(raw: 7), HostCallResultCode.OK.rawValue) - // TODO: update after pvm memory is updated to have pages and able to allocate with accessiblity - try context.pvms[regs[0]]!.memory.write( - address: UInt32(truncatingIfNeeded: regs[1] * UInt64(config.value.pvmMemoryPageSize)), - values: Data(repeating: 0, count: Int(regs[2] * UInt64(config.value.pvmMemoryPageSize))) - ) + try context.pvms[regs[0]]!.memory.void(pageIndex: UInt32(truncatingIfNeeded: regs[1]), pages: Int(regs[2])) } } } diff --git a/JAMTests/Package.resolved b/JAMTests/Package.resolved index 9520d3bc..8a4e0f98 100644 --- a/JAMTests/Package.resolved +++ b/JAMTests/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "b09552c8dbc0d6746d5c30f4bd3bce579055c1ec4333006ae939b3251732c315", + "originHash" : "d1c16170b6583ee1b137012237d778a580dd5133ce298fe722c0dd6851f6018d", "pins" : [ { "identity" : "blake2.swift", @@ -10,6 +10,15 @@ "version" : "0.2.0" } }, + { + "identity" : "lrucache", + "kind" : "remoteSourceControl", + "location" : "https://github.com/nicklockwood/LRUCache.git", + "state" : { + "revision" : "542f0449556327415409ededc9c43a4bd0a397dc", + "version" : "1.0.7" + } + }, { "identity" : "swift-crypto", "kind" : "remoteSourceControl", @@ -49,7 +58,7 @@ { "identity" : "swift-numerics", "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-numerics.git", + "location" : "https://github.com/apple/swift-numerics", "state" : { "branch" : "main", "revision" : "e30276bff2ff5ed80566fbdca49f50aa160b0e83" diff --git a/PolkaVM/Package.resolved b/PolkaVM/Package.resolved index abeed02f..a57c3ba5 100644 --- a/PolkaVM/Package.resolved +++ b/PolkaVM/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "a91ad177156caefa2e488081e3e97d998b7265492cacff52c8348bf4ef81e168", + "originHash" : "5f5f6662e0a341792801f498df286aec16ade8bbccd3121439d34662fc774385", "pins" : [ { "identity" : "blake2.swift", @@ -10,6 +10,15 @@ "version" : "0.2.0" } }, + { + "identity" : "lrucache", + "kind" : "remoteSourceControl", + "location" : "https://github.com/nicklockwood/LRUCache.git", + "state" : { + "revision" : "542f0449556327415409ededc9c43a4bd0a397dc", + "version" : "1.0.7" + } + }, { "identity" : "swift-crypto", "kind" : "remoteSourceControl", @@ -49,7 +58,7 @@ { "identity" : "swift-numerics", "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-numerics.git", + "location" : "https://github.com/apple/swift-numerics", "state" : { "branch" : "main", "revision" : "e30276bff2ff5ed80566fbdca49f50aa160b0e83" diff --git a/PolkaVM/Package.swift b/PolkaVM/Package.swift index b1c019f6..88e789e1 100644 --- a/PolkaVM/Package.swift +++ b/PolkaVM/Package.swift @@ -19,6 +19,7 @@ let package = Package( .package(path: "../TracingUtils"), .package(url: "https://github.com/apple/swift-testing.git", branch: "0.10.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.6.0"), + .package(url: "https://github.com/nicklockwood/LRUCache.git", from: "1.0.7"), ], targets: [ .target( @@ -27,6 +28,7 @@ let package = Package( "Utils", "TracingUtils", .product(name: "Logging", package: "swift-log"), + .product(name: "LRUCache", package: "LRUCache"), ] ), .testTarget( diff --git a/PolkaVM/Sources/PolkaVM/Instruction.swift b/PolkaVM/Sources/PolkaVM/Instruction.swift index 543d5283..f1979187 100644 --- a/PolkaVM/Sources/PolkaVM/Instruction.swift +++ b/PolkaVM/Sources/PolkaVM/Instruction.swift @@ -35,7 +35,7 @@ extension Instruction { } logger.debug("execution success! updating pc...") return updatePC(context: context, skip: skip) - } catch let e as Memory.Error { + } catch let e as MemoryError { logger.debug("memory error: \(e)") return .exit(.pageFault(e.address)) } catch let e { diff --git a/PolkaVM/Sources/PolkaVM/Memory.swift b/PolkaVM/Sources/PolkaVM/Memory.swift index d4951173..e85d3183 100644 --- a/PolkaVM/Sources/PolkaVM/Memory.swift +++ b/PolkaVM/Sources/PolkaVM/Memory.swift @@ -1,271 +1,505 @@ import Foundation +import LRUCache + +public enum MemoryError: Swift.Error { + case pageFault(UInt32) + case notReadable(UInt32) + case notWritable(UInt32) + case outOfMemory(UInt32) + + public var address: UInt32 { + switch self { + case let .pageFault(address): + address + case let .notWritable(address): + address + case let .outOfMemory(address): + address + case let .notReadable(address): + address + } + } +} -public class MemorySection { - /// lowest address bound - public let startAddressBound: UInt32 - /// highest address bound - public let endAddressBound: UInt32 - /// is the section writable - public let isWritable: Bool - /// allocated data - fileprivate var data: Data - - /// current data end address, also the start address of empty space - public var currentEnd: UInt32 { - startAddressBound + UInt32(data.count) +public enum PageAccess { + case readOnly + case readWrite + case noAccess + + public func isReadable() -> Bool { + switch self { + case .readOnly: + true + case .readWrite: + true + case .noAccess: + false + } } - public init(startAddressBound: UInt32, endAddressBound: UInt32, data: Data, isWritable: Bool) { - self.startAddressBound = startAddressBound - self.endAddressBound = endAddressBound - self.data = data - self.isWritable = isWritable + public func isWritable() -> Bool { + switch self { + case .readWrite: + true + default: + false + } } } -extension MemorySection { - public func read(address: UInt32, length: Int) throws(Memory.Error) -> Data { - guard startAddressBound <= address, address + UInt32(length) < endAddressBound else { - throw Memory.Error.pageFault(address) +public protocol Memory { + var pageMap: PageMap { get } + + func isReadable(address: UInt32, length: Int) -> Bool + func isWritable(address: UInt32, length: Int) -> Bool + func isReadable(pageStart: UInt32, pages: Int) -> Bool + func isWritable(pageStart: UInt32, pages: Int) -> Bool + + func read(address: UInt32) throws -> UInt8 + func read(address: UInt32, length: Int) throws -> Data + func write(address: UInt32, value: UInt8) throws + func write(address: UInt32, values: some Sequence) throws + + func zero(pageIndex: UInt32, pages: Int) throws + func void(pageIndex: UInt32, pages: Int) throws + func sbrk(_ increment: UInt32) throws -> UInt32 +} + +public class PageMap { + private var pageTable: [UInt32: PageAccess] = [:] + private let config: PvmConfig + + // cache for multi page queries + private var isReadableCache: LRUCache, Bool> + private var isWritableCache: LRUCache, Bool> + + public init(pageMap: [(address: UInt32, length: UInt32, access: PageAccess)], config: PvmConfig) { + self.config = config + isReadableCache = LRUCache, Bool>(totalCostLimit: 0, countLimit: 1024) + isWritableCache = LRUCache, Bool>(totalCostLimit: 0, countLimit: 1024) + + for entry in pageMap { + let startIndex = entry.address / UInt32(config.pvmMemoryPageSize) + let pages = StandardProgram.alignToPageSize(size: entry.length, config: config) + + for i in startIndex ..< startIndex + pages { + pageTable[i] = entry.access + } } - let start = address - startAddressBound - let end = start + UInt32(length) + } - let validCount = min(end, UInt32(data.count)) - let dataToRead = data[start ..< validCount] + public func isReadable(pageStart: UInt32, pages: Int) -> Bool { + let pageRange = pageStart ..< pageStart + UInt32(pages) - let zeroCount = max(0, Int(end - validCount)) - let zeros = Data(repeating: 0, count: zeroCount) + let cacheValue = isReadableCache.value(forKey: pageRange) + if let cacheValue { + return cacheValue + } - return dataToRead + zeros + var result = false + for i in pageRange { + result = result || (pageTable[i]?.isReadable() ?? false) + } + isReadableCache.setValue(result, forKey: pageRange) + return result } - public func write(address: UInt32, values: some Sequence) throws(Memory.Error) { - let valuesData = Data(values) - guard isWritable else { - throw Memory.Error.notWritable(address) + public func isReadable(address: UInt32, length: Int) -> Bool { + let startPageIndex = address / UInt32(config.pvmMemoryPageSize) + let pages = StandardProgram.alignToPageSize(size: UInt32(length), config: config) + return isReadable(pageStart: startPageIndex, pages: Int(pages)) + } + + public func isWritable(pageStart: UInt32, pages: Int) -> Bool { + let pageRange = pageStart ..< pageStart + UInt32(pages) + + let cacheValue = isWritableCache.value(forKey: pageRange) + if let cacheValue { + return cacheValue } - guard startAddressBound <= address, address + UInt32(valuesData.count) < endAddressBound else { - throw Memory.Error.notWritable(address) + + var result = false + for i in pageRange { + result = result && (pageTable[i]?.isWritable() ?? false) } + isWritableCache.setValue(result, forKey: pageRange) + return result + } + + public func isWritable(address: UInt32, length: Int) -> Bool { + let startPageIndex = address / UInt32(config.pvmMemoryPageSize) + let pages = StandardProgram.alignToPageSize(size: UInt32(length), config: config) + return isWritable(pageStart: startPageIndex, pages: Int(pages)) + } - let start = address - startAddressBound - let end = start + UInt32(valuesData.count) - guard end < data.count else { - throw Memory.Error.notWritable(address) + public func update(address: UInt32, length: Int, access: PageAccess) { + let startPageIndex = address / UInt32(config.pvmMemoryPageSize) + let pages = StandardProgram.alignToPageSize(size: UInt32(length), config: config) + let pageRange = startPageIndex ..< startPageIndex + pages + + for i in pageRange { + pageTable[i] = access } - data[start ..< end] = valuesData + isReadableCache.removeAllValues() + isWritableCache.removeAllValues() + } + + public func update(pageIndex: UInt32, pages: Int, access: PageAccess) { + for i in pageIndex ..< pageIndex + UInt32(pages) { + pageTable[i] = access + } + isReadableCache.removeAllValues() + isWritableCache.removeAllValues() } } -public class Memory { - public enum Error: Swift.Error { - case pageFault(UInt32) - case notWritable(UInt32) - case outOfMemory(UInt32) - - public var address: UInt32 { - switch self { - case let .pageFault(address): - address - case let .notWritable(address): - address - case let .outOfMemory(address): - address - } +public class MemoryChunk { + public var startAddress: UInt32 + public var endAddress: UInt32 + public private(set) var data: Data + + public init(startAddress: UInt32, endAddress: UInt32, data: Data) { + self.startAddress = startAddress + self.endAddress = endAddress + self.data = data + } + + public func read(address: UInt32, length: Int) throws(MemoryError) -> Data { + guard startAddress <= address, address + UInt32(length) < endAddress else { + throw .pageFault(address) } + let startIndex = address - startAddress + let endIndex = startIndex + UInt32(length) + + let validCount = min(endIndex, UInt32(data.count)) + let dataToRead = data[startIndex ..< validCount] + + let zeroCount = max(0, Int(endIndex - validCount)) + let zeros = Data(repeating: 0, count: zeroCount) + + return dataToRead + zeros } - // standard program sections - private var readOnly: MemorySection? - private var heap: MemorySection? - private var stack: MemorySection? - private var argument: MemorySection? - - // general program sections - private var memorySections: [MemorySection] = [] - - /// General program init with a fixed page map and some initial data - public init(pageMap: [(address: UInt32, length: UInt32, writable: Bool)], chunks: [(address: UInt32, data: Data)]) { - readOnly = nil - heap = nil - stack = nil - argument = nil - memorySections = [] - - let sortedPageMap = pageMap.sorted(by: { $0.address < $1.address }) - let sortedChunks = chunks.sorted(by: { $0.address < $1.address }) - - for (address, length, writable) in sortedPageMap { - var data = Data(repeating: 0, count: Int(length)) - if sortedChunks.count != 0 { - let chunkIndex = Memory.binarySearch(array: sortedChunks.map(\.address), value: address) - let chunk = sortedChunks[chunkIndex] - if address <= chunk.address, chunk.address + UInt32(chunk.data.count) <= address + length { - data = chunk.data - } - } - let section = MemorySection( - startAddressBound: address, - endAddressBound: address + length, - data: data, - isWritable: writable - ) - memorySections.append(section) + public func write(address: UInt32, values: some Sequence) throws(MemoryError) { + let valuesData = Data(values) + guard startAddress <= address, address + UInt32(valuesData.count) < endAddress else { + throw .pageFault(address) + } + + let startIndex = address - startAddress + let endIndex = startIndex + UInt32(valuesData.count) + guard endIndex < data.count else { + throw .notWritable(address) } + + data[startIndex ..< endIndex] = valuesData } +} + +/// Standard Program Memory +public class StandardMemory: Memory { + public private(set) var pageMap: PageMap + private let config: PvmConfig + + private var readOnly: MemoryChunk + private var heap: MemoryChunk + private var stack: MemoryChunk + private var argument: MemoryChunk - /// Standard Program init - /// - /// Init with some read only data, writable data and argument data public init(readOnlyData: Data, readWriteData: Data, argumentData: Data, heapEmptyPagesSize: UInt32, stackSize: UInt32) { let config = DefaultPvmConfig() let P = StandardProgram.alignToPageSize let Q = StandardProgram.alignToZoneSize let ZZ = UInt32(config.pvmProgramInitZoneSize) + let readOnlyLen = UInt32(readOnlyData.count) let readWriteLen = UInt32(readWriteData.count) - let argumentDataLen = UInt32(argumentData.count) let heapStart = 2 * ZZ + Q(readOnlyLen, config) + let heapDataLen = P(readWriteLen, config) + let stackPageAlignedSize = P(stackSize, config) + let stackBaseAddr = UInt32(config.pvmProgramInitStackBaseAddress) - stackPageAlignedSize + + let argumentDataLen = UInt32(argumentData.count) - readOnly = MemorySection( - startAddressBound: ZZ, - endAddressBound: ZZ + P(readOnlyLen, config), - data: readWriteData, - isWritable: false + readOnly = MemoryChunk( + startAddress: ZZ, + endAddress: ZZ + P(readOnlyLen, config), + data: readOnlyData ) - heap = MemorySection( - startAddressBound: heapStart, - endAddressBound: heapStart + P(readWriteLen, config) + heapEmptyPagesSize, - data: readWriteData, - isWritable: true + + heap = MemoryChunk( + startAddress: heapStart, + endAddress: heapStart + heapDataLen + heapEmptyPagesSize, + data: readWriteData ) - stack = MemorySection( - startAddressBound: UInt32(config.pvmProgramInitStackBaseAddress) - stackPageAlignedSize, - endAddressBound: UInt32(config.pvmProgramInitStackBaseAddress), - // TODO: check is this necessary - data: Data(repeating: 0, count: Int(stackPageAlignedSize)), - isWritable: true + stack = MemoryChunk( + startAddress: stackBaseAddr, + endAddress: UInt32(config.pvmProgramInitStackBaseAddress), + data: Data(repeating: 0, count: Int(stackPageAlignedSize)) ) - argument = MemorySection( - startAddressBound: UInt32(config.pvmProgramInitInputStartAddress), - endAddressBound: UInt32(config.pvmProgramInitInputStartAddress) + P(argumentDataLen, config), - data: argumentData, - isWritable: false + argument = MemoryChunk( + startAddress: UInt32(config.pvmProgramInitInputStartAddress), + endAddress: UInt32(config.pvmProgramInitInputStartAddress) + P(argumentDataLen, config), + data: argumentData ) + + pageMap = PageMap(pageMap: [ + (ZZ, P(readOnlyLen, config), .readOnly), + (heapStart, heapDataLen + heapEmptyPagesSize, .readWrite), + (stackBaseAddr, stackPageAlignedSize, .readWrite), + (UInt32(config.pvmProgramInitInputStartAddress), P(argumentDataLen, config), .readOnly), + ], config: config) + + self.config = config + } + + private func getChunk(forAddress: UInt32) throws(MemoryError) -> MemoryChunk { + let address = forAddress & UInt32.max + if address >= readOnly.startAddress, address < readOnly.endAddress { + return readOnly + } else if address >= heap.startAddress, address < heap.endAddress { + return heap + } else if address >= stack.startAddress, address < stack.endAddress { + return stack + } else if address >= argument.startAddress, address < argument.endAddress { + return argument + } + throw .pageFault(address) } - /// if value not in array, return the index of the previous element or 0 - static func binarySearch(array: [UInt32], value: UInt32) -> Int { + public func read(address: UInt32) throws(MemoryError) -> UInt8 { + guard isReadable(address: address, length: 1) else { + throw .notReadable(address) + } + return try getChunk(forAddress: address).read(address: address, length: 1).first ?? 0 + } + + public func read(address: UInt32, length: Int) throws(MemoryError) -> Data { + guard isReadable(address: address, length: length) else { + throw .notReadable(address) + } + return try getChunk(forAddress: address).read(address: address, length: length) + } + + public func write(address: UInt32, value: UInt8) throws(MemoryError) { + guard isWritable(address: address, length: 1) else { + throw .notWritable(address) + } + try getChunk(forAddress: address).write(address: address, values: [value]) + } + + public func write(address: UInt32, values: some Sequence) throws(MemoryError) { + guard isWritable(address: address, length: values.underestimatedCount) else { + throw .notWritable(address) + } + try getChunk(forAddress: address).write(address: address, values: values) + } + + public func zero(pageIndex: UInt32, pages: Int) throws { + let address = pageIndex * UInt32(config.pvmMemoryPageSize) + try getChunk(forAddress: address).write( + address: address, + values: Data(repeating: 0, count: Int(pages * config.pvmMemoryPageSize)) + ) + pageMap.update(pageIndex: pageIndex, pages: pages, access: .readWrite) + } + + public func void(pageIndex: UInt32, pages: Int) throws { + let address = pageIndex * UInt32(config.pvmMemoryPageSize) + try getChunk(forAddress: address).write( + address: address, + values: Data(repeating: 0, count: Int(pages * config.pvmMemoryPageSize)) + ) + pageMap.update(pageIndex: pageIndex, pages: pages, access: .noAccess) + } + + public func sbrk(_ increment: UInt32) throws(MemoryError) -> UInt32 { + let prevHeapEnd = heap.endAddress + guard prevHeapEnd + increment < stack.startAddress else { + throw .outOfMemory(prevHeapEnd) + } + pageMap.update(address: prevHeapEnd, length: Int(increment), access: .readWrite) + heap.endAddress += increment + return prevHeapEnd + } +} + +/// General Program Memory +public class GeneralMemory: Memory { + public private(set) var pageMap: PageMap + private let config: PvmConfig + private var chunks: [MemoryChunk] = [] + + public init(pageMap: [(address: UInt32, length: UInt32, writable: Bool)], chunks: [(address: UInt32, data: Data)]) throws { + let config = DefaultPvmConfig() + self.pageMap = PageMap( + pageMap: pageMap.map { (address: $0.address, length: $0.length, access: $0.writable ? .readWrite : .readOnly) }, + config: config + ) + for chunk in chunks { + try GeneralMemory.insertChunk(address: chunk.address, data: chunk.data, chunks: &self.chunks) + } + self.config = config + } + + // modify chunks array, always merge adjacent chunks, overwrite existing data + private static func insertChunk(address: UInt32, data: Data, chunks: inout [MemoryChunk]) throws { + let newEnd = address + UInt32(data.count) + + // find overlapping chunks + var firstIndex = searchChunk(for: address, in: chunks).index + if firstIndex > 0, chunks[firstIndex - 1].endAddress > address { + firstIndex -= 1 + } + var lastIndex = firstIndex + while lastIndex < chunks.count, chunks[lastIndex].startAddress < newEnd { + lastIndex += 1 + } + + // no overlaps + if firstIndex == lastIndex { + chunks.insert(MemoryChunk(startAddress: address, endAddress: newEnd, data: data), at: firstIndex) + return + } + + // have overlaps + // calculate merged chunk boundaries + let startAddr = min(chunks[firstIndex].startAddress, address) + let endAddr = max(chunks[lastIndex - 1].endAddress, newEnd) + let newChunk = MemoryChunk(startAddress: startAddr, endAddress: endAddr, data: Data()) + // copy existing chunks + for i in firstIndex ..< lastIndex { + let chunk = chunks[i] + try newChunk.write( + address: chunk.startAddress, + values: chunk.data + ) + } + // overwrite existing data with input + try newChunk.write(address: address, values: data) + // replace old chunks + chunks.replaceSubrange(firstIndex ..< lastIndex, with: [newChunk]) + } + + // binary search for the index containing the address or index to to inserted + private static func searchChunk(for address: UInt32, in chunks: [MemoryChunk]) -> (index: Int, found: Bool) { var low = 0 - var high = array.count - 1 - while low <= high { - let mid = (low + high) / 2 - if array[mid] < value { + var high = chunks.endIndex + while low < high { + let mid = low + (high - low) / 2 + if chunks[mid].startAddress < address { low = mid + 1 - } else if array[mid] > value { - high = mid - 1 + } else if chunks[mid].startAddress > address { + high = mid } else { - return mid + return (mid, true) } } - return max(0, low - 1) + return (low, false) } - private func getSection(forAddress: UInt32) throws(Error) -> MemorySection { - let address = forAddress & UInt32.max - if memorySections.count != 0 { - return memorySections[Memory.binarySearch(array: memorySections.map(\.startAddressBound), value: address)] - } else if let readOnly { - if address >= readOnly.startAddressBound, address < readOnly.endAddressBound { - return readOnly - } - } else if let heap { - if address >= heap.startAddressBound, address < heap.endAddressBound { - return heap - } - } else if let stack { - if address >= stack.startAddressBound, address < stack.endAddressBound { - return stack - } - } else if let argument { - if address >= argument.startAddressBound, address < argument.endAddressBound { - return argument - } + private func getChunk(forAddress: UInt32) throws(MemoryError) -> MemoryChunk { + let (index, found) = GeneralMemory.searchChunk(for: forAddress, in: chunks) + if found { + return chunks[index] } - throw Error.pageFault(address) + throw .pageFault(forAddress) } - public func isReadable(address: UInt32, length: Int) -> Bool { - do { - let section = try getSection(forAddress: address) - return section.startAddressBound <= address && address + UInt32(length) < section.endAddressBound - } catch { - return false + public func read(address: UInt32) throws(MemoryError) -> UInt8 { + guard isReadable(address: address, length: 1) else { + throw .notReadable(address) } + return try getChunk(forAddress: address).read(address: address, length: 1).first ?? 0 } - public func read(address: UInt32) throws(Error) -> UInt8 { - try getSection(forAddress: address).read(address: address, length: 1).first ?? 0 + public func read(address: UInt32, length: Int) throws(MemoryError) -> Data { + guard isReadable(address: address, length: length) else { + throw .notReadable(address) + } + return try getChunk(forAddress: address).read(address: address, length: length) } - public func read(address: UInt32, length: Int) throws(Error) -> Data { - try getSection(forAddress: address).read(address: address, length: length) + public func write(address: UInt32, value: UInt8) throws(MemoryError) { + guard isWritable(address: address, length: 1) else { + throw .notWritable(address) + } + try getChunk(forAddress: address).write(address: address, values: [value]) } - public func isWritable(address: UInt32, length: Int) -> Bool { - do { - let section = try getSection(forAddress: address) - return section.isWritable && section.startAddressBound <= address && address + UInt32(length) < section.endAddressBound - } catch { - return false + public func write(address: UInt32, values: some Sequence) throws(MemoryError) { + guard isWritable(address: address, length: values.underestimatedCount) else { + throw .notWritable(address) } + try getChunk(forAddress: address).write(address: address, values: values) } - public func write(address: UInt32, value: UInt8) throws(Error) { - try getSection(forAddress: address).write(address: address, values: Data([value])) + public func zero(pageIndex: UInt32, pages: Int) throws { + let address = pageIndex * UInt32(config.pvmMemoryPageSize) + try GeneralMemory.insertChunk( + address: address, + data: Data(repeating: 0, count: Int(pages * config.pvmMemoryPageSize)), + chunks: &chunks + ) + pageMap.update(pageIndex: pageIndex, pages: pages, access: .readWrite) } - public func write(address: UInt32, values: some Sequence) throws(Error) { - try getSection(forAddress: address).write(address: address, values: values) + public func void(pageIndex: UInt32, pages: Int) throws { + let address = pageIndex * UInt32(config.pvmMemoryPageSize) + try GeneralMemory.insertChunk( + address: address, + data: Data(repeating: 0, count: Int(pages * config.pvmMemoryPageSize)), + chunks: &chunks + ) + pageMap.update(pageIndex: pageIndex, pages: pages, access: .noAccess) } - public func sbrk(_ increment: UInt32) throws -> UInt32 { - var section: MemorySection - if let heap { - section = heap - } else if memorySections.count != 0 { - section = memorySections.last! - } else { - throw Error.pageFault(0) - } - - let oldSectionEnd = section.currentEnd - guard section.isWritable, oldSectionEnd + increment < section.endAddressBound else { - throw Error.outOfMemory(oldSectionEnd) + public func sbrk(_ increment: UInt32) throws(MemoryError) -> UInt32 { + for i in 0 ..< chunks.count - 1 { + if chunks[i].endAddress + increment < chunks[i + 1].startAddress { + let prevEnd = chunks[i].endAddress + chunks[i].endAddress += increment + pageMap.update(address: prevEnd, length: Int(increment), access: .readWrite) + return prevEnd + } } - section.data.append(Data(repeating: 0, count: Int(increment))) - return oldSectionEnd + throw .outOfMemory(chunks.last?.endAddress ?? 0) } } extension Memory { - public class Readonly { - private let memory: Memory + public func isReadable(address: UInt32, length: Int) -> Bool { + pageMap.isReadable(address: address, length: length) + } - public init(_ memory: Memory) { - self.memory = memory - } + public func isWritable(address: UInt32, length: Int) -> Bool { + pageMap.isWritable(address: address, length: length) + } - public func read(address: UInt32) throws -> UInt8 { - try memory.read(address: address) - } + public func isReadable(pageStart: UInt32, pages: Int) -> Bool { + pageMap.isReadable(pageStart: pageStart, pages: pages) + } - public func read(address: UInt32, length: Int) throws -> Data { - try memory.read(address: address, length: length) - } + public func isWritable(pageStart: UInt32, pages: Int) -> Bool { + pageMap.isWritable(pageStart: pageStart, pages: pages) + } +} + +public class ReadonlyMemory { + private let memory: Memory + + public init(_ memory: Memory) { + self.memory = memory + } + + public func read(address: UInt32) throws -> UInt8 { + try memory.read(address: address) + } + + public func read(address: UInt32, length: Int) throws -> Data { + try memory.read(address: address, length: length) } } diff --git a/PolkaVM/Sources/PolkaVM/StandardProgram.swift b/PolkaVM/Sources/PolkaVM/StandardProgram.swift index 37c0feaa..1f7420a8 100644 --- a/PolkaVM/Sources/PolkaVM/StandardProgram.swift +++ b/PolkaVM/Sources/PolkaVM/StandardProgram.swift @@ -59,7 +59,7 @@ public class StandardProgram { initialRegisters = Registers(config: config, argumentData: argumentData) - initialMemory = Memory( + initialMemory = StandardMemory( readOnlyData: readOnlyData, readWriteData: readWriteData, argumentData: argumentData ?? Data(), @@ -68,12 +68,12 @@ public class StandardProgram { ) } - static func alignToPageSize(size: UInt32, config: PvmConfig) -> UInt32 { + public static func alignToPageSize(size: UInt32, config: PvmConfig) -> UInt32 { let pageSize = UInt32(config.pvmMemoryPageSize) return (size + pageSize - 1) / pageSize * pageSize } - static func alignToZoneSize(size: UInt32, config: PvmConfig) -> UInt32 { + public static func alignToZoneSize(size: UInt32, config: PvmConfig) -> UInt32 { let zoneSize = UInt32(config.pvmProgramInitZoneSize) return (size + zoneSize - 1) / zoneSize * zoneSize } diff --git a/PolkaVM/Sources/PolkaVM/VMState.swift b/PolkaVM/Sources/PolkaVM/VMState.swift index 5db7c318..78d1de5a 100644 --- a/PolkaVM/Sources/PolkaVM/VMState.swift +++ b/PolkaVM/Sources/PolkaVM/VMState.swift @@ -36,14 +36,18 @@ public class VMState { gas } - public func getMemory() -> Memory.Readonly { - Memory.Readonly(memory) + public func getMemory() -> ReadonlyMemory { + ReadonlyMemory(memory) } public func getMemoryUnsafe() -> Memory { memory } + public func isMemoryReadable(address: some FixedWidthInteger, length: Int) -> Bool { + memory.isReadable(address: UInt32(truncatingIfNeeded: address), length: length) + } + public func readMemory(address: some FixedWidthInteger) throws -> UInt8 { try memory.read(address: UInt32(truncatingIfNeeded: address)) } @@ -52,10 +56,6 @@ public class VMState { try memory.read(address: UInt32(truncatingIfNeeded: address), length: length) } - public func isMemoryReadable(address: some FixedWidthInteger, length: Int) -> Bool { - memory.isReadable(address: UInt32(truncatingIfNeeded: address), length: length) - } - public func isMemoryWritable(address: some FixedWidthInteger, length: Int) -> Bool { memory.isWritable(address: UInt32(truncatingIfNeeded: address), length: length) } diff --git a/boka.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/boka.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 6da8821a..77a4407b 100644 --- a/boka.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/boka.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "df8d708ee631530dce6fd56811e5100a965c55290ef02b21f0ffdb94c192448c", + "originHash" : "2e1e8e50337213cfabbfa2873f36444e76badd3762318552a0add386e1cc8e55", "pins" : [ { "identity" : "async-channels", @@ -55,6 +55,15 @@ "version" : "1.23.0" } }, + { + "identity" : "lrucache", + "kind" : "remoteSourceControl", + "location" : "https://github.com/nicklockwood/LRUCache.git", + "state" : { + "revision" : "542f0449556327415409ededc9c43a4bd0a397dc", + "version" : "1.0.7" + } + }, { "identity" : "multipart-kit", "kind" : "remoteSourceControl", @@ -247,7 +256,7 @@ { "identity" : "swift-numerics", "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-numerics", + "location" : "https://github.com/apple/swift-numerics.git", "state" : { "branch" : "main", "revision" : "e30276bff2ff5ed80566fbdca49f50aa160b0e83" @@ -292,10 +301,10 @@ { "identity" : "swift-syntax", "kind" : "remoteSourceControl", - "location" : "https://github.com/swiftlang/swift-syntax.git", + "location" : "https://github.com/apple/swift-syntax.git", "state" : { - "revision" : "4c6cc0a3b9e8f14b3ae2307c5ccae4de6167ac2c", - "version" : "600.0.0-prerelease-2024-06-12" + "revision" : "0687f71944021d616d34d922343dcef086855920", + "version" : "600.0.1" } }, { From 45e94f2e02cbeea19df59faa5f8beee220306dda Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Fri, 27 Dec 2024 11:20:13 +0800 Subject: [PATCH 2/7] fix lint --- PolkaVM/Sources/PolkaVM/Memory.swift | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/PolkaVM/Sources/PolkaVM/Memory.swift b/PolkaVM/Sources/PolkaVM/Memory.swift index e85d3183..261a0cd7 100644 --- a/PolkaVM/Sources/PolkaVM/Memory.swift +++ b/PolkaVM/Sources/PolkaVM/Memory.swift @@ -458,13 +458,11 @@ public class GeneralMemory: Memory { } public func sbrk(_ increment: UInt32) throws(MemoryError) -> UInt32 { - for i in 0 ..< chunks.count - 1 { - if chunks[i].endAddress + increment < chunks[i + 1].startAddress { - let prevEnd = chunks[i].endAddress - chunks[i].endAddress += increment - pageMap.update(address: prevEnd, length: Int(increment), access: .readWrite) - return prevEnd - } + for i in 0 ..< chunks.count - 1 where chunks[i].endAddress + increment < chunks[i + 1].startAddress { + let prevEnd = chunks[i].endAddress + chunks[i].endAddress += increment + pageMap.update(address: prevEnd, length: Int(increment), access: .readWrite) + return prevEnd } throw .outOfMemory(chunks.last?.endAddress ?? 0) } From bd9c2eaf480720aad7173ebcdaaf2e82045b557f Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Fri, 27 Dec 2024 13:40:39 +0800 Subject: [PATCH 3/7] improve --- PolkaVM/Sources/PolkaVM/Memory.swift | 91 +++++++++++++++++++++------- 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/PolkaVM/Sources/PolkaVM/Memory.swift b/PolkaVM/Sources/PolkaVM/Memory.swift index 261a0cd7..44547afe 100644 --- a/PolkaVM/Sources/PolkaVM/Memory.swift +++ b/PolkaVM/Sources/PolkaVM/Memory.swift @@ -2,20 +2,26 @@ import Foundation import LRUCache public enum MemoryError: Swift.Error { - case pageFault(UInt32) + case chunkNotFound(UInt32) + case exceedChunkBoundary(UInt32) case notReadable(UInt32) case notWritable(UInt32) case outOfMemory(UInt32) + case notContiguous(UInt32) public var address: UInt32 { switch self { - case let .pageFault(address): + case let .chunkNotFound(address): + address + case let .exceedChunkBoundary(address): + address + case let .notReadable(address): address case let .notWritable(address): address case let .outOfMemory(address): address - case let .notReadable(address): + case let .notContiguous(address): address } } @@ -155,8 +161,8 @@ public class PageMap { } public class MemoryChunk { - public var startAddress: UInt32 - public var endAddress: UInt32 + public private(set) var startAddress: UInt32 + public private(set) var endAddress: UInt32 public private(set) var data: Data public init(startAddress: UInt32, endAddress: UInt32, data: Data) { @@ -167,7 +173,7 @@ public class MemoryChunk { public func read(address: UInt32, length: Int) throws(MemoryError) -> Data { guard startAddress <= address, address + UInt32(length) < endAddress else { - throw .pageFault(address) + throw .exceedChunkBoundary(address) } let startIndex = address - startAddress let endIndex = startIndex + UInt32(length) @@ -184,7 +190,7 @@ public class MemoryChunk { public func write(address: UInt32, values: some Sequence) throws(MemoryError) { let valuesData = Data(values) guard startAddress <= address, address + UInt32(valuesData.count) < endAddress else { - throw .pageFault(address) + throw .exceedChunkBoundary(address) } let startIndex = address - startAddress @@ -195,6 +201,31 @@ public class MemoryChunk { data[startIndex ..< endIndex] = valuesData } + + public func incrementEnd(size increment: UInt32) throws(MemoryError) { + guard endAddress + increment <= UInt32.max else { + throw .outOfMemory(endAddress) + } + endAddress += increment + } + + public func merge(chunk newChunk: MemoryChunk) throws(MemoryError) { + guard newChunk.endAddress <= UInt32.max else { + throw .outOfMemory(endAddress) + } + guard endAddress == newChunk.startAddress else { + throw .notContiguous(newChunk.startAddress) + } + endAddress = newChunk.endAddress + zeroPad() + data.append(newChunk.data) + } + + private func zeroPad() { + if data.count < Int(endAddress - startAddress) { + data.append(Data(repeating: 0, count: Int(endAddress - startAddress) - data.count)) + } + } } /// Standard Program Memory @@ -267,7 +298,7 @@ public class StandardMemory: Memory { } else if address >= argument.startAddress, address < argument.endAddress { return argument } - throw .pageFault(address) + throw .chunkNotFound(address) } public func read(address: UInt32) throws(MemoryError) -> UInt8 { @@ -322,7 +353,7 @@ public class StandardMemory: Memory { throw .outOfMemory(prevHeapEnd) } pageMap.update(address: prevHeapEnd, length: Int(increment), access: .readWrite) - heap.endAddress += increment + try heap.incrementEnd(size: increment) return prevHeapEnd } } @@ -331,6 +362,7 @@ public class StandardMemory: Memory { public class GeneralMemory: Memory { public private(set) var pageMap: PageMap private let config: PvmConfig + // TODO: can be improved by using a more efficient data structure private var chunks: [MemoryChunk] = [] public init(pageMap: [(address: UInt32, length: UInt32, writable: Bool)], chunks: [(address: UInt32, data: Data)]) throws { @@ -346,6 +378,7 @@ public class GeneralMemory: Memory { } // modify chunks array, always merge adjacent chunks, overwrite existing data + // note: caller should rmb to handle corresponding page map updates private static func insertChunk(address: UInt32, data: Data, chunks: inout [MemoryChunk]) throws { let newEnd = address + UInt32(data.count) @@ -370,13 +403,9 @@ public class GeneralMemory: Memory { let startAddr = min(chunks[firstIndex].startAddress, address) let endAddr = max(chunks[lastIndex - 1].endAddress, newEnd) let newChunk = MemoryChunk(startAddress: startAddr, endAddress: endAddr, data: Data()) - // copy existing chunks + // merge existing chunks for i in firstIndex ..< lastIndex { - let chunk = chunks[i] - try newChunk.write( - address: chunk.startAddress, - values: chunk.data - ) + try newChunk.merge(chunk: chunks[i]) } // overwrite existing data with input try newChunk.write(address: address, values: data) @@ -406,7 +435,7 @@ public class GeneralMemory: Memory { if found { return chunks[index] } - throw .pageFault(forAddress) + throw .chunkNotFound(forAddress) } public func read(address: UInt32) throws(MemoryError) -> UInt8 { @@ -438,9 +467,8 @@ public class GeneralMemory: Memory { } public func zero(pageIndex: UInt32, pages: Int) throws { - let address = pageIndex * UInt32(config.pvmMemoryPageSize) try GeneralMemory.insertChunk( - address: address, + address: pageIndex * UInt32(config.pvmMemoryPageSize), data: Data(repeating: 0, count: Int(pages * config.pvmMemoryPageSize)), chunks: &chunks ) @@ -448,9 +476,8 @@ public class GeneralMemory: Memory { } public func void(pageIndex: UInt32, pages: Int) throws { - let address = pageIndex * UInt32(config.pvmMemoryPageSize) try GeneralMemory.insertChunk( - address: address, + address: pageIndex * UInt32(config.pvmMemoryPageSize), data: Data(repeating: 0, count: Int(pages * config.pvmMemoryPageSize)), chunks: &chunks ) @@ -458,9 +485,27 @@ public class GeneralMemory: Memory { } public func sbrk(_ increment: UInt32) throws(MemoryError) -> UInt32 { - for i in 0 ..< chunks.count - 1 where chunks[i].endAddress + increment < chunks[i + 1].startAddress { - let prevEnd = chunks[i].endAddress - chunks[i].endAddress += increment + // find a gap if any + for i in 0 ..< chunks.count - 1 { + let currentChunk = chunks[i] + let nextChunk = chunks[i + 1] + // check if there's enough space between the current and next chunk + if currentChunk.endAddress + increment < nextChunk.startAddress { + let prevEnd = currentChunk.endAddress + try currentChunk.incrementEnd(size: increment) + pageMap.update(address: prevEnd, length: Int(increment), access: .readWrite) + // merge with the next chunk if they become adjacent + if currentChunk.endAddress == nextChunk.startAddress { + try currentChunk.merge(chunk: nextChunk) + chunks.remove(at: i + 1) + } + return prevEnd + } + } + // extend last chunk + if let lastChunk = chunks.last { + let prevEnd = lastChunk.endAddress + try lastChunk.incrementEnd(size: increment) pageMap.update(address: prevEnd, length: Int(increment), access: .readWrite) return prevEnd } From 309fa4f999e648f4ed511beb85264e97a3cbb30f Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Fri, 27 Dec 2024 16:02:49 +0800 Subject: [PATCH 4/7] page map tests --- PolkaVM/Sources/PolkaVM/Memory.swift | 27 +++++--- PolkaVM/Tests/PolkaVMTests/MemoryTests.swift | 71 ++++++++++++++++++++ 2 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 PolkaVM/Tests/PolkaVMTests/MemoryTests.swift diff --git a/PolkaVM/Sources/PolkaVM/Memory.swift b/PolkaVM/Sources/PolkaVM/Memory.swift index 44547afe..2ed31d83 100644 --- a/PolkaVM/Sources/PolkaVM/Memory.swift +++ b/PolkaVM/Sources/PolkaVM/Memory.swift @@ -81,12 +81,12 @@ public class PageMap { public init(pageMap: [(address: UInt32, length: UInt32, access: PageAccess)], config: PvmConfig) { self.config = config - isReadableCache = LRUCache, Bool>(totalCostLimit: 0, countLimit: 1024) - isWritableCache = LRUCache, Bool>(totalCostLimit: 0, countLimit: 1024) + isReadableCache = .init(totalCostLimit: 0, countLimit: 1024) + isWritableCache = .init(totalCostLimit: 0, countLimit: 1024) for entry in pageMap { let startIndex = entry.address / UInt32(config.pvmMemoryPageSize) - let pages = StandardProgram.alignToPageSize(size: entry.length, config: config) + let pages = alignToNumberOfPages(size: entry.length) for i in startIndex ..< startIndex + pages { pageTable[i] = entry.access @@ -94,17 +94,22 @@ public class PageMap { } } + private func alignToNumberOfPages(size: UInt32) -> UInt32 { + let pageSize = UInt32(config.pvmMemoryPageSize) + return (size + pageSize - 1) / pageSize + } + public func isReadable(pageStart: UInt32, pages: Int) -> Bool { + if pages == 0 { return false } let pageRange = pageStart ..< pageStart + UInt32(pages) - let cacheValue = isReadableCache.value(forKey: pageRange) if let cacheValue { return cacheValue } - var result = false + var result = true for i in pageRange { - result = result || (pageTable[i]?.isReadable() ?? false) + result = result && (pageTable[i]?.isReadable() ?? false) } isReadableCache.setValue(result, forKey: pageRange) return result @@ -112,19 +117,19 @@ public class PageMap { public func isReadable(address: UInt32, length: Int) -> Bool { let startPageIndex = address / UInt32(config.pvmMemoryPageSize) - let pages = StandardProgram.alignToPageSize(size: UInt32(length), config: config) + let pages = alignToNumberOfPages(size: UInt32(length)) return isReadable(pageStart: startPageIndex, pages: Int(pages)) } public func isWritable(pageStart: UInt32, pages: Int) -> Bool { + if pages == 0 { return false } let pageRange = pageStart ..< pageStart + UInt32(pages) - let cacheValue = isWritableCache.value(forKey: pageRange) if let cacheValue { return cacheValue } - var result = false + var result = true for i in pageRange { result = result && (pageTable[i]?.isWritable() ?? false) } @@ -134,13 +139,13 @@ public class PageMap { public func isWritable(address: UInt32, length: Int) -> Bool { let startPageIndex = address / UInt32(config.pvmMemoryPageSize) - let pages = StandardProgram.alignToPageSize(size: UInt32(length), config: config) + let pages = alignToNumberOfPages(size: UInt32(length)) return isWritable(pageStart: startPageIndex, pages: Int(pages)) } public func update(address: UInt32, length: Int, access: PageAccess) { let startPageIndex = address / UInt32(config.pvmMemoryPageSize) - let pages = StandardProgram.alignToPageSize(size: UInt32(length), config: config) + let pages = alignToNumberOfPages(size: UInt32(length)) let pageRange = startPageIndex ..< startPageIndex + pages for i in pageRange { diff --git a/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift b/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift new file mode 100644 index 00000000..2b681692 --- /dev/null +++ b/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift @@ -0,0 +1,71 @@ +import Foundation +import Testing +import Utils + +@testable import PolkaVM + +enum MemoryTests { + @Suite struct PageMapTests { + private let config = DefaultPvmConfig() + + @Test func emptyPageMap() { + let pageMap = PageMap(pageMap: [], config: config) + #expect(pageMap.isReadable(pageStart: 0, pages: 1) == false) + #expect(pageMap.isReadable(address: 0, length: 0) == false) + #expect(pageMap.isReadable(address: 0, length: 1) == false) + #expect(pageMap.isReadable(address: 1, length: 1) == false) + #expect(pageMap.isWritable(pageStart: 0, pages: 1) == false) + #expect(pageMap.isWritable(address: 0, length: 0) == false) + #expect(pageMap.isWritable(address: 0, length: 1) == false) + #expect(pageMap.isWritable(address: 1, length: 1) == false) + } + + @Test func initIncompletePage() { + let pageMap = PageMap(pageMap: [(address: 0, length: 1, access: .readOnly)], config: config) + + #expect(pageMap.isReadable(pageStart: 0, pages: 1) == true) + #expect(pageMap.isWritable(pageStart: 0, pages: 1) == false) + + #expect(pageMap.isReadable(address: 0, length: 1) == true) + #expect(pageMap.isReadable(address: UInt32(config.pvmMemoryPageSize) - 1, length: 1) == true) + #expect(pageMap.isReadable(address: UInt32(config.pvmMemoryPageSize), length: 1) == false) + } + + @Test func updatePageMap() { + let pageMap = PageMap( + pageMap: [ + (address: 0, length: UInt32(config.pvmMemoryPageSize), access: .readOnly), + (address: UInt32(config.pvmMemoryPageSize), length: UInt32(config.pvmMemoryPageSize), access: .readOnly), + ], + config: config + ) + + #expect(pageMap.isReadable(pageStart: 0, pages: 1) == true) + #expect(pageMap.isWritable(pageStart: 0, pages: 1) == false) + #expect(pageMap.isReadable(pageStart: 1, pages: 1) == true) + #expect(pageMap.isWritable(pageStart: 1, pages: 1) == false) + + pageMap.update(pageIndex: 1, pages: 1, access: .readWrite) + + #expect(pageMap.isReadable(pageStart: 1, pages: 1) == true) + #expect(pageMap.isWritable(pageStart: 0, pages: 1) == false) + #expect(pageMap.isWritable(pageStart: 1, pages: 1) == true) + + pageMap.update(address: 0, length: config.pvmMemoryPageSize, access: .noAccess) + + #expect(pageMap.isReadable(pageStart: 0, pages: 1) == false) + #expect(pageMap.isWritable(pageStart: 0, pages: 1) == false) + #expect(pageMap.isReadable(pageStart: 1, pages: 1) == true) + } + } + + @Suite struct MemoryChunkTests { + private var config = DefaultPvmConfig() + } + + @Suite struct StandardMemoryTests { + private var config = DefaultPvmConfig() + } + + @Suite struct GeneralMemoryTests {} +} From faeb2f370bfc959a03eab537b9013af6544f4186 Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Fri, 27 Dec 2024 16:42:36 +0800 Subject: [PATCH 5/7] fix var --- PolkaVM/Sources/PolkaVM/Memory.swift | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/PolkaVM/Sources/PolkaVM/Memory.swift b/PolkaVM/Sources/PolkaVM/Memory.swift index 2ed31d83..9f5f8918 100644 --- a/PolkaVM/Sources/PolkaVM/Memory.swift +++ b/PolkaVM/Sources/PolkaVM/Memory.swift @@ -76,8 +76,8 @@ public class PageMap { private let config: PvmConfig // cache for multi page queries - private var isReadableCache: LRUCache, Bool> - private var isWritableCache: LRUCache, Bool> + private let isReadableCache: LRUCache, Bool> + private let isWritableCache: LRUCache, Bool> public init(pageMap: [(address: UInt32, length: UInt32, access: PageAccess)], config: PvmConfig) { self.config = config @@ -235,13 +235,13 @@ public class MemoryChunk { /// Standard Program Memory public class StandardMemory: Memory { - public private(set) var pageMap: PageMap + public let pageMap: PageMap private let config: PvmConfig - private var readOnly: MemoryChunk - private var heap: MemoryChunk - private var stack: MemoryChunk - private var argument: MemoryChunk + private let readOnly: MemoryChunk + private let heap: MemoryChunk + private let stack: MemoryChunk + private let argument: MemoryChunk public init(readOnlyData: Data, readWriteData: Data, argumentData: Data, heapEmptyPagesSize: UInt32, stackSize: UInt32) { let config = DefaultPvmConfig() @@ -365,7 +365,7 @@ public class StandardMemory: Memory { /// General Program Memory public class GeneralMemory: Memory { - public private(set) var pageMap: PageMap + public let pageMap: PageMap private let config: PvmConfig // TODO: can be improved by using a more efficient data structure private var chunks: [MemoryChunk] = [] From 79b4e31096c649c4f8d25b97694568bbcbf8596d Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Tue, 31 Dec 2024 15:51:34 +0800 Subject: [PATCH 6/7] more tests and fixes --- PolkaVM/Sources/PolkaVM/Memory.swift | 136 ++++++---- PolkaVM/Sources/PolkaVM/StandardProgram.swift | 2 +- PolkaVM/Tests/PolkaVMTests/MemoryTests.swift | 234 +++++++++++++++++- 3 files changed, 314 insertions(+), 58 deletions(-) diff --git a/PolkaVM/Sources/PolkaVM/Memory.swift b/PolkaVM/Sources/PolkaVM/Memory.swift index 9f5f8918..1fb43cfa 100644 --- a/PolkaVM/Sources/PolkaVM/Memory.swift +++ b/PolkaVM/Sources/PolkaVM/Memory.swift @@ -1,13 +1,14 @@ import Foundation import LRUCache -public enum MemoryError: Swift.Error { +public enum MemoryError: Error, Equatable { case chunkNotFound(UInt32) case exceedChunkBoundary(UInt32) case notReadable(UInt32) case notWritable(UInt32) case outOfMemory(UInt32) case notContiguous(UInt32) + case invalidChunk(UInt32) public var address: UInt32 { switch self { @@ -23,6 +24,8 @@ public enum MemoryError: Swift.Error { address case let .notContiguous(address): address + case let .invalidChunk(address): + address } } } @@ -170,45 +173,50 @@ public class MemoryChunk { public private(set) var endAddress: UInt32 public private(set) var data: Data - public init(startAddress: UInt32, endAddress: UInt32, data: Data) { + public init(startAddress: UInt32, endAddress: UInt32, data: Data) throws(MemoryError) { + guard startAddress <= endAddress, endAddress - startAddress >= UInt32(data.count) else { + throw .invalidChunk(startAddress) + } self.startAddress = startAddress self.endAddress = endAddress self.data = data } public func read(address: UInt32, length: Int) throws(MemoryError) -> Data { - guard startAddress <= address, address + UInt32(length) < endAddress else { + guard startAddress <= address, address + UInt32(length) <= endAddress else { throw .exceedChunkBoundary(address) } let startIndex = address - startAddress - let endIndex = startIndex + UInt32(length) - let validCount = min(endIndex, UInt32(data.count)) - let dataToRead = data[startIndex ..< validCount] + if startIndex >= data.count { + return Data(repeating: 0, count: length) + } else { + let validCount = min(length, data.count - Int(startIndex)) + let dataToRead = data.count > 0 ? data[startIndex ..< startIndex + UInt32(validCount)] : Data() - let zeroCount = max(0, Int(endIndex - validCount)) - let zeros = Data(repeating: 0, count: zeroCount) + let zeroCount = max(0, length - validCount) + let zeros = Data(repeating: 0, count: zeroCount) - return dataToRead + zeros + return dataToRead + zeros + } } public func write(address: UInt32, values: some Sequence) throws(MemoryError) { let valuesData = Data(values) - guard startAddress <= address, address + UInt32(valuesData.count) < endAddress else { + guard startAddress <= address, address + UInt32(valuesData.count) <= endAddress else { throw .exceedChunkBoundary(address) } let startIndex = address - startAddress let endIndex = startIndex + UInt32(valuesData.count) - guard endIndex < data.count else { - throw .notWritable(address) - } + + try zeroPad(until: startAddress + endIndex) data[startIndex ..< endIndex] = valuesData } public func incrementEnd(size increment: UInt32) throws(MemoryError) { - guard endAddress + increment <= UInt32.max else { + guard UInt32.max - endAddress >= increment else { throw .outOfMemory(endAddress) } endAddress += increment @@ -221,14 +229,18 @@ public class MemoryChunk { guard endAddress == newChunk.startAddress else { throw .notContiguous(newChunk.startAddress) } + try zeroPad() endAddress = newChunk.endAddress - zeroPad() data.append(newChunk.data) } - private func zeroPad() { - if data.count < Int(endAddress - startAddress) { - data.append(Data(repeating: 0, count: Int(endAddress - startAddress) - data.count)) + private func zeroPad(until address: UInt32? = nil) throws(MemoryError) { + let end = address ?? endAddress + guard end >= startAddress, end <= endAddress else { + throw .exceedChunkBoundary(end) + } + if data.count < Int(end - startAddress) { + data.append(Data(repeating: 0, count: Int(end - startAddress) - data.count)) } } } @@ -243,40 +255,40 @@ public class StandardMemory: Memory { private let stack: MemoryChunk private let argument: MemoryChunk - public init(readOnlyData: Data, readWriteData: Data, argumentData: Data, heapEmptyPagesSize: UInt32, stackSize: UInt32) { + public init(readOnlyData: Data, readWriteData: Data, argumentData: Data, heapEmptyPagesSize: UInt32, stackSize: UInt32) throws { let config = DefaultPvmConfig() let P = StandardProgram.alignToPageSize - let Q = StandardProgram.alignToZoneSize + let Z = StandardProgram.alignToZoneSize let ZZ = UInt32(config.pvmProgramInitZoneSize) let readOnlyLen = UInt32(readOnlyData.count) let readWriteLen = UInt32(readWriteData.count) - let heapStart = 2 * ZZ + Q(readOnlyLen, config) - let heapDataLen = P(readWriteLen, config) + let heapStart = 2 * ZZ + Z(readOnlyLen, config) + let heapDataPagesLen = P(readWriteLen, config) let stackPageAlignedSize = P(stackSize, config) - let stackBaseAddr = UInt32(config.pvmProgramInitStackBaseAddress) - stackPageAlignedSize + let stackStartAddr = UInt32(config.pvmProgramInitStackBaseAddress) - stackPageAlignedSize let argumentDataLen = UInt32(argumentData.count) - readOnly = MemoryChunk( + readOnly = try MemoryChunk( startAddress: ZZ, endAddress: ZZ + P(readOnlyLen, config), data: readOnlyData ) - heap = MemoryChunk( + heap = try MemoryChunk( startAddress: heapStart, - endAddress: heapStart + heapDataLen + heapEmptyPagesSize, + endAddress: heapStart + heapDataPagesLen + heapEmptyPagesSize, data: readWriteData ) - stack = MemoryChunk( - startAddress: stackBaseAddr, + stack = try MemoryChunk( + startAddress: stackStartAddr, endAddress: UInt32(config.pvmProgramInitStackBaseAddress), data: Data(repeating: 0, count: Int(stackPageAlignedSize)) ) - argument = MemoryChunk( + argument = try MemoryChunk( startAddress: UInt32(config.pvmProgramInitInputStartAddress), endAddress: UInt32(config.pvmProgramInitInputStartAddress) + P(argumentDataLen, config), data: argumentData @@ -284,16 +296,15 @@ public class StandardMemory: Memory { pageMap = PageMap(pageMap: [ (ZZ, P(readOnlyLen, config), .readOnly), - (heapStart, heapDataLen + heapEmptyPagesSize, .readWrite), - (stackBaseAddr, stackPageAlignedSize, .readWrite), + (heapStart, heapDataPagesLen + heapEmptyPagesSize, .readWrite), + (stackStartAddr, stackPageAlignedSize, .readWrite), (UInt32(config.pvmProgramInitInputStartAddress), P(argumentDataLen, config), .readOnly), ], config: config) self.config = config } - private func getChunk(forAddress: UInt32) throws(MemoryError) -> MemoryChunk { - let address = forAddress & UInt32.max + private func getChunk(address: UInt32) throws(MemoryError) -> MemoryChunk { if address >= readOnly.startAddress, address < readOnly.endAddress { return readOnly } else if address >= heap.startAddress, address < heap.endAddress { @@ -310,42 +321,44 @@ public class StandardMemory: Memory { guard isReadable(address: address, length: 1) else { throw .notReadable(address) } - return try getChunk(forAddress: address).read(address: address, length: 1).first ?? 0 + return try getChunk(address: address).read(address: address, length: 1).first ?? 0 } public func read(address: UInt32, length: Int) throws(MemoryError) -> Data { guard isReadable(address: address, length: length) else { throw .notReadable(address) } - return try getChunk(forAddress: address).read(address: address, length: length) + return try getChunk(address: address).read(address: address, length: length) } public func write(address: UInt32, value: UInt8) throws(MemoryError) { guard isWritable(address: address, length: 1) else { throw .notWritable(address) } - try getChunk(forAddress: address).write(address: address, values: [value]) + try getChunk(address: address).write(address: address, values: [value]) } public func write(address: UInt32, values: some Sequence) throws(MemoryError) { guard isWritable(address: address, length: values.underestimatedCount) else { throw .notWritable(address) } - try getChunk(forAddress: address).write(address: address, values: values) + try getChunk(address: address).write(address: address, values: values) } + // TODO: check whether need this public func zero(pageIndex: UInt32, pages: Int) throws { let address = pageIndex * UInt32(config.pvmMemoryPageSize) - try getChunk(forAddress: address).write( + try getChunk(address: address).write( address: address, values: Data(repeating: 0, count: Int(pages * config.pvmMemoryPageSize)) ) pageMap.update(pageIndex: pageIndex, pages: pages, access: .readWrite) } + // TODO: check whether need this public func void(pageIndex: UInt32, pages: Int) throws { let address = pageIndex * UInt32(config.pvmMemoryPageSize) - try getChunk(forAddress: address).write( + try getChunk(address: address).write( address: address, values: Data(repeating: 0, count: Int(pages * config.pvmMemoryPageSize)) ) @@ -365,6 +378,7 @@ public class StandardMemory: Memory { /// General Program Memory public class GeneralMemory: Memory { + // TODO: check if need paged aligned access for general memory public let pageMap: PageMap private let config: PvmConfig // TODO: can be improved by using a more efficient data structure @@ -382,11 +396,24 @@ public class GeneralMemory: Memory { self.config = config } + // assume chunks is sorted by address // modify chunks array, always merge adjacent chunks, overwrite existing data // note: caller should rmb to handle corresponding page map updates private static func insertChunk(address: UInt32, data: Data, chunks: inout [MemoryChunk]) throws { let newEnd = address + UInt32(data.count) + // new item at last index + if address >= chunks.last?.endAddress ?? UInt32.max { + let chunk = try MemoryChunk(startAddress: address, endAddress: newEnd, data: data) + if chunks.last?.endAddress == address { + try chunks.last?.merge(chunk: chunk) + return + } else { + chunks.append(chunk) + return + } + } + // find overlapping chunks var firstIndex = searchChunk(for: address, in: chunks).index if firstIndex > 0, chunks[firstIndex - 1].endAddress > address { @@ -399,20 +426,19 @@ public class GeneralMemory: Memory { // no overlaps if firstIndex == lastIndex { - chunks.insert(MemoryChunk(startAddress: address, endAddress: newEnd, data: data), at: firstIndex) + try chunks.insert(MemoryChunk(startAddress: address, endAddress: newEnd, data: data), at: firstIndex) return } // have overlaps - // calculate merged chunk boundaries + // calculate overlapping chunk boundaries let startAddr = min(chunks[firstIndex].startAddress, address) let endAddr = max(chunks[lastIndex - 1].endAddress, newEnd) - let newChunk = MemoryChunk(startAddress: startAddr, endAddress: endAddr, data: Data()) - // merge existing chunks + let newChunk = try MemoryChunk(startAddress: startAddr, endAddress: endAddr, data: Data()) for i in firstIndex ..< lastIndex { - try newChunk.merge(chunk: chunks[i]) + try newChunk.write(address: chunks[i].startAddress, values: chunks[i].data) } - // overwrite existing data with input + // lastly, overwrite existing data with input try newChunk.write(address: address, values: data) // replace old chunks chunks.replaceSubrange(firstIndex ..< lastIndex, with: [newChunk]) @@ -424,51 +450,51 @@ public class GeneralMemory: Memory { var high = chunks.endIndex while low < high { let mid = low + (high - low) / 2 - if chunks[mid].startAddress < address { - low = mid + 1 + if chunks[mid].startAddress <= address, address < chunks[mid].endAddress { + return (mid, true) } else if chunks[mid].startAddress > address { high = mid } else { - return (mid, true) + low = mid + 1 } } return (low, false) } - private func getChunk(forAddress: UInt32) throws(MemoryError) -> MemoryChunk { - let (index, found) = GeneralMemory.searchChunk(for: forAddress, in: chunks) + private func getChunk(address: UInt32) throws(MemoryError) -> MemoryChunk { + let (index, found) = GeneralMemory.searchChunk(for: address, in: chunks) if found { return chunks[index] } - throw .chunkNotFound(forAddress) + throw .chunkNotFound(address) } public func read(address: UInt32) throws(MemoryError) -> UInt8 { guard isReadable(address: address, length: 1) else { throw .notReadable(address) } - return try getChunk(forAddress: address).read(address: address, length: 1).first ?? 0 + return try getChunk(address: address).read(address: address, length: 1).first ?? 0 } public func read(address: UInt32, length: Int) throws(MemoryError) -> Data { guard isReadable(address: address, length: length) else { throw .notReadable(address) } - return try getChunk(forAddress: address).read(address: address, length: length) + return try getChunk(address: address).read(address: address, length: length) } public func write(address: UInt32, value: UInt8) throws(MemoryError) { guard isWritable(address: address, length: 1) else { throw .notWritable(address) } - try getChunk(forAddress: address).write(address: address, values: [value]) + try getChunk(address: address).write(address: address, values: [value]) } public func write(address: UInt32, values: some Sequence) throws(MemoryError) { guard isWritable(address: address, length: values.underestimatedCount) else { throw .notWritable(address) } - try getChunk(forAddress: address).write(address: address, values: values) + try getChunk(address: address).write(address: address, values: values) } public func zero(pageIndex: UInt32, pages: Int) throws { diff --git a/PolkaVM/Sources/PolkaVM/StandardProgram.swift b/PolkaVM/Sources/PolkaVM/StandardProgram.swift index 1f7420a8..7ddc373d 100644 --- a/PolkaVM/Sources/PolkaVM/StandardProgram.swift +++ b/PolkaVM/Sources/PolkaVM/StandardProgram.swift @@ -59,7 +59,7 @@ public class StandardProgram { initialRegisters = Registers(config: config, argumentData: argumentData) - initialMemory = StandardMemory( + initialMemory = try StandardMemory( readOnlyData: readOnlyData, readWriteData: readWriteData, argumentData: argumentData ?? Data(), diff --git a/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift b/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift index 2b681692..6f7e3b56 100644 --- a/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift +++ b/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift @@ -61,11 +61,241 @@ enum MemoryTests { @Suite struct MemoryChunkTests { private var config = DefaultPvmConfig() + + @Test func invalidChunk() throws { + #expect(throws: MemoryError.invalidChunk(10)) { try MemoryChunk(startAddress: 10, endAddress: 9, data: Data([])) } + #expect(throws: MemoryError.invalidChunk(0)) { try MemoryChunk(startAddress: 0, endAddress: 0, data: Data([0])) } + #expect(throws: MemoryError.invalidChunk(0)) { try MemoryChunk(startAddress: 0, endAddress: 1, data: Data([0, 0])) } + } + + @Test func read() throws { + let chunk = try MemoryChunk(startAddress: 1, endAddress: 10, data: Data()) + #expect(try chunk.read(address: 1, length: 1) == Data([0])) + #expect(try chunk.read(address: 2, length: 1) == Data([0])) + #expect(try chunk.read(address: 2, length: 0) == Data()) + + #expect(throws: MemoryError.exceedChunkBoundary(0)) { try chunk.read(address: 0, length: 1) } + #expect(throws: MemoryError.exceedChunkBoundary(11)) { try chunk.read(address: 11, length: 0) } + } + + @Test func write() throws { + let chunk = try MemoryChunk(startAddress: 0, endAddress: 10, data: Data()) + try chunk.write(address: 0, values: [1]) + #expect(chunk.data == Data([1])) + try chunk.write(address: 1, values: Data([2])) + #expect(chunk.data == Data([1, 2])) + try chunk.write(address: 1, values: Data([3])) + #expect(chunk.data == Data([1, 3])) + + #expect(throws: MemoryError.exceedChunkBoundary(11)) { try chunk.write(address: 11, values: Data([0])) } + #expect(throws: MemoryError.exceedChunkBoundary(9)) { try chunk.write(address: 9, values: Data([0, 0])) } + } + + @Test func incrementEnd() throws { + let chunk = try MemoryChunk(startAddress: 0, endAddress: UInt32.max - 5, data: Data()) + try chunk.incrementEnd(size: 5) + #expect(chunk.endAddress == UInt32.max) + + #expect(throws: MemoryError.outOfMemory(UInt32.max)) { try chunk.incrementEnd(size: 5) } + } + + @Test func merge() throws { + let chunk1 = try MemoryChunk(startAddress: 0, endAddress: 5, data: Data([1, 2, 3])) + let chunk2 = try MemoryChunk(startAddress: 5, endAddress: 8, data: Data([5, 6, 7])) + try chunk1.merge(chunk: chunk2) + #expect(chunk1.data == Data([1, 2, 3, 0, 0, 5, 6, 7])) + #expect(chunk2.data == Data([5, 6, 7])) + #expect(chunk1.endAddress == 8) + + let chunk3 = try MemoryChunk(startAddress: 10, endAddress: 15, data: Data([4, 5, 6])) + #expect(throws: MemoryError.notContiguous(10)) { try chunk1.merge(chunk: chunk3) } + } } @Suite struct StandardMemoryTests { - private var config = DefaultPvmConfig() + let config = DefaultPvmConfig() + let memory: StandardMemory + + let readOnlyStart: UInt32 + let readOnlyEnd: UInt32 + let heapStart: UInt32 + let heapEnd: UInt32 + let stackStart: UInt32 + let stackEnd: UInt32 + let argumentStart: UInt32 + let argumentEnd: UInt32 + + init() throws { + let readOnlyData = Data([1, 2, 3]) + let readWriteData = Data([4, 5, 6]) + let argumentData = Data([7, 8, 9]) + memory = try StandardMemory( + readOnlyData: readOnlyData, + readWriteData: readWriteData, + argumentData: argumentData, + heapEmptyPagesSize: 100 * UInt32(config.pvmMemoryPageSize), + stackSize: 1024 + ) + readOnlyStart = UInt32(config.pvmProgramInitZoneSize) + readOnlyEnd = UInt32(config.pvmProgramInitZoneSize) + UInt32(config.pvmMemoryPageSize) + heapStart = 2 * UInt32(config.pvmProgramInitZoneSize) + UInt32(config.pvmProgramInitZoneSize) + heapEnd = heapStart + UInt32(config.pvmMemoryPageSize) + 100 * UInt32(config.pvmMemoryPageSize) + stackStart = UInt32(config.pvmProgramInitStackBaseAddress) - UInt32(config.pvmMemoryPageSize) + stackEnd = UInt32(config.pvmProgramInitStackBaseAddress) + argumentStart = UInt32(config.pvmProgramInitInputStartAddress) + argumentEnd = UInt32(config.pvmProgramInitInputStartAddress) + UInt32(config.pvmMemoryPageSize) + } + + @Test func read() throws { + // readonly + #expect(throws: MemoryError.notReadable(0)) { try memory.read(address: 0) } + #expect(throws: MemoryError.notReadable(readOnlyStart - 1)) { try memory.read(address: readOnlyStart - 1) } + #expect(memory.isReadable(address: 0, length: config.pvmProgramInitZoneSize) == false) + #expect(try memory.read(address: readOnlyStart, length: 4) == Data([1, 2, 3, 0])) + #expect(try memory.read(address: readOnlyStart, length: 4) == Data([1, 2, 3, 0])) + #expect(throws: MemoryError.notReadable(readOnlyEnd)) { try memory.read( + address: readOnlyEnd, + length: Int(heapStart - readOnlyEnd) + ) } + + // heap + #expect(memory.isReadable(address: heapStart - 1, length: 1) == false) + #expect(memory.isReadable(address: heapStart, length: Int(heapEnd - heapStart)) == true) + #expect(try memory.read(address: heapStart, length: 4) == Data([4, 5, 6, 0])) + #expect(try memory.read(address: heapEnd - 3, length: 3) == Data([0, 0, 0])) + + // stack + #expect(memory.isReadable(address: stackStart - 1, length: 1) == false) + #expect(memory.isReadable(address: stackStart, length: Int(stackEnd - stackStart)) == true) + #expect(try memory.read(address: stackStart, length: 2) == Data([0, 0])) + #expect(try memory.read(address: stackEnd - 3, length: 3) == Data([0, 0, 0])) + + // argument + #expect(memory.isReadable(address: argumentStart - 1, length: 1) == false) + #expect(memory.isReadable(address: argumentStart, length: Int(argumentEnd - argumentStart)) == true) + #expect(try memory.read(address: argumentStart, length: 4) == Data([7, 8, 9, 0])) + #expect(try memory.read(address: argumentEnd - 3, length: 3) == Data([0, 0, 0])) + #expect(throws: MemoryError.notReadable(argumentEnd)) { try memory.read(address: argumentEnd, length: 1) } + } + + @Test func write() throws { + // readonly + #expect(throws: MemoryError.notWritable(0)) { try memory.write(address: 0, value: 0) } + #expect(throws: MemoryError.notWritable(readOnlyStart - 1)) { try memory.write(address: readOnlyStart - 1, value: 0) } + #expect(memory.isWritable(address: 0, length: config.pvmProgramInitZoneSize) == false) + #expect(throws: MemoryError.notWritable(readOnlyStart)) { try memory.write(address: readOnlyStart, value: 4) } + #expect(try memory.read(address: readOnlyStart, length: 4) == Data([1, 2, 3, 0])) + + // heap + #expect(memory.isWritable(address: heapStart - 1, length: 1) == false) + #expect(memory.isWritable(address: heapStart, length: Int(heapEnd - heapStart)) == true) + try memory.write(address: heapStart, value: 44) + #expect(try memory.read(address: heapStart, length: 4) == Data([44, 5, 6, 0])) + try memory.write(address: heapEnd - 1, value: 1) + #expect(try memory.read(address: heapEnd - 2, length: 2) == Data([0, 1])) + + // stack + #expect(memory.isWritable(address: stackStart - 1, length: 1) == false) + #expect(memory.isWritable(address: stackStart, length: Int(stackEnd - stackStart)) == true) + try memory.write(address: stackStart, value: 1) + #expect(try memory.read(address: stackStart, length: 2) == Data([1, 0])) + try memory.write(address: stackEnd - 2, values: [1, 2]) + #expect(try memory.read(address: stackEnd - 4, length: 4) == Data([0, 0, 1, 2])) + + // argument + #expect(memory.isReadable(address: argumentStart - 1, length: 1) == false) + #expect(memory.isReadable(address: argumentStart, length: Int(argumentEnd - argumentStart)) == true) + #expect(memory.isReadable(address: argumentEnd, length: Int(UInt32.max - argumentEnd)) == false) + #expect(throws: MemoryError.notWritable(argumentStart)) { try memory.write(address: argumentStart, value: 4) } + } + + @Test func sbrk() throws { + #expect(memory.isReadable(address: heapEnd, length: Int(stackEnd - heapEnd)) == false) + #expect(try memory.sbrk(100) == heapEnd) + #expect(memory.isReadable(address: heapEnd, length: config.pvmMemoryPageSize) == true) + #expect(memory.isWritable(address: heapEnd, length: config.pvmMemoryPageSize) == true) + #expect(try memory.sbrk(UInt32(config.pvmMemoryPageSize)) == heapEnd + UInt32(config.pvmMemoryPageSize)) + #expect(memory.isWritable(address: heapEnd, length: config.pvmMemoryPageSize * 2) == true) + #expect(memory.isWritable(address: heapEnd, length: config.pvmMemoryPageSize * 2 + 1) == false) + } } - @Suite struct GeneralMemoryTests {} + @Suite struct GeneralMemoryTests { + let config = DefaultPvmConfig() + let memory: GeneralMemory + + init() throws { + memory = try GeneralMemory( + pageMap: [ + (address: 0, length: UInt32(config.pvmMemoryPageSize), writable: true), + (address: UInt32(config.pvmMemoryPageSize) + 2, length: UInt32(config.pvmMemoryPageSize), writable: false), + (address: UInt32(config.pvmMemoryPageSize) * 4, length: UInt32(config.pvmMemoryPageSize) / 2, writable: true), + ], + chunks: [ + (address: 0, data: Data([1, 2, 3, 4])), + (address: 4, data: Data([5, 6, 7])), + (address: 2048, data: Data([1, 2, 3])), + (address: UInt32(config.pvmMemoryPageSize), data: Data([1, 2, 3])), + ] + ) + } + + @Test func pageMap() throws { + #expect(memory.isReadable(pageStart: 0, pages: 1) == true) + #expect(memory.isReadable(pageStart: 1, pages: 1) == true) + #expect(memory.isReadable(pageStart: 2, pages: 1) == false) + #expect(memory.isReadable(pageStart: 4, pages: 1) == true) + + #expect(memory.isWritable(pageStart: 0, pages: 1) == true) + #expect(memory.isWritable(pageStart: 1, pages: 1) == false) + #expect(memory.isWritable(pageStart: 2, pages: 1) == false) + #expect(memory.isWritable(pageStart: 4, pages: 1) == true) + } + + @Test func read() throws { + #expect(try memory.read(address: 0, length: 4) == Data([1, 2, 3, 4])) + #expect(throws: MemoryError.chunkNotFound(1024)) { try memory.read(address: 1024, length: 4) } + #expect(try memory.read(address: 2048, length: 2) == Data([1, 2])) + #expect(throws: MemoryError.exceedChunkBoundary(2048)) { try memory.read(address: 2048, length: 10) } + } + + @Test func write() throws { + try memory.write(address: 2, values: [9, 8]) + #expect(try memory.read(address: 0, length: 4) == Data([1, 2, 9, 8])) + #expect(throws: MemoryError.notWritable(4096)) { try memory.write(address: 4096, values: [0]) } + } + + @Test func sbrk() throws { + let oldEnd = try memory.sbrk(512) + + #expect(memory.isWritable(address: oldEnd, length: 512) == true) + #expect(memory.isWritable(address: 0, length: Int(oldEnd)) == true) + + try memory.write(address: oldEnd, values: [1, 2, 3]) + #expect(try memory.read(address: oldEnd - 1, length: 5) == Data([7, 1, 2, 3, 0])) + } + + @Test func zero() throws { + #expect(try memory.read(address: 4096, length: 3) == Data([1, 2, 3])) + try memory.zero(pageIndex: 1, pages: 1) + #expect(try memory.read(address: 4096, length: 3) == Data([0, 0, 0])) + + #expect(memory.isReadable(address: 4096 * 2, length: 3) == false) + try memory.zero(pageIndex: 2, pages: 1) + #expect(memory.isReadable(address: 4096 * 2, length: 4096) == true) + #expect(memory.isWritable(address: 4096 * 2, length: 4096) == true) + } + + @Test func void() throws { + #expect(try memory.read(address: 4096, length: 3) == Data([1, 2, 3])) + try memory.void(pageIndex: 1, pages: 1) + #expect(memory.isReadable(address: 4096, length: 4096) == false) + + #expect(memory.isReadable(address: 4096 * 4, length: 3) == true) + #expect(memory.isWritable(address: 4096 * 4, length: 3) == true) + try memory.void(pageIndex: 4, pages: 1) + #expect(memory.isReadable(address: 4096 * 4, length: 4096) == false) + #expect(memory.isWritable(address: 4096 * 4, length: 4096) == false) + } + } } From cb38eb824e801b3a6ccbed31a0d26fbf677dc587 Mon Sep 17 00:00:00 2001 From: Qiwei Yang Date: Tue, 31 Dec 2024 16:38:05 +0800 Subject: [PATCH 7/7] fix sbrk --- PolkaVM/Sources/PolkaVM/Memory.swift | 7 ++++--- PolkaVM/Tests/PolkaVMTests/MemoryTests.swift | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/PolkaVM/Sources/PolkaVM/Memory.swift b/PolkaVM/Sources/PolkaVM/Memory.swift index 1fb43cfa..ec06e116 100644 --- a/PolkaVM/Sources/PolkaVM/Memory.swift +++ b/PolkaVM/Sources/PolkaVM/Memory.swift @@ -367,11 +367,12 @@ public class StandardMemory: Memory { public func sbrk(_ increment: UInt32) throws(MemoryError) -> UInt32 { let prevHeapEnd = heap.endAddress - guard prevHeapEnd + increment < stack.startAddress else { + let incrementAlignToPage = StandardProgram.alignToPageSize(size: increment, config: config) + guard prevHeapEnd + incrementAlignToPage < stack.startAddress else { throw .outOfMemory(prevHeapEnd) } - pageMap.update(address: prevHeapEnd, length: Int(increment), access: .readWrite) - try heap.incrementEnd(size: increment) + pageMap.update(address: prevHeapEnd, length: Int(incrementAlignToPage), access: .readWrite) + try heap.incrementEnd(size: incrementAlignToPage) return prevHeapEnd } } diff --git a/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift b/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift index 6f7e3b56..9f7e4469 100644 --- a/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift +++ b/PolkaVM/Tests/PolkaVMTests/MemoryTests.swift @@ -210,7 +210,7 @@ enum MemoryTests { } @Test func sbrk() throws { - #expect(memory.isReadable(address: heapEnd, length: Int(stackEnd - heapEnd)) == false) + #expect(memory.isReadable(address: heapEnd, length: config.pvmMemoryPageSize) == false) #expect(try memory.sbrk(100) == heapEnd) #expect(memory.isReadable(address: heapEnd, length: config.pvmMemoryPageSize) == true) #expect(memory.isWritable(address: heapEnd, length: config.pvmMemoryPageSize) == true)