Binary serialization for payments data (#739)

This is symmetrical with what we already have for channel data.

The goal is to offer ready-made serializers for library integrators, taking care of backward compatibility when the model is updated, instead of pushing the work to integrators.

Initially I explored a json-based approach in branch [db-type-module](https://github.com/ACINQ/lightning-kmp/tree/db-type-module) (see module `lightning-kmp-db-types`). But it turned out to be tedious and verbose, with a lot of class definitions and boiler plate code to handle model migrations.

These binary codecs are much lighter to write and maintain, and probably faster (although it was not the main objective). The main drawbacks is that the serialized data is not human readable.

Author: pm47

modules/core/src/commonMain/kotlin/fr/acinq/lightning/channel/states/Channel.kt

@@ -17,7 +17,7 @@ import fr.acinq.lightning.channel.Helpers.Closing.getRemotePerCommitmentSecret
 import fr.acinq.lightning.crypto.KeyManager
 import fr.acinq.lightning.db.ChannelClosingType
 import fr.acinq.lightning.logging.*
-import fr.acinq.lightning.serialization.Encryption.from
+import fr.acinq.lightning.serialization.channel.Encryption.from
 import fr.acinq.lightning.transactions.Transactions.TransactionWithInputInfo.ClosingTx
 import fr.acinq.lightning.utils.*
 import fr.acinq.lightning.wire.*

modules/core/src/commonMain/kotlin/fr/acinq/lightning/db/PaymentsDb.kt

@@ -303,6 +303,7 @@ sealed class OutgoingPayment : WalletPayment()
  * @param parts list of partial child payments that have actually been sent.
  * @param status current status of the payment.
  */
+@Suppress("DEPRECATION")
 data class LightningOutgoingPayment(
     override val id: UUID,
     val recipientAmount: MilliSatoshi,
@@ -361,6 +362,7 @@ data class LightningOutgoingPayment(
          * Swap-out payments send a lightning payment to a swap server, which will send an on-chain transaction to a given address.
          * The swap-out fee is taken by the swap server to cover the miner fee.
          */
+        @Deprecated("Legacy trusted swap-out, kept for backwards-compatibility with existing databases.")
         data class SwapOut(val address: String, override val paymentRequest: PaymentRequest, val swapOutFee: Satoshi) : Details() {
             override val paymentHash: ByteVector32 = paymentRequest.paymentHash
         }

modules/core/src/commonMain/kotlin/fr/acinq/lightning/io/Peer.kt

@@ -19,8 +19,8 @@ import fr.acinq.lightning.logging.MDCLogger
 import fr.acinq.lightning.logging.mdc
 import fr.acinq.lightning.logging.withMDC
 import fr.acinq.lightning.payment.*
-import fr.acinq.lightning.serialization.Encryption.from
-import fr.acinq.lightning.serialization.Serialization.DeserializationResult
+import fr.acinq.lightning.serialization.channel.Encryption.from
+import fr.acinq.lightning.serialization.channel.Serialization.DeserializationResult
 import fr.acinq.lightning.transactions.Scripts
 import fr.acinq.lightning.transactions.Transactions
 import fr.acinq.lightning.utils.*

modules/core/src/commonMain/kotlin/fr/acinq/lightning/serialization/InputExtensions.kt

@@ -0,0 +1,55 @@
+package fr.acinq.lightning.serialization
+
+import fr.acinq.bitcoin.*
+import fr.acinq.bitcoin.io.Input
+import fr.acinq.bitcoin.utils.Either
+import fr.acinq.lightning.utils.UUID
+import fr.acinq.lightning.wire.LightningCodecs
+import fr.acinq.lightning.wire.LightningMessage
+
+object InputExtensions {
+
+    fun Input.readNumber(): Long = LightningCodecs.bigSize(this)
+
+    fun Input.readBoolean(): Boolean = read() == 1
+
+    fun Input.readString(): String = readDelimitedByteArray().decodeToString()
+
+    fun Input.readByteVector32(): ByteVector32 = ByteVector32(ByteArray(32).also { read(it, 0, it.size) })
+
+    fun Input.readByteVector64(): ByteVector64 = ByteVector64(ByteArray(64).also { read(it, 0, it.size) })
+
+    fun Input.readPublicKey() = PublicKey(ByteArray(33).also { read(it, 0, it.size) })
+
+    fun Input.readPrivateKey() = PrivateKey(ByteArray(32).also { read(it, 0, it.size) })
+
+    fun Input.readTxId(): TxId = TxId(readByteVector32())
+
+    fun Input.readUuid(): UUID = UUID.fromBytes(ByteArray(16).also { read(it, 0, it.size) })
+
+    fun Input.readDelimitedByteArray(): ByteArray {
+        val size = readNumber().toInt()
+        return ByteArray(size).also { read(it, 0, size) }
+    }
+
+    fun Input.readLightningMessage() = LightningMessage.decode(readDelimitedByteArray())
+
+    fun <T> Input.readCollection(readElem: () -> T): Collection<T> {
+        val size = readNumber()
+        return buildList {
+            repeat(size.toInt()) {
+                add(readElem())
+            }
+        }
+    }
+
+    fun <L, R> Input.readEither(readLeft: () -> L, readRight: () -> R): Either<L, R> = when (read()) {
+        0 -> Either.Left(readLeft())
+        else -> Either.Right(readRight())
+    }
+
+    fun <T : Any> Input.readNullable(readNotNull: () -> T): T? = when (read()) {
+        1 -> readNotNull()
+        else -> null
+    }
+}

modules/core/src/commonMain/kotlin/fr/acinq/lightning/serialization/OutputExtensions.kt

@@ -0,0 +1,71 @@
+package fr.acinq.lightning.serialization
+
+import fr.acinq.bitcoin.*
+import fr.acinq.bitcoin.io.Output
+import fr.acinq.bitcoin.utils.Either
+import fr.acinq.lightning.utils.UUID
+import fr.acinq.lightning.wire.LightningCodecs
+import fr.acinq.lightning.wire.LightningMessage
+
+object OutputExtensions {
+
+    fun Output.writeNumber(o: Number): Unit = LightningCodecs.writeBigSize(o.toLong(), this)
+
+    fun Output.writeBoolean(o: Boolean): Unit = if (o) write(1) else write(0)
+
+    fun Output.writeString(o: String): Unit = writeDelimited(o.encodeToByteArray())
+
+    fun Output.writeByteVector32(o: ByteVector32) = write(o.toByteArray())
+
+    fun Output.writeByteVector64(o: ByteVector64) = write(o.toByteArray())
+
+    fun Output.writePublicKey(o: PublicKey) = write(o.value.toByteArray())
+
+    fun Output.writePrivateKey(o: PrivateKey) = write(o.value.toByteArray())
+
+    fun Output.writeTxId(o: TxId) = write(o.value.toByteArray())
+
+    fun Output.writeUuid(o: UUID) = o.run {
+        // NB: copied from kotlin source code (https://github.com/JetBrains/kotlin/blob/v2.1.0/libraries/stdlib/src/kotlin/uuid/Uuid.kt) in order to be forward compatible
+        fun Long.toByteArray(dst: ByteArray, dstOffset: Int) {
+            for (index in 0 until 8) {
+                val shift = 8 * (7 - index)
+                dst[dstOffset + index] = (this ushr shift).toByte()
+            }
+        }
+        val bytes = ByteArray(16)
+        mostSignificantBits.toByteArray(bytes, 0)
+        leastSignificantBits.toByteArray(bytes, 8)
+        write(bytes)
+    }
+
+    fun Output.writeDelimited(o: ByteArray) {
+        writeNumber(o.size)
+        write(o)
+    }
+
+    fun <T : BtcSerializable<T>> Output.writeBtcObject(o: T): Unit = writeDelimited(o.serializer().write(o))
+
+    fun Output.writeLightningMessage(o: LightningMessage) = writeDelimited(LightningMessage.encode(o))
+
+    fun <T> Output.writeCollection(o: Collection<T>, writeElem: (T) -> Unit) {
+        writeNumber(o.size)
+        o.forEach { writeElem(it) }
+    }
+
+    fun <L, R> Output.writeEither(o: Either<L, R>, writeLeft: (L) -> Unit, writeRight: (R) -> Unit) = when (o) {
+        is Either.Left -> {
+            write(0); writeLeft(o.value)
+        }
+        is Either.Right -> {
+            write(1); writeRight(o.value)
+        }
+    }
+
+    fun <T : Any> Output.writeNullable(o: T?, writeNotNull: (T) -> Unit) = when (o) {
+        is T -> {
+            write(1); writeNotNull(o)
+        }
+        else -> write(0)
+    }
+}

modules/core/src/commonMain/kotlin/fr/acinq/lightning/serialization/Encryption.kt renamed to modules/core/src/commonMain/kotlin/fr/acinq/lightning/serialization/channel/Encryption.kt

@@ -1,4 +1,4 @@
-package fr.acinq.lightning.serialization
+package fr.acinq.lightning.serialization.channel
 
 import fr.acinq.bitcoin.ByteVector32
 import fr.acinq.bitcoin.Crypto

modules/core/src/commonMain/kotlin/fr/acinq/lightning/serialization/Serialization.kt renamed to modules/core/src/commonMain/kotlin/fr/acinq/lightning/serialization/channel/Serialization.kt

@@ -1,18 +1,18 @@
-package fr.acinq.lightning.serialization
+package fr.acinq.lightning.serialization.channel
 
 import fr.acinq.bitcoin.crypto.Pack
 import fr.acinq.lightning.channel.states.PersistedChannelState
 
 object Serialization {
 
     fun serialize(state: PersistedChannelState): ByteArray {
-        return fr.acinq.lightning.serialization.v4.Serialization.serialize(state)
+        return fr.acinq.lightning.serialization.channel.v4.Serialization.serialize(state)
     }
 
     fun deserialize(bin: ByteArray): DeserializationResult {
         return when {
             // v4 uses a 1-byte version discriminator
-            bin[0].toInt() == 4 -> DeserializationResult.Success(fr.acinq.lightning.serialization.v4.Deserialization.deserialize(bin))
+            bin[0].toInt() == 4 -> DeserializationResult.Success(fr.acinq.lightning.serialization.channel.v4.Deserialization.deserialize(bin))
             // v2/v3 use a 4-bytes version discriminator
             Pack.int32BE(bin) == 3 -> DeserializationResult.Success(fr.acinq.lightning.serialization.v3.Serialization.deserialize(bin))
             Pack.int32BE(bin) == 2 -> DeserializationResult.Success(fr.acinq.lightning.serialization.v2.Serialization.deserialize(bin))

modules/core/src/commonMain/kotlin/fr/acinq/lightning/serialization/v2/ChannelState.kt renamed to modules/core/src/commonMain/kotlin/fr/acinq/lightning/serialization/channel/v2/ChannelState.kt

@@ -1,7 +1,6 @@
-package fr.acinq.lightning.serialization.v4
+package fr.acinq.lightning.serialization.channel.v4
 
 import fr.acinq.bitcoin.*
-import fr.acinq.bitcoin.crypto.musig2.IndividualNonce
 import fr.acinq.bitcoin.io.ByteArrayInput
 import fr.acinq.bitcoin.io.Input
 import fr.acinq.bitcoin.io.readNBytes
@@ -13,6 +12,19 @@ import fr.acinq.lightning.blockchain.fee.FeeratePerKw
 import fr.acinq.lightning.channel.*
 import fr.acinq.lightning.channel.states.*
 import fr.acinq.lightning.crypto.ShaChain
+import fr.acinq.lightning.serialization.InputExtensions.readBoolean
+import fr.acinq.lightning.serialization.InputExtensions.readByteVector32
+import fr.acinq.lightning.serialization.InputExtensions.readByteVector64
+import fr.acinq.lightning.serialization.InputExtensions.readCollection
+import fr.acinq.lightning.serialization.InputExtensions.readDelimitedByteArray
+import fr.acinq.lightning.serialization.InputExtensions.readEither
+import fr.acinq.lightning.serialization.InputExtensions.readLightningMessage
+import fr.acinq.lightning.serialization.InputExtensions.readNullable
+import fr.acinq.lightning.serialization.InputExtensions.readNumber
+import fr.acinq.lightning.serialization.InputExtensions.readPublicKey
+import fr.acinq.lightning.serialization.InputExtensions.readString
+import fr.acinq.lightning.serialization.InputExtensions.readTxId
+import fr.acinq.lightning.serialization.common.liquidityads.Deserialization.readLiquidityPurchase
 import fr.acinq.lightning.transactions.*
 import fr.acinq.lightning.transactions.Transactions.TransactionWithInputInfo.*
 import fr.acinq.lightning.utils.UUID
@@ -26,7 +38,7 @@ object Deserialization {
     fun deserialize(bin: ByteArray): PersistedChannelState {
         val input = ByteArrayInput(bin)
         val version = input.read()
-        require(version == Serialization.versionMagic) { "incorrect version $version, expected ${Serialization.versionMagic}" }
+        require(version == Serialization.VERSION_MAGIC) { "incorrect version $version, expected ${Serialization.VERSION_MAGIC}" }
         return input.readPersistedChannelState()
     }
 
@@ -435,31 +447,6 @@ object Deserialization {
         )
     )
 
-    private fun Input.readLiquidityFees(): LiquidityAds.Fees = LiquidityAds.Fees(miningFee = readNumber().sat, serviceFee = readNumber().sat)
-
-    private fun Input.readLiquidityPurchase(): LiquidityAds.Purchase = when (val discriminator = read()) {
-        0x00 -> LiquidityAds.Purchase.Standard(
-            amount = readNumber().sat,
-            fees = readLiquidityFees(),
-            paymentDetails = readLiquidityAdsPaymentDetails()
-        )
-        0x01 -> LiquidityAds.Purchase.WithFeeCredit(
-            amount = readNumber().sat,
-            fees = readLiquidityFees(),
-            feeCreditUsed = readNumber().msat,
-            paymentDetails = readLiquidityAdsPaymentDetails()
-        )
-        else -> error("unknown discriminator $discriminator for class ${LiquidityAds.Purchase::class}")
-    }
-
-    private fun Input.readLiquidityAdsPaymentDetails(): LiquidityAds.PaymentDetails = when (val discriminator = read()) {
-        0x00 -> LiquidityAds.PaymentDetails.FromChannelBalance
-        0x80 -> LiquidityAds.PaymentDetails.FromFutureHtlc(readCollection { readByteVector32() }.toList())
-        0x81 -> LiquidityAds.PaymentDetails.FromFutureHtlcWithPreimage(readCollection { readByteVector32() }.toList())
-        0x82 -> LiquidityAds.PaymentDetails.FromChannelBalanceForFutureHtlc(readCollection { readByteVector32() }.toList())
-        else -> error("unknown discriminator $discriminator for class ${LiquidityAds.PaymentDetails::class}")
-    }
-
     private fun Input.skipLegacyLiquidityLease() {
         readNumber() // amount
         readNumber() // mining fee
@@ -713,8 +700,6 @@ object Deserialization {
 
     private fun Input.readOutPoint(): OutPoint = OutPoint.read(readDelimitedByteArray())
 
-    private fun Input.readTxOut(): TxOut = TxOut.read(readDelimitedByteArray())
-
     private fun Input.readTransaction(): Transaction = Transaction.read(readDelimitedByteArray())
 
     private fun Input.readTransactionWithInputInfo(): Transactions.TransactionWithInputInfo = when (val discriminator = read()) {
@@ -741,47 +726,4 @@ object Deserialization {
         min = FeeratePerKw(readNumber().sat),
         max = FeeratePerKw(readNumber().sat)
     )
-
-    private fun Input.readNumber(): Long = LightningCodecs.bigSize(this)
-
-    private fun Input.readBoolean(): Boolean = read() == 1
-
-    private fun Input.readString(): String = readDelimitedByteArray().decodeToString()
-
-    private fun Input.readByteVector32(): ByteVector32 = ByteVector32(ByteArray(32).also { read(it, 0, it.size) })
-
-    private fun Input.readByteVector64(): ByteVector64 = ByteVector64(ByteArray(64).also { read(it, 0, it.size) })
-
-    private fun Input.readPublicKey() = PublicKey(ByteArray(33).also { read(it, 0, it.size) })
-
-    private fun Input.readTxId(): TxId = TxId(readByteVector32())
-
-    private fun Input.readPublicNonce() = IndividualNonce(ByteArray(66).also { read(it, 0, it.size) })
-
-    private fun Input.readDelimitedByteArray(): ByteArray {
-        val size = readNumber().toInt()
-        return ByteArray(size).also { read(it, 0, size) }
-    }
-
-    private fun Input.readLightningMessage() = LightningMessage.decode(readDelimitedByteArray())
-
-    private fun <T> Input.readCollection(readElem: () -> T): Collection<T> {
-        val size = readNumber()
-        return buildList {
-            repeat(size.toInt()) {
-                add(readElem())
-            }
-        }
-    }
-
-    private fun <L, R> Input.readEither(readLeft: () -> L, readRight: () -> R): Either<L, R> = when (read()) {
-        0 -> Either.Left(readLeft())
-        else -> Either.Right(readRight())
-    }
-
-    private fun <T : Any> Input.readNullable(readNotNull: () -> T): T? = when (read()) {
-        1 -> readNotNull()
-        else -> null
-    }
-
 }