diff --git a/convex-cli/GENESIS.md b/convex-cli/GENESIS.md index 0cec1c371..fad3087db 100644 --- a/convex-cli/GENESIS.md +++ b/convex-cli/GENESIS.md @@ -38,6 +38,11 @@ alias convex="java -jar ~/convex.jar" ### Upload keystore +### Critical public keys: + +Genesis/Admin Key: `0xc1d3b0104d55ddf7680181a46e93422e49e2ea9298e37794860f1ef1128427f7` +Governance key: `0xaE9C747a9730D63Fc16BcccEBd12B5dD4c8fBe1328e9a953025e8C02164Ed5E6` +mikera key: `0x89b5142678bfef7a2245af5ae5b9ab1e10c282b375fa297c5aaeccc48ac97cac` ### Managing with screen diff --git a/convex-cli/src/main/java/convex/cli/mixins/RemotePeerMixin.java b/convex-cli/src/main/java/convex/cli/mixins/RemotePeerMixin.java index 1a7dbed2b..eaddb8597 100644 --- a/convex-cli/src/main/java/convex/cli/mixins/RemotePeerMixin.java +++ b/convex-cli/src/main/java/convex/cli/mixins/RemotePeerMixin.java @@ -43,6 +43,9 @@ public Convex connect() { throw new CLIError("Timeout while attempting to connect to peer: "+hostname,e); } catch (IOException e) { throw new CLIError("IO Error: "+e.getMessage(),e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new CLIError("Connection interrupted",e); } } diff --git a/convex-core/src/main/cvx/convex/asset/multi-token.cvx b/convex-core/src/main/cvx/convex/asset/multi-token.cvx index b19fdbf51..d838237b3 100644 --- a/convex-core/src/main/cvx/convex/asset/multi-token.cvx +++ b/convex-core/src/main/cvx/convex/asset/multi-token.cvx @@ -160,7 +160,7 @@ (assoc rec 1 (assoc os receiver quantity)))] (set-holding *caller* (assoc hs id nrec))) (do ;; create a new record with given offer - (or (get tokens id) (fail "token does not exist")) + (or (get tokens id) (fail :STATE "token does not exist")) (set-holding *caller* {id [0 {receiver quantity}]}))) quantity ;; return value is quantity offered )) @@ -203,7 +203,7 @@ [a b] (let [a (cond a (int a) 0) b (cond b (int b) 0)] - (if (>= a b) (- a b) 0))) + (cond (>= a b) (- a b) 0))) (defn quantity-subset? ^{:callable true} diff --git a/convex-core/src/main/cvx/convex/torus/exchange.cvx b/convex-core/src/main/cvx/convex/torus/exchange.cvx index 5729ee1c3..4e3447fc3 100644 --- a/convex-core/src/main/cvx/convex/torus/exchange.cvx +++ b/convex-core/src/main/cvx/convex/torus/exchange.cvx @@ -8,6 +8,7 @@ ;;;;;;;;;; Imports +(set-controller #2) (import convex.asset :as asset) (import convex.fungible :as fungible) @@ -45,6 +46,8 @@ (def torus ~torus) (def token-balance 0) + (set-controller torus) + (defn -qc [q] (cond (int? q) q ;; base case, quantity should always be an integer diff --git a/convex-core/src/main/java/convex/core/Constants.java b/convex-core/src/main/java/convex/core/Constants.java index c9a4e9665..8aba13405 100644 --- a/convex-core/src/main/java/convex/core/Constants.java +++ b/convex-core/src/main/java/convex/core/Constants.java @@ -19,7 +19,7 @@ public class Constants { /** * Initial timestamp for new States */ - public static final long INITIAL_TIMESTAMP = Instant.parse("2020-12-06T05:08:13.0864Z").toEpochMilli(); + public static final long INITIAL_TIMESTAMP = Instant.parse("2024-12-20T02:21:42.0200Z").toEpochMilli(); // public static final long INITIAL_TIMESTAMP = Instant.parse("2024-12-06T05:08:13.0864Z").toEpochMilli(); /** @@ -38,15 +38,15 @@ public class Constants { public static final long INITIAL_MEMORY_PRICE = 1000000L; /** - * Memory Pool of growth increment 1mb + * Memory Pool of growth increment 40kb / hour i.e. approx. 1mb per day */ - public static final long MEMORY_POOL_GROWTH = 1000000L; + public static final long MEMORY_POOL_GROWTH = 40000L; /** - * Memory Pool of growth interval (once per day). This means regular price drops + * Memory Pool of growth interval (once per hour). This means regular price drops * in memory pool */ - public static final long MEMORY_POOL_GROWTH_INTERVAL = 1000L * 24 * 3600; + public static final long MEMORY_POOL_GROWTH_INTERVAL = 1000L * 3600; /** * Max juice allowable during execution of a single transaction. @@ -161,7 +161,7 @@ public class Constants { * Flag to omit filling in stack traces on validation exceptions. This helps * performance against DoS attacks */ - public static final boolean OMIT_VALIDATION_STACKTRACES = true; + public static final boolean OMIT_VALIDATION_STACKTRACES = false; public static final int PBE_ITERATIONS = 100000; diff --git a/convex-core/src/main/java/convex/core/Result.java b/convex-core/src/main/java/convex/core/Result.java index b9ae37f6c..d9ef9c715 100644 --- a/convex-core/src/main/java/convex/core/Result.java +++ b/convex-core/src/main/java/convex/core/Result.java @@ -129,6 +129,10 @@ public static Result error(Keyword errorCode, AString message) { return error(errorCode,message,null); } + public static Result value(ACell value) { + return create(null,value); + } + public static Result error(Keyword errorCode, String message) { return error(errorCode,Strings.create(message),null); } @@ -320,7 +324,7 @@ public boolean isError() { * @return New Result instance */ - public static Result fromContext(CVMLong id,ResultContext rc) { + public static Result fromContext(ACell id,ResultContext rc) { Context ctx=rc.context; Object result=ctx.getValue(); ACell errorCode=null; @@ -377,6 +381,7 @@ public static Result fromContext(Context ctx) { * @return Updated Result */ public Result withID(ACell id) { + if (Cells.equals(id, getID())) return this; return withValues(values.assoc(ID_POS, id)); } @@ -386,7 +391,9 @@ public Result withID(ACell id) { * @return Result instance representing the exception (will be an error) */ public static Result fromException(Throwable e) { - if (e==null) return Result.error(ErrorCodes.EXCEPTION,Strings.NIL); + if (e==null) { + return Result.error(ErrorCodes.EXCEPTION,Strings.NIL); + } if (e instanceof TimeoutException) { String msg=e.getMessage(); return Result.error(ErrorCodes.TIMEOUT,Strings.create(msg)); @@ -426,6 +433,12 @@ public static Result fromException(Throwable e) { // Note interrupts are always caused by CLIENT from a local perspective private static final Result INTERRUPTED_RESULT=Result.error(ErrorCodes.INTERRUPTED,Strings.create("Interrupted!")).withSource(SourceCodes.CLIENT); private static final Result MISSING_RESULT=Result.error(ErrorCodes.MISSING,Strings.create("Missing Data!")).withSource(SourceCodes.CLIENT); + public static final Result CLOSED_CONNECTION = Result.error(ErrorCodes.CONNECT,Strings.create("Connection Closed")).withSource(SourceCodes.COMM); + public static final Result SENT_MESSAGE = Result.value(Strings.intern("Sent")); + public static final Result FULL_CLIENT_BUFFER = Result.error(ErrorCodes.LOAD, Strings.FULL_BUFFER).withSource(SourceCodes.COMM); + public static final Result BAD_FORMAT = Result.error(ErrorCodes.FORMAT, "Bad format"); + + /** * Returns a Result representing a thread interrupt, AND sets the interrupt status on the current thread @@ -436,6 +449,8 @@ private static Result interruptThread() { return INTERRUPTED_RESULT; } + + /** * Converts this result to a JSON representation. WARNING: some information may be lost because JSON is a terrible format. * @@ -461,6 +476,7 @@ public HashMap toJSON() { private static final StringShort RESULT_TAG=StringShort.create("#Result"); + @Override public boolean print(BlobBuilder sb, long limit) { sb.append(RESULT_TAG); @@ -532,6 +548,11 @@ public static Result fromJSON(Object json) { return Result.create(id, value, errorCode); } + public static ACell peekResultID(Blob messageData, int i) throws BadFormatException { + Result r=Result.read(messageData, i); + return r.getID(); + } + } diff --git a/convex-core/src/main/java/convex/core/cpos/Belief.java b/convex-core/src/main/java/convex/core/cpos/Belief.java index c0de24317..f014e13db 100644 --- a/convex-core/src/main/java/convex/core/cpos/Belief.java +++ b/convex-core/src/main/java/convex/core/cpos/Belief.java @@ -237,7 +237,7 @@ public static Collection> extractOrders(ACell payload) { @Override public void validateStructure() throws InvalidDataException { super.validateStructure(); - if (!(values.get(IX_ORDERS) instanceof Index)) { + if (!(values.get(IX_ORDERS) instanceof Index)) { throw new InvalidDataException("Orders should be an Index",this); } } @@ -248,20 +248,25 @@ public void validateStructure() throws InvalidDataException { * @param signedBlock Signed Block of transactions * @return Updated Belief with new Order */ - public Belief proposeBlock(AKeyPair kp, SignedData signedBlock) { + @SuppressWarnings("unchecked") + public Belief proposeBlock(AKeyPair kp, SignedData... signedBlocks) { AccountKey peerKey=kp.getAccountKey(); Index> orders = getOrders(); SignedData mySO=orders.get(peerKey); Order myOrder; if (mySO==null) { - myOrder=Order.create(); + throw new IllegalStateException("Trying to propose block without a current ordering for peer "+peerKey); } else { myOrder=mySO.getValue(); } // Create new order with signed Block - Order newOrder = myOrder.append(signedBlock); + Order newOrder = myOrder; + int n=signedBlocks.length; + for (int i=0; i newSignedOrder = kp.signData(newOrder); Index> newOrders = orders.assoc(peerKey, newSignedOrder); diff --git a/convex-core/src/main/java/convex/core/cpos/BeliefMerge.java b/convex-core/src/main/java/convex/core/cpos/BeliefMerge.java index d6f8daaa7..99a754fdc 100644 --- a/convex-core/src/main/java/convex/core/cpos/BeliefMerge.java +++ b/convex-core/src/main/java/convex/core/cpos/BeliefMerge.java @@ -428,9 +428,7 @@ private final AVector> appendNewBlocks(AVector> newBlocks = new HashSet<>(); newBlocks.addAll(newBlocksOrdered); - // exclude new blocks already in the base Order - // TODO: what about blocks already in consensus? - // Probably need to check last block time from Peer + // exclude new blocks already in the base consensus Order long scanStart=Math.min(blocks.count(), consensusPoint); Iterator> it = blocks.listIterator(scanStart); while (it.hasNext()) { @@ -483,7 +481,8 @@ public Long apply(Order c) { // in order to sort by length of matched proposals long blockMatch = proposedBlocks.commonPrefixLength(c.getBlocks()); - long minPrevious = Math.min(winnningOrder.getConsensusPoint(level-1), c.getConsensusPoint(level-1)); + int prevLevel=level-1; + long minPrevious = Math.min(winnningOrder.getConsensusPoint(prevLevel), c.getConsensusPoint(prevLevel)); // Match length is how many blocks agree with winning order at previous consensus level long match = Math.min(blockMatch, minPrevious); @@ -561,22 +560,19 @@ public static boolean compareOrders(Order oldOrder, Order newOrder) { // new Order is more recent, so switch to this return true; } else { + // timestamps are the same + // Don't replace if equal if (oldOrder.equals(newOrder)) return false; // This probably shouldn't happen if peers are sticking to timestamps // But we compare anyway - // Prefer advanced consensus - for (int level=CPoSConstants.CONSENSUS_LEVELS-1; level>=1; level--) { + // Prefer more blocks / advanced consensus + for (int level=0; leveloldOrder.getConsensusPoint(level)) return true; } - - // Finally prefer more blocks - AVector> abs=oldOrder.getBlocks(); - AVector> bbs=newOrder.getBlocks(); - if(abs.count()> transactions; + + private static final int IX_TIMESTAMP= 0; + private static final int IX_TRANSACTIONS = 1; + + private static final long NUM_FIELDS = FORMAT.count(); /** * Comparator to sort blocks by timestamp @@ -48,7 +54,10 @@ public final class Block extends ARecordGeneric { static final Comparator> TIMESTAMP_COMPARATOR = new Comparator<>() { @Override public int compare(SignedData a, SignedData b) { - int sig = Long.compare(a.getValue().getTimeStamp(), b.getValue().getTimeStamp()); + Block ba=a.getValue(); + Block bb=b.getValue(); + + int sig = Long.compare(ba.getTimeStamp(), bb.getTimeStamp()); return sig; } }; @@ -59,15 +68,16 @@ private Block(long timestamp, AVector> transactions) { this.transactions = transactions; } - public Block(AVector values) { + private Block(AVector values) { super(CVMTag.BLOCK,FORMAT,values); - this.timestamp=RT.ensureLong(values.get(0)).longValue(); + this.timestamp=RT.ensureLong(values.get(IX_TIMESTAMP)).longValue(); + } @Override public ACell get(Keyword k) { - if (Keywords.TIMESTAMP.equals(k)) return CVMLong.create(timestamp); - if (Keywords.TRANSACTIONS.equals(k)) return transactions; + if (Keywords.TIMESTAMP.equals(k)) return values.get(IX_TIMESTAMP); + if (Keywords.TRANSACTIONS.equals(k)) return getTransactions(); return null; } @@ -151,7 +161,7 @@ public static Block read(Blob b, int pos) throws BadFormatException { * @return Vector of transactions */ public AVector> getTransactions() { - if (transactions==null) transactions=RT.ensureVector(values.get(1)); + if (transactions==null) transactions=RT.ensureVector(values.get(IX_TRANSACTIONS)); return transactions; } @@ -162,7 +172,18 @@ public boolean isCanonical() { @Override public void validateCell() throws InvalidDataException { - // nothing to do + if (values.count()!=NUM_FIELDS) throw new InvalidDataException("Wrong field count",this); + } + + @Override + public void validateStructure() throws InvalidDataException { + AVector> txs=getTransactions(); + if (txs==null) throw new InvalidDataException("No transactions",this); + if (txs.count()>Constants.MAX_TRANSACTIONS_PER_BLOCK) { + throw new InvalidDataException("Too many transactions: "+txs.count(),this); + } + // We don't validate individual transactions here + // This gets enforced latter when transactions are applied } @Override diff --git a/convex-core/src/main/java/convex/core/cpos/CPoSConstants.java b/convex-core/src/main/java/convex/core/cpos/CPoSConstants.java index dc7534370..f207e88cb 100644 --- a/convex-core/src/main/java/convex/core/cpos/CPoSConstants.java +++ b/convex-core/src/main/java/convex/core/cpos/CPoSConstants.java @@ -50,7 +50,7 @@ public class CPoSConstants { public static final long INITIAL_PEER_TIMESTAMP = -1L; /** - * Minimum stake for a Peer to be considered by other Peers in consensus + * Minimum stake balance for a Peer to be considered by other Peers in consensus */ public static final long MINIMUM_EFFECTIVE_STAKE = Coin.GOLD * 1000; /** @@ -69,8 +69,15 @@ public class CPoSConstants { /** * Maximum allowed number of missing hashes in missing data request + * + * (2 header values short of 256, so that request vector is 2 levels at max size) */ - public static final long MISSING_LIMIT = 256; + public static final int MISSING_LIMIT = 254; + + /** + * Milliseconds time between blocks for a peer to collect maximum rewards (10 mins) + */ + public static final long MAX_REWARD_TIME = 10*60*1000; } diff --git a/convex-core/src/main/java/convex/core/cpos/Order.java b/convex-core/src/main/java/convex/core/cpos/Order.java index 057ceb46c..31f39a8f2 100644 --- a/convex-core/src/main/java/convex/core/cpos/Order.java +++ b/convex-core/src/main/java/convex/core/cpos/Order.java @@ -37,6 +37,8 @@ public class Order extends ARecordGeneric { private static final int IX_TIMESTAMP = 0; private static final int IX_CONSENSUS = 1; private static final int IX_BLOCKS = 2; + + private static final long NUM_FIELDS=FORMAT.count(); /** * Timestamp of this Order, i.e. the timestamp of the peer at the time it was created @@ -58,7 +60,7 @@ private Order(AVector values) { this.consensusPoints = RT.toLongArray((AVector)values.get(IX_CONSENSUS)); } - private Order(AVector> blocks, long[] consensusPoints, long timestamp) { + private Order(long timestamp, long[] consensusPoints, AVector> blocks) { super(CVMTag.ORDER,FORMAT,Vectors.create(CVMLong.create(timestamp),Vectors.createLongs(consensusPoints),blocks)); this.timestamp = timestamp; this.consensusPoints=consensusPoints; @@ -77,7 +79,7 @@ private static Order create(AVector> blocks, long proposalPoin consensusPoints[1] = proposalPoint; consensusPoints[2] = consensusPoint; - return new Order(blocks, consensusPoints,timestamp); + return new Order(timestamp, consensusPoints,blocks); } /** @@ -99,7 +101,7 @@ public static Order create(long proposalPoint, long consensusPoint, SignedData values = Vectors.read(b, pos); + if (values.count()!=NUM_FIELDS) throw new BadFormatException("Wrong number of Order fields"); long epos=pos+values.getEncodingLength(); Order result=new Order(values); diff --git a/convex-core/src/main/java/convex/core/cvm/ARecordGeneric.java b/convex-core/src/main/java/convex/core/cvm/ARecordGeneric.java index f72699518..318c78ae4 100644 --- a/convex-core/src/main/java/convex/core/cvm/ARecordGeneric.java +++ b/convex-core/src/main/java/convex/core/cvm/ARecordGeneric.java @@ -124,7 +124,7 @@ protected void validateCell() throws InvalidDataException { Cells.validateCell(values); } - protected void validateStructure() throws InvalidDataException { + public void validateStructure() throws InvalidDataException { super.validateStructure(); if (values.count()!=format.count()) { throw new InvalidDataException("Expected "+format.count()+ "Record values but was: "+values.count(),this); diff --git a/convex-core/src/main/java/convex/core/cvm/CVMEncoder.java b/convex-core/src/main/java/convex/core/cvm/CVMEncoder.java index 4cd44499e..8d4465277 100644 --- a/convex-core/src/main/java/convex/core/cvm/CVMEncoder.java +++ b/convex-core/src/main/java/convex/core/cvm/CVMEncoder.java @@ -23,7 +23,10 @@ public ACell read(Blob encoding,int offset) throws BadFormatException { protected ACell readExtension(byte tag, Blob blob, int offset) throws BadFormatException { // We expect a VLQ Count following the tag long code=Format.readVLQCount(blob,offset+1); - if (tag == CVMTag.CORE_DEF) return Core.fromCode(code); + if (tag == CVMTag.CORE_DEF) { + ACell cc=Core.fromCode(code); + if (cc!=null) return cc; + } if (tag == CVMTag.ADDRESS) return Address.create(code); return ExtensionValue.create(tag, code); diff --git a/convex-core/src/main/java/convex/core/cvm/Peer.java b/convex-core/src/main/java/convex/core/cvm/Peer.java index 52c938ff6..493ba09f8 100644 --- a/convex-core/src/main/java/convex/core/cvm/Peer.java +++ b/convex-core/src/main/java/convex/core/cvm/Peer.java @@ -593,6 +593,7 @@ public BlockResult getBlockResult(long i) { public Peer proposeBlock(Block block) { SignedData signedBlock=sign(block); + @SuppressWarnings("unchecked") Belief newBelief=belief.proposeBlock(keyPair, signedBlock); Peer result=this; @@ -661,4 +662,18 @@ public long getStatePosition() { public State getGenesisState() { return genesis; } + + /** + * Checks if the Peer is ready to publish a Block. Requires sufficient stake + * @return true if ready to publish, false otherwise + */ + public boolean isReadyToPublish() { + // If we are the only peer, always allow publishing + if (state.getPeers().count()<=1) return true; + + PeerStatus ps=state.getPeer(peerKey); + if (ps==null) return false; + if (ps.getBalance() EMPTY_STAKES = Index.none(); + private final int IX_TIMESTAMP=5; + /** * Per controller address */ @@ -87,7 +89,7 @@ public PeerStatus(AVector values) { this.controller = RT.ensureAddress(values.get(0)); this.peerStake = RT.ensureLong(values.get(1)).longValue(); this.delegatedStake = RT.ensureLong(values.get(3)).longValue(); - this.timestamp = RT.ensureLong(values.get(5)).longValue(); + this.timestamp = RT.ensureLong(values.get(IX_TIMESTAMP)).longValue(); this.balance = RT.ensureLong(values.get(6)).longValue(); } @@ -263,9 +265,9 @@ private PeerStatus withBalance(long newBalance) { return new PeerStatus(values.assoc(6,CVMLong.create(newBalance))); } - private PeerStatus withTimestamp(CVMLong newTimestamp) { - if (timestamp==newTimestamp.longValue()) return this; - return new PeerStatus(values.assoc(5,newTimestamp)); + private PeerStatus withTimestamp(long newTimestamp) { + if (timestamp==newTimestamp) return this; + return new PeerStatus(values.assoc(IX_TIMESTAMP,CVMLong.create(newTimestamp))); } /** @@ -352,7 +354,7 @@ public PeerStatus distributeBlockReward(State state, long peerFees) { CVMLong timestamp=state.getTimestamp(); long newTime=timestamp.longValue(); if (oldTime0) { - state=state.putAccount(Address.ZERO,rewardPool.withBalance(rewardPool.getBalance()+fees)); + state=state.putAccount(Address.ZERO,rewardPool.withBalance(poolBalance+fees-timeReward)); } return state; } diff --git a/convex-core/src/main/java/convex/core/cvm/Syntax.java b/convex-core/src/main/java/convex/core/cvm/Syntax.java index 122e271c1..e7607ba56 100644 --- a/convex-core/src/main/java/convex/core/cvm/Syntax.java +++ b/convex-core/src/main/java/convex/core/cvm/Syntax.java @@ -243,8 +243,8 @@ public void validateStructure() throws InvalidDataException { if (datum instanceof Syntax) { throw new InvalidDataException("Cannot double-wrap a Syntax value",this); } - if (!datum.isCVMValue()) throw new InvalidDataException("Syntax can only wrap CVM values",this); } + meta.validateStructure(); } @Override diff --git a/convex-core/src/main/java/convex/core/cvm/transactions/ATransaction.java b/convex-core/src/main/java/convex/core/cvm/transactions/ATransaction.java index 07d28f418..bf1ba51d7 100644 --- a/convex-core/src/main/java/convex/core/cvm/transactions/ATransaction.java +++ b/convex-core/src/main/java/convex/core/cvm/transactions/ATransaction.java @@ -33,12 +33,15 @@ public abstract class ATransaction extends ARecordGeneric { protected final Address origin; protected final long sequence; + + protected static final int IX_ORIGIN=0; + protected static final int IX_SEQUENCE=1; protected ATransaction(byte tag,RecordFormat format, AVector values) { super(tag,format,values); - this.origin=RT.ensureAddress(values.get(0)); + this.origin=RT.ensureAddress(values.get(IX_ORIGIN)); if (origin==null) throw new IllegalArgumentException("Null Origin Address for transaction"); - this.sequence = RT.ensureLong(values.get(1)).longValue(); + this.sequence = RT.ensureLong(values.get(IX_SEQUENCE)).longValue(); } /** @@ -85,7 +88,7 @@ public AType getType() { @Override public ACell get(Keyword key) { if (Keywords.ORIGIN.equals(key)) return origin; - if (Keywords.SEQUENCE.equals(key)) return values.get(1); + if (Keywords.SEQUENCE.equals(key)) return values.get(IX_SEQUENCE); return null; } diff --git a/convex-core/src/main/java/convex/core/cvm/transactions/Multi.java b/convex-core/src/main/java/convex/core/cvm/transactions/Multi.java index f00fc2d37..d5c3f412e 100644 --- a/convex-core/src/main/java/convex/core/cvm/transactions/Multi.java +++ b/convex-core/src/main/java/convex/core/cvm/transactions/Multi.java @@ -60,11 +60,15 @@ public class Multi extends ATransaction { * Transactions beyond the first failure will not be attempted. */ public static final int MODE_UNTIL=3; - + private static final Keyword[] KEYS = new Keyword[] { Keywords.ORIGIN, Keywords.SEQUENCE,Keywords.MODE,Keywords.TXS}; private static final RecordFormat FORMAT = RecordFormat.of(KEYS); + // Record key indexes + protected static final int IX_MODE=2; + protected static final int IX_TXS=3; + protected Multi(Address origin, long sequence, int mode, AVector txs) { super(CVMTag.MULTI,FORMAT, Vectors.create(origin,CVMLong.create(sequence),CVMLong.create(mode),txs)); @@ -74,7 +78,7 @@ protected Multi(Address origin, long sequence, int mode, AVector t protected Multi(AVector values) { super(CVMTag.MULTI,FORMAT, values); - this.mode=Utils.checkedInt(RT.ensureLong(values.get(2)).longValue()); + this.mode=Utils.checkedInt(RT.ensureLong(values.get(IX_MODE)).longValue()); if (!isValidMode(mode)) throw new IllegalArgumentException("Bad mode"); } @@ -142,7 +146,7 @@ public Context apply(Context ctx) { private AVector getTransactions() { if (txs==null) { - txs=RT.ensureVector(values.get(3)); + txs=RT.ensureVector(values.get(IX_TXS)); } return txs; } @@ -179,13 +183,12 @@ public ATransaction withOrigin(Address newAddress) { @Override public void validateCell() throws InvalidDataException { - // TODO Auto-generated method stub if ((modeMODE_UNTIL)) throw new InvalidDataException("Illegal mode: "+mode,this); } @Override public ACell get(Keyword key) { - if (Keywords.MODE.equals(key)) return CVMLong.create(mode); + if (Keywords.MODE.equals(key)) return values.get(IX_MODE); if (Keywords.TXS.equals(key)) return getTransactions(); return super.get(key); // covers origin and sequence } diff --git a/convex-core/src/main/java/convex/core/data/AArrayBlob.java b/convex-core/src/main/java/convex/core/data/AArrayBlob.java index 2ab25693e..aeea51e53 100644 --- a/convex-core/src/main/java/convex/core/data/AArrayBlob.java +++ b/convex-core/src/main/java/convex/core/data/AArrayBlob.java @@ -370,7 +370,8 @@ public void validateCell() throws InvalidDataException { } @Override - protected void validateStructure() { + public void validateStructure() throws InvalidDataException { + super.validateStructure(); // nothing to do by default } diff --git a/convex-core/src/main/java/convex/core/data/ACAD3Record.java b/convex-core/src/main/java/convex/core/data/ACAD3Record.java index 55d5a6fbc..dd7a4248a 100644 --- a/convex-core/src/main/java/convex/core/data/ACAD3Record.java +++ b/convex-core/src/main/java/convex/core/data/ACAD3Record.java @@ -31,7 +31,8 @@ public void validateCell() throws InvalidDataException { } @Override - protected void validateStructure() throws InvalidDataException { + public void validateStructure() throws InvalidDataException { + super.validateStructure(); // Nothing to do, any child refs are valid } diff --git a/convex-core/src/main/java/convex/core/data/ACell.java b/convex-core/src/main/java/convex/core/data/ACell.java index 1461acf69..94ea878b5 100644 --- a/convex-core/src/main/java/convex/core/data/ACell.java +++ b/convex-core/src/main/java/convex/core/data/ACell.java @@ -52,7 +52,7 @@ public void validate() throws InvalidDataException { * * @throws InvalidDataException If the Cell is invalid */ - protected void validateStructure() throws InvalidDataException { + public void validateStructure() throws InvalidDataException { // nothing by default } @@ -207,7 +207,11 @@ protected final Blob createEncoding() { */ @Override public String toString() { - return print().toString(); + try { + return print().toString(); + } catch (Exception e) { + return Utils.getClassName(e)+ " Print failed: "+e.getMessage(); + } } /** @@ -399,7 +403,7 @@ public int getRefCount() { } /** - * Gets the number of Branches referenced from this Cell. This number is + * Gets the number of non-embedded Branches referenced from this Cell. This number is * final / immutable for any given instance and is defined by the Cell encoding rules. * * @return The number of Branches from this Cell @@ -422,8 +426,7 @@ public int getBranchCount() { } /** - * Gets the number of Branches referenced from this Cell. This number is - * final / immutable for any given instance and is defined by the Cell encoding rules. + * Gets a non-embedded Branch referenced from this Cell. * * @return The Ref for the branch, or null if an invalid index */ diff --git a/convex-core/src/main/java/convex/core/data/AExtensionValue.java b/convex-core/src/main/java/convex/core/data/AExtensionValue.java index b0069ed02..4cdbb87e6 100644 --- a/convex-core/src/main/java/convex/core/data/AExtensionValue.java +++ b/convex-core/src/main/java/convex/core/data/AExtensionValue.java @@ -11,7 +11,7 @@ public abstract class AExtensionValue extends ABlobLike { /** - * Length of an Address in bytes (when considered as a Blob) + * Length of an extension value in bytes (when considered as a Blob) */ protected static final int BYTE_LENGTH = 8; @@ -88,6 +88,11 @@ public AExtensionValue empty() { return null; } + @Override + public boolean isEmbedded() { + return true; + } + @Override protected final long calcMemorySize() { // always embedded and no child Refs, so memory size == 0 diff --git a/convex-core/src/main/java/convex/core/data/Cells.java b/convex-core/src/main/java/convex/core/data/Cells.java index 6f4ff341a..a26923252 100644 --- a/convex-core/src/main/java/convex/core/data/Cells.java +++ b/convex-core/src/main/java/convex/core/data/Cells.java @@ -9,6 +9,7 @@ import convex.core.data.impl.DummyCell; import convex.core.exceptions.InvalidDataException; import convex.core.exceptions.ParseException; +import convex.core.exceptions.TODOException; import convex.core.store.AStore; import convex.core.store.Stores; import convex.core.util.Utils; @@ -31,6 +32,8 @@ public class Cells { public static final ACell DUMMY = new DummyCell(); + public static final ACell NIL = null; + /** * Equality method allowing for nulls * @@ -334,11 +337,22 @@ public static void validate(ACell cell) throws InvalidDataException { try { cell.validateCell(); cell.validateStructure(); - } catch (ClassCastException | NullPointerException e) { + // Cells.markValidated(cell); + } catch (Exception e) { throw new InvalidDataException("Invalid due to failure "+e.getMessage(),cell); } } + /** + * Marks a cell as being validated + * @param cell + */ + public static void markValidated(ACell cell) { + + cell.getRef().mergeFlags(Ref.VALIDATED); + throw new TODOException("Probably not safe?"); + } + /** * Gets the caches hash for a cell if available * @param k Any cell (can be null) diff --git a/convex-core/src/main/java/convex/core/data/CodedValue.java b/convex-core/src/main/java/convex/core/data/CodedValue.java index e6ac8842d..ab8a04562 100644 --- a/convex-core/src/main/java/convex/core/data/CodedValue.java +++ b/convex-core/src/main/java/convex/core/data/CodedValue.java @@ -32,7 +32,7 @@ public void validateCell() throws InvalidDataException { } @Override - protected void validateStructure() throws InvalidDataException { + public void validateStructure() throws InvalidDataException { // Nothing to do, any child refs are valid } diff --git a/convex-core/src/main/java/convex/core/data/Format.java b/convex-core/src/main/java/convex/core/data/Format.java index e942decd3..93aedc80d 100644 --- a/convex-core/src/main/java/convex/core/data/Format.java +++ b/convex-core/src/main/java/convex/core/data/Format.java @@ -526,7 +526,10 @@ private static ACell readExtension(byte tag, Blob blob, int offset) throws BadFo // We expect a VLQ Count following the tag long code=readVLQCount(blob,offset+1); - if (tag == CVMTag.CORE_DEF) return Core.fromCode(code); + if (tag == CVMTag.CORE_DEF) { + ACell cc=Core.fromCode(code); + if (cc!=null) return cc; + } if ((tag == CVMTag.OP_SPECIAL)&&(code spec= Special.create((int)code); @@ -639,7 +642,7 @@ static T read(byte tag, Blob blob, int offset) throws BadForma } catch (Exception e) { throw new BadFormatException("Unexpected Exception when decoding ("+tag+"): "+e.getMessage(), e); } - throw new BadFormatException(ErrorMessages.badTagMessage(tag)); + throw new BadFormatException(ErrorMessages.badTagMessage(tag,blob,offset)); } @@ -922,7 +925,9 @@ public static void decodeCells(HashMap acc, Blob data) throws BadFor ix+=Format.getVLQCountLength(encLength); Blob enc=data.slice(ix, ix+encLength); - if (enc==null) throw new BadFormatException("Incomplete encoding"); + if (enc==null) { + throw new BadFormatException("Incomplete encoding"); + } Hash h=enc.getContentHash(); // Check store for Ref - avoids duplicate objects in many cases @@ -966,14 +971,14 @@ public static Blob encodeMultiCell(ACell a, boolean everything) { return encodeMultiCell(a,cells,everything); } - private static Blob encodeMultiCell(ACell a, ArrayList> cells, boolean everything) { - Blob topCellEncoding=Cells.encode(a); - Consumer> addToStackFunc=r->{cells.add(r);}; + private static Blob encodeMultiCell(ACell topCell, ArrayList> branches, boolean everything) { + Blob topCellEncoding=Cells.encode(topCell); + Consumer> addToStackFunc=r->{branches.add(r);}; // Visit refs in stack to add to message, accumulating message size required int[] ml=new int[] {topCellEncoding.size()}; // Array mutation trick for accumulator. Ugly but works.... HashSet> refs=new HashSet<>(); - Trees.visitStack(cells, cr->{ + Trees.visitStack(branches, cr->{ if (!refs.contains(cr)) { ACell c=cr.getValue(); int encLength=c.getEncodingLength(); @@ -1016,8 +1021,17 @@ private static Blob encodeMultiCell(ACell a, ArrayList> cells, boolean ev * @param v * @return */ - public static Blob encodeDataVector(AVector v) { + public static Blob encodeDataResult(Result result) { + AVector v=RT.ensureVector(result.getValue()); + if (v==null) throw new IllegalArgumentException("Data result must contain a vector value"); + ArrayList> cells=new ArrayList>(); + + // Add the top level vector as a branch iff it is not embedded in the Result + if (!v.isEmbedded()) { + cells.add(v.getRef()); + } + v.visitAllChildren(vc->{ Ref r=vc.getRef(); if (!r.isEmbedded()) { @@ -1033,7 +1047,8 @@ public static Blob encodeDataVector(AVector v) { }; }); - return encodeMultiCell(v,cells,false); + // Note false to prevent traversing all extra branches + return encodeMultiCell(result,cells,false); } diff --git a/convex-core/src/main/java/convex/core/data/Ref.java b/convex-core/src/main/java/convex/core/data/Ref.java index c4e30211b..8efa9b69a 100644 --- a/convex-core/src/main/java/convex/core/data/Ref.java +++ b/convex-core/src/main/java/convex/core/data/Ref.java @@ -333,7 +333,7 @@ public static Ref get(T value) { * @param hash The hash value for this Ref to refer to * @return Ref for the specific hash. */ - public static RefSoft forHash(Hash hash) { + public static Ref forHash(Hash hash) { return RefSoft.createForHash(hash); } @@ -365,7 +365,7 @@ public Ref setFlags(int newFlags) { */ public static Ref readRaw(Blob b, int pos) throws BadFormatException { Hash h = Hash.wrap(b,pos); - if (h==null) throw new BadFormatException("Insufficient bytes to read Ref as position: "+pos); + if (h==null) throw new BadFormatException("Insufficient bytes to read Ref at position: "+pos); Ref ref=Ref.forHash(h); ref=ref.markEmbedded(false); return ref; @@ -694,6 +694,12 @@ public static int mergeFlags(int a, int b) { int flagsPart=((a|b)&~STATUS_MASK); return statusPart|flagsPart; } + + protected int mergeFlags(int flags) { + int newFlags=mergeFlags(this.flags,flags); + setFlags(newFlags); + return newFlags; + } /** * Ensures this Ref is canonical @@ -733,4 +739,6 @@ public boolean isInternal() { public boolean isValidated() { return getStatus()>=VALIDATED; } + + } diff --git a/convex-core/src/main/java/convex/core/data/RefSoft.java b/convex-core/src/main/java/convex/core/data/RefSoft.java index 618305ed1..7770a6f2d 100644 --- a/convex-core/src/main/java/convex/core/data/RefSoft.java +++ b/convex-core/src/main/java/convex/core/data/RefSoft.java @@ -81,9 +81,9 @@ public static RefSoft create(AStore store,T value, int flag * * @param Type of value * @param hash Hash ID of value. - * @return New RefSoft instance + * @return New RefSoft instance, or cached Refsoft from current store */ - public static RefSoft createForHash(Hash hash) { + public static Ref createForHash(Hash hash) { return new RefSoft(Stores.current(),hash); } @@ -91,6 +91,8 @@ public static RefSoft createForHash(Hash hash) { public T getValue() { T result = softRef.get(); if (result == null) { + if (store==null) throw new MissingDataException(null,hash); + Ref storeRef = store.refForHash(hash); if (storeRef == null) { throw new MissingDataException(store,hash); diff --git a/convex-core/src/main/java/convex/core/data/SignedData.java b/convex-core/src/main/java/convex/core/data/SignedData.java index 2e872457e..337c77515 100644 --- a/convex-core/src/main/java/convex/core/data/SignedData.java +++ b/convex-core/src/main/java/convex/core/data/SignedData.java @@ -319,11 +319,6 @@ public void validateSignature() throws BadSignatureException { if (!checkSignature()) throw new BadSignatureException("Signature not valid!", this); } - @Override - public boolean isCanonical() { - return true; - } - @Override public final int getRefCount() { // Value Ref only @@ -366,16 +361,6 @@ public void validateCell() throws InvalidDataException { public Ref getValueRef() { return valueRef; } - - /** - * SignedData is not embedded. - * main reason: We always want to persist in store to cache verification status - * - * @return Always false - */ - public boolean isEmbedded() { - return false; - } @Override public RecordFormat getFormat() { diff --git a/convex-core/src/main/java/convex/core/data/Strings.java b/convex-core/src/main/java/convex/core/data/Strings.java index 379fd1494..684659d2e 100644 --- a/convex-core/src/main/java/convex/core/data/Strings.java +++ b/convex-core/src/main/java/convex/core/data/Strings.java @@ -64,6 +64,10 @@ public class Strings { public static final StringShort LONG_MIN_VALUE = StringShort.create("-9223372036854775808"); + public static final StringShort SENT = StringShort.create("Sent"); + + public static final StringShort FULL_BUFFER = StringShort.create("Buffer full"); + /** * Reads a String from a Blob encoding. @@ -127,11 +131,15 @@ public static AString create(Object o) { return create(o.toString()); } - public static T intern(T value) { return Cells.intern(value); } + @SuppressWarnings("unchecked") + public static T intern(String value) { + return (T) intern(create(value)); + } + public static AString create(CVMChar c) { return create(c.toUTFBlob()); } diff --git a/convex-core/src/main/java/convex/core/data/VectorLeaf.java b/convex-core/src/main/java/convex/core/data/VectorLeaf.java index 63f1ca7d0..9785f5c7b 100644 --- a/convex-core/src/main/java/convex/core/data/VectorLeaf.java +++ b/convex-core/src/main/java/convex/core/data/VectorLeaf.java @@ -601,7 +601,7 @@ public boolean equals(AVector a) { AVector v=(AVector) a; if (v.count()!=count) return false; - // It's a vector of same length, but not canonical? + // It's a vector of same length, but not canonical, so check encoding return a.getEncoding().equals(this.getEncoding()); } @@ -732,9 +732,9 @@ public static final boolean isValidCount(long count) { @Override protected void visitAllChildren(Consumer> visitor) { if (hasPrefix()) { - prefix.getValue().visitAllChildren(visitor); + AVector child=prefix.getValue(); + child.visitAllChildren(visitor); + visitor.accept(child); } } - - } diff --git a/convex-core/src/main/java/convex/core/data/VectorTree.java b/convex-core/src/main/java/convex/core/data/VectorTree.java index de434edde..a09518c11 100644 --- a/convex-core/src/main/java/convex/core/data/VectorTree.java +++ b/convex-core/src/main/java/convex/core/data/VectorTree.java @@ -571,7 +571,7 @@ public VectorTree updateRefs(IRefFunction func) { Ref> newChild = (Ref>) func.apply(current); if (newChild!=current) { - if (children==newChildren) newChildren=children.clone(); + if (children==newChildren) newChildren=newChildren.clone(); newChildren[i] = newChild; } } @@ -685,7 +685,7 @@ protected void validateCell() throws InvalidDataException { } @Override - protected void validateStructure() throws InvalidDataException { + public void validateStructure() throws InvalidDataException { super.validateStructure(); long c = 0; int blen = children.length; diff --git a/convex-core/src/main/java/convex/core/exceptions/FastRuntimeException.java b/convex-core/src/main/java/convex/core/exceptions/FastRuntimeException.java index be99f4e7e..97e85dcd3 100644 --- a/convex-core/src/main/java/convex/core/exceptions/FastRuntimeException.java +++ b/convex-core/src/main/java/convex/core/exceptions/FastRuntimeException.java @@ -1,5 +1,7 @@ package convex.core.exceptions; +import convex.core.Constants; + @SuppressWarnings("serial") public class FastRuntimeException extends RuntimeException { @@ -10,6 +12,8 @@ public FastRuntimeException(String message) { // Don't fill in a stack trace for fast exceptions. We are going to catch and ignore it anyway..... @Override public Throwable fillInStackTrace() { - return this; + if (Constants.OMIT_VALIDATION_STACKTRACES) return this; + return super.fillInStackTrace(); + } } diff --git a/convex-core/src/main/java/convex/core/exceptions/MissingDataException.java b/convex-core/src/main/java/convex/core/exceptions/MissingDataException.java index ac3780630..88911cf4b 100644 --- a/convex-core/src/main/java/convex/core/exceptions/MissingDataException.java +++ b/convex-core/src/main/java/convex/core/exceptions/MissingDataException.java @@ -24,7 +24,7 @@ public MissingDataException(AStore store, Hash hash) { } public String getMessage() { - return "Missing hash:" + hash + " in store " + store.toString(); + return "Missing hash:" + hash + ((store==null)?" (null store)": "in store " + store.toString()); } /** diff --git a/convex-core/src/main/java/convex/core/init/Init.java b/convex-core/src/main/java/convex/core/init/Init.java index 975731198..4ef43cfc9 100644 --- a/convex-core/src/main/java/convex/core/init/Init.java +++ b/convex-core/src/main/java/convex/core/init/Init.java @@ -1,6 +1,7 @@ package convex.core.init; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import convex.core.Coin; @@ -62,6 +63,7 @@ public class Init { // First user of Protonet, i.e. @mikera public static final Address FIRST_USER_ADDRESS = Address.create(13); + public static final AccountKey FIRST_USER_KEY = AccountKey.fromHex("89b5142678bfef7a2245af5ae5b9ab1e10c282b375fa297c5aaeccc48ac97cac"); // Constants private static final Index EMPTY_PEERS = Index.none(); @@ -72,6 +74,8 @@ public class Init { */ private static final long GENESIS_COINS=1000000*Coin.GOLD; + public static final AccountKey DEFAULT_GOV_KEY = AccountKey.fromHex("aE9C747a9730D63Fc16BcccEBd12B5dD4c8fBe1328e9a953025e8C02164Ed5E6"); + /** * Creates the base genesis state (before deployment of standard libraries and actors). This is the minimum state required for Convex operation. @@ -209,6 +213,9 @@ public static State createBaseState(AccountKey governanceKey, AccountKey genesis } assert(userFunds == 0L); } + + // Initial user account, follows genesis peer controller(s) + accts=addAccount(accts,Address.create(accts.count()),FIRST_USER_KEY,0); // Finally add initial peers @@ -234,6 +241,10 @@ public static State createBaseState(AccountKey governanceKey, AccountKey genesis s = s.withAccounts(accts); // Add peers to the State s = s.withPeers(peers); + + s = register(s, GENESIS_ADDRESS, "Genesis User", "User account for genesis"); + s = register(s, GENESIS_PEER_ADDRESS, "Genesis Peer", "Genesis peer contrroller account"); + { // Test total funds after creating user / peer accounts long total = s.computeTotalBalance(); @@ -272,14 +283,14 @@ private static State addStaticLibraries(State s) { { // Register core library now that registry exists Context ctx = Context.create(s, INIT_ADDRESS); s = ctx.getState(); - s = register(s, CORE_ADDRESS, "Convex Core Library", "Core utilities accessible by default in any account."); + s = register(s, CORE_ADDRESS, "Convex Core Library", "Core library accessible by default in any account"); - s = register(s, FOUNDATION_ADDRESS, "Foundation Governance", "Master network governance account."); - s = register(s, RESERVE_ADDRESS, "Foundation Reserve", "Unreleased Foundation coin reserve."); - s = register(s, UNRELEASED_ADDRESS, "Release Curve Reserve", "Release curve unreleased coins."); - s = register(s, DISTRIBUTION_ADDRESS, "Release Curve Pre-Distribution", "Release curve coins for future distribution."); - s = register(s, GOVERNANCE_ADDRESS, "Network Governance", "Network governance account."); - s = register(s, ADMIN_ADDRESS, "Network Admin", "Network admin accouns."); + s = register(s, FOUNDATION_ADDRESS, "Foundation Governance", "Master network governance account"); + s = register(s, RESERVE_ADDRESS, "Foundation Reserve", "Unreleased Foundation coin reserve"); + s = register(s, UNRELEASED_ADDRESS, "Release Curve Reserve", "Release curve unreleased coins"); + s = register(s, DISTRIBUTION_ADDRESS, "Release Curve Pre-Distribution", "Release curve coins for future distribution"); + s = register(s, GOVERNANCE_ADDRESS, "Network Governance", "Network governance account"); + s = register(s, ADMIN_ADDRESS, "Network Admin", "Network admin accouns"); } return s; @@ -290,7 +301,7 @@ public static State createState(List genesisKeys) { } public static State createState(AccountKey genesisKey,List peerKeys) { - return createState(genesisKey,genesisKey,peerKeys); + return createState(DEFAULT_GOV_KEY,genesisKey,peerKeys); } public static State createState(AccountKey governanceKey, AccountKey genesisKey,List peerKeys) { @@ -324,32 +335,43 @@ private static State addTestingCurrencies(State s) { private static State addStandardLibraries(State s) { s = doActorDeploy(s, "/convex/asset/fungible.cvx"); - s = doActorDeploy(s, "/convex/lab/trusted-oracle/actor.cvx"); - s = doActorDeploy(s, "/convex/lab/oracle.cvx"); s = doActorDeploy(s, "/convex/asset/asset.cvx"); s = doActorDeploy(s, "/convex/torus/exchange.cvx"); s = doActorDeploy(s, "/convex/asset/nft/simple.cvx"); s = doActorDeploy(s, "/convex/asset/nft/basic.cvx"); - s = doActorDeploy(s, "/convex/asset/nft/tokens.cvx"); s = doActorDeploy(s, "/convex/asset/box/actor.cvx"); s = doActorDeploy(s, "/convex/asset/box.cvx"); s = doActorDeploy(s, "/convex/asset/multi-token.cvx"); - s = doActorDeploy(s, "/convex/asset/share.cvx"); - s = doActorDeploy(s, "/convex/asset/market/trade.cvx"); s = doActorDeploy(s, "/convex/asset/wrap/convex.cvx"); - s = doActorDeploy(s, "/convex/lab/play.cvx"); - s = doActorDeploy(s, "/convex/lab/did.cvx"); - s = doActorDeploy(s, "/convex/lab/curation-market.cvx"); s = doActorDeploy(s, "/convex/trust/ownership-monitor.cvx"); s = doActorDeploy(s, "/convex/trust/delegate.cvx"); s = doActorDeploy(s, "/convex/trust/whitelist.cvx"); s = doActorDeploy(s, "/convex/trust/monitors.cvx"); s = doActorDeploy(s, "/convex/trust/governance.cvx"); - s = doActorDeploy(s, "/convex/asset/spatial.cvx"); // s = doActorDeploy(s, "convex/user.cvx"); return s; } + /** + * Add extra libraries for testing purposes, not part of official genesis + * @param peerKeys + * @return + */ + public static State createTestState(ArrayList peerKeys) { + State s=createState(peerKeys); + s = doActorDeploy(s, "/convex/asset/nft/tokens.cvx"); + s = doActorDeploy(s, "/convex/lab/play.cvx"); + s = doActorDeploy(s, "/convex/lab/did.cvx"); + s = doActorDeploy(s, "/convex/lab/curation-market.cvx"); + s = doActorDeploy(s, "/convex/lab/trusted-oracle/actor.cvx"); + s = doActorDeploy(s, "/convex/lab/oracle.cvx"); + s = doActorDeploy(s, "/convex/asset/share.cvx"); + s = doActorDeploy(s, "/convex/asset/market/trade.cvx"); + s = doActorDeploy(s, "/convex/asset/spatial.cvx"); + + return s; + } + private static State addCNSBaseTree(State s) { Context ctx=Context.create(s, GOVERNANCE_ADDRESS); ctx=ctx.eval(Reader.read("(*registry*/create 'convex)")); @@ -392,14 +414,6 @@ private static State addCNSExtraTree(State s) { s=ctx.getState(); return s; } - - public static Address calcPeerAddress(int userCount, int index) { - return Address.create(GENESIS_ADDRESS.longValue() + userCount + index); - } - - public static Address calcUserAddress(int index) { - return Address.create(GENESIS_ADDRESS.longValue() + index); - } private static State doActorDeploy(State s, String resource) { return doActorDeploy(s,resource,true); @@ -464,6 +478,7 @@ private static State doCurrencyDeploy(State s, AVector row) { + "(import convex.fungible :as fun) " + "(deploy " + "'(call *registry* (register "+metaString+"))" + + "'(set-controller #2)" + "(fun/build-token {:supply " + supply + " :decimals "+decimals+"})" +")" + ")")); @@ -518,4 +533,5 @@ private static AVector addCoreLibrary(AVector acct } + } diff --git a/convex-core/src/main/java/convex/core/lang/Core.java b/convex-core/src/main/java/convex/core/lang/Core.java index 5a813741b..f7c3a9c34 100644 --- a/convex-core/src/main/java/convex/core/lang/Core.java +++ b/convex-core/src/main/java/convex/core/lang/Core.java @@ -3009,14 +3009,12 @@ private static Context applyDocumentation(Context ctx) throws IOException { * Read a Core definition from an encoding * @param b Blob containing encoding * @param pos Position to read Core code function - * @return Singleton cell representing the Core value - * @throws BadFormatException In case of encoding error + * @return Singleton cell representing the Core value, or null if not defined */ public static ACell fromCode(long code) throws BadFormatException { - if (code <0 || code>=CODE_MAP.length) throw new BadFormatException("Core code out of range: "+code); + if (code <0 || code>=CODE_MAP.length) return null; ACell o = CODE_MAP[(int)code]; - if (o == null) throw new BadFormatException("Core code definition not found: " + code); return o; } diff --git a/convex-core/src/main/java/convex/core/lang/RT.java b/convex-core/src/main/java/convex/core/lang/RT.java index 69a51db58..97e6f36f4 100644 --- a/convex-core/src/main/java/convex/core/lang/RT.java +++ b/convex-core/src/main/java/convex/core/lang/RT.java @@ -1414,6 +1414,17 @@ public static Keyword castKeyword(ACell a) { Keyword k = Keyword.create(name); return k; } + + /** + * Casts to a Keyword + * @param a + * @return + */ + public static Keyword ensureKeyword(ACell a) { + if (a instanceof Keyword) + return (Keyword) a; + return null; + } /** * Ensures the argument is a Symbol. @@ -1496,7 +1507,7 @@ public static void validate(Object o) throws InvalidDataException { if (o == null) return; if (o instanceof ACell) { - ((ACell) o).validate(); + Cells.validate((ACell) o); } else if (o instanceof Ref) { ((Ref) o).validate(); } else { @@ -1883,4 +1894,6 @@ public static long[] toLongArray(AVector v) { + + } diff --git a/convex-core/src/main/java/convex/core/store/Stores.java b/convex-core/src/main/java/convex/core/store/Stores.java index fa1df2cea..927709dfc 100644 --- a/convex-core/src/main/java/convex/core/store/Stores.java +++ b/convex-core/src/main/java/convex/core/store/Stores.java @@ -1,7 +1,10 @@ package convex.core.store; import java.io.IOException; +import java.util.concurrent.Callable; +import convex.core.util.UnsafeRunnable; +import convex.core.util.Utils; import convex.etch.EtchStore; public class Stores { @@ -73,4 +76,28 @@ public static void setGlobalStore(EtchStore store) { if (store==null) throw new IllegalArgumentException("Cannot set global store to null)"); globalStore=store; } + + public static void runWithMemoryStore(UnsafeRunnable r) { + AStore saved=Stores.current(); + try { + Stores.setCurrent(new MemoryStore()); + r.run(); + } catch (Exception e) { + throw Utils.sneakyThrow(e); + } finally { + Stores.setCurrent(saved); + } + } + + public static V runWithMemoryStore(Callable r) { + AStore saved=Stores.current(); + try { + Stores.setCurrent(new MemoryStore()); + return r.call(); + } catch (Exception e) { + throw Utils.sneakyThrow(e); + } finally { + Stores.setCurrent(saved); + } + } } diff --git a/convex-core/src/main/java/convex/core/text/PrintUtils.java b/convex-core/src/main/java/convex/core/text/PrintUtils.java new file mode 100644 index 000000000..1b811c4e0 --- /dev/null +++ b/convex-core/src/main/java/convex/core/text/PrintUtils.java @@ -0,0 +1,50 @@ +package convex.core.text; + +import convex.core.data.ACell; +import convex.core.data.ACountable; +import convex.core.data.Cells; +import convex.core.data.Ref; +import convex.core.lang.RT; +import convex.core.util.Utils; + +public class PrintUtils { + + public static String printRefTree(Ref ref) { + StringBuilder sb=new StringBuilder(); + + printRefTree(sb,ref,0); + return sb.toString(); + } + + private static void printRefTree(StringBuilder sb, Ref ref, int level) { + + + String prefix=Text.whiteSpace(level); + + if (ref.isMissing()) { + sb.append(prefix); + sb.append("MISSING: "+ref.getHash()); + sb.append('\n'); + } else { + ACell val=ref.getValue(); + String name=Utils.getClassName(val); + if (Cells.isCompletelyEncoded(val)) { + name+=" = " +RT.toString(val); + } else { + if (val instanceof ACountable) { + name+=" [count=" +RT.count(val)+"]"; + } + } + + + sb.append(prefix); + sb.append(name); + sb.append('\n'); + + int rc=Cells.refCount(val); + for (int i=0; i { + AStore saved=Stores.current(); + try { + func.run(); + } finally { + Stores.setCurrent(saved); + } + }); + } + } diff --git a/convex-core/src/main/java/convex/core/util/UnsafeRunnable.java b/convex-core/src/main/java/convex/core/util/UnsafeRunnable.java new file mode 100644 index 000000000..157d56de3 --- /dev/null +++ b/convex-core/src/main/java/convex/core/util/UnsafeRunnable.java @@ -0,0 +1,5 @@ +package convex.core.util; + +public interface UnsafeRunnable { + void run () throws Exception; +} diff --git a/convex-core/src/main/java/convex/etch/EtchStore.java b/convex-core/src/main/java/convex/etch/EtchStore.java index 9fe5db72e..67631079e 100644 --- a/convex-core/src/main/java/convex/etch/EtchStore.java +++ b/convex-core/src/main/java/convex/etch/EtchStore.java @@ -2,6 +2,7 @@ import java.io.File; import java.io.IOException; +import java.nio.channels.ClosedChannelException; import java.util.function.Consumer; import org.slf4j.Logger; @@ -113,7 +114,7 @@ public static EtchStore createTemp() throws IOException { @Override public Ref refForHash(Hash hash) { try { - Ref existing = (Ref) refCache.getCell(hash); + Ref existing = checkCache(hash); if (existing != null) return (Ref) existing; @@ -121,8 +122,12 @@ public Ref refForHash(Hash hash) { return (Ref) Ref.NULL_VALUE; existing = readStoreRef(hash); return (Ref) existing; + } catch (ClosedChannelException e) { + log.debug("Etch store closed during refForHAsh lookup"); + return null; } catch (IOException e) { - throw Utils.sneakyThrow(e); + log.warn("IO Error in Etch store: "+e); + return null; } } diff --git a/convex-core/src/test/java/convex/core/cpos/BeliefMergeTest.java b/convex-core/src/test/java/convex/core/cpos/BeliefMergeTest.java index 99af97131..cbd0b4f98 100644 --- a/convex-core/src/test/java/convex/core/cpos/BeliefMergeTest.java +++ b/convex-core/src/test/java/convex/core/cpos/BeliefMergeTest.java @@ -26,7 +26,6 @@ import convex.core.data.AccountKey; import convex.core.data.Cells; import convex.core.data.EncodingTest; -import convex.core.data.Hash; import convex.core.data.Index; import convex.core.data.RecordTest; import convex.core.data.SignedData; @@ -455,8 +454,7 @@ public void testGossipConsensus() throws Exception { if (ANALYSIS) printAccounts(accounts); // final state checks - assertEquals(Hash.fromHex("5e16756d0e3b5ab5ac4a6649858b165b41d19de6277fab0e30c2ff3e0460065e"),finalState.getHash()); - + // assertEquals(Hash.fromHex("95915ffa2d983b763cb504a92cbc5d31622b5f01d4943ecda91d1efd1d5f0214"),finalState.getHash()); int expectedTxCount=NUM_PEERS * (NUM_INITIAL_TRANS+TX_ROUNDS); assertEquals(expectedTxCount, bs4[PROPOSER].getOrder(RKEY).getConsensusPoint()); diff --git a/convex-core/src/test/java/convex/core/cpos/BeliefVotingTest.java b/convex-core/src/test/java/convex/core/cpos/BeliefVotingTest.java index 10d913e6b..a5a44eabd 100644 --- a/convex-core/src/test/java/convex/core/cpos/BeliefVotingTest.java +++ b/convex-core/src/test/java/convex/core/cpos/BeliefVotingTest.java @@ -151,8 +151,8 @@ public void testTieBreak() throws BadSignatureException, InvalidDataException { SignedData o1=or(1, TS, 0,0,A,B); SignedData o2=or(2, TS, 0,0,B); SignedData o3=or(3, TS, 0,0,B,A); - SignedData o4=or(4, TS, 0,0,B,A,C,G); - SignedData o5=or(5, TS, 0,0,B,A,E,F,D); // should win, hash based? + SignedData o4=or(4, TS, 0,0,B,A,C,G); // should win, hash based? + SignedData o5=or(5, TS, 0,0,B,A,E,F,D); Belief b=Belief.create(o0,o1,o2,o3,o4,o5); BeliefMerge mc=BeliefMerge.create(b, kps[0], TS, s); @@ -161,11 +161,13 @@ public void testTieBreak() throws BadSignatureException, InvalidDataException { Order order=so.getValue(); assertEquals(7,order.getBlockCount()); assertEquals(B,order.getBlock(0)); - assertEquals(F,order.getBlock(3)); + assertEquals(G,order.getBlock(3)); assertEquals(0,order.getProposalPoint()); // 66.66..% just short of proposal threshold assertEquals(0,order.getConsensusPoint()); // Note C,G not in winning Order so sorted by timestamp order - assertEquals(Vectors.of(B,A,E,F,D,C,G),order.getBlocks()); + + assertEquals(Vectors.of(2,1,3,7,4,5,6),order.getBlocks().map(sb->CVMLong.create(sb.getValue().getTimeStamp()))); + assertEquals(Vectors.of(B,A,C,G,D,E,F),order.getBlocks()); } { @@ -183,10 +185,9 @@ public void testTieBreak() throws BadSignatureException, InvalidDataException { Order order=so.getValue(); assertEquals(7,order.getBlockCount()); assertEquals(B,order.getBlock(0)); - assertEquals(F,order.getBlock(3)); assertEquals(1,order.getConsensusPoint(1)); // Enough for proposal assertEquals(0,order.getConsensusPoint(2)); - assertEquals(Vectors.of(B,A,E,F,G,C,D),order.getBlocks()); + assertEquals(Vectors.of(B,A,C,D,E,F,G),order.getBlocks()); } { diff --git a/convex-core/src/test/java/convex/core/data/AdversarialDataTest.java b/convex-core/src/test/java/convex/core/data/AdversarialDataTest.java index 59727ea82..b2d818d78 100644 --- a/convex-core/src/test/java/convex/core/data/AdversarialDataTest.java +++ b/convex-core/src/test/java/convex/core/data/AdversarialDataTest.java @@ -278,6 +278,12 @@ public void testBadBelief() { invalidEncoding(CVMTag.BELIEF,"d401b6"); // Byteflag instead of Index } + @Test + public void testBadBlobs() throws BadFormatException { + invalidEncoding("828041000000000000"); + assertEquals(CVMLong.create(0x8041000000000000l),Format.read("188041000000000000")); + } + @Test public void testBadSet() { invalidEncoding("83851d3ff0000000000000"); diff --git a/convex-core/src/test/java/convex/core/data/CAD3Test.java b/convex-core/src/test/java/convex/core/data/CAD3Test.java index 2e32209b1..3cb0bc5c7 100644 --- a/convex-core/src/test/java/convex/core/data/CAD3Test.java +++ b/convex-core/src/test/java/convex/core/data/CAD3Test.java @@ -63,6 +63,9 @@ public class CAD3Test extends ACVMTest { assertSame(Vectors.empty(),Reader.read("#[8000]")); assertNull(Reader.read("#[00]")); assertEquals(ExtensionValue.create((byte) 0xe5, 0),Reader.read("#[e500]")); + + assertEquals(Core.QUOTE,Reader.read("#[ed00]")); + assertEquals(ExtensionValue.create(CVMTag.CORE_DEF,1280),Reader.read("#[ed8a00]")); } @Test public void testDenseRecords() { diff --git a/convex-core/src/test/java/convex/core/data/EncodingTest.java b/convex-core/src/test/java/convex/core/data/EncodingTest.java index 1fb0c948c..d9a8d77d9 100644 --- a/convex-core/src/test/java/convex/core/data/EncodingTest.java +++ b/convex-core/src/test/java/convex/core/data/EncodingTest.java @@ -335,7 +335,6 @@ public void testSignedDataEncoding() throws BadFormatException { ABlob bigBlob=Blob.createRandom(new Random(123), 10000).toCanonical(); Invoke trans=Invoke.create(Address.create(607), 6976, Vectors.of(1,bigBlob,2,bigBlob)); SignedData strans=Samples.KEY_PAIR.signData(trans); - assertFalse(strans.isEmbedded()); AVector v=Vectors.of(strans); Blob enc=Format.encodeMultiCell(v,true); diff --git a/convex-core/src/test/java/convex/core/data/ObjectsTest.java b/convex-core/src/test/java/convex/core/data/ObjectsTest.java index c3ef85644..1eebdc98f 100644 --- a/convex-core/src/test/java/convex/core/data/ObjectsTest.java +++ b/convex-core/src/test/java/convex/core/data/ObjectsTest.java @@ -45,8 +45,8 @@ public static void doAnyValueTests(ACell a) { fail(e); } Ref r = Ref.get(a); - assertEquals(h,r.getHash()); assertSame(a, r.getValue()); // shouldn't get GC'd because we have a strong reference + assertEquals(h,r.getHash()); doAnyEncodingTests(a); doCellTests(a); diff --git a/convex-core/src/test/java/convex/core/data/RefTest.java b/convex-core/src/test/java/convex/core/data/RefTest.java index ed28a9ca6..deb4642e8 100644 --- a/convex-core/src/test/java/convex/core/data/RefTest.java +++ b/convex-core/src/test/java/convex/core/data/RefTest.java @@ -26,6 +26,7 @@ import convex.core.exceptions.MissingDataException; import convex.core.lang.Core; import convex.core.lang.RT; +import convex.core.store.Stores; import convex.core.util.Utils; import convex.test.Samples; @@ -377,11 +378,8 @@ public static void checkInternal(T a) { assertTrue(ref.isInternal(),()->"Not internal ref: "+a+" of type "+Utils.getClass(a)); assertSame(a,ref.getValue()); - try { - assertSame(a,Cells.persist(a)); - } catch (IOException e) { - throw Utils.sneakyThrow(e); - } + Stores.runWithMemoryStore(()->assertSame(a,Cells.persist(a))); + } @Test @@ -397,7 +395,7 @@ public void testNullRef() { @Test public void testReadRefSoft() throws BadFormatException { Hash h=Hash.wrap(Blobs.createRandom(32)); - RefSoft ref=RefSoft.createForHash(h); + Ref ref=RefSoft.createForHash(h); ref.markEmbedded(false); // needed to ensure indirect encoding is assumed Blob b=ref.getEncoding(); assertEquals(Ref.INDIRECT_ENCODING_LENGTH,b.count()); diff --git a/convex-core/src/test/java/convex/core/data/SyntaxTest.java b/convex-core/src/test/java/convex/core/data/SyntaxTest.java index 1b08e30e2..114ce5b14 100644 --- a/convex-core/src/test/java/convex/core/data/SyntaxTest.java +++ b/convex-core/src/test/java/convex/core/data/SyntaxTest.java @@ -7,6 +7,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; + import org.junit.jupiter.api.Test; import convex.core.cvm.Address; @@ -75,6 +77,19 @@ private void doSyntaxTest(Syntax s) { } + @Test public void testSytaxEncode() throws BadFormatException, IOException { + + Syntax s=Reader.read("^{} {#1 -9223372036854775808}"); + Blob b=s.getEncoding(); + Syntax s2=Format.read(b); + + assertEquals(s,s2); + + Ref pref=Cells.persist(s).getRef(); + assertEquals(s,pref.getValue()); + + } + @Test public void testSyntaxPrintRegression() { String s="^{} 0xa89e59cc8ab9fc6a13785a37938c85b306b24663415effc01063a6e25ef52ebcd3647d3a77e0a33908a372146fdccab6"; int n=s.length(); diff --git a/convex-core/src/test/java/convex/core/data/VectorsTest.java b/convex-core/src/test/java/convex/core/data/VectorsTest.java index a9030035b..4b1edbfc4 100644 --- a/convex-core/src/test/java/convex/core/data/VectorsTest.java +++ b/convex-core/src/test/java/convex/core/data/VectorsTest.java @@ -392,9 +392,10 @@ public void testPrefixLength() throws BadFormatException { @Test public void testVisitors() { VisitCounter> vc=new VisitCounter<>(); - Samples.INT_VECTOR_300.visitAllChildren(vc); + AVector v=Samples.INT_VECTOR_300; + v.visitAllChildren(vc); - assertEquals(16+1+3,vc.count); // 16*(16) + 1 *(256) + 2 *(16) + 1 *(32) [== whole prefix] + assertEquals(16+1+2+1+1,vc.count); // 16 (16) + 1 (256) + 2 (16) + 1 (32) + 1 [== whole prefix] } @Test diff --git a/convex-core/src/test/java/convex/core/init/InitTest.java b/convex-core/src/test/java/convex/core/init/InitTest.java index da9b0ff9a..3f1a9db6f 100644 --- a/convex-core/src/test/java/convex/core/init/InitTest.java +++ b/convex-core/src/test/java/convex/core/init/InitTest.java @@ -60,7 +60,7 @@ public class InitTest extends ACVMTest { public static State createState() { try { - return Init.createState(PEER_KEYS); + return Init.createTestState(PEER_KEYS); } catch (Throwable e) { e.printStackTrace(); throw e; @@ -95,6 +95,11 @@ public void testDeploy() { assertEquals(Init.REGISTRY_ADDRESS, eval("(resolve convex.registry)")); assertEquals(Init.TRUST_ADDRESS, eval("(resolve convex.trust)")); } + + @Test + public void testNames() { + assertEquals("Convex Core Library",evalS("(:name (call *registry* (lookup #8)))")); + } @Test public void testInitState() throws InvalidDataException { diff --git a/convex-core/src/test/java/convex/store/EtchStoreTest.java b/convex-core/src/test/java/convex/store/EtchStoreTest.java index bca967c7a..5809c491f 100644 --- a/convex-core/src/test/java/convex/store/EtchStoreTest.java +++ b/convex-core/src/test/java/convex/store/EtchStoreTest.java @@ -216,7 +216,7 @@ public void testBeliefAnnounce() throws IOException { // Persist belief counter.set(0L); Ref prb=srb.persist(noveltyHandler); - assertEquals(4L,counter.get()); + assertEquals(3L,counter.get()); // Persist again. Should be no new novelty counter.set(0L); @@ -228,7 +228,7 @@ public void testBeliefAnnounce() throws IOException { counter.set(0L); Ref arb=Cells.announce(belief,noveltyHandler).getRef(); assertEquals(srb,arb); - assertEquals(4L,counter.get()); + assertEquals(3L,counter.get()); // Announce again. Should be no new novelty counter.set(0L); diff --git a/convex-gui/src/main/java/convex/gui/components/ConnectPanel.java b/convex-gui/src/main/java/convex/gui/components/ConnectPanel.java index afa64b2b6..c5823e23c 100644 --- a/convex-gui/src/main/java/convex/gui/components/ConnectPanel.java +++ b/convex-gui/src/main/java/convex/gui/components/ConnectPanel.java @@ -1,10 +1,7 @@ package convex.gui.components; import java.awt.Color; -import java.io.IOException; -import java.net.ConnectException; import java.net.InetSocketAddress; -import java.util.concurrent.TimeoutException; import javax.swing.JComponent; import javax.swing.JLabel; @@ -15,6 +12,7 @@ import org.slf4j.LoggerFactory; import convex.api.Convex; +import convex.api.ConvexRemote; import convex.core.crypto.wallet.AWalletEntry; import convex.core.cvm.Address; import convex.core.init.Init; @@ -84,7 +82,7 @@ public static Convex tryConnect(JComponent parent,String prompt) { String target=pan.hostField.getText(); InetSocketAddress sa=IPUtils.toInetSocketAddress(target); log.info("Attempting connect to: "+sa); - Convex convex=Convex.connect(sa); + Convex convex=ConvexRemote.connectNetty(sa); convex.setAddress(addr); HostCombo.registerGoodConnection(target); @@ -103,13 +101,13 @@ public static Convex tryConnect(JComponent parent,String prompt) { } } return convex; - } catch (ConnectException e) { - log.info("Failed to connect"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + e.printStackTrace(); + } catch (Exception e) { + log.info("Failed to connect",e); Toast.display(parent, e.getMessage(), Color.RED); - } catch (TimeoutException | IOException e) { - Toast.display(parent, e.getMessage(), Color.RED); - e.printStackTrace(); - } + } } else { log.info("Connect cancelled by user"); } diff --git a/convex-gui/src/main/java/convex/gui/components/account/KeyPairCombo.java b/convex-gui/src/main/java/convex/gui/components/account/KeyPairCombo.java index 9e37cd5ed..9bd341a25 100644 --- a/convex-gui/src/main/java/convex/gui/components/account/KeyPairCombo.java +++ b/convex-gui/src/main/java/convex/gui/components/account/KeyPairCombo.java @@ -1,6 +1,7 @@ package convex.gui.components.account; import java.awt.Component; +import java.util.NoSuchElementException; import javax.swing.ComboBoxModel; import javax.swing.DefaultListModel; @@ -153,6 +154,16 @@ public static KeyPairCombo forConvex(Convex convex) { AKeyPair kp=convex.getKeyPair(); return create(kp); } + + public static KeyPairCombo create() { + KeyPairModel model=new KeyPairModel(); + try { + model.setSelectedItem(model.underlying.firstElement()); + } catch (NoSuchElementException e) { + // ignore + } + return new KeyPairCombo(model); + } public static KeyPairCombo create(AKeyPair kp) { KeyPairModel model=new KeyPairModel(); diff --git a/convex-gui/src/main/java/convex/gui/models/AccountsTableModel.java b/convex-gui/src/main/java/convex/gui/models/AccountsTableModel.java index 2661f52bf..0f560f57b 100644 --- a/convex-gui/src/main/java/convex/gui/models/AccountsTableModel.java +++ b/convex-gui/src/main/java/convex/gui/models/AccountsTableModel.java @@ -60,7 +60,8 @@ public Object getValueAt(int rowIndex, int columnIndex) { case 3: return as.getBalance(); case 4: { - ACell o = as.getHolding(Init.REGISTRY_ADDRESS); + AccountStatus registry = state.getAccount(Init.REGISTRY_ADDRESS); + ACell o = registry.getHolding(address); if (o == null) return ""; if (!(o instanceof AMap)) return ""; AMap a = (AMap) o; diff --git a/convex-gui/src/main/java/convex/gui/peer/PeerComponent.java b/convex-gui/src/main/java/convex/gui/peer/PeerComponent.java index 144acebf2..a8347c25f 100644 --- a/convex-gui/src/main/java/convex/gui/peer/PeerComponent.java +++ b/convex-gui/src/main/java/convex/gui/peer/PeerComponent.java @@ -1,8 +1,6 @@ package convex.gui.peer; -import java.io.IOException; import java.util.Objects; -import java.util.concurrent.TimeoutException; import javax.swing.JButton; import javax.swing.JMenuItem; @@ -12,13 +10,13 @@ import convex.api.Convex; import convex.api.ConvexLocal; import convex.api.ConvexRemote; +import convex.core.crypto.AKeyPair; +import convex.core.cvm.Address; import convex.core.cvm.Peer; +import convex.core.cvm.PeerStatus; import convex.core.cvm.State; -import convex.core.crypto.AKeyPair; import convex.core.data.ACell; import convex.core.data.AccountKey; -import convex.core.cvm.Address; -import convex.core.cvm.PeerStatus; import convex.core.text.Text; import convex.etch.EtchStore; import convex.gui.components.BaseImageButton; @@ -190,7 +188,7 @@ protected void launchClientWindow(Convex peer) { AKeyPair kp=peer.getKeyPair();; convex.setAddress(addr,kp); new REPLClient(convex).run(); - } catch (IOException | TimeoutException e) { + } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } diff --git a/convex-gui/src/main/java/convex/gui/peer/PeerGUI.java b/convex-gui/src/main/java/convex/gui/peer/PeerGUI.java index 636f6c1ec..df23459db 100644 --- a/convex-gui/src/main/java/convex/gui/peer/PeerGUI.java +++ b/convex-gui/src/main/java/convex/gui/peer/PeerGUI.java @@ -21,16 +21,17 @@ import convex.api.Convex; import convex.api.ConvexLocal; -import convex.core.cpos.Order; -import convex.core.cvm.Peer; import convex.core.Result; -import convex.core.cvm.State; +import convex.core.cpos.Order; import convex.core.crypto.AKeyPair; +import convex.core.crypto.wallet.AWalletEntry; import convex.core.crypto.wallet.HotWalletEntry; -import convex.core.data.AccountKey; import convex.core.cvm.Address; -import convex.core.data.Keyword; import convex.core.cvm.Keywords; +import convex.core.cvm.Peer; +import convex.core.cvm.State; +import convex.core.data.AccountKey; +import convex.core.data.Keyword; import convex.core.init.Init; import convex.core.store.AStore; import convex.core.store.Stores; @@ -56,8 +57,6 @@ public class PeerGUI extends AbstractGUI { protected JFrame frame; - public List KEYPAIRS=new ArrayList<>(); - private List PEERKEYS; public static final int DEFAULT_NUM_PEERS=3; @@ -65,6 +64,9 @@ public class PeerGUI extends AbstractGUI { public State genesisState; private StateModel latestState = StateModel.create(genesisState); public StateModel tickState = StateModel.create(0L); + + protected DefaultListModel peerList = new DefaultListModel(); + /** * Launch the application. @@ -80,12 +82,60 @@ public static void main(String[] args) throws Exception { System.exit(0); } - public static PeerGUI launchPeerGUI(int peerNum, AKeyPair genesis, boolean topLevel) throws InterruptedException, PeerException { - PeerGUI manager = new PeerGUI(peerNum,genesis); + public static PeerGUI launchPeerGUI(int peerNum, AKeyPair genesisKey, boolean topLevel) throws InterruptedException, PeerException { + PeerGUI manager = create(peerNum,genesisKey); manager.run(); return manager; } + + public static PeerGUI launchPeerGUI(InetSocketAddress sa, AWalletEntry we) throws InterruptedException, PeerException { + DefaultListModel peerList=new DefaultListModel<>(); + + HashMap config=new HashMap<>(); + config.put(Keywords.KEYPAIR,we.getKeyPair()); + config.put(Keywords.SOURCE,sa); + Server server=API.launchPeer(config); + ConvexLocal convex=ConvexLocal.connect(server); + peerList.addElement(convex); + PeerGUI manager = new PeerGUI(peerList); + manager.run(); + return manager; + } + public static PeerGUI create(int peerCount, AKeyPair genesisKey) throws PeerException { + DefaultListModel peerList=launchAllPeers(peerCount,genesisKey); + return new PeerGUI(peerList); + } + + + private static DefaultListModel launchAllPeers(int peerCount, AKeyPair genesisKey) throws PeerException { + List KEYPAIRS=new ArrayList<>(); + List PEERKEYS; + for (int i=0; ikp.getAccountKey()).collect(Collectors.toList()); + State genesisState=Init.createState(PEERKEYS); + + try { + DefaultListModel peerList=new DefaultListModel<>(); + List serverList = API.launchLocalPeers(KEYPAIRS,genesisState); + for (Server server: serverList) { + ConvexLocal convex=Convex.connect(server, server.getPeerController(), server.getKeyPair()); + peerList.addElement(convex); + + // initial wallet list + HotWalletEntry we = HotWalletEntry.create(server.getKeyPair(),"Peer key pair"); + KeyRingPanel.addWalletEntry(we); + } + return peerList; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new PeerException("Peer launch interrupted",e); + } + } /* * Main component panel */ @@ -98,7 +148,6 @@ public static PeerGUI launchPeerGUI(int peerNum, AKeyPair genesis, boolean topLe JTabbedPane tabs; RESTServer restServer; - AKeyPair genesisKey; /** * Create the application. @@ -106,25 +155,22 @@ public static PeerGUI launchPeerGUI(int peerNum, AKeyPair genesis, boolean topLe * @param peerCount number of peers to initialise in genesis * @throws PeerException If peer startup fails */ - public PeerGUI(int peerCount, AKeyPair genesis) throws PeerException { + private PeerGUI(DefaultListModel peerList) throws PeerException { super ("Peer Manager"); - // Create key pairs for peers, use genesis key as first keypair - genesisKey=genesis; - for (int i=0; ikp.getAccountKey()).collect(Collectors.toList()); - genesisState=Init.createState(PEERKEYS); + // Create key pairs for peers, use genesis key as first keypair + Server firstServer=peerList.get(0).getLocalServer(); + genesisState=firstServer.getPeer().getGenesisState(); + latestState = StateModel.create(genesisState); tickState = StateModel.create(0L); - // launch local peers + + serverPanel= new ServerListPanel(this); keyRingPanel = new KeyRingPanel(); - launchAllPeers(); Server first=peerList.firstElement().getLocalServer(); ConvexLocal convex=Convex.connect(first); @@ -210,7 +256,6 @@ public void run() { } }; - protected DefaultListModel peerList = new DefaultListModel(); public DefaultListModel getPeerList() { return peerList; @@ -244,8 +289,9 @@ public KeyRingPanel getWalletPanel() { * @return Convex connection instance * @throws IOException If IO error occurs during connection attempt * @throws TimeoutException If attempt to connect times out + * @throws InterruptedException */ - public Convex makeConnection(Address address,AKeyPair kp) throws IOException, TimeoutException { + public Convex makeConnection(Address address,AKeyPair kp) throws IOException, TimeoutException, InterruptedException { InetSocketAddress host = getDefaultConvex().getHostAddress(); return Convex.connect(host,address, kp); } @@ -271,7 +317,7 @@ public Convex getClientConvex(Address contract) { return Convex.connect(getPrimaryServer(),contract,null); } - public Convex connectClient(Address address, AKeyPair keyPair) throws IOException, TimeoutException { + public Convex connectClient(Address address, AKeyPair keyPair) throws IOException, TimeoutException, InterruptedException { return makeConnection(address,keyPair); } @@ -333,22 +379,6 @@ public Server getPrimaryServer() { return null; } - public void launchAllPeers() throws PeerException { - try { - List serverList = API.launchLocalPeers(KEYPAIRS,genesisState); - for (Server server: serverList) { - ConvexLocal convex=Convex.connect(server, server.getPeerController(), server.getKeyPair()); - peerList.addElement(convex); - - // initial wallet list - HotWalletEntry we = HotWalletEntry.create(server.getKeyPair(),"Peer key pair"); - KeyRingPanel.addWalletEntry(we); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new PeerException("Peer launch interrupted",e); - } - } public void launchExtraPeer() { AKeyPair kp=AKeyPair.generate(); @@ -415,4 +445,8 @@ public void setupFrame(JFrame frame) { frame.getContentPane().add(this,"dock center"); } + + + + } \ No newline at end of file diff --git a/convex-gui/src/main/java/convex/gui/peer/PeerLaunchDialog.java b/convex-gui/src/main/java/convex/gui/peer/PeerLaunchDialog.java index 5138ab1fa..5366731a6 100644 --- a/convex-gui/src/main/java/convex/gui/peer/PeerLaunchDialog.java +++ b/convex-gui/src/main/java/convex/gui/peer/PeerLaunchDialog.java @@ -1,6 +1,7 @@ package convex.gui.peer; import java.awt.Color; +import java.net.InetSocketAddress; import javax.swing.JButton; import javax.swing.JComponent; @@ -8,39 +9,43 @@ import javax.swing.JOptionPane; import javax.swing.JPanel; import javax.swing.JSpinner; +import javax.swing.JTabbedPane; import javax.swing.SpinnerNumberModel; import convex.core.crypto.AKeyPair; import convex.core.crypto.wallet.AWalletEntry; import convex.core.crypto.wallet.HotWalletEntry; +import convex.gui.components.HostCombo; import convex.gui.components.Toast; import convex.gui.components.account.KeyPairCombo; import convex.gui.keys.UnlockWalletDialog; import convex.gui.utils.SymbolIcon; import convex.gui.utils.Toolkit; +import convex.net.IPUtils; import net.miginfocom.swing.MigLayout; public class PeerLaunchDialog { public static void runLaunchDialog(JComponent parent) { - JPanel pan=new JPanel(); - pan.setLayout(new MigLayout("fill,wrap 3","","[fill]10[fill][40]")); + // Local testnet options - pan.add(new JLabel("Number of Peers:")); + JPanel testNetPanel=new JPanel(); + testNetPanel.setLayout(new MigLayout("fill,wrap 3","","[fill]10[fill][40]")); + + testNetPanel.add(new JLabel("Number of Peers:")); JSpinner peerCountSpinner = new JSpinner(); // Note: about 300 max number of clients before hitting juice limits for account creation peerCountSpinner.setModel(new SpinnerNumberModel(PeerGUI.DEFAULT_NUM_PEERS, 1, 100, 1)); - pan.add(peerCountSpinner); - pan.add(Toolkit.makeHelp("Select a number of peers to include in the genesis state and launch initially. More can be added later. 3-5 recommended for local devnet testing")); + testNetPanel.add(peerCountSpinner); + testNetPanel.add(Toolkit.makeHelp("Select a number of peers to include in the genesis state and launch initially. More can be added later. 3-5 recommended for local devnet testing")); - pan.add(new JLabel("Genesis Key: ")); - AKeyPair kp=AKeyPair.generate(); - KeyPairCombo keyField=KeyPairCombo.create(kp); + testNetPanel.add(new JLabel("Genesis Key: ")); + KeyPairCombo keyField=KeyPairCombo.create(); - pan.add(keyField); - pan.add(Toolkit.makeHelp("Select genesis key for the network. The genesis key will be the key used for the first peer and initial governance accounts.")); + testNetPanel.add(keyField); + testNetPanel.add(Toolkit.makeHelp("Select genesis key for the network. The genesis key will be the key used for the first peer and initial governance accounts.")); - pan.add(new JPanel()); + testNetPanel.add(new JPanel()); JButton randomise=new JButton("Randomise",SymbolIcon.get(0xe863,Toolkit.SMALL_ICON_SIZE)); randomise.addActionListener(e->{ @@ -50,31 +55,60 @@ public static void runLaunchDialog(JComponent parent) { // setting a selected item to something not in the list when not editable keyField.getModel().setSelectedItem(HotWalletEntry.create(newKP,"Random genesis key pair for testing")); }); - pan.add(randomise); - pan.add(Toolkit.makeHelp("Randomise the genesis key. Fine for testing purposes.")); - - int result = JOptionPane.showConfirmDialog(parent, pan, + testNetPanel.add(randomise); + testNetPanel.add(Toolkit.makeHelp("Randomise the genesis key. Fine for testing purposes.")); + + + // Temporary peer options + JPanel joinPanel=new JPanel(); + joinPanel.setLayout(new MigLayout("fill,wrap 3","","[fill]10[fill][40]")); + joinPanel.add(new JLabel("Source Peer:")); + HostCombo hostField=new HostCombo(); + hostField.setToolTipText("Enter a peer address to join e.g. peer.convex.live:18888"); + joinPanel.add(hostField); + joinPanel.add(Toolkit.makeHelp("Select an existing peer to join with the new peer. Should be a trusted source for the global state and current consensus ordering.")); + + joinPanel.add(new JLabel("Peer Key: ")); + KeyPairCombo peerKeyField=KeyPairCombo.create(); + joinPanel.add(peerKeyField); + joinPanel.add(Toolkit.makeHelp("Select peer key for the new peer.")); + + JTabbedPane tabs=new JTabbedPane(); + tabs.add(testNetPanel,"Local Testnet"); + tabs.add(joinPanel,"Join Network"); + + int result = JOptionPane.showConfirmDialog(parent, tabs, "Peer Launch Details", JOptionPane.OK_CANCEL_OPTION, JOptionPane.QUESTION_MESSAGE, SymbolIcon.get(0xeb9b,Toolkit.ICON_SIZE)); if (result == JOptionPane.OK_OPTION) { try { - int numPeers=(Integer)peerCountSpinner.getValue(); - AWalletEntry we=keyField.getWalletEntry(); - if (we==null) throw new IllegalStateException("No key pair selected"); - - if (we.isLocked()) { - boolean unlocked= UnlockWalletDialog.offerUnlock(parent,we); - if (!unlocked) { - Toast.display(parent, "Launch cancelled: Locked genesis key", Color.RED); - return; - } - } - - kp=we.getKeyPair(); - - PeerGUI.launchPeerGUI(numPeers, kp,false); + if (tabs.getSelectedComponent()==testNetPanel) { + int numPeers=(Integer)peerCountSpinner.getValue(); + AWalletEntry we=keyField.getWalletEntry(); + if (we==null) throw new IllegalStateException("No key pair selected"); + + if (we.isLocked()) { + boolean unlocked= UnlockWalletDialog.offerUnlock(parent,we); + if (!unlocked) { + Toast.display(parent, "Launch cancelled: Locked genesis key", Color.RED); + return; + } + } + + AKeyPair kp=we.getKeyPair(); + PeerGUI.launchPeerGUI(numPeers, kp,false); + } else if (tabs.getSelectedComponent()==joinPanel) { + String host=hostField.getText(); + InetSocketAddress sa=IPUtils.toInetSocketAddress(host); + if (sa==null) throw new IllegalArgumentException("Invalid host address for joining"); + + AWalletEntry we=peerKeyField.getWalletEntry(); + if (we==null) throw new IllegalArgumentException("No peer key selected"); + + PeerGUI.launchPeerGUI(sa,we); + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } catch (Exception e) { diff --git a/convex-gui/src/main/java/convex/gui/repl/REPLPanel.java b/convex-gui/src/main/java/convex/gui/repl/REPLPanel.java index 5e24e0398..fc50c5109 100644 --- a/convex-gui/src/main/java/convex/gui/repl/REPLPanel.java +++ b/convex-gui/src/main/java/convex/gui/repl/REPLPanel.java @@ -171,7 +171,6 @@ public REPLPanel(Convex convex) { output.setFont(OUTPUT_FONT); //outputArea.setForeground(Color.GREEN); output.setBackground(new Color(10,10,10)); - output.setFocusable(false); output.setToolTipText("Output from transaction execution"); //DefaultCaret caret = (DefaultCaret)(outputArea.getCaret()); //caret.setUpdatePolicy(DefaultCaret.ALWAYS_UPDATE); diff --git a/convex-gui/src/main/java/convex/gui/server/StressPanel.java b/convex-gui/src/main/java/convex/gui/server/StressPanel.java index 144ef0e02..2856266d6 100644 --- a/convex-gui/src/main/java/convex/gui/server/StressPanel.java +++ b/convex-gui/src/main/java/convex/gui/server/StressPanel.java @@ -11,6 +11,7 @@ import java.util.HashMap; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeoutException; @@ -214,6 +215,9 @@ protected String doInBackground() { running=repeatCheckBox.isSelected(); if (running) Thread.sleep(((Integer)(repeatTimeSpinner.getValue()))*1000); }; + } catch (ExecutionException e) { + log.info("Stress test worker terminated",e); + resultArea.setText("Test Error: "+e); } catch (Exception e) { log.warn("Stress test worker terminated unexpectedly",e); resultArea.setText("Test Error: "+e); @@ -352,7 +356,7 @@ private void setupClients() throws IOException, TimeoutException { } } - protected void connectClients(AVector
clientAddresses) throws IOException, TimeoutException { + protected void connectClients(AVector
clientAddresses) throws IOException, TimeoutException, InterruptedException { for (int i=0; i query(ACell form, Address address) { } @Override - public CompletableFuture message(Blob message) { + public CompletableFuture messageRaw(Blob message) { throw new TODOException(); } diff --git a/convex-peer/pom.xml b/convex-peer/pom.xml index 9e092b4cd..6e7bd4ecc 100644 --- a/convex-peer/pom.xml +++ b/convex-peer/pom.xml @@ -39,6 +39,12 @@ slf4j-api ${slf4j.version} + + io.netty + netty-all + 4.1.115.Final + compile + org.junit.jupiter diff --git a/convex-peer/src/main/java/convex/api/Acquiror.java b/convex-peer/src/main/java/convex/api/Acquiror.java index 2bc738e3d..5fe523b89 100644 --- a/convex-peer/src/main/java/convex/api/Acquiror.java +++ b/convex-peer/src/main/java/convex/api/Acquiror.java @@ -4,12 +4,12 @@ import java.util.HashSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import convex.core.Result; import convex.core.cpos.CPoSConstants; import convex.core.data.ACell; import convex.core.data.AVector; @@ -19,13 +19,12 @@ import convex.core.data.prim.CVMLong; import convex.core.exceptions.BadFormatException; import convex.core.exceptions.MissingDataException; +import convex.core.exceptions.ResultException; import convex.core.lang.RT; import convex.core.store.AStore; -import convex.core.store.Stores; import convex.core.util.ThreadUtils; import convex.core.util.Utils; import convex.net.Message; -import convex.net.MessageType; /** * Utility class for acquiring data remotely @@ -33,7 +32,9 @@ public class Acquiror { private static final Logger log = LoggerFactory.getLogger(Acquiror.class.getName()); - private static final int ACQUIRE_LOOP_TIMEOUT=2000; + + // Probably don't need this, can time out the future + // private static final int ACQUIRE_LOOP_TIMEOUT=2000; private Hash hash; @@ -66,9 +67,8 @@ public CompletableFuture getFuture() { return f; } log.trace("Trying to acquire remotely: {}",hash); - - ThreadUtils.runVirtual(()-> { - Stores.setCurrent(store); // use store for calling thread + + ThreadUtils.runWithStore(store,()-> { try { HashSet missingSet = new HashSet<>(); @@ -94,35 +94,29 @@ public CompletableFuture getFuture() { return; } } - CVMLong id=CVMLong.create(source.connection.getNextID()); + CVMLong id=CVMLong.create(source.getNextID()); Message dataRequest=Message.createDataRequest(id, missingSet.toArray(Utils.EMPTY_HASHES)); - CompletableFuture cf=new CompletableFuture(); - synchronized (source.awaiting) { - boolean sent=source.connection.sendMessage(dataRequest); - if (!sent) { - log.warn("Unable to send data acquisition request"); - continue; - } - cf=cf.orTimeout(ACQUIRE_LOOP_TIMEOUT,TimeUnit.MILLISECONDS); - // Store future for completion by result message - source.awaiting.put(id,cf); - } + CompletableFuture cf=source.message(dataRequest); try { - Message resp=cf.get(); - if (resp.getType()==MessageType.DATA) { - log.trace("Got acquire response: {} ",resp); - AVector v=resp.getPayload(); - for (int i=1; i reqv=dataRequest.getPayload(); - f.completeExceptionally(new MissingDataException(store,RT.ensureHash(reqv.get(i)))); - continue; - } - Cells.store(v.get(i), store); + Result resp=cf.get(); + if (resp.isError()) { + f.completeExceptionally(new ResultException(resp)); + log.info("Failed to request missing data: "+resp); + return; + } + + AVector v=RT.ensureVector(resp.getValue()); + if (v==null) throw new BadFormatException("Expected Vector in data result for id "+id+" but was: "+resp); + for (int i=0; i reqv=dataRequest.getPayload(); + Hash expectedHash=RT.ensureHash(reqv.get(i+2)); + f.completeExceptionally(new MissingDataException(store,expectedHash)); + continue; } - } else { - log.warn("Unexpected data response type: "+resp.getType()); + Cells.store(val, store); } } catch (ExecutionException e) { if (e.getCause() instanceof TimeoutException) { diff --git a/convex-peer/src/main/java/convex/api/Convex.java b/convex-peer/src/main/java/convex/api/Convex.java index cb17603dd..fc9f473e7 100644 --- a/convex-peer/src/main/java/convex/api/Convex.java +++ b/convex-peer/src/main/java/convex/api/Convex.java @@ -3,6 +3,7 @@ import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.util.HashMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -51,11 +52,13 @@ import convex.peer.Server; /** - * Class representing a client API to the Convex network. + * Class providing a client API to the Convex network. * - * An instance of the type Convex represents a stateful client connection to the + * An instance of the type Convex manages a stateful client connection to the * Convex network that can issue transactions both synchronously and - * asynchronously. This can be used by both peers and JVM-based clients. + * asynchronously via a peer. + * + * This can be used by both peers and JVM-based clients. * * "I'm doing a (free) operating system (just a hobby, won't be big and * professional like gnu)" - Linus Torvalds @@ -93,49 +96,11 @@ public abstract class Convex implements AutoCloseable { */ protected Long sequence = null; + /** - * Map of results awaiting completion. - */ - protected HashMap> awaiting = new HashMap<>(); - - /** - * Result Consumer for messages received back from a client connection + * Counter for outgoing message IDs. Used to give an ID to requests that expect a Result */ - protected final Consumer messageHandler = new ResultConsumer() { - @Override - protected synchronized void handleResult(ACell id, Result v) { - ACell ec=v.getErrorCode(); - - if ((ec!=null)&&(!SourceCodes.CODE.equals(v.getSource()))) { - // if our result didn't result in user code execution - // then we probably have a wrong sequence number now. Kill the stored value. - sequence = null; - } - } - - @Override - public void accept(Message m) { - // Check if we are waiting for a Result with this ID for this connection - synchronized (awaiting) { - ACell id=m.getID(); - CompletableFuture cf = (id==null)?null:awaiting.remove(id); - if (cf != null) { - // log.info("Return message received for message ID: {} with type: {} "+m.toString(), id,m.getType()); - if (cf.complete(m)) return; - } - } - - if (delegatedHandler!=null) { - delegatedHandler.accept(m); - } else { - // default handling - super.accept(m); - } - } - }; - - private Consumer delegatedHandler=null; - + protected long idCounter=0; protected Convex(Address address, AKeyPair keyPair) { this.keyPair = keyPair; @@ -146,8 +111,9 @@ protected Convex(Address address, AKeyPair keyPair) { * Attempts best possible connection * @throws TimeoutException * @throws IOException + * @throws InterruptedException */ - public static Convex connect(Object host) throws IOException, TimeoutException { + public static Convex connect(Object host) throws IOException, TimeoutException, InterruptedException { if (host instanceof Convex) return (Convex)host; if (host instanceof Convex) return connect((Server)host); @@ -165,8 +131,9 @@ public static Convex connect(Object host) throws IOException, TimeoutException { * @return New Convex client instance * @throws IOException If IO Error occurs * @throws TimeoutException If connection attempt times out + * @throws InterruptedException */ - public static ConvexRemote connect(InetSocketAddress hostAddress) throws IOException, TimeoutException { + public static ConvexRemote connect(InetSocketAddress hostAddress) throws IOException, TimeoutException, InterruptedException { return connect(hostAddress, (Address) null, (AKeyPair) null); } @@ -180,30 +147,19 @@ public static ConvexRemote connect(InetSocketAddress hostAddress) throws IOExcep * @return New Convex client instance * @throws IOException If connection fails due to IO error * @throws TimeoutException If connection attempt times out + * @throws InterruptedException */ public static ConvexRemote connect(InetSocketAddress peerAddress, Address address, AKeyPair keyPair) - throws IOException, TimeoutException { - return Convex.connect(peerAddress, address, keyPair, Stores.current()); - } - - /** - * Create a Convex client by connecting to the specified Peer using the given - * key pair and using a given store - * - * @param peerAddress Address of Peer - * @param address Address of Account to use for Client - * @param keyPair Key pair to use for client transactions - * @param store Store to use for this connection - * @return New Convex client instance - * @throws IOException If connection fails due to IO error - * @throws TimeoutException If connection attempt times out - */ - public static ConvexRemote connect(InetSocketAddress peerAddress, Address address, AKeyPair keyPair, AStore store) - throws IOException, TimeoutException { - ConvexRemote convex = new ConvexRemote(address, keyPair); - convex.connectToPeer(peerAddress, store); + throws IOException, TimeoutException, InterruptedException { + ConvexRemote convex = ConvexRemote.connect(peerAddress); + convex.setAddress(address); + convex.setKeyPair(keyPair); return convex; } + + protected long getNextID() { + return idCounter++; + } /** * Sets the Address for this connection. This will be used for subsequent @@ -240,15 +196,6 @@ public void setNextSequence(long nextSequence) { this.sequence = nextSequence - 1L; } - /** - * Sets a handler for messages that are received but not otherwise processed (transaction/query results will - * be relayed instead to the appropriate handler ) - * @param handler Handler for received messaged - */ - public void setHandler(Consumer handler) { - this.delegatedHandler = handler; - } - /** * Gets the current sequence number for this Client, which is the sequence * number of the last transaction observed for the current client's Account. @@ -447,6 +394,7 @@ public SignedData prepareTransaction(ACell code) throws ResultExce if (compResult.isError()) throw new ResultException(compResult); code=compResult.getValue(); } + if (address==null) throw new ResultException(Result.error(ErrorCodes.STATE,"No origin address for transaction")); transaction=Invoke.create(address, ATransaction.UNKNOWN_SEQUENCE, code); } return prepareTransaction(transaction); @@ -727,16 +675,15 @@ public CompletableFuture query(String query) { * @param message Raw message data * @return A Future for the result of the query */ - public abstract CompletableFuture message(Blob message); + public abstract CompletableFuture messageRaw(Blob message); /** - * Submits a Message to the Convex network, returning a Future for any Result + * Submits a Message to the connected peer, returning a Future for any Result * * @param message Message data - * @return A Future for the result of the query + * @return A Future for the Result of the query. May just be "Sent" if no other result expected, or an immediate error if sending failed. */ public abstract CompletableFuture message(Message message); - /** * Attempts to resolve a CNS name @@ -799,37 +746,6 @@ public Result requestStatusSync(long timeoutMillis) { */ public abstract CompletableFuture requestStatus(); - /** - * Method to start waiting for a complete result. Must be called with lock on - * `awaiting` map to prevent risk of missing results before it is called. - * - * @param id ID of result message to await - * @return - */ - protected CompletableFuture awaitResult(ACell id, long timeout) { - CompletableFuture cf = new CompletableFuture(); - if (timeout>0) { - cf=cf.orTimeout(timeout, TimeUnit.MILLISECONDS); - } - CompletableFuture cr=cf.handle((m,e)->{ - synchronized(awaiting) { - awaiting.remove(id); - } - // clear sequence if something went wrong. It is probably invalid now.... - if (e!=null) { - sequence=null; - return Result.fromException(e); - } - Result r=m.toResult(); - if (r.getErrorCode()!=null) { - sequence=null; - } - return r; - }); - awaiting.put(id, cf); - return cr; - } - /** * Request a challenge. This is request is made by any peer that needs to find * out if another peer can be trusted. @@ -900,7 +816,7 @@ protected Result querySync(ACell query, Address address, long timeoutMillis) thr try { result = cf.get(timeoutMillis, TimeUnit.MILLISECONDS); } catch (ExecutionException | TimeoutException e) { - return Result.fromException(e.getCause()); + return Result.fromException(e); } finally { cf.cancel(true); } @@ -1023,7 +939,8 @@ public static ConvexLocal connect(Server server, Address address, AKeyPair keyPa * @return New Client Connection */ public static ConvexLocal connect(Server server) { - return ConvexLocal.create(server, null, null); + ConvexLocal convex= ConvexLocal.create(server, null, null); + return convex; } /** diff --git a/convex-peer/src/main/java/convex/api/ConvexLocal.java b/convex-peer/src/main/java/convex/api/ConvexLocal.java index a49b69029..f64244a8d 100644 --- a/convex-peer/src/main/java/convex/api/ConvexLocal.java +++ b/convex-peer/src/main/java/convex/api/ConvexLocal.java @@ -19,10 +19,9 @@ import convex.core.data.Hash; import convex.core.data.Ref; import convex.core.data.SignedData; -import convex.core.data.Vectors; -import convex.core.data.prim.CVMLong; import convex.core.exceptions.MissingDataException; import convex.core.store.AStore; +import convex.core.store.Stores; import convex.core.util.ThreadUtils; import convex.net.Message; import convex.net.MessageType; @@ -41,6 +40,10 @@ protected ConvexLocal(Server server, Address address, AKeyPair keyPair) { this.preCompile=true; // pre-compile by default if local peer } + public static ConvexLocal create(Server server) { + return new ConvexLocal(server, null,null); + } + public static ConvexLocal create(Server server, Address address, AKeyPair keyPair) { return new ConvexLocal(server, address,keyPair); } @@ -73,14 +76,14 @@ public CompletableFuture acquire(Hash hash, AStore store) { @Override public CompletableFuture requestStatus() { - return makeMessageFuture(MessageType.STATUS,CVMLong.create(makeID())); + return makeMessageFuture(Message.createStatusRequest(getNextID())); } @Override public CompletableFuture transact(SignedData signed) { maybeUpdateSequence(signed); - CompletableFuture r= makeMessageFuture(MessageType.TRANSACT,Vectors.of(makeID(),signed)); + CompletableFuture r= makeMessageFuture(Message.createTransaction(getNextID(),signed)); return r; } @@ -92,14 +95,10 @@ public CompletableFuture requestChallenge(SignedData data) { @Override public CompletableFuture query(ACell query, Address address) { - return makeMessageFuture(Message.createQuery(makeID(),query,address)); + return makeMessageFuture(Message.createQuery(getNextID(),query,address)); } - private long idCounter=0; - - private long makeID() { - return idCounter++; - } + private CompletableFuture makeMessageFuture(MessageType type, ACell payload) { Message ml=Message.create(type,payload); @@ -120,15 +119,46 @@ private CompletableFuture makeMessageFuture(Message message) { } private Predicate makeResultHandler(CompletableFuture cf) { + AStore senderStore=Stores.current(); return m->{ - Result r=m.toResult(); - if (r.getErrorCode()!=null) { - sequence=null; + // Protect message reading in sender store + AStore savedStore=Stores.current(); + try { + Stores.setCurrent(senderStore); + Result r=m.toResult(); + if (r.getErrorCode()!=null) { + sequence=null; + } + cf.complete(r); + return true; + } finally { + Stores.setCurrent(savedStore); } - cf.complete(r); - return true; }; } + + @Override + public CompletableFuture messageRaw(Blob rawData) { + try { + Message m=Message.create(rawData); + return message(m); + } catch (Exception e) { + return CompletableFuture.completedFuture(Result.fromException(e).withSource(SourceCodes.CLIENT)); + } + } + + @Override + public CompletableFuture message(Message message) { + ACell id=message.getRequestID(); + if (id==null) { + // directly forward message to Server + server.getReceiveAction().accept(message); + return CompletableFuture.completedFuture(Result.SENT_MESSAGE); + } + + // We are expecting a return message, so build a completable future for it + return makeMessageFuture(message); + } @Override public void close() { @@ -180,19 +210,6 @@ public Long getBalance() { return server.getPeer().getConsensusState().getBalance(address); } - @Override - public CompletableFuture message(Blob rawData) { - try { - Message m=Message.create(rawData); - return message(m); - } catch (Exception e) { - return CompletableFuture.completedFuture(Result.fromException(e).withSource(SourceCodes.CLIENT)); - } - } - @Override - public CompletableFuture message(Message message) { - return makeMessageFuture(message); - } } diff --git a/convex-peer/src/main/java/convex/api/ConvexRemote.java b/convex-peer/src/main/java/convex/api/ConvexRemote.java index 9a3e8bb41..b107f216e 100644 --- a/convex-peer/src/main/java/convex/api/ConvexRemote.java +++ b/convex-peer/src/main/java/convex/api/ConvexRemote.java @@ -2,8 +2,11 @@ import java.io.IOException; import java.net.InetSocketAddress; +import java.util.HashMap; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,23 +23,29 @@ import convex.core.data.Blob; import convex.core.data.Hash; import convex.core.data.SignedData; -import convex.core.data.prim.CVMLong; import convex.core.exceptions.ResultException; import convex.core.exceptions.TODOException; import convex.core.lang.RT; import convex.core.store.AStore; import convex.core.store.Stores; -import convex.net.Connection; +import convex.net.AConnection; import convex.net.Message; +import convex.net.impl.netty.NettyConnection; +import convex.net.impl.nio.Connection; +import convex.peer.Config; import convex.peer.Server; +/** + * Convex client API implementation for peers accessed over a network connection using the Convex binary peer protocol + * + */ public class ConvexRemote extends Convex { /** * Current Connection to a Peer, may be null or a closed connection. */ - protected Connection connection; + protected AConnection connection; - private static final Logger log = LoggerFactory.getLogger(ConvexRemote.class.getName()); + protected static final Logger log = LoggerFactory.getLogger(ConvexRemote.class.getName()); protected InetSocketAddress remoteAddress; @@ -50,16 +59,126 @@ protected ConvexRemote(Address address, AKeyPair keyPair) { super(address, keyPair); } - protected void connectToPeer(InetSocketAddress peerAddress, AStore store) throws IOException, TimeoutException { + protected void connectToPeer(InetSocketAddress peerAddress) throws IOException, TimeoutException, InterruptedException { remoteAddress=peerAddress; - setConnection(Connection.connect(peerAddress, messageHandler, store)); + if (Config.USE_NETTY_CLIENT) { + setConnection(NettyConnection.connect(peerAddress, returnMessageHandler)); + } else { + setConnection(Connection.connect(peerAddress, returnMessageHandler)); + } + // setConnection(NettyConnection.connect(peerAddress, returnMessageHandler)); + } + + public static ConvexRemote connect(InetSocketAddress peerAddress) throws IOException, TimeoutException, InterruptedException { + ConvexRemote convex=new ConvexRemote(null,null); + convex.connectToPeer(peerAddress); + return convex; + } + + public static ConvexRemote connectNetty(InetSocketAddress sa) throws InterruptedException { + ConvexRemote convex=new ConvexRemote(null,null); + convex.remoteAddress=sa; + convex.setConnection(NettyConnection.connect(sa, convex.returnMessageHandler)); + return convex; + } + + public static ConvexRemote connectNIO(InetSocketAddress sa) throws InterruptedException, IOException, TimeoutException { + ConvexRemote convex=new ConvexRemote(null,null); + convex.remoteAddress=sa; + convex.setConnection(Connection.connect(sa, convex.returnMessageHandler)); + return convex; + } + + /** + * Map of results awaiting completion. + */ + private HashMap> awaiting = new HashMap<>(); + + /** + * Method to start waiting for a return Message. + * + * Must be called with lock on + * `awaiting` map to prevent risk of missing results before it is called. + * + * @param resultID ID of result message to await + * @return + */ + private CompletableFuture awaitResult(ACell resultID, long timeout) { + if (resultID==null) throw new IllegalArgumentException("Non-null return ID required"); + + // Save store from the sending thread. We want to decode the Result on this store! + AStore awaitingStore=Stores.current(); + + CompletableFuture cf = new CompletableFuture(); + awaiting.put(resultID, cf); + + if (timeout>0) { + cf=cf.orTimeout(timeout, TimeUnit.MILLISECONDS); + } + CompletableFuture cr=cf.handle((m,e)->{ + synchronized(awaiting) { + // no longer want to wait for this result + // either we go a result back, or the future failed + awaiting.remove(resultID); + } + + // Set the store. Likely to be needed by anyone waiting on the future + // We don't need to restore it because the return message handler does that for us + Stores.setCurrent(awaitingStore); + + // clear sequence if something went wrong. It is probably invalid now.... + if (e!=null) { + sequence=null; + return Result.fromException(e); + } + + Result r=m.toResult(); + if (r.getErrorCode()!=null) { + sequence=null; + } + return r; + }); + return cr; } + + /** + * Result handler for Messages received back from a remote connection + */ + protected final Consumer returnMessageHandler = m-> { + ACell id=m.getResultID(); + + if (id!=null) { + // Check if we are waiting for a Result with this ID for this connection + synchronized (awaiting) { + // We save and restore the Store, since completing the future might change it + AStore savedStore=Stores.current(); + try { + CompletableFuture cf = awaiting.get(id); + if (cf != null) { + // log.info("Return message received for message ID: {} with type: {} "+m.toString(), id,m.getType()); + boolean didComplete = cf.complete(m); + if (!didComplete) { + log.warn("Message return future already completed with value: "+cf.join()); + } + awaiting.remove(id); + } + } catch (Exception e) { + log.warn("Unexpected error completing result",e); + } finally { + Stores.setCurrent(savedStore); + } + + } + } else { + // Ignore the message, we are a client side connection so not interested. + } + }; - public void reconnect() throws IOException, TimeoutException { - Connection curr=connection; - AStore store=(curr==null)?Stores.current():curr.getStore(); + + + public synchronized void reconnect() throws IOException, TimeoutException, InterruptedException { close(); - setConnection(Connection.connect(remoteAddress, messageHandler, store)); + connectToPeer(remoteAddress); } /** @@ -67,9 +186,9 @@ public void reconnect() throws IOException, TimeoutException { * * @param conn Connection value to use */ - protected void setConnection(Connection conn) { - Connection curr=this.connection; - if (curr == conn) return; + protected void setConnection(AConnection conn) { + AConnection curr=this.connection; + if (curr == conn) return; // no change if (curr!=null) close(); this.connection = conn; } @@ -80,18 +199,10 @@ protected void setConnection(Connection conn) { * @return true if connected, false otherwise */ public boolean isConnected() { - Connection c = this.connection; + AConnection c = this.connection; return (c != null) && (!c.isClosed()); } - /** - * Close without affecting the underlying connection (will be unlinked but not closed) - */ - public void closeButMaintainConnection() { - this.connection = null; - close(); - } - @Override public CompletableFuture acquireState() { AStore store=Stores.current(); @@ -106,117 +217,71 @@ public CompletableFuture acquireState() { } @Override - public synchronized CompletableFuture transact(SignedData signed) { - long id = -1; - long wait=10; - - // loop until request is queued. We need this for backpressure - while (true) { - if (connection.isClosed()) return closedResult; - - try { - synchronized (awaiting) { - id = connection.sendTransaction(signed); - if (id>=0) { - // Store future for completion by result message - maybeUpdateSequence(signed); - CompletableFuture cf = awaitResult(CVMLong.create(id),timeout); - log.trace("Sent transaction with message ID: {} awaiting count = {}", id, awaiting.size()); - return cf; - } - } - - Thread.sleep(wait); - wait+=1+wait/3; // slow exponential backoff - } catch (InterruptedException e) { - // we honour the interruption, but return a failed result - Result r=Result.fromException(e); - return CompletableFuture.completedFuture(r); - } catch (IOException e) { - Result r=Result.fromException(e).withInfo(Keywords.SOURCE,SourceCodes.COMM); - return CompletableFuture.completedFuture(r); - } - } + public CompletableFuture transact(SignedData signed) { + Message m=Message.createTransaction(getNextID(), signed); + return message(m); } - private static CompletableFuture closedResult=CompletableFuture.completedFuture(Result.error(ErrorCodes.CLOSED, "Transaction interrupted before sending").withSource(SourceCodes.COMM)); - @Override public CompletableFuture query(ACell query, Address address) { - long wait=10; - - // loop until request is queued. We need this for backpressure - while (true) { - if (connection.isClosed()) return closedResult; - - // If we can't send yet, block briefly and try again - try { - synchronized (awaiting) { - long id = connection.sendQuery(query, address); - if(id>=0) { - CompletableFuture cf= awaitResult(CVMLong.create(id),timeout); - return cf; - } - } - - Thread.sleep(wait); - wait+=1+wait/3; // slow exponential backoff - } catch (InterruptedException e) { - // This handles interrupts correctly, returning a failed result - Result r= Result.fromException(e); - return CompletableFuture.completedFuture(r); - } catch (IOException e) { - Result r=Result.fromException(e).withInfo(Keywords.SOURCE,SourceCodes.COMM); - return CompletableFuture.completedFuture(r); - } - } + Message m=Message.createQuery(getNextID(), query,address); + return message(m); } @Override - public CompletableFuture message(Blob message) { + public CompletableFuture messageRaw(Blob message) { throw new TODOException(); } @Override - public CompletableFuture message(Message message) { - throw new TODOException(); - } - - @Override - public CompletableFuture requestStatus() { + public CompletableFuture message(Message m) { + AConnection conn=connection; + if (conn==null) { + return CompletableFuture.completedFuture(Result.CLOSED_CONNECTION); + } + + ACell id=m.getRequestID(); try { + if (id==null) { + // Not expecting any return message, so just report sending + boolean sent = conn.sendMessage(m); + if (sent) { + // log.info("Sent message: "+m); + } else { + return CompletableFuture.completedFuture(Result.FULL_CLIENT_BUFFER); + } + return CompletableFuture.completedFuture(Result.SENT_MESSAGE); + } + synchronized (awaiting) { - long id = connection.sendStatusRequest(); - if (id < 0) { - return CompletableFuture.completedFuture(Result.error(ErrorCodes.LOAD, "Full buffer, can't send status request").withSource(SourceCodes.COMM)); + boolean sent = conn.sendMessage(m); + if (sent) { + // All OK + // log.info("Sent message: "+m); + } else { + return CompletableFuture.completedFuture(Result.FULL_CLIENT_BUFFER); } - CompletableFuture cf = awaitResult(CVMLong.create(id),timeout); + // Make sure we call this while synchronised on awaiting map + CompletableFuture cf = awaitResult(id,timeout); return cf; } - } catch (IOException e) { + } catch (Exception e) { Result r=Result.fromException(e).withInfo(Keywords.SOURCE,SourceCodes.COMM); return CompletableFuture.completedFuture(r); } } + @Override + public CompletableFuture requestStatus() { + Message m=Message.createStatusRequest(getNextID()); + return message(m); + } + @Override public CompletableFuture requestChallenge(SignedData data) { - synchronized (awaiting) { - long id; - try { - id = connection.sendChallenge(data); - } catch (IOException e) { - return CompletableFuture.completedFuture(Result.error(ErrorCodes.IO, "Error requesting challenge")); - } - if (id < 0) { - // TODO: too fragile? - return CompletableFuture.completedFuture(Result.error(ErrorCodes.IO, "Full buffer while requesting challenge")); - } - - // Store future for completion by result message - return awaitResult(CVMLong.create(id),timeout); - } + Message m=Message.createChallenge(data); + return message(m); } @Override @@ -230,8 +295,9 @@ public CompletableFuture acquire(Hash hash, AStore store) { * Disconnects the client from the network, closing the underlying connection. */ public synchronized void close() { - Connection c = this.connection; + AConnection c = this.connection; if (c != null) { + // log.info("Connection closed",new Exception()); c.close(); } connection = null; @@ -249,4 +315,6 @@ public Server getLocalServer() { } + + } diff --git a/convex-peer/src/main/java/convex/net/AConnection.java b/convex-peer/src/main/java/convex/net/AConnection.java new file mode 100644 index 000000000..95674775f --- /dev/null +++ b/convex-peer/src/main/java/convex/net/AConnection.java @@ -0,0 +1,53 @@ +package convex.net; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import convex.core.data.AccountKey; + +public abstract class AConnection { + + + private AccountKey trustedKey=null; + + public boolean isTrusted() { + return trustedKey!=null; + } + + /** + * Sends a message over this connection + * + * @param msg Message to send + * @return true if message buffered successfully, false if failed due to full buffer + * @throws IOException If IO error occurs while sending + */ + public abstract boolean sendMessage(Message m) throws IOException; + + /** + * Returns the remote SocketAddress associated with this connection, or null if + * not available + * + * @return An InetSocketAddress if associated, otherwise null + */ + public abstract InetSocketAddress getRemoteAddress(); + + /** + * Sets the trusted remote key for this connection. Only do this f the other side has successfully responded to an authentication challenge + * @param key + */ + public void setTrustedKey(AccountKey key) { + this.trustedKey=key; + } + + /** + * Checks if this connection is closed (i.e. the underlying channel is closed) + * + * @return true if the channel is closed, false otherwise. + */ + public abstract boolean isClosed(); + + public abstract void close(); + + public abstract long getReceivedCount(); + +} diff --git a/convex-peer/src/main/java/convex/net/AServer.java b/convex-peer/src/main/java/convex/net/AServer.java new file mode 100644 index 000000000..6b464295f --- /dev/null +++ b/convex-peer/src/main/java/convex/net/AServer.java @@ -0,0 +1,34 @@ +package convex.net; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +public abstract class AServer implements Closeable { + + /** + * Gets the port that this server instance is configured to listen on + * + * @return Port number, may be null if not set + */ + public Integer getPort() { + return port; + } + + private Integer port=null; + + @Override + public abstract void close(); + + public abstract InetSocketAddress getHostAddress(); + + /** + * Sets the port for this server. Should be called prior to launch + * @param port + */ + public void setPort(Integer port) { + this.port=port; + } + + public abstract void launch() throws IOException, InterruptedException; +} diff --git a/convex-peer/src/main/java/convex/net/ChallengeRequest.java b/convex-peer/src/main/java/convex/net/ChallengeRequest.java index cd9c12ba8..fb9f4cd36 100644 --- a/convex-peer/src/main/java/convex/net/ChallengeRequest.java +++ b/convex-peer/src/main/java/convex/net/ChallengeRequest.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.security.SecureRandom; import java.util.concurrent.TimeUnit; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,9 +12,9 @@ import convex.core.data.AVector; import convex.core.data.AccountKey; import convex.core.data.Blob; -import convex.core.data.Vectors; import convex.core.data.Hash; import convex.core.data.SignedData; +import convex.core.data.Vectors; public class ChallengeRequest { @@ -46,9 +47,9 @@ public static ChallengeRequest create(AccountKey peerKey, int timeoutSeconds) { * Sends out a single challenge to the remote peer. * @param connection Connection * @param peer This Peer - * @return ID of message sent, or negative value if sending fails + * @return Boolean true if sending succeeded */ - public long send(Connection connection, Peer peer) { + public boolean send(AConnection connection, Peer peer) { AVector values = null; try { SecureRandom random = new SecureRandom(); @@ -61,12 +62,13 @@ public long send(Connection connection, Peer peer) { values = Vectors.of(token, peer.getNetworkID(), peerKey); SignedData challenge = peer.sign(values); sendHash = challenge.getHash(); - return connection.sendChallenge(challenge); + Message m=Message.createChallenge(challenge); + return connection.sendMessage(m); } catch (IOException e) { log.warn("Cannot send challenge to remote peer at {}", connection.getRemoteAddress()); values = null; } - return -1; + return false; } public AccountKey getPeerKey() { diff --git a/convex-peer/src/main/java/convex/net/Message.java b/convex-peer/src/main/java/convex/net/Message.java index 79379cf72..111cd4939 100644 --- a/convex-peer/src/main/java/convex/net/Message.java +++ b/convex-peer/src/main/java/convex/net/Message.java @@ -12,12 +12,15 @@ import convex.core.cpos.Belief; import convex.core.cpos.CPoSConstants; import convex.core.cvm.Address; +import convex.core.cvm.CVMTag; +import convex.core.cvm.transactions.ATransaction; import convex.core.data.ACell; import convex.core.data.AString; import convex.core.data.AVector; import convex.core.data.Blob; import convex.core.data.Format; import convex.core.data.Hash; +import convex.core.data.Keyword; import convex.core.data.Ref; import convex.core.data.SignedData; import convex.core.data.Strings; @@ -29,6 +32,7 @@ import convex.core.lang.RT; import convex.core.lang.Reader; import convex.core.store.AStore; +import convex.core.text.PrintUtils; import convex.core.util.Utils; /** @@ -45,6 +49,8 @@ public class Message { protected static final Logger log = LoggerFactory.getLogger(Message.class.getName()); + private static final Message BYE_MESSAGE = Message.create(MessageType.GOODBYE,Vectors.create(MessageTag.BYE)); + protected ACell payload; protected Blob messageData; // encoding of payload (possibly multi-cell) protected MessageType type; @@ -57,14 +63,7 @@ protected Message(MessageType type, ACell payload, Blob data, Predicate this.returnHandler=handler; } - public static Message create(Connection conn, MessageType type, Blob data) { - Predicate handler=t -> { - try { - return conn.sendMessage(t); - } catch (IOException e) { - return false; - } - }; + public static Message create(Predicate handler, MessageType type, Blob data) { return new Message(type, null,data,handler); } @@ -82,23 +81,22 @@ public static Message create(MessageType type,ACell payload, Blob data) { } public static Message createDataResponse(ACell id, ACell... cells) { - int n=cells.length; - ACell[] cs=new ACell[n+1]; - cs[0]=id; - for (int i=0; i response) { } public static Message createGoodBye() { - return create(MessageType.GOODBYE, null); + return BYE_MESSAGE; } @SuppressWarnings("unchecked") @@ -144,27 +142,14 @@ public T getPayload() throws BadFormatException { */ public Blob getMessageData() { if (messageData!=null) return messageData; - - // default to single cell encoding - // TODO: alternative depths for different types + MessageType type=getType(); switch (type) { - case MessageType.RESULT: - case MessageType.QUERY: - case MessageType.TRANSACT: - case MessageType.REQUEST_DATA: - messageData=Format.encodeMultiCell(payload,true); - break; - - case MessageType.DATA: - @SuppressWarnings("unchecked") - AVector v=(AVector) payload; - messageData=Format.encodeDataVector(v); - break; - - default: - messageData=Format.encodeMultiCell(payload,true); - } + case MessageType.BELIEF: + // throw new Error("Received belief message should already have partial data encoding"); + default: + messageData=Format.encodeMultiCell(payload,true); + } return messageData; } @@ -178,11 +163,39 @@ public MessageType getType() { } private MessageType inferType() { + if (hasData()) { + // These can be inferred directly from top encoding tag + byte tag=messageData.byteAt(0); + if (tag==CVMTag.BELIEF) return MessageType.BELIEF; + if (tag==Tag.SIGNED_DATA) return MessageType.BELIEF; // i.e. a SignedData or similar + if (tag==CVMTag.RESULT) return MessageType.RESULT; + } + try { ACell payload=getPayload(); if (payload instanceof Result) return MessageType.RESULT; + + if (payload instanceof Belief) return MessageType.BELIEF; + if (payload instanceof SignedData) return MessageType.BELIEF; + + if (payload instanceof AVector) { + Keyword mt=RT.ensureKeyword(((AVector)payload).get(0)); + if (mt==null) return MessageType.UNKNOWN; + if (MessageTag.STATUS_REQUEST.equals(mt)) return MessageType.STATUS; + if (MessageTag.QUERY.equals(mt)) return MessageType.QUERY; + if (MessageTag.BYE.equals(mt)) return MessageType.GOODBYE; + if (MessageTag.TRANSACT.equals(mt)) return MessageType.TRANSACT; + if (MessageTag.DATA_REQUEST.equals(mt)) return MessageType.DATA_REQUEST; + } } catch (Exception e) { // default fall-through to UNKNOWN. We don't know what it is supposed to be! + try { + ACell payload=getPayload(); + System.out.println(PrintUtils.printRefTree(payload.getRef())); + log.info("Can't infer message type with object "+Utils.getClassName(payload),e); + } catch (Exception ex) { + ex.printStackTrace(); + } } return MessageType.UNKNOWN; @@ -196,7 +209,7 @@ public String toString() { if (ps==null) return (""); return ps.toString(); } catch (MissingDataException e) { - return ""; + return ""; } catch (BadFormatException e) { return ": "+e.getMessage(); } @@ -218,34 +231,70 @@ public boolean equals(Object o) { * @return Message ID, or null if the message does not have a message ID */ public ACell getID() { + if (payload==null) throw new IllegalStateException("Attempting to get ID of message before Payload is decoded"); + switch (getType()) { + // Result is a special record type + case RESULT: return getResultID(); + + default: return getRequestID(); + } + } + + /** + * Gets the request ID for this message, assuming it is a request expecting a response + * @return + */ + public ACell getRequestID() { + // if (payload==null) throw new IllegalStateException("Attempting to get ID of message before Payload is decoded"); try { - switch (type) { - // Query and transact use a vector [ID ...] + switch (getType()) { + + // ID in position 1 + case STATUS: + case TRANSACT: case QUERY: - case TRANSACT: return ((AVector)getPayload()).get(0); - - // Result is a special record type - case RESULT: return ((Result)getPayload()).getID(); - - // Status ID is the single value - case STATUS: return (getPayload()); - - case DATA: { - ACell o=getPayload(); - if (o instanceof AVector) { - AVector v = (AVector)o; - if (v.count()==0) return null; - // first element might be ID, otherwise null - return RT.ensureLong(v.get(0)); - } + case DATA_REQUEST:{ + AVector v=RT.ensureVector(getPayload()); + if (v.count()<2) return null; + return RT.ensureLong(v.get(1)); } default: return null; } } catch (Exception e) { - // defensive coding + log.warn("Unexpected error getting request ID",e); + return null; + } + } + + /** + * Gets the result ID for this message, assuming it is a Result + * + * This needs to work even if the payload is not yet decoded, for message routing (possibly with a different store) + * + * @return + */ + public ACell getResultID() { + if (payload!=null) { + if (payload instanceof Result) { + return ((Result)payload).getID(); + } + return null; + } + + if (hasData()) try { + // Check tag is a Result + byte tag=messageData.byteAt(0); + if (tag!=CVMTag.RESULT) return null; + + // Peek at Result ID without loading whole payload + return Result.peekResultID(messageData,0); + } catch (Exception e) { + log.warn("Unexpected error getting result ID: "+e.getMessage()); return null; } + + return null; } /** @@ -256,27 +305,23 @@ public ACell getID() { @SuppressWarnings("unchecked") public Message withID(ACell id) { try { - switch (type) { - // Query and transact use a vector [ID ...] - case QUERY: - case TRANSACT: - return Message.create(type, ((AVector)getPayload()).assoc(0, id)); + switch (getType()) { // Result is a special record type case RESULT: return Message.create(type, ((Result)getPayload()).withID(id)); - - // Status ID is the single value + + // Using a vector [key ID ...] case STATUS: - return Message.create(type, id); - - case DATA: { + case TRANSACT: + case QUERY: + case DATA_REQUEST: { ACell o=getPayload(); if (o instanceof AVector) { AVector v = (AVector)o; - if (v.count()==0) return null; + if (v.count()<2) return null; // first element assumed to be ID - return Message.create(type, v.assoc(0, id)); + return Message.create(type, v.assoc(1, id)); } } @@ -295,13 +340,18 @@ public Message withID(ACell id) { * * @param res Result record * @return True if reported successfully, false otherwise + * @throws IllegalStateException if original message did not specify a return ID */ public boolean returnResult(Result res) { - ACell id=getID(); - if (id!=null) res=res.withID(id); - - Message msg=Message.createResult(res); - return returnMessage(msg); + ACell id=getRequestID(); // what was the request ID of original message? + if (id!=null) { + // Make sure Result has correct result ID + res=res.withID(id); + Message msg=Message.createResult(res); + return returnMessage(msg); + } else { + throw new IllegalStateException("Trying to return result with no original request ID"); + } } /** @@ -318,6 +368,10 @@ public boolean returnMessage(Message m) { return handler.test(m); } + /** + * Return true if there is encoded message data + * @return + */ public boolean hasData() { return messageData!=null; } @@ -326,7 +380,7 @@ public static Message createResult(Result res) { return create(MessageType.RESULT,res); } - public static Message createResult(CVMLong id, ACell value, ACell error) { + public static Message createResult(ACell id, ACell value, ACell error) { Result r=Result.create(id, value,error); return createResult(r); } @@ -339,19 +393,24 @@ public void closeConnection() { } public Message makeDataResponse(AStore store) throws BadFormatException { + final int HEADER_OFFSET=2; // offset of hashes in request vector + AVector v = RT.ensureVector(getPayload()); if ((v == null)||(v.isEmpty())) { throw new BadFormatException("Invalid data request payload"); }; - if (v.count()>CPoSConstants.MISSING_LIMIT+1) { + if (v.count()>CPoSConstants.MISSING_LIMIT+HEADER_OFFSET) { throw new BadFormatException("Too many elements in Missing data request"); } + + ACell id=v.get(1); // location of ID in request record //System.out.println("DATA REQ:"+ v); - int n=v.size(); + + int n=v.size()-HEADER_OFFSET; // number of values requested (ignore header elements) + ACell[] vals=new ACell[n]; - vals[0]=v.get(0); - for (int i=1; i v=Vectors.create(MessageTag.QUERY,CVMLong.create(id),code,address); + return create(MessageType.QUERY,v); } + public static Message createTransaction(long id, SignedData signed) { + AVector v=Vectors.create(MessageTag.TRANSACT,CVMLong.create(id),signed); + return create(MessageType.TRANSACT,v); + } + + /** + * Sends a STATUS Request Message on this connection. + * + * @return The ID of the message sent, or -1 if send buffer is full. + * @throws IOException If IO error occurs + */ + public static Message createStatusRequest(long id) { + CVMLong idPayload = CVMLong.create(id); + AVector v=Vectors.create(MessageTag.STATUS_REQUEST,idPayload); + return create(MessageType.STATUS,v); + } + + /** + * Return the Hash of the Message payload + * @return Hash, or null if message format is invalid + */ + public Hash getHash() { + try { + return getPayload().getHash(); + } catch (BadFormatException e) { + return null; + } + } + + } diff --git a/convex-peer/src/main/java/convex/net/MessageReceiver.java b/convex-peer/src/main/java/convex/net/MessageReceiver.java index 57da734a3..33527b911 100644 --- a/convex-peer/src/main/java/convex/net/MessageReceiver.java +++ b/convex-peer/src/main/java/convex/net/MessageReceiver.java @@ -4,6 +4,7 @@ import java.nio.ByteBuffer; import java.nio.channels.ReadableByteChannel; import java.util.function.Consumer; +import java.util.function.Predicate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,17 +45,18 @@ public class MessageReceiver { private final Consumer action; private Consumer hook=null; - private final Connection connection; + private final Predicate returnHandler; private long receivedMessageCount = 0; private static final Logger log = LoggerFactory.getLogger(MessageReceiver.class.getName()); - public MessageReceiver(Consumer receiveAction, Connection pc) { + public MessageReceiver(Consumer receiveAction, Predicate returnHandler) { this.action = receiveAction; - this.connection = pc; + this.returnHandler = returnHandler; } + /** * Get the number of messages received in total by this Receiver * @return Count of messages received @@ -150,11 +152,7 @@ public synchronized int receiveFromChannel(ReadableByteChannel chan) throws BadF private void receiveMessage(Blob messageData) throws BadFormatException, HandlerException { if (messageData.count()<1) throw new BadFormatException("Empty message"); - byte mType=messageData.byteAtUnchecked(0); - MessageType type=MessageType.decode(mType); - - Blob encoding=messageData.slice(1); - Message message = Message.create(connection, type, encoding); + Message message = Message.create(returnHandler, null, messageData); // call the receiver hook, if registered maybeCallHook(message); diff --git a/convex-peer/src/main/java/convex/net/MessageTag.java b/convex-peer/src/main/java/convex/net/MessageTag.java new file mode 100644 index 000000000..80a03e2cc --- /dev/null +++ b/convex-peer/src/main/java/convex/net/MessageTag.java @@ -0,0 +1,19 @@ +package convex.net; + +import convex.core.data.Keyword; + +/** + * Constant tags used to identify general purpose messages + */ +public class MessageTag { + + public static final Keyword STATUS_REQUEST = Keyword.intern("SR"); + public static final Keyword QUERY=Keyword.intern("Q"); + public static final Keyword TRANSACT=Keyword.intern("TX"); + + public static final Keyword DATA_REQUEST=Keyword.intern("DR"); + + public static final Keyword BYE=Keyword.intern("BYE"); + + +} diff --git a/convex-peer/src/main/java/convex/net/MessageType.java b/convex-peer/src/main/java/convex/net/MessageType.java index 49aacae84..3f16d0205 100644 --- a/convex-peer/src/main/java/convex/net/MessageType.java +++ b/convex-peer/src/main/java/convex/net/MessageType.java @@ -23,8 +23,8 @@ public enum MessageType { /** * A message relaying data. * - * Payload is a vector: - * - [id content] + * Payload is a Result: + * - Result * * Data is presented "as-is", and may be: * - the result of a missing data request @@ -49,17 +49,17 @@ public enum MessageType { * peers. Peers under load may ignore data requests. * * Payload is a Vector containing ID plus one or more hashes - * i.e [id hash1 hash2 ......] + * i.e [:DR id hash1 hash2 ......] * * Receiver should respond with a DATA message if the specified data is * available in their store, and they are willing to fulfil the request */ - REQUEST_DATA(5), + DATA_REQUEST(5), /** * A request to perform the specified query and return results. * - * Payload is: [id form address?] + * Payload is: [:Q id form address?] * * Receiver may may determine policies regarding whether to accept or reject * queries, typically receiver will want to authenticate the sender and ensure @@ -71,7 +71,7 @@ public enum MessageType { * A message requesting a transaction be accepted by the receiving peer and * included in the next available block. * - * Payload is: [id signed-data] + * Payload is: [:TX id signed-data] */ TRANSACT(7), @@ -79,7 +79,7 @@ public enum MessageType { * Message containing the Result for a corresponding COMMAND, QUERY or TRANSACT * message. * - * Payload is: [id result error-code] + * Payload is: Result * * Where: * - Result is the result of the request, or the message if an error occurred @@ -107,8 +107,8 @@ public enum MessageType { * Communication of an intention to shutdown the connection. This is optional * * Payload can be: - * - nil for no reason - * - A human readable string as a reason + * - [:BYE] for generic close + * - [:BYE message] for close with a reason */ GOODBYE(11), @@ -145,7 +145,7 @@ public static MessageType decode(int i) throws BadFormatException { case 4: return COMMAND; case 5: - return REQUEST_DATA; + return DATA_REQUEST; case 6: return QUERY; case 7: diff --git a/convex-peer/src/main/java/convex/net/ResultConsumer.java b/convex-peer/src/main/java/convex/net/ResultConsumer.java index ea28749fa..b84bd87ce 100644 --- a/convex-peer/src/main/java/convex/net/ResultConsumer.java +++ b/convex-peer/src/main/java/convex/net/ResultConsumer.java @@ -26,7 +26,7 @@ public void accept(Message m) { handleDataProvided(m); break; } - case REQUEST_DATA: { + case DATA_REQUEST: { handleDataRequest(m); break; } @@ -35,7 +35,7 @@ public void accept(Message m) { break; } default: { - log.error("Message type ignored: ", type); + log.error("Message type ignored type: {} value: {}", type, m); } } } diff --git a/convex-peer/src/main/java/convex/net/impl/netty/NettyConnection.java b/convex-peer/src/main/java/convex/net/impl/netty/NettyConnection.java new file mode 100644 index 000000000..54249853d --- /dev/null +++ b/convex-peer/src/main/java/convex/net/impl/netty/NettyConnection.java @@ -0,0 +1,144 @@ +package convex.net.impl.netty; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.function.Consumer; + +import convex.core.data.Vectors; +import convex.core.util.Shutdown; +import convex.net.AConnection; +import convex.net.Message; +import convex.net.MessageType; +import convex.peer.Config; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; + +public class NettyConnection extends AConnection { + + /** + * Static client connection worker + */ + static EventLoopGroup workerGroup = null; + + static Bootstrap clientBootstrap = null; + + private Channel channel; + + private NettyInboundHandler inboundHandler; + + private NettyConnection(Channel channel, NettyInboundHandler inbound) { + this.channel = channel; + this.inboundHandler=inbound; + } + + protected static EventLoopGroup getEventLoopGroup() { + if (workerGroup != null) + return workerGroup; + + synchronized (NettyConnection.class) { + if (workerGroup != null) + return workerGroup; + workerGroup = new NioEventLoopGroup(); + + Shutdown.addHook(Shutdown.CONNECTION, () -> { + if (workerGroup != null) { + workerGroup.shutdownGracefully(); + } + }); + return workerGroup; + } + } + + protected static Bootstrap getClientBootstrap() { + if (clientBootstrap != null) + return clientBootstrap; + + synchronized (NettyConnection.class) { + if (clientBootstrap != null) + return clientBootstrap; + Bootstrap b = new Bootstrap(); + b.group(getEventLoopGroup()); + b.channel(NioSocketChannel.class); + b.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) Config.DEFAULT_CLIENT_TIMEOUT); + b.option(ChannelOption.SO_KEEPALIVE, true); + + b.handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + // nothing to add, connect will do this + } + }); + + clientBootstrap = b; + return clientBootstrap; + } + } + + public static NettyConnection connect(SocketAddress sa, Consumer receiveAction) throws InterruptedException { + Bootstrap b = getClientBootstrap(); + ChannelFuture f = b.connect(sa).sync(); // (5) + + Channel chan = f.channel(); + NettyInboundHandler inbound=new NettyInboundHandler(receiveAction,null); + f.channel().pipeline().addLast(inbound,new NettyOutboundHandler()); + + NettyConnection client = new NettyConnection(chan,inbound); + return client; + } + + protected ChannelFuture send(Message m) { + return channel.writeAndFlush(m); + } + + public static void main(String... args) throws Exception { + NettyConnection client = connect(new InetSocketAddress("localhost", 8000),m->{ + System.err.println("Client received:" + m); + }); + + client.send(Message.create(MessageType.QUERY,Vectors.of(1,2,3,4))).sync(); + } + + @Override + public boolean sendMessage(Message m) { + if (!channel.isActive()) return false; + + // Note: never call await here as might block the IO thread + @SuppressWarnings("unused") + ChannelFuture cf=channel.writeAndFlush(m); + // cf.syncUninterruptibly(); + return true; + } + + @Override + public InetSocketAddress getRemoteAddress() { + if (channel==null) return null; + return (InetSocketAddress) channel.remoteAddress(); + } + + @Override + public boolean isClosed() { + if (channel==null) return true; + return !channel.isOpen(); + } + + @Override + public void close() { + if (channel!=null) { + channel.close(); + channel=null; + } + } + + @Override + public long getReceivedCount() { + return inboundHandler.getReceivedCount(); + } + +} diff --git a/convex-peer/src/main/java/convex/net/impl/netty/NettyInboundHandler.java b/convex-peer/src/main/java/convex/net/impl/netty/NettyInboundHandler.java new file mode 100644 index 000000000..62408c1bc --- /dev/null +++ b/convex-peer/src/main/java/convex/net/impl/netty/NettyInboundHandler.java @@ -0,0 +1,103 @@ +package convex.net.impl.netty; + +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Predicate; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import convex.core.cpos.CPoSConstants; +import convex.core.data.Blob; +import convex.core.exceptions.BadFormatException; +import convex.net.Message; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; + +class NettyInboundHandler extends ByteToMessageDecoder { + + static final Logger log = LoggerFactory.getLogger(NettyInboundHandler.class.getName()); + + + private final Consumer receiveAction; + + private Predicate returnAction; + + private long receivedCount=0; + + public NettyInboundHandler(Consumer receiveAction, Predicate returnAction) { + this.receiveAction=receiveAction; + this.returnAction=returnAction; + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { // (4) + // Close the connection when an exception is raised. + log.info("Closed Netty channel due to: "+cause.getMessage(),cause); + ctx.close(); + } + + public long getReceivedCount() { + return receivedCount; + } + + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { + ByteBuf buf=(ByteBuf)in; + buf.markReaderIndex(); // mark reader position, may revert to this if no complete message + + int mlen=0; + int newBytes=buf.readableBytes(); + if (newBytes==0) return; + + for (int i=0; ; i++) { + if (i>=newBytes) { + // insufficient bytes for message length, need to wait for more + buf.resetReaderIndex(); + return; + } + + byte b=buf.readByte(); + + if ((i==0)&&(b==0x80)) { + byte[] bytes=new byte[newBytes]; + bytes[0]=b; + buf.readBytes(bytes, 1, newBytes-1); + Blob tmp=Blob.wrap(bytes); + throw new BadFormatException("Zero leading bits in message length, content: "+tmp); + } + + int bm=(b&0x7f); // new bits for length + mlen=(mlen<<7)+bm; + if (mlen>CPoSConstants.MAX_MESSAGE_LENGTH) throw new BadFormatException("Message too long: "+mlen); + if ((b&0x80)==0) { + // we have a complete message length + break; + }; + } + + if (buf.readableBytes()= 1; i--) { + byte single = (byte) (0x80 | (x >>> (7 * i))); // 7 bits + bb.writeByte(single); + } + byte end = (byte) (x & 0x7F); // last 7 bits of long, high bit zero + bb.writeByte(end); + } +} \ No newline at end of file diff --git a/convex-peer/src/main/java/convex/net/impl/netty/NettyServer.java b/convex-peer/src/main/java/convex/net/impl/netty/NettyServer.java new file mode 100644 index 000000000..be0380497 --- /dev/null +++ b/convex-peer/src/main/java/convex/net/impl/netty/NettyServer.java @@ -0,0 +1,141 @@ +package convex.net.impl.netty; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.function.Consumer; +import java.util.function.Predicate; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import convex.core.Constants; +import convex.core.data.ACell; +import convex.core.util.Shutdown; +import convex.net.AServer; +import convex.net.Message; +import convex.peer.Server; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; + +public class NettyServer extends AServer { + + static final Logger log = LoggerFactory.getLogger(NettyServer.class.getName()); + + static EventLoopGroup bossGroup=null; + + protected synchronized static EventLoopGroup getEventLoopGroup() { + if (bossGroup!=null) return bossGroup; + bossGroup=new NioEventLoopGroup(); + Shutdown.addHook(Shutdown.SERVER,()->{ + if (bossGroup!=null) { + bossGroup.shutdownGracefully(); + } + }); + return bossGroup; + } + + private Consumer receiveAction=m->{ + try { + ACell payload=m.getPayload(); + m.returnMessage(Message.createResult(m.getRequestID(), payload, null)); + } catch (Exception e) { + log.warn("Unexpected exception handling message receipt",e); + } + }; + + private Channel channel; + + public NettyServer(Integer port) { + setPort(port); + } + + + public static NettyServer create(Server server) { + NettyServer ns=new NettyServer(null); + ns.receiveAction=server.getReceiveAction(); + return ns; + } + + + public void launch() throws IOException,InterruptedException { + EventLoopGroup bossGroup = NettyServer.getEventLoopGroup(); + EventLoopGroup workerGroup = NettyConnection.getEventLoopGroup(); + ServerBootstrap b = new ServerBootstrap(); + b.group(bossGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + Predicate returnHandler=m->{ + ch.writeAndFlush(m); + return true; + }; + NettyInboundHandler inbound=new NettyInboundHandler(getReceiveAction(),returnHandler); + ch.pipeline().addLast(inbound,new NettyOutboundHandler()); + } + }) + .option(ChannelOption.SO_BACKLOG, 128) // Backlog of incoming connection requests + .childOption(ChannelOption.SO_KEEPALIVE, true); + + ChannelFuture f=null; + Integer port=getPort(); + if (port==null) try { + f = b.bind(Constants.DEFAULT_PEER_PORT).sync(); + port=Constants.DEFAULT_PEER_PORT; + } catch (Exception e) { + // failed so try with random port + port=0; + } + + if (f==null) { + f = b.bind(port).sync(); + } + // Check local port + InetSocketAddress localAddress=(InetSocketAddress) f.channel().localAddress(); + setPort(localAddress.getPort()); + log.debug("Netty Server started on port: "+getPort()); + + this.channel=f.channel(); + } + + protected Consumer getReceiveAction() { + return receiveAction; + } + + public static void main(String... args) throws Exception { + try (NettyServer server=new NettyServer(8000)) { + server.launch(); + + server.waitForClose(); + } + } + + @Override + public void close() { + if (channel!=null) { + channel.close(); + } + } + + public void waitForClose() throws InterruptedException { + channel.closeFuture().sync(); + } + + @Override + public InetSocketAddress getHostAddress() { + return (InetSocketAddress) channel.localAddress(); + } + + + public void setReceiveAction(Consumer handler) { + receiveAction=handler; + } + +} diff --git a/convex-peer/src/main/java/convex/net/Connection.java b/convex-peer/src/main/java/convex/net/impl/nio/Connection.java similarity index 76% rename from convex-peer/src/main/java/convex/net/Connection.java rename to convex-peer/src/main/java/convex/net/impl/nio/Connection.java index 18e7d9002..40a8014e3 100644 --- a/convex-peer/src/main/java/convex/net/Connection.java +++ b/convex-peer/src/main/java/convex/net/impl/nio/Connection.java @@ -1,4 +1,4 @@ -package convex.net; +package convex.net.impl.nio; import java.io.IOException; import java.net.InetSocketAddress; @@ -18,6 +18,8 @@ import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.function.Consumer; +import java.util.function.Predicate; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,6 +45,10 @@ import convex.core.util.Counters; import convex.core.util.Utils; import convex.core.util.Shutdown; +import convex.net.AConnection; +import convex.net.Message; +import convex.net.MessageReceiver; +import convex.net.MessageSender; import convex.net.impl.HandlerException; import convex.peer.Config; @@ -65,7 +71,7 @@ *

*/ @SuppressWarnings("unused") -public class Connection { +public class Connection extends AConnection { final ByteChannel channel; @@ -79,12 +85,6 @@ public class Connection { */ private long lastActivity; - /** - * Store to use for this connection. Required for responding to incoming - * messages. - */ - private final AStore store; - /** * If trusted, the Account Key of the remote peer. */ @@ -95,12 +95,19 @@ public class Connection { private final MessageReceiver receiver; private final MessageSender sender; - private Connection(ByteChannel channel, Consumer receiveAction, AStore store, + private Connection(ByteChannel channel, Consumer receiveAction, AccountKey trustedPeerKey) { this.channel = channel; - receiver = new MessageReceiver(receiveAction, this); + Predicate handler=t -> { + try { + return sendMessage(t); + } catch (IOException e) { + return false; + } + }; + + receiver = new MessageReceiver(receiveAction, handler); sender = new MessageSender(channel); - this.store = store; this.lastActivity=Utils.getCurrentTimestamp(); this.trustedPeerKey = trustedPeerKey; } @@ -117,12 +124,12 @@ private Connection(ByteChannel channel, Consumer receiveAction, AStore * @return New Connection instance * @throws IOException If IO error occurs */ - public static Connection create(ByteChannel channel, Consumer receiveAction, AStore store, + public static Connection create(ByteChannel channel, Consumer receiveAction, AccountKey trustedPeerKey) throws IOException { // Needed in case server has incoming connections but no outbound? ensureSelectorLoop(); - return new Connection(channel, receiveAction, store, trustedPeerKey); + return new Connection(channel, receiveAction, trustedPeerKey); } /** @@ -137,9 +144,9 @@ public static Connection create(ByteChannel channel, Consumer receiveAc * @throws TimeoutException If connection cannot be established within an * acceptable time (~5s) */ - public static Connection connect(InetSocketAddress socketAddress, Consumer receiveAction, AStore store) + public static Connection connect(InetSocketAddress socketAddress, Consumer receiveAction) throws IOException, TimeoutException { - return connect(socketAddress, receiveAction, store, null); + return connect(socketAddress, receiveAction, null); } /** @@ -156,9 +163,9 @@ public static Connection connect(InetSocketAddress socketAddress, Consumer receiveAction, AStore store, + public static Connection connect(InetSocketAddress socketAddress, Consumer receiveAction, AccountKey trustedPeerKey) throws IOException, TimeoutException { - return connect(socketAddress,receiveAction,store,trustedPeerKey,Config.SOCKET_SEND_BUFFER_SIZE,Config.SOCKET_RECEIVE_BUFFER_SIZE); + return connect(socketAddress,receiveAction,trustedPeerKey,Config.SOCKET_SEND_BUFFER_SIZE,Config.SOCKET_RECEIVE_BUFFER_SIZE); } /** @@ -177,12 +184,10 @@ public static Connection connect(InetSocketAddress socketAddress, Consumer receiveAction, AStore store, + public static Connection connect(InetSocketAddress socketAddress, Consumer receiveAction, AccountKey trustedPeerKey, int sendBufferSize, int receiveBufferSize) throws IOException, TimeoutException { ensureSelectorLoop(); - if (store == null) - throw new Error("Connection requires a store"); SocketChannel clientChannel = SocketChannel.open(); clientChannel.configureBlocking(false); clientChannel.socket().setReceiveBufferSize(receiveBufferSize); @@ -208,7 +213,7 @@ public static Connection connect(InetSocketAddress socketAddress, Consumer hook) { receiver.setHook(hook); } - /** - * Returns the remote SocketAddress associated with this connection, or null if - * not available - * - * @return An InetSocketAddress if associated, otherwise null - */ + @Override public InetSocketAddress getRemoteAddress() { if (!(channel instanceof SocketChannel)) return null; @@ -242,14 +242,6 @@ public InetSocketAddress getRemoteAddress() { return null; } } - - /** - * Gets the store associated with this Connection - * @return Store instance - */ - public AStore getStore() { - return store; - } /** * Returns the local SocketAddress associated with this connection, or null if @@ -280,48 +272,7 @@ public InetSocketAddress getLocalAddress() { */ public boolean sendData(Blob data) throws IOException { log.trace("Sending data: {}", data); - return sendBuffer(MessageType.DATA, data); - } - - /** - * Sends a QUERY Message on this connection with a null Address - * - * @param form A data object representing the query form - * @return The ID of the message sent, or -1 if send buffer is full. - * @throws IOException If IO error occurs - */ - public long sendQuery(ACell form) throws IOException { - return sendQuery(form, null); - } - - /** - * Sends a QUERY Message on this connection. - * - * @param form A data object representing the query source form - * @param address The address with which to run the query, which may be null - * @return The ID of the message sent, or -1 if send buffer is full. - * @throws IOException If IO error occurs - */ - public long sendQuery(ACell form, Address address) throws IOException { - AStore temp = Stores.current(); - long id = ++idCounter; - AVector v = Vectors.of(id, form, address); - boolean sent = sendObject(MessageType.QUERY, v); - return sent ? id : -1; - } - - /** - * Sends a STATUS Request Message on this connection. - * - * @return The ID of the message sent, or -1 if send buffer is full. - * @throws IOException If IO error occurs - */ - public long sendStatusRequest() throws IOException { - AStore temp = Stores.current(); - long id = ++idCounter; - CVMLong idPayload = CVMLong.create(id); - boolean sent=sendObject(MessageType.STATUS, idPayload); - return sent? id:-1; + return sendBuffer(data); } /** @@ -338,7 +289,7 @@ public long sendChallenge(SignedData challenge) throws IOException { AStore temp = Stores.current(); try { long id = ++idCounter; - boolean sent = sendObject(MessageType.CHALLENGE, challenge); + boolean sent = sendObject(challenge); return (sent) ? id : -1; } finally { Stores.setCurrent(temp); @@ -358,39 +309,16 @@ public long sendResponse(SignedData response) throws IOException { AStore temp = Stores.current(); try { long id = ++idCounter; - boolean sent = sendObject(MessageType.RESPONSE, response); + boolean sent = sendObject(response); return (sent) ? id : -1; } finally { Stores.setCurrent(temp); } } - /** - * Sends a transaction if possible, returning the message ID (greater than zero) - * if successful. - * - * Returns -1 if the message could not be sent because of a full buffer. - * - * @param signed Signed transaction - * @return Message ID of the transaction request, or -1 if send buffer is full. - * @throws IOException In the event of an IO error, e.g. closed connection - */ - public long sendTransaction(SignedData signed) throws IOException { - long id = getNextID(); - AVector v = Vectors.of(id, signed); - boolean sent = sendObject(MessageType.TRANSACT, v); - return (sent) ? id : -1; - } - - /** - * Sends a message over this connection - * - * @param msg Message to send - * @return true if message buffered successfully, false if failed due to full buffer - * @throws IOException If IO error occurs while sending - */ + @Override public boolean sendMessage(Message msg) throws IOException { - return sendBuffer(msg.getType(),msg.getMessageData()); + return sendBuffer(msg.getMessageData()); } /** @@ -401,15 +329,15 @@ public boolean sendMessage(Message msg) throws IOException { * @return true if message queued successfully, false otherwise * @throws IOException If IO error occurs */ - private boolean sendObject(MessageType type, ACell payload) throws IOException { + private boolean sendObject(ACell payload) throws IOException { Counters.sendCount++; Blob enc = Format.encodeMultiCell(payload,true); if (log.isTraceEnabled()) { - log.trace("Sending message: " + type + " :: " + payload + " to " + getRemoteAddress() + " format: " + log.trace("Sending message: " + payload + " to " + getRemoteAddress() + " format: " + Cells.encode(payload).toHexString()); } - boolean sent = sendBuffer(type, enc); + boolean sent = sendBuffer(enc); return sent; } @@ -421,14 +349,14 @@ private boolean sendObject(MessageType type, ACell payload) throws IOException { * @return true if message sent, false otherwise * @throws IOException */ - private boolean sendBuffer(MessageType type, Blob data) throws IOException { + private boolean sendBuffer(Blob data) throws IOException { // synchronise on sender synchronized (sender) { if (!sender.canSendMessage()) return false; int dataLength = Utils.checkedInt(data.count()); - // Total message length field is one byte for message code + encoded object length - int messageLength = dataLength + 1; + // Total message length field is encoded object length + int messageLength = dataLength; boolean sent; int headerLength; // ensure frameBuf is clear and ready for writing @@ -436,7 +364,6 @@ private boolean sendBuffer(MessageType type, Blob data) throws IOException { // write message header (length plus message code) Format.writeMessageLength(frameBuf, messageLength); - frameBuf.put(type.getMessageCode()); headerLength = frameBuf.position(); // now write message @@ -463,12 +390,12 @@ private boolean sendBuffer(MessageType type, Blob data) throws IOException { } if (log.isTraceEnabled()) { - log.trace("Sent message " + type + " of length: " + dataLength + " Connection ID: " + log.trace("Sent message of length: " + dataLength + " Connection ID: " + System.identityHashCode(this)); } } else { - log.warn("sendBuffer failed with message {} of length: {} Connection ID: {}" - , type, dataLength, System.identityHashCode(this)); + log.warn("sendBuffer failed with message of length: {} Connection ID: {}" + , dataLength, System.identityHashCode(this)); } return sent; } @@ -476,6 +403,7 @@ private boolean sendBuffer(MessageType type, Blob data) throws IOException { } + @Override public synchronized void close() { SocketChannel chan = (SocketChannel) channel; if (chan != null) { @@ -492,11 +420,7 @@ public void finalize() { close(); } - /** - * Checks if this connection is closed (i.e. the underlying channel is closed) - * - * @return true if the channel is closed, false otherwise. - */ + @Override public boolean isClosed() { return !channel.isOpen(); } @@ -612,7 +536,7 @@ protected static void selectRead(SelectionKey key) throws IOException { log.trace("Channel closed from: {}", conn.getRemoteAddress()); key.cancel(); } catch (BadFormatException e) { - log.debug("Cancelled connection to Peer: Bad data format from: " + conn.getRemoteAddress() + " " + log.info("Cancelled connection to Peer: Bad data format from: " + conn.getRemoteAddress() + " " + e.getMessage()); key.cancel(); } catch (HandlerException e) { @@ -635,20 +559,14 @@ protected static void selectRead(SelectionKey key) throws IOException { */ public int handleChannelRecieve() throws IOException, BadFormatException, HandlerException { AStore savedStore = Stores.current(); - try { - // set the current store for handling incoming messages - Stores.setCurrent(store); - int recd= receiver.receiveFromChannel(channel); - int total =recd; - while (recd>0) { - recd=receiver.receiveFromChannel(channel); - total+=recd; - } - if (recd>0) lastActivity=System.currentTimeMillis(); - return total; - } finally { - Stores.setCurrent(savedStore); + int recd= receiver.receiveFromChannel(channel); + int total =recd; + while (recd>0) { + recd=receiver.receiveFromChannel(channel); + total+=recd; } + if (recd>0) lastActivity=System.currentTimeMillis(); + return total; } /** @@ -684,19 +602,7 @@ public boolean flushBytes() throws IOException { @Override public String toString() { - return "PeerConnection: " + channel; - } - - public AccountKey getTrustedPeerKey() { - return trustedPeerKey; - } - - public void setTrustedPeerKey(AccountKey value) { - trustedPeerKey = value; - } - - public boolean isTrusted() { - return trustedPeerKey != null; + return "NIO Connection: " + channel; } public long getLastActivity() { diff --git a/convex-peer/src/main/java/convex/net/NIOServer.java b/convex-peer/src/main/java/convex/net/impl/nio/NIOServer.java similarity index 82% rename from convex-peer/src/main/java/convex/net/NIOServer.java rename to convex-peer/src/main/java/convex/net/impl/nio/NIOServer.java index ea7f099a8..4aa11683a 100644 --- a/convex-peer/src/main/java/convex/net/NIOServer.java +++ b/convex-peer/src/main/java/convex/net/impl/nio/NIOServer.java @@ -1,6 +1,5 @@ -package convex.net; +package convex.net.impl.nio; -import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; import java.net.ServerSocket; @@ -14,15 +13,18 @@ import java.nio.channels.SocketChannel; import java.util.Iterator; import java.util.Set; +import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import convex.core.Constants; import convex.core.exceptions.BadFormatException; +import convex.core.store.AStore; import convex.core.store.Stores; import convex.core.util.Utils; -import convex.net.impl.HandlerException; +import convex.net.AServer; +import convex.net.Message; import convex.peer.Config; import convex.peer.Server; @@ -36,7 +38,7 @@ * receive queue is full (thereby applying back-pressure to clients) * */ -public class NIOServer implements Closeable { +public class NIOServer extends AServer { public static final int DEFAULT_PORT = 18888; private static final Logger log = LoggerFactory.getLogger(NIOServer.class.getName()); @@ -51,10 +53,18 @@ public class NIOServer implements Closeable { private boolean running = false; - private final Server server; + private final Consumer receiveAction; - private NIOServer(Server server) { - this.server = server; + private final AStore store; + + protected NIOServer(AStore store, Consumer receiveAction) { + this.store=store; + this.receiveAction=receiveAction; + } + + + private AStore getStore() { + return store; } /** @@ -64,7 +74,7 @@ private NIOServer(Server server) { * @return New NIOServer instance */ public static NIOServer create(Server server) { - return new NIOServer(server); + return new NIOServer(server.getStore(),server.getReceiveAction()); } /** @@ -74,38 +84,41 @@ public static NIOServer create(Server server) { * @param port Port to use. If 0 or null, a default port will be used, with fallback to a random port * @throws IOException in case of IO problem */ - public void launch(String bindAddress, Integer port) throws IOException { + public void launch() throws IOException { ssc = ServerSocketChannel.open(); // Set receive buffer size ssc.socket().setReceiveBufferSize(Config.SOCKET_SERVER_BUFFER_SIZE); ssc.socket().setReuseAddress(true); - bindAddress = (bindAddress == null) ? "::" : bindAddress; + String bindAddress = "::"; // Bind to a port - InetSocketAddress bindSA; - if (port == null) { - port = 0; - } - if (port==0) { - try { - bindSA = new InetSocketAddress(bindAddress, Constants.DEFAULT_PEER_PORT); - ssc.bind(bindSA); - } catch (IOException e) { - // try again with random port - bindSA = new InetSocketAddress(bindAddress, 0); + { + InetSocketAddress bindSA; + Integer port=getPort(); + if (port == null) { + port = 0; + } + if (port<=0) { + try { + bindSA = new InetSocketAddress(bindAddress, Constants.DEFAULT_PEER_PORT); + ssc.bind(bindSA); + } catch (IOException e) { + // try again with random port + bindSA = new InetSocketAddress(bindAddress, 0); + ssc.bind(bindSA); + } + } else { + bindSA = new InetSocketAddress(bindAddress, port); ssc.bind(bindSA); } - } else { - bindSA = new InetSocketAddress(bindAddress, port); - ssc.bind(bindSA); + + // Find out which port we actually bound to + bindSA = (InetSocketAddress) ssc.getLocalAddress(); + setPort(ssc.socket().getLocalPort()); } - // Find out which port we actually bound to - bindSA = (InetSocketAddress) ssc.getLocalAddress(); - port = ssc.socket().getLocalPort(); - // change to bnon-blocking mode ssc.configureBlocking(false); @@ -117,10 +130,10 @@ public void launch(String bindAddress, Integer port) throws IOException { // set running status now, so that loops don't terminate running = true; - Thread selectorThread = new Thread(selectorLoop, "NIO Server loop on port: " + port); + Thread selectorThread = new Thread(selectorLoop, "NIO Server loop on port: " + getPort()); selectorThread.setDaemon(true); // daemon thread so it doesn't stop shutdown selectorThread.start(); - log.debug("NIO server started on port {}", port); + log.debug("NIO server started on port {}", getPort()); } long lastConnectionPrune=0; @@ -133,7 +146,7 @@ public void launch(String bindAddress, Integer port) throws IOException { @Override public void run() { // Use the store configured for the owning server. - Stores.setCurrent(server.getStore()); + Stores.setCurrent(getStore()); try { // loop unless we are interrupted while (running && !Thread.currentThread().isInterrupted()) { @@ -176,8 +189,8 @@ public void run() { // keys.clear(); } - } catch (IOException e) { - log.error("Unexpected IO Exception, terminating selector loop: ", e); + } catch (Exception e) { + log.error("Unexpected Exception, terminating selector loop: ", e); } finally { try { // close all client channels @@ -205,14 +218,13 @@ public void run() { } log.debug("Selector loop ended on port: " + getPort()); } + + }; - /** - * Gets the port that this server instance is listening on. - * - * @return Port number, or 0 if a server socket is not bound. - */ - public int getPort() { + + @Override + public Integer getPort() { if (ssc == null) return 0; ServerSocket socket = ssc.socket(); @@ -261,7 +273,11 @@ private Connection ensureConnection(SelectionKey key) throws IOException { } private Connection createClientConnection(SocketChannel sc) throws IOException { - return Connection.create(sc, server.getReceiveAction(), server.getStore(), null); + return Connection.create(sc, getReceiveAction(), null); + } + + protected Consumer getReceiveAction() { + return receiveAction; } protected void selectRead(SelectionKey key) throws IOException { @@ -269,7 +285,7 @@ protected void selectRead(SelectionKey key) throws IOException { // log.info("Connection read from: "+sc.getRemoteAddress()+" with key:"+key); Connection conn = ensureConnection(key); if (conn == null) - throw new IOException("No Connection in selecion key"); + throw new IOException("No Connection in selection key"); try { int n = conn.handleChannelRecieve(); if (n < 0) { @@ -286,7 +302,7 @@ protected void selectRead(SelectionKey key) throws IOException { e.getMessage()); // TODO: blacklist peer? key.cancel(); - } catch (HandlerException e) { + } catch (Exception e) { log.warn("Unexpected exception in receive handler", e.getCause()); key.cancel(); } diff --git a/convex-peer/src/main/java/convex/peer/API.java b/convex-peer/src/main/java/convex/peer/API.java index 027f066ec..f39e95e46 100644 --- a/convex-peer/src/main/java/convex/peer/API.java +++ b/convex-peer/src/main/java/convex/peer/API.java @@ -47,7 +47,7 @@ public class API { *
  • :store (optional, AStore or String filename) - AStore instance. Defaults to the configured global store *
  • :keystore (optional, Keystore or string filename) - Keystore instance. Read only, used for key lookup if necessary. *
  • :storepass (optional, string) - Integrity password for keystore. If omitted, no integrity check is performed - *
  • :source (optional, String) - URL for Peer to replicate initial State/Belief from. + *
  • :source (optional, String or Socket Address) - URL for Peer to replicate initial State/Belief from. *
  • :state (optional, State) - Genesis state. Defaults to a fresh genesis state for the Peer if neither :source nor :state is specified *
  • :restore (optional, Boolean) - Boolean Flag to restore from existing store. Default to true *
  • :persist (optional, Boolean) - Boolean flag to determine if peer state should be persisted in store at server close. Default true. diff --git a/convex-peer/src/main/java/convex/peer/BeliefPropagator.java b/convex-peer/src/main/java/convex/peer/BeliefPropagator.java index ccfc26c0d..b8cd85f5d 100644 --- a/convex-peer/src/main/java/convex/peer/BeliefPropagator.java +++ b/convex-peer/src/main/java/convex/peer/BeliefPropagator.java @@ -26,6 +26,7 @@ import convex.core.data.Index; import convex.core.data.Ref; import convex.core.data.SignedData; +import convex.core.data.Vectors; import convex.core.exceptions.BadFormatException; import convex.core.exceptions.InvalidDataException; import convex.core.exceptions.MissingDataException; @@ -46,7 +47,7 @@ public class BeliefPropagator extends AThreadedComponent { /** * Wait period for beliefs received in each iteration of Server Belief Merge loop. */ - private static final long AWAIT_BELIEFS_PAUSE = 60L; + private static final long AWAIT_BELIEFS_PAUSE = 30L; public static final int BELIEF_REBROADCAST_DELAY=300; @@ -88,7 +89,7 @@ public BeliefPropagator(Server server) { * Check if the propagator wants the latest Belief for rebroadcast * @return True is rebroadcast is due */ - public boolean isRebroadcastDue() { + boolean isRebroadcastDue() { return (lastBroadcastTime+BELIEF_REBROADCAST_DELAY) signedBlock= server.transactionHandler.maybeGetBlock(); boolean published=false; - if (signedBlock!=null) { - belief=belief.proposeBlock(server.getKeyPair(),signedBlock); + SignedData[] signedBlocks= server.transactionHandler.maybeGenerateBlocks(); + if (signedBlocks!=null) { + belief=belief.proposeBlock(server.getKeyPair(),signedBlocks); + published=true; + if (log.isDebugEnabled()) { - Block bl=signedBlock.getValue(); - log.debug("Block proposed: {} tx(s), size={}, hash={}", bl.getTransactions().count(), signedBlock.getMemorySize(),signedBlock.getHash()); + log.debug("Blocks proposed: "+Vectors.of((Object[])signedBlocks).map(sb->sb.getHash())); } - published=true; } // Return true iff we published a new Block or updated our own Order @@ -252,6 +253,7 @@ private void observeBeliefUpdate(Belief b) { * @return True if Peer Belief Order was changed, false otherwise. */ protected boolean maybeMergeBeliefs(Belief... newBeliefs) { + if ((newBeliefs==null)||(newBeliefs.length==0)) return false; try { long ts=Utils.getCurrentTimestamp(); AKeyPair kp=server.getKeyPair(); @@ -287,7 +289,9 @@ protected boolean maybeMergeBeliefs(Belief... newBeliefs) { } /** - * Await an incoming Belief for belief merge / potential update + * Await incoming Belief for all incoming belief merges / potential update. This merges multiple incoming beliefs into a single Belief + * which compacts the number of incoming orders for the upcoming Belief Merge + * * @return Incoming Belief, or null if nothing arrived within time window * @throws InterruptedException */ @@ -300,10 +304,17 @@ private Belief awaitBelief() throws InterruptedException { LoadMonitor.up(); if (firstEvent==null) return null; // nothing arrived + // Drain queue of all incoming Beliefs beliefMessages.add(firstEvent); beliefQueue.drainTo(beliefMessages); + + if (log.isDebugEnabled()) { + log.debug("Belief Messages received: "+beliefMessages.size()); + } + + // Build a Map of current Orders. We compare incoming Orders to this + // So that we can identify new information HashMap> newOrders=belief.getOrdersHashMap(); - // log.info("Merging Beliefs: "+allBeliefs.size()); boolean anyOrderChanged=false; for (Message m: beliefMessages) { @@ -313,18 +324,26 @@ private Belief awaitBelief() throws InterruptedException { if (!anyOrderChanged) return null; Belief newBelief= Belief.create(newOrders); + // log.info("New Belief received"); return newBelief; } - + /** + * Merge a single Belief message into a map of accumulated latest Orders + * @param orders + * @param m + * @return true if there was any updated order Order, false otherwise + */ protected boolean mergeBeliefMessage(HashMap> orders, Message m) { boolean changed=false; AccountKey myKey=server.getPeerKey(); + try { // Add to map of new Beliefs received for each Peer beliefReceivedCount++; try { ACell payload=m.getPayload(); + // log.info("Merging Belief message: "+Cells.getHash(payload)); Collection> a = Belief.extractOrders(payload); for (SignedData so:a ) { AccountKey key=so.getAccountKey(); @@ -335,6 +354,8 @@ protected boolean mergeBeliefMessage(HashMap> orde if (orders.containsKey(key)) { Order newOrder=so.getValue(); Order oldOrder=orders.get(key).getValue(); + + boolean replace=BeliefMerge.compareOrders(oldOrder, newOrder); if (!replace) continue; } @@ -348,22 +369,26 @@ protected boolean mergeBeliefMessage(HashMap> orde break; }; + // Ensure we can persist newly received Order so=Cells.persist(so); observeOrderUpdate(so); orders.put(key, so); changed=true; } catch (MissingDataException e) { + // Something missing in received Belief. This is expected for + // Partial Belief update messages server.getConnectionManager().alertMissing(m,e,key); } catch (IOException e) { // This is pretty bad, probably we lost the store? // We certainly can't propagate the newly received order // throw new Error(e); - log.warn("IO exception tryin to merge Order",e); + log.warn("IO exception trying to merge Order",e); return changed; } } } catch (MissingDataException e) { + log.debug("Missing data in Belief message "+m.getHash()); server.getConnectionManager().alertMissing(m,e,null); } } catch (ClassCastException | BadFormatException e) { @@ -379,11 +404,6 @@ private void observeOrderUpdate(SignedData so) { obs.accept(so); } } - - private void doBroadcast(Message msg) throws InterruptedException { - server.manager.broadcast(msg); - beliefBroadcastCount++; - } private Message createFullUpdateMessage() throws IOException { ArrayList novelty=new ArrayList<>(); @@ -401,7 +421,7 @@ private Message createFullUpdateMessage() throws IOException { belief=Cells.announce(belief, noveltyHandler); lastFullBroadcastBelief=belief; - Message msg = createBelief(belief, novelty); + Message msg = createPartialBelief(belief, novelty); long messageSize=msg.getMessageData().count(); if (messageSize>=CPoSConstants.MAX_MESSAGE_LENGTH*0.95) { log.warn("Long Belief Delta message: "+messageSize); @@ -433,7 +453,7 @@ private Message createQuickUpdateMessage() throws IOException { orders=orders.assoc(key, order); belief=belief.withOrders(orders); - Message msg = createBelief(order, novelty); + Message msg = createPartialBelief(order, novelty); long messageSize=msg.getMessageData().count(); if (messageSize>=CPoSConstants.MAX_MESSAGE_LENGTH*0.95) { log.warn("Long Belief Delta message: "+messageSize); @@ -447,17 +467,17 @@ private Message createQuickUpdateMessage() throws IOException { * @param belief Belief top level Cell to encode * @return Message instance */ - private static Message createBelief(ACell payload, List novelty) { + private static Message createPartialBelief(ACell payload, List novelty) { int n=novelty.size(); if (n==0) { //log.warn("No novelty in Belief"); - novelty.add(n, payload); + novelty.add(payload); } else if (!payload.equals(novelty.get(n-1))) { //log.warn("Last element not Belief out of "+novelty.size()); - novelty.add(n, payload); + novelty.add(payload); } Blob data=Format.encodeDelta(novelty); - return Message.create(MessageType.BELIEF,null,data); + return Message.create(MessageType.BELIEF,payload,data); } private Belief lastFullBroadcastBelief; diff --git a/convex-peer/src/main/java/convex/peer/Config.java b/convex-peer/src/main/java/convex/peer/Config.java index 5e603bebb..3c6ec3949 100644 --- a/convex-peer/src/main/java/convex/peer/Config.java +++ b/convex-peer/src/main/java/convex/peer/Config.java @@ -41,6 +41,16 @@ public class Config { * Size of default client socket send buffer */ public static final int SOCKET_SEND_BUFFER_SIZE = 2*65536; + + /** + * Flag to use Netty client connections + */ + public static final boolean USE_NETTY_CLIENT = true; + + /** + * Flag to use Netty server implementation + */ + public static final boolean USE_NETTY_SERVER = true; /** * Delay before rebroadcasting Belief if not in consensus @@ -88,7 +98,7 @@ public class Config { /** * Size of incoming Belief queue */ - public static final int BELIEF_QUEUE_SIZE = 500; + public static final int BELIEF_QUEUE_SIZE = 200; /** * Checks if the config specifies a valid store diff --git a/convex-peer/src/main/java/convex/peer/ConnectionManager.java b/convex-peer/src/main/java/convex/peer/ConnectionManager.java index bcdcfc947..3513f42d9 100644 --- a/convex-peer/src/main/java/convex/peer/ConnectionManager.java +++ b/convex-peer/src/main/java/convex/peer/ConnectionManager.java @@ -8,8 +8,9 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.*; +import java.util.concurrent.TimeoutException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +35,8 @@ import convex.core.lang.RT; import convex.core.util.LoadMonitor; import convex.core.util.Utils; +import convex.net.AConnection; import convex.net.ChallengeRequest; -import convex.net.Connection; import convex.net.IPUtils; import convex.net.Message; @@ -70,15 +71,10 @@ public class ConnectionManager extends AThreadedComponent { */ static final long POLL_ACQUIRE_TIMEOUT_MILLIS = 12000; - /** - * Timeout for a regular Belief delta broadcast - */ - private static final long BROADCAST_TIMEOUT = 1000; - /** * Map of current connections. */ - private final HashMap connections = new HashMap<>(); + private final HashMap connections = new HashMap<>(); /** * The list of outgoing challenges that are being made to remote peers @@ -93,13 +89,13 @@ public class ConnectionManager extends AThreadedComponent { * Celled by the connection manager to ensure we are tracking latest Beliefs on the network * @throws InterruptedException */ - private void pollBelief() throws InterruptedException { + private void maybePollBelief() throws InterruptedException { try { // Poll only if no recent consensus updates long lastConsensus = server.getPeer().getConsensusState().getTimestamp().longValue(); if (lastConsensus + pollDelay >= Utils.getCurrentTimestamp()) return; - ArrayList conns = new ArrayList<>(connections.values()); + ArrayList conns = new ArrayList<>(connections.values()); if (conns.size() == 0) { // Nothing to do return; @@ -107,22 +103,33 @@ private void pollBelief() throws InterruptedException { // TODO: probably shouldn't make a new connection? // Maybe use Convex instance instead of Connection? - Connection c = conns.get(random.nextInt(conns.size())); + Convex c = conns.get(random.nextInt(conns.size())); - if (c.isClosed()) return; - ; - try (Convex convex = Convex.connect(c.getRemoteAddress())) { - // use requestStatusSync to auto acquire hash of the status instead of the value - Result result=convex.requestStatusSync(POLL_TIMEOUT_MILLIS); - AVector status = result.getValue(); + if (!c.isConnected()) { + log.warn("Attempted to poll from closed connection"); + return; + } - Hash h=RT.ensureHash(status.get(0)); + Convex convex = c; + + // use requestStatusSync to auto acquire hash of the status instead of the value + Result result=convex.requestStatusSync(POLL_TIMEOUT_MILLIS); + if (result.isError()) { + log.warn("Failure requesting status during polling: "+result); + return; + } + + AVector status = result.getValue(); + if (status==null) { + log.warn("Dubious status response message: "+result); + return; + } + Hash h=RT.ensureHash(status.get(0)); - Belief sb=(Belief) convex.acquire(h).get(POLL_ACQUIRE_TIMEOUT_MILLIS,TimeUnit.MILLISECONDS); + Belief sb=(Belief) convex.acquire(h).get(POLL_ACQUIRE_TIMEOUT_MILLIS,TimeUnit.MILLISECONDS); - server.queueBelief(Message.createBelief(sb)); - } - } catch (RuntimeException | TimeoutException | ExecutionException | IOException t) { + server.queueBelief(Message.createBelief(sb)); + } catch (Exception t) { if (server.isLive()) { log.warn("Belief Polling failed: {}",t.getClass().toString()+" : "+t.getMessage()); } @@ -143,11 +150,9 @@ protected void maintainConnections() throws InterruptedException { AccountKey[] peers = connections.keySet().toArray(new AccountKey[currentPeerCount]); for (AccountKey p: peers) { - Connection conn=connections.get(p); - - // Remove closed connections. No point keeping these - if ((conn==null)||(conn.isClosed())) { - closeConnection(p); + Convex conn=getConnection(p); + if (conn==null) { + // Must have lost this connection currentPeerCount--; continue; } @@ -157,8 +162,8 @@ protected void maintainConnections() throws InterruptedException { * withdrawn, have trivial stake or are slashed from current consideration. */ PeerStatus ps=s.getPeer(p); - if ((ps==null)||(ps.getTotalStake()<=CPoSConstants.MINIMUM_EFFECTIVE_STAKE)) { - closeConnection(p); + if ((ps==null)||(ps.getBalance()<=CPoSConstants.MINIMUM_EFFECTIVE_STAKE)) { + closeConnection(p,"Insufficient stake"); currentPeerCount--; continue; } @@ -176,7 +181,7 @@ protected void maintainConnections() throws InterruptedException { double dropRate=millisSinceLastUpdate/(double)Config.PEER_CONNECTION_DROP_TIME; if (random.nextDouble()<(dropRate*(1.0-keepChance))) { - closeConnection(p); + closeConnection(p,"Dropping minor peers"); currentPeerCount--; continue; } @@ -185,7 +190,7 @@ protected void maintainConnections() throws InterruptedException { // send request for a trusted peer connection if necessary // TODO: need to find out why the response message is not being received by the peers - requestChallenge(p, conn, server.getPeer()); + // requestChallenge(p, conn, server.getPeer()); } // refresh peers list @@ -199,7 +204,7 @@ protected void maintainConnections() throws InterruptedException { private void tryRandomConnect(State s) throws InterruptedException { // Connect to a random peer with host address by stake - // SECURITY: stake weighted connection is important to avoid bad peers + // SECURITY: stake weighted connection is important to avoid bad / irrelevant peers // influencing the connection pool Set potentialPeers=s.getPeers().keySet(); @@ -214,9 +219,10 @@ private void tryRandomConnect(State s) throws InterruptedException { PeerStatus ps=s.getPeers().get(peerKey); if (ps==null) continue; // skip AString hostName=ps.getHostname(); - if (hostName==null) continue; + if (hostName==null) continue; // don't now where to connect to! InetSocketAddress maybeAddress=IPUtils.toInetSocketAddress(hostName.toString()); if (maybeAddress==null) continue; + long peerStake=ps.getPeerStake(); if (peerStake>CPoSConstants.MINIMUM_EFFECTIVE_STAKE) { double t=random.nextDouble()*(accStake+peerStake); @@ -263,9 +269,10 @@ public ConnectionManager(Server server) { * @param peerKey Peer key linked to the connection to close and remove. * */ - public void closeConnection(AccountKey peerKey) { - Connection conn=connections.get(peerKey); + public void closeConnection(AccountKey peerKey,String reason) { + Convex conn=connections.get(peerKey); if (conn!=null) { + log.info("Removed peer connection to "+peerKey+ " Reason="+reason); conn.close(); connections.remove(peerKey); } @@ -275,8 +282,9 @@ public void closeConnection(AccountKey peerKey) { * Close all outgoing connections from this Peer */ public void closeAllConnections() { - for (Connection conn:connections.values()) { - if (conn!=null) conn.close(); + HashMap conns=new HashMap<>(getConnections()); + for (AccountKey peerKey:conns.keySet()) { + closeConnection(peerKey,"Closing all connections"); } connections.clear(); } @@ -286,7 +294,7 @@ public void closeAllConnections() { * * @return Set of connections */ - public HashMap getConnections() { + public HashMap getConnections() { return connections; } @@ -307,9 +315,16 @@ public boolean isConnected(AccountKey peerKey) { * * @return Connection instance, or null if not found */ - public Connection getConnection(AccountKey peerKey) { - if (!connections.containsKey(peerKey)) return null; - return connections.get(peerKey); + public Convex getConnection(AccountKey peerKey) { + Convex c=connections.get(peerKey); + if (c==null) return null; + + if (!c.isConnected()) { + closeConnection(peerKey,"Removing already closed connection"); + return null; + } + + return c; } /** @@ -320,21 +335,6 @@ public int getConnectionCount() { return connections.size(); } - /** - * Returns the number of trusted connections - * @return Number of trusted connections - * - */ - public int getTrustedConnectionCount() { - int result = 0; - for (Connection connection : connections.values()) { - if (connection.isTrusted()) { - result ++; - } - } - return result; - } - public void processChallenge(Message m, Peer thisPeer) { SignedData> signedData=null; try { @@ -479,9 +479,9 @@ AccountKey processResponse(Message m, Peer thisPeer) { // remove from list incase this fails, we can generate another challenge challengeList.remove(fromPeer); - Connection connection = getConnection(fromPeer); + Convex connection = getConnection(fromPeer); if (connection != null) { - connection.setTrustedPeerKey(fromPeer); + // connection.setTrustedKey(fromPeer); } // return the trusted peer key @@ -498,7 +498,7 @@ AccountKey processResponse(Message m, Peer thisPeer) { * @param thisPeer Source peer that the challenge is issued from * */ - public void requestChallenge(AccountKey toPeerKey, Connection connection, Peer thisPeer) { + public void requestChallenge(AccountKey toPeerKey, AConnection connection, Peer thisPeer) { synchronized(challengeList) { if (connection.isTrusted()) { return; @@ -513,7 +513,7 @@ public void requestChallenge(AccountKey toPeerKey, Connection connection, Peer t challengeList.remove(toPeerKey); } ChallengeRequest request = ChallengeRequest.create(toPeerKey); - if (request.send(connection, thisPeer)>=0) { + if (request.send(connection, thisPeer)) { challengeList.put(toPeerKey, request); } else { // TODO: check OK to do nothing and send later? @@ -528,55 +528,54 @@ public void requestChallenge(AccountKey toPeerKey, Connection connection, Peer t * @throws InterruptedException If broadcast is interrupted * */ - public void broadcast(Message msg) throws InterruptedException { - HashMap hm=getCurrentConnections(); + public void broadcast(Message msg) { + HashMap hm=getCurrentConnections(); + + if (hm.isEmpty()) { + log.debug("No connections to broadcast to from "+server.getPeerKey()); + return; + } - long start=Utils.getCurrentTimestamp(); - while ((!hm.isEmpty())&&(start+BROADCAST_TIMEOUT>Utils.getCurrentTimestamp())) { - ArrayList> left=new ArrayList<>(hm.entrySet()); + ArrayList> left=new ArrayList<>(hm.entrySet()); - // Shuffle order for sending - Utils.shuffle(left); + // Shuffle order for sending + Utils.shuffle(left); - for (Map.Entry me: left) { - Connection pc=me.getValue(); - boolean sent; - try { - sent = pc.sendMessage(msg); - } catch (IOException e) { - // Something went wrong, lose this connection - closeConnection(me.getKey()); - sent=false; - } - if (sent) { - hm.remove(me.getKey()); - } else { - // log.warn("Broadcast failed!"); + for (Map.Entry me: left) { + Convex pc=me.getValue(); + CompletableFuture sent=pc.message(msg); + if (sent.isDone()) { + Result r=sent.join(); + if (r.isError()) { + // log.warn("Immediate error broadcasting: "+r); } } - - // terminate loop if everything is successfully sent - if (hm.isEmpty()) break; - - // Avoid a busy wait if buffers are full and still have things to send - LoadMonitor.down(); - Thread.sleep(10); - LoadMonitor.up(); } - if ((!hm.isEmpty())&&server.isLive()) { - ArrayList> left=new ArrayList<>(hm.entrySet()); - Map.Entry drop=left.get(random.nextInt(left.size())); - AccountKey dropKey=drop.getKey(); - closeConnection(dropKey); - log.warn("Unable to send broadcast to "+hm.size()+" peers, dropped one connection to: "+dropKey); - } +// If we couldn't broadcast to everyone, we are probably overloaded. drop a random connection +// if ((!hm.isEmpty())&&server.isLive()) { +// ArrayList> left=new ArrayList<>(hm.entrySet()); +// Map.Entry drop=left.get(random.nextInt(left.size())); +// AccountKey dropKey=drop.getKey(); +// closeConnection(dropKey); +// log.warn("Unable to send broadcast to "+hm.size()+" peers, dropped one connection to: "+dropKey+ "remaining = "+getConnectionCount()); +// } } - private HashMap getCurrentConnections() { - synchronized(connections) { - return new HashMap<>(connections); + /** + * Gets a new hashmap of connection which are currently live + * @return + */ + private synchronized HashMap getCurrentConnections() { + HashMap liveConnections=new HashMap<>(); + for (Map.Entry me: connections.entrySet()) { + AccountKey peerKey=me.getKey(); + Convex c=me.getValue(); + if ((c!=null)&&(c.isConnected())) { + liveConnections.put(peerKey,c); + } } + return liveConnections; } /** @@ -585,39 +584,47 @@ private HashMap getCurrentConnections() { * @return new Connection, or null if attempt fails * @throws InterruptedException */ - public Connection connectToPeer(InetSocketAddress hostAddress) throws InterruptedException { - Connection newConn = null; + public Convex connectToPeer(InetSocketAddress hostAddress) throws InterruptedException { try { - // Use temp client connection to query status Convex convex=Convex.connect(hostAddress); Result result = convex.requestStatusSync(Config.DEFAULT_CLIENT_TIMEOUT); if (result.isError()) { - log.info("Bad status message from remote Peer"); + log.info("Bad status message from remote Peer: "+result); + convex.close(); return null; + } else { + log.debug("Got status from peer: "+result); } AVector status = result.getValue(); // close the temp connection to Convex API - convex.close(); AccountKey peerKey =RT.ensureAccountKey(status.get(3)); if (peerKey==null) return null; - Connection existing=connections.get(peerKey); - if ((existing!=null)&&!existing.isClosed()) return existing; - synchronized(connections) { - // reopen with connection to the peer and handle server messages - newConn = Connection.connect(hostAddress, server.receiveAction, server.getStore(), null,Config.SOCKET_PEER_BUFFER_SIZE,Config.SOCKET_PEER_BUFFER_SIZE); - connections.put(peerKey, newConn); + Convex existing=getConnection(peerKey); + if ((existing!=null)&&existing.isConnected()) { + log.info("Trying to connect with existing connection"); + convex.close(); + return existing; + } else { + addConnection(peerKey, convex); } + return convex; } catch (IOException | TimeoutException e) { - // ignore any errors from the peer connections + log.warn("Error connecting to peer: ",e); return null; } catch (UnresolvedAddressException e) { log.info("Unable to resolve host address: "+hostAddress); return null; } - return newConn; + } + + public synchronized void addConnection(AccountKey peerKey, Convex convex) { + synchronized(connections) { + log.debug("Connected to Peer: "+peerKey+ " at "+convex.getHostAddress()); + connections.put(peerKey, convex); + } } @Override @@ -626,9 +633,6 @@ public void close() { try { Message msg = Message.createGoodBye(); broadcast(msg); - } catch (InterruptedException e ) { - // maintain interrupt status - Thread.currentThread().interrupt(); } finally { super.close(); } @@ -648,7 +652,7 @@ protected void loop() throws InterruptedException { Thread.sleep(ConnectionManager.SERVER_CONNECTION_PAUSE); LoadMonitor.up(); maintainConnections(); - pollBelief(); + maybePollBelief(); } @Override @@ -673,13 +677,21 @@ public void alertBadMessage(Message m, String reason) { * @param peerKey Peer key which triggered missing data */ public void alertMissing(Message m, MissingDataException e, AccountKey peerKey) { - if (log.isDebugEnabled()) { - String message= "Missing data "+e.getMissingHash(); - log.debug(message); + try { + Convex conn=getConnection(peerKey); + if (conn==null) return; // No outbound connection to this peer, so just ignore + + if (log.isDebugEnabled()) { + String message= "Missing data alert "+e.getMissingHash(); + // String s=PrintUtils.printRefTree(m.getPayload().getRef()); + // System.out.println(s); + log.info(message); + } + + // TODO: possibly fire off request to specific Peer? Unclear if this improves things generally, but might avoid polling + } catch (Exception ex) { + log.warn("Unexpected error responding to missing data",ex); } - - // TODO: possibly fire off request to specific Peer? Unclear if this improves things generally, but might avoid polling - } } diff --git a/convex-peer/src/main/java/convex/peer/QueryHandler.java b/convex-peer/src/main/java/convex/peer/QueryHandler.java index 8336712e2..bfa06d436 100644 --- a/convex-peer/src/main/java/convex/peer/QueryHandler.java +++ b/convex-peer/src/main/java/convex/peer/QueryHandler.java @@ -12,7 +12,6 @@ import convex.core.cvm.Address; import convex.core.data.ACell; import convex.core.data.AVector; -import convex.core.data.prim.CVMLong; import convex.core.exceptions.BadFormatException; import convex.core.lang.RT; import convex.core.util.LoadMonitor; @@ -55,7 +54,7 @@ protected void loop() throws InterruptedException { case QUERY: handleQuery(m); break; - case REQUEST_DATA: + case DATA_REQUEST: handleDataRequest(m); break; default: @@ -82,7 +81,7 @@ protected void handleDataRequest(Message m) { } } catch (BadFormatException e) { log.warn("Unable to deliver missing data due badly formatted DATA_REQUEST: {}", m); - } catch (RuntimeException e) { + } catch (Exception e) { log.warn("Unable to deliver missing data due to exception:", e); } } @@ -91,11 +90,11 @@ private void handleQuery(Message m) { try { // query is a vector [id , form, address?] AVector v= m.getPayload(); - CVMLong id = (CVMLong) v.get(0); - ACell form = v.get(1); + ACell id = v.get(1); + ACell form = v.get(2); // extract the Address, might be null - Address address = RT.ensureAddress(v.get(2)); + Address address = RT.ensureAddress(v.get(3)); log.debug( "Processing query: {} with address: {}" , form, address); // log.log(LEVEL_MESSAGE, "Processing query: " + form + " with address: " + diff --git a/convex-peer/src/main/java/convex/peer/Server.java b/convex-peer/src/main/java/convex/peer/Server.java index b9387e3f6..d1f861dae 100644 --- a/convex-peer/src/main/java/convex/peer/Server.java +++ b/convex-peer/src/main/java/convex/peer/Server.java @@ -38,19 +38,19 @@ import convex.core.data.Strings; import convex.core.data.Vectors; import convex.core.data.prim.CVMLong; -import convex.core.exceptions.BadFormatException; import convex.core.exceptions.InvalidDataException; import convex.core.exceptions.MissingDataException; import convex.core.init.Init; import convex.core.lang.RT; import convex.core.store.AStore; import convex.core.store.Stores; -import convex.core.util.Counters; import convex.core.util.Shutdown; import convex.core.util.Utils; +import convex.net.AServer; import convex.net.Message; import convex.net.MessageType; -import convex.net.NIOServer; +import convex.net.impl.netty.NettyServer; +import convex.net.impl.nio.NIOServer; /** @@ -130,13 +130,20 @@ public class Server implements Closeable { /** * NIO Server instance */ - private NIOServer nio = NIOServer.create(this); + private AServer nio; private Server(HashMap config) throws ConfigException { this.config = config; // Critical to ensure we have the store set up before anything else. Stuff might break badly otherwise! this.store = Config.ensureStore(config); + + if (Config.USE_NETTY_SERVER) { + this.nio=NettyServer.create(this); + } else { + this.nio= NIOServer.create(this); + } + } // This doesn't actually do anything useful? Do we need this? @@ -239,7 +246,9 @@ public Peer syncPeer(AKeyPair keyPair, Convex convex) throws LaunchException, In } log.info("Retrieved Peer Belief: "+beliefHash+ " with memory size: "+belF.getMemorySize()); - convex.close(); + // Add the new connection since it seems good + getConnectionManager().addConnection(remoteKey,convex); + SignedData peerOrder=belF.getOrders().get(remoteKey); if (peerOrder!=null) { SignedData newOrder=keyPair.signData(peerOrder.getValue()); @@ -334,7 +343,8 @@ public void launch() throws LaunchException, InterruptedException { Object p = config.get(Keywords.PORT); Integer port = (p == null) ? null : Utils.toInt(p); - nio.launch((String)config.get(Keywords.BIND_ADDRESS), port); + nio.setPort(port); + nio.launch(); port = nio.getPort(); // Get the actual port (may be auto-allocated) // set running status now, so that loops don't immediately terminate @@ -372,15 +382,16 @@ private void goLive() { * * SECURITY: Should anticipate malicious messages * - * Runs on receiver thread, so we want to offload to a queue ASAP + * Runs on receiver thread, so we want to offload to a queue ASAP, never block * * @param m */ protected void processMessage(Message m) { - MessageType type = m.getType(); + // log.info("Message received: "+m.getMessageData()); AStore tempStore=Stores.current(); try { Stores.setCurrent(this.store); + MessageType type = m.getType(); switch (type) { case BELIEF: processBelief(m); @@ -393,16 +404,15 @@ protected void processMessage(Message m) { break; case COMMAND: break; - case DATA: - processData(m); - break; - case REQUEST_DATA: + case DATA_REQUEST: processQuery(m); // goes on Query handler break; case QUERY: processQuery(m); break; case RESULT: + // We aren't expecting results here (on an inbound connection) + log.debug("unexpected Result received"); break; case TRANSACT: processTransact(m); @@ -414,13 +424,14 @@ protected void processMessage(Message m) { processStatus(m); break; default: - Result r=Result.create(m.getID(), Strings.create("Bad Message Type: "+type), ErrorCodes.ARGUMENT); - m.returnResult(r); + log.info("Unrecognised message: "+m); break; } } catch (MissingDataException e) { Hash missingHash = e.getMissingHash(); - log.trace("Missing data: {} in message of type {}" , missingHash,type); + log.info("Missing data: {} in message", missingHash); + } catch (Exception e) { + log.warn("Unexpected error processing peer message",e); } finally { Stores.setCurrent(tempStore); } @@ -542,26 +553,6 @@ protected void processQuery(Message m) { m.returnResult(r); } } - - private void processData(Message m) { - ACell payload; - try { - payload = m.getPayload(); - Counters.peerDataReceived++; - - Ref r = Ref.get(payload); - if (r.isEmbedded()) { - log.warn("DATA with embedded value: "+payload); - return; - } - r = r.persistShallow(); - } catch (BadFormatException | IOException e) { - log.debug("Error processing data: "+e.getMessage()); - m.closeConnection(); - return; - } - - } /** * Process an incoming message that represents a Belief @@ -578,7 +569,7 @@ protected void processBelief(Message m) { * Gets the port that this Server is currently accepting connections on * @return Port number */ - public int getPort() { + public Integer getPort() { return nio.getPort(); } diff --git a/convex-peer/src/main/java/convex/peer/TransactionHandler.java b/convex-peer/src/main/java/convex/peer/TransactionHandler.java index c72d5f9b2..ab7f95201 100644 --- a/convex-peer/src/main/java/convex/peer/TransactionHandler.java +++ b/convex-peer/src/main/java/convex/peer/TransactionHandler.java @@ -2,12 +2,10 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Consumer; -import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,12 +52,12 @@ */ public class TransactionHandler extends AThreadedComponent { - static final Logger log = LoggerFactory.getLogger(BeliefPropagator.class.getName()); + static final Logger log = LoggerFactory.getLogger(TransactionHandler.class.getName()); /** * Default minimum delay between proposing own transactions as a peer */ - private static final long OWN_BLOCK_DELAY=2000; + private static final long OWN_BLOCK_DELAY=10000; /** * Default minimum delay between proposing a block as a peer @@ -110,42 +108,52 @@ private void registerInterest(Hash signedTransactionHash, Message m) { interests.put(signedTransactionHash, m); } - private void processMessages() throws InterruptedException { - Result problem=checkPeerState(); - for (Message msg: messages) { - if (problem==null) { - processMessage(msg); - } else { - msg.returnResult(problem); - } - } - } + private static final Result ERR_NOT_LIVE=Result.error(ErrorCodes.STATE, Strings.create("Server is not live")).withSource(SourceCodes.PEER); + private static final Result ERR_NOT_REGISTERED=Result.error(ErrorCodes.STATE, Strings.create("Peer not registered in global state")).withSource(SourceCodes.PEER); + private static final Result ERR_NOT_STAKED=Result.error(ErrorCodes.STATE, Strings.create("Peer not sufficiently staked to publish transactions")).withSource(SourceCodes.PEER); private Result checkPeerState() { try { if (!server.isLive()) { - return Result.error(ErrorCodes.STATE, Strings.create("Server is not live")).withSource(SourceCodes.PEER); + return ERR_NOT_LIVE; } Peer p=server.getPeer(); - State s=server.getPeer().getConsensusState(); + State s=p.getConsensusState(); PeerStatus ps=s.getPeers().get(p.getPeerKey()); if (ps==null) { - return Result.error(ErrorCodes.STATE, Strings.create("Peer not registered in global state")).withSource(SourceCodes.PEER); + return ERR_NOT_REGISTERED; } + if (ps.getBalance() v = m.getPayload(); @SuppressWarnings("unchecked") - SignedData sd = (SignedData) v.get(1); + SignedData sd = (SignedData) v.get(2); // Check our transaction is valid and we want to process it Result error=checkTransaction(sd); @@ -153,9 +161,6 @@ protected void processMessage(Message m) throws InterruptedException { m.returnResult(error.withSource(SourceCodes.PEER)); return; } - - // Persist the signed transaction. Might throw MissingDataException? - sd=Cells.persist(sd); // Put on Server's transaction queue. We are OK to block here LoadMonitor.down(); @@ -165,7 +170,7 @@ protected void processMessage(Message m) throws InterruptedException { this.clientTransactionCount++; registerInterest(sd.getHash(), m); - } catch (BadFormatException | IOException e) { + } catch (BadFormatException e) { log.warn("Unhandled exception in transaction handler",e); m.closeConnection(); } catch (MissingDataException e) { @@ -259,8 +264,8 @@ private void reportTransactions(Block block, BlockResult br, long blockNum) { Hash h = t.getHash(); Message m = interests.get(h); if (m != null) { - ACell id = m.getID(); - log.trace("Returning transaction result ID {}", id); + // ACell id = m.getID(); + // log.info("Returning transaction result ID {}", id); Result res = null; try { @@ -300,14 +305,16 @@ private void observeTransactionResponse(SignedData sd, Result r) { } /** - * Checks for pending transactions, and if found propose them as a new Block. + * Gets the next Blocks for publication, or null if nothing to publish + * Checks for pending transactions, and if found propose them as new Block(s). * * @return New signed Block, or null if nothing to publish yet */ - protected SignedData maybeGenerateBlock(Peer peer) { + protected SignedData[] maybeGenerateBlocks() { + Peer peer=server.getPeer(); long timestamp=Utils.getCurrentTimestamp(); - if (!readyToPublish(peer)) return null; + if (!peer.isReadyToPublish()) return null; long minBlockTime=getMinBlockTime(); @@ -317,37 +324,43 @@ protected SignedData maybeGenerateBlock(Peer peer) { maybeGetOwnTransactions(peer); // possibly have client transactions to publish - transactionQueue.drainTo(newTransactions,Constants.MAX_TRANSACTIONS_PER_BLOCK); + transactionQueue.drainTo(newTransactions); - int n = newTransactions.size(); - if (n == 0) return null; + if (newTransactions.isEmpty()) return null; - // TODO: smaller block if too many transactions? - Block block = Block.create(timestamp, (List>) newTransactions); + int ntrans=newTransactions.size(); + int bsize=Constants.MAX_TRANSACTIONS_PER_BLOCK; + int nblocks=((ntrans-1)/bsize)+1; + + @SuppressWarnings("unchecked") + SignedData[] signedBlocks=new SignedData[nblocks]; + + for (int i=0; i signedBlock=peer.getKeyPair().signData(block); + + try { + Cells.persist(signedBlock); + } catch (Exception e) { + log.warn("Exception preparing new block",e); + return null; + } + signedBlocks[i]=signedBlock; + } newTransactions.clear(); - lastBlockPublishedTime=Utils.getCurrentTimestamp(); - SignedData signedBlock=peer.getKeyPair().signData(block); - return signedBlock; - } - - /** - * Gets the next Block for publication, or null if not yet ready - * @return New Block, or null if not yet produced - */ - public SignedData maybeGetBlock() { - return maybeGenerateBlock(server.getPeer()); - } - - /** - * Checks if the Peer is ready to publish a Block - * @param peer Current Peer instance - * @return true if ready to publish, false otherwise - */ - private boolean readyToPublish(Peer peer) { - return true; + lastBlockPublishedTime=timestamp; + return signedBlocks; } Long minBlockTime=null; + + /** + * Get the minimum time between proposing blocks. Default 10ms. + * @return + */ private long getMinBlockTime() { if (minBlockTime==null) { HashMap config = server.getConfig(); @@ -388,22 +401,22 @@ void maybeGetOwnTransactions(Peer p) { // NOTE: beyond this point we only execute stuff when AUTO_MANAGE is set if (!Utils.bool(server.getConfig().get(Keywords.AUTO_MANAGE))) return; + + // No point publishing if low staked etc. + if (!p.isReadyToPublish()) return; State s=p.getConsensusState(); AccountKey peerKey=p.getPeerKey(); PeerStatus ps=s.getPeer(peerKey); if (ps==null) return; // No peer record in consensus state? - // No point setting this if low staked - if (ps.getPeerStake()[] rs = new Future[n]; for (int i = 0; i < n; i++) { Future f = convex.transact(Invoke.create(ADDRESS, 0, Constant.of(i))); rs[i] = f; } for (int i = 0; i < n; i++) { - Result r = rs[i].get(6000, TimeUnit.MILLISECONDS); - assertNull(r.getErrorCode(), ()->"Error:" + r.toString()); + Result r = rs[i].get(10000, TimeUnit.MILLISECONDS); + final int ri=i; + assertNull(r.getErrorCode(), ()->"Error on result: "+ri+" = " + r.toString()); } } } @@ -167,7 +168,7 @@ public void testManyTransactions() throws IOException, TimeoutException, Interru public void testReceivedCount() throws IOException, TimeoutException, InterruptedException, ResultException { synchronized (network.SERVER) { ConvexRemote convex = Convex.connect(network.SERVER.getHostAddress(), ADDRESS, KEYPAIR); - Connection conn=convex.connection; + AConnection conn=convex.connection; long seq=convex.getSequence(); assertEquals(1,conn.getReceivedCount()); diff --git a/convex-peer/src/test/java/convex/net/ConnectionTest.java b/convex-peer/src/test/java/convex/net/ConnectionTest.java index de12321a8..98e849dfa 100644 --- a/convex-peer/src/test/java/convex/net/ConnectionTest.java +++ b/convex-peer/src/test/java/convex/net/ConnectionTest.java @@ -9,9 +9,9 @@ import convex.core.data.prim.CVMLong; import convex.core.exceptions.BadFormatException; -import convex.core.store.Stores; import convex.core.util.Utils; import convex.net.impl.HandlerException; +import convex.net.impl.nio.Connection; /** * Tests for the low level Connection class @@ -23,7 +23,7 @@ public void testMessageFlood() throws IOException, BadFormatException, Interrupt final ArrayList received = new ArrayList<>(); MemoryByteChannel chan = MemoryByteChannel.create(100); - Connection conn=Connection.create(chan, null, Stores.current(), null); + Connection conn=Connection.create(chan, null, null); // create a custom PeerConnection and MessageReceiver for testing // null Queue OK, we aren't queueing with our custom receive action @@ -31,7 +31,7 @@ public void testMessageFlood() throws IOException, BadFormatException, Interrupt synchronized (received) { received.add(a); } - }, conn); + }, null); Thread receiveThread=new Thread(()-> { while (!Thread.currentThread().isInterrupted()) { diff --git a/convex-peer/src/test/java/convex/net/MessageTest.java b/convex-peer/src/test/java/convex/net/MessageTest.java index 7b55e1a0b..0eae6b0d6 100644 --- a/convex-peer/src/test/java/convex/net/MessageTest.java +++ b/convex-peer/src/test/java/convex/net/MessageTest.java @@ -1,51 +1,220 @@ package convex.net; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; +import convex.core.data.Format; + +import java.util.Arrays; +import java.util.Random; import org.junit.jupiter.api.*; +import convex.core.cpos.Order; + +import convex.core.crypto.AKeyPair; +import convex.core.cvm.Address; +import convex.core.cvm.Symbols; +import convex.core.cvm.transactions.ATransaction; +import convex.core.cvm.transactions.Invoke; import convex.core.data.ACell; import convex.core.data.AVector; import convex.core.data.Blob; import convex.core.data.Blobs; import convex.core.data.Cells; import convex.core.data.Hash; +import convex.core.data.SignedData; import convex.core.data.Vectors; import convex.core.data.prim.CVMLong; import convex.core.exceptions.BadFormatException; +import convex.core.lang.RT; import convex.core.store.Stores; +import convex.core.Result; +import convex.core.cpos.Belief; +import convex.core.cpos.Block; +import convex.core.cpos.CPoSConstants; public class MessageTest { - Hash BAD_HASH=Hash.EMPTY_HASH; + static Hash BAD_HASH=Hash.EMPTY_HASH; + static AKeyPair KP=AKeyPair.createSeeded(1567856); @Test public void testMissingResponse() throws BadFormatException, IOException { // non-embedded Blob which might be a missing branch Blob b=Blobs.createRandom(400); Message mq=Message.createDataResponse(CVMLong.ONE, b); - assertEquals(MessageType.DATA,mq.getType()); + assertEquals(MessageType.RESULT,mq.getType()); + doMessageTest(mq); Blob enc=mq.getMessageData(); Message mr=Message.create(enc); - AVector v=mr.getPayload(); + AVector v=mr.toResult().getValue(); - assertEquals(b,v.get(1)); + assertEquals(b,v.get(0)); Cells.persist(b); Message md=Message.createDataRequest(CVMLong.ONE, b.getHash()); + doMessageTest(md); Message mdr=md.makeDataResponse(Stores.current()); assertEquals(mq,mdr); + doMessageTest(mdr); + } + + @SuppressWarnings("unchecked") + @Test public void testBeliefMessage() throws BadFormatException { + long TS=120; + Block b=Block.of(TS); + + Belief belief=Belief.create(KP, Order.create(0,0,KP.signData(b))); + + Message m=Message.createBelief(belief); + + assertNull(m.getRequestID()); + assertEquals(MessageType.BELIEF,m.getType()); + + Blob enc=m.getMessageData(); + + ACell b2=Format.decodeMultiCell(enc); + assertEquals(belief.getHash(),b2.getHash()); + + Message m2=Message.create(enc); + assertNull(m.getResultID()); + + assertEquals(MessageType.BELIEF,m2.getType()); + assertEquals(belief,m2.getPayload()); + + + } + + @Test public void testBigMissingResponse() throws BadFormatException, IOException { + // non-embedded Blob which might be a missing branch + Blob b=Blobs.createRandom(400); + Hash[] hashes=new Hash[CPoSConstants.MISSING_LIMIT]; + Arrays.fill(hashes,b.getHash()); + + Message mr=Message.createDataRequest(b,hashes); + assertEquals(MessageType.DATA_REQUEST,mr.getType()); + doMessageTest(mr); + + Blob enc=mr.getMessageData(); + + Message mrs=Message.create(enc); + AVector v=mrs.getPayload(); + assertEquals(2+CPoSConstants.MISSING_LIMIT,v.count()); + assertEquals(MessageType.DATA_REQUEST,mrs.getType()); + assertEquals(MessageTag.DATA_REQUEST,v.get(0)); + doMessageTest(mrs); } + + @Test public void testLostMissingResponse() throws BadFormatException, IOException { Message md=Message.createDataRequest(CVMLong.ONE, BAD_HASH); - assertEquals(Vectors.of(1,BAD_HASH),md.getPayload()); + assertEquals(Vectors.of(MessageTag.DATA_REQUEST,1,BAD_HASH),md.getPayload()); Message mdr=md.makeDataResponse(Stores.current()); - assertEquals(Vectors.of(1,null),mdr.getPayload()); + assertEquals(Result.create(CVMLong.ONE, Vectors.of(Cells.NIL)),mdr.getPayload()); + } + + @Test + public void testTypes() throws BadFormatException { + MessageType[] types = MessageType.values(); + + for (MessageType t : types) { + assertSame(t, MessageType.decode(t.getMessageCode())); + } + } + + @Test public void testQuery() { + Message m=Message.createQuery(0, Symbols.STAR_BALANCE, Address.ZERO); + doMessageTest(m); + } + + @Test public void testTransact() { + ATransaction tx=Invoke.create(Address.create(134564), 124334, Symbols.STAR_BALANCE); + SignedData stx=KP.signData(tx); + Message m=Message.createTransaction(12, stx); + doMessageTest(m); + } + + + @Test + public void testBadCode() { + assertThrows(BadFormatException.class, () -> MessageType.decode(-1)); + } + + @Test + public void testDataMessages() throws BadFormatException, IOException { + Blob b=Blob.createRandom(new Random(1256785), 1000); + Cells.persist(b); + + Message m=Message.createDataRequest(CVMLong.ONE, b.getHash()); + Message r=Message.createDataResponse(CVMLong.ONE, b); + + assertEquals(r,m.makeDataResponse(Stores.current())); + + // Check Result + Result res=r.toResult(); + assertEquals(CVMLong.ONE,res.getID()); + + AVector v=res.getValue(); + assertEquals(b,v.get(0)); + doMessageTest(m); + doMessageTest(r); + } + + @Test + public void testStatusMessage() { + Message m=Message.createStatusRequest(2); + assertEquals(RT.cvm(2),m.getID()); + doMessageTest(m); + } + + /** + * Generic tests for any valid message + * @param m + */ + public void doMessageTest(Message m) { + MessageType type=m.getType(); + assertNotNull(type); + + ACell id=m.getID(); + ACell reqID=m.getRequestID(); + ACell resultID=m.getResultID(); + if (reqID!=null) { + assertNull(resultID); // should be both a request and a result + assertEquals(id,reqID); + } + + if (resultID!=null) { + assertEquals(MessageType.RESULT,type); + assertNull(reqID); + assertEquals(id,resultID); + } + + try { + ACell payload=m.getPayload(); + + Blob data=m.getMessageData(); + assertTrue(data.count()>0); + + ACell dp=Format.decodeMultiCell(data); + Message m2=Message.create(null, dp); + + assertEquals(type,m2.getType()); + assertEquals(id,m2.getID()); + assertEquals(payload,m2.getPayload()); + + } catch (BadFormatException e) { + fail("Bad format: "+m,e); + } } } diff --git a/convex-peer/src/test/java/convex/net/impl/netty/NettyServerTest.java b/convex-peer/src/test/java/convex/net/impl/netty/NettyServerTest.java new file mode 100644 index 000000000..f0fd50378 --- /dev/null +++ b/convex-peer/src/test/java/convex/net/impl/netty/NettyServerTest.java @@ -0,0 +1,105 @@ +package convex.net.impl.netty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; + +import convex.api.Convex; +import convex.api.ConvexRemote; +import convex.core.Result; +import convex.core.cvm.Address; +import convex.core.cvm.Keywords; +import convex.core.data.AVector; +import convex.core.data.Blob; +import convex.core.data.Blobs; +import convex.core.data.Vectors; +import convex.core.exceptions.BadFormatException; +import convex.core.lang.RT; +import convex.net.Message; +import convex.net.MessageTag; + +@TestInstance(Lifecycle.PER_CLASS) +public class NettyServerTest { + + @Test public void testServerSetup() throws IOException, InterruptedException, TimeoutException { + try (NettyServer server = new NettyServer(0)) { + server.launch(); + Integer port=server.getPort(); + + assertNotNull(port); + InetSocketAddress addr=server.getHostAddress(); + assertEquals(port,addr.getPort()); + + CompletableFuture rec=new CompletableFuture<>(); + NettyConnection client=NettyConnection.connect(addr, m->{ + rec.complete(m); + }); + + client.send(Message.createQuery(10, "*address*", Address.create(17))); + + Message m=rec.join(); + assertEquals(RT.cvm(10),m.getResultID()); + + { // Regular client + Convex convex=Convex.connect(addr); + Result r=convex.query(Keywords.FOO).join(); + assertFalse(r.isError()); + AVector v= RT.ensureVector(r.getValue()); + + AVector expected=Vectors.of(MessageTag.QUERY,0,Keywords.FOO,null); + assertEquals(expected,v); + } + + { // Netty client + Convex convex=ConvexRemote.connectNetty(addr); + Result r=convex.query(":hello").join(); + assertFalse(r.isError()); + } + } + } + + @Test public void testBigMessage() throws IOException, InterruptedException, TimeoutException { + try (NettyServer server = new NettyServer(0)) { + server.launch(); + server.setReceiveAction(m->{ + try { + Result r=Result.create(m.getRequestID(), m.getPayload(), null); + m.returnResult(r); + } catch (BadFormatException e) { + m.returnResult(Result.BAD_FORMAT); + } + }); + Integer port=server.getPort(); + + assertNotNull(port); + InetSocketAddress socketAddr=server.getHostAddress(); + assertEquals(port,socketAddr.getPort()); + + ArrayBlockingQueue queue=new ArrayBlockingQueue<>(100); + + NettyConnection client=NettyConnection.connect(socketAddr, m->{ + queue.add(m); + }); + + Blob blob=Blobs.createRandom(100000).toFlatBlob(); + + Message mq=Message.createQuery(10, blob, Address.create(17)); + client.send(mq); + + Message m=queue.take(); + assertEquals(RT.cvm(10),m.getResultID()); + + + } + } +} diff --git a/convex-peer/src/test/java/convex/net/impl/nio/NIOServerTest.java b/convex-peer/src/test/java/convex/net/impl/nio/NIOServerTest.java new file mode 100644 index 000000000..2e29bd088 --- /dev/null +++ b/convex-peer/src/test/java/convex/net/impl/nio/NIOServerTest.java @@ -0,0 +1,53 @@ +package convex.net.impl.nio; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; + +import org.junit.jupiter.api.Test; + +import convex.api.Convex; +import convex.api.ConvexRemote; +import convex.core.Result; +import convex.core.cpos.Belief; +import convex.core.cvm.Keywords; +import convex.core.store.Stores; +import convex.net.Message; + +public class NIOServerTest { + + @Test public void testNIOServer() throws IOException, TimeoutException, InterruptedException { + ArrayList recd=new ArrayList(); + Consumer rec=m->{ + recd.add(m); + }; + + try (NIOServer s = new NIOServer(Stores.current(),rec)) { + s.launch(); + InetSocketAddress sa=s.getHostAddress(); + + Convex c=ConvexRemote.connectNIO(sa); + assertTrue(c.isConnected()); + + c.query(Keywords.FOO); + + CompletableFuture bm=c.message(Message.createBelief(Belief.initial())); + assertFalse(bm.join().isError()); + + c.query(Keywords.BAR); + + int EXP=3; + while(recd.size() received = new ArrayList<>(); - - MemoryByteChannel chan = MemoryByteChannel.create(10000); - Connection pc = Connection.create(chan, null, Stores.current(), null); - - // create a custom PeerConnection and MessageReceiver for testing - // null Queue OK, we aren't queueing with our custom receive action - MessageReceiver mr = new MessageReceiver(a -> received.add(a), pc); - - ACell msg1 = RT.cvm("Hello World!"); - assertTrue(pc.sendMessage(Message.createDataResponse(CVMLong.ZERO,msg1))); - ACell msg2 = RT.cvm(13L); - assertTrue(pc.sendMessage(Message.createDataResponse(CVMLong.ZERO,msg2))); - - // need to call sendBytes to flush send buffer to channel - // since we aren't using a Selector / SocketChannel here - assertTrue(pc.flushBytes()); - - // receive messages - mr.receiveFromChannel(chan); - assertEquals(2, received.size()); - assertEquals(Vectors.of(0,msg1), received.get(0).getPayload()); - assertEquals(Vectors.of(0,msg2), received.get(1).getPayload()); - - Message m1 = received.get(0); - assertEquals(MessageType.DATA, m1.getType()); - } - - @Test - public void testBigMessage() throws IOException, BadFormatException, HandlerException { - final ArrayList received = new ArrayList<>(); - - MemoryByteChannel chan = MemoryByteChannel.create(1000); - Connection pc = Connection.create(chan, null, Stores.current(), null); - - // create a custom PeerConnection and MessageReceiver for testing - // null Queue OK, we aren't queueing with our custom receive action - MessageReceiver mr = new MessageReceiver(a -> received.add(a), pc); - - ABlob blob = Blobs.createRandom(new Random(), 100000).getCanonical(); - Result r=Result.create(CVMLong.ONE, blob); - Message msg=Message.createResult(r); - pc.sendMessage(msg); - - // receive message - while (!pc.flushBytes()) { - mr.receiveFromChannel(chan); - } - mr.receiveFromChannel(chan); // complete receiving if needed - - assertEquals(1,received.size()); - - Message rec=received.get(0); - assertEquals(MessageType.RESULT, rec.getType()); - - ACell r2=rec.getPayload(); - assertEquals(r,r2); - } -} diff --git a/convex-peer/src/test/java/convex/peer/MessageTest.java b/convex-peer/src/test/java/convex/peer/MessageTest.java deleted file mode 100644 index 382ede819..000000000 --- a/convex-peer/src/test/java/convex/peer/MessageTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package convex.peer; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.io.IOException; -import java.util.Random; - -import org.junit.jupiter.api.Test; - -import convex.core.data.ACell; -import convex.core.data.AVector; -import convex.core.data.Blob; -import convex.core.data.Cells; -import convex.core.data.prim.CVMLong; -import convex.core.exceptions.BadFormatException; -import convex.core.store.Stores; -import convex.net.Message; -import convex.net.MessageType; - -public class MessageTest { - - @Test - public void testTypes() throws BadFormatException { - MessageType[] types = MessageType.values(); - - for (MessageType t : types) { - assertSame(t, MessageType.decode(t.getMessageCode())); - } - } - - @Test - public void testBadCode() { - assertThrows(BadFormatException.class, () -> MessageType.decode(-1)); - } - - @Test - public void testDataMessages() throws BadFormatException, IOException { - Blob b=Blob.createRandom(new Random(1256785), 1000); - Cells.persist(b); - - Message m=Message.createDataRequest(CVMLong.ONE, b.getHash()); - Message r=Message.createDataResponse(CVMLong.ONE, b); - - assertEquals(r,m.makeDataResponse(Stores.current())); - - AVector v=r.getPayload(); - assertEquals(CVMLong.ONE,v.get(0)); - assertEquals(b,v.get(1)); - } -} diff --git a/convex-peer/src/test/java/convex/peer/ServerTest.java b/convex-peer/src/test/java/convex/peer/ServerTest.java index 1462a128d..4acb4deda 100644 --- a/convex-peer/src/test/java/convex/peer/ServerTest.java +++ b/convex-peer/src/test/java/convex/peer/ServerTest.java @@ -34,7 +34,6 @@ import convex.core.data.Maps; import convex.core.data.Ref; import convex.core.data.Refs; -import convex.core.data.SignedData; import convex.core.data.prim.CVMLong; import convex.core.exceptions.BadSignatureException; import convex.core.exceptions.ResultException; @@ -43,7 +42,6 @@ import convex.core.lang.Reader; import convex.core.store.AStore; import convex.core.store.Stores; -import convex.net.Connection; /** * Tests for a fresh standalone server cluster instance @@ -82,7 +80,6 @@ public void testLocalConnect() throws Exception { r=convex.transactSync("(do (transfer "+user+" 100000) *balance*)"); assertEquals("10000000",r.getValue().toString()); - } @Test @@ -104,7 +101,7 @@ public void testServerFlood() throws IOException, InterruptedException, TimeoutE } @Test - public void testBalanceQuery() throws IOException, TimeoutException, ResultException { + public void testBalanceQuery() throws IOException, TimeoutException, ResultException, InterruptedException { Convex convex=Convex.connect(network.SERVER.getHostAddress(),network.VILLAIN,network.VILLAIN_KEYPAIR); // test the connection is still working @@ -131,10 +128,13 @@ public void testConvexAPI() throws IOException, InterruptedException, ExecutionE Convex convex=network.getClient(); Future f=convex.query(Symbols.STAR_BALANCE); + assertTrue(f.get().getValue() instanceof CVMLong); + convex.core.Result f2=convex.querySync(Symbols.STAR_ADDRESS); + assertFalse(f2.isError(),()->"Bad query result: "+f2); assertEquals(convex.getAddress(),f2.getValue()); - assertTrue(f.get().getValue() instanceof CVMLong); + // Note difference by argument type. `nil` code can make a valid transaction assertThrows(IllegalArgumentException.class,()->convex.transact((ATransaction)null)); @@ -233,11 +233,4 @@ public void testAcquireState() throws IOException, InterruptedException, Executi assertTrue(s instanceof State); } } - - public long checkSent(Connection pc,SignedData st) throws IOException { - long x=pc.sendTransaction(st); - assertTrue(x>=0); - return x; - } - }