Skip to content

add NonBlockingByteBufferJsonParser (and refactor some NonBlockingJsonParser code to make it more extensible) #795

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 24 commits into from
Jul 25, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
a2049e9
refactor some code to make it feasible to have a separate ByteBuffer …
pjfanning Jul 12, 2022
97909d7
Merge remote-tracking branch 'upstream/2.14' into byte-buffer
pjfanning Jul 12, 2022
314f5ec
Merge branch '2.14' into byte-buffer
pjfanning Jul 13, 2022
b841dba
getNextSignedByteFromBuffer()
pjfanning Jul 13, 2022
31cb693
Update NonBlockingJsonParser.java
pjfanning Jul 13, 2022
0085974
Create NonBlockingUtf8JsonParserBase.java
pjfanning Jul 13, 2022
b01df73
more refactor
pjfanning Jul 13, 2022
a39015a
more refactor
pjfanning Jul 13, 2022
ad82e9d
more refactor
pjfanning Jul 13, 2022
9c219ed
create ByteBuffer parser
pjfanning Jul 13, 2022
10cc16c
Update NonBlockingJsonParser.java
pjfanning Jul 13, 2022
422afce
add byte buffer method to factory
pjfanning Jul 13, 2022
5c016eb
Merge branch 'byte-buffer' into byte-buffer2
pjfanning Jul 13, 2022
466abd6
fix issue
pjfanning Jul 13, 2022
fd5dfd5
Merge branch 'byte-buffer' into byte-buffer2
pjfanning Jul 13, 2022
135c3ca
Update AsyncLocationTest.java
pjfanning Jul 14, 2022
e843ae1
Update AsyncLocationTest.java
pjfanning Jul 14, 2022
76c2513
Merge branch 'byte-buffer' into byte-buffer2
pjfanning Jul 15, 2022
fd7b7ed
wip
pjfanning Jul 16, 2022
ba9f819
add byte buffer test
pjfanning Jul 16, 2022
f8b623a
Update TokenStreamFactory.java
pjfanning Jul 16, 2022
8927be7
unused imports
pjfanning Jul 16, 2022
4335783
Merge remote-tracking branch 'upstream/2.14' into byte-buffer
pjfanning Jul 25, 2022
4fee704
Update NonBlockingUtf8JsonParserBase.java
pjfanning Jul 25, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions src/main/java/com/fasterxml/jackson/core/JsonFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import com.fasterxml.jackson.core.format.MatchStrength;
import com.fasterxml.jackson.core.io.*;
import com.fasterxml.jackson.core.json.*;
import com.fasterxml.jackson.core.json.async.NonBlockingByteBufferJsonParser;
import com.fasterxml.jackson.core.json.async.NonBlockingJsonParser;
import com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer;
import com.fasterxml.jackson.core.sym.CharsToNameCanonicalizer;
Expand Down Expand Up @@ -1241,6 +1242,33 @@ public JsonParser createNonBlockingByteArrayParser() throws IOException
return new NonBlockingJsonParser(ctxt, _parserFeatures, can);
}

/**
* Optional method for constructing parser for non-blocking parsing
* via {@link com.fasterxml.jackson.core.async.ByteBufferFeeder}
* interface (accessed using {@link JsonParser#getNonBlockingInputFeeder()}
* from constructed instance).
*<p>
* If this factory does not support non-blocking parsing (either at all,
* or from byte array),
* will throw {@link UnsupportedOperationException}.
*<p>
* Note that JSON-backed factory only supports parsing of UTF-8 encoded JSON content
* (and US-ASCII since it is proper subset); other encodings are not supported
* at this point.
*
* @since 2.14
*/
@Override
public JsonParser createNonBlockingByteBufferParser() throws IOException
{
// 17-May-2017, tatu: Need to take care not to accidentally create JSON parser
// for non-JSON input:
_requireJSONFactory("Non-blocking source not (yet?) supported for this format (%s)");
IOContext ctxt = _createNonBlockingContext(null);
ByteQuadsCanonicalizer can = _byteSymbolCanonicalizer.makeChild(_factoryFeatures);
return new NonBlockingByteBufferJsonParser(ctxt, _parserFeatures, can);
}

/*
/**********************************************************
/* Generator factories
Expand Down
34 changes: 34 additions & 0 deletions src/main/java/com/fasterxml/jackson/core/TokenStreamFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,42 @@ public abstract class TokenStreamFactory
public abstract JsonParser createParser(String content) throws IOException;
public abstract JsonParser createParser(URL url) throws IOException;

/**
* Optional method for constructing parser for non-blocking parsing
* via {@link com.fasterxml.jackson.core.async.ByteArrayFeeder}
* interface (accessed using {@link JsonParser#getNonBlockingInputFeeder()}
* from constructed instance).
*<p>
* If this factory does not support non-blocking parsing (either at all,
* or from byte array),
* will throw {@link UnsupportedOperationException}.
*<p>
* Note that JSON-backed factory only supports parsing of UTF-8 encoded JSON content
* (and US-ASCII since it is proper subset); other encodings are not supported
* at this point.
*
* @since 2.9
*/
public abstract JsonParser createNonBlockingByteArrayParser() throws IOException;

/**
* Optional method for constructing parser for non-blocking parsing
* via {@link com.fasterxml.jackson.core.async.ByteBufferFeeder}
* interface (accessed using {@link JsonParser#getNonBlockingInputFeeder()}
* from constructed instance).
*<p>
* If this factory does not support non-blocking parsing (either at all,
* or from byte array),
* will throw {@link UnsupportedOperationException}.
*<p>
* Note that JSON-backed factory only supports parsing of UTF-8 encoded JSON content
* (and US-ASCII since it is proper subset); other encodings are not supported
* at this point.
*
* @since 2.14
*/
public abstract JsonParser createNonBlockingByteBufferParser() throws IOException;

/*
/**********************************************************************
/* Factory methods, generators
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package com.fasterxml.jackson.core.json.async;

import com.fasterxml.jackson.core.async.ByteBufferFeeder;
import com.fasterxml.jackson.core.async.NonBlockingInputFeeder;
import com.fasterxml.jackson.core.io.IOContext;
import com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer;

import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;

/**
* Non-blocking parser implementation for JSON content.
*<p>
* NOTE: only supports parsing of UTF-8 encoded content (and 7-bit US-ASCII since
* it is strict subset of UTF-8): other encodings are not supported.
*/
public class NonBlockingByteBufferJsonParser
extends NonBlockingUtf8JsonParserBase
implements ByteBufferFeeder {

private ByteBuffer _inputBuffer = ByteBuffer.wrap(NO_BYTES);

public NonBlockingByteBufferJsonParser(IOContext ctxt, int parserFeatures,
ByteQuadsCanonicalizer sym) {
super(ctxt, parserFeatures, sym);
}

@Override
public NonBlockingInputFeeder getNonBlockingInputFeeder() {
return this;
}

@Override
public void feedInput(final ByteBuffer byteBuffer) throws IOException {
// Must not have remaining input
if (_inputPtr < _inputEnd) {
_reportError("Still have %d undecoded bytes, should not call 'feedInput'", _inputEnd - _inputPtr);
}

final int start = byteBuffer.position();
final int end = byteBuffer.limit();

if (end < start) {
_reportError("Input end (%d) may not be before start (%d)", end, start);
}
// and shouldn't have been marked as end-of-input
if (_endOfInput) {
_reportError("Already closed, can not feed more input");
}
// Time to update pointers first
_currInputProcessed += _origBufferLen;

// Also need to adjust row start, to work as if it extended into the past wrt new buffer
_currInputRowStart = start - (_inputEnd - _currInputRowStart);

// And then update buffer settings
_currBufferStart = start;
_inputBuffer = byteBuffer;
_inputPtr = start;
_inputEnd = end;
_origBufferLen = end - start;
}

@Override
public int releaseBuffered(final OutputStream out) throws IOException {
final int avail = _inputEnd - _inputPtr;
if (avail > 0) {
final WritableByteChannel channel = Channels.newChannel(out);
channel.write(_inputBuffer);
}
return avail;
}

@Override
protected byte getNextSignedByteFromBuffer() {
return _inputBuffer.get(_inputPtr++);
}

@Override
protected int getNextUnsignedByteFromBuffer() {
return _inputBuffer.get(_inputPtr++) & 0xFF;
}

@Override
protected byte getByteFromBuffer(final int ptr) {
return _inputBuffer.get(ptr);
}
}
Loading