Skip to content

Commit 57dc14c

Browse files
authored
Fix issue with Jackson 2.15: Can not write a field name, expecting a value (#15564)
This commit fixes the issue with Jackson > 2.15 and `log.format=json`: "Can not write a field name, expecting a value", by adding a default serializers for JRuby's objects.
1 parent db8f87b commit 57dc14c

File tree

6 files changed

+475
-22
lines changed

6 files changed

+475
-22
lines changed

logstash-core/src/main/java/org/logstash/ObjectMappers.java

+15-3
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,12 @@
3737
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
3838
import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
3939
import com.fasterxml.jackson.dataformat.cbor.CBORGenerator;
40-
import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator;
40+
4141
import java.io.IOException;
4242
import java.math.BigDecimal;
4343
import java.math.BigInteger;
4444
import java.util.HashMap;
45+
import org.apache.logging.log4j.core.jackson.Log4jJsonObjectMapper;
4546
import org.jruby.RubyBignum;
4647
import org.jruby.RubyBoolean;
4748
import org.jruby.RubyFixnum;
@@ -51,11 +52,14 @@
5152
import org.jruby.RubySymbol;
5253
import org.jruby.ext.bigdecimal.RubyBigDecimal;
5354
import org.logstash.ext.JrubyTimestampExtLibrary;
55+
import org.logstash.log.RubyBasicObjectSerializer;
5456

5557
public final class ObjectMappers {
5658

59+
static final String RUBY_SERIALIZERS_MODULE_ID = "RubySerializers";
60+
5761
private static final SimpleModule RUBY_SERIALIZERS =
58-
new SimpleModule("RubySerializers")
62+
new SimpleModule(RUBY_SERIALIZERS_MODULE_ID)
5963
.addSerializer(RubyString.class, new RubyStringSerializer())
6064
.addSerializer(RubySymbol.class, new RubySymbolSerializer())
6165
.addSerializer(RubyFloat.class, new RubyFloatSerializer())
@@ -71,7 +75,15 @@ public final class ObjectMappers {
7175
.addDeserializer(RubyNil.class, new RubyNilDeserializer());
7276

7377
public static final ObjectMapper JSON_MAPPER =
74-
new ObjectMapper().registerModule(RUBY_SERIALIZERS);
78+
new ObjectMapper().registerModule(RUBY_SERIALIZERS);
79+
80+
static String RUBY_BASIC_OBJECT_SERIALIZERS_MODULE_ID = "RubyBasicObjectSerializers";
81+
82+
// The RubyBasicObjectSerializer must be registered first, so it has a lower priority
83+
// over other more specific serializers.
84+
public static final ObjectMapper LOG4J_JSON_MAPPER = new Log4jJsonObjectMapper()
85+
.registerModule(new SimpleModule(RUBY_BASIC_OBJECT_SERIALIZERS_MODULE_ID).addSerializer(new RubyBasicObjectSerializer()))
86+
.registerModule(RUBY_SERIALIZERS);
7587

7688
/* TODO use this validator instead of LaissezFaireSubTypeValidator
7789
public static final PolymorphicTypeValidator TYPE_VALIDATOR = BasicPolymorphicTypeValidator.builder()

logstash-core/src/main/java/org/logstash/log/CustomLogEventSerializer.java

+53-16
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,26 @@
2121
package org.logstash.log;
2222

2323
import com.fasterxml.jackson.core.JsonGenerator;
24+
import com.fasterxml.jackson.core.io.SegmentedStringWriter;
25+
import com.fasterxml.jackson.core.util.BufferRecycler;
2426
import com.fasterxml.jackson.databind.JsonMappingException;
2527
import com.fasterxml.jackson.databind.JsonSerializer;
2628
import com.fasterxml.jackson.databind.SerializerProvider;
27-
29+
import com.google.common.primitives.Primitives;
30+
import org.apache.logging.log4j.LogManager;
31+
import org.apache.logging.log4j.Logger;
2832
import java.io.IOException;
2933
import java.util.Map;
3034

35+
import static org.logstash.ObjectMappers.LOG4J_JSON_MAPPER;
36+
3137
/**
3238
* Json serializer for logging messages, use in json appender.
33-
* */
39+
*/
3440
public class CustomLogEventSerializer extends JsonSerializer<CustomLogEvent> {
41+
42+
private static final Logger LOGGER = LogManager.getLogger(CustomLogEventSerializer.class);
43+
3544
@Override
3645
public void serialize(CustomLogEvent event, JsonGenerator generator, SerializerProvider provider) throws IOException {
3746
generator.writeStartObject();
@@ -41,20 +50,9 @@ public void serialize(CustomLogEvent event, JsonGenerator generator, SerializerP
4150
generator.writeObjectField("thread", event.getThreadName());
4251
generator.writeFieldName("logEvent");
4352
generator.writeStartObject();
44-
if (event.getMessage() instanceof StructuredMessage) {
45-
StructuredMessage message = (StructuredMessage) event.getMessage();
46-
generator.writeStringField("message", message.getMessage());
47-
if (message.getParams() != null) {
48-
for (Map.Entry<Object, Object> entry : message.getParams().entrySet()) {
49-
Object value = entry.getValue();
50-
try {
51-
generator.writeObjectField(entry.getKey().toString(), value);
52-
} catch (JsonMappingException e) {
53-
generator.writeObjectField(entry.getKey().toString(), value.toString());
54-
}
55-
}
56-
}
5753

54+
if (event.getMessage() instanceof StructuredMessage) {
55+
writeStructuredMessage((StructuredMessage) event.getMessage(), generator);
5856
} else {
5957
generator.writeStringField("message", event.getMessage().getFormattedMessage());
6058
}
@@ -63,9 +61,48 @@ public void serialize(CustomLogEvent event, JsonGenerator generator, SerializerP
6361
generator.writeEndObject();
6462
}
6563

64+
private void writeStructuredMessage(StructuredMessage message, JsonGenerator generator) throws IOException {
65+
generator.writeStringField("message", message.getMessage());
66+
67+
if (message.getParams() == null || message.getParams().isEmpty()) {
68+
return;
69+
}
70+
71+
for (final Map.Entry<Object, Object> entry : message.getParams().entrySet()) {
72+
final String paramName = entry.getKey().toString();
73+
final Object paramValue = entry.getValue();
74+
75+
try {
76+
if (isValueSafeToWrite(paramValue)) {
77+
generator.writeObjectField(paramName, paramValue);
78+
continue;
79+
}
80+
81+
// Create a new Jackson's generator for each entry, that way, the main generator is not compromised/invalidated
82+
// in case any key/value fails to write. It also uses the JSON_LOGGER_MAPPER instead of the default Log4's one,
83+
// leveraging all necessary custom Ruby serializers.
84+
try (final SegmentedStringWriter entryJsonWriter = new SegmentedStringWriter(new BufferRecycler());
85+
final JsonGenerator entryGenerator = LOG4J_JSON_MAPPER.getFactory().createGenerator(entryJsonWriter)) {
86+
entryGenerator.writeObject(paramValue);
87+
generator.writeFieldName(paramName);
88+
generator.writeRawValue(entryJsonWriter.getAndClear());
89+
}
90+
} catch (JsonMappingException e) {
91+
LOGGER.debug("Failed to serialize message param type {}", paramValue.getClass(), e);
92+
generator.writeObjectField(paramName, paramValue.toString());
93+
}
94+
}
95+
}
96+
97+
private boolean isValueSafeToWrite(Object value) {
98+
return value == null ||
99+
value instanceof String ||
100+
value.getClass().isPrimitive() ||
101+
Primitives.isWrapperType(value.getClass());
102+
}
103+
66104
@Override
67105
public Class<CustomLogEvent> handledType() {
68-
69106
return CustomLogEvent.class;
70107
}
71108
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* Licensed to Elasticsearch B.V. under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch B.V. licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.logstash.log;
21+
22+
import com.fasterxml.jackson.core.JsonGenerator;
23+
import com.fasterxml.jackson.databind.JsonMappingException;
24+
import com.fasterxml.jackson.databind.JsonSerializer;
25+
import com.fasterxml.jackson.databind.SerializerProvider;
26+
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
27+
import com.fasterxml.jackson.databind.util.ClassUtil;
28+
import org.apache.logging.log4j.LogManager;
29+
import org.apache.logging.log4j.Logger;
30+
import org.jruby.RubyBasicObject;
31+
import org.jruby.RubyMethod;
32+
import org.jruby.RubyString;
33+
import org.jruby.exceptions.NameError;
34+
import org.logstash.RubyUtil;
35+
36+
import java.io.IOException;
37+
import java.util.Optional;
38+
39+
/**
40+
* Default serializer for {@link org.jruby.RubyBasicObject} since Jackson can't handle that type natively.
41+
* Arrays, Collections and Maps are delegated to the default Jackson's serializer - which might end-up invoking this
42+
* serializer for its elements.
43+
* Values which inspect method is implemented and owned by the LogStash module will be serialized using this method return.
44+
* If none of the above conditions match, it gets the serialized value by invoking the Ruby's {@code to_s} method, falling back
45+
* to {@link RubyBasicObject#to_s()} and {@link RubyBasicObject#anyToString()} in case of errors.
46+
*/
47+
public final class RubyBasicObjectSerializer extends StdSerializer<RubyBasicObject> {
48+
49+
private static final long serialVersionUID = -5557562960691452054L;
50+
private static final Logger LOGGER = LogManager.getLogger(RubyBasicObjectSerializer.class);
51+
private static final String METHOD_INSPECT = "inspect";
52+
private static final String METHOD_TO_STRING = "to_s";
53+
54+
public RubyBasicObjectSerializer() {
55+
super(RubyBasicObject.class);
56+
}
57+
58+
@Override
59+
public void serialize(final RubyBasicObject value, final JsonGenerator gen, final SerializerProvider provider) throws IOException {
60+
final Optional<JsonSerializer<Object>> serializer = findTypeSerializer(value, provider);
61+
if (serializer.isPresent()) {
62+
try {
63+
serializer.get().serialize(value, gen, provider);
64+
return;
65+
} catch (IOException e) {
66+
LOGGER.debug("Failed to serialize value type {} using default serializer {}", value.getClass(), serializer.get().getClass(), e);
67+
}
68+
}
69+
70+
if (isCustomInspectMethodDefined(value)) {
71+
try {
72+
gen.writeString(value.callMethod(METHOD_INSPECT).asJavaString());
73+
return;
74+
} catch (Exception e) {
75+
LOGGER.debug("Failed to serialize value type {} using the custom `inspect` method", value.getMetaClass(), e);
76+
}
77+
}
78+
79+
try {
80+
gen.writeString(value.callMethod(METHOD_TO_STRING).asJavaString());
81+
return;
82+
} catch (Exception e) {
83+
LOGGER.debug("Failed to serialize value type {} using `to_s` method", value.getMetaClass(), e);
84+
}
85+
86+
try {
87+
gen.writeString(value.to_s().asJavaString());
88+
} catch (Exception e) {
89+
LOGGER.debug("Failed to serialize value type {} using `RubyBasicObject#to_s()` method", value.getMetaClass(), e);
90+
gen.writeString(value.anyToString().asJavaString());
91+
}
92+
}
93+
94+
private Optional<JsonSerializer<Object>> findTypeSerializer(final RubyBasicObject value, final SerializerProvider provider) {
95+
if (ClassUtil.isCollectionMapOrArray(value.getClass())) {
96+
try {
97+
// Delegates the serialization to the Jackson's default serializers, which might
98+
// end up using this serializer for its elements.
99+
return Optional.ofNullable(provider.findTypedValueSerializer(value.getJavaClass(), false, null));
100+
} catch (JsonMappingException e) {
101+
// Ignored
102+
}
103+
}
104+
105+
return Optional.empty();
106+
}
107+
108+
private boolean isCustomInspectMethodDefined(final RubyBasicObject value) {
109+
try {
110+
final Object candidate = value.method(RubyString.newString(RubyUtil.RUBY, METHOD_INSPECT));
111+
return candidate instanceof RubyMethod && ((RubyMethod) candidate).owner(RubyUtil.RUBY.getCurrentContext()).toString().toLowerCase().startsWith("logstash");
112+
} catch (NameError e) {
113+
return false;
114+
}
115+
}
116+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package org.logstash;
2+
3+
import com.fasterxml.jackson.databind.JavaType;
4+
import com.fasterxml.jackson.databind.JsonSerializer;
5+
import com.fasterxml.jackson.databind.ObjectMapper;
6+
import com.fasterxml.jackson.databind.ser.BeanSerializerFactory;
7+
import com.fasterxml.jackson.databind.ser.Serializers;
8+
import com.fasterxml.jackson.databind.type.TypeFactory;
9+
import org.jruby.RubyBasicObject;
10+
import org.junit.Test;
11+
import org.logstash.log.RubyBasicObjectSerializer;
12+
13+
import java.util.LinkedList;
14+
15+
import static org.junit.Assert.assertNotNull;
16+
import static org.junit.Assert.assertTrue;
17+
import static org.logstash.ObjectMappers.RUBY_BASIC_OBJECT_SERIALIZERS_MODULE_ID;
18+
import static org.logstash.ObjectMappers.RUBY_SERIALIZERS_MODULE_ID;
19+
20+
public class ObjectMappersTest {
21+
22+
@Test
23+
public void testLog4jOMRegisterRubySerializersModule() {
24+
assertTrue(ObjectMappers.LOG4J_JSON_MAPPER.getRegisteredModuleIds().contains(RUBY_SERIALIZERS_MODULE_ID));
25+
}
26+
27+
@Test
28+
public void testLog4jOMRegisterRubyBasicObjectSerializersModule() {
29+
assertTrue(ObjectMappers.LOG4J_JSON_MAPPER.getRegisteredModuleIds().contains(RUBY_BASIC_OBJECT_SERIALIZERS_MODULE_ID));
30+
}
31+
32+
@Test
33+
public void testLog4jOMRegisterRubyBasicObjectSerializersFirst() {
34+
final ObjectMapper mapper = ObjectMappers.LOG4J_JSON_MAPPER;
35+
final BeanSerializerFactory factory = (BeanSerializerFactory) mapper.getSerializerFactory();
36+
37+
final LinkedList<Serializers> list = new LinkedList<>();
38+
for (Serializers serializer : factory.getFactoryConfig().serializers()) {
39+
list.add(serializer);
40+
}
41+
42+
// RubyBasicObjectSerializer + Log4jJsonModule
43+
assertTrue(list.size() > 1);
44+
45+
final Serializers rubyBasicObjectSerializer = list.get(list.size() - 2);
46+
final JavaType valueType = TypeFactory.defaultInstance().constructType(RubyBasicObject.class);
47+
final JsonSerializer<?> found = rubyBasicObjectSerializer.findSerializer(mapper.getSerializationConfig(), valueType, null);
48+
49+
assertNotNull(found);
50+
assertTrue("RubyBasicObjectSerializer must be registered before others non-default serializers", found instanceof RubyBasicObjectSerializer);
51+
}
52+
}

0 commit comments

Comments
 (0)