[BACKPORT] Calculate the initial frame size of the client messages correctly (#499)

mdumandag · web-flow · commit f1db480a5d2d · 2021-10-14T13:03:10.000+03:00
* Calculate the initial frame size of the client messages correctly

While calculating the initial frame size of the client messages,
we were adding `SIZE_OF_FRAME_LENGTH_AND_FLAGS` twice. That extra
addition is removed from the `create_initial_buffer`.

To make the `create_initial_buffer_custom` alike, I also removed
the addition from there, although there was no such problem for that
method. To make it work again, I have modified the protocol template
to perform one less subtraction of `SIZE_OF_FRAME_LENGTH_AND_FLAGS`
from the initial frame size.

Also, while I was there, I have updated the `sql_execute_codec`, and
added support for the newly added parameter.

* add a unit test that verifies the encoded requests produce the same bytearray as the Java client

* bump client version

* add v5 to branch filter in test runner
diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml
@@ -3,11 +3,11 @@ on:
   push:
     branches:
       - master
-      - 4.*.z
+      - '[45].*.z'
   pull_request:
     branches:
       - master
-      - 4.*.z
+      - '[45].*.z'
 jobs:
   run-tests:
     runs-on: ${{ matrix.os }}
diff --git a/docs/conf.py b/docs/conf.py
@@ -72,9 +72,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = "5.0"
+version = "5.0.1"
 # The full version, including alpha/beta/rc tags.
-release = "5.0"
+release = "5.0.1"
 
 autodoc_member_order = "bysource"
 autoclass_content = "both"
diff --git a/hazelcast/__init__.py b/hazelcast/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "5.0"
+__version__ = "5.0.1"
 
 # Set the default handler to "hazelcast" loggers
 # to avoid "No handlers could be found" warnings.
diff --git a/hazelcast/protocol/client_message.py b/hazelcast/protocol/client_message.py
@@ -34,7 +34,6 @@
 
 # For codecs
 def create_initial_buffer(size, message_type, is_final=False):
-    size += SIZE_OF_FRAME_LENGTH_AND_FLAGS
     buf = bytearray(size)
     LE_INT.pack_into(buf, 0, size)
     flags = _UNFRAGMENTED_MESSAGE_FLAGS
@@ -48,7 +47,6 @@ def create_initial_buffer(size, message_type, is_final=False):
 
 # For custom codecs
 def create_initial_buffer_custom(size, is_begin_frame=False):
-    size += SIZE_OF_FRAME_LENGTH_AND_FLAGS
     if is_begin_frame:
         # Needed for custom codecs that does not have initial frame at first
         # but requires later due to new fix sized parameters
diff --git a/hazelcast/protocol/codec/custom/address_codec.py b/hazelcast/protocol/codec/custom/address_codec.py
@@ -6,7 +6,7 @@
 
 _PORT_ENCODE_OFFSET = 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
 _PORT_DECODE_OFFSET = 0
-_INITIAL_FRAME_SIZE = _PORT_ENCODE_OFFSET + INT_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _PORT_ENCODE_OFFSET + INT_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class AddressCodec(object):
diff --git a/hazelcast/protocol/codec/custom/bitmap_index_options_codec.py b/hazelcast/protocol/codec/custom/bitmap_index_options_codec.py
@@ -6,7 +6,7 @@
 
 _UNIQUE_KEY_TRANSFORMATION_ENCODE_OFFSET = 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
 _UNIQUE_KEY_TRANSFORMATION_DECODE_OFFSET = 0
-_INITIAL_FRAME_SIZE = _UNIQUE_KEY_TRANSFORMATION_ENCODE_OFFSET + INT_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _UNIQUE_KEY_TRANSFORMATION_ENCODE_OFFSET + INT_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class BitmapIndexOptionsCodec(object):
diff --git a/hazelcast/protocol/codec/custom/endpoint_qualifier_codec.py b/hazelcast/protocol/codec/custom/endpoint_qualifier_codec.py
@@ -6,7 +6,7 @@
 
 _TYPE_ENCODE_OFFSET = 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
 _TYPE_DECODE_OFFSET = 0
-_INITIAL_FRAME_SIZE = _TYPE_ENCODE_OFFSET + INT_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _TYPE_ENCODE_OFFSET + INT_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class EndpointQualifierCodec(object):
diff --git a/hazelcast/protocol/codec/custom/error_holder_codec.py b/hazelcast/protocol/codec/custom/error_holder_codec.py
@@ -8,7 +8,7 @@
 
 _ERROR_CODE_ENCODE_OFFSET = 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
 _ERROR_CODE_DECODE_OFFSET = 0
-_INITIAL_FRAME_SIZE = _ERROR_CODE_ENCODE_OFFSET + INT_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _ERROR_CODE_ENCODE_OFFSET + INT_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class ErrorHolderCodec(object):
diff --git a/hazelcast/protocol/codec/custom/index_config_codec.py b/hazelcast/protocol/codec/custom/index_config_codec.py
@@ -8,7 +8,7 @@
 
 _TYPE_ENCODE_OFFSET = 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
 _TYPE_DECODE_OFFSET = 0
-_INITIAL_FRAME_SIZE = _TYPE_ENCODE_OFFSET + INT_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _TYPE_ENCODE_OFFSET + INT_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class IndexConfigCodec(object):
diff --git a/hazelcast/protocol/codec/custom/member_info_codec.py b/hazelcast/protocol/codec/custom/member_info_codec.py
@@ -12,7 +12,7 @@
 _UUID_DECODE_OFFSET = 0
 _LITE_MEMBER_ENCODE_OFFSET = _UUID_ENCODE_OFFSET + UUID_SIZE_IN_BYTES
 _LITE_MEMBER_DECODE_OFFSET = _UUID_DECODE_OFFSET + UUID_SIZE_IN_BYTES
-_INITIAL_FRAME_SIZE = _LITE_MEMBER_ENCODE_OFFSET + BOOLEAN_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _LITE_MEMBER_ENCODE_OFFSET + BOOLEAN_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class MemberInfoCodec(object):
diff --git a/hazelcast/protocol/codec/custom/member_version_codec.py b/hazelcast/protocol/codec/custom/member_version_codec.py
@@ -9,7 +9,7 @@
 _MINOR_DECODE_OFFSET = _MAJOR_DECODE_OFFSET + BYTE_SIZE_IN_BYTES
 _PATCH_ENCODE_OFFSET = _MINOR_ENCODE_OFFSET + BYTE_SIZE_IN_BYTES
 _PATCH_DECODE_OFFSET = _MINOR_DECODE_OFFSET + BYTE_SIZE_IN_BYTES
-_INITIAL_FRAME_SIZE = _PATCH_ENCODE_OFFSET + BYTE_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _PATCH_ENCODE_OFFSET + BYTE_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class MemberVersionCodec(object):
diff --git a/hazelcast/protocol/codec/custom/paging_predicate_holder_codec.py b/hazelcast/protocol/codec/custom/paging_predicate_holder_codec.py
@@ -11,7 +11,7 @@
 _PAGE_DECODE_OFFSET = _PAGE_SIZE_DECODE_OFFSET + INT_SIZE_IN_BYTES
 _ITERATION_TYPE_ID_ENCODE_OFFSET = _PAGE_ENCODE_OFFSET + INT_SIZE_IN_BYTES
 _ITERATION_TYPE_ID_DECODE_OFFSET = _PAGE_DECODE_OFFSET + INT_SIZE_IN_BYTES
-_INITIAL_FRAME_SIZE = _ITERATION_TYPE_ID_ENCODE_OFFSET + BYTE_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _ITERATION_TYPE_ID_ENCODE_OFFSET + BYTE_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class PagingPredicateHolderCodec(object):
diff --git a/hazelcast/protocol/codec/custom/raft_group_id_codec.py b/hazelcast/protocol/codec/custom/raft_group_id_codec.py
@@ -8,7 +8,7 @@
 _SEED_DECODE_OFFSET = 0
 _ID_ENCODE_OFFSET = _SEED_ENCODE_OFFSET + LONG_SIZE_IN_BYTES
 _ID_DECODE_OFFSET = _SEED_DECODE_OFFSET + LONG_SIZE_IN_BYTES
-_INITIAL_FRAME_SIZE = _ID_ENCODE_OFFSET + LONG_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _ID_ENCODE_OFFSET + LONG_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class RaftGroupIdCodec(object):
diff --git a/hazelcast/protocol/codec/custom/simple_entry_view_codec.py b/hazelcast/protocol/codec/custom/simple_entry_view_codec.py
@@ -24,7 +24,7 @@
 _TTL_DECODE_OFFSET = _VERSION_DECODE_OFFSET + LONG_SIZE_IN_BYTES
 _MAX_IDLE_ENCODE_OFFSET = _TTL_ENCODE_OFFSET + LONG_SIZE_IN_BYTES
 _MAX_IDLE_DECODE_OFFSET = _TTL_DECODE_OFFSET + LONG_SIZE_IN_BYTES
-_INITIAL_FRAME_SIZE = _MAX_IDLE_ENCODE_OFFSET + LONG_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _MAX_IDLE_ENCODE_OFFSET + LONG_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class SimpleEntryViewCodec(object):
diff --git a/hazelcast/protocol/codec/custom/sql_column_metadata_codec.py b/hazelcast/protocol/codec/custom/sql_column_metadata_codec.py
@@ -8,7 +8,7 @@
 _TYPE_DECODE_OFFSET = 0
 _NULLABLE_ENCODE_OFFSET = _TYPE_ENCODE_OFFSET + INT_SIZE_IN_BYTES
 _NULLABLE_DECODE_OFFSET = _TYPE_DECODE_OFFSET + INT_SIZE_IN_BYTES
-_INITIAL_FRAME_SIZE = _NULLABLE_ENCODE_OFFSET + BOOLEAN_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _NULLABLE_ENCODE_OFFSET + BOOLEAN_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class SqlColumnMetadataCodec(object):
diff --git a/hazelcast/protocol/codec/custom/sql_error_codec.py b/hazelcast/protocol/codec/custom/sql_error_codec.py
@@ -8,7 +8,7 @@
 _CODE_DECODE_OFFSET = 0
 _ORIGINATING_MEMBER_ID_ENCODE_OFFSET = _CODE_ENCODE_OFFSET + INT_SIZE_IN_BYTES
 _ORIGINATING_MEMBER_ID_DECODE_OFFSET = _CODE_DECODE_OFFSET + INT_SIZE_IN_BYTES
-_INITIAL_FRAME_SIZE = _ORIGINATING_MEMBER_ID_ENCODE_OFFSET + UUID_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _ORIGINATING_MEMBER_ID_ENCODE_OFFSET + UUID_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class SqlErrorCodec(object):
diff --git a/hazelcast/protocol/codec/custom/sql_query_id_codec.py b/hazelcast/protocol/codec/custom/sql_query_id_codec.py
@@ -11,7 +11,7 @@
 _LOCAL_ID_HIGH_DECODE_OFFSET = _MEMBER_ID_LOW_DECODE_OFFSET + LONG_SIZE_IN_BYTES
 _LOCAL_ID_LOW_ENCODE_OFFSET = _LOCAL_ID_HIGH_ENCODE_OFFSET + LONG_SIZE_IN_BYTES
 _LOCAL_ID_LOW_DECODE_OFFSET = _LOCAL_ID_HIGH_DECODE_OFFSET + LONG_SIZE_IN_BYTES
-_INITIAL_FRAME_SIZE = _LOCAL_ID_LOW_ENCODE_OFFSET + LONG_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _LOCAL_ID_LOW_ENCODE_OFFSET + LONG_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class SqlQueryIdCodec(object):
diff --git a/hazelcast/protocol/codec/custom/stack_trace_element_codec.py b/hazelcast/protocol/codec/custom/stack_trace_element_codec.py
@@ -6,7 +6,7 @@
 
 _LINE_NUMBER_ENCODE_OFFSET = 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
 _LINE_NUMBER_DECODE_OFFSET = 0
-_INITIAL_FRAME_SIZE = _LINE_NUMBER_ENCODE_OFFSET + INT_SIZE_IN_BYTES - 2 * SIZE_OF_FRAME_LENGTH_AND_FLAGS
+_INITIAL_FRAME_SIZE = _LINE_NUMBER_ENCODE_OFFSET + INT_SIZE_IN_BYTES - SIZE_OF_FRAME_LENGTH_AND_FLAGS
 
 
 class StackTraceElementCodec(object):
diff --git a/hazelcast/protocol/codec/sql_execute_codec.py b/hazelcast/protocol/codec/sql_execute_codec.py
@@ -18,15 +18,17 @@
 _REQUEST_TIMEOUT_MILLIS_OFFSET = REQUEST_HEADER_SIZE
 _REQUEST_CURSOR_BUFFER_SIZE_OFFSET = _REQUEST_TIMEOUT_MILLIS_OFFSET + LONG_SIZE_IN_BYTES
 _REQUEST_EXPECTED_RESULT_TYPE_OFFSET = _REQUEST_CURSOR_BUFFER_SIZE_OFFSET + INT_SIZE_IN_BYTES
-_REQUEST_INITIAL_FRAME_SIZE = _REQUEST_EXPECTED_RESULT_TYPE_OFFSET + BYTE_SIZE_IN_BYTES
+_REQUEST_SKIP_UPDATE_STATISTICS_OFFSET = _REQUEST_EXPECTED_RESULT_TYPE_OFFSET + BYTE_SIZE_IN_BYTES
+_REQUEST_INITIAL_FRAME_SIZE = _REQUEST_SKIP_UPDATE_STATISTICS_OFFSET + BOOLEAN_SIZE_IN_BYTES
 _RESPONSE_UPDATE_COUNT_OFFSET = RESPONSE_HEADER_SIZE
 
 
-def encode_request(sql, parameters, timeout_millis, cursor_buffer_size, schema, expected_result_type, query_id):
+def encode_request(sql, parameters, timeout_millis, cursor_buffer_size, schema, expected_result_type, query_id, skip_update_statistics):
     buf = create_initial_buffer(_REQUEST_INITIAL_FRAME_SIZE, _REQUEST_MESSAGE_TYPE)
     FixSizedTypesCodec.encode_long(buf, _REQUEST_TIMEOUT_MILLIS_OFFSET, timeout_millis)
     FixSizedTypesCodec.encode_int(buf, _REQUEST_CURSOR_BUFFER_SIZE_OFFSET, cursor_buffer_size)
     FixSizedTypesCodec.encode_byte(buf, _REQUEST_EXPECTED_RESULT_TYPE_OFFSET, expected_result_type)
+    FixSizedTypesCodec.encode_boolean(buf, _REQUEST_SKIP_UPDATE_STATISTICS_OFFSET, skip_update_statistics)
     StringCodec.encode(buf, sql)
     ListMultiFrameCodec.encode_contains_nullable(buf, parameters, DataCodec.encode)
     CodecUtil.encode_nullable(buf, schema, StringCodec.encode)
diff --git a/hazelcast/sql.py b/hazelcast/sql.py
@@ -1234,6 +1234,7 @@ def execute(self, sql, params, kwargs):
                 statement.schema,
                 statement.expected_result_type,
                 query_id,
+                False,
             )
 
             invocation = Invocation(
diff --git a/tests/unit/client_message_test.py b/tests/unit/client_message_test.py
@@ -52,6 +52,77 @@ def test_copy(self):
         self.assertEqual(99, message.buf[0])
         self.assertEqual(0, copy.buf[0])  # should be a deep copy
 
+    def test_encode(self):
+        msg = client_authentication_codec.encode_request(
+            "dev",
+            "username",
+            "password",
+            uuid.UUID(hex="1862c7d2-f89c-4151-981d-07a6287089d3"),
+            "PYH",
+            1,
+            "5.0",
+            "hz.client_0",
+            ["label"],
+        )
+
+        # fmt: off
+        expected = [
+            # initial frame
+            40, 0, 0, 0,  # length
+            0, 192,  # flags
+            0, 1, 0, 0,  # message type
+            0, 0, 0, 0, 0, 0, 0, 0,  # correlation id
+            255, 255, 255, 255,  # partition id
+            0, 81, 65, 156, 248, 210, 199, 98, 24, 211, 137, 112, 40, 166, 7, 29, 152,  # uuid
+            1,  # serialization version
+
+            # cluster name frame
+            9, 0, 0, 0,  # length
+            0, 0,  # flags
+            100, 101, 118,  # cluster name
+
+            # username frame
+            14, 0, 0, 0,  # length
+            0, 0,  # length
+            117, 115, 101, 114, 110, 97, 109, 101,  # username
+
+            # password frame
+            14, 0, 0, 0,  # length
+            0, 0,  # flags
+            112, 97, 115, 115, 119, 111, 114, 100,  # password
+
+            # client type frame
+            9, 0, 0, 0,  # length
+            0, 0,  # flags
+            80, 89, 72,  # client type
+
+            # client version frame
+            9, 0, 0, 0,  # length
+            0, 0,  # flags
+            53, 46, 48,  # version
+
+            # client name frame
+            17, 0, 0, 0,  # length
+            0, 0,  # flags
+            104, 122, 46, 99, 108, 105, 101, 110, 116, 95, 48,  # client name
+
+            # labels begin frame
+            6, 0, 0, 0,  # length
+            0, 16,  # flags
+
+            # labels[0] frame
+            11, 0, 0, 0,  # length
+            0, 0,  # flags
+            108, 97, 98, 101, 108,  # labels[0]
+
+            # labels end frame
+            6, 0, 0, 0,  # length
+            0, 40,  # flags
+        ]
+        # fmt: on
+
+        self.assertEqual(bytearray(expected), msg.buf)
+
 
 BEGIN_FRAME = Frame(bytearray(0), 1 << 12)
 END_FRAME = Frame(bytearray(), 1 << 11)

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-__version__ = "5.0"`
	`1`	`+__version__ = "5.0.1"`
`2`	`2`
`3`	`3`	`# Set the default handler to "hazelcast" loggers`
`4`	`4`	`# to avoid "No handlers could be found" warnings.`