Skip to content

Commit b4059ce

Browse files
willdonnellywilldonnelly
committed
source-mysql-batch: Fix CHAR(n) COLLATE *_bin handling
Previously we added logic to serialize string/blob result types differently depending on the value of the `BINARY_FLAG` flat bit on the result. This was incorrect, because that flag actually means that a binary collation order is used for a given column. The correct way to check whether a column contains binary data vs text data is by looking for character set number 63. See https://dev.mysql.com/doc/c-api/8.0/en/c-api-data-structures.html which explicitly says that: > To distinguish between binary and nonbinary data for string data > types, check whether the charsetnr value is 63. If so, the character > set is binary, which indicates binary rather than nonbinary data. > This enables you to distinguish BINARY from CHAR, VARBINARY from > VARCHAR, and the BLOB types from the TEXT types.
1 parent c5a814f commit b4059ce

5 files changed

Lines changed: 429 additions & 17 deletions

File tree

‎source-mysql-batch/.snapshots/TestCharCollations-Capture‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
# ================================
2+
# Collection "acmeCo/test/test/charcollations_382659": 5 Documents
3+
# ================================
4+
{
5+
"type": "object",
6+
"required": [
7+
"_meta",
8+
"char_binary",
9+
"char_default",
10+
"char_utf8mb4_bin",
11+
"id",
12+
"text_default",
13+
"text_utf8mb4_bin",
14+
"varchar_default",
15+
"varchar_utf8mb4_bin"
16+
],
17+
"additionalProperties": false,
18+
"properties": {
19+
"_meta": {
20+
"$schema": "https://json-schema.org/draft/2020-12/schema",
21+
"$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata",
22+
"properties": {
23+
"polled": {
24+
"type": "string",
25+
"format": "date-time",
26+
"title": "Polled Timestamp",
27+
"description": "The time at which the update query which produced this document as executed."
28+
},
29+
"index": {
30+
"type": "integer",
31+
"title": "Result Index",
32+
"description": "The index of this document within the query execution which produced it."
33+
},
34+
"row_id": {
35+
"type": "integer",
36+
"title": "Row ID",
37+
"description": "Row ID of the Document"
38+
},
39+
"op": {
40+
"type": "string",
41+
"enum": [
42+
"c",
43+
"u",
44+
"d"
45+
],
46+
"title": "Change Operation",
47+
"description": "Operation type (c: Create / u: Update / d: Delete)",
48+
"default": "u"
49+
},
50+
"source": {
51+
"properties": {
52+
"resource": {
53+
"type": "string",
54+
"description": "Resource name of the binding from which this document was captured."
55+
},
56+
"schema": {
57+
"type": "string",
58+
"description": "Database schema from which the document was read."
59+
},
60+
"table": {
61+
"type": "string",
62+
"description": "Database table from which the document was read."
63+
},
64+
"tag": {
65+
"type": "string",
66+
"description": "Optional 'Source Tag' property as defined in the endpoint configuration."
67+
}
68+
},
69+
"additionalProperties": false,
70+
"type": "object",
71+
"required": [
72+
"resource"
73+
]
74+
}
75+
},
76+
"type": "object",
77+
"required": [
78+
"polled",
79+
"index",
80+
"row_id",
81+
"source"
82+
],
83+
"additionalProperties": false
84+
},
85+
"char_binary": {
86+
"maxLength": 16,
87+
"description": "(source type: binary)",
88+
"type": [
89+
"string"
90+
]
91+
},
92+
"char_default": {
93+
"maxLength": 10,
94+
"description": "(source type: char)",
95+
"type": [
96+
"string"
97+
]
98+
},
99+
"char_utf8mb4_bin": {
100+
"maxLength": 10,
101+
"description": "(source type: char)",
102+
"type": [
103+
"string"
104+
]
105+
},
106+
"id": {
107+
"type": "integer",
108+
"description": "(source type: non-nullable int)"
109+
},
110+
"text_default": {
111+
"description": "(source type: text)",
112+
"type": [
113+
"string"
114+
]
115+
},
116+
"text_utf8mb4_bin": {
117+
"description": "(source type: text)",
118+
"type": [
119+
"string"
120+
]
121+
},
122+
"varchar_default": {
123+
"maxLength": 255,
124+
"description": "(source type: varchar)",
125+
"type": [
126+
"string"
127+
]
128+
},
129+
"varchar_utf8mb4_bin": {
130+
"maxLength": 255,
131+
"description": "(source type: varchar)",
132+
"type": [
133+
"string"
134+
]
135+
}
136+
},
137+
"x-infer-schema": true
138+
}
139+
{
140+
"_meta": {
141+
"polled": "<TIMESTAMP>",
142+
"index": 0,
143+
"row_id": 0,
144+
"source": {
145+
"resource": "test_charcollations_382659",
146+
"schema": "test",
147+
"table": "charcollations_382659"
148+
}
149+
},
150+
"char_binary": "aGVsbG8AAAAAAA==",
151+
"char_default": "hello",
152+
"char_utf8mb4_bin": "hello",
153+
"id": 0,
154+
"text_default": "text_val",
155+
"text_utf8mb4_bin": "text_val",
156+
"varchar_default": "world",
157+
"varchar_utf8mb4_bin": "world"
158+
}
159+
{
160+
"_meta": {
161+
"polled": "<TIMESTAMP>",
162+
"index": 1,
163+
"row_id": 1,
164+
"source": {
165+
"resource": "test_charcollations_382659",
166+
"schema": "test",
167+
"table": "charcollations_382659"
168+
}
169+
},
170+
"char_binary": "c2hvcnQAAAAAAA==",
171+
"char_default": "short",
172+
"char_utf8mb4_bin": "short",
173+
"id": 1,
174+
"text_default": "short",
175+
"text_utf8mb4_bin": "short",
176+
"varchar_default": "short",
177+
"varchar_utf8mb4_bin": "short"
178+
}
179+
{
180+
"_meta": {
181+
"polled": "<TIMESTAMP>",
182+
"index": 2,
183+
"row_id": 2,
184+
"source": {
185+
"resource": "test_charcollations_382659",
186+
"schema": "test",
187+
"table": "charcollations_382659"
188+
}
189+
},
190+
"char_binary": "AAAAAAAAAAAAAA==",
191+
"char_default": "",
192+
"char_utf8mb4_bin": "",
193+
"id": 2,
194+
"text_default": "",
195+
"text_utf8mb4_bin": "",
196+
"varchar_default": "",
197+
"varchar_utf8mb4_bin": ""
198+
}
199+
{
200+
"_meta": {
201+
"polled": "<TIMESTAMP>",
202+
"index": 3,
203+
"row_id": 3,
204+
"source": {
205+
"resource": "test_charcollations_382659",
206+
"schema": "test",
207+
"table": "charcollations_382659"
208+
}
209+
},
210+
"char_binary": null,
211+
"char_default": null,
212+
"char_utf8mb4_bin": null,
213+
"id": 3,
214+
"text_default": null,
215+
"text_utf8mb4_bin": null,
216+
"varchar_default": null,
217+
"varchar_utf8mb4_bin": null
218+
}
219+
# ================================
220+
# Final State Checkpoint
221+
# ================================
222+
{"bindingStateV1":{"test_charcollations_382659":{"CursorNames":["id"],"CursorValues":[3],"DocumentCount":4,"LastPolled":"<TIMESTAMP>"}}}
223+

‎source-mysql-batch/.snapshots/TestCharCollations-Discovery‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
Binding 0:
2+
{
3+
"resource_config_json": {
4+
"name": "test_charcollations_382659",
5+
"schema": "test",
6+
"table": "charcollations_382659",
7+
"cursor": [
8+
"id"
9+
]
10+
},
11+
"resource_path": [
12+
"test_charcollations_382659"
13+
],
14+
"collection": {
15+
"name": "acmeCo/test/test/charcollations_382659",
16+
"read_schema_json": {
17+
"type": "object",
18+
"required": [
19+
"_meta",
20+
"id"
21+
],
22+
"properties": {
23+
"_meta": {
24+
"$schema": "https://json-schema.org/draft/2020-12/schema",
25+
"$id": "https://github.com/estuary/connectors/source-mysql-batch/document-metadata",
26+
"properties": {
27+
"polled": {
28+
"type": "string",
29+
"format": "date-time",
30+
"title": "Polled Timestamp",
31+
"description": "The time at which the update query which produced this document as executed."
32+
},
33+
"index": {
34+
"type": "integer",
35+
"title": "Result Index",
36+
"description": "The index of this document within the query execution which produced it."
37+
},
38+
"row_id": {
39+
"type": "integer",
40+
"title": "Row ID",
41+
"description": "Row ID of the Document"
42+
},
43+
"op": {
44+
"type": "string",
45+
"enum": [
46+
"c",
47+
"u",
48+
"d"
49+
],
50+
"title": "Change Operation",
51+
"description": "Operation type (c: Create / u: Update / d: Delete)",
52+
"default": "u"
53+
},
54+
"source": {
55+
"properties": {
56+
"resource": {
57+
"type": "string",
58+
"description": "Resource name of the binding from which this document was captured."
59+
},
60+
"schema": {
61+
"type": "string",
62+
"description": "Database schema from which the document was read."
63+
},
64+
"table": {
65+
"type": "string",
66+
"description": "Database table from which the document was read."
67+
},
68+
"tag": {
69+
"type": "string",
70+
"description": "Optional 'Source Tag' property as defined in the endpoint configuration."
71+
}
72+
},
73+
"type": "object",
74+
"required": [
75+
"resource"
76+
]
77+
}
78+
},
79+
"type": "object",
80+
"required": [
81+
"polled",
82+
"index",
83+
"row_id",
84+
"source"
85+
]
86+
},
87+
"char_binary": {
88+
"maxLength": 16,
89+
"description": "(source type: binary)",
90+
"type": [
91+
"string",
92+
"null"
93+
]
94+
},
95+
"char_default": {
96+
"maxLength": 10,
97+
"description": "(source type: char)",
98+
"type": [
99+
"string",
100+
"null"
101+
]
102+
},
103+
"char_utf8mb4_bin": {
104+
"maxLength": 10,
105+
"description": "(source type: char)",
106+
"type": [
107+
"string",
108+
"null"
109+
]
110+
},
111+
"id": {
112+
"type": "integer",
113+
"description": "(source type: non-nullable int)"
114+
},
115+
"text_default": {
116+
"description": "(source type: text)",
117+
"type": [
118+
"string",
119+
"null"
120+
]
121+
},
122+
"text_utf8mb4_bin": {
123+
"description": "(source type: text)",
124+
"type": [
125+
"string",
126+
"null"
127+
]
128+
},
129+
"varchar_default": {
130+
"maxLength": 255,
131+
"description": "(source type: varchar)",
132+
"type": [
133+
"string",
134+
"null"
135+
]
136+
},
137+
"varchar_utf8mb4_bin": {
138+
"maxLength": 255,
139+
"description": "(source type: varchar)",
140+
"type": [
141+
"string",
142+
"null"
143+
]
144+
}
145+
},
146+
"x-infer-schema": true
147+
},
148+
"key": [
149+
"/id"
150+
],
151+
"projections": null
152+
},
153+
"state_key": "test_charcollations_382659"
154+
}
155+

‎source-mysql-batch/driver.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -630,7 +630,7 @@ func (c *capture) poll(ctx context.Context, binding *bindingInfo) error {
630630
rowValues[0] = metadata
631631
for idx, val := range row {
632632
var field = result.Fields[idx]
633-
var translatedValue, err = translateMySQLValue(val.Value(), field.Type, field.Flag)
633+
var translatedValue, err = translateMySQLValue(val.Value(), field.Type, field.Charset)
634634
if err != nil {
635635
return fmt.Errorf("error translating column %q value: %w", string(field.Name), err)
636636
}

0 commit comments

Comments
 (0)