Support null in json string cols (#7963)

lhoestq · web-flow · commit 0fcfcf3ea320 · 2026-01-26T18:48:44.000+01:00
* support null in json string cols

* style
diff --git a/src/datasets/packaged_modules/json/json.py b/src/datasets/packaged_modules/json/json.py
@@ -105,7 +105,8 @@ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
                         .to_json(orient="records", lines=True)
                     )
                     string_array = pa.array(
-                        ("{" + x.rstrip() for x in ("\n" + jsonl).split("\n{") if x), type=pa.string()
+                        (None if x.strip() == "null" else x.strip() for x in jsonl.split("\n") if x.strip()),
+                        type=pa.string(),
                     )
                     pa_table = pa_table.set_column(i, column_name, string_array)
             # more expensive cast to support nested structures with keys in a different order

Original file line number	Diff line number	Diff line change
`@@ -105,7 +105,8 @@ def _cast_table(self, pa_table: pa.Table) -> pa.Table:`
`105`	`105`	`.to_json(orient="records", lines=True)`
`106`	`106`	`)`
`107`	`107`	`string_array = pa.array(`
`108`		`- ("{" + x.rstrip() for x in ("\n" + jsonl).split("\n{") if x), type=pa.string()`
	`108`	`+ (None if x.strip() == "null" else x.strip() for x in jsonl.split("\n") if x.strip()),`
	`109`	`+ type=pa.string(),`
`109`	`110`	`)`
`110`	`111`	`pa_table = pa_table.set_column(i, column_name, string_array)`
`111`	`112`	`# more expensive cast to support nested structures with keys in a different order`