Cannot append to a partitioned table with a map(string, string) column

[ericseguin-northstar]

Apache Iceberg version

iceberg-go version: v0.4.0
arrow-go version: v18.4.1

Please describe the bug 🐞

When I call tbl.Append() on a partitioned table with a map(string, string) column where the value is nullable, I get the following error:

not implemented: function 'array_take' has no kernel matching input types (map<utf8, utf8, items_nullable>, int64)

If the table is unpartitioned or if the map column is removed the operation works without issue.

To reproduce

Here's a very minimal example that reproduces the issue on my machine:

package reproduce

import (
	"database/sql"
	"testing"
	"time"

	"github.com/apache/arrow-go/v18/arrow"
	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/memory"
	"github.com/apache/iceberg-go"
	"github.com/apache/iceberg-go/catalog/rest"
	"github.com/apache/iceberg-go/table"
	"github.com/stretchr/testify/require"
	"github.com/trinodb/trino-go-client/trino"
)

func TestReproduce(t *testing.T) {
	ctx := t.Context()

	// create a table with a MAP column and an arbitrary TIMESTAMP column for partitioning
	// use Trino so we can enable partitioning
	trinoCfg := trino.Config{
		Catalog:   "<catalog name in Trino>",
		ServerURI: "<trino endpoint>",
		Schema:    "public",
	}
	trinoDSN, err := trinoCfg.FormatDSN()
	require.NoError(t, err)

	db, err := sql.Open("trino", trinoDSN)
	require.NoError(t, err)

	_, err = db.ExecContext(ctx, `
		CREATE TABLE my_table (
			date TIMESTAMP,
			my_map MAP(VARCHAR, VARCHAR)
		) WITH (
			partitioning = ARRAY['day(date)']
		)`)
	require.NoError(t, err)
	defer db.ExecContext(ctx, "DROP TABLE my_table")

	// get the Iceberg table
	cat, err := rest.NewCatalog(ctx, "rest", "<catalog REST endpoint>") // I'm using nessie
	require.NoError(t, err)

	tbl, err := cat.LoadTable(ctx, []string{"public", "my_table"})
	require.NoError(t, err)

	// print obtained schema in case it's helpful for investigation
	arrowSchema, err := table.SchemaToArrowSchema(tbl.Schema(), nil, true, false)
	require.NoError(t, err)

	t.Logf("Iceberg schema:\n%s", tbl.Schema().String())
	t.Logf("Arrow schema:\n%s", arrowSchema.String())

	// append a dummy record
	rb := array.NewRecordBuilder(memory.NewGoAllocator(), arrowSchema)
	defer rb.Release()

	rb.Field(0).(*array.TimestampBuilder).Append(arrow.Timestamp(time.Now().UnixMicro()))
	mb := rb.Field(1).(*array.MapBuilder)
	mb.Append(true)
	mb.KeyBuilder().(*array.StringBuilder).Append("key")
	mb.ItemBuilder().(*array.StringBuilder).Append("val")

	rec := rb.NewRecordBatch()
	defer rec.Release()

	rr, err := array.NewRecordReader(arrowSchema, []arrow.RecordBatch{rec})
	require.NoError(t, err)
	defer rr.Release()

	_, err = tbl.Append(ctx, rr, iceberg.Properties{})
	require.NoError(t, err) // fails here
}

This fails at the tbl.Append() call as marked:

--- FAIL: TestReproduce (0.31s)
    <...>/reproduce_test.go:74: Iceberg schema:
        table {
        	1: date: required timestamp
        	2: my_map: required map<string, string>
        }
    <...>/reproduce_test.go:75: Arrow schema:
        schema:
          fields: 2
            - date: type=timestamp[us]
              metadata: ["PARQUET:field_id": "1"]
            - my_map: type=map<utf8, utf8, items_nullable>
                metadata: ["PARQUET:field_id": "2"]
    <...>/reproduce_test.go:94: 
        	Error Trace:	<...>/reproduce_test.go:94
        	Error:      	Received unexpected error:
        	            	not implemented: function 'array_take' has no kernel matching input types (map<utf8, utf8, items_nullable>, int64)
        	Test:       	TestReproduce

[ericseguin-northstar]

For completeness, here's the output of SHOW CREATE TABLE with Trino:

trino:public> show create table my_table;
                                          Create Table                                           
-------------------------------------------------------------------------------------------------
 CREATE TABLE my_catalog.public.my_table (                                             
    date timestamp(6) NOT NULL,                                                                  
    my_map map(varchar, varchar) NOT NULL                                                        
 )                                                                                               
 WITH (                                                                                          
    format = 'PARQUET',                                                                          
    format_version = 2,                                                                          
    location = 's3://my_bucket/public/my_table_ffe70918-14b0-47eb-8bf6-c1e5cb661e3d', 
    max_commit_retry = 4,                                                                        
    partitioning = ARRAY['day(date)']                                                            
 )                                                                                               
(1 row)

[zeroshade]

Can you try upgrading your arrow-go version to 18.5.0? I believe that's where we added the relevant kernel

[ericseguin-northstar]

Confirmed (and found the associated PR), the append works as expected with arrow-go 18.5.0.

Unfortunately using iceberg-go v0.4.0 and arrow-go v18.5.0 seems to result in a compatibility issue with the substrait dependency (v4 in iceberg-go, v7 in arrow-go):

# github.com/apache/iceberg-go/table/substrait
<home>/go/pkg/mod/github.com/apache/iceberg-go@v0.4.0/table/substrait/substrait.go:49:38: cannot use expr.NewEmptyExtensionRegistry(collection) (value of struct type "github.com/substrait-io/substrait-go/v4/expr".ExtensionRegistry) as "github.com/substrait-io/substrait-go/v7/expr".ExtensionRegistry value in argument to exprs.NewExtensionSetDefault

This goes away when using the latest iceberg-go commit (Go ref v0.4.1-0.20260105163041-9e4a5b74ffc2), so it looks like we just need a new stable iceberg-go release :)