Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions catalog/glue/glue_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ var testNonIcebergGlueTable = types.Table{
},
}

var testSchema = iceberg.NewSchemaWithIdentifiers(0, []int{},
var testSchema = iceberg.MustNewSchemaWithIdentifiers(0, []int{},
iceberg.NestedField{ID: 1, Name: "foo", Type: iceberg.PrimitiveTypes.String},
iceberg.NestedField{ID: 2, Name: "bar", Type: iceberg.PrimitiveTypes.Int32, Required: true},
iceberg.NestedField{ID: 3, Name: "baz", Type: iceberg.PrimitiveTypes.Bool})
Expand Down Expand Up @@ -1005,10 +1005,11 @@ func TestGlueCreateTableInvalidMetadataRollback(t *testing.T) {
func TestGlueCreateTableRollbackOnInvalidMetadata(t *testing.T) {
assert := require.New(t)
mockGlueSvc := &mockGlueClient{}
schema := iceberg.NewSchemaWithIdentifiers(1, []int{1},
schema, err := iceberg.NewSchemaWithIdentifiers(1, []int{1},
iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.Int64Type{}, Required: true},
iceberg.NestedField{ID: 2, Name: "name", Type: iceberg.StringType{}, Required: true},
)
assert.NoError(err)
mockGlueSvc.On("CreateTable", mock.Anything, mock.Anything, mock.Anything).Return(&glue.CreateTableOutput{}, nil)
mockGlueSvc.On("GetTable", mock.Anything, &glue.GetTableInput{
DatabaseName: aws.String("test_database"),
Expand All @@ -1026,7 +1027,7 @@ func TestGlueCreateTableRollbackOnInvalidMetadata(t *testing.T) {
glueSvc: mockGlueSvc,
awsCfg: &aws.Config{},
}
_, err := glueCatalog.CreateTable(context.TODO(),
_, err = glueCatalog.CreateTable(context.TODO(),
TableIdentifier("test_database", "test_rollback_table"),
schema,
catalog.WithLocation("s3://non-existent-test-bucket"))
Expand Down Expand Up @@ -1094,10 +1095,11 @@ func TestAlterTableIntegration(t *testing.T) {
metadataLocation := os.Getenv("TEST_TABLE_LOCATION")
tbName := fmt.Sprintf("table_%d", time.Now().UnixNano())
tbIdent := TableIdentifier(dbName, tbName)
schema := iceberg.NewSchemaWithIdentifiers(0, []int{},
schema, err := iceberg.NewSchemaWithIdentifiers(0, []int{},
iceberg.NestedField{ID: 1, Name: "foo", Type: iceberg.PrimitiveTypes.String},
iceberg.NestedField{ID: 2, Name: "bar", Type: iceberg.PrimitiveTypes.Int32},
iceberg.NestedField{ID: 3, Name: "baz", Type: iceberg.PrimitiveTypes.Bool})
assert.NoError(err)

awsCfg, err := config.LoadDefaultConfig(context.TODO(), config.WithClientLogMode(aws.LogRequest|aws.LogResponse))
assert.NoError(err)
Expand Down Expand Up @@ -1169,7 +1171,9 @@ func TestAlterTableIntegration(t *testing.T) {
}
newFields := append(currentSchema.Fields(), addField) // add column 'new_col'
newFields = append(newFields[:1], newFields[2:]...) // drop column 'bar'
updateColumns := table.NewAddSchemaUpdate(iceberg.NewSchemaWithIdentifiers(newSchemaId, currentSchema.IdentifierFieldIDs, newFields...))
newSchema, err := iceberg.NewSchemaWithIdentifiers(newSchemaId, currentSchema.IdentifierFieldIDs, newFields...)
require.NoError(t, err)
updateColumns := table.NewAddSchemaUpdate(newSchema)
setSchema := table.NewSetCurrentSchemaUpdate(newSchemaId)

_, _, err = ctlg.CommitTable(
Expand Down Expand Up @@ -1342,9 +1346,10 @@ func TestCommitTableOptimisticLockingIntegration(t *testing.T) {
tbName := fmt.Sprintf("optimistic_lock_test_%d", time.Now().UnixNano())
tbIdent := TableIdentifier(dbName, tbName)

schema := iceberg.NewSchemaWithIdentifiers(0, []int{},
schema, err := iceberg.NewSchemaWithIdentifiers(0, []int{},
iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64},
iceberg.NestedField{ID: 2, Name: "data", Type: iceberg.PrimitiveTypes.String})
assert.NoError(err)

awsCfg, err := config.LoadDefaultConfig(context.TODO(), config.WithClientLogMode(aws.LogRequest|aws.LogResponse))
assert.NoError(err)
Expand Down
76 changes: 39 additions & 37 deletions catalog/glue/schema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,8 @@
Doc: "User tags",
},
}
schema := iceberg.NewSchema(1, fields...)
schema, err := iceberg.NewSchema(1, fields...)
require.NoError(t, err)

Check failure on line 353 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / windows-latest go1.24.9

undefined: require (typecheck)

Check failure on line 353 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / ubuntu-latest go1.24.9

undefined: require (typecheck)

Check failure on line 353 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / macos-latest go1.23.6

undefined: require (typecheck)

Check failure on line 353 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / macos-latest go1.24.9

undefined: require (typecheck)

Check failure on line 353 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / ubuntu-latest go1.23.6

undefined: require (typecheck)

Check failure on line 353 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / windows-latest go1.23.6

undefined: require (typecheck)
columns := schemaToGlueColumns(schema, true)
assert.Equal(t, 4, len(columns))
assert.Equal(t, "id", aws.ToString(columns[0].Name))
Expand All @@ -365,42 +366,43 @@
}

func TestSchemasToGlueColumns(t *testing.T) {
schemas := []*iceberg.Schema{
iceberg.NewSchema(0,
iceberg.NestedField{
ID: 1,
Name: "id",
Type: iceberg.Int64Type{},
Required: true,
},
iceberg.NestedField{
ID: 2,
Name: "name",
Type: iceberg.StringType{},
Required: true,
},
iceberg.NestedField{
ID: 3,
Name: "address",
Type: iceberg.StringType{},
Required: false,
},
),
iceberg.NewSchema(1,
iceberg.NestedField{
ID: 1,
Name: "id",
Type: iceberg.Int64Type{},
Required: true,
},
iceberg.NestedField{
ID: 2,
Name: "name",
Type: iceberg.StringType{},
Required: true,
},
),
}
schema0, err := iceberg.NewSchema(0,
iceberg.NestedField{
ID: 1,
Name: "id",
Type: iceberg.Int64Type{},
Required: true,
},
iceberg.NestedField{
ID: 2,
Name: "name",
Type: iceberg.StringType{},
Required: true,
},
iceberg.NestedField{
ID: 3,
Name: "address",
Type: iceberg.StringType{},
Required: false,
},
)
require.NoError(t, err)

Check failure on line 389 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / windows-latest go1.24.9

undefined: require (typecheck)

Check failure on line 389 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / ubuntu-latest go1.24.9

undefined: require (typecheck)

Check failure on line 389 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / macos-latest go1.23.6

undefined: require (typecheck)

Check failure on line 389 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / macos-latest go1.24.9

undefined: require (typecheck)

Check failure on line 389 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / ubuntu-latest go1.23.6

undefined: require (typecheck)

Check failure on line 389 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / windows-latest go1.23.6

undefined: require (typecheck)
schema1, err := iceberg.NewSchema(1,
iceberg.NestedField{
ID: 1,
Name: "id",
Type: iceberg.Int64Type{},
Required: true,
},
iceberg.NestedField{
ID: 2,
Name: "name",
Type: iceberg.StringType{},
Required: true,
},
)
require.NoError(t, err)

Check failure on line 404 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / windows-latest go1.24.9

undefined: require (typecheck)

Check failure on line 404 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / ubuntu-latest go1.24.9

undefined: require (typecheck)

Check failure on line 404 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / macos-latest go1.23.6

undefined: require (typecheck)

Check failure on line 404 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / macos-latest go1.24.9

undefined: require (typecheck)

Check failure on line 404 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / ubuntu-latest go1.23.6

undefined: require (typecheck)

Check failure on line 404 in catalog/glue/schema_test.go

View workflow job for this annotation

GitHub Actions / windows-latest go1.23.6

undefined: require (typecheck)
schemas := []*iceberg.Schema{schema0, schema1}

expectedColumns := []types.Column{
{
Expand Down
11 changes: 7 additions & 4 deletions catalog/rest/rest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -972,7 +972,7 @@ var (
}
}`, exampleTableMetadataNoSnapshotV1)

tableSchemaSimple = iceberg.NewSchemaWithIdentifiers(1, []int{2},
tableSchemaSimple = iceberg.MustNewSchemaWithIdentifiers(1, []int{2},
iceberg.NestedField{ID: 1, Name: "foo", Type: iceberg.StringType{}, Required: false},
iceberg.NestedField{ID: 2, Name: "bar", Type: iceberg.PrimitiveTypes.Int32, Required: true},
iceberg.NestedField{ID: 3, Name: "baz", Type: iceberg.PrimitiveTypes.Bool, Required: false},
Expand Down Expand Up @@ -2151,12 +2151,13 @@ func (r *RestCatalogSuite) TestCreateView200() {
ns := "ns"
viewName := "view"
identifier := table.Identifier{ns, viewName}
schema := iceberg.NewSchemaWithIdentifiers(0, []int{1}, iceberg.NestedField{
schema, err := iceberg.NewSchemaWithIdentifiers(0, []int{1}, iceberg.NestedField{
ID: 1,
Name: "id",
Type: iceberg.PrimitiveTypes.Int32,
Required: true,
})
r.Require().NoError(err)
config := iceberg.Properties{
"comment": "Example view created via REST catalog",
"owner": "admin",
Expand Down Expand Up @@ -2206,12 +2207,13 @@ func (r *RestCatalogSuite) TestCreateView409() {
ns := "ns"
viewName := "view"
identifier := table.Identifier{ns, viewName}
schema := iceberg.NewSchema(1, iceberg.NestedField{
schema, err := iceberg.NewSchema(1, iceberg.NestedField{
ID: 1,
Name: "id",
Type: iceberg.PrimitiveTypes.Int32,
Required: true,
})
r.Require().NoError(err)
sql := "SELECT * FROM table"
reprs := []view.Representation{view.NewRepresentation(sql, "default")}
version, err := view.NewVersion(1, 1, reprs, table.Identifier{ns})
Expand Down Expand Up @@ -2240,12 +2242,13 @@ func (r *RestCatalogSuite) TestCreateView404() {
ns := "ns"
viewName := "view"
identifier := table.Identifier{ns, viewName}
schema := iceberg.NewSchema(1, iceberg.NestedField{
schema, err := iceberg.NewSchema(1, iceberg.NestedField{
ID: 1,
Name: "id",
Type: iceberg.PrimitiveTypes.Int32,
Required: true,
})
r.Require().NoError(err)
sql := "SELECT * FROM table"
reprs := []view.Representation{
view.NewRepresentation(sql, "spark"),
Expand Down
2 changes: 1 addition & 1 deletion catalog/sql/sql_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
"github.com/apache/arrow-go/v18/parquet/pqarrow"
"github.com/apache/iceberg-go"
"github.com/apache/iceberg-go/catalog"
"github.com/apache/iceberg-go/catalog/internal"

Check failure on line 37 in catalog/sql/sql_test.go

View workflow job for this annotation

GitHub Actions / windows-latest go1.24.9

could not import github.com/apache/iceberg-go/catalog/internal (-: # github.com/apache/iceberg-go/catalog/internal

Check failure on line 37 in catalog/sql/sql_test.go

View workflow job for this annotation

GitHub Actions / ubuntu-latest go1.24.9

could not import github.com/apache/iceberg-go/catalog/internal (-: # github.com/apache/iceberg-go/catalog/internal

Check failure on line 37 in catalog/sql/sql_test.go

View workflow job for this annotation

GitHub Actions / macos-latest go1.23.6

could not import github.com/apache/iceberg-go/catalog/internal (-: # github.com/apache/iceberg-go/catalog/internal

Check failure on line 37 in catalog/sql/sql_test.go

View workflow job for this annotation

GitHub Actions / macos-latest go1.24.9

could not import github.com/apache/iceberg-go/catalog/internal (-: # github.com/apache/iceberg-go/catalog/internal

Check failure on line 37 in catalog/sql/sql_test.go

View workflow job for this annotation

GitHub Actions / ubuntu-latest go1.23.6

could not import github.com/apache/iceberg-go/catalog/internal (-: # github.com/apache/iceberg-go/catalog/internal

Check failure on line 37 in catalog/sql/sql_test.go

View workflow job for this annotation

GitHub Actions / windows-latest go1.23.6

could not import github.com/apache/iceberg-go/catalog/internal (-: # github.com/apache/iceberg-go/catalog/internal
sqlcat "github.com/apache/iceberg-go/catalog/sql"
"github.com/apache/iceberg-go/table"
"github.com/stretchr/testify/assert"
Expand All @@ -42,7 +42,7 @@
"github.com/uptrace/bun/driver/sqliteshim"
)

var tableSchemaNested = iceberg.NewSchemaWithIdentifiers(1,
var tableSchemaNested = iceberg.MustNewSchemaWithIdentifiers(1,
[]int{1},
iceberg.NestedField{
ID: 1, Name: "foo", Type: iceberg.PrimitiveTypes.String, Required: false,
Expand Down
27 changes: 18 additions & 9 deletions exprs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,18 +68,22 @@ func TestUnaryExpr(t *testing.T) {
assert.True(t, n.Equals(exp))
})

sc := iceberg.NewSchema(1, iceberg.NestedField{
sc, err := iceberg.NewSchema(1, iceberg.NestedField{
ID: 2, Name: "a", Type: iceberg.PrimitiveTypes.Int32,
})
sc2 := iceberg.NewSchema(1, iceberg.NestedField{
require.NoError(t, err)
sc2, err := iceberg.NewSchema(1, iceberg.NestedField{
ID: 2, Name: "a", Type: iceberg.PrimitiveTypes.Float64,
})
sc3 := iceberg.NewSchema(1, iceberg.NestedField{
require.NoError(t, err)
sc3, err := iceberg.NewSchema(1, iceberg.NestedField{
ID: 2, Name: "a", Type: iceberg.PrimitiveTypes.Int32, Required: true,
})
sc4 := iceberg.NewSchema(1, iceberg.NestedField{
require.NoError(t, err)
sc4, err := iceberg.NewSchema(1, iceberg.NestedField{
ID: 2, Name: "a", Type: iceberg.PrimitiveTypes.Float32, Required: true,
})
require.NoError(t, err)

t.Run("isnull and notnull", func(t *testing.T) {
t.Run("bind", func(t *testing.T) {
Expand Down Expand Up @@ -202,7 +206,7 @@ func TestRefBindingCaseSensitive(t *testing.T) {
}

func TestRefTypes(t *testing.T) {
sc := iceberg.NewSchema(1,
sc, err := iceberg.NewSchema(1,
iceberg.NestedField{ID: 1, Name: "a", Type: iceberg.PrimitiveTypes.Bool},
iceberg.NestedField{ID: 2, Name: "b", Type: iceberg.PrimitiveTypes.Int32},
iceberg.NestedField{ID: 3, Name: "c", Type: iceberg.PrimitiveTypes.Int64},
Expand All @@ -215,7 +219,9 @@ func TestRefTypes(t *testing.T) {
iceberg.NestedField{ID: 10, Name: "j", Type: iceberg.PrimitiveTypes.String},
iceberg.NestedField{ID: 11, Name: "k", Type: iceberg.PrimitiveTypes.Binary},
iceberg.NestedField{ID: 12, Name: "l", Type: iceberg.PrimitiveTypes.UUID},
iceberg.NestedField{ID: 13, Name: "m", Type: iceberg.FixedTypeOf(5)})
iceberg.NestedField{ID: 13, Name: "m", Type: iceberg.FixedTypeOf(5)},
)
require.NoError(t, err)

t.Run("bind term", func(t *testing.T) {
for i := 0; i < sc.NumFields(); i++ {
Expand Down Expand Up @@ -635,7 +641,7 @@ func TestBoundReferenceToString(t *testing.T) {
}

func TestToString(t *testing.T) {
schema := iceberg.NewSchema(1,
schema, err := iceberg.NewSchema(1,
iceberg.NestedField{ID: 1, Name: "a", Type: iceberg.PrimitiveTypes.String},
iceberg.NestedField{ID: 2, Name: "b", Type: iceberg.PrimitiveTypes.String},
iceberg.NestedField{ID: 3, Name: "c", Type: iceberg.PrimitiveTypes.String},
Expand All @@ -647,7 +653,9 @@ func TestToString(t *testing.T) {
iceberg.NestedField{ID: 9, Name: "i", Type: iceberg.PrimitiveTypes.UUID},
iceberg.NestedField{ID: 10, Name: "j", Type: iceberg.PrimitiveTypes.Bool},
iceberg.NestedField{ID: 11, Name: "k", Type: iceberg.PrimitiveTypes.Bool},
iceberg.NestedField{ID: 12, Name: "l", Type: iceberg.PrimitiveTypes.Binary})
iceberg.NestedField{ID: 12, Name: "l", Type: iceberg.PrimitiveTypes.Binary},
)
require.NoError(t, err)

null := iceberg.IsNull(iceberg.Reference("a"))
nan := iceberg.IsNaN(iceberg.Reference("g"))
Expand Down Expand Up @@ -770,10 +778,11 @@ func TestToString(t *testing.T) {
}

func TestBindAboveBelowIntMax(t *testing.T) {
sc := iceberg.NewSchema(1,
sc, err := iceberg.NewSchema(1,
iceberg.NestedField{ID: 1, Name: "a", Type: iceberg.PrimitiveTypes.Int32},
iceberg.NestedField{ID: 2, Name: "b", Type: iceberg.PrimitiveTypes.Float32},
)
require.NoError(t, err)

ref, ref2 := iceberg.Reference("a"), iceberg.Reference("b")
above, below := int64(math.MaxInt32)+1, int64(math.MinInt32)-1
Expand Down
2 changes: 1 addition & 1 deletion manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -2315,7 +2315,7 @@ type ManifestEntry interface {
wrap(status ManifestEntryStatus, snapshotID, seqNum, fileSeqNum *int64, datafile DataFile) ManifestEntry
}

var PositionalDeleteSchema = NewSchema(0,
var PositionalDeleteSchema = MustNewSchema(0,
NestedField{ID: 2147483546, Type: PrimitiveTypes.String, Name: "file_path", Required: true},
NestedField{ID: 2147483545, Type: PrimitiveTypes.Int32, Name: "pos", Required: true},
)
32 changes: 24 additions & 8 deletions manifest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package iceberg

import (
"bytes"
"fmt"
"io"
"testing"
"time"
Expand Down Expand Up @@ -443,7 +444,12 @@ var (
},
}

testSchema = NewSchema(0,
testSchema *Schema
)

func init() {
var err error
testSchema, err = NewSchema(0,
NestedField{ID: 1, Name: "VendorID", Type: PrimitiveTypes.Int32, Required: true},
NestedField{ID: 2, Name: "tpep_pickup_datetime", Type: PrimitiveTypes.Timestamp, Required: true},
NestedField{ID: 3, Name: "tpep_dropoff_datetime", Type: PrimitiveTypes.Timestamp, Required: true},
Expand All @@ -462,9 +468,12 @@ var (
NestedField{ID: 16, Name: "improvement_surcharge", Type: PrimitiveTypes.Float64, Required: false},
NestedField{ID: 17, Name: "total_amount", Type: PrimitiveTypes.Float64, Required: true},
NestedField{ID: 18, Name: "congestion_surcharge", Type: PrimitiveTypes.Float64, Required: false},
NestedField{ID: 19, Name: "VendorID", Type: PrimitiveTypes.Int32, Required: false},
NestedField{ID: 19, Name: "vendor_id_alt", Type: PrimitiveTypes.Int32, Required: false},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why change the field name? Isn't this intended to test the case of multiple fields with the same name but different IDs?

)
)
if err != nil {
panic(fmt.Sprintf("failed to create testSchema: %v", err))
}
}

type ManifestTestSuite struct {
suite.Suite
Expand Down Expand Up @@ -943,10 +952,16 @@ func (m *ManifestTestSuite) TestReadManifestIncompleteSchema() {
file, err := WriteManifest(
"s3://bucket/namespace/table/metadata/abcd-0123.avro", &buf, 2,
partitionSpec,
NewSchema(123,
NestedField{ID: 1, Name: "id", Type: Int64Type{}},
NestedField{ID: 2, Name: "name", Type: StringType{}},
),
func() *Schema {
sch, err := NewSchema(123,
NestedField{ID: 1, Name: "id", Type: Int64Type{}},
NestedField{ID: 2, Name: "name", Type: StringType{}},
)
if err != nil {
panic(fmt.Sprintf("failed to create schema: %v", err))
}
return sch
}(),
snapshotID,
[]ManifestEntry{NewManifestEntry(
EntryStatusADDED,
Expand Down Expand Up @@ -1361,7 +1376,8 @@ func (m *ManifestTestSuite) TestManifestEntryBuilder() {
}

func (m *ManifestTestSuite) TestManifestWriterMeta() {
sch := NewSchema(0, NestedField{ID: 0, Name: "test01", Type: StringType{}})
sch, err := NewSchema(0, NestedField{ID: 0, Name: "test01", Type: StringType{}})
m.Require().NoError(err)
w, err := NewManifestWriter(2, io.Discard, *UnpartitionedSpec, sch, 1)
m.Require().NoError(err)
md, err := w.meta()
Expand Down
2 changes: 1 addition & 1 deletion name_mapping.go
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ func ApplyNameMapping(schemaWithoutIDs *Schema, nameMapping NameMapping) (*Schem
}

return NewSchema(schemaWithoutIDs.ID,
top.Type.(*StructType).FieldList...), nil
top.Type.(*StructType).FieldList...)
}

type createMapping struct{}
Expand Down
Loading
Loading