Feature/custom analzyers (#216)

* Added test for views in backup.
* Added internal support for custom analzyers.

Author: Lars Maier

go.mod

@@ -6,5 +6,6 @@ require (
 	github.com/arangodb/go-velocypack v0.0.0-20190129082528-7896a965b4ad
 	github.com/coreos/go-iptables v0.4.3
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/stretchr/testify v1.2.2
 	github.com/pkg/errors v0.8.1
 )

test/view_test.go

@@ -28,6 +28,8 @@ import (
 	"testing"
 
 	driver "github.com/arangodb/go-driver"
+
+	"github.com/stretchr/testify/require"
 )
 
 // ensureArangoSearchView is a helper to check if an arangosearch view exists and create it if needed.
@@ -694,3 +696,72 @@ func TestArangoSearchPrimarySort(t *testing.T) {
 		})
 	}
 }
+
+func newBool(v bool) *bool {
+	return &v
+}
+
+// TestArangoSearchViewProperties353 tests for custom analyzers.
+func TestArangoSearchViewProperties353(t *testing.T) {
+	ctx := context.Background()
+	c := createClientFromEnv(t, true)
+	skipNoCluster(c, t) // analyzers can only be read in the
+	skipBelowVersion(c, "3.5.3", t)
+	db := ensureDatabase(ctx, c, "view_test", nil, t)
+	colname := "someCol"
+	ensureCollection(ctx, db, colname, nil, t)
+	name := "test_get_asview_353"
+	analyzerName := "myanalyzer"
+	opts := &driver.ArangoSearchViewProperties{
+		Links: driver.ArangoSearchLinks{
+			colname: driver.ArangoSearchElementProperties{
+				AnalyzerDefinitions: []driver.ArangoSearchAnalyzerDefinition{
+					driver.ArangoSearchAnalyzerDefinition{
+						Name: analyzerName,
+						Type: driver.ArangoSearchAnalyzerTypeNorm,
+						Properties: driver.ArangoSearchAnalyzerProperties{
+							Locale: "en_US.utf-8",
+							Case:   driver.ArangoSearchCaseLower,
+						},
+						Features: []driver.ArangoSearchAnalyzerFeature{
+							driver.ArangoSearchAnalyzerFeaturePosition,
+							driver.ArangoSearchAnalyzerFeatureFrequency,
+						},
+					},
+				},
+				IncludeAllFields: newBool(true),
+			},
+		},
+	}
+	_, err := db.CreateArangoSearchView(ctx, name, opts)
+	require.NoError(t, err)
+	// Get view
+	v, err := db.View(ctx, name)
+	require.NoError(t, err)
+	asv, err := v.ArangoSearchView()
+	require.NoError(t, err)
+	// Check asv properties
+	p, err := asv.Properties(ctx)
+	require.NoError(t, err)
+	require.Contains(t, p.Links, colname)
+
+	// get cluster inventory
+	cluster, err := c.Cluster(ctx)
+	require.NoError(t, err)
+	inv, err := cluster.DatabaseInventory(ctx, db)
+	require.NoError(t, err)
+	p2, found := inv.ViewByName(name)
+	require.True(t, found)
+
+	require.Contains(t, p2.Links, colname)
+	link := p2.Links[colname]
+	require.Len(t, link.AnalyzerDefinitions, 2)
+	analyzer := &link.AnalyzerDefinitions[1]
+	require.EqualValues(t, analyzer.Name, analyzerName)
+	require.EqualValues(t, analyzer.Type, driver.ArangoSearchAnalyzerTypeNorm)
+	require.Len(t, analyzer.Features, 2)
+	require.EqualValues(t, analyzer.Features[0], driver.ArangoSearchAnalyzerFeaturePosition)
+	require.EqualValues(t, analyzer.Features[1], driver.ArangoSearchAnalyzerFeatureFrequency)
+	require.EqualValues(t, analyzer.Properties.Locale, "en_US.utf-8")
+	require.EqualValues(t, analyzer.Properties.Case, driver.ArangoSearchCaseLower)
+}

view_arangosearch.go

@@ -39,6 +39,83 @@ type ArangoSearchView interface {
 	SetProperties(ctx context.Context, options ArangoSearchViewProperties) error
 }
 
+// ArangoSearchAnalyzerType specifies type of an analyzer
+type ArangoSearchAnalyzerType string
+
+const (
+	// ArangoSearchAnalyzerTypeIdentity treat value as atom (no transformation)
+	ArangoSearchAnalyzerTypeIdentity ArangoSearchAnalyzerType = "identity"
+	// ArangoSearchAnalyzerTypeDelimiter split into tokens at user-defined character
+	ArangoSearchAnalyzerTypeDelimiter ArangoSearchAnalyzerType = "delimiter"
+	// ArangoSearchAnalyzerTypeStem apply stemming to the value as a whole
+	ArangoSearchAnalyzerTypeStem ArangoSearchAnalyzerType = "stem"
+	// ArangoSearchAnalyzerTypeNorm apply normalization to the value as a whole
+	ArangoSearchAnalyzerTypeNorm ArangoSearchAnalyzerType = "norm"
+	// ArangoSearchAnalyzerTypeNGram create n-grams from value with user-defined lengths
+	ArangoSearchAnalyzerTypeNGram ArangoSearchAnalyzerType = "ngram"
+	// ArangoSearchAnalyzerTypeText tokenize into words, optionally with stemming, normalization and stop-word filtering
+	ArangoSearchAnalyzerTypeText ArangoSearchAnalyzerType = "text"
+)
+
+// ArangoSearchAnalyzerFeature specifies a feature to an analyzer
+type ArangoSearchAnalyzerFeature string
+
+const (
+	// ArangoSearchAnalyzerFeatureFrequency how often a term is seen, required for PHRASE()
+	ArangoSearchAnalyzerFeatureFrequency ArangoSearchAnalyzerFeature = "frequency"
+	// ArangoSearchAnalyzerFeatureNorm the field normalization factor
+	ArangoSearchAnalyzerFeatureNorm ArangoSearchAnalyzerFeature = "norm"
+	// ArangoSearchAnalyzerFeaturePosition sequentially increasing term position, required for PHRASE(). If present then the frequency feature is also required
+	ArangoSearchAnalyzerFeaturePosition ArangoSearchAnalyzerFeature = "position"
+)
+
+type ArangoSearchCaseType string
+
+const (
+	// ArangoSearchCaseUpper to convert to all lower-case characters
+	ArangoSearchCaseUpper ArangoSearchCaseType = "upper"
+	// ArangoSearchCaseLower to convert to all upper-case characters
+	ArangoSearchCaseLower ArangoSearchCaseType = "lower"
+	// ArangoSearchCaseNone to not change character case (default)
+	ArangoSearchCaseNone ArangoSearchCaseType = "none"
+)
+
+// ArangoSearchAnalyzerProperties specifies options for the analyzer. Which fields are required and
+// respected depends on the analyzer type.
+// more information can be found here: https://www.arangodb.com/docs/stable/arangosearch-analyzers.html#analyzer-properties
+type ArangoSearchAnalyzerProperties struct {
+	// Locale used by Stem, Norm, Text
+	Locale string `json:"locale,omitempty"`
+	// Delimiter used by Delimiter
+	Delimiter string `json:"delimiter,omitempty"`
+	// Accent used by Norm, Text
+	Accent *bool `json:"accent,omitempty"`
+	// Case used by Norm, Text
+	Case ArangoSearchCaseType `json:"case,omitempty"`
+
+	// Min used by NGram
+	Min *int64 `json:"min,omitempty"`
+	// Max used by NGram
+	Max *int64 `json:"max,omitempty"`
+	// PreserveOriginal used by NGram
+	PreserveOriginal *int64 `json:"preserveOriginal,omitempty"`
+
+	// Stemming used by Text
+	Stemming *bool `json:"stemming,omitempty"`
+	// Stopword used by Text
+	Stopwords []string `json:"stopwords,omitempty"`
+	// StopwordsPath used by Text
+	StopwordsPath []string `json:"stopwordsPath,omitempty"`
+}
+
+// ArangoSearchAnalyzerDefinition provides definition of an analyzer
+type ArangoSearchAnalyzerDefinition struct {
+	Name       string                         `json:"name,omitempty"`
+	Type       ArangoSearchAnalyzerType       `json:"type,omitempty"`
+	Properties ArangoSearchAnalyzerProperties `json:"properties,omitempty"`
+	Features   []ArangoSearchAnalyzerFeature  `json:"features,omitempty"`
+}
+
 // ArangoSearchViewProperties contains properties an an ArangoSearch view.
 type ArangoSearchViewProperties struct {
 	// CleanupIntervalStep specifies the minimum number of commits to wait between
@@ -96,7 +173,9 @@ type ArangoSearchViewProperties struct {
 type ArangoSearchSortDirection string
 
 const (
-	ArangoSearchSortDirectionAsc  ArangoSearchSortDirection = "ASC"
+	// ArangoSearchSortDirectionAsc sort ascending
+	ArangoSearchSortDirectionAsc ArangoSearchSortDirection = "ASC"
+	// ArangoSearchSortDirectionDesc sort descending
 	ArangoSearchSortDirectionDesc ArangoSearchSortDirection = "DESC"
 )
 
@@ -184,6 +263,7 @@ type ArangoSearchFields map[string]ArangoSearchElementProperties
 // Note that this structure is recursive. Settings not specified (nil)
 // at a given level will inherit their setting from a lower level.
 type ArangoSearchElementProperties struct {
+	AnalyzerDefinitions []ArangoSearchAnalyzerDefinition `json:"analyzerDefinitions,omitempty"`
 	// The list of analyzers to be used for indexing of string values. Defaults to ["identify"].
 	Analyzers []string `json:"analyzers,omitempty"`
 	// If set to true, all fields of this element will be indexed. Defaults to false.