Merge pull request #114 from lukaszbudnik/dev-v2020.1.3

v2020.1.3

README.md

@@ -70,7 +70,7 @@ Sample HTTP response:
 
 API v2 was introduced in migrator v2020.1.0. API v2 is a GraphQL API.
 
-API v2 also introduced a formal concept of DB versions. Every migrator action creates a new DB version. Version logically groups all applied DB migrations for auditing and compliance purposes. You can browse versions together with executed DB migrations using the GraphQL API.
+API v2 introduced a formal concept of a DB version. Every migrator action creates a new DB version. Version logically groups all applied DB migrations for auditing and compliance purposes. You can browse versions together with executed DB migrations using the GraphQL API.
 
 ## GET /v2/config
 
@@ -113,31 +113,6 @@ Sample request:
 curl -v http://localhost:8080/v2/schema
 ```
 
-Sample HTTP response (truncated):
-
-```
-< HTTP/1.1 200 OK
-< Content-Type: text/plain; charset=utf-8
-< Date: Mon, 02 Mar 2020 20:12:20 GMT
-< Transfer-Encoding: chunked
-<
-schema {
-  query: Query
-  mutation: Mutation
-}
-enum MigrationType {
-  SingleMigration
-  TenantMigration
-  SingleScript
-  TenantScript
-}
-...
-```
-
-## POST /v2/service
-
-This is a GraphQL endpoint which handles both query and mutation requests.
-
 The API v2 GraphQL schema and its description is as follows:
 
 ```graphql
@@ -273,9 +248,13 @@ type Mutation {
 }
 ```
 
+## POST /v2/service
+
+This is a GraphQL endpoint which handles both query and mutation requests.
+
 There are code generators available which can generate client code based on GraphQL schema. This would be the preferred way of consuming migrator's GraphQL endpoint.
 
-In Quick Start Guide there are a few curl examples to get you started.
+In [Quick Start Guide](#quick-start-guide) there are a few curl examples to get you started.
 
 ## /v1 - REST API
 
@@ -516,9 +495,11 @@ port: 8080
 # then all HTTP requests should be prefixed with that path, for example: /migrator/v1/config, /migrator/v1/migrations/source, etc.
 pathPrefix: /migrator
 # the webhook configuration section is optional
-# URL and template are required if at least one of them is empty noop notifier is used
-# the default content type header sent is application/json (can be overridden via webHookHeaders below)
+# the default Content-Type header is application/json but can be overridden via webHookHeaders below
 webHookURL: https://your.server.com/services/TTT/BBB/XXX
+# if the webhook expects a payload in a specific format there is an option to provide a payload template
+# see webhook template for more information
+webHookTemplate: '{"text": "New version: ${summary.versionId} started at: ${summary.startedAt} and took ${summary.duration}. Full results are: ${summary}"}'
 # should you need more control over HTTP headers use below
 webHookHeaders:
   - "Authorization: Basic QWxhZGRpbjpPcGVuU2VzYW1l"
@@ -536,6 +517,21 @@ webHookHeaders:
   - "X-Security-Token: ${SECURITY_TOKEN}"
 ```
 
+## WebHook template
+
+By default when a webhook is configured migrator will post a JSON representation of `Summary` struct to its endpoint.
+
+If your webhook expects a payload in a specific format (say Slack or MS Teams incoming webhooks) there is an option to configure a `webHookTemplate` property in migrator's configuration file. The template can have the following placeholders:
+
+* `${summary}` - will be replaced by a JSON representation of `Summary` struct, all double quotes will be escaped so that the template remains a valid JSON document
+* `${summary.field}` - will be replaced by a given field of `Summary` struct
+
+Placeholders can be mixed:
+
+```yaml
+webHookTemplate: '{"text": "New version created: ${summary.versionId} started at: ${summary.startedAt} and took ${summary.duration}. Migrations/scripts total: ${summary.migrationsGrandTotal}/${summary.scriptsGrandTotal}. Full results are: ${summary}"}'
+```
+
 ## Source migrations
 
 Migrations can be read from local disk, AWS S3, Azure Blob Containers. I'm open to contributions to add more cloud storage options.
@@ -634,7 +630,7 @@ schemaPlaceHolder: :tenant
 
 Before switching from a legacy tool you need to synchronise source migrations to migrator. migrator has no knowledge of migrations applied by other tools and as such will attempt to apply all found source migrations.
 
-Synchronising will load all source migrations and mark them as applied. This can be done by `CreateVersion` operation with `action: "Sync"`.
+Synchronising will load all source migrations and mark them as applied. This can be done by `CreateVersion` operation with action set to `Sync`.
 
 Once the initial synchronisation is done you can use migrator for all the consecutive DB migrations.
 

codecov.yml

@@ -0,0 +1,6 @@
+coverage:
+    status:
+      project:
+        default:
+          threshold: 5%
+          only_pulls: true

config/config.go

@@ -28,6 +28,7 @@ type Config struct {
 	PathPrefix        string   `yaml:"pathPrefix,omitempty"`
 	WebHookURL        string   `yaml:"webHookURL,omitempty"`
 	WebHookHeaders    []string `yaml:"webHookHeaders,omitempty"`
+	WebHookTemplate   string   `yaml:"webHookTemplate,omitempty"`
 }
 
 func (config Config) String() string {

config/config_test.go

@@ -57,7 +57,7 @@ func TestWithEnvFromFile(t *testing.T) {
 }
 
 func TestConfigString(t *testing.T) {
-	config := &Config{"", "/opt/app/migrations", "postgres", "user=p dbname=db host=localhost", "select abc", "insert into table", ":tenant", []string{"ref"}, []string{"tenants"}, []string{"procedures"}, []string{}, "8181", "", "https://hooks.slack.com/services/TTT/BBB/XXX", []string{}}
+	config := &Config{"", "/opt/app/migrations", "postgres", "user=p dbname=db host=localhost", "select abc", "insert into table", ":tenant", []string{"ref"}, []string{"tenants"}, []string{"procedures"}, []string{}, "8181", "", "https://hooks.slack.com/services/TTT/BBB/XXX", []string{}, `{"text": "Results are: ${summary}"}`}
 	// check if go naming convention applies
 	expected := `baseLocation: /opt/app/migrations
 driver: postgres
@@ -72,7 +72,8 @@ tenantMigrations:
 singleScripts:
 - procedures
 port: "8181"
-webHookURL: https://hooks.slack.com/services/TTT/BBB/XXX`
+webHookURL: https://hooks.slack.com/services/TTT/BBB/XXX
+webHookTemplate: '{"text": "Results are: ${summary}"}'`
 	actual := fmt.Sprintf("%v", config)
 	assert.Equal(t, expected, actual)
 }

coordinator/coordinator.go

@@ -2,7 +2,6 @@ package coordinator
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"reflect"
 
@@ -303,9 +302,7 @@ func (c *coordinator) filterTenantMigrations(sourceMigrations []types.Migration)
 // errors are silently discarded, adding tenant or applying migrations
 // must not fail because of notification error
 func (c *coordinator) sendNotification(results *types.MigrationResults) {
-	bytes, _ := json.Marshal(results)
-	text := string(bytes)
-	if resp, err := c.notifier.Notify(text); err != nil {
+	if resp, err := c.notifier.Notify(results); err != nil {
 		common.LogError(c.ctx, "Notifier error: %v", err.Error())
 	} else {
 		common.LogInfo(c.ctx, "Notifier response: %v", resp)

coordinator/coordinator_mocks.go

@@ -40,7 +40,7 @@ type mockedNotifier struct {
 	returnError bool
 }
 
-func (m *mockedNotifier) Notify(message string) (string, error) {
+func (m *mockedNotifier) Notify(summary *types.Summary) (string, error) {
 	if m.returnError {
 		return "", errors.New("algo salió terriblemente mal")
 	}

db/db.go

@@ -354,8 +354,8 @@ func (bc *baseConnector) CreateVersion(versionName string, action types.Action,
 		}
 	}()
 
-	results, versionID := bc.applyMigrationsInTx(tx, versionName, action, tenants, migrations)
-	version := bc.getVersionByIDInTx(tx, int32(versionID))
+	results := bc.applyMigrationsInTx(tx, versionName, action, tenants, migrations)
+	version := bc.getVersionByIDInTx(tx, results.VersionID)
 
 	return results, version
 }
@@ -404,9 +404,9 @@ func (bc *baseConnector) CreateTenant(versionName string, action types.Action, d
 	}
 
 	tenantStruct := types.Tenant{Name: tenant}
-	results, versionID := bc.applyMigrationsInTx(tx, versionName, action, []types.Tenant{tenantStruct}, migrations)
+	results := bc.applyMigrationsInTx(tx, versionName, action, []types.Tenant{tenantStruct}, migrations)
 
-	version := bc.getVersionByIDInTx(tx, int32(versionID))
+	version := bc.getVersionByIDInTx(tx, results.VersionID)
 
 	return results, version
 }
@@ -437,7 +437,7 @@ func (bc *baseConnector) getSchemaPlaceHolder() string {
 	return schemaPlaceHolder
 }
 
-func (bc *baseConnector) applyMigrationsInTx(tx *sql.Tx, versionName string, action types.Action, tenants []types.Tenant, migrations []types.Migration) (*types.MigrationResults, int64) {
+func (bc *baseConnector) applyMigrationsInTx(tx *sql.Tx, versionName string, action types.Action, tenants []types.Tenant, migrations []types.Migration) *types.MigrationResults {
 
 	results := &types.MigrationResults{
 		StartedAt: graphql.Time{Time: time.Now()},
@@ -514,5 +514,7 @@ func (bc *baseConnector) applyMigrationsInTx(tx *sql.Tx, versionName string, act
 
 	}
 
-	return results, versionID
+	results.VersionID = int32(versionID)
+
+	return results
 }

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/lib/pq v1.3.0
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/stretchr/testify v1.4.0
+	github.com/thedevsaddam/gojsonq/v2 v2.5.2
 	golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect
 	gopkg.in/go-playground/validator.v9 v9.29.1
 	gopkg.in/yaml.v2 v2.2.8

go.sum

@@ -63,10 +63,15 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/thedevsaddam/gojsonq v1.9.1 h1:zQulEP43nwmq5EKrNWyIgJVbqDeMdC1qzXM/f5O15a0=
+github.com/thedevsaddam/gojsonq v2.3.0+incompatible h1:i2lFTvGY4LvoZ2VUzedsFlRiyaWcJm3Uh6cQ9+HyQA8=
+github.com/thedevsaddam/gojsonq/v2 v2.5.2 h1:CoMVaYyKFsVj6TjU6APqAhAvC07hTI6IQen8PHzHYY0=
+github.com/thedevsaddam/gojsonq/v2 v2.5.2/go.mod h1:bv6Xa7kWy82uT0LnXPE2SzGqTj33TAEeR560MdJkiXs=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=

notifications/notifications.go

@@ -3,32 +3,60 @@ package notifications
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"regexp"
 	"strings"
 
+	gojsonq "github.com/thedevsaddam/gojsonq/v2"
+
 	"github.com/lukaszbudnik/migrator/config"
+	"github.com/lukaszbudnik/migrator/types"
 )
 
 const (
 	defaultContentType = "application/json"
-	textPlaceHolder    = "{text}"
 )
 
 // Notifier interface abstracts all notifications performed by migrator
 type Notifier interface {
-	Notify(string) (string, error)
+	Notify(*types.Summary) (string, error)
 }
 
 // baseNotifier type is a base struct embedded by all implementations of Notifier interface
 type baseNotifier struct {
 	config *config.Config
 }
 
-func (bn *baseNotifier) Notify(message string) (string, error) {
-	reader := bytes.NewReader([]byte(message))
+func (bn *baseNotifier) Notify(summary *types.Summary) (string, error) {
+	summaryJSON, err := json.MarshalIndent(summary, "", "    ")
+	if err != nil {
+		return "", err
+	}
+
+	payload := string(summaryJSON)
+
+	if template := bn.config.WebHookTemplate; len(template) > 0 {
+		// ${summary} will be replaced with the JSON object
+		if strings.Contains(template, "${summary}") {
+			template = strings.Replace(template, "${summary}", strings.ReplaceAll(payload, "\"", "\\\""), -1)
+		}
+		// migrator also supports parsing individual fields using ${summary.field} syntax
+		if strings.Contains(template, "${summary.") {
+			r, _ := regexp.Compile("\\${summary.([a-zA-Z]+)}")
+			matches := r.FindAllStringSubmatch(template, -1)
+			for _, m := range matches {
+				value := gojsonq.New().FromString(payload).Find(m[1])
+				valueString := fmt.Sprintf("%v", value)
+				template = strings.Replace(template, m[0], valueString, -1)
+			}
+		}
+		payload = template
+	}
 
+	reader := bytes.NewReader([]byte(payload))
 	url := bn.config.WebHookURL
 
 	req, err := http.NewRequest(http.MethodPost, url, reader)
@@ -65,7 +93,7 @@ type noopNotifier struct {
 	baseNotifier
 }
 
-func (sn *noopNotifier) Notify(text string) (string, error) {
+func (sn *noopNotifier) Notify(summary *types.Summary) (string, error) {
 	return "noop", nil
 }