From d5d830289e4b21f7f68b52edab6e3f4fcde3f9e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Tue, 29 Sep 2020 16:09:56 +0200 Subject: [PATCH 1/6] adding builds for go 1.14 and 1.15 on Travis, updating go modules --- .travis.yml | 2 ++ go.mod | 2 +- go.sum | 28 ++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 84e7739..dd153b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,8 @@ services: go: - "1.13" + - "1.14" + - "1.15" before_script: - sh -c "if [ '$DB' = 'postgresql' ]; then psql -U postgres -c 'create database migrator_test'; fi" diff --git a/go.mod b/go.mod index 8bc27a1..aeaef52 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/lukaszbudnik/migrator -go 1.12 +go 1.13 require ( github.com/Azure/azure-storage-blob-go v0.8.0 diff --git a/go.sum b/go.sum index ac8caf6..87b0993 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ +github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= @@ -10,59 +12,85 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/aws/aws-sdk-go v1.28.14 h1:ZeFS5GVtsJMZ0TBJ5n4HYwB/4MpY0hWkRthNNZkIzNo= github.com/aws/aws-sdk-go v1.28.14/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e h1:LzwWXEScfcTu7vUZNlDDWDARoSGEtvlDKK2BYHowNeE= github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.5.0 h1:fi+bqFAx/oLK54somfCtEZs9HeH1LHVoEPUgARpTqyc= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/graph-gophers/graphql-go v0.0.0-20200207002730-8334863f2c8b h1:fRjb9ncV+Aad/w56TstaCM/xGusFsfDfeGhhc+k4IBg= github.com/graph-gophers/graphql-go v0.0.0-20200207002730-8334863f2c8b/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 873a63c2b9fd62e31291b29821c4093cba8c930e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Sat, 3 Oct 2020 00:11:58 +0200 Subject: [PATCH 2/6] simplifies running tests, docker-compose to provision 5 different database containers, fixes #98 --- .travis.yml | 22 +- APIv1.md | 309 +++++++++ README.md | 596 ++++-------------- db/db_dialect_test.go | 16 +- db/db_integration_test.go | 323 ++++++++++ db/db_mssql_test.go | 22 +- db/db_mysql_test.go | 16 +- db/db_postgresql_test.go | 16 +- db/db_test.go | 233 +------ loader/azureblob_loader_test.go | 9 +- server/server_test.go | 2 +- test/docker-compose.yaml | 104 +++ test/docker/create-and-setup-container.sh | 29 - test/docker/destroy-container.sh | 15 - test/docker/scripts/destroy-container.sh | 7 - .../mssql-create-and-setup-container.sh | 32 - .../mysql-create-and-setup-container.sh | 32 - .../postgresql-create-and-setup-container.sh | 31 - test/migrator-docker.yaml | 8 + test/migrator-mariadb.yaml | 8 + test/migrator-mssql.yaml | 2 +- test/migrator-mysql.yaml | 2 +- test/migrator-percona.yaml | 8 + test/migrator-postgresql.yaml | 2 +- test/performance/generate-test-migrations.sh | 21 +- test/performance/migrator-performance.yaml | 8 + test/performance/test.sh | 75 ++- ultimate-coverage.sh | 26 - 28 files changed, 1012 insertions(+), 962 deletions(-) create mode 100644 APIv1.md create mode 100644 db/db_integration_test.go create mode 100644 test/docker-compose.yaml delete mode 100755 test/docker/create-and-setup-container.sh delete mode 100755 test/docker/destroy-container.sh delete mode 100644 test/docker/scripts/destroy-container.sh delete mode 100644 test/docker/scripts/mssql-create-and-setup-container.sh delete mode 100644 test/docker/scripts/mysql-create-and-setup-container.sh delete mode 100644 test/docker/scripts/postgresql-create-and-setup-container.sh create mode 100644 test/migrator-docker.yaml create mode 100644 test/migrator-mariadb.yaml create mode 100644 test/migrator-percona.yaml create mode 100644 test/performance/migrator-performance.yaml delete mode 100755 ultimate-coverage.sh diff --git a/.travis.yml b/.travis.yml index dd153b6..c4f8e67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,11 @@ language: go -addons: - postgresql: '9.6' +services: + - docker env: - matrix: - - DB=postgresql - - DB=mysql - global: - - secure: Wvmf5FAySAGXuugkWg/lNQuPfTDA7PYlZB8Izt+bhaVZDp3Re0xz8rhomUw/ZXP/2r81HHPTvDU+eNHxpneyBeHvSXvvhzMVyU+GFLeXdHA0rjvKittkkKq5XuUkj5oaSYJIh0YRn2UdfMaQ/R+y8ZgL2uP41slG6jWpfuMeFe+oqvJCgJ7qovQ7xUataDQdGCyWCpsTlBdpGz+YFUHNrcMK3ppvm4WhzDeMGXHPuTPcWZ+edO8QVWJkxVXamuWgq95tFB3EAXx5tfcg4CS5cC/l/K2tMH7MTsXvWOCRSU+f79cSNlrztP1Mhc/TPBZe87ubmkLOQGRK//ZBqbmSXjsyjv4qkq3z223Dmg+MRkt6ip01zsSVE6vmEvtVHGxtj8VKfWbwWusAXuLa1gtzXa/vVgisF++aj1ZEhApH5mu95gMqy9tOY/mX4Pm4iqalLujvlzVHKDDCmZsyF2O8mZCL2cDH7xlQTLeq9eyRPXNcm13WDDAcOhgVSpWNVObggUMZ67YDGV0yPQKGQBOCCqV3jCKgii/4m7n1H2mn1b8VpYOdK3ncA3U3bcHDPy3Nh44EmdJnj7QiUZtN+RyQGCz83h3X2owGjNG3qvGxrBPy87DbQU+FTN88zB4DdgO/TOYEzg3Wbz24D3thZUDR8rWvu9t9/PvH+CroTIpW4Iw= - - secure: d5+t1ysBsHPGKvrbxsN4M+4ns4Q3YHgAk3gapaPNxlX3pDKLgZ1Kd/XBnQ3dMSCP7FWejgmeLDxloGQMeieq6exEM9kmf8Ui/oFB97j2+ODXPv6+X69+aDDGMT94J77v5DwdJzyULI4vn3ZF2V6fM5/tlle4hf4uwCBNh30Svar7ZXsp/aZL7wpZzUG+efLFJJZ4oSD2NteWR9IeZbNqQtV0fUTHLk3g4Suze6Yt2OmZiFH4pn8Ihftjyr84nke65aPws8XKzoaVmghoZXmKNPNyiH3Dyu6uyGa2OhMspj0zjXHP+sMlgMrp5cvAGchRBz9NPL18VHtgID/hEPLfwb6ZZ6HypgXu5+4/O/J2vGHTFV0q2aluNBJi+sTzklYkHoCiBXtjdZxI/+mwK1VIeMejrhSjNtzdkryEbOQM/CoBcDacQIw4gUOloCG4AcFNXzQZifHZckIgc4JChYmNHhpEu3U1Zil8H7QDQ7MGqJG+HWuK/s/leoJx112jHLmCgPsq9hmMozDdQi8rVSS8Eo6bg7NTOuE9LK/uoweF31NJYITYV3vxmqluwT7+yn7V9avXRFTV1qTcavsjAYkMS3PlLX4f+w1HtZUIOjtdABcK5mg1JjZ92xHmNFudu8b2n6j3A3nOYuA0NsX5BlCuIG84MozNsMleOl6svbRKW+4= - -services: - - postgresql - - mysql + - secure: Wvmf5FAySAGXuugkWg/lNQuPfTDA7PYlZB8Izt+bhaVZDp3Re0xz8rhomUw/ZXP/2r81HHPTvDU+eNHxpneyBeHvSXvvhzMVyU+GFLeXdHA0rjvKittkkKq5XuUkj5oaSYJIh0YRn2UdfMaQ/R+y8ZgL2uP41slG6jWpfuMeFe+oqvJCgJ7qovQ7xUataDQdGCyWCpsTlBdpGz+YFUHNrcMK3ppvm4WhzDeMGXHPuTPcWZ+edO8QVWJkxVXamuWgq95tFB3EAXx5tfcg4CS5cC/l/K2tMH7MTsXvWOCRSU+f79cSNlrztP1Mhc/TPBZe87ubmkLOQGRK//ZBqbmSXjsyjv4qkq3z223Dmg+MRkt6ip01zsSVE6vmEvtVHGxtj8VKfWbwWusAXuLa1gtzXa/vVgisF++aj1ZEhApH5mu95gMqy9tOY/mX4Pm4iqalLujvlzVHKDDCmZsyF2O8mZCL2cDH7xlQTLeq9eyRPXNcm13WDDAcOhgVSpWNVObggUMZ67YDGV0yPQKGQBOCCqV3jCKgii/4m7n1H2mn1b8VpYOdK3ncA3U3bcHDPy3Nh44EmdJnj7QiUZtN+RyQGCz83h3X2owGjNG3qvGxrBPy87DbQU+FTN88zB4DdgO/TOYEzg3Wbz24D3thZUDR8rWvu9t9/PvH+CroTIpW4Iw= + - secure: d5+t1ysBsHPGKvrbxsN4M+4ns4Q3YHgAk3gapaPNxlX3pDKLgZ1Kd/XBnQ3dMSCP7FWejgmeLDxloGQMeieq6exEM9kmf8Ui/oFB97j2+ODXPv6+X69+aDDGMT94J77v5DwdJzyULI4vn3ZF2V6fM5/tlle4hf4uwCBNh30Svar7ZXsp/aZL7wpZzUG+efLFJJZ4oSD2NteWR9IeZbNqQtV0fUTHLk3g4Suze6Yt2OmZiFH4pn8Ihftjyr84nke65aPws8XKzoaVmghoZXmKNPNyiH3Dyu6uyGa2OhMspj0zjXHP+sMlgMrp5cvAGchRBz9NPL18VHtgID/hEPLfwb6ZZ6HypgXu5+4/O/J2vGHTFV0q2aluNBJi+sTzklYkHoCiBXtjdZxI/+mwK1VIeMejrhSjNtzdkryEbOQM/CoBcDacQIw4gUOloCG4AcFNXzQZifHZckIgc4JChYmNHhpEu3U1Zil8H7QDQ7MGqJG+HWuK/s/leoJx112jHLmCgPsq9hmMozDdQi8rVSS8Eo6bg7NTOuE9LK/uoweF31NJYITYV3vxmqluwT7+yn7V9avXRFTV1qTcavsjAYkMS3PlLX4f+w1HtZUIOjtdABcK5mg1JjZ92xHmNFudu8b2n6j3A3nOYuA0NsX5BlCuIG84MozNsMleOl6svbRKW+4= go: - "1.13" @@ -21,11 +13,7 @@ go: - "1.15" before_script: - - sh -c "if [ '$DB' = 'postgresql' ]; then psql -U postgres -c 'create database migrator_test'; fi" - - sh -c "if [ '$DB' = 'postgresql' ]; then psql -U postgres -d migrator_test -f test/create-test-tenants.sql; fi" - - sh -c "if [ '$DB' = 'mysql' ]; then mysql -u root -e 'create database migrator_test'; fi" - - sh -c "if [ '$DB' = 'mysql' ]; then mysql -u root -D migrator_test < test/create-test-tenants.sql; fi" - - sh -c "cp test/migrator-$DB.yaml.travis test/migrator.yaml" + - docker-compose -f test/docker-compose.yaml up script: - ./coverage.sh diff --git a/APIv1.md b/APIv1.md new file mode 100644 index 0000000..9249035 --- /dev/null +++ b/APIv1.md @@ -0,0 +1,309 @@ +## /v1 + +**As of migrator v2020.1.0 API v1 is deprecated and will sunset in v2021.1.0.** + +API v1 is available in migrator v4.x and v2020.x. + +## GET /v1/config + +Returns migrator's config as `application/x-yaml`. + +Sample request: + +``` +curl -v http://localhost:8080/v1/config +``` + +Sample HTTP response: + +``` +< HTTP/1.1 200 OK +< Content-Type: application/x-yaml; charset=utf-8 +< Date: Wed, 01 Jan 2020 17:31:57 GMT +< Content-Length: 277 + +baseLocation: test/migrations +driver: postgres +dataSource: user=postgres dbname=migrator_test host=127.0.0.1 port=32776 sslmode=disable + connect_timeout=1 +singleMigrations: +- ref +- config +tenantMigrations: +- tenants +singleScripts: +- config-scripts +tenantScripts: +- tenants-scripts +``` +## GET /v1/migrations/source + +Returns list of all source migrations. Response is a list of JSON representation of `Migration` struct. + +Sample request: + +``` +curl -v http://localhost:8080/v1/migrations/source +``` + +Sample HTTP response: + +``` +< HTTP/1.1 200 OK +< Content-Type: application/json; charset=utf-8 +< Date: Tue, 31 Dec 2019 11:27:48 GMT +< Transfer-Encoding: chunked + +[ + { + "name": "201602160002.sql", + "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config", + "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config/201602160002.sql", + "migrationType": 1, + "contents": "create table {schema}.config (\n id integer,\n k varchar(100),\n v varchar(100),\n primary key (id)\n);\n", + "checkSum": "58db38d8f6c197ab290212470a82fe1f5b1f3cacadbe00ac59cd68a3bfa98baf" + }, + { + "name": "201602160002.sql", + "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants", + "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants/201602160002.sql", + "migrationType": 2, + "contents": "create table {schema}.module (id integer, id_config integer, foreign key (id_config) references config.config(id));\n", + "checkSum": "56c4c1d8f82f3dedade5116be46267edee01a4889c6359ef03c39dc73ca653a8" + } +] +``` + +`Migration` JSON contains the following fields: + +* `name` - migration file name +* `sourceDir` - absolute path to source directory +* `file` - absolute path to migration file (concatenation of `sourceDir` and `name`) +* `migrationType` - type of migration, values are: + * 1 - single migration applied once for a given schema + * 2 - multi-tenant migration applied once but for all tenants/schemas + * 3 - single script - special type of migration applied always for a given schema + * 4 - multi-tenant script - special type of migration applied always for all tenants/schemas +* `contents` - contents of the migration file +* `checkSum` - sha256 checksum of migration file contents + +## GET /v1/migrations/applied + +Returns list of all applied migrations. Response is a list of JSON representation of `MigrationDB` struct. + +Sample request: + +``` +curl -v http://localhost:8080/v1/migrations/applied +``` + +Sample HTTP response: + +``` +< HTTP/1.1 200 OK +< Content-Type: application/json; charset=utf-8 +< Date: Wed, 01 Jan 2020 17:32:49 GMT +< Transfer-Encoding: chunked + +[ + { + "name": "201602160001.sql", + "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config", + "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config/201602160001.sql", + "migrationType": 1, + "contents": "create schema config;\n", + "checkSum": "c1380af7a054ec75778252f539e1e9f914d2c5b1f441ea1df18c2140c6c3380a", + "schema": "config", + "appliedAt": "2020-01-01T17:29:13.169306Z" + }, + { + "name": "201602160002.sql", + "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config", + "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config/201602160002.sql", + "migrationType": 1, + "contents": "create table {schema}.config (\n id integer,\n k varchar(100),\n v varchar(100),\n primary key (id)\n);\n", + "checkSum": "58db38d8f6c197ab290212470a82fe1f5b1f3cacadbe00ac59cd68a3bfa98baf", + "schema": "config", + "appliedAt": "2020-01-01T17:29:13.169306Z" + }, + { + "name": "201602160002.sql", + "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants", + "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants/201602160002.sql", + "migrationType": 2, + "contents": "create table {schema}.module (id integer, id_config integer, foreign key (id_config) references config.config(id));\n", + "checkSum": "56c4c1d8f82f3dedade5116be46267edee01a4889c6359ef03c39dc73ca653a8", + "schema": "abc", + "appliedAt": "2020-01-01T17:29:13.169306Z" + }, + { + "name": "201602160002.sql", + "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants", + "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants/201602160002.sql", + "migrationType": 2, + "contents": "create table {schema}.module (id integer, id_config integer, foreign key (id_config) references config.config(id));\n", + "checkSum": "56c4c1d8f82f3dedade5116be46267edee01a4889c6359ef03c39dc73ca653a8", + "schema": "def", + "appliedAt": "2020-01-01T17:29:13.169306Z" + }, + { + "name": "201602160002.sql", + "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants", + "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants/201602160002.sql", + "migrationType": 2, + "contents": "create table {schema}.module (id integer, id_config integer, foreign key (id_config) references config.config(id));\n", + "checkSum": "56c4c1d8f82f3dedade5116be46267edee01a4889c6359ef03c39dc73ca653a8", + "schema": "xyz", + "appliedAt": "2020-01-01T17:29:13.169306Z" + } +] +``` + +`MigrationDB` JSON contains all the fields from `Migration` struct and adds the following ones: + +* `schema` - schema for which given migration was applied, for single migrations this is equal to source dir name, for multi-tenant ones this is the name of the actual tenant schema +* `appliedAt` - date time migration was applied + + +## POST /v1/migrations + +Applies new source migrations to DB and returns summary results and a list of applied migrations. + +This operation requires as an input the following JSON payload: + +* `mode` - defines mode in which migrator will execute migrations, valid values are: + * `apply` - applies migrations + * `sync` - synchronises all source migrations with internal migrator's table, this action loads and marks all source migrations as applied but does not apply them + * `dry-run` - instead of calling commit, calls rollback at the end of the operation +* `response` - controls how much information is returned by migrator, valid values are: + * `full` - the response will contain both summary results and a list of applied migrations/scripts + * `list` - the response will contain both summary results and a list of applied migrations/scripts but without their contents (introduced in migrator `v4.1.1` and a part of API v1; does not break API v1 contract - existing integrations will continue to work) + * `summary` - the response will contain only summary results + +Sample request: + +``` +curl -v -X POST -H "Content-Type: application/json" -d '{"mode": "apply", "response": "full"}' http://localhost:8080/v1/migrations +``` + +Sample HTTP response: + +``` +{ + "results": { + "startedAt": "2020-01-01T18:29:13.14682+01:00", + "duration": 51637303, + "tenants": 3, + "singleMigrations": 4, + "tenantMigrations": 4, + "tenantMigrationsTotal": 12, + "migrationsGrandTotal": 16, + "singleScripts": 1, + "tenantScripts": 1, + "tenantScriptsTotal": 3, + "scriptsGrandTotal": 4 + }, + "appliedMigrations": [ + { + "name": "201602160001.sql", + "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config", + "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config/201602160001.sql", + "migrationType": 1, + "contents": "create schema config;\n", + "checkSum": "c1380af7a054ec75778252f539e1e9f914d2c5b1f441ea1df18c2140c6c3380a" + } + ] +} +``` + +`appliedMigrations` is a list of JSON representation of `Migration` struct as already described above. + +`results` is a JSON representation of `Results` struct. `Results` JSON contains the following fields: + +* `startedAt` - date time the operation started +* `duration` - how long the operation took in nanoseconds +* `tenants` - number of tenants in the system +* `singleMigrations` - number of identified and applied single migrations +* `tenantMigrations` - number of identified tenant migrations +* `tenantMigrationsTotal` - number of all tenant migrations applied (equals to `tenants` * `tenantMigrations`) +* `migrationsGrandTotal` - sum of `singleMigrations` and `tenantMigrationsTotal` +* `singleScripts` - number of read and applied single scripts +* `tenantScripts` - number of read tenant scripts +* `tenantScriptsTotal` - number of all tenant scripts applied (equals to `tenants` * `tenantScripts`) +* `scriptsGrandTotal` - sum of `singleScripts` and `tenantScriptsTotal` + + +## GET /v1/tenants + +Returns list of all tenants. + +Sample request: + +``` +curl -v http://localhost:8080/v1/tenants +``` + +Sample HTTP response: + + +``` +< HTTP/1.1 200 OK +< Content-Type: application/json; charset=utf-8 +< Date: Wed, 01 Jan 2020 17:16:09 GMT +< Content-Length: 58 + +["abc","def","xyz","new_test_tenant_1577793069634018000"] +``` + +## POST /v1/tenants + +Adds a new tenant and applies all tenant migrations and scripts for newly created tenant. Returns summary results and a list of applied migrations. + +This operation requires as an input the following JSON payload: + +* `name` - the name of the new tenant +* `mode` - same as `mode` for [POST /v1/migrations](#post-v1migrations) +* `response` - same as `response` for [POST /v1/migrations](#post-v1migrations) + +Sample request: + +``` +curl -v -X POST -H "Content-Type: application/json" -d '{"name": "new_test_tenant", "mode": "apply", "response": "full"}' http://localhost:8080/v1/tenants +``` + +Sample HTTP response. + +``` +< HTTP/1.1 200 OK +< Content-Type: application/json; charset=utf-8 +< Date: Wed, 01 Jan 2020 17:45:00 GMT +< Transfer-Encoding: chunked + +{ + "results": { + "startedAt": "2020-01-01T18:45:00.174152+01:00", + "duration": 12426788, + "tenants": 1, + "singleMigrations": 0, + "tenantMigrations": 4, + "tenantMigrationsTotal": 4, + "migrationsGrandTotal": 4, + "singleScripts": 0, + "tenantScripts": 1, + "tenantScriptsTotal": 1, + "scriptsGrandTotal": 1 + }, + "appliedMigrations": [ + { + "name": "201602160002.sql", + "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants", + "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants/201602160002.sql", + "migrationType": 2, + "contents": "create table {schema}.module (id integer, id_config integer, foreign key (id_config) references config.config(id));\n", + "checkSum": "56c4c1d8f82f3dedade5116be46267edee01a4889c6359ef03c39dc73ca653a8" + } + ] +} +``` + +The response is identical to the one of [POST /v1/migrations](#post-v1migrations). When adding new tenant only tenant migrations and scripts are applied and only for the newly created tenant. That is why `singleMigrations` and `singleScripts` are always 0 and `tenants` is always 1. \ No newline at end of file diff --git a/README.md b/README.md index f3502a6..4ae0c57 100644 --- a/README.md +++ b/README.md @@ -10,30 +10,19 @@ The official docker image is available on docker hub at [lukasz/migrator](https: # Table of contents -* [Usage](#usage) - * [GET /](#get-) +* [API](#api) * [/v2 - GraphQL API](#v2---graphql-api) * [GET /v2/config](#get-v2config) * [GET /v2/schema](#get-v2schema) * [POST /v2/service](#post-v2service) - * [/v1](#v1) - * [GET /v1/config](#get-v1config) - * [GET /v1/migrations/source](#get-v1migrationssource) - * [GET /v1/migrations/applied](#get-v1migrationsapplied) - * [POST /v1/migrations](#post-v1migrations) - * [GET /v1/tenants](#get-v1tenants) - * [POST /v1/tenants](#post-v1tenants) + * [/v1 - REST API](#v1---rest-api) * [Request tracing](#request-tracing) * [Quick Start Guide](#quick-start-guide) * [1. Get the migrator project](#1-get-the-migrator-project) - * [2. Setup test DB container](#2-setup-test-db-container) - * [3. Build and run migrator](#3-build-and-run-migrator) - * [4. Run migrator from official docker image](#4-run-migrator-from-official-docker-image) + * [2. Start test DB containers](#2-start-test-db-containers) + * [3. Run migrator from official docker image](#3-run-migrator-from-official-docker-image) + * [4. Build and run migrator](#4-build-and-run-migrator) * [5. Play around with migrator](#5-play-around-with-migrator) -* [Tutorials](#tutorials) - * [Deploying migrator to AWS ECS](#deploying-migrator-to-aws-ecs) - * [Deploying migrator to AWS EKS](#deploying-migrator-to-aws-eks) - * [Deploying migrator to Azure AKS](#deploying-migrator-to-azure-aks) * [Configuration](#configuration) * [migrator.yaml](#migratoryaml) * [Env variables substitution](#env-variables-substitution) @@ -47,20 +36,20 @@ The official docker image is available on docker hub at [lukasz/migrator](https: * [Custom schema placeholder](#custom-schema-placeholder) * [Synchonising legacy migrations to migrator](#synchonising-legacy-migrations-to-migrator) * [Final comments](#final-comments) +* [Tutorials](#tutorials) + * [Deploying migrator to AWS ECS](#deploying-migrator-to-aws-ecs) + * [Deploying migrator to AWS EKS](#deploying-migrator-to-aws-eks) + * [Deploying migrator to Azure AKS](#deploying-migrator-to-azure-aks) * [Performance](#performance) * [Change log](#change-log) * [Contributing, code style, running unit & integration tests](#contributing-code-style-running-unit--integration-tests) * [License](#license) -# Usage +# API -migrator exposes a simple REST API described below. +migrator exposes a REST and GraphQL APIs described below. -## GET / - -Migrator returns build information together with a list of supported API versions. - -Sample request: +To return build information together with a list of supported API versions execute: ``` curl -v http://localhost:8080/ @@ -149,7 +138,7 @@ enum MigrationType { This is a GraphQL endpoint which handles both query and mutation requests. -The current GraphQL schema together with description in comments is as follows: +The API v2 GraphQL schema and its description is as follows: ```graphql schema { @@ -286,9 +275,81 @@ type Mutation { There are code generators available which can generate client code based on GraphQL schema. This would be the preferred way of consuming migrator's GraphQL endpoint. -However, below are a few curl examples to get you started. +In Quick Start Guide there are a few curl examples to get you started. + +## /v1 - REST API + +**As of migrator v2020.1.0 API v1 is deprecated and will sunset in v2021.1.0.** + +API v1 is available in migrator v4.x and v2020.x. + +The documentation is available in a separate document [API v1](APIv1.md). + +## Request tracing + +migrator uses request tracing via `X-Request-ID` header. This header can be used with all requests for tracing and/or auditing purposes. If this header is absent migrator will generate one for you. + +# Quick Start Guide + +You can apply your first migrations with migrator in literally a couple of minutes. There are some test migrations which are located in `test` directory as well as docker-compose for setting up test databases. + +The quick start guide shows you how to either use the official docker image or build migrator locally. + +## 1. Get the migrator project + +Get the source code the usual go way: + +``` +go get -d -v github.com/lukaszbudnik/migrator +cd $GOPATH/src/github.com/lukaszbudnik/migrator +``` + +migrator aims to support 3 latest Go versions (built automatically on Travis). + +## 2. Start test DB containers + +Start and setup test DB containers: -Create new version: +``` +docker-compose -f ./test/dokcer-compose.yaml up +``` + +docker-compose will start and configure 5 different database containers (3 MySQL flavours, PostgreSQL, and MSSQL). Every database container has a ready-to-use migrator config in `test` directory. + +## 3. Run migrator from official docker image + +The official migrator docker image is available on docker hub [lukasz/migrator](https://hub.docker.com/r/lukasz/migrator). + +In order to run the docker container remember to: + +1. mount a volume with migrations, for example: `/data` +2. specify location of migrator configuration file, for convenience it is usually located under `/data` directory; it defaults to `/data/migrator.yaml` and can be overridden by setting environment variable `MIGRATOR_YAML` + +The docker-compose which starts test DB containers also starts latest migrator with sample configuration and test migrations. See `docker-compose.yaml` for details. + +## 4. Build and run migrator + +migrator uses go modules to manage dependencies. When building & running migrator from source code simply execute: + +``` +go build +./migrator -configFile test/migrator-postgresql.yaml +``` + +> Note: There are 3 git variables injected into the production build. When migrator is built like above it prints empty branch/tag, commit sha, and commit date. This is OK for local development. If you want to inject proper values take a look at `Dockerfile` for details. + +## 5. Play around with migrator + +If you started migrator in point 3 - migrator listens on port 8181 and connects to mysql server. +If you started migrator in point 4 - migrator listens on port 8080 and connects to postgresql server. + +Set the port accordingly: + +``` +MIGRATOR_PORT=8181 +``` + +Create new version, return version id and name together with operation summary: ``` # versionName parameter is required and can be: @@ -323,10 +384,10 @@ cat < create_version.txt } EOF # and now execute the above query -curl -d @create_version.txt http://localhost:8080/v2/service +curl -d @create_version.txt http://localhost:$MIGRATOR_PORT/v2/service ``` -Create new tenant, run in dry run and instead of default `Apply`, run `Sync` action, also include DB migrations in output: +Create new tenant, run in dry-run mode, run `Sync` action (instead of default `Apply`), return version id and name, DB migrations, and operation summary: ``` # versionName parameter is required and can be: @@ -362,6 +423,8 @@ cat < create_tenant.txt "operationName": "CreateTenant", "variables": { "input": { + "dryRun": true, + "action": "Sync", "versionName": "$COMMIT_SHA - $TENANT_NAME", "tenantName": "$TENANT_NAME" } @@ -369,10 +432,10 @@ cat < create_tenant.txt } EOF # and now execute the above query -curl -d @create_tenant.txt http://localhost:8080/v2/service +curl -d @create_tenant.txt http://localhost:$MIGRATOR_PORT/v2/service ``` -Query data (yes, migrator supports multiple operations in a single GraphQL query): +Migrator supports multiple operations in a single GraphQL query. Let's fetch source single migrations, source tenant migrations, and tenants in a single GraphQL query: ``` # new lines are used for readability but have to be removed from the actual request @@ -405,432 +468,11 @@ cat < query.txt } EOF # and now execute the above query -curl -d @query.txt http://localhost:8080/v2/service +curl -d @query.txt http://localhost:$MIGRATOR_PORT/v2/service ``` For more GraphQL query and mutation examples see `data/graphql_test.go`. -## /v1 - -**Deprecation**: As of migrator v2020.1.0 API v1 is deprecated and will sunset in v2021.1.0. - -API v1 is available in migrator v4.x and v2020.x. - -## GET /v1/config - -Returns migrator's config as `application/x-yaml`. - -Sample request: - -``` -curl -v http://localhost:8080/v1/config -``` - -Sample HTTP response: - -``` -< HTTP/1.1 200 OK -< Content-Type: application/x-yaml; charset=utf-8 -< Date: Wed, 01 Jan 2020 17:31:57 GMT -< Content-Length: 277 - -baseLocation: test/migrations -driver: postgres -dataSource: user=postgres dbname=migrator_test host=127.0.0.1 port=32776 sslmode=disable - connect_timeout=1 -singleMigrations: -- ref -- config -tenantMigrations: -- tenants -singleScripts: -- config-scripts -tenantScripts: -- tenants-scripts -``` -## GET /v1/migrations/source - -Returns list of all source migrations. Response is a list of JSON representation of `Migration` struct. - -Sample request: - -``` -curl -v http://localhost:8080/v1/migrations/source -``` - -Sample HTTP response: - -``` -< HTTP/1.1 200 OK -< Content-Type: application/json; charset=utf-8 -< Date: Tue, 31 Dec 2019 11:27:48 GMT -< Transfer-Encoding: chunked - -[ - { - "name": "201602160002.sql", - "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config", - "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config/201602160002.sql", - "migrationType": 1, - "contents": "create table {schema}.config (\n id integer,\n k varchar(100),\n v varchar(100),\n primary key (id)\n);\n", - "checkSum": "58db38d8f6c197ab290212470a82fe1f5b1f3cacadbe00ac59cd68a3bfa98baf" - }, - { - "name": "201602160002.sql", - "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants", - "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants/201602160002.sql", - "migrationType": 2, - "contents": "create table {schema}.module (id integer, id_config integer, foreign key (id_config) references config.config(id));\n", - "checkSum": "56c4c1d8f82f3dedade5116be46267edee01a4889c6359ef03c39dc73ca653a8" - } -] -``` - -`Migration` JSON contains the following fields: - -* `name` - migration file name -* `sourceDir` - absolute path to source directory -* `file` - absolute path to migration file (concatenation of `sourceDir` and `name`) -* `migrationType` - type of migration, values are: - * 1 - single migration applied once for a given schema - * 2 - multi-tenant migration applied once but for all tenants/schemas - * 3 - single script - special type of migration applied always for a given schema - * 4 - multi-tenant script - special type of migration applied always for all tenants/schemas -* `contents` - contents of the migration file -* `checkSum` - sha256 checksum of migration file contents - -## GET /v1/migrations/applied - -Returns list of all applied migrations. Response is a list of JSON representation of `MigrationDB` struct. - -Sample request: - -``` -curl -v http://localhost:8080/v1/migrations/applied -``` - -Sample HTTP response: - -``` -< HTTP/1.1 200 OK -< Content-Type: application/json; charset=utf-8 -< Date: Wed, 01 Jan 2020 17:32:49 GMT -< Transfer-Encoding: chunked - -[ - { - "name": "201602160001.sql", - "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config", - "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config/201602160001.sql", - "migrationType": 1, - "contents": "create schema config;\n", - "checkSum": "c1380af7a054ec75778252f539e1e9f914d2c5b1f441ea1df18c2140c6c3380a", - "schema": "config", - "appliedAt": "2020-01-01T17:29:13.169306Z" - }, - { - "name": "201602160002.sql", - "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config", - "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config/201602160002.sql", - "migrationType": 1, - "contents": "create table {schema}.config (\n id integer,\n k varchar(100),\n v varchar(100),\n primary key (id)\n);\n", - "checkSum": "58db38d8f6c197ab290212470a82fe1f5b1f3cacadbe00ac59cd68a3bfa98baf", - "schema": "config", - "appliedAt": "2020-01-01T17:29:13.169306Z" - }, - { - "name": "201602160002.sql", - "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants", - "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants/201602160002.sql", - "migrationType": 2, - "contents": "create table {schema}.module (id integer, id_config integer, foreign key (id_config) references config.config(id));\n", - "checkSum": "56c4c1d8f82f3dedade5116be46267edee01a4889c6359ef03c39dc73ca653a8", - "schema": "abc", - "appliedAt": "2020-01-01T17:29:13.169306Z" - }, - { - "name": "201602160002.sql", - "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants", - "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants/201602160002.sql", - "migrationType": 2, - "contents": "create table {schema}.module (id integer, id_config integer, foreign key (id_config) references config.config(id));\n", - "checkSum": "56c4c1d8f82f3dedade5116be46267edee01a4889c6359ef03c39dc73ca653a8", - "schema": "def", - "appliedAt": "2020-01-01T17:29:13.169306Z" - }, - { - "name": "201602160002.sql", - "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants", - "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants/201602160002.sql", - "migrationType": 2, - "contents": "create table {schema}.module (id integer, id_config integer, foreign key (id_config) references config.config(id));\n", - "checkSum": "56c4c1d8f82f3dedade5116be46267edee01a4889c6359ef03c39dc73ca653a8", - "schema": "xyz", - "appliedAt": "2020-01-01T17:29:13.169306Z" - } -] -``` - -`MigrationDB` JSON contains all the fields from `Migration` struct and adds the following ones: - -* `schema` - schema for which given migration was applied, for single migrations this is equal to source dir name, for multi-tenant ones this is the name of the actual tenant schema -* `appliedAt` - date time migration was applied - - -## POST /v1/migrations - -Applies new source migrations to DB and returns summary results and a list of applied migrations. - -This operation requires as an input the following JSON payload: - -* `mode` - defines mode in which migrator will execute migrations, valid values are: - * `apply` - applies migrations - * `sync` - synchronises all source migrations with internal migrator's table, this action loads and marks all source migrations as applied but does not apply them - * `dry-run` - instead of calling commit, calls rollback at the end of the operation -* `response` - controls how much information is returned by migrator, valid values are: - * `full` - the response will contain both summary results and a list of applied migrations/scripts - * `list` - the response will contain both summary results and a list of applied migrations/scripts but without their contents (introduced in migrator `v4.1.1` and a part of API v1; does not break API v1 contract - existing integrations will continue to work) - * `summary` - the response will contain only summary results - -Sample request: - -``` -curl -v -X POST -H "Content-Type: application/json" -d '{"mode": "apply", "response": "full"}' http://localhost:8080/v1/migrations -``` - -Sample HTTP response: - -``` -{ - "results": { - "startedAt": "2020-01-01T18:29:13.14682+01:00", - "duration": 51637303, - "tenants": 3, - "singleMigrations": 4, - "tenantMigrations": 4, - "tenantMigrationsTotal": 12, - "migrationsGrandTotal": 16, - "singleScripts": 1, - "tenantScripts": 1, - "tenantScriptsTotal": 3, - "scriptsGrandTotal": 4 - }, - "appliedMigrations": [ - { - "name": "201602160001.sql", - "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config", - "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/config/201602160001.sql", - "migrationType": 1, - "contents": "create schema config;\n", - "checkSum": "c1380af7a054ec75778252f539e1e9f914d2c5b1f441ea1df18c2140c6c3380a" - } - ] -} -``` - -`appliedMigrations` is a list of JSON representation of `Migration` struct as already described above. - -`results` is a JSON representation of `Results` struct. `Results` JSON contains the following fields: - -* `startedAt` - date time the operation started -* `duration` - how long the operation took in nanoseconds -* `tenants` - number of tenants in the system -* `singleMigrations` - number of identified and applied single migrations -* `tenantMigrations` - number of identified tenant migrations -* `tenantMigrationsTotal` - number of all tenant migrations applied (equals to `tenants` * `tenantMigrations`) -* `migrationsGrandTotal` - sum of `singleMigrations` and `tenantMigrationsTotal` -* `singleScripts` - number of read and applied single scripts -* `tenantScripts` - number of read tenant scripts -* `tenantScriptsTotal` - number of all tenant scripts applied (equals to `tenants` * `tenantScripts`) -* `scriptsGrandTotal` - sum of `singleScripts` and `tenantScriptsTotal` - - -## GET /v1/tenants - -Returns list of all tenants. - -Sample request: - -``` -curl -v http://localhost:8080/v1/tenants -``` - -Sample HTTP response: - - -``` -< HTTP/1.1 200 OK -< Content-Type: application/json; charset=utf-8 -< Date: Wed, 01 Jan 2020 17:16:09 GMT -< Content-Length: 58 - -["abc","def","xyz","new_test_tenant_1577793069634018000"] -``` - -## POST /v1/tenants - -Adds a new tenant and applies all tenant migrations and scripts for newly created tenant. Returns summary results and a list of applied migrations. - -This operation requires as an input the following JSON payload: - -* `name` - the name of the new tenant -* `mode` - same as `mode` for [POST /v1/migrations](#post-v1migrations) -* `response` - same as `response` for [POST /v1/migrations](#post-v1migrations) - -Sample request: - -``` -curl -v -X POST -H "Content-Type: application/json" -d '{"name": "new_test_tenant", "mode": "apply", "response": "full"}' http://localhost:8080/v1/tenants -``` - -Sample HTTP response. - -``` -< HTTP/1.1 200 OK -< Content-Type: application/json; charset=utf-8 -< Date: Wed, 01 Jan 2020 17:45:00 GMT -< Transfer-Encoding: chunked - -{ - "results": { - "startedAt": "2020-01-01T18:45:00.174152+01:00", - "duration": 12426788, - "tenants": 1, - "singleMigrations": 0, - "tenantMigrations": 4, - "tenantMigrationsTotal": 4, - "migrationsGrandTotal": 4, - "singleScripts": 0, - "tenantScripts": 1, - "tenantScriptsTotal": 1, - "scriptsGrandTotal": 1 - }, - "appliedMigrations": [ - { - "name": "201602160002.sql", - "sourceDir": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants", - "file": "/Users/lukasz/go/src/github.com/lukaszbudnik/migrator/test/migrations/tenants/201602160002.sql", - "migrationType": 2, - "contents": "create table {schema}.module (id integer, id_config integer, foreign key (id_config) references config.config(id));\n", - "checkSum": "56c4c1d8f82f3dedade5116be46267edee01a4889c6359ef03c39dc73ca653a8" - } - ] -} -``` - -The response is identical to the one of [POST /v1/migrations](#post-v1migrations). When adding new tenant only tenant migrations and scripts are applied and only for the newly created tenant. That is why `singleMigrations` and `singleScripts` are always 0 and `tenants` is always 1. - -## Request tracing - -migrator uses request tracing via `X-Request-ID` header. This header can be used with all requests for tracing and/or auditing purposes. If this header is absent migrator will generate one for you. - -# Quick Start Guide - -You can apply your first migrations with migrator in literally a couple of minutes. There are some test migrations which are located in `test` directory as well as some docker scripts for setting up test databases. - -The quick start guide shows you how to either build the migrator locally or use the official docker image. - -Steps 1 & 2 are required either way (migrator source code contains sample configuration & setup files together with some test migrations). -Step 3 is for building migrator locally, step 4 is for running the migrator container. -Step 5 is running examples and enjoying migrator ;) - -## 1. Get the migrator project - -Get the source code the usual go way: - -``` -go get -d -v github.com/lukaszbudnik/migrator -cd $GOPATH/src/github.com/lukaszbudnik/migrator -``` - -migrator aims to support 3 latest Go versions (built automatically on Travis). - -## 2. Setup test DB container - -migrator comes with helper scripts to setup test DB containers. Let's use postgres (see `ultimate-coverage.sh` for all supported containers). - -``` -./test/docker/create-and-setup-container.sh postgres -``` - -Script will start container called `migrator-postgres`. - -Further, apart of starting test DB container, the script also generates a ready-to-use test config file. We will use it later. - -## 3. Build and run migrator - -migrator uses go modules to manage dependencies. When building & running migrator from source code simply execute: - -``` -go build -./migrator -configFile test/migrator.yaml -``` - -> Note: There are 3 git variables injected into the production build (branch/tag together with commit sha & commit date). When migrator is built like above it prints empty branch/tag, commit sha and date. This is OK for local development. If you want to inject proper values take a look at `Dockerfile` for details. - -## 4. Run migrator from official docker image - -The official migrator docker image is available on docker hub [lukasz/migrator](https://hub.docker.com/r/lukasz/migrator). - -All migrator releases are automatically available as docker images on docker hub [lukasz/migrator/tags](https://hub.docker.com/r/lukasz/migrator/tags). - -To start a migrator container you need to: - -1. mount a volume with migrations, for example: `/data` -2. specify location of migrator configuration file, for convenience it is usually located under `/data` directory; it defaults to `/data/migrator.yaml` and can be overridden by setting environment variable `MIGRATOR_YAML` - -When running migrator from docker we need to update `migrator.yaml` (generated in step 2) as well as provide a link to `migrator-postgres` container: - -``` -sed -i "s/host=[^ ]* port=[^ ]*/host=migrator-postgres port=5432/g" test/migrator.yaml -sed -i "s/baseLocation: .*/baseLocation: \/data\/migrations/g" test/migrator.yaml -docker run --name migrator-test -p 8080:8080 -v $PWD/test:/data -e MIGRATOR_YAML=/data/migrator.yaml -d --link migrator-postgres lukasz/migrator -``` - -## 5. Play around with migrator - -Happy path: - -``` -curl -v http://localhost:8080/v1/config -curl -v http://localhost:8080/v1/migrations/source -curl -v http://localhost:8080/v1/tenants -curl -v http://localhost:8080/v1/migrations/applied -curl -v -X POST -H "Content-Type: application/json" -d '{"mode": "apply", "response": "full"}' http://localhost:8080/v1/migrations -curl -v -X POST -H "Content-Type: application/json" -d '{"name": "new_tenant", "mode": "apply", "response": "full"}' http://localhost:8080/v1/tenants -curl -v http://localhost:8080/v1/migrations/applied -``` - -And some errors. For example let's break a checksum of the first migration and try to apply migrations or add new tenant. - -``` -echo " " >> test/migrations/config/201602160001.sql -curl -v -X POST -H "Content-Type: application/json" -d '{"mode": "apply", "response": "full"}' http://localhost:8080/v1/migrations -curl -v -X POST -H "Content-Type: application/json" -d '{"name": "new_tenant", "mode": "apply", "response": "full"}' http://localhost:8080/v1/tenants -``` - -# Tutorials - -In this section I provide links to more in-depth migrator tutorials. - -## Deploying migrator to AWS ECS - -The goal of this tutorial is to deploy migrator to AWS ECS, load migrations from AWS S3 and apply them to AWS RDS DB while storing env variables securely in AWS Secrets Manager. The list of all AWS services used is: IAM, ECS, ECR, Secrets Manager, RDS, and S3. - -You can find it in [tutorials/aws-ecs](tutorials/aws-ecs). - -## Deploying migrator to AWS EKS - -The goal of this tutorial is to deploy migrator to AWS EKS, load migrations from AWS S3 and apply them to AWS RDS DB. The list of AWS services used is: IAM, EKS, ECR, RDS, and S3. - -You can find it in [tutorials/aws-eks](contrib/aws-eks). - -## Deploying migrator to Azure AKS - -The goal of this tutorial is to publish migrator image to Azure ACR private container repository, deploy migrator to Azure AKS, load migrations from Azure Blob Container and apply them to Azure Database for PostgreSQL. The list of Azure services used is: AKS, ACR, Blob Storage, and Azure Database for PostgreSQL. - -You can find it in [tutorials/azure-aks](tutorials/azure-aks). - # Configuration Let's see how to configure migrator. @@ -896,7 +538,7 @@ webHookHeaders: ## Source migrations -Migrations can be read either from local disk or from S3 (I'm open to contributions to add more cloud storage options). +Migrations can be read from local disk, AWS S3, Azure Blob Containers. I'm open to contributions to add more cloud storage options. ### Local storage @@ -960,7 +602,7 @@ If you have an existing way of storing information about your tenants you can co In the config file you need to provide 2 configuration properties: * `tenantSelectSQL` - a select statement which returns names of the tenants -* `tenantInsertSQL` - an insert statement which creates a new tenant entry, the insert statement should be a valid prepared statement for the SQL driver/database you use, it must accept the name of the new tenant as a parameter; finally should your table require additional columns you need to provide default values for them too +* `tenantInsertSQL` - an insert statement which creates a new tenant entry, the insert statement should be a valid prepared statement for the SQL driver/database you use, it must accept the name of the new tenant as a parameter; finally should your table require additional columns you need to provide default values for them Here is an example: @@ -990,26 +632,42 @@ schemaPlaceHolder: :tenant ## Synchonising legacy migrations to migrator -Before switching from a legacy framework to migrator you need to synchronise source migrations to migrator. - -This can be done using the POST /v1/migrations endpoint and setting the `mode` param to `sync`: - -``` -curl -v -X POST -H "Content-Type: application/json" -d '{"mode": "sync", "response": "full"}' http://localhost:8080/v1/migrations -``` +Before switching from a legacy tool you need to synchronise source migrations to migrator. migrator has no knowledge of migrations applied by other tools and as such will attempt to apply all found source migrations. -migrator will load and synchronise all source migrations with internal migrator's table, this action loads and marks all source migrations as applied but does not apply them. +Synchronising will load all source migrations and mark them as applied. This can be done by `CreateVersion` operation with `action: "Sync"`. -Once the initial sync is done you can move to migrator for all the consecutive DB migrations. +Once the initial synchronisation is done you can use migrator for all the consecutive DB migrations. ## Final comments When using migrator please remember that: -* migrator creates `migrator` schema together with `migrator_migrations` table automatically -* if you're not using [Custom tenants support](#custom-tenants-support) migrator creates `migrator_tenants` table automatically; just like `migrator_migrations` this table is created inside the `migrator` schema -* when adding a new tenant migrator creates a new DB schema and applies all tenant migrations and scripts - no need to apply them manually -* single schemas are not created automatically, you must add initial migration with `create schema {schema}` SQL statement (see examples above) +* migrator creates `migrator` schema together with `migrator_versions` and `migrator_migrations` tables automatically +* if you're not using [Custom tenants support](#custom-tenants-support) migrator creates `migrator_tenants` table automatically +* when adding a new tenant migrator creates a new DB schema and applies all tenant migrations and scripts +* single schemas are not created automatically, you must add initial migration with `create schema {schema}` SQL statement (see sample migrations in `test` folder) + +# Tutorials + +In this section I provide links to more in-depth migrator tutorials. + +## Deploying migrator to AWS ECS + +The goal of this tutorial is to deploy migrator to AWS ECS, load migrations from AWS S3 and apply them to AWS RDS DB while storing env variables securely in AWS Secrets Manager. The list of all AWS services used is: IAM, ECS, ECR, Secrets Manager, RDS, and S3. + +You can find it in [tutorials/aws-ecs](tutorials/aws-ecs). + +## Deploying migrator to AWS EKS + +The goal of this tutorial is to deploy migrator to AWS EKS, load migrations from AWS S3 and apply them to AWS RDS DB. The list of AWS services used is: IAM, EKS, ECR, RDS, and S3. + +You can find it in [tutorials/aws-eks](contrib/aws-eks). + +## Deploying migrator to Azure AKS + +The goal of this tutorial is to publish migrator image to Azure ACR private container repository, deploy migrator to Azure AKS, load migrations from Azure Blob Container and apply them to Azure Database for PostgreSQL. The list of Azure services used is: AKS, ACR, Blob Storage, and Azure Database for PostgreSQL. + +You can find it in [tutorials/azure-aks](tutorials/azure-aks). # Performance @@ -1049,10 +707,10 @@ Code should be formatted, checked, and tested using the following commands: ``` ./fmt-lint-vet.sh -./ultimate-coverage.sh +./coverage.sh ``` -The `ultimate-coverage.sh` script loops through 5 different containers (3 MySQL flavours, PostgreSQL, and MSSQL) creates db docker container, executes `coverage.sh` script, and finally tears down given db docker container. +The `db/db_integration_test.go` uses go subtests and runs all tests agains 5 different database containers (3 MySQL flavours, PostgreSQL, and MSSQL). # License diff --git a/db/db_dialect_test.go b/db/db_dialect_test.go index f64f474..7875741 100644 --- a/db/db_dialect_test.go +++ b/db/db_dialect_test.go @@ -8,11 +8,9 @@ import ( ) func TestBaseDialectGetCreateTenantsTableSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) - config.Driver = "postgres" - dialect := newDialect(config) createTenantsTableSQL := dialect.GetCreateTenantsTableSQL() @@ -29,11 +27,9 @@ create table if not exists migrator.migrator_tenants ( } func TestBaseDialectGetCreateMigrationsTableSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) - config.Driver = "postgres" - dialect := newDialect(config) createMigrationsTableSQL := dialect.GetCreateMigrationsTableSQL() @@ -56,11 +52,9 @@ create table if not exists migrator.migrator_migrations ( } func TestBaseDialectGetCreateSchemaSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) - config.Driver = "postgres" - dialect := newDialect(config) createSchemaSQL := dialect.GetCreateSchemaSQL("abc") @@ -71,11 +65,9 @@ func TestBaseDialectGetCreateSchemaSQL(t *testing.T) { } func TestBaseDialectGetVersionsSelectSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) - config.Driver = "postgres" - dialect := newDialect(config) versionsSelectSQL := dialect.GetVersionsSelectSQL() diff --git a/db/db_integration_test.go b/db/db_integration_test.go new file mode 100644 index 0000000..8539d06 --- /dev/null +++ b/db/db_integration_test.go @@ -0,0 +1,323 @@ +package db + +import ( + "fmt" + "testing" + "time" + + "github.com/lukaszbudnik/migrator/config" + "github.com/lukaszbudnik/migrator/types" + "github.com/stretchr/testify/assert" +) + +func getSupportedDatabases() []string { + return []string{"postgresql", "mysql", "mariadb", "percona", "mssql"} +} + +func TestGetTenants(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + connector := New(newTestContext(), config) + defer connector.Dispose() + + tenants := connector.GetTenants() + + assert.True(t, len(tenants) >= 3) + assert.Contains(t, tenants, types.Tenant{Name: "abc"}) + assert.Contains(t, tenants, types.Tenant{Name: "def"}) + assert.Contains(t, tenants, types.Tenant{Name: "xyz"}) + }) + } +} + +func TestCreateVersion(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + connector := New(newTestContext(), config) + defer connector.Dispose() + + tenants := connector.GetTenants() + noOfTenants := len(tenants) + + dbMigrationsBefore := connector.GetAppliedMigrations() + lenBefore := len(dbMigrationsBefore) + + p1 := time.Now().UnixNano() + p2 := time.Now().UnixNano() + p3 := time.Now().UnixNano() + p4 := time.Now().UnixNano() + p5 := time.Now().UnixNano() + t1 := time.Now().UnixNano() + t2 := time.Now().UnixNano() + t3 := time.Now().UnixNano() + t4 := time.Now().UnixNano() + + // public migrations + public1 := types.Migration{Name: fmt.Sprintf("%v.sql", p1), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p1), MigrationType: types.MigrationTypeSingleMigration, Contents: "drop table if exists modules"} + public2 := types.Migration{Name: fmt.Sprintf("%v.sql", p2), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p2), MigrationType: types.MigrationTypeSingleMigration, Contents: "create table modules ( k int, v text )"} + public3 := types.Migration{Name: fmt.Sprintf("%v.sql", p3), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p3), MigrationType: types.MigrationTypeSingleMigration, Contents: "insert into modules values ( 123, '123' )"} + + // public scripts + public4 := types.Migration{Name: fmt.Sprintf("%v.sql", p4), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p4), MigrationType: types.MigrationTypeSingleScript, Contents: "insert into modules values ( 1234, '1234' )"} + public5 := types.Migration{Name: fmt.Sprintf("%v.sql", p5), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p5), MigrationType: types.MigrationTypeSingleScript, Contents: "insert into modules values ( 12345, '12345' )"} + + // tenant migrations + tenant1 := types.Migration{Name: fmt.Sprintf("%v.sql", t1), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t1), MigrationType: types.MigrationTypeTenantMigration, Contents: "drop table if exists {schema}.settings"} + tenant2 := types.Migration{Name: fmt.Sprintf("%v.sql", t2), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t2), MigrationType: types.MigrationTypeTenantMigration, Contents: "create table {schema}.settings (k int, v text)"} + tenant3 := types.Migration{Name: fmt.Sprintf("%v.sql", t3), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t3), MigrationType: types.MigrationTypeTenantMigration, Contents: "insert into {schema}.settings values (456, '456') "} + + // tenant scripts + tenant4 := types.Migration{Name: fmt.Sprintf("%v.sql", t4), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t4), MigrationType: types.MigrationTypeTenantScript, Contents: "insert into {schema}.settings values (456, '456') "} + + migrationsToApply := []types.Migration{public1, public2, public3, tenant1, tenant2, tenant3, public4, public5, tenant4} + + results, version := connector.CreateVersion("commit-sha", types.ActionApply, false, migrationsToApply) + + assert.NotNil(t, version) + assert.True(t, version.ID > 0) + assert.Equal(t, "commit-sha", version.Name) + assert.Equal(t, results.MigrationsGrandTotal+results.ScriptsGrandTotal, int32(len(version.DBMigrations))) + assert.Equal(t, int32(noOfTenants), results.Tenants) + assert.Equal(t, int32(3), results.SingleMigrations) + assert.Equal(t, int32(2), results.SingleScripts) + assert.Equal(t, int32(3), results.TenantMigrations) + assert.Equal(t, int32(1), results.TenantScripts) + assert.Equal(t, int32(noOfTenants*3), results.TenantMigrationsTotal) + assert.Equal(t, int32(noOfTenants*1), results.TenantScriptsTotal) + assert.Equal(t, int32(noOfTenants*3+3), results.MigrationsGrandTotal) + assert.Equal(t, int32(noOfTenants*1+2), results.ScriptsGrandTotal) + + dbMigrationsAfter := connector.GetAppliedMigrations() + lenAfter := len(dbMigrationsAfter) + + // 3 tenant migrations * no of tenants + 3 public + // 1 tenant script * no of tenants + 2 public scripts + expected := (3*noOfTenants + 3) + (1*noOfTenants + 2) + assert.Equal(t, expected, lenAfter-lenBefore) + }) + } +} + +func TestCreateVersionEmptyMigrationArray(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + connector := New(newTestContext(), config) + defer connector.Dispose() + + migrationsToApply := []types.Migration{} + + results, version := connector.CreateVersion("commit-sha", types.ActionApply, false, migrationsToApply) + // empty migrations slice - no version created + assert.Nil(t, version) + assert.Equal(t, int32(0), results.MigrationsGrandTotal) + assert.Equal(t, int32(0), results.ScriptsGrandTotal) + }) + } +} + +func TestGetTenantsSQLDefault(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + dialect := newDialect(config) + connector := baseConnector{newTestContext(), config, dialect, nil} + defer connector.Dispose() + + tenantSelectSQL := connector.getTenantSelectSQL() + + assert.Equal(t, "select name from migrator.migrator_tenants", tenantSelectSQL) + }) + } +} + +func TestCreateTenant(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + connector := New(newTestContext(), config) + defer connector.Dispose() + + t1 := time.Now().UnixNano() + t2 := time.Now().UnixNano() + t3 := time.Now().UnixNano() + + tenant1 := types.Migration{Name: fmt.Sprintf("%v.sql", t1), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t1), MigrationType: types.MigrationTypeTenantMigration, Contents: "drop table if exists {schema}.settings"} + tenant2 := types.Migration{Name: fmt.Sprintf("%v.sql", t2), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t2), MigrationType: types.MigrationTypeTenantMigration, Contents: "create table {schema}.settings (k int, v text)"} + tenant3 := types.Migration{Name: fmt.Sprintf("%v.sql", t3), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t3), MigrationType: types.MigrationTypeTenantMigration, Contents: "insert into {schema}.settings values (456, '456')"} + + migrationsToApply := []types.Migration{tenant1, tenant2, tenant3} + + uniqueTenant := fmt.Sprintf("new_test_tenant_%v", time.Now().UnixNano()) + + results, version := connector.CreateTenant("commit-sha", types.ActionApply, false, uniqueTenant, migrationsToApply) + + assert.NotNil(t, version) + assert.True(t, version.ID > 0) + assert.Equal(t, "commit-sha", version.Name) + assert.Equal(t, results.MigrationsGrandTotal+results.ScriptsGrandTotal, int32(len(version.DBMigrations))) + + // applied only for one tenant - the newly added one + assert.Equal(t, int32(1), results.Tenants) + // just one tenant so total number of tenant migrations is equal to tenant migrations + assert.Equal(t, int32(3), results.TenantMigrations) + assert.Equal(t, int32(3), results.TenantMigrationsTotal) + }) + } +} + +func TestGetVersions(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + connector := New(newTestContext(), config) + defer connector.Dispose() + + versions := connector.GetVersions() + + assert.True(t, len(versions) >= 2) + // versions are sorted from newest (highest ID) to oldest (lowest ID) + assert.True(t, versions[0].ID > versions[1].ID) + }) + } +} + +func TestGetVersionsByFile(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + connector := New(newTestContext(), config) + defer connector.Dispose() + + versions := connector.GetVersions() + existingVersion := versions[0] + + versions = connector.GetVersionsByFile(versions[0].DBMigrations[0].File) + version := versions[0] + assert.Equal(t, existingVersion.ID, version.ID) + assert.Equal(t, existingVersion.DBMigrations[0].File, version.DBMigrations[0].File) + assert.True(t, len(version.DBMigrations) > 0) + }) + } +} + +func TestGetVersionByID(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + connector := New(newTestContext(), config) + defer connector.Dispose() + + versions := connector.GetVersions() + existingVersion := versions[0] + + version, err := connector.GetVersionByID(existingVersion.ID) + assert.Nil(t, err) + assert.Equal(t, existingVersion.ID, version.ID) + assert.True(t, len(version.DBMigrations) > 0) + }) + } +} + +func TestGetVersionByIDNotFound(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + connector := New(newTestContext(), config) + defer connector.Dispose() + + version, err := connector.GetVersionByID(-1) + assert.Nil(t, version) + assert.Equal(t, "Version not found ID: -1", err.Error()) + }) + } +} + +func TestGetDBMigrationByID(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + connector := New(newTestContext(), config) + defer connector.Dispose() + + versions := connector.GetVersions() + existingVersion := versions[0] + existingDBMigration := existingVersion.DBMigrations[0] + + dbMigration, err := connector.GetDBMigrationByID(existingDBMigration.ID) + assert.Nil(t, err) + assert.Equal(t, existingDBMigration.ID, dbMigration.ID) + }) + } +} + +func TestGetDBMigrationByIDNotFound(t *testing.T) { + supportedDatabases := getSupportedDatabases() + + for _, database := range supportedDatabases { + t.Run(database, func(t *testing.T) { + configFile := fmt.Sprintf("../test/migrator-%s.yaml", database) + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + connector := New(newTestContext(), config) + defer connector.Dispose() + + dbMigration, err := connector.GetDBMigrationByID(-1) + assert.Nil(t, dbMigration) + assert.Equal(t, "DB migration not found ID: -1", err.Error()) + }) + } +} diff --git a/db/db_mssql_test.go b/db/db_mssql_test.go index 922b491..88b96ba 100644 --- a/db/db_mssql_test.go +++ b/db/db_mssql_test.go @@ -15,7 +15,7 @@ func TestDBCreateDialectMSSQLDriver(t *testing.T) { } func TestMSSQLLastInsertIdSupported(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" @@ -27,7 +27,7 @@ func TestMSSQLLastInsertIdSupported(t *testing.T) { } func TestMSSQLGetMigrationInsertSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" @@ -40,7 +40,7 @@ func TestMSSQLGetMigrationInsertSQL(t *testing.T) { } func TestMSSQLGetTenantInsertSQLDefault(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" @@ -54,7 +54,7 @@ func TestMSSQLGetTenantInsertSQLDefault(t *testing.T) { } func TestMSSQLDialectGetCreateTenantsTableSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" @@ -78,7 +78,7 @@ END } func TestMSSQLDialectGetCreateMigrationsTableSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" @@ -108,7 +108,7 @@ END } func TestMSSQLDialectGetCreateSchemaSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" @@ -128,7 +128,7 @@ END } func TestMSSQLGetVersionInsertSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" @@ -140,7 +140,7 @@ func TestMSSQLGetVersionInsertSQL(t *testing.T) { } func TestMSSQLGetCreateVersionsTableSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" @@ -181,7 +181,7 @@ end } func TestMSSQLGetVersionsByFileSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" @@ -193,7 +193,7 @@ func TestMSSQLGetVersionsByFileSQL(t *testing.T) { } func TestMSSQLGetVersionByIDSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" @@ -205,7 +205,7 @@ func TestMSSQLGetVersionByIDSQL(t *testing.T) { } func TestMSSQLGetMigrationByIDSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mssql.yaml") assert.Nil(t, err) config.Driver = "sqlserver" diff --git a/db/db_mysql_test.go b/db/db_mysql_test.go index c30d807..106fa28 100644 --- a/db/db_mysql_test.go +++ b/db/db_mysql_test.go @@ -15,7 +15,7 @@ func TestDBCreateDialectMysqlDriver(t *testing.T) { } func TestMySQLLastInsertIdSupported(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mysql.yaml") assert.Nil(t, err) config.Driver = "mysql" @@ -26,7 +26,7 @@ func TestMySQLLastInsertIdSupported(t *testing.T) { } func TestMySQLGetMigrationInsertSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mysql.yaml") assert.Nil(t, err) config.Driver = "mysql" @@ -39,7 +39,7 @@ func TestMySQLGetMigrationInsertSQL(t *testing.T) { } func TestMySQLGetTenantInsertSQLDefault(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mysql.yaml") assert.Nil(t, err) config.Driver = "mysql" @@ -53,7 +53,7 @@ func TestMySQLGetTenantInsertSQLDefault(t *testing.T) { } func TestMySQLGetVersionInsertSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mysql.yaml") assert.Nil(t, err) config.Driver = "mysql" @@ -65,7 +65,7 @@ func TestMySQLGetVersionInsertSQL(t *testing.T) { } func TestMySQLGetCreateVersionsTableSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mysql.yaml") assert.Nil(t, err) config.Driver = "mysql" @@ -105,7 +105,7 @@ end; } func TestMySQLGetVersionsByFileSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mysql.yaml") assert.Nil(t, err) config.Driver = "mysql" @@ -117,7 +117,7 @@ func TestMySQLGetVersionsByFileSQL(t *testing.T) { } func TestMySQLGetVersionByIDSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mysql.yaml") assert.Nil(t, err) config.Driver = "mysql" @@ -129,7 +129,7 @@ func TestMySQLGetVersionByIDSQL(t *testing.T) { } func TestMySQLGetMigrationByIDSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-mysql.yaml") assert.Nil(t, err) config.Driver = "mysql" diff --git a/db/db_postgresql_test.go b/db/db_postgresql_test.go index 88372fb..86048d2 100644 --- a/db/db_postgresql_test.go +++ b/db/db_postgresql_test.go @@ -15,7 +15,7 @@ func TestDBCreateDialectPostgreSQLDriver(t *testing.T) { } func TestPostgreSQLLastInsertIdSupported(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) config.Driver = "postgres" @@ -26,7 +26,7 @@ func TestPostgreSQLLastInsertIdSupported(t *testing.T) { } func TestPostgreSQLGetMigrationInsertSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) config.Driver = "postgres" @@ -39,7 +39,7 @@ func TestPostgreSQLGetMigrationInsertSQL(t *testing.T) { } func TestPostgreSQLGetTenantInsertSQLDefault(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) config.Driver = "postgres" @@ -53,7 +53,7 @@ func TestPostgreSQLGetTenantInsertSQLDefault(t *testing.T) { } func TestPostgreSQLGetVersionInsertSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) config.Driver = "postgres" @@ -65,7 +65,7 @@ func TestPostgreSQLGetVersionInsertSQL(t *testing.T) { } func TestPostgreSQLGetCreateVersionsTableSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) config.Driver = "postgres" @@ -101,7 +101,7 @@ end $$; } func TestPostgreSQLGetVersionsByFileSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) config.Driver = "postgres" @@ -113,7 +113,7 @@ func TestPostgreSQLGetVersionsByFileSQL(t *testing.T) { } func TestPostgreSQLGetVersionByIDSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) config.Driver = "postgres" @@ -125,7 +125,7 @@ func TestPostgreSQLGetVersionByIDSQL(t *testing.T) { } func TestPostgreSQLGetMigrationByIDSQL(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) config.Driver = "postgres" diff --git a/db/db_test.go b/db/db_test.go index bbe202e..66e820e 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -14,11 +14,6 @@ import ( "github.com/stretchr/testify/assert" ) -var ( - existingVersion types.Version - existingDBMigration types.DBMigration -) - func newTestContext() context.Context { ctx := context.TODO() ctx = context.WithValue(ctx, common.RequestIDKey{}, time.Now().Nanosecond()) @@ -35,7 +30,7 @@ func TestDBCreateConnectorPanicUnknownDriver(t *testing.T) { } func TestConnectorInitPanicConnectionError(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) config.DataSource = strings.Replace(config.DataSource, "127.0.0.1", "1.0.0.1", -1) @@ -57,104 +52,6 @@ func TestConnectorInitPanicConnectionError(t *testing.T) { assert.Contains(t, message, "Failed to connect to database") } -func TestGetTenants(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - connector := New(newTestContext(), config) - defer connector.Dispose() - - tenants := connector.GetTenants() - - assert.True(t, len(tenants) >= 3) - assert.Contains(t, tenants, types.Tenant{Name: "abc"}) - assert.Contains(t, tenants, types.Tenant{Name: "def"}) - assert.Contains(t, tenants, types.Tenant{Name: "xyz"}) -} - -func TestCreateVersion(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - connector := New(newTestContext(), config) - defer connector.Dispose() - - tenants := connector.GetTenants() - noOfTenants := len(tenants) - - dbMigrationsBefore := connector.GetAppliedMigrations() - lenBefore := len(dbMigrationsBefore) - - p1 := time.Now().UnixNano() - p2 := time.Now().UnixNano() - p3 := time.Now().UnixNano() - p4 := time.Now().UnixNano() - p5 := time.Now().UnixNano() - t1 := time.Now().UnixNano() - t2 := time.Now().UnixNano() - t3 := time.Now().UnixNano() - t4 := time.Now().UnixNano() - - // public migrations - public1 := types.Migration{Name: fmt.Sprintf("%v.sql", p1), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p1), MigrationType: types.MigrationTypeSingleMigration, Contents: "drop table if exists modules"} - public2 := types.Migration{Name: fmt.Sprintf("%v.sql", p2), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p2), MigrationType: types.MigrationTypeSingleMigration, Contents: "create table modules ( k int, v text )"} - public3 := types.Migration{Name: fmt.Sprintf("%v.sql", p3), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p3), MigrationType: types.MigrationTypeSingleMigration, Contents: "insert into modules values ( 123, '123' )"} - - // public scripts - public4 := types.Migration{Name: fmt.Sprintf("%v.sql", p4), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p4), MigrationType: types.MigrationTypeSingleScript, Contents: "insert into modules values ( 1234, '1234' )"} - public5 := types.Migration{Name: fmt.Sprintf("%v.sql", p5), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p5), MigrationType: types.MigrationTypeSingleScript, Contents: "insert into modules values ( 12345, '12345' )"} - - // tenant migrations - tenant1 := types.Migration{Name: fmt.Sprintf("%v.sql", t1), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t1), MigrationType: types.MigrationTypeTenantMigration, Contents: "drop table if exists {schema}.settings"} - tenant2 := types.Migration{Name: fmt.Sprintf("%v.sql", t2), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t2), MigrationType: types.MigrationTypeTenantMigration, Contents: "create table {schema}.settings (k int, v text)"} - tenant3 := types.Migration{Name: fmt.Sprintf("%v.sql", t3), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t3), MigrationType: types.MigrationTypeTenantMigration, Contents: "insert into {schema}.settings values (456, '456') "} - - // tenant scripts - tenant4 := types.Migration{Name: fmt.Sprintf("%v.sql", t4), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t4), MigrationType: types.MigrationTypeTenantScript, Contents: "insert into {schema}.settings values (456, '456') "} - - migrationsToApply := []types.Migration{public1, public2, public3, tenant1, tenant2, tenant3, public4, public5, tenant4} - - results, version := connector.CreateVersion("commit-sha", types.ActionApply, false, migrationsToApply) - - assert.NotNil(t, version) - assert.True(t, version.ID > 0) - assert.Equal(t, "commit-sha", version.Name) - assert.Equal(t, results.MigrationsGrandTotal+results.ScriptsGrandTotal, int32(len(version.DBMigrations))) - assert.Equal(t, int32(noOfTenants), results.Tenants) - assert.Equal(t, int32(3), results.SingleMigrations) - assert.Equal(t, int32(2), results.SingleScripts) - assert.Equal(t, int32(3), results.TenantMigrations) - assert.Equal(t, int32(1), results.TenantScripts) - assert.Equal(t, int32(noOfTenants*3), results.TenantMigrationsTotal) - assert.Equal(t, int32(noOfTenants*1), results.TenantScriptsTotal) - assert.Equal(t, int32(noOfTenants*3+3), results.MigrationsGrandTotal) - assert.Equal(t, int32(noOfTenants*1+2), results.ScriptsGrandTotal) - - dbMigrationsAfter := connector.GetAppliedMigrations() - lenAfter := len(dbMigrationsAfter) - - // 3 tenant migrations * no of tenants + 3 public - // 1 tenant script * no of tenants + 2 public scripts - expected := (3*noOfTenants + 3) + (1*noOfTenants + 2) - assert.Equal(t, expected, lenAfter-lenBefore) -} - -func TestCreateVersionEmptyMigrationArray(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - connector := New(newTestContext(), config) - defer connector.Dispose() - - migrationsToApply := []types.Migration{} - - results, version := connector.CreateVersion("commit-sha", types.ActionApply, false, migrationsToApply) - // empty migrations slice - no version created - assert.Nil(t, version) - assert.Equal(t, int32(0), results.MigrationsGrandTotal) - assert.Equal(t, int32(0), results.ScriptsGrandTotal) -} - func TestCreateVersionDryRunMode(t *testing.T) { db, mock, err := sqlmock.New() assert.Nil(t, err) @@ -237,19 +134,6 @@ func TestCreateVersionSyncMode(t *testing.T) { } } -func TestGetTenantsSQLDefault(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - dialect := newDialect(config) - connector := baseConnector{newTestContext(), config, dialect, nil} - defer connector.Dispose() - - tenantSelectSQL := connector.getTenantSelectSQL() - - assert.Equal(t, "select name from migrator.migrator_tenants", tenantSelectSQL) -} - func TestGetTenantsSQLOverride(t *testing.T) { config, err := config.FromFile("../test/migrator-overrides.yaml") assert.Nil(t, err) @@ -264,7 +148,7 @@ func TestGetTenantsSQLOverride(t *testing.T) { } func TestGetSchemaPlaceHolderDefault(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") + config, err := config.FromFile("../test/migrator-postgresql.yaml") assert.Nil(t, err) dialect := newDialect(config) @@ -289,39 +173,6 @@ func TestGetSchemaPlaceHolderOverride(t *testing.T) { assert.Equal(t, "[schema]", placeholder) } -func TestCreateTenant(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - connector := New(newTestContext(), config) - defer connector.Dispose() - - t1 := time.Now().UnixNano() - t2 := time.Now().UnixNano() - t3 := time.Now().UnixNano() - - tenant1 := types.Migration{Name: fmt.Sprintf("%v.sql", t1), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t1), MigrationType: types.MigrationTypeTenantMigration, Contents: "drop table if exists {schema}.settings"} - tenant2 := types.Migration{Name: fmt.Sprintf("%v.sql", t2), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t2), MigrationType: types.MigrationTypeTenantMigration, Contents: "create table {schema}.settings (k int, v text)"} - tenant3 := types.Migration{Name: fmt.Sprintf("%v.sql", t3), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t3), MigrationType: types.MigrationTypeTenantMigration, Contents: "insert into {schema}.settings values (456, '456')"} - - migrationsToApply := []types.Migration{tenant1, tenant2, tenant3} - - uniqueTenant := fmt.Sprintf("new_test_tenant_%v", time.Now().UnixNano()) - - results, version := connector.CreateTenant("commit-sha", types.ActionApply, false, uniqueTenant, migrationsToApply) - - assert.NotNil(t, version) - assert.True(t, version.ID > 0) - assert.Equal(t, "commit-sha", version.Name) - assert.Equal(t, results.MigrationsGrandTotal+results.ScriptsGrandTotal, int32(len(version.DBMigrations))) - - // applied only for one tenant - the newly added one - assert.Equal(t, int32(1), results.Tenants) - // just one tenant so total number of tenant migrations is equal to tenant migrations - assert.Equal(t, int32(3), results.TenantMigrations) - assert.Equal(t, int32(3), results.TenantMigrationsTotal) -} - func TestCreateTenantDryRunMode(t *testing.T) { db, mock, err := sqlmock.New() assert.Nil(t, err) @@ -421,83 +272,3 @@ func TestGetTenantInsertSQLOverride(t *testing.T) { assert.Equal(t, "insert into someschema.sometable (somename) values ($1)", tenantInsertSQL) } - -func TestGetVersions(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - connector := New(newTestContext(), config) - defer connector.Dispose() - - versions := connector.GetVersions() - - assert.True(t, len(versions) >= 2) - // versions are sorted from newest (highest ID) to oldest (lowest ID) - assert.True(t, versions[0].ID > versions[1].ID) - - existingVersion = versions[0] - existingDBMigration = existingVersion.DBMigrations[0] -} - -func TestGetVersionsByFile(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - connector := New(newTestContext(), config) - defer connector.Dispose() - - versions := connector.GetVersionsByFile(existingVersion.DBMigrations[0].File) - version := versions[0] - assert.Equal(t, existingVersion.ID, version.ID) - assert.Equal(t, existingVersion.DBMigrations[0].File, version.DBMigrations[0].File) - assert.True(t, len(version.DBMigrations) > 0) -} - -func TestGetVersionByID(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - connector := New(newTestContext(), config) - defer connector.Dispose() - - version, err := connector.GetVersionByID(existingVersion.ID) - assert.Nil(t, err) - assert.Equal(t, existingVersion.ID, version.ID) - assert.True(t, len(version.DBMigrations) > 0) -} - -func TestGetVersionByIDNotFound(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - connector := New(newTestContext(), config) - defer connector.Dispose() - - version, err := connector.GetVersionByID(-1) - assert.Nil(t, version) - assert.Equal(t, "Version not found ID: -1", err.Error()) -} - -func TestGetDBMigrationByID(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - connector := New(newTestContext(), config) - defer connector.Dispose() - - dbMigration, err := connector.GetDBMigrationByID(existingDBMigration.ID) - assert.Nil(t, err) - assert.Equal(t, existingDBMigration.ID, dbMigration.ID) -} - -func TestGetDBMigrationByIDNotFound(t *testing.T) { - config, err := config.FromFile("../test/migrator.yaml") - assert.Nil(t, err) - - connector := New(newTestContext(), config) - defer connector.Dispose() - - dbMigration, err := connector.GetDBMigrationByID(-1) - assert.Nil(t, dbMigration) - assert.Equal(t, "DB migration not found ID: -1", err.Error()) -} diff --git a/loader/azureblob_loader_test.go b/loader/azureblob_loader_test.go index 0984795..2286ce5 100644 --- a/loader/azureblob_loader_test.go +++ b/loader/azureblob_loader_test.go @@ -11,11 +11,18 @@ import ( ) func TestAzureGetSourceMigrations(t *testing.T) { + + accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT"), os.Getenv("AZURE_STORAGE_ACCESS_KEY") + + if len(accountName) == 0 || len(accountKey) == 0 { + t.Skip("skipping test AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY not set") + } + // migrator implements env variable substitution and normally we would use: // "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/mycontainer" // however below we are creating the Config struct directly // and that's why we need to build correct URL ourselves - baseLocation := fmt.Sprintf("https://%v.blob.core.windows.net/mycontainer", os.Getenv("AZURE_STORAGE_ACCOUNT")) + baseLocation := fmt.Sprintf("https://%v.blob.core.windows.net/mycontainer", accountName) config := &config.Config{ BaseLocation: baseLocation, diff --git a/server/server_test.go b/server/server_test.go index 275f8ae..aa26064 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -18,7 +18,7 @@ import ( ) var ( - configFile = "../test/migrator.yaml" + configFile = "../test/migrator-postgresql.yaml" configFileOverrides = "../test/migrator-overrides.yaml" ) diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml new file mode 100644 index 0000000..639145d --- /dev/null +++ b/test/docker-compose.yaml @@ -0,0 +1,104 @@ +version: "3.4" +services: + postgres: + image: "postgres" + ports: + - "5432:5432" + environment: + - POSTGRES_PASSWORD=supersecret + - POSTGRES_DB=migrator + volumes: + - ./create-test-tenants.sql:/docker-entrypoint-initdb.d/create-test-tenants.sql + pgadmin: + image: "thajeztah/pgadmin4" + depends_on: + - postgres + ports: + - "5050:5050" + links: + - postgres + mysql: + image: "mysql" + ports: + - "3306:3306" + environment: + - MYSQL_ROOT_PASSWORD=supersecret + volumes: + - ./create-test-tenants.sql:/docker-entrypoint-initdb.d/create-test-tenants.sql + mariadb: + image: "mariadb" + ports: + - "13306:3306" + environment: + - MYSQL_ROOT_PASSWORD=supersecret + volumes: + - ./create-test-tenants.sql:/docker-entrypoint-initdb.d/create-test-tenants.sql + percona: + image: "percona" + ports: + - "23306:3306" + environment: + - MYSQL_ROOT_PASSWORD=supersecret + volumes: + - ./create-test-tenants.sql:/docker-entrypoint-initdb.d/create-test-tenants.sql + phpmyadmin: + image: "phpmyadmin/phpmyadmin" + depends_on: + - mysql + - mariadb + - percona + ports: + - "8888:80" + environment: + - PMA_HOSTS=mysql,mariadb,percona + - PMA_USER=root + - PMA_PASSWORD=supersecret + links: + - mysql + - mariadb + - percona + mssql: + image: mcr.microsoft.com/mssql/server:2017-latest + ports: + - "1433:1433" + environment: + - SA_PASSWORD=Super5ecret + - ACCEPT_EULA=Y + volumes: + - ./create-test-tenants-mssql.sql:/docker-entrypoint-initdb.d/create-test-tenants-mssql.sql + command: + - /bin/bash + - -c + - | + /opt/mssql/bin/sqlservr & + PID=$$! + is_up=-1 + while [ $$is_up -ne 0 ] && [ $$is_up -ne 16 ] ; do + /opt/mssql-tools/bin/sqlcmd -l 30 -S localhost -h-1 -V1 -U sa -P $$SA_PASSWORD -Q "CREATE DATABASE migrator" + is_up=$$? + sleep 5 + done + if [ $$is_up -eq 0 ]; then + for script in /docker-entrypoint-initdb.d/*.sql + do /opt/mssql-tools/bin/sqlcmd -U sa -P $$SA_PASSWORD -d migrator -l 30 -e -i $$script + done + fi + wait $$PID + migrator: + image: "lukasz/migrator" + depends_on: + - mysql + - mariadb + - percona + - postgres + ports: + - "8181:8080" + environment: + - MIGRATOR_YAML=/data/migrator-docker.yaml + volumes: + - .:/data + links: + - mysql + - mariadb + - percona + - postgres diff --git a/test/docker/create-and-setup-container.sh b/test/docker/create-and-setup-container.sh deleted file mode 100755 index 69ad9f4..0000000 --- a/test/docker/create-and-setup-container.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -if [[ ! -z "$DEBUG" ]]; then - set -x -fi - -cd `dirname $0` - -CONTAINER_TYPE=$1 - -for SCRIPT in scripts/*.sh; do - source "$SCRIPT" -done - -case $CONTAINER_TYPE in - postgres ) - postgresql_start $CONTAINER_TYPE - ;; - mysql|mariadb|percona ) - mysql_start $CONTAINER_TYPE - ;; - mssql ) - mssql_start $CONTAINER_TYPE - ;; - * ) - >&2 echo "Unknown container type $CONTAINER_TYPE" - exit 1 - -esac diff --git a/test/docker/destroy-container.sh b/test/docker/destroy-container.sh deleted file mode 100755 index a34fc37..0000000 --- a/test/docker/destroy-container.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -if [[ ! -z "$DEBUG" ]]; then - set -x -fi - -cd `dirname $0` - -CONTAINER_TYPE=$1 - -for SCRIPT in scripts/*.sh; do - source "${SCRIPT}" -done - -destroy_container $CONTAINER_TYPE diff --git a/test/docker/scripts/destroy-container.sh b/test/docker/scripts/destroy-container.sh deleted file mode 100644 index 9087a83..0000000 --- a/test/docker/scripts/destroy-container.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -function destroy_container() { - name=$1 - docker stop "migrator-$name" - docker rm "migrator-$name" -} diff --git a/test/docker/scripts/mssql-create-and-setup-container.sh b/test/docker/scripts/mssql-create-and-setup-container.sh deleted file mode 100644 index a09abca..0000000 --- a/test/docker/scripts/mssql-create-and-setup-container.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -function mssql_start() { - name=mssql - - ip=127.0.0.1 - port=1433 - database=migratortest - password=YourStrongPassw0rd - - docker run -d \ - --name "migrator-$name" \ - -e 'ACCEPT_EULA=Y' -e "SA_PASSWORD=$password" \ - -P \ - mcr.microsoft.com/mssql/server:2017-latest - - sleep 15 - - running=$(docker inspect -f {{.State.Running}} "migrator-$name") - - if [[ "true" == "$running" ]]; then - docker_port=$(docker port "migrator-$name" | grep "^$port/tcp" | awk -F ':' '{print $2}') - - sqlcmd -S "$ip,$docker_port" -U SA -P $password -Q "CREATE DATABASE $database" - sqlcmd -S "$ip,$docker_port" -U SA -P $password -d $database -i ../create-test-tenants-mssql.sql - - cat ../migrator-mssql.yaml | sed "s/A/sqlserver:\/\/SA:$password@$ip:$docker_port\/?database=$database\&connection+timeout=1\&dial+timeout=1/g" > ../migrator.yaml - else - >&2 echo "Could not setup mssql-$name" - exit 1 - fi -} diff --git a/test/docker/scripts/mysql-create-and-setup-container.sh b/test/docker/scripts/mysql-create-and-setup-container.sh deleted file mode 100644 index c5c324a..0000000 --- a/test/docker/scripts/mysql-create-and-setup-container.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -function mysql_start() { - flavour=$1 - name=${flavour//\//-} - - ip=127.0.0.1 - port=3306 - database=migrator_test - - docker run -d \ - --name "migrator-$flavour" \ - -e MYSQL_ALLOW_EMPTY_PASSWORD=yes \ - -P \ - "$flavour" - - sleep 30 - - running=$(docker inspect -f {{.State.Running}} "migrator-$name") - - if [[ "true" == "$running" ]]; then - docker_port=$(docker port "migrator-$name" | grep "^$port/tcp" | awk -F ':' '{print $2}') - - mysql -u root -h $ip -P $docker_port -e "create database $database" - mysql -u root -h $ip -P $docker_port -D $database < ../create-test-tenants.sql - cat ../migrator-mysql.yaml | sed "s/A/root:@tcp($ip:$docker_port)\/$database?parseTime=true\&timeout=1s/g" > ../migrator.yaml - else - >&2 echo "Could not setup mysql-$name" - exit 1 - fi - -} diff --git a/test/docker/scripts/postgresql-create-and-setup-container.sh b/test/docker/scripts/postgresql-create-and-setup-container.sh deleted file mode 100644 index 40a3570..0000000 --- a/test/docker/scripts/postgresql-create-and-setup-container.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -function postgresql_start() { - flavour=$1 - name=${flavour//\//-} - - ip=127.0.0.1 - port=5432 - database=migrator_test - - docker run -d \ - --name "migrator-$flavour" \ - -P \ - "$flavour" - - sleep 10 - - running=$(docker inspect -f {{.State.Running}} "migrator-$name") - - if [[ "true" == "$running" ]]; then - docker_port=$(docker port "migrator-$name" | grep "^$port/tcp" | awk -F ':' '{print $2}') - - psql -U postgres -h $ip -p $docker_port -c "create database $database" - psql -U postgres -h $ip -p $docker_port -d $database -f ../create-test-tenants.sql - - cat ../migrator-postgresql.yaml | sed "s/dbname=[^ ]* host=[^ ]* port=[^ ]*/dbname=$database host=$ip port=$docker_port/g" > ../migrator.yaml - else - >&2 echo "Could not setup postgresql-$name" - exit 1 - fi -} diff --git a/test/migrator-docker.yaml b/test/migrator-docker.yaml new file mode 100644 index 0000000..a8a8a79 --- /dev/null +++ b/test/migrator-docker.yaml @@ -0,0 +1,8 @@ +baseLocation: /data/migrations +driver: mysql +dataSource: "root:supersecret@tcp(mysql:3306)/migrator?parseTime=true&timeout=1s" +singleMigrations: + - ref + - config +tenantMigrations: + - tenants diff --git a/test/migrator-mariadb.yaml b/test/migrator-mariadb.yaml new file mode 100644 index 0000000..ef9c329 --- /dev/null +++ b/test/migrator-mariadb.yaml @@ -0,0 +1,8 @@ +baseLocation: test/migrations +driver: mysql +dataSource: "root:supersecret@tcp(127.0.0.1:13306)/migrator?parseTime=true&timeout=1s" +singleMigrations: + - ref + - config +tenantMigrations: + - tenants diff --git a/test/migrator-mssql.yaml b/test/migrator-mssql.yaml index 3aac7c5..fcd9bb3 100644 --- a/test/migrator-mssql.yaml +++ b/test/migrator-mssql.yaml @@ -1,6 +1,6 @@ baseLocation: test/migrations driver: sqlserver -dataSource: "A" +dataSource: "sqlserver://SA:Super5ecret@127.0.0.1/?database=migrator&connection+timeout=1&dial+timeout=1" singleMigrations: - ref - config diff --git a/test/migrator-mysql.yaml b/test/migrator-mysql.yaml index 8bd4ac8..74aa4c4 100644 --- a/test/migrator-mysql.yaml +++ b/test/migrator-mysql.yaml @@ -1,6 +1,6 @@ baseLocation: test/migrations driver: mysql -dataSource: "A" +dataSource: "root:supersecret@tcp(127.0.0.1:3306)/migrator?parseTime=true&timeout=1s" singleMigrations: - ref - config diff --git a/test/migrator-percona.yaml b/test/migrator-percona.yaml new file mode 100644 index 0000000..8dce17f --- /dev/null +++ b/test/migrator-percona.yaml @@ -0,0 +1,8 @@ +baseLocation: test/migrations +driver: mysql +dataSource: "root:supersecret@tcp(127.0.0.1:23306)/migrator?parseTime=true&timeout=1s" +singleMigrations: + - ref + - config +tenantMigrations: + - tenants diff --git a/test/migrator-postgresql.yaml b/test/migrator-postgresql.yaml index dabf3c4..a6a4a7a 100644 --- a/test/migrator-postgresql.yaml +++ b/test/migrator-postgresql.yaml @@ -1,6 +1,6 @@ baseLocation: test/migrations driver: postgres -dataSource: "user=postgres dbname=A host=B port=C sslmode=disable connect_timeout=1" +dataSource: "user=postgres password=supersecret dbname=migrator host=127.0.0.1 port=5432 sslmode=disable connect_timeout=1" singleMigrations: - ref - config diff --git a/test/performance/generate-test-migrations.sh b/test/performance/generate-test-migrations.sh index 277f0a5..0a2e132 100755 --- a/test/performance/generate-test-migrations.sh +++ b/test/performance/generate-test-migrations.sh @@ -1,5 +1,7 @@ #!/bin/bash +cd "$(dirname "$0")" + remove_tenant_placeholder=0 append=0 @@ -53,6 +55,19 @@ insert into ${tenantprefixplaceholder}table_for_inserts values ($i); EOL } +function generate_public_table { + timestamp=$(date +'%Y%m%d%H%M%S%N') + file="migrations/public/V${timestamp}.sql" + cat > $file <> migrator.yaml - # generate test migrtations -./generate-test-migrations.sh $NO_OF_MIGRATIONS - -cat migrator.yaml - -psql -U postgres -h 127.0.0.1 -p 55432 -d migrator_test -tAq -c "delete from public.migrator_migrations" - +./test/performance/generate-test-migrations.sh -n $NO_OF_MIGRATIONS + +export PGPASSWORD=supersecret +psql -U postgres -h 127.0.0.1 -p 5432 -d migrator -tAq -c "delete from migrator.migrator_versions" + +./migrator -configFile=test/performance/migrator-performance.yaml & +PID=$! +sleep 5 + +COMMIT_SHA="performance-tests" +# new lines are used for readability but have to be removed from the actual request +cat < create_version.txt +{ + "query": " + mutation CreateVersion(\$input: VersionInput!) { + createVersion(input: \$input) { + version { + id, + name, + } + summary { + startedAt + tenants + migrationsGrandTotal + scriptsGrandTotal + } + } + }", + "operationName": "CreateVersion", + "variables": { + "input": { + "versionName": "$COMMIT_SHA" + } + } +} +EOF +# and now execute the above query start=`date +%s` -migrator +curl -d @create_version.txt http://localhost:8888/v2/service end=`date +%s` -output=$(psql -U postgres -h 127.0.0.1 -p 55432 -d migrator_test -tAq -c "select name from public.migrator_tenants") +output=$(psql -U postgres -h 127.0.0.1 -p 5432 -d migrator -tAq -c "select name from migrator.migrator_tenants") IFS=$'\n'; tenants=($output); unset IFS; for tenant in "${tenants[@]}"; do - count=$(psql -U postgres -h 127.0.0.1 -p 55432 -d migrator_test -tAq -c "select count(distinct col) from $tenant.table_for_inserts") - tables_count=$(psql -U postgres -h 127.0.0.1 -p 55432 -d migrator_test -tAq -c "select count(*) from information_schema.tables where table_schema = '$tenant' and table_name like 'table_%';") + count=$(psql -U postgres -h 127.0.0.1 -p 5432 -d migrator -tAq -c "select count(distinct col) from $tenant.table_for_inserts") + tables_count=$(psql -U postgres -h 127.0.0.1 -p 5432 -d migrator -tAq -c "select count(*) from information_schema.tables where table_schema = '$tenant' and table_name like 'table_%';") if [[ $count -ne $NO_OF_MIGRATIONS+1 ]]; then echo "[migrations inserts] error for $tenant, got $count, expected: $((NO_OF_MIGRATIONS+1))" @@ -44,10 +68,7 @@ done echo "Test took $((end-start)) seconds" -# remove generated migrations and test config -rm -rf migrations -rm migrator.yaml - -# stop and destroy the test container -cd ../.. -./docker/postgresql-destroy-container.sh +# remove generated migrations and test request and kill migrator +rm -rf test/performance/migrations +rm create_version.txt +kill $PID diff --git a/ultimate-coverage.sh b/ultimate-coverage.sh deleted file mode 100755 index fc23a44..0000000 --- a/ultimate-coverage.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -dbs="postgres mysql mariadb percona mssql" - -fail=0 - -for db in $dbs -do - echo "1. destroy $db docker (just in case it's already created)" - ./test/docker/destroy-container.sh $db - echo "2. create and setup $db docker" - ./test/docker/create-and-setup-container.sh $db - echo "3. run all tests" - ./coverage.sh - if [[ $? -ne 0 ]]; then - fail=1 - fi - echo "4. destroy $db docker (cleanup)" - ./test/docker/destroy-container.sh $db -done - -if [[ $fail -ne 0 ]]; then - echo "There are errors in tests. Please review." -fi - -exit $fail From 06728819cda3b2fb8b026cfd829fea29d7ddf776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Sat, 3 Oct 2020 00:43:42 +0200 Subject: [PATCH 3/6] running docker-compose in background and adding 5 minutes sleep to start all containers --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c4f8e67..4b3691c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,8 @@ go: - "1.15" before_script: - - docker-compose -f test/docker-compose.yaml up + - docker-compose -f test/docker-compose.yaml up -d + - sleep 300 script: - ./coverage.sh From 8232b51439f4ac4bb49c3f6d468c02e59a1bc285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Sat, 3 Oct 2020 01:23:19 +0200 Subject: [PATCH 4/6] travis: separated up into pull and up so that up takes less time and requires shorter sleep --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4b3691c..1f1a3a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,9 @@ go: - "1.15" before_script: + - docker-compose -f test/docker-compose.yaml pull - docker-compose -f test/docker-compose.yaml up -d - - sleep 300 + - sleep 60 script: - ./coverage.sh From e1b9f253aa8c9ecd56f9f602292573ca305516e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Sat, 3 Oct 2020 14:58:34 +0200 Subject: [PATCH 5/6] restored env.global so that both secure envs are picked up at the same time (and not cause matrix test) --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1f1a3a6..a723111 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,9 @@ services: - docker env: - - secure: Wvmf5FAySAGXuugkWg/lNQuPfTDA7PYlZB8Izt+bhaVZDp3Re0xz8rhomUw/ZXP/2r81HHPTvDU+eNHxpneyBeHvSXvvhzMVyU+GFLeXdHA0rjvKittkkKq5XuUkj5oaSYJIh0YRn2UdfMaQ/R+y8ZgL2uP41slG6jWpfuMeFe+oqvJCgJ7qovQ7xUataDQdGCyWCpsTlBdpGz+YFUHNrcMK3ppvm4WhzDeMGXHPuTPcWZ+edO8QVWJkxVXamuWgq95tFB3EAXx5tfcg4CS5cC/l/K2tMH7MTsXvWOCRSU+f79cSNlrztP1Mhc/TPBZe87ubmkLOQGRK//ZBqbmSXjsyjv4qkq3z223Dmg+MRkt6ip01zsSVE6vmEvtVHGxtj8VKfWbwWusAXuLa1gtzXa/vVgisF++aj1ZEhApH5mu95gMqy9tOY/mX4Pm4iqalLujvlzVHKDDCmZsyF2O8mZCL2cDH7xlQTLeq9eyRPXNcm13WDDAcOhgVSpWNVObggUMZ67YDGV0yPQKGQBOCCqV3jCKgii/4m7n1H2mn1b8VpYOdK3ncA3U3bcHDPy3Nh44EmdJnj7QiUZtN+RyQGCz83h3X2owGjNG3qvGxrBPy87DbQU+FTN88zB4DdgO/TOYEzg3Wbz24D3thZUDR8rWvu9t9/PvH+CroTIpW4Iw= - - secure: d5+t1ysBsHPGKvrbxsN4M+4ns4Q3YHgAk3gapaPNxlX3pDKLgZ1Kd/XBnQ3dMSCP7FWejgmeLDxloGQMeieq6exEM9kmf8Ui/oFB97j2+ODXPv6+X69+aDDGMT94J77v5DwdJzyULI4vn3ZF2V6fM5/tlle4hf4uwCBNh30Svar7ZXsp/aZL7wpZzUG+efLFJJZ4oSD2NteWR9IeZbNqQtV0fUTHLk3g4Suze6Yt2OmZiFH4pn8Ihftjyr84nke65aPws8XKzoaVmghoZXmKNPNyiH3Dyu6uyGa2OhMspj0zjXHP+sMlgMrp5cvAGchRBz9NPL18VHtgID/hEPLfwb6ZZ6HypgXu5+4/O/J2vGHTFV0q2aluNBJi+sTzklYkHoCiBXtjdZxI/+mwK1VIeMejrhSjNtzdkryEbOQM/CoBcDacQIw4gUOloCG4AcFNXzQZifHZckIgc4JChYmNHhpEu3U1Zil8H7QDQ7MGqJG+HWuK/s/leoJx112jHLmCgPsq9hmMozDdQi8rVSS8Eo6bg7NTOuE9LK/uoweF31NJYITYV3vxmqluwT7+yn7V9avXRFTV1qTcavsjAYkMS3PlLX4f+w1HtZUIOjtdABcK5mg1JjZ92xHmNFudu8b2n6j3A3nOYuA0NsX5BlCuIG84MozNsMleOl6svbRKW+4= + global: + - secure: Wvmf5FAySAGXuugkWg/lNQuPfTDA7PYlZB8Izt+bhaVZDp3Re0xz8rhomUw/ZXP/2r81HHPTvDU+eNHxpneyBeHvSXvvhzMVyU+GFLeXdHA0rjvKittkkKq5XuUkj5oaSYJIh0YRn2UdfMaQ/R+y8ZgL2uP41slG6jWpfuMeFe+oqvJCgJ7qovQ7xUataDQdGCyWCpsTlBdpGz+YFUHNrcMK3ppvm4WhzDeMGXHPuTPcWZ+edO8QVWJkxVXamuWgq95tFB3EAXx5tfcg4CS5cC/l/K2tMH7MTsXvWOCRSU+f79cSNlrztP1Mhc/TPBZe87ubmkLOQGRK//ZBqbmSXjsyjv4qkq3z223Dmg+MRkt6ip01zsSVE6vmEvtVHGxtj8VKfWbwWusAXuLa1gtzXa/vVgisF++aj1ZEhApH5mu95gMqy9tOY/mX4Pm4iqalLujvlzVHKDDCmZsyF2O8mZCL2cDH7xlQTLeq9eyRPXNcm13WDDAcOhgVSpWNVObggUMZ67YDGV0yPQKGQBOCCqV3jCKgii/4m7n1H2mn1b8VpYOdK3ncA3U3bcHDPy3Nh44EmdJnj7QiUZtN+RyQGCz83h3X2owGjNG3qvGxrBPy87DbQU+FTN88zB4DdgO/TOYEzg3Wbz24D3thZUDR8rWvu9t9/PvH+CroTIpW4Iw= + - secure: d5+t1ysBsHPGKvrbxsN4M+4ns4Q3YHgAk3gapaPNxlX3pDKLgZ1Kd/XBnQ3dMSCP7FWejgmeLDxloGQMeieq6exEM9kmf8Ui/oFB97j2+ODXPv6+X69+aDDGMT94J77v5DwdJzyULI4vn3ZF2V6fM5/tlle4hf4uwCBNh30Svar7ZXsp/aZL7wpZzUG+efLFJJZ4oSD2NteWR9IeZbNqQtV0fUTHLk3g4Suze6Yt2OmZiFH4pn8Ihftjyr84nke65aPws8XKzoaVmghoZXmKNPNyiH3Dyu6uyGa2OhMspj0zjXHP+sMlgMrp5cvAGchRBz9NPL18VHtgID/hEPLfwb6ZZ6HypgXu5+4/O/J2vGHTFV0q2aluNBJi+sTzklYkHoCiBXtjdZxI/+mwK1VIeMejrhSjNtzdkryEbOQM/CoBcDacQIw4gUOloCG4AcFNXzQZifHZckIgc4JChYmNHhpEu3U1Zil8H7QDQ7MGqJG+HWuK/s/leoJx112jHLmCgPsq9hmMozDdQi8rVSS8Eo6bg7NTOuE9LK/uoweF31NJYITYV3vxmqluwT7+yn7V9avXRFTV1qTcavsjAYkMS3PlLX4f+w1HtZUIOjtdABcK5mg1JjZ92xHmNFudu8b2n6j3A3nOYuA0NsX5BlCuIG84MozNsMleOl6svbRKW+4= go: - "1.13" From 2fece2fbe23af02564169c55c11436c6ae7d906c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Sat, 3 Oct 2020 15:17:45 +0200 Subject: [PATCH 6/6] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..75bc5a8 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + branches: [master] + pull_request: + # The branches below must be a subset of the branches above + branches: [master] + schedule: + - cron: '0 7 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['go'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1