Skip to content

Commit

Permalink
Add RDS IAM authentication support
Browse files Browse the repository at this point in the history
  • Loading branch information
DJAndries committed Oct 18, 2024
1 parent c751555 commit 61bcf21
Show file tree
Hide file tree
Showing 5 changed files with 196 additions and 17 deletions.
52 changes: 38 additions & 14 deletions datastore/migrations/20240904202925_init.up.sql
Original file line number Diff line number Diff line change
Expand Up @@ -23,28 +23,52 @@ CREATE TABLE entities (
mtime BIGINT NOT NULL,
version BIGINT NOT NULL,
data_type INTEGER NOT NULL,
specifics BYTEA STORAGE EXTERNAL NOT NULL,
client_defined_unique_tag TEXT STORAGE PLAIN,
server_defined_unique_tag TEXT STORAGE PLAIN,
name TEXT STORAGE PLAIN,
originator_cache_guid TEXT STORAGE PLAIN,
originator_client_item_id TEXT STORAGE PLAIN,
parent_id TEXT STORAGE PLAIN,
non_unique_name TEXT STORAGE PLAIN,
unique_position BYTEA STORAGE PLAIN,
specifics BYTEA NOT NULL,
client_defined_unique_tag TEXT,
server_defined_unique_tag TEXT,
name TEXT,
originator_cache_guid TEXT,
originator_client_item_id TEXT,
parent_id TEXT,
non_unique_name TEXT,
unique_position BYTEA,
folder BOOLEAN,
deleted BOOLEAN NOT NULL,
PRIMARY KEY (id, chain_id),
UNIQUE (chain_id, client_defined_unique_tag)
)
PARTITION BY RANGE (chain_id);

ALTER TABLE entities ALTER specifics SET STORAGE EXTERNAL;
ALTER TABLE entities ALTER client_defined_unique_tag SET STORAGE PLAIN;
ALTER TABLE entities ALTER server_defined_unique_tag SET STORAGE PLAIN;
ALTER TABLE entities ALTER name SET STORAGE PLAIN;
ALTER TABLE entities ALTER originator_cache_guid SET STORAGE PLAIN;
ALTER TABLE entities ALTER originator_client_item_id SET STORAGE PLAIN;
ALTER TABLE entities ALTER parent_id SET STORAGE PLAIN;
ALTER TABLE entities ALTER non_unique_name SET STORAGE PLAIN;
ALTER TABLE entities ALTER unique_position SET STORAGE PLAIN;

CREATE INDEX entities_chain_id_data_type_mtime_idx ON entities (chain_id, data_type, mtime);

SELECT partman.create_parent(
p_parent_table := 'public.entities',
p_control := 'chain_id',
p_interval := '3500'
);
DO $$
BEGIN
-- for vanilla postgres
PERFORM partman.create_parent(
p_parent_table := 'public.entities',
p_control := 'chain_id',
p_interval := '3500',
p_type := 'range'
);
EXCEPTION WHEN OTHERS THEN
-- for Aurora
PERFORM partman.create_parent(
p_parent_table := 'public.entities',
p_control := 'chain_id',
p_interval := '3500',
p_type := 'native'
);
END $$;

CREATE EXTENSION IF NOT EXISTS pg_cron;

Expand Down
93 changes: 93 additions & 0 deletions datastore/rds.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package datastore

import (
"context"
"database/sql/driver"
"fmt"
"net/url"
"os"
"sync"
"time"

"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/rds/auth"
"github.com/jackc/pgx/stdlib"
)

const defaultRegion = "us-west-2"

const (
rdsPortKey = "RDS_DATABASE_PORT"
rdsHostKey = "RDS_WRITER_ENDPOINT"
rdsUserKey = "RDS_USER"
rdsDbNameKey = "RDS_DATABASE_NAME"
regionKey = "AWS_REGION"
)

type rdsConnector struct {
hostAndPort string
dbName string
user string
token string
region string
tokenCacheTime time.Time
mu sync.Mutex
}

func newRDSConnector() *rdsConnector {
port := os.Getenv(rdsPortKey)
host := os.Getenv(rdsHostKey)
user := os.Getenv(rdsUserKey)
dbName := os.Getenv(rdsDbNameKey)
region := os.Getenv(regionKey)

if region == "" {
region = defaultRegion
}
hostAndPort := fmt.Sprintf("%s:%s", host, port)
return &rdsConnector{
hostAndPort: hostAndPort,
dbName: dbName,
user: user,
region: region,
}
}

func (c *rdsConnector) getConnectionString(ctx context.Context) (string, error) {
c.mu.Lock()
defer c.mu.Unlock()

if time.Since(c.tokenCacheTime) > 10*time.Minute {
cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
return "", fmt.Errorf("failed to load AWS config")
}

token, err := auth.BuildAuthToken(
ctx, c.hostAndPort, c.region, c.user, cfg.Credentials)
if err != nil {
return "", fmt.Errorf("failed to create authentication token: %w", err)
}
c.token = url.QueryEscape(token)
c.tokenCacheTime = time.Now()
}

return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=require", c.user, c.token, c.hostAndPort, c.dbName), nil
}

func (c *rdsConnector) Connect(ctx context.Context) (driver.Conn, error) {
connStr, err := c.getConnectionString(ctx)
if err != nil {
return nil, err
}

return stdlib.GetDefaultDriver().Open(connStr)
}

func (c *rdsConnector) Driver() driver.Driver {
return c
}

func (c *rdsConnector) Open(_ string) (driver.Conn, error) {
return nil, fmt.Errorf("open method unsupported")
}
26 changes: 23 additions & 3 deletions datastore/sql.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package datastore

import (
"context"
"database/sql"
"embed"
"errors"
"fmt"
Expand All @@ -11,6 +13,7 @@ import (
// import postgres package for migrations
_ "github.com/golang-migrate/migrate/v4/database/postgres"
"github.com/golang-migrate/migrate/v4/source/iofs"

// import pgx so it can be used with sqlx
_ "github.com/jackc/pgx/stdlib"
"github.com/jmoiron/sqlx"
Expand Down Expand Up @@ -58,12 +61,22 @@ func NewSQLDB(isTesting bool) (*SQLDB, error) {
envKey = sqlURLEnvKey
}

var rdsConnector *rdsConnector
if os.Getenv(rdsHostKey) != "" {
rdsConnector = newRDSConnector()
}

sqlURL := os.Getenv(envKey)
if sqlURL == "" {
if rdsConnector != nil {
sqlURL, err = rdsConnector.getConnectionString(context.Background())
if err != nil {
return nil, err
}
} else if sqlURL == "" {
if isTesting {
sqlURL = defaultSQLTestURL
} else {
return nil, fmt.Errorf("%s must be defined", envKey)
return nil, fmt.Errorf("%s or %s must be defined", envKey, rdsHostKey)
}
}
iofsDriver, err := iofs.New(migrationFiles, "migrations")
Expand All @@ -82,9 +95,16 @@ func NewSQLDB(isTesting bool) (*SQLDB, error) {
if !errors.Is(err, migrate.ErrNoChange) {
return nil, fmt.Errorf("Failed to run migrations: %w", err)
}
err = nil
}

db, err := sqlx.Connect("pgx", sqlURL)
var db *sqlx.DB
if rdsConnector != nil {
baseDB := sql.OpenDB(rdsConnector)
db = sqlx.NewDb(baseDB, "pgx")
} else {
db, err = sqlx.Connect("pgx", sqlURL)
}
if err != nil {
return nil, fmt.Errorf("Failed to connect to SQL DB: %w", err)
}
Expand Down
14 changes: 14 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,20 @@ require (

require (
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect
github.com/aws/aws-sdk-go-v2/config v1.28.0 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect
github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.4.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect
github.com/aws/smithy-go v1.22.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/brave-intl/bat-go/libs v0.0.0-20240909083638-be56e4a5398e // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
Expand Down
28 changes: 28 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,34 @@ github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbV
github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI=
github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ=
github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc=
github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8=
github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw=
github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.4.21 h1:wRH9E07mfYqZ1EPphNTUIkrZ/7wcbZAGcjhrBlkWy4c=
github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.4.21/go.mod h1:6m/MDzT+aFxaIo46f2MYV4d+qG9J9keLlHL0qKnQFgA=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI=
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo=
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo=
github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/brave-intl/bat-go/libs v0.0.0-20231020145457-cc9860c87bae h1:CGUFAtMXAsGajLeobq6ep+5wREYS+lepZSdPckY+Ba0=
Expand Down

0 comments on commit 61bcf21

Please sign in to comment.