From 74d842311afab7362e3c4291831584b38370e0c4 Mon Sep 17 00:00:00 2001 From: yuli Date: Sat, 24 Aug 2024 21:11:47 +0300 Subject: [PATCH] refactor databunker init steps --- src/bunker.go | 262 +------------------------------------------------ src/service.go | 215 ++++++++++++++++++++++++++++++++++++++++ src/utils.go | 42 ++++++++ 3 files changed, 258 insertions(+), 261 deletions(-) create mode 100644 src/service.go diff --git a/src/bunker.go b/src/bunker.go index b85bce27..1e3b52c9 100644 --- a/src/bunker.go +++ b/src/bunker.go @@ -5,32 +5,20 @@ package main import ( "bytes" "compress/gzip" - "context" - "crypto/md5" - "crypto/tls" - "encoding/hex" - "errors" - "flag" "io" - "io/ioutil" "log" "math/rand" "net/http" - "os" - "os/signal" "strings" - "syscall" "time" "github.com/gobuffalo/packr" "github.com/julienschmidt/httprouter" - "github.com/kelseyhightower/envconfig" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/securitybunker/databunker/src/autocontext" "github.com/securitybunker/databunker/src/storage" "go.mongodb.org/mongo-driver/bson" - yaml "gopkg.in/yaml.v2" ) var version string @@ -332,33 +320,6 @@ func (e mainEnv) setupRouter() *httprouter.Router { return router } -// readConfFile() read configuration file. -func readConfFile(cfg *Config, filepath *string) error { - confFile := "databunker.yaml" - if filepath != nil { - if len(*filepath) > 0 { - confFile = *filepath - } - } - log.Printf("Databunker configuration file: %s\n", confFile) - f, err := os.Open(confFile) - if err != nil { - return err - } - decoder := yaml.NewDecoder(f) - err = decoder.Decode(cfg) - if err != nil { - return err - } - return nil -} - -// readEnv() process environment variables. -func readEnv(cfg *Config) error { - err := envconfig.Process("", cfg) - return err -} - // dbCleanup() is used to run cron jobs. func (e mainEnv) dbCleanupDo() { log.Printf("db cleanup timeout\n") @@ -494,230 +455,9 @@ func reqMiddleware(handler http.Handler) http.Handler { }) } -func setupDB(dbPtr *string, masterKeyPtr *string, customRootToken string) (*dbcon, string, error) { - log.Println("Databunker init") - var masterKey []byte - var err error - if variableProvided("DATABUNKER_MASTERKEY", masterKeyPtr) == true { - masterKey, err = masterkeyGet(masterKeyPtr) - if err != nil { - log.Printf("Failed to parse master key: %s\n", err) - os.Exit(0) - } - log.Println("Master key: ****") - } else { - masterKey, err = generateMasterKey() - if err != nil { - log.Printf("Failed to generate master key: %s", err) - os.Exit(0) - } - log.Printf("Master key: %x\n", masterKey) - } - hash := md5.Sum(masterKey) - log.Println("Init database") - store, err := storage.InitDB(dbPtr) - for numAttempts := 60; err != nil && numAttempts > 0; numAttempts-- { - time.Sleep(1 * time.Second) - log.Printf("Trying to init db: %d\n", 61-numAttempts) - store, err = storage.InitDB(dbPtr) - } - if err != nil { - //log.Panic("error %s", err.Error()) - log.Fatalf("Databunker failed to init database, error %s\n\n", err.Error()) - os.Exit(0) - } - db := &dbcon{store, masterKey, hash[:]} - rootToken, err := db.createRootXtoken(customRootToken) - if err != nil { - //log.Panic("error %s", err.Error()) - log.Printf("Failed to init root token: %s", err.Error()) - os.Exit(0) - } - log.Println("Creating default legal basis records") - db.createLegalBasis("core-send-email-on-login", "", "login", "Send email on login", - "Confirm to allow sending access code using 3rd party email gateway", "consent", - "This consent is required to give you our service.", "active", true, true) - db.createLegalBasis("core-send-sms-on-login", "", "login", "Send SMS on login", - "Confirm to allow sending access code using 3rd party SMS gateway", "consent", - "This consent is required to give you our service.", "active", true, true) - if len(customRootToken) > 0 && customRootToken != "DEMO" { - log.Println("API Root token: ****") - } else { - log.Printf("API Root token: %s\n", rootToken) - } - return db, rootToken, err -} - -func variableProvided(vname string, masterKeyPtr *string) bool { - if masterKeyPtr != nil && len(*masterKeyPtr) > 0 { - return true - } - if len(os.Getenv(vname)) > 0 { - return true - } - if len(os.Getenv(vname+"_FILE")) > 0 { - return true - } - return false -} - -func masterkeyGet(masterKeyPtr *string) ([]byte, error) { - masterKeyStr := "" - if masterKeyPtr != nil && len(*masterKeyPtr) > 0 { - masterKeyStr = *masterKeyPtr - } else if len(os.Getenv("DATABUNKER_MASTERKEY")) > 0 { - masterKeyStr = os.Getenv("DATABUNKER_MASTERKEY") - } else if len(os.Getenv("DATABUNKER_MASTERKEY_FILE")) > 0 { - content, err := ioutil.ReadFile(os.Getenv("DATABUNKER_MASTERKEY_FILE")) - if err != nil { - return nil, err - } - // Convert []byte to string - masterKeyStr = strings.TrimSpace(string(content)) - // we will TRY to delete secret file when running inside container/kubernetes - if isContainer() == true { - os.Remove(os.Getenv("DATABUNKER_MASTERKEY_FILE")) - } - } - if len(masterKeyStr) == 0 { - return nil, errors.New("Master key environment variable/parameter is missing") - } - if len(masterKeyStr) != 48 { - return nil, errors.New("Master key length is wrong") - } - if isValidHex(masterKeyStr) == false { - return nil, errors.New("Master key is not valid hex string") - } - masterKey, err := hex.DecodeString(masterKeyStr) - if err != nil { - return nil, errors.New("Failed to decode master key") - } - return masterKey, nil -} - // main application function func main() { rand.Seed(time.Now().UnixNano()) lockMemory() - initPtr := flag.Bool("init", false, "Generate master key and init database") - demoPtr := flag.Bool("demoinit", false, "Generate master key with a DEMO root access token") - startPtr := flag.Bool("start", false, "Start databunker service. Provide additional --masterkey value or set it up using evironment variable: DATABUNKER_MASTERKEY") - masterKeyPtr := flag.String("masterkey", "", "Specify master key - main database encryption key") - dbPtr := flag.String("db", "databunker", "Specify database name/file") - confPtr := flag.String("conf", "", "Configuration file name to use") - rootTokenKeyPtr := flag.String("roottoken", "", "Specify custom root token to use during database init. It must be in UUID format.") - versionPtr := flag.Bool("version", false, "Print version information") - flag.Parse() - - if *versionPtr { - log.Printf("Databunker version: %s\n", version) - os.Exit(0) - } - - var cfg Config - readEnv(&cfg) - readConfFile(&cfg, confPtr) - - customRootToken := "" - if *demoPtr { - customRootToken = "DEMO" - } else if variableProvided("DATABUNKER_ROOTTOKEN", rootTokenKeyPtr) == true { - if rootTokenKeyPtr != nil && len(*rootTokenKeyPtr) > 0 { - customRootToken = *rootTokenKeyPtr - } else { - customRootToken = os.Getenv("DATABUNKER_ROOTTOKEN") - } - } - if *initPtr || *demoPtr { - if storage.DBExists(dbPtr) == true { - log.Println("Database is alredy initialized.") - } else { - db, _, _ := setupDB(dbPtr, masterKeyPtr, customRootToken) - db.store.CloseDB() - } - os.Exit(0) - } - dbExists := storage.DBExists(dbPtr) - for numAttempts := 60; dbExists == false && numAttempts > 0; numAttempts-- { - time.Sleep(1 * time.Second) - log.Printf("Trying to open db [%d]\n", 61-numAttempts) - dbExists = storage.DBExists(dbPtr) - } - if dbExists == false { - log.Println("Database is not initialized") - log.Println(`Run "databunker -init" for the first time to generate keys and init database.`) - os.Exit(0) - } - if masterKeyPtr == nil && *startPtr == false { - log.Println(`Run "databunker -start" will load DATABUNKER_MASTERKEY environment variable.`) - log.Println(`For testing "databunker -masterkey MASTER_KEY_VALUE" can be used. Not recommended for production.`) - os.Exit(0) - } - err := loadUserSchema(cfg, confPtr) - if err != nil { - log.Printf("Failed to load user schema: %s\n", err) - os.Exit(0) - } - masterKey, masterKeyErr := masterkeyGet(masterKeyPtr) - if masterKeyErr != nil { - log.Printf("Error: %s", masterKeyErr) - os.Exit(0) - } - store, err := storage.OpenDB(dbPtr) - if err != nil { - log.Printf("Filed to open db: %s", err) - os.Exit(0) - } - hash := md5.Sum(masterKey) - db := &dbcon{store, masterKey, hash[:]} - e := mainEnv{db, cfg, make(chan struct{})} - e.dbCleanup() - initGeoIP() - initCaptcha(hash) - router := e.setupRouter() - router = e.setupConfRouter(router) - tlsConfig := &tls.Config{ - MinVersion: tls.VersionTLS12, - CipherSuites: []uint16{ - tls.TLS_AES_256_GCM_SHA384, - tls.TLS_CHACHA20_POLY1305_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - //tls.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, - //tls.TLS_DHE_RSA_WITH_AES_256_CCM_8, - //tls.TLS_DHE_RSA_WITH_AES_256_CCM, - //tls.TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384, - //tls.TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - }, - } - srv := &http.Server{Addr: cfg.Server.Host + ":" + cfg.Server.Port, Handler: reqMiddleware(router), TLSConfig: tlsConfig} - - stop := make(chan os.Signal, 2) - signal.Notify(stop, os.Interrupt, syscall.SIGTERM) - // Waiting for SIGINT (pkill -2) - go func() { - <-stop - log.Println("Closing app...") - close(e.stopChan) - time.Sleep(1 * time.Second) - srv.Shutdown(context.TODO()) - db.store.CloseDB() - }() - - if _, err := os.Stat(cfg.Ssl.SslCertificate); !os.IsNotExist(err) { - log.Printf("Loading ssl\n") - err := srv.ListenAndServeTLS(cfg.Ssl.SslCertificate, cfg.Ssl.SslCertificateKey) - if err != nil { - log.Printf("ListenAndServeSSL: %s\n", err) - } - } else { - log.Println("Loading server") - err := srv.ListenAndServe() - if err != nil { - log.Printf("ListenAndServe(): %s\n", err) - } - } + loadService() } diff --git a/src/service.go b/src/service.go new file mode 100644 index 00000000..8ffbc1c6 --- /dev/null +++ b/src/service.go @@ -0,0 +1,215 @@ +package main + +import ( + "context" + "crypto/md5" + "crypto/tls" + "encoding/hex" + "errors" + "flag" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/securitybunker/databunker/src/storage" +) + +func loadService() { + initPtr := flag.Bool("init", false, "Generate master key and init database") + demoPtr := flag.Bool("demoinit", false, "Generate master key with a DEMO root access token") + startPtr := flag.Bool("start", false, "Start databunker service. Provide additional --masterkey value or set it up using evironment variable: DATABUNKER_MASTERKEY") + masterKeyPtr := flag.String("masterkey", "", "Specify master key - main database encryption key") + dbPtr := flag.String("db", "databunker", "Specify database name/file") + confPtr := flag.String("conf", "", "Configuration file name to use") + rootTokenKeyPtr := flag.String("roottoken", "", "Specify custom root token to use during database init. It must be in UUID format.") + versionPtr := flag.Bool("version", false, "Print version information") + flag.Parse() + + if *versionPtr { + log.Printf("Databunker version: %s\n", version) + os.Exit(0) + } + log.Printf("Databunker version: %s\n", version) + var cfg Config + readEnv(&cfg) + readConfFile(&cfg, confPtr) + + customRootToken := "" + if *demoPtr { + customRootToken = "DEMO" + } else { + customRootToken = getArgEnvFileVariable("DATABUNKER_ROOTTOKEN", rootTokenKeyPtr) + } + if *initPtr || *demoPtr { + if storage.DBExists(dbPtr) == true { + log.Println("Database is alredy initialized.") + } else { + db, _, _ := setupDB(dbPtr, masterKeyPtr, customRootToken) + db.store.CloseDB() + } + os.Exit(0) + } + dbExists := storage.DBExists(dbPtr) + for numAttempts := 60; dbExists == false && numAttempts > 0; numAttempts-- { + time.Sleep(1 * time.Second) + log.Printf("Trying to open db [%d]\n", 61-numAttempts) + dbExists = storage.DBExists(dbPtr) + } + if dbExists == false { + log.Println("Database is not initialized") + log.Println(`Run "databunker -init" for the first time to generate keys and init database.`) + os.Exit(0) + } + masterKeyStr := getArgEnvFileVariable("DATABUNKER_MASTERKEY", masterKeyPtr) + if *startPtr == false { + log.Println(`'databunker -start' command is missing.`) + os.Exit(0) + } + if len(masterKeyStr) == 0 { + log.Println(`ENV['DATABUNKER_MASTERKEY'] or 'databunker -masterkey value' must be present.`) + os.Exit(0) + } + err := loadUserSchema(cfg, confPtr) + if err != nil { + log.Printf("Failed to load user schema: %s\n", err) + os.Exit(0) + } + masterKey, masterKeyErr := decodeMasterkey(masterKeyStr) + if masterKeyErr != nil { + log.Printf("Error: %s", masterKeyErr) + os.Exit(0) + } + store, err := storage.OpenDB(dbPtr) + if err != nil { + log.Printf("Filed to open db: %s", err) + os.Exit(0) + } + hash := md5.Sum(masterKey) + db := &dbcon{store, masterKey, hash[:]} + e := mainEnv{db, cfg, make(chan struct{})} + e.dbCleanup() + initGeoIP() + initCaptcha(hash) + router := e.setupRouter() + router = e.setupConfRouter(router) + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + //tls.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + //tls.TLS_DHE_RSA_WITH_AES_256_CCM_8, + //tls.TLS_DHE_RSA_WITH_AES_256_CCM, + //tls.TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384, + //tls.TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + }, + } + listener := cfg.Server.Host + ":" + cfg.Server.Port + srv := &http.Server{Addr: listener, Handler: reqMiddleware(router), TLSConfig: tlsConfig} + + stop := make(chan os.Signal, 2) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + // Waiting for SIGINT (pkill -2) + go func() { + <-stop + log.Println("Closing app...") + close(e.stopChan) + time.Sleep(1 * time.Second) + srv.Shutdown(context.TODO()) + db.store.CloseDB() + }() + + if _, err := os.Stat(cfg.Ssl.SslCertificate); !os.IsNotExist(err) { + log.Printf("Open HTTPS listener %s\n", listener) + err := srv.ListenAndServeTLS(cfg.Ssl.SslCertificate, cfg.Ssl.SslCertificateKey) + if err != nil { + log.Printf("ListenAndServeSSL: %s\n", err) + } + } else { + log.Printf("Open HTTP listener %s\n", listener) + err := srv.ListenAndServe() + if err != nil { + log.Printf("ListenAndServe(): %s\n", err) + } + } +} + +func decodeMasterkey(masterKeyStr string) ([]byte, error) { + if len(masterKeyStr) == 0 { + return nil, errors.New("Master key environment variable/parameter is missing") + } + if len(masterKeyStr) != 48 { + return nil, errors.New("Master key length is wrong") + } + if isValidHex(masterKeyStr) == false { + return nil, errors.New("Master key is not valid hex string") + } + masterKey, err := hex.DecodeString(masterKeyStr) + if err != nil { + return nil, errors.New("Failed to decode master key") + } + return masterKey, nil +} + +func setupDB(dbPtr *string, masterKeyPtr *string, customRootToken string) (*dbcon, string, error) { + log.Println("Databunker init") + var masterKey []byte + var err error + masterKeyString := getArgEnvFileVariable("DATABUNKER_MASTERKEY", masterKeyPtr) + if len(masterKeyString) > 0 { + masterKey, err = decodeMasterkey(masterKeyString) + if err != nil { + log.Printf("Failed to parse master key: %s\n", err) + os.Exit(0) + } + log.Println("Master key: ****") + } else { + masterKey, err = generateMasterKey() + if err != nil { + log.Printf("Failed to generate master key: %s", err) + os.Exit(0) + } + log.Printf("Master key: %x\n", masterKey) + } + hash := md5.Sum(masterKey) + log.Println("Init database") + store, err := storage.InitDB(dbPtr) + for numAttempts := 60; err != nil && numAttempts > 0; numAttempts-- { + time.Sleep(1 * time.Second) + log.Printf("Trying to init db: %d\n", 61-numAttempts) + store, err = storage.InitDB(dbPtr) + } + if err != nil { + //log.Panic("error %s", err.Error()) + log.Fatalf("Databunker failed to init database, error %s\n\n", err.Error()) + os.Exit(0) + } + db := &dbcon{store, masterKey, hash[:]} + rootToken, err := db.createRootXtoken(customRootToken) + if err != nil { + //log.Panic("error %s", err.Error()) + log.Printf("Failed to init root token: %s", err.Error()) + os.Exit(0) + } + log.Println("Creating default legal basis records") + db.createLegalBasis("core-send-email-on-login", "", "login", "Send email on login", + "Confirm to allow sending access code using 3rd party email gateway", "consent", + "This consent is required to give you our service.", "active", true, true) + db.createLegalBasis("core-send-sms-on-login", "", "login", "Send SMS on login", + "Confirm to allow sending access code using 3rd party SMS gateway", "consent", + "This consent is required to give you our service.", "active", true, true) + if len(customRootToken) > 0 && customRootToken != "DEMO" { + log.Println("API Root token: ****") + } else { + log.Printf("API Root token: %s\n", rootToken) + } + return db, rootToken, err +} diff --git a/src/utils.go b/src/utils.go index c0772fb4..81ad53d8 100644 --- a/src/utils.go +++ b/src/utils.go @@ -19,8 +19,10 @@ import ( "syscall" "time" + "github.com/kelseyhightower/envconfig" "github.com/ttacon/libphonenumber" "golang.org/x/sys/unix" + "gopkg.in/yaml.v2" ) var ( @@ -120,6 +122,46 @@ func getInt64Value(records map[string]interface{}, key string) int64 { return 0 } +// readConfFile() read configuration file. +func readConfFile(cfg *Config, filepath *string) error { + confFile := "databunker.yaml" + if filepath != nil { + if len(*filepath) > 0 { + confFile = *filepath + } + } + log.Printf("Loading configuration file: %s\n", confFile) + f, err := os.Open(confFile) + if err != nil { + return err + } + decoder := yaml.NewDecoder(f) + err = decoder.Decode(cfg) + if err != nil { + return err + } + return nil +} + +// readEnv() process environment variables. +func readEnv(cfg *Config) error { + err := envconfig.Process("", cfg) + return err +} + +func getArgEnvFileVariable(vname string, masterKeyPtr *string) string { + strvalue := "" + if masterKeyPtr != nil && len(*masterKeyPtr) > 0 { + strvalue = *masterKeyPtr + } else if len(os.Getenv(vname)) > 0 { + strvalue = os.Getenv(vname) + } else if len(os.Getenv(vname+"_FILE")) > 0 { + data, _ := os.ReadFile(os.Getenv(vname + "_FILE")) + strvalue = string(data) + } + return strings.TrimSpace(strvalue) +} + func hashString(md5Salt []byte, src string) string { stringToHash := append(md5Salt, []byte(src)...) hashed := sha256.Sum256(stringToHash)