diff --git a/build/lido-exporter b/build/lido-exporter new file mode 100644 index 000000000..55d24101a Binary files /dev/null and b/build/lido-exporter differ diff --git a/build/sedg.exee b/build/sedg.exee new file mode 100644 index 000000000..49d897665 Binary files /dev/null and b/build/sedg.exee differ diff --git a/build/sedge.exe b/build/sedge.exe new file mode 100644 index 000000000..b8c0f54ef Binary files /dev/null and b/build/sedge.exe differ diff --git a/cli/cli.go b/cli/cli.go index c23f5533a..e5de749e9 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -24,7 +24,6 @@ import ( "time" eth2 "github.com/protolambda/zrnt/eth2/configs" - "github.com/NethermindEth/sedge/internal/pkg/clients" "github.com/NethermindEth/sedge/internal/pkg/dependencies" "github.com/NethermindEth/sedge/internal/pkg/generate" @@ -84,9 +83,10 @@ type CliCmdOptions struct { numberOfValidators int64 existingValidators int64 installDependencies bool + enableMonitoring bool } -func CliCmd(p ui.Prompter, actions actions.SedgeActions, depsMgr dependencies.DependenciesManager) *cobra.Command { +func CliCmd(p ui.Prompter, actions actions.SedgeActions, depsMgr dependencies.DependenciesManager, monitoringMgr MonitoringManager) *cobra.Command { o := new(CliCmdOptions) cmd := &cobra.Command{ Use: "cli", @@ -116,13 +116,13 @@ using docker compose command behind the scenes. } switch o.nodeType { case NodeTypeFullNode: - return setupFullNode(p, o, actions, depsMgr) + return setupFullNode(p, o, actions, depsMgr, monitoringMgr) case NodeTypeExecution: - return setupExecutionNode(p, o, actions, depsMgr) + return setupExecutionNode(p, o, actions, depsMgr, monitoringMgr) case NodeTypeConsensus: - return setupConsensusNode(p, o, actions, depsMgr) + return setupConsensusNode(p, o, actions, depsMgr, monitoringMgr) case NodeTypeValidator: - return setupValidatorNode(p, o, actions, depsMgr) + return setupValidatorNode(p, o, actions, depsMgr, monitoringMgr) } return nil }, @@ -130,7 +130,7 @@ using docker compose command behind the scenes. return cmd } -func setupFullNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsManager dependencies.DependenciesManager) (err error) { +func setupFullNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsManager dependencies.DependenciesManager, monitoringMgr MonitoringManager) (err error) { o.genData.Services = []string{"execution", "consensus"} if err := confirmWithValidator(p, o); err != nil { return err @@ -193,6 +193,9 @@ func setupFullNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, deps if err := setupJWT(p, o, false); err != nil { return err } + if err := confirmEnableMonitoring(p, o); err != nil { + return err + } // Call generate action o.genData, err = a.Generate(actions.GenerateOptions{ GenerationData: o.genData, @@ -201,10 +204,10 @@ func setupFullNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, deps if err != nil { return err } - return postGenerate(p, o, a, depsManager) + return postGenerate(p, o, a, depsManager, monitoringMgr) } -func setupExecutionNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsManager dependencies.DependenciesManager) (err error) { +func setupExecutionNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsManager dependencies.DependenciesManager, monitoringMgr MonitoringManager) (err error) { o.genData.Services = []string{"execution"} if err := selectExecutionClient(p, o); err != nil { return err @@ -223,6 +226,9 @@ func setupExecutionNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, if err := setupJWT(p, o, true); err != nil { return err } + if err := confirmEnableMonitoring(p, o); err != nil { + return err + } o.genData, err = a.Generate(actions.GenerateOptions{ GenerationData: o.genData, GenerationPath: o.generationPath, @@ -230,10 +236,10 @@ func setupExecutionNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, if err != nil { return err } - return postGenerate(p, o, a, depsManager) + return postGenerate(p, o, a, depsManager, monitoringMgr) } -func setupConsensusNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsManager dependencies.DependenciesManager) (err error) { +func setupConsensusNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsManager dependencies.DependenciesManager, monitoringMgr MonitoringManager) (err error) { o.genData.Services = []string{"consensus"} if err := selectConsensusClient(p, o); err != nil { return err @@ -268,6 +274,9 @@ func setupConsensusNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, if err := setupJWT(p, o, true); err != nil { return err } + if err := confirmEnableMonitoring(p, o); err != nil { + return err + } o.genData, err = a.Generate(actions.GenerateOptions{ GenerationData: o.genData, GenerationPath: o.generationPath, @@ -275,10 +284,10 @@ func setupConsensusNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, if err != nil { return err } - return postGenerate(p, o, a, depsManager) + return postGenerate(p, o, a, depsManager, monitoringMgr) } -func setupValidatorNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsManager dependencies.DependenciesManager) (err error) { +func setupValidatorNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsManager dependencies.DependenciesManager, monitoringMgr MonitoringManager) (err error) { o.genData.Services = []string{"validator"} if err := selectValidatorClient(p, o); err != nil { return err @@ -306,6 +315,9 @@ func setupValidatorNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, } o.genData.MevBoostOnValidator = o.withMevBoost } + if err := confirmEnableMonitoring(p, o); err != nil { + return err + } o.genData, err = a.Generate(actions.GenerateOptions{ GenerationData: o.genData, GenerationPath: o.generationPath, @@ -313,7 +325,7 @@ func setupValidatorNode(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, if err != nil { return err } - return postGenerate(p, o, a, depsManager) + return postGenerate(p, o, a, depsManager, monitoringMgr) } func setupJWT(p ui.Prompter, o *CliCmdOptions, skip bool) error { @@ -345,7 +357,7 @@ func setupJWT(p ui.Prompter, o *CliCmdOptions, skip bool) error { return nil } -func postGenerate(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsMgr dependencies.DependenciesManager) error { +func postGenerate(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsMgr dependencies.DependenciesManager, monitoringMgr MonitoringManager) error { if o.withValidator || o.nodeType == NodeTypeValidator { if err := generateKeystore(p, o, a, depsMgr); err != nil { return err @@ -385,6 +397,11 @@ func postGenerate(p ui.Prompter, o *CliCmdOptions, a actions.SedgeActions, depsM }); err != nil { return err } + if o.enableMonitoring { + if err := InitMonitoring(true, true, monitoringMgr, nil); err != nil { + return err + } + } if o.withValidator { log.Info(configs.HappyStakingRun) } else { @@ -818,6 +835,11 @@ func confirmEnableMEVBoost(p ui.Prompter, o *CliCmdOptions) (err error) { return } +func confirmEnableMonitoring(p ui.Prompter, o *CliCmdOptions) (err error) { + o.enableMonitoring, err = p.Confirm("Do you want to enable the monitoring stack?", false) + return +} + func inputCustomNetworkConfig(p ui.Prompter, o *CliCmdOptions) (err error) { o.genData.CustomNetworkConfigPath, err = p.InputFilePath("Custom network config file path", "", true, ".yml", ".yaml") if err != nil { diff --git a/cli/cli_test.go b/cli/cli_test.go index 5c6f68385..abf0359d9 100644 --- a/cli/cli_test.go +++ b/cli/cli_test.go @@ -117,6 +117,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().EthAddress("Please enter the Fee Recipient address", "", true).Return("0x2d07a21ebadde0c13e6b91022a7e5722eb6bf5d5", nil), prompter.EXPECT().Confirm("Do you want to expose all ports?", false).Return(true, nil), prompter.EXPECT().Select("Select JWT source", "", []string{SourceTypeCreate, SourceTypeExisting}).Return(0, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -182,6 +183,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().EthAddress("Please enter the Fee Recipient address (press enter to skip it)", "", false).Return("0x2d07a21ebadde0c13e6b91022a7e5722eb6bf5d5", nil), prompter.EXPECT().Confirm("Do you want to expose all ports?", false).Return(true, nil), prompter.EXPECT().Select("Select JWT source", "", []string{SourceTypeCreate, SourceTypeExisting}).Return(0, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -226,6 +228,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().EthAddress("Please enter the Fee Recipient address (press enter to skip it)", "", false).Return("0x2d07a21ebadde0c13e6b91022a7e5722eb6bf5d5", nil), prompter.EXPECT().Confirm("Do you want to expose all ports?", false).Return(true, nil), prompter.EXPECT().Select("Select JWT source", "", []string{SourceTypeCreate, SourceTypeExisting}).Return(0, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -259,6 +262,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().Select("Select execution client", "", ETHClients["execution"]).Return(0, nil), prompter.EXPECT().Confirm("Do you want to expose all ports?", false).Return(true, nil), prompter.EXPECT().Select("Select JWT source", "", []string{SourceTypeCreate, SourceTypeExisting, SourceTypeSkip}).Return(2, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -303,6 +307,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().Select("Select execution client", "", ETHClients["execution"]).Return(0, nil), prompter.EXPECT().Confirm("Do you want to expose all ports?", false).Return(true, nil), prompter.EXPECT().Select("Select JWT source", "", []string{SourceTypeCreate, SourceTypeExisting, SourceTypeSkip}).Return(2, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -358,6 +363,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().EthAddress("Please enter the Fee Recipient address (press enter to skip it)", "", false).Return("0x2d07a21ebadde0c13e8b91022a7e5732eb6bf5d5", nil), prompter.EXPECT().Confirm("Do you want to expose all ports?", false).Return(true, nil), prompter.EXPECT().Select("Select JWT source", "", []string{SourceTypeCreate, SourceTypeExisting, SourceTypeSkip}).Return(0, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -402,6 +408,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().EthAddress("Please enter the Fee Recipient address (press enter to skip it)", "", false).Return("0x2d07a21ebadde0c13e8b91022a7e5732eb6bf5d5", nil), prompter.EXPECT().Confirm("Do you want to expose all ports?", false).Return(false, nil), prompter.EXPECT().Select("Select JWT source", "", []string{SourceTypeCreate, SourceTypeExisting, SourceTypeSkip}).Return(0, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -444,6 +451,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().InputInt64("Validator grace period. This is the number of epochs the validator will wait for security reasons before starting", int64(1)).Return(int64(2), nil), prompter.EXPECT().EthAddress("Please enter the Fee Recipient address", "", true).Return("0x2d07a31ebadce0a13e8a91022a5e5732eb6bf5d5", nil), prompter.EXPECT().Confirm("Enable MEV Boost?", true).Return(true, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -509,6 +517,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().EthAddress("Please enter the Fee Recipient address (press enter to skip it)", "", false).Return("0x2d07a21ebadde0c13e8b91022a7e5732eb6bf5d5", nil), prompter.EXPECT().Confirm("Do you want to expose all ports?", false).Return(true, nil), prompter.EXPECT().Select("Select JWT source", "", []string{SourceTypeCreate, SourceTypeExisting, SourceTypeSkip}).Return(0, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -567,6 +576,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().InputURL("Checkpoint sync URL", configs.NetworksConfigs()[genData.Network].CheckpointSyncURL, false).Return("http://checkpoint.sync", nil), prompter.EXPECT().Confirm("Do you want to expose all ports?", false).Return(true, nil), prompter.EXPECT().Select("Select JWT source", "", []string{SourceTypeCreate, SourceTypeExisting}).Return(0, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -645,6 +655,7 @@ func TestCli(t *testing.T) { prompter.EXPECT().InputURL("Checkpoint sync URL", configs.NetworksConfigs()[genData.Network].CheckpointSyncURL, false).Return("http://checkpoint.sync", nil), prompter.EXPECT().Confirm("Do you want to expose all ports?", false).Return(true, nil), prompter.EXPECT().Select("Select JWT source", "", []string{SourceTypeCreate, SourceTypeExisting}).Return(0, nil), + prompter.EXPECT().Confirm("Do you want to enable the monitoring stack?", false).Return(false, nil), sedgeActions.EXPECT().Generate(gomock.Eq(actions.GenerateOptions{ GenerationPath: generationPath, GenerationData: genData, @@ -681,11 +692,12 @@ func TestCli(t *testing.T) { sedgeActions := sedge_mocks.NewMockSedgeActions(ctrl) prompter := sedge_mocks.NewMockPrompter(ctrl) depsMgr := sedge_mocks.NewMockDependenciesManager(ctrl) + monitoringMgr := sedge_mocks.NewMockMonitoringManager(ctrl) defer ctrl.Finish() tt.setup(t, sedgeActions, prompter, depsMgr) - c := CliCmd(prompter, sedgeActions, depsMgr) + c := CliCmd(prompter, sedgeActions, depsMgr, monitoringMgr) c.Execute() }) } diff --git a/cli/monitoring.go b/cli/monitoring.go index 64e11d32d..8ea643def 100644 --- a/cli/monitoring.go +++ b/cli/monitoring.go @@ -15,40 +15,116 @@ limitations under the License. */ package cli +// TODO: Add prompt for monitoring in cli, add lido flag here to run lido-exporter, +// implement Add target in serviceAPI as adding that service as target to prometheus (node & lido exporters) ,use image we created for lido-exporter import ( "errors" "fmt" + "strconv" + "time" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/NethermindEth/sedge/configs" "github.com/NethermindEth/sedge/internal/common" "github.com/NethermindEth/sedge/internal/monitoring" + "github.com/NethermindEth/sedge/internal/ui" + "github.com/NethermindEth/sedge/internal/utils" + sedgeOpts "github.com/NethermindEth/sedge/internal/pkg/options" + lidoExporter "github.com/NethermindEth/sedge/internal/monitoring/services/lido_exporter" +) + +var ( + lido bool +) +const ( + initMonitoring = "init" + cleanMonitoring = "clean" ) func MonitoringCmd(mgr MonitoringManager) *cobra.Command { - cmd := cobra.Command{ + cmd := &cobra.Command{ Use: "monitoring [init|clean]", Short: "Manage the monitoring stack", Long: "Manage the monitoring stack. Use 'init' to install and run, or 'clean' to stop and uninstall.", - Args: cobra.ExactArgs(1), + } + cmd.AddCommand(InitSubCmd(mgr)) + cmd.AddCommand(CleanSubCmd(mgr)) + + return cmd +} + +func InitSubCmd(mgr MonitoringManager) *cobra.Command { + var additionalServices []monitoring.ServiceAPI + cmd := &cobra.Command{ + Use: "init", + Short: "Initialize the monitoring stack", + } + cmd.AddCommand(DefaultSubCmd(mgr, additionalServices)) + cmd.AddCommand(LidoSubCmd(mgr, additionalServices)) + + return cmd +} + +func CleanSubCmd(mgr MonitoringManager) *cobra.Command { + return &cobra.Command{ + Use: "clean", + Short: "Clean the monitoring stack", RunE: func(cmd *cobra.Command, args []string) error { - switch args[0] { - case "init": - return InitMonitoring(true, true, mgr) - case "clean": - return CleanMonitoring(mgr) - default: - return fmt.Errorf("invalid argument: %s. Use 'init' or 'clean'", args[0]) + return CleanMonitoring(mgr) + }, + } +} + +func LidoSubCmd(mgr MonitoringManager, additionalServices []monitoring.ServiceAPI) *cobra.Command { + lido:= &lidoExporter.LidoExporterParams{} + cmd:= &cobra.Command{ + Use: "lido", + Short: "Configure Lido CSM Node monitoring", + Args: cobra.NoArgs, + PreRunE: func(cmd *cobra.Command, args []string) error { + if lido.NodeOperatorID == "" && lido.RewardAddress == "" { + return errors.New("Node Operator ID or Reward Address is required") } + if err := ui.EthAddressValidator(rewardAddress, false); err != nil && len(args) != 0 { + return err + } + additionalServices = append(additionalServices, lidoExporter.NewLidoExporter(*lido)) + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + return InitMonitoring(true, true, mgr, additionalServices) }, } - return &cmd + cmd.Flags().StringVar(&lido.NodeOperatorID,"node-operator-id", "", "Node Operator ID") + cmd.Flags().StringVar(&lido.RewardAddress,"reward-address", "", "Reward address of Node Operator. It is used to calculate Node Operator ID if not set") + cmd.Flags().StringVar(&lido.Network,"network", "holesky", "Network name") + cmd.Flags().StringSliceVar(&lido.RPCEndpoints,"rpc-endpoints", nil, "List of Ethereum HTTP RPC endpoints") + cmd.Flags().StringSliceVar(&lido.WSEndpoints,"ws-endpoints", nil, "List of Ethereum WebSocket RPC endpoints") + cmd.Flags().Uint16Var(&lido.Port,"port", 8080, "Port where the metrics will be exported.") + cmd.Flags().DurationVar(&lido.ScrapeTime,"scrape-time", 30*time.Second, "Time interval for scraping metrics. Values should be in the format of 10s, 1m, 1h, etc.") + cmd.Flags().StringVar(&logLevel, "log-level", "info", "Set Log Level, e.g panic, fatal, error, warn, warning, info, debug, trace") + + return cmd } +func DefaultSubCmd(mgr MonitoringManager, additionalServices []monitoring.ServiceAPI) *cobra.Command { + cmd:= &cobra.Command{ + Use: "default", + Short: "Default monitoring configuration", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return InitMonitoring(true, true, mgr, nil) + }, + } + return cmd +} + + // Init initializes the Monitoring Stack. If install is true, it will install the Monitoring Stack if it is not installed. // If run is true, it will run the Monitoring Stack if it is not running. -func InitMonitoring(install, run bool, monitoringMgr MonitoringManager) error { +func InitMonitoring(install, run bool, monitoringMgr MonitoringManager, additionalServices []monitoring.ServiceAPI) error { // Check if the monitoring stack is installed. installStatus, err := monitoringMgr.InstallationStatus() if err != nil { @@ -68,6 +144,14 @@ func InitMonitoring(install, run bool, monitoringMgr MonitoringManager) error { return err } } + + // Add additional services to the monitoring manager + for _, service := range additionalServices { + if err := monitoringMgr.AddService(service); err != nil { + return fmt.Errorf("failed to add service %s: %w", service.Name(), err) + } + } + // Check if the monitoring stack is running. status, err := monitoringMgr.Status() if err != nil { @@ -107,3 +191,81 @@ func CleanMonitoring(monitoringMgr MonitoringManager) error { } return nil } + +func InputLidoExporterParams(p ui.Prompter) (*lidoExporter.LidoExporterParams, error) { + lido := sedgeOpts.CreateSedgeOptions(sedgeOpts.LidoNode) + params := &lidoExporter.LidoExporterParams{} + var err error + + // Node Operator ID + params.NodeOperatorID, err = p.Input("Enter Node Operator ID (leave empty if using Reward Address)", "", false, nil) + if err != nil { + return params, err + } + + // Reward Address + if params.NodeOperatorID == "" { + params.RewardAddress, err = p.EthAddress("Enter Reward Address of Node Operator", "", true) + if err != nil { + return params, err + } + } + + // Network + options := lido.SupportedNetworks() + index, err := p.Select("Select network", "holesky", options) + if err != nil { + return params, err + } + params.Network = options[index] + + // RPC URLs + defaultRPCURLs, err := configs.GetPublicRPCs(params.Network) + rpcURLs, err := p.InputList("Insert Ethereum HTTP RPC endpoints if you don't want to use the default values listed below", defaultRPCURLs, func(list []string) error { + badUri, ok := utils.UriValidator(list) + if !ok { + return fmt.Errorf(configs.InvalidUrlFlagError, "rpc", badUri) + } + return nil + }) + params.RPCEndpoints = rpcURLs + + // WSs URLs + defaultWSURLs, err := configs.GetPublicWSs(params.Network) + wsURLs, err := p.InputList("Insert Ethereum WebSocket RPC endpoints if you don't want to use the default values listed below", defaultWSURLs, nil) + if err != nil { + return params, err + } + params.WSEndpoints = wsURLs + + // Port + portStr, err := p.Input("Enter port for exporting metrics", "8080", false, nil) + if err != nil { + return params, err + } + port64, err := strconv.ParseUint(portStr, 10, 16) + if err != nil { + return params, fmt.Errorf("invalid port number: %v", err) + } + params.Port = uint16(port64) + + // Scrape time + scrapeTimeStr, err := p.Input("Enter scrape time interval (e.g., 30s, 1m, 1h)", "30s", false, nil) + if err != nil { + return params, err + } + params.ScrapeTime, err = time.ParseDuration(scrapeTimeStr) + if err != nil { + return params, fmt.Errorf("invalid scrape time: %v", err) + } + + //Log level + logOptions := []string{"panic", "fatal", "error", "warn", "warning", "info", "debug", "trace"} + index, err = p.Select("Select log level", "info", logOptions) + if err != nil { + return params, err + } + params.LogLevel = logOptions[index] + + return params, nil +} \ No newline at end of file diff --git a/cli/monitoringManager.go b/cli/monitoringManager.go index a02f4d0e9..cb8928af9 100644 --- a/cli/monitoringManager.go +++ b/cli/monitoringManager.go @@ -5,6 +5,7 @@ package cli import ( "github.com/NethermindEth/sedge/internal/common" "github.com/NethermindEth/sedge/internal/monitoring/services/types" + "github.com/NethermindEth/sedge/internal/monitoring" ) type MonitoringManager interface { @@ -39,4 +40,7 @@ type MonitoringManager interface { // ServiceEndpoints returns the endpoints of the monitoring services. ServiceEndpoints() map[string]string + + // AddService adds a new service to the monitoring stack. + AddService(service monitoring.ServiceAPI) error } diff --git a/cli/monitoring_test.go b/cli/monitoring_test.go index 63e091efd..4bc3e402b 100644 --- a/cli/monitoring_test.go +++ b/cli/monitoring_test.go @@ -38,8 +38,8 @@ func TestMonitoringCmd(t *testing.T) { isErr bool }{ { - name: "valid monitoring init", - flags: []string{"init"}, + name: "valid monitoring init: default", + flags: []string{"init", "default"}, mocker: func(t *testing.T, ctrl *gomock.Controller) *sedge_mocks.MockMonitoringManager { mockManager := sedge_mocks.NewMockMonitoringManager(ctrl) gomock.InOrder( @@ -67,8 +67,25 @@ func TestMonitoringCmd(t *testing.T) { isErr: false, }, { - name: "invalid action", - flags: []string{"invalid"}, + name: "valid monitoring init: lido", + flags: []string{"init", "lido", "--node-operator-id", "1"}, + mocker: func(t *testing.T, ctrl *gomock.Controller) *sedge_mocks.MockMonitoringManager { + mockManager := sedge_mocks.NewMockMonitoringManager(ctrl) + gomock.InOrder( + mockManager.EXPECT().InstallationStatus().Return(common.NotInstalled, nil).AnyTimes(), + mockManager.EXPECT().InstallStack().Return(nil).AnyTimes(), + mockManager.EXPECT().AddService(gomock.Any()).Return(nil).AnyTimes(), + mockManager.EXPECT().Status().Return(common.Created, nil).AnyTimes(), + mockManager.EXPECT().Run().Return(nil).AnyTimes(), + mockManager.EXPECT().Init().Return(nil).AnyTimes(), + ) + return mockManager + }, + isErr: false, + }, + { + name: "invalid monitoring init: lido, no nodeID or reward address", + flags: []string{"init", "lido"}, mocker: func(t *testing.T, ctrl *gomock.Controller) *sedge_mocks.MockMonitoringManager { return sedge_mocks.NewMockMonitoringManager(ctrl) }, @@ -285,7 +302,7 @@ func TestInitMonitoring(t *testing.T) { // Get monitoring manager mock monitoringMgr := tt.mocker(t, ctrl) - err := InitMonitoring(true, true, monitoringMgr) + err := InitMonitoring(true, true, monitoringMgr, nil) if tt.wantErr { require.Error(t, err) } else { diff --git a/cmd/sedge/main.go b/cmd/sedge/main.go index f35d816f4..0fba9000a 100644 --- a/cmd/sedge/main.go +++ b/cmd/sedge/main.go @@ -76,7 +76,7 @@ func main() { prometheus.NewPrometheus(), node_exporter.NewNodeExporter(), } - monitoringManager := monitoring.NewMonitoringManager( + monitoringMgr := monitoring.NewMonitoringManager( monitoringServices, composeManager, dockerServiceManager, @@ -93,7 +93,7 @@ func main() { sedgeCmd := cli.RootCmd() sedgeCmd.AddCommand( - cli.CliCmd(prompt, sedgeActions, depsMgr), + cli.CliCmd(prompt, sedgeActions, depsMgr, monitoringMgr), cli.KeysCmd(cmdRunner, prompt), cli.DownCmd(cmdRunner, sedgeActions, depsMgr), cli.ClientsCmd(), @@ -108,7 +108,7 @@ func main() { cli.DependenciesCommand(depsMgr), cli.ShowCmd(cmdRunner, sedgeActions, depsMgr), cli.LidoStatusCmd(), - cli.MonitoringCmd(monitoringManager), + cli.MonitoringCmd(monitoringMgr), ) sedgeCmd.SilenceErrors = true sedgeCmd.SilenceUsage = true diff --git a/e2e/sedge/monitoring_stack_test.go b/e2e/sedge/monitoring_stack_test.go index aeea81a43..982cf872a 100644 --- a/e2e/sedge/monitoring_stack_test.go +++ b/e2e/sedge/monitoring_stack_test.go @@ -62,7 +62,7 @@ func TestE2E_MonitoringStack_Init(t *testing.T) { nil, // Act func(t *testing.T, binaryPath string, dataDirPath string) { - runErr = base.RunCommand(t, binaryPath, "sedge", "monitoring", "init") + runErr = base.RunCommand(t, binaryPath, "sedge", "monitoring", "init","default") }, // Assert func(t *testing.T, dataDirPath string) { @@ -90,7 +90,7 @@ func TestE2E_MonitoringStack_NotReinstalled(t *testing.T) { t, // Arrange func(t *testing.T, sedgePath string) error { - err := base.RunCommand(t, sedgePath, "sedge", "monitoring", "init") + err := base.RunCommand(t, sedgePath, "sedge", "monitoring", "init","default") if err != nil { return err } @@ -141,7 +141,7 @@ func TestE2E_MonitoringStack_Clean(t *testing.T) { t, // Arrange func(t *testing.T, sedgePath string) error { - return base.RunCommand(t, sedgePath, "sedge", "monitoring", "init") + return base.RunCommand(t, sedgePath, "sedge", "monitoring", "init","default") }, // Act func(t *testing.T, binaryPath string, dataDirPath string) { diff --git a/internal/monitoring/constants.go b/internal/monitoring/constants.go index 0434f2329..d24d092ba 100644 --- a/internal/monitoring/constants.go +++ b/internal/monitoring/constants.go @@ -22,6 +22,8 @@ const ( GrafanaContainerName = "sedge_grafana" NodeExporterServiceName = "node_exporter" NodeExporterContainerName = "sedge_node_exporter" + LidoExporterServiceName = "lido_exporter" + LidoExporterContainerName = "sedge_lido_exporter" monitoringPath = "monitoring" InstanceIDLabel = "instance_id" ) diff --git a/internal/monitoring/data/monitoring.go b/internal/monitoring/data/monitoring.go index c9a03f926..35270975a 100644 --- a/internal/monitoring/data/monitoring.go +++ b/internal/monitoring/data/monitoring.go @@ -16,12 +16,14 @@ limitations under the License. package data import ( + "bytes" "errors" "fmt" "io" "io/fs" "os" "path/filepath" + "text/template" "github.com/NethermindEth/sedge/internal/monitoring/locker" "github.com/spf13/afero" @@ -68,48 +70,72 @@ func (m *MonitoringStack) unlock() error { } // Setup sets up the monitoring stack with the given environment variables and -// docker-compose.yml file. +// docker-compose_base.tmpl file. func (m *MonitoringStack) Setup(env map[string]string, monitoringFs fs.FS) (err error) { - err = m.lock() + err = m.lock() + if err != nil { + return err + } + defer func() { + unlockErr := m.unlock() + if err == nil { + err = unlockErr + } + }() + + // Create .env file + envFile, err := m.fs.Create(filepath.Join(m.path, ".env")) + if err != nil { + return err + } + for k, v := range env { + _, err = envFile.WriteString(fmt.Sprintf("%s=%s\n", k, v)) + if err != nil { + return fmt.Errorf("failed to write to .env file: %w", err) + } + } + defer envFile.Close() + + // Read the main Docker Compose template + rawBaseTmp, err := monitoringFs.Open("services/docker-compose_base.tmpl") if err != nil { - return err + return fmt.Errorf("error opening docker-compose template: %w", err) } - defer func() { - unlockErr := m.unlock() - if err == nil { - err = unlockErr - } - }() + defer rawBaseTmp.Close() - // Create .env file - envFile, err := m.fs.Create(filepath.Join(m.path, ".env")) + // Read the content of the template file + rawBaseTmpContent, err := io.ReadAll(rawBaseTmp) if err != nil { - return err - } - for k, v := range env { - _, err = envFile.WriteString(fmt.Sprintf("%s=%s\n", k, v)) - if err != nil { - return fmt.Errorf("failed to write to .env file: %w", err) - } + return fmt.Errorf("error reading docker-compose template: %w", err) } - defer envFile.Close() - // Copy docker-compose.yml - mComposeFile, err := monitoringFs.Open("script/docker-compose.yml") + // Parse the base template + baseTmp, err := template.New("docker-compose").Parse(string(rawBaseTmpContent)) if err != nil { - return err + return fmt.Errorf("error parsing docker-compose template: %w", err) + } + + // Create a buffer to hold the executed template content + var buf bytes.Buffer + + // Execute the base template without any additional data + if err := baseTmp.Execute(&buf, nil); err != nil { + return fmt.Errorf("error executing docker-compose template: %w", err) } - defer mComposeFile.Close() + + // Write the executed template content to the final Docker Compose file composeFile, err := m.fs.Create(filepath.Join(m.path, "docker-compose.yml")) if err != nil { - return err + return fmt.Errorf("error creating docker-compose.yml file: %w", err) } defer composeFile.Close() - if _, err := io.Copy(composeFile, mComposeFile); err != nil { - return err + + // Write the buffer content to the file + if _, err := io.Copy(composeFile, &buf); err != nil { + return fmt.Errorf("error writing docker-compose.yml file: %w", err) } - return nil + return nil } // CreateDir creates a new directory in the monitoring stack at the given path. @@ -241,3 +267,4 @@ func (m *MonitoringStack) Cleanup(force bool) (err error) { } return m.fs.RemoveAll(m.path) } + diff --git a/internal/monitoring/data/monitoring_test.go b/internal/monitoring/data/monitoring_test.go index 1f3f704a4..0e7133fdc 100644 --- a/internal/monitoring/data/monitoring_test.go +++ b/internal/monitoring/data/monitoring_test.go @@ -16,16 +16,17 @@ limitations under the License. package data import ( + "bytes" "errors" - "io/fs" "path/filepath" "strings" "sync" "testing" + "text/template" "time" - "github.com/NethermindEth/sedge/internal/monitoring/data/testdata" mocks "github.com/NethermindEth/sedge/internal/monitoring/locker/mocks" + "github.com/NethermindEth/sedge/internal/monitoring/services/templates" "github.com/NethermindEth/sedge/internal/monitoring/utils" "github.com/golang/mock/gomock" "github.com/spf13/afero" @@ -63,142 +64,141 @@ func TestInit(t *testing.T) { } func TestSetup(t *testing.T) { - t.Parallel() - - okLocker := func(t *testing.T) *mocks.MockLocker { - // Create a mock locker - ctrl := gomock.NewController(t) - locker := mocks.NewMockLocker(ctrl) - - // Expect the lock to be acquired - gomock.InOrder( - locker.EXPECT().Lock().Return(nil), - locker.EXPECT().Locked().Return(true), - locker.EXPECT().Unlock().Return(nil), - ) - return locker - } - - tests := []struct { - name string - env map[string]string - testFs fs.FS - mocker func(*testing.T) *mocks.MockLocker - wantErr bool - }{ - { - name: "success", - env: map[string]string{ - "NODE_NAME": "node1", - }, - testFs: testdata.TestData, - mocker: okLocker, - wantErr: false, - }, - { - name: "missing docker-compose.yml", - env: map[string]string{ - "ERROR": "error", - }, - testFs: testdata.Empty, - mocker: okLocker, - wantErr: true, - }, - { - name: "empty .env", - env: map[string]string{}, - testFs: testdata.TestData, - mocker: okLocker, - wantErr: false, - }, - { - name: "unlock error", - env: map[string]string{ - "ERROR": "error", - }, - testFs: testdata.TestData, - mocker: func(t *testing.T) *mocks.MockLocker { - // Create a mock locker - ctrl := gomock.NewController(t) - locker := mocks.NewMockLocker(ctrl) - - // Expect the lock to be acquired - gomock.InOrder( - locker.EXPECT().Lock().Return(nil), - locker.EXPECT().Locked().Return(false), - ) - return locker - }, - wantErr: true, - }, - { - name: "lock error", - env: map[string]string{ - "ERROR": "error", - }, - testFs: testdata.TestData, - mocker: func(t *testing.T) *mocks.MockLocker { - // Create a mock locker - ctrl := gomock.NewController(t) - locker := mocks.NewMockLocker(ctrl) - - // Expect the lock to be acquired - locker.EXPECT().Lock().Return(errors.New("lock error")) - return locker - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Create an in-memory filesystem - afs := afero.NewMemMapFs() - - // Create a new MonitoringStack with the in-memory filesystem - stack := &MonitoringStack{ - path: "/", - l: tt.mocker(t), - fs: afs, - } - - err := stack.Setup(tt.env, tt.testFs) - if tt.wantErr { - assert.Error(t, err) - return - } else { - assert.NoError(t, err) - } - - // Check that the files were created - exists, err := afero.Exists(afs, "/.env") - assert.NoError(t, err) - assert.True(t, exists) - - // Parse .env file and compare with the expected values - env, error := afero.ReadFile(afs, "/.env") - require.NoError(t, error) - gotEnv := make(map[string]string) - for _, line := range strings.Split(string(env), "\n") { - parts := strings.Split(line, "=") - if len(parts) == 2 { - gotEnv[parts[0]] = parts[1] - } - } - assert.EqualValues(t, tt.env, gotEnv) - - exists, err = afero.Exists(afs, "/docker-compose.yml") - assert.NoError(t, err) - assert.True(t, exists) - - // Compare docker-compose.yml with the expected file - gotCmp, err := afero.ReadFile(afs, "/docker-compose.yml") - require.NoError(t, err) - wantCmp, err := fs.ReadFile(tt.testFs, "script/docker-compose.yml") - require.NoError(t, err) - assert.Equal(t, string(wantCmp), string(gotCmp)) - }) - } + t.Parallel() + + okLocker := func(t *testing.T) *mocks.MockLocker { + ctrl := gomock.NewController(t) + locker := mocks.NewMockLocker(ctrl) + gomock.InOrder( + locker.EXPECT().Lock().Return(nil), + locker.EXPECT().Locked().Return(true), + locker.EXPECT().Unlock().Return(nil), + ) + return locker + } + + mockTemplate := `{{/* docker-compose_base.tmpl */}} +{{ define "docker-compose" }} +services: + service1: + container_name: service1 + image: ${IMAGE} + ports: + - ${PORT}:9090 + networks: + - sedge +networks: + sedge: + name: sedge-network + external: true +{{ end }} +` + + tests := []struct { + name string + env map[string]string + mocker func(*testing.T) *mocks.MockLocker + wantErr bool + }{ + { + name: "success", + env: map[string]string{ + "IMAGE": "myimage:latest", + "PORT": "8080", + }, + mocker: okLocker, + wantErr: false, + }, + { + name: "empty .env", + env: map[string]string{}, + mocker: okLocker, + wantErr: false, + }, + { + name: "unlock error", + env: map[string]string{ + "ERROR": "error", + }, + mocker: func(t *testing.T) *mocks.MockLocker { + ctrl := gomock.NewController(t) + locker := mocks.NewMockLocker(ctrl) + gomock.InOrder( + locker.EXPECT().Lock().Return(nil), + locker.EXPECT().Locked().Return(false), + ) + return locker + }, + wantErr: true, + }, + { + name: "lock error", + env: map[string]string{ + "ERROR": "error", + }, + mocker: func(t *testing.T) *mocks.MockLocker { + ctrl := gomock.NewController(t) + locker := mocks.NewMockLocker(ctrl) + locker.EXPECT().Lock().Return(errors.New("lock error")) + return locker + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + afs := afero.NewMemMapFs() + mockFs := &afero.MemMapFs{} + afero.WriteFile(mockFs, "services/docker-compose_base.tmpl", []byte(mockTemplate), 0644) + + stack := &MonitoringStack{ + path: "/", + l: tt.mocker(t), + fs: afs, + } + + err := stack.Setup(tt.env, afero.NewIOFS(mockFs)) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + + // Check .env file + exists, err := afero.Exists(afs, "/.env") + assert.NoError(t, err) + assert.True(t, exists) + + env, err := afero.ReadFile(afs, "/.env") + require.NoError(t, err) + gotEnv := make(map[string]string) + for _, line := range strings.Split(string(env), "\n") { + parts := strings.Split(line, "=") + if len(parts) == 2 { + gotEnv[parts[0]] = parts[1] + } + } + assert.EqualValues(t, tt.env, gotEnv) + + // Check docker-compose.yml + exists, err = afero.Exists(afs, "/docker-compose.yml") + assert.NoError(t, err) + assert.True(t, exists) + + gotCmp, err := afero.ReadFile(afs, "/docker-compose.yml") + require.NoError(t, err) + + // Parse the template with the test data + tmpl, err := template.New("test").Parse(mockTemplate) + require.NoError(t, err) + var expectedBuf bytes.Buffer + err = tmpl.ExecuteTemplate(&expectedBuf, "docker-compose", nil) + require.NoError(t, err) + + assert.Equal(t, expectedBuf.String(), string(gotCmp)) + }) + } } func TestCreateDir(t *testing.T) { @@ -816,114 +816,112 @@ func TestPath(t *testing.T) { } func TestCleanup(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - mocker func(*testing.T) (*mocks.MockLocker, afero.Fs) - force bool - notInstall bool - wantErr bool - }{ - { - name: "ok, force false", - mocker: func(t *testing.T) (*mocks.MockLocker, afero.Fs) { - // Create a mock locker - ctrl := gomock.NewController(t) - locker := mocks.NewMockLocker(ctrl) - - // Expect the lock to be acquired - gomock.InOrder( - locker.EXPECT().New(utils.PathMatcher{Expected: filepath.Join("monitoring", ".lock")}).Return(locker), - locker.EXPECT().Lock().Return(nil), - locker.EXPECT().Locked().Return(true), - locker.EXPECT().Unlock().Return(nil), - ) - locker.EXPECT().Lock().Return(nil) - return locker, afero.NewMemMapFs() - }, - force: false, - }, - { - name: "ok, force true", - mocker: func(t *testing.T) (*mocks.MockLocker, afero.Fs) { - // Create a mock locker - ctrl := gomock.NewController(t) - locker := mocks.NewMockLocker(ctrl) - - // Expect the lock to be acquired - gomock.InOrder( - locker.EXPECT().New(utils.PathMatcher{Expected: filepath.Join("monitoring", ".lock")}).Return(locker), - locker.EXPECT().Lock().Return(nil), - locker.EXPECT().Locked().Return(true), - locker.EXPECT().Unlock().Return(nil), - ) - return locker, afero.NewMemMapFs() - }, - force: true, - }, - { - name: "not installed", - mocker: func(t *testing.T) (*mocks.MockLocker, afero.Fs) { - // Create a mock locker - ctrl := gomock.NewController(t) - locker := mocks.NewMockLocker(ctrl) - - // Expect the lock to be acquired - locker.EXPECT().Lock().Return(nil) - return locker, afero.NewMemMapFs() - }, - notInstall: true, - }, - { - name: "lock error", - mocker: func(t *testing.T) (*mocks.MockLocker, afero.Fs) { - // Create a mock locker - ctrl := gomock.NewController(t) - locker := mocks.NewMockLocker(ctrl) - - // Expect the lock to be acquired - gomock.InOrder( - locker.EXPECT().New(utils.PathMatcher{Expected: filepath.Join("monitoring", ".lock")}).Return(locker), - locker.EXPECT().Lock().Return(nil), - locker.EXPECT().Locked().Return(true), - locker.EXPECT().Unlock().Return(nil), - ) - locker.EXPECT().Lock().Return(errors.New("lock error")) - return locker, afero.NewMemMapFs() - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - locker, fs := tt.mocker(t) - stack := &MonitoringStack{ - path: "/monitoring", - l: locker, - fs: fs, - } - - // Install the stack - var err error - if !tt.notInstall { - err = stack.Init() - require.NoError(t, err) - err = stack.Setup(map[string]string{"NODE_NAME": "test"}, testdata.TestData) - require.NoError(t, err) - } - - err = stack.Cleanup(tt.force) - if tt.wantErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - // Check that monitoring stack has been removed - exists, err := afero.DirExists(fs, "/monitoring") - assert.NoError(t, err) - assert.False(t, exists) - } - }) - } + t.Parallel() + + tests := []struct { + name string + mocker func(*testing.T) (*mocks.MockLocker, afero.Fs) + force bool + notInstall bool + wantErr bool + }{ + { + name: "ok, force false", + mocker: func(t *testing.T) (*mocks.MockLocker, afero.Fs) { + ctrl := gomock.NewController(t) + locker := mocks.NewMockLocker(ctrl) + gomock.InOrder( + locker.EXPECT().New(utils.PathMatcher{Expected: filepath.Join("monitoring", ".lock")}).Return(locker), + locker.EXPECT().Lock().Return(nil), + locker.EXPECT().Locked().Return(true), + locker.EXPECT().Unlock().Return(nil), + ) + locker.EXPECT().Lock().Return(nil) + return locker, afero.NewMemMapFs() + }, + force: false, + }, + { + name: "ok, force true", + mocker: func(t *testing.T) (*mocks.MockLocker, afero.Fs) { + ctrl := gomock.NewController(t) + locker := mocks.NewMockLocker(ctrl) + gomock.InOrder( + locker.EXPECT().New(utils.PathMatcher{Expected: filepath.Join("monitoring", ".lock")}).Return(locker), + locker.EXPECT().Lock().Return(nil), + locker.EXPECT().Locked().Return(true), + locker.EXPECT().Unlock().Return(nil), + ) + return locker, afero.NewMemMapFs() + }, + force: true, + }, + { + name: "not installed", + mocker: func(t *testing.T) (*mocks.MockLocker, afero.Fs) { + ctrl := gomock.NewController(t) + locker := mocks.NewMockLocker(ctrl) + locker.EXPECT().Lock().Return(nil) + return locker, afero.NewMemMapFs() + }, + notInstall: true, + }, + { + name: "lock error", + mocker: func(t *testing.T) (*mocks.MockLocker, afero.Fs) { + ctrl := gomock.NewController(t) + locker := mocks.NewMockLocker(ctrl) + gomock.InOrder( + locker.EXPECT().New(utils.PathMatcher{Expected: filepath.Join("monitoring", ".lock")}).Return(locker), + locker.EXPECT().Lock().Return(nil), + locker.EXPECT().Locked().Return(true), + locker.EXPECT().Unlock().Return(nil), + ) + locker.EXPECT().Lock().Return(errors.New("lock error")) + return locker, afero.NewMemMapFs() + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + locker, fs := tt.mocker(t) + stack := &MonitoringStack{ + path: "/monitoring", + l: locker, + fs: fs, + } + + // Install the stack + var err error + if !tt.notInstall { + err = stack.Init() + require.NoError(t, err) + + // Write the template file to the in-memory filesystem + afero.WriteFile(fs, "services/docker-compose_base.tmpl", []byte(` +services: + {{.ServiceName}}: + image: {{.Image}} + ports: + - "{{.Port}}:{{.Port}}" +`), 0644) + + err = stack.Setup(map[string]string{"ServiceName": "myservice", "Image": "myimage:latest", "Port": "8080"}, templates.Services) + require.NoError(t, err) + } + + err = stack.Cleanup(tt.force) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + // Check that monitoring stack has been removed + exists, err := afero.DirExists(fs, "/monitoring") + assert.NoError(t, err) + assert.False(t, exists) + } + }) + } } diff --git a/internal/monitoring/data/testdata/empty/empty.txt b/internal/monitoring/data/testdata/empty/empty.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/internal/monitoring/data/testdata/env/empty-env b/internal/monitoring/data/testdata/env/empty-env deleted file mode 100644 index e69de29bb..000000000 diff --git a/internal/monitoring/data/testdata/env/with-values b/internal/monitoring/data/testdata/env/with-values deleted file mode 100644 index 282f239ea..000000000 --- a/internal/monitoring/data/testdata/env/with-values +++ /dev/null @@ -1,3 +0,0 @@ -MAIN_SERVICE_NAME=main-service -MAIN_PORT=8080 -NETWORK_NAME=sedge diff --git a/internal/monitoring/data/testdata/script/docker-compose.yml b/internal/monitoring/data/testdata/script/docker-compose.yml deleted file mode 100644 index 4399818e8..000000000 --- a/internal/monitoring/data/testdata/script/docker-compose.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: '3' - -services: - test: - image: alpine - command: sleep 1000 - networks: - - test - -networks: - test: - driver: bridge \ No newline at end of file diff --git a/internal/monitoring/data/testdata/testdata.go b/internal/monitoring/data/testdata/testdata.go deleted file mode 100644 index 0252c668e..000000000 --- a/internal/monitoring/data/testdata/testdata.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2022 Nethermind - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package testdata - -import ( - "embed" - "io" - "io/fs" - "testing" - - "github.com/spf13/afero" - "github.com/stretchr/testify/require" -) - -//go:embed all:* -var TestData embed.FS - -//go:embed empty -var Empty embed.FS - -func SetupProfileFS(t *testing.T, instanceName string, afs afero.Fs) string { - t.Helper() - instanceFs, err := fs.Sub(TestData, instanceName) - if err != nil { - t.Fatalf("failed to setup instance filesystem: %v", err) - } - - tempPath := t.TempDir() - - err = fs.WalkDir(instanceFs, ".", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - targetPath := tempPath + "/" + path - if d.IsDir() { - if err := afs.MkdirAll(targetPath, 0o755); err != nil { - return err - } - } else { - file, err := instanceFs.Open(path) - if err != nil { - return err - } - defer file.Close() - targetFile, err := afs.Create(targetPath) - if err != nil { - return err - } - defer targetFile.Close() - if _, err := io.Copy(targetFile, file); err != nil { - return err - } - } - return nil - }) - require.NoError(t, err) - - return tempPath -} - -func GetEnv(t *testing.T, envName string) io.ReadCloser { - t.Helper() - file, err := TestData.Open("env/" + envName) - require.NoError(t, err, "failed to open env file %s", envName) - return file -} diff --git a/internal/monitoring/gen.go b/internal/monitoring/gen.go index e55e3efd5..d13eb56f7 100644 --- a/internal/monitoring/gen.go +++ b/internal/monitoring/gen.go @@ -15,8 +15,8 @@ limitations under the License. */ package monitoring -//go:generate mockgen -package=sedge_mocks -destination=../../mocks/monitoringService.go github.com/NethermindEth/sedge/internal/monitoring ServiceAPI +//go:generate mockgen -package=mocks -destination=./mocks/monitoringService.go github.com/NethermindEth/sedge/internal/monitoring ServiceAPI -//go:generate mockgen -package=sedge_mocks -destination=../../mocks/composeManager.go github.com/NethermindEth/sedge/internal/monitoring ComposeManager +//go:generate mockgen -package=mocks -destination=./mocks/composeManager.go github.com/NethermindEth/sedge/internal/monitoring ComposeManager -//go:generate mockgen -package=sedge_mocks -destination=../../mocks/dockerManager.go github.com/NethermindEth/sedge/internal/monitoring DockerServiceManager +//go:generate mockgen -package=mocks -destination=./mocks/dockerManager.go github.com/NethermindEth/sedge/internal/monitoring DockerServiceManager diff --git a/internal/monitoring/monitoring.go b/internal/monitoring/monitoring.go index 61970d525..5bcb61a97 100644 --- a/internal/monitoring/monitoring.go +++ b/internal/monitoring/monitoring.go @@ -17,9 +17,10 @@ package monitoring import ( "bytes" - "embed" "fmt" + "html/template" "net" + "os" "path/filepath" "strconv" "strings" @@ -27,6 +28,7 @@ import ( "github.com/NethermindEth/sedge/internal/common" "github.com/NethermindEth/sedge/internal/monitoring/data" "github.com/NethermindEth/sedge/internal/monitoring/locker" + "github.com/NethermindEth/sedge/internal/monitoring/services/templates" "github.com/NethermindEth/sedge/internal/monitoring/services/types" "github.com/NethermindEth/sedge/internal/monitoring/utils" "github.com/NethermindEth/sedge/internal/pkg/commands" @@ -35,8 +37,6 @@ import ( funk "github.com/thoas/go-funk" ) -//go:embed script -var script embed.FS // MonitoringManager manages the monitoring services. It provides methods for initializing the monitoring stack, // adding and removing targets, running and stopping the monitoring stack, and checking the status of the monitoring stack. @@ -113,6 +113,7 @@ func (m *MonitoringManager) InstallStack() error { // Merge all dotEnv dotEnv := make(map[string]string) defaultPorts := make(map[string]uint16) + for _, service := range m.services { for k, v := range service.DotEnv() { dotEnv[k] = v @@ -147,7 +148,7 @@ func (m *MonitoringManager) InstallStack() error { } } - if err = m.stack.Setup(dotEnv, script); err != nil { + if err = m.stack.Setup(dotEnv, templates.Services); err != nil { return fmt.Errorf("%w: %w", ErrInstallingMonitoringMngr, err) } @@ -247,10 +248,9 @@ func (m *MonitoringManager) Stop() error { // Status checks the status of the containers in the monitoring stack and returns the status. func (m *MonitoringManager) Status() (status common.Status, err error) { - containers := []string{ - GrafanaContainerName, - PrometheusContainerName, - NodeExporterContainerName, + var containers []string + for _, service := range m.services{ + containers = append(containers, service.ContainerName()) } for _, container := range containers { @@ -281,7 +281,7 @@ func (m *MonitoringManager) InstallationStatus() (common.Status, error) { return common.NotInstalled, nil } -// Cleanup removes the monitoring stack. If force is true, it bypasses locks and removes the stack without running 'docker compose down'. +// Cleanup removes the monitoring stack. func (m *MonitoringManager) Cleanup() error { log.Info("Shutting down monitoring stack...") if err := m.composeManager.Down(commands.DockerComposeDownOptions{Path: filepath.Join(m.stack.Path(), "docker-compose.yml")}); err != nil { @@ -328,3 +328,125 @@ func (m *MonitoringManager) saveServiceIP() error { } return nil } + +// AddService adds a new service to the monitoring stack dynamically. +func (m *MonitoringManager) AddService(service ServiceAPI) error { + // Check if the service already exists + for _, existingService := range m.services { + if existingService.ContainerName() == service.ContainerName() { + return fmt.Errorf("service %s already exists", service.ContainerName()) + } + } + + // Add the new service to the list + m.services = append(m.services, service) + + // Get the new service's environment variables + dotEnv := service.DotEnv() + + // Initialize the new service + if err := service.Init(types.ServiceOptions{ + Stack: m.stack, + Dotenv: dotEnv, + }); err != nil { + return fmt.Errorf("failed to initialize service %s: %w", service.Name(), err) + } + // Setup the new service + if err := service.Setup(dotEnv); err != nil { + return fmt.Errorf("failed to setup service %s: %w", service.Name(), err) + } + + // Update the .env file in the stack + if err := m.updateEnvFile(dotEnv); err != nil { + return fmt.Errorf("failed to update .env file: %w", err) + } + + // Update the docker-compose.yml file + if err := m.updateDockerComposeFile(service); err != nil { + return fmt.Errorf("failed to update docker-compose.yml: %w", err) + } + // Create and start the new service's container + if err := m.composeManager.Create(commands.DockerComposeCreateOptions{Path: filepath.Join(m.stack.Path(), "docker-compose.yml")}); err != nil { + return fmt.Errorf("failed to create service container: %w", err) + } + + if err := m.composeManager.Up(commands.DockerComposeUpOptions{Path: filepath.Join(m.stack.Path(), "docker-compose.yml")}); err != nil { + return fmt.Errorf("failed to start service container: %w", err) + } + + // Save the new service's IP + if err := m.saveServiceIP(); err != nil { + return fmt.Errorf("failed to save service IP: %w", err) + } + + return nil +} + +// Helper method to update the .env file +func (m *MonitoringManager) updateEnvFile(newEnv map[string]string) error { + currentEnv, err := m.stack.ReadFile(".env") + if err != nil { + return err + } + + // Parse current env + env := make(map[string]string) + for _, line := range bytes.Split(currentEnv, []byte("\n")) { + parts := bytes.SplitN(line, []byte("="), 2) + if len(parts) == 2 { + env[string(parts[0])] = string(parts[1]) + } + } + + // Merge new env + for k, v := range newEnv { + env[k] = v + } + + // Write updated env + var buf bytes.Buffer + for k, v := range env { + buf.WriteString(fmt.Sprintf("%s=%s\n", k, v)) + } + + return m.stack.WriteFile(".env", buf.Bytes()) +} + +// Helper method to update the docker-compose.yml file +func (m *MonitoringManager) updateDockerComposeFile(service ServiceAPI) error { + + // Read the main Docker Compose template + rawBaseTmp, err := templates.Services.ReadFile(filepath.Join("services", "docker-compose_base.tmpl")) + if err != nil { + return err + } + + baseTmp, err := template.New("docker-compose").Parse(string(rawBaseTmp)) + if err != nil { + return err + } + + serviceTmp, err := templates.Services.ReadFile(filepath.Join("services",service.Name()+".tmpl")) + if err != nil { + return fmt.Errorf("error reading lido_exporter template: %w", err) + } + + baseTmp, err = baseTmp.Parse(string(serviceTmp)) + if err != nil { + return fmt.Errorf("error parsing service %s template: %w", service.Name(),err) + } + + // Create a buffer to hold the merged content + var buf bytes.Buffer + + data:= types.ServiceTemplateData{ + LidoExporter: service.Name() == LidoExporterServiceName, + } + + // Execute the main template with the service template as data + if err := baseTmp.Execute(&buf, data); err != nil { + return err + } + // Write the merged content to the final Docker Compose file + return os.WriteFile(filepath.Join(m.stack.Path(), "docker-compose.yml"), buf.Bytes(), 0644) +} diff --git a/internal/monitoring/monitoring_test.go b/internal/monitoring/monitoring_test.go index e97319d05..241e80fbb 100644 --- a/internal/monitoring/monitoring_test.go +++ b/internal/monitoring/monitoring_test.go @@ -31,10 +31,11 @@ import ( "github.com/NethermindEth/sedge/internal/common" "github.com/NethermindEth/sedge/internal/monitoring/data" mock_locker "github.com/NethermindEth/sedge/internal/monitoring/locker/mocks" + mocks "github.com/NethermindEth/sedge/internal/monitoring/mocks" + "github.com/NethermindEth/sedge/internal/monitoring/services/templates" "github.com/NethermindEth/sedge/internal/monitoring/services/types" "github.com/NethermindEth/sedge/internal/monitoring/utils" "github.com/NethermindEth/sedge/internal/pkg/commands" - mocks "github.com/NethermindEth/sedge/mocks" "github.com/golang/mock/gomock" log "github.com/sirupsen/logrus" "github.com/spf13/afero" @@ -1272,78 +1273,103 @@ func TestStatus(t *testing.T) { tests := []struct { name string - mocker func(t *testing.T, ctrl *gomock.Controller) *mocks.MockDockerServiceManager + mocker func(t *testing.T, ctrl *gomock.Controller) ([]ServiceAPI, *mocks.MockDockerServiceManager) want common.Status wantErr bool }{ { name: "ok", - mocker: func(t *testing.T, ctrl *gomock.Controller) *mocks.MockDockerServiceManager { + mocker: func(t *testing.T, ctrl *gomock.Controller) ([]ServiceAPI, *mocks.MockDockerServiceManager) { + services := []ServiceAPI{ + mocks.NewMockServiceAPI(ctrl), + mocks.NewMockServiceAPI(ctrl), + mocks.NewMockServiceAPI(ctrl), + } dockerServiceManager := mocks.NewMockDockerServiceManager(ctrl) - // Expect the docker manager to be triggered - gomock.InOrder( - dockerServiceManager.EXPECT().ContainerStatus(GrafanaContainerName).Return(common.Running, nil), - dockerServiceManager.EXPECT().ContainerStatus(PrometheusContainerName).Return(common.Running, nil), - dockerServiceManager.EXPECT().ContainerStatus(NodeExporterContainerName).Return(common.Running, nil), - ) - return dockerServiceManager + + for i, service := range services { + mockService := service.(*mocks.MockServiceAPI) + containerName := fmt.Sprintf("service%d", i+1) + mockService.EXPECT().ContainerName().Return(containerName) + dockerServiceManager.EXPECT().ContainerStatus(containerName).Return(common.Running, nil) + } + + return services, dockerServiceManager }, want: common.Running, }, { name: "error", - mocker: func(t *testing.T, ctrl *gomock.Controller) *mocks.MockDockerServiceManager { + mocker: func(t *testing.T, ctrl *gomock.Controller) ([]ServiceAPI, *mocks.MockDockerServiceManager) { + service := mocks.NewMockServiceAPI(ctrl) dockerServiceManager := mocks.NewMockDockerServiceManager(ctrl) - // Expect the docker manager to be triggered - dockerServiceManager.EXPECT().ContainerStatus(GrafanaContainerName).Return(common.Unknown, errors.New("error")) - return dockerServiceManager + + service.EXPECT().ContainerName().Return("service1") + dockerServiceManager.EXPECT().ContainerStatus("service1").Return(common.Unknown, errors.New("error")) + + return []ServiceAPI{service}, dockerServiceManager }, want: common.Unknown, wantErr: true, }, { name: "Restarting", - mocker: func(t *testing.T, ctrl *gomock.Controller) *mocks.MockDockerServiceManager { + mocker: func(t *testing.T, ctrl *gomock.Controller) ([]ServiceAPI, *mocks.MockDockerServiceManager) { + services := []ServiceAPI{ + mocks.NewMockServiceAPI(ctrl), + mocks.NewMockServiceAPI(ctrl), + } dockerServiceManager := mocks.NewMockDockerServiceManager(ctrl) - // Expect the docker manager to be triggered - gomock.InOrder( - dockerServiceManager.EXPECT().ContainerStatus(GrafanaContainerName).Return(common.Restarting, nil), - dockerServiceManager.EXPECT().ContainerStatus(PrometheusContainerName).Return(common.Restarting, nil), - dockerServiceManager.EXPECT().ContainerStatus(NodeExporterContainerName).Return(common.Restarting, nil), - ) - return dockerServiceManager + + for i, service := range services { + mockService := service.(*mocks.MockServiceAPI) + containerName := fmt.Sprintf("service%d", i+1) + mockService.EXPECT().ContainerName().Return(containerName) + dockerServiceManager.EXPECT().ContainerStatus(containerName).Return(common.Restarting, nil) + } + + return services, dockerServiceManager }, want: common.Restarting, }, { name: "Paused", - mocker: func(t *testing.T, ctrl *gomock.Controller) *mocks.MockDockerServiceManager { + mocker: func(t *testing.T, ctrl *gomock.Controller) ([]ServiceAPI, *mocks.MockDockerServiceManager) { + service := mocks.NewMockServiceAPI(ctrl) dockerServiceManager := mocks.NewMockDockerServiceManager(ctrl) - // Expect the docker manager to be triggered - dockerServiceManager.EXPECT().ContainerStatus(GrafanaContainerName).Return(common.Paused, nil) - return dockerServiceManager + + service.EXPECT().ContainerName().Return("service1") + dockerServiceManager.EXPECT().ContainerStatus("service1").Return(common.Paused, nil) + + return []ServiceAPI{service}, dockerServiceManager }, want: common.Broken, wantErr: true, }, { name: "Exited", - mocker: func(t *testing.T, ctrl *gomock.Controller) *mocks.MockDockerServiceManager { + mocker: func(t *testing.T, ctrl *gomock.Controller) ([]ServiceAPI, *mocks.MockDockerServiceManager) { + service := mocks.NewMockServiceAPI(ctrl) dockerServiceManager := mocks.NewMockDockerServiceManager(ctrl) - // Expect the docker manager to be triggered - dockerServiceManager.EXPECT().ContainerStatus(GrafanaContainerName).Return(common.Exited, nil) - return dockerServiceManager + + service.EXPECT().ContainerName().Return("service1") + dockerServiceManager.EXPECT().ContainerStatus("service1").Return(common.Exited, nil) + + return []ServiceAPI{service}, dockerServiceManager }, want: common.Broken, wantErr: true, }, { name: "Dead", - mocker: func(t *testing.T, ctrl *gomock.Controller) *mocks.MockDockerServiceManager { + mocker: func(t *testing.T, ctrl *gomock.Controller) ([]ServiceAPI, *mocks.MockDockerServiceManager) { + service := mocks.NewMockServiceAPI(ctrl) dockerServiceManager := mocks.NewMockDockerServiceManager(ctrl) - // Expect the docker manager to be triggered - dockerServiceManager.EXPECT().ContainerStatus(GrafanaContainerName).Return(common.Dead, nil) - return dockerServiceManager + + service.EXPECT().ContainerName().Return("service1") + dockerServiceManager.EXPECT().ContainerStatus("service1").Return(common.Dead, nil) + + return []ServiceAPI{service}, dockerServiceManager }, want: common.Broken, wantErr: true, @@ -1352,19 +1378,16 @@ func TestStatus(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Create a mock controller ctrl := gomock.NewController(t) - // Create a mock locker locker := mock_locker.NewMockLocker(ctrl) - // Expect the lock to be acquired locker.EXPECT().New(utils.PathMatcher{Expected: filepath.Join(userDataHome, ".sedge", "monitoring", ".lock")}).Return(locker) - // Create a monitoring manager + services, dockerServiceManager := tt.mocker(t, ctrl) manager := NewMonitoringManager( - []ServiceAPI{mocks.NewMockServiceAPI(ctrl)}, + services, mocks.NewMockComposeManager(ctrl), - tt.mocker(t, ctrl), + dockerServiceManager, afero.NewMemMapFs(), locker, ) @@ -1596,7 +1619,7 @@ func TestCleanup(t *testing.T) { ) if !tt.noInstall { - err := manager.stack.Setup(map[string]string{"NODE_NAME": "test"}, script) + err := manager.stack.Setup(map[string]string{"NODE_NAME": "test"}, templates.Services) require.NoError(t, err) } @@ -1646,3 +1669,256 @@ func TestServiceEndpoints(t *testing.T) { endpoints := manager.ServiceEndpoints() assert.Equal(t, want, endpoints) } + + +func TestAddService(t *testing.T) { + // Silence logger + log.SetOutput(io.Discard) + + userDataHome := os.Getenv("XDG_DATA_HOME") + if userDataHome == "" { + userHome, err := os.UserHomeDir() + require.NoError(t, err) + userDataHome = filepath.Join(userHome, ".local", "share") + } + + tests := []struct { + name string + mocker func(t *testing.T, ctrl *gomock.Controller) (*MonitoringManager, *mocks.MockServiceAPI) + wantErr bool + }{ + { + name: "add new service successfully", + mocker: func(t *testing.T, ctrl *gomock.Controller) (*MonitoringManager, *mocks.MockServiceAPI) { + fs := afero.NewMemMapFs() + locker := mock_locker.NewMockLocker(ctrl) + composeManager := mocks.NewMockComposeManager(ctrl) + dockerServiceManager := mocks.NewMockDockerServiceManager(ctrl) + + newService := mocks.NewMockServiceAPI(ctrl) + newService.EXPECT().ContainerName().Return("new-service").AnyTimes() + newService.EXPECT().Name().Return("new-service").AnyTimes() + newService.EXPECT().DotEnv().Return(map[string]string{"NEW_SERVICE_PORT": "8080"}) + newService.EXPECT().Init(gomock.Any()).Return(nil) + newService.EXPECT().Setup(gomock.Any()).Return(nil) + + locker.EXPECT().New(gomock.Any()).Return(locker).AnyTimes() + locker.EXPECT().Lock().Return(nil).AnyTimes() + locker.EXPECT().Locked().Return(true).AnyTimes() + locker.EXPECT().Unlock().Return(nil).AnyTimes() + + composeManager.EXPECT().Create(newService).Return(nil) + composeManager.EXPECT().Up(newService).Return(nil) + + dockerServiceManager.EXPECT().ContainerIP("new-service").Return("172.0.0.2", nil) + + // Mock the template files + err := afero.WriteFile(fs, filepath.Join(userDataHome, ".sedge", "monitoring", ".env"), []byte("EXISTING_VAR=value"), 0644) + require.NoError(t, err) + err = afero.WriteFile(fs, filepath.Join(userDataHome, ".sedge", "monitoring", "docker-compose.yml"), []byte("version: '3'"), 0644) + require.NoError(t, err) + + manager := NewMonitoringManager( + []ServiceAPI{}, + composeManager, + dockerServiceManager, + fs, + locker, + ) + + return manager, newService + }, + wantErr: false, + }, + { + name: "service already exists", + mocker: func(t *testing.T, ctrl *gomock.Controller) (*MonitoringManager, *mocks.MockServiceAPI) { + fs := afero.NewMemMapFs() + locker := mock_locker.NewMockLocker(ctrl) + composeManager := mocks.NewMockComposeManager(ctrl) + dockerServiceManager := mocks.NewMockDockerServiceManager(ctrl) + + existingService := mocks.NewMockServiceAPI(ctrl) + existingService.EXPECT().ContainerName().Return("existing-service").AnyTimes() + + manager := NewMonitoringManager( + []ServiceAPI{existingService}, + composeManager, + dockerServiceManager, + fs, + locker, + ) + + newService := mocks.NewMockServiceAPI(ctrl) + newService.EXPECT().ContainerName().Return("existing-service") + + return manager, newService + }, + wantErr: true, + }, + + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + manager, newService := tt.mocker(t, ctrl) + + err := manager.AddService(newService) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Contains(t, manager.services, newService) + } + }) + } +} + +func TestUpdateEnvFile(t *testing.T) { + // Silence logger + log.SetOutput(io.Discard) + + tests := []struct { + name string + initialEnv string + newEnv map[string]string + expectedEnv string + wantErr bool + }{ + { + name: "add new variables", + initialEnv: "EXISTING_VAR=value\n", + newEnv: map[string]string{"NEW_VAR": "new_value"}, + expectedEnv: "EXISTING_VAR=value\nNEW_VAR=new_value\n", + wantErr: false, + }, + { + name: "update existing variable", + initialEnv: "EXISTING_VAR=old_value\n", + newEnv: map[string]string{"EXISTING_VAR": "new_value"}, + expectedEnv: "EXISTING_VAR=new_value\n", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := afero.NewMemMapFs() + ctrl := gomock.NewController(t) + mockLocker := mock_locker.NewMockLocker(ctrl) + + // Mock the locker behavior + mockLocker.EXPECT().New(gomock.Any()).Return(mockLocker).AnyTimes() + mockLocker.EXPECT().Lock().Return(nil).AnyTimes() + mockLocker.EXPECT().Locked().Return(true).AnyTimes() + mockLocker.EXPECT().Unlock().Return(nil).AnyTimes() + + // Create a monitoring manager + manager := NewMonitoringManager( + []ServiceAPI{}, + mocks.NewMockComposeManager(ctrl), + mocks.NewMockDockerServiceManager(ctrl), + fs, + mockLocker, + ) + + err := afero.WriteFile(fs, filepath.Join(manager.stack.Path(), ".env"), []byte(tt.initialEnv), 0644) + require.NoError(t, err) + + err = manager.updateEnvFile(tt.newEnv) + + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + content, err := afero.ReadFile(fs, filepath.Join(manager.stack.Path(), ".env")) + assert.NoError(t, err) + assert.Equal(t, tt.expectedEnv, string(content)) + } + }) + } +} + +func TestUpdateDockerComposeFile(t *testing.T) { + tests := []struct { + name string + serviceName string + wantErr bool + }{ + { + name: "add lido exporter service", + serviceName: LidoExporterServiceName, + wantErr: false, + }, + { + name: "non-existent service", + serviceName: "non-existent-service", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := afero.NewMemMapFs() + ctrl := gomock.NewController(t) + mockLocker := mock_locker.NewMockLocker(ctrl) + + // Mock the locker behavior + mockLocker.EXPECT().New(gomock.Any()).Return(mockLocker).AnyTimes() + mockLocker.EXPECT().Lock().Return(nil).AnyTimes() + mockLocker.EXPECT().Locked().Return(true).AnyTimes() + mockLocker.EXPECT().Unlock().Return(nil).AnyTimes() + + // Create a monitoring manager + manager := NewMonitoringManager( + []ServiceAPI{}, + mocks.NewMockComposeManager(ctrl), + mocks.NewMockDockerServiceManager(ctrl), + fs, + mockLocker, + ) + service := mocks.NewMockServiceAPI(ctrl) + service.EXPECT().Name().Return(tt.serviceName).AnyTimes() + + // Write an initial docker-compose.yml file + initialContent := `version: '3' +services: + grafana: + # ... (grafana config) + prometheus: + # ... (prometheus config) + node-exporter: + # ... (node-exporter config) +volumes: + grafana-storage: +networks: + sedge: + name: sedge-network + external: true` + err := afero.WriteFile(fs, filepath.Join(manager.stack.Path(), "docker-compose.yml"), []byte(initialContent), 0644) + require.NoError(t, err) + + err = manager.updateDockerComposeFile(service) + + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + content, err := afero.ReadFile(fs, filepath.Join(manager.stack.Path(), "docker-compose.yml")) + assert.NoError(t, err) + assert.Contains(t, string(content), "services:") + assert.Contains(t, string(content), "grafana:") + assert.Contains(t, string(content), "prometheus:") + assert.Contains(t, string(content), "node-exporter:") + if tt.serviceName == LidoExporterServiceName { + assert.Contains(t, string(content), "lido_exporter:") + } else { + assert.NotContains(t, string(content), "lido_exporter:") + } + } + }) + } +} \ No newline at end of file diff --git a/internal/monitoring/service.go b/internal/monitoring/service.go index d4ee135d2..4139f7243 100644 --- a/internal/monitoring/service.go +++ b/internal/monitoring/service.go @@ -50,4 +50,7 @@ type ServiceAPI interface { // Endpoint returns the endpoint of the service. Endpoint() string + + // Name returns the name of the service. + Name() string } diff --git a/internal/monitoring/services/grafana/service.go b/internal/monitoring/services/grafana/service.go index 2be637087..19b8c2cd9 100644 --- a/internal/monitoring/services/grafana/service.go +++ b/internal/monitoring/services/grafana/service.go @@ -197,3 +197,7 @@ func (g *GrafanaService) ContainerName() string { func (g *GrafanaService) Endpoint() string { return fmt.Sprintf("http://%s:%d", g.containerIP, g.port) } + +func (g *GrafanaService) Name() string { + return monitoring.GrafanaServiceName +} diff --git a/internal/monitoring/services/lido_exporter/dotenv.go b/internal/monitoring/services/lido_exporter/dotenv.go new file mode 100644 index 000000000..2a37b8897 --- /dev/null +++ b/internal/monitoring/services/lido_exporter/dotenv.go @@ -0,0 +1,28 @@ +/* +Copyright 2022 Nethermind + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package lido_exporter + +var dotEnv map[string]string = map[string]string{ + "LIDO_EXPORTER_IMAGE": "nethermindeth/lido-exporter:v1.0.0", + "LIDO_EXPORTER_PORT": "8080", + "LIDO_EXPORTER_NODE_OPERATOR_ID": "", + "LIDO_EXPORTER_REWARD_ADDRESS": "", + "LIDO_EXPORTER_NETWORK": "", + "LIDO_EXPORTER_RPC_ENDPOINTS": "", + "LIDO_EXPORTER_WS_ENDPOINTS": "", + "LIDO_EXPORTER_SCRAPE_TIME": "", + "LIDO_EXPORTER_LOG_LEVEL": "", +} \ No newline at end of file diff --git a/internal/monitoring/services/lido_exporter/error.go b/internal/monitoring/services/lido_exporter/error.go new file mode 100644 index 000000000..284cf0c16 --- /dev/null +++ b/internal/monitoring/services/lido_exporter/error.go @@ -0,0 +1,20 @@ +/* +Copyright 2022 Nethermind + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package lido_exporter + +import "errors" + +var ErrInvalidOptions = errors.New("invalid options for lido exporter setup") diff --git a/internal/monitoring/services/lido_exporter/service.go b/internal/monitoring/services/lido_exporter/service.go new file mode 100644 index 000000000..1fa94847a --- /dev/null +++ b/internal/monitoring/services/lido_exporter/service.go @@ -0,0 +1,110 @@ +/* +Copyright 2022 Nethermind + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package lido_exporter + +import ( + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/NethermindEth/sedge/internal/monitoring" + "github.com/NethermindEth/sedge/internal/monitoring/services/types" +) + +var _ monitoring.ServiceAPI = &LidoExporterService{} + +type LidoExporterParams struct { + NodeOperatorID string + RewardAddress string + Network string + RPCEndpoints []string + WSEndpoints []string + Port uint16 + ScrapeTime time.Duration + LogLevel string +} + + +type LidoExporterService struct { + containerIP net.IP + params LidoExporterParams +} + +func NewLidoExporter(params LidoExporterParams) *LidoExporterService { + // Set other Lido Exporter parameters + dotEnv["LIDO_EXPORTER_NODE_OPERATOR_ID"] = params.NodeOperatorID + dotEnv["LIDO_EXPORTER_REWARD_ADDRESS"] = params.RewardAddress + dotEnv["LIDO_EXPORTER_NETWORK"] = params.Network + dotEnv["LIDO_EXPORTER_RPC_ENDPOINTS"] = strings.Join(params.RPCEndpoints, ",") + dotEnv["LIDO_EXPORTER_WS_ENDPOINTS"] = strings.Join(params.WSEndpoints, ",") + dotEnv["LIDO_EXPORTER_SCRAPE_TIME"] = params.ScrapeTime.String() + dotEnv["LIDO_EXPORTER_LOG_LEVEL"] = params.LogLevel + + return &LidoExporterService{ + params: params, + } +} + +func (n *LidoExporterService) Init(opts types.ServiceOptions) error { + // Validate dotEnv + lidoExporterPort, ok := opts.Dotenv["LIDO_EXPORTER_PORT"] + if !ok { + return fmt.Errorf("%w: %s missing in options", ErrInvalidOptions, "LIDO_EXPORTER_PORT") + } else if lidoExporterPort == "" { + return fmt.Errorf("%w: %s can't be empty", ErrInvalidOptions, "LIDO_EXPORTER_PORT") + } + + port, err := strconv.ParseUint(opts.Dotenv["LIDO_EXPORTER_PORT"], 10, 16) + if err != nil { + return fmt.Errorf("%w: %s is not a valid port", ErrInvalidOptions, "LIDO_EXPORTER_PORT") + } + n.params.Port = uint16(port) + return nil +} + +func (l *LidoExporterService) AddTarget(target types.MonitoringTarget, labels map[string]string, jobName string) error { + return nil +} + +func (l *LidoExporterService) RemoveTarget(instanceID string) (string, error) { + return "", nil +} + +func (l *LidoExporterService) DotEnv() map[string]string { + return dotEnv +} + +func (l *LidoExporterService) Setup(options map[string]string) error { + return nil +} + +func (l *LidoExporterService) SetContainerIP(ip net.IP) { + l.containerIP = ip +} + +func (l *LidoExporterService) ContainerName() string { + return monitoring.LidoExporterContainerName +} + +func (l *LidoExporterService) Endpoint() string { + return fmt.Sprintf("http://%s:%d", l.containerIP, l.params.Port) +} + +func (l *LidoExporterService) Name() string { + return monitoring.LidoExporterServiceName +} \ No newline at end of file diff --git a/internal/monitoring/services/lido_exporter/service_test.go b/internal/monitoring/services/lido_exporter/service_test.go new file mode 100644 index 000000000..b4ee7a5bd --- /dev/null +++ b/internal/monitoring/services/lido_exporter/service_test.go @@ -0,0 +1,139 @@ +/* +Copyright 2022 Nethermind + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package lido_exporter + +import ( + "net" + "path/filepath" + "strconv" + "testing" + + "github.com/NethermindEth/sedge/internal/monitoring" + "github.com/NethermindEth/sedge/internal/monitoring/data" + mocks "github.com/NethermindEth/sedge/internal/monitoring/locker/mocks" + "github.com/NethermindEth/sedge/internal/monitoring/services/types" + "github.com/NethermindEth/sedge/internal/monitoring/utils" + "github.com/golang/mock/gomock" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInit(t *testing.T) { + // Create an in-memory filesystem + afs := afero.NewMemMapFs() + + // Create a mock locker + ctrl := gomock.NewController(t) + locker := mocks.NewMockLocker(ctrl) + + // Expect the lock to be acquired + locker.EXPECT().New(utils.PathMatcher{Expected: filepath.Join("monitoring", ".lock")}).Return(locker) + + // Create a new DataDir with the in-memory filesystem + dataDir, err := data.NewDataDir("/", afs, locker) + require.NoError(t, err) + stack, err := dataDir.MonitoringStack() + require.NoError(t, err) + + tests := []struct { + name string + options types.ServiceOptions + wantErr bool + }{ + { + name: "ok", + options: types.ServiceOptions{ + Dotenv: map[string]string{ + "LIDO_EXPORTER_PORT": "6666", + }, + Stack: stack, + }, + }, + { + name: "missing lido exporter port", + options: types.ServiceOptions{ + Dotenv: map[string]string{}, + Stack: stack, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lidoExporter := NewLidoExporter(LidoExporterParams{}) + err := lidoExporter.Init(tt.options) + if tt.wantErr { + require.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.options.Dotenv["LIDO_EXPORTER_PORT"], strconv.Itoa(int(lidoExporter.params.Port))) + } + }) + } +} + +func TestSetContainerIP(t *testing.T) { + tests := []struct { + name string + ip net.IP + }{ + { + name: "ok", + ip: net.ParseIP("127.0.0.1"), + }, + { + name: "empty", + ip: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a new Lido Exporter service + lidoExporter := NewLidoExporter(LidoExporterParams{}) + lidoExporter.SetContainerIP(tt.ip) + assert.Equal(t, tt.ip, lidoExporter.containerIP) + }) + } +} + +func TestContainerName(t *testing.T) { + want := monitoring.NodeExporterContainerName + + // Create a new Lido Exporter service + lidoExporter := NewLidoExporter(LidoExporterParams{}) + assert.Equal(t, want, lidoExporter.ContainerName()) +} + +func TestEndpoint(t *testing.T) { + dotenv := map[string]string{ + "LIDO_EXPORTER_PORT": "6666", + } + want := "http://168.77.88.99:6666" + + // Create a new Node exporter service + lidoExporter := NewLidoExporter(LidoExporterParams{}) + err := lidoExporter.Init(types.ServiceOptions{ + Dotenv: dotenv, + }) + require.NoError(t, err) + lidoExporter.SetContainerIP(net.ParseIP("168.77.88.99")) + + endpoint := lidoExporter.Endpoint() + assert.Equal(t, want, endpoint) +} diff --git a/internal/monitoring/services/node_exporter/service.go b/internal/monitoring/services/node_exporter/service.go index 863241938..bca1c931f 100644 --- a/internal/monitoring/services/node_exporter/service.go +++ b/internal/monitoring/services/node_exporter/service.go @@ -79,3 +79,7 @@ func (n *NodeExporterService) ContainerName() string { func (n *NodeExporterService) Endpoint() string { return fmt.Sprintf("http://%s:%d", n.containerIP, n.port) } + +func (n *NodeExporterService) Name() string { + return monitoring.NodeExporterServiceName +} \ No newline at end of file diff --git a/internal/monitoring/services/prometheus/service.go b/internal/monitoring/services/prometheus/service.go index e164a4823..ac329bf0c 100644 --- a/internal/monitoring/services/prometheus/service.go +++ b/internal/monitoring/services/prometheus/service.go @@ -303,3 +303,7 @@ func (p *PrometheusService) reloadConfig() error { return err } + +func (p *PrometheusService) Name() string { + return monitoring.PrometheusServiceName +} \ No newline at end of file diff --git a/internal/monitoring/script/docker-compose.yml b/internal/monitoring/services/templates/services/docker-compose_base.tmpl similarity index 89% rename from internal/monitoring/script/docker-compose.yml rename to internal/monitoring/services/templates/services/docker-compose_base.tmpl index 65dd7793b..307d8491a 100644 --- a/internal/monitoring/script/docker-compose.yml +++ b/internal/monitoring/services/templates/services/docker-compose_base.tmpl @@ -1,3 +1,6 @@ +{{/* docker-compose_base.tmpl */}} +{{ define "docker-compose" }} + services: grafana: container_name: sedge_grafana @@ -47,9 +50,15 @@ services: networks: - sedge -networks: - sedge: - name: sedge-network +{{ if .LidoExporter }} + {{ template "lido_exporter" . }} +{{ end }} volumes: grafana-storage: + +networks: + sedge: + name: sedge-network + external: true +{{ end }} \ No newline at end of file diff --git a/internal/monitoring/services/templates/services/lido_exporter.tmpl b/internal/monitoring/services/templates/services/lido_exporter.tmpl new file mode 100644 index 000000000..8cede80a4 --- /dev/null +++ b/internal/monitoring/services/templates/services/lido_exporter.tmpl @@ -0,0 +1,18 @@ +{{ define "lido_exporter" }} + lido_exporter: + container_name: sedge_lido_exporter + image: ${LIDO_EXPORTER_IMAGE} + restart: unless-stopped + ports: + - ${LIDO_EXPORTER_PORT}:8080 + environment: + - LIDO_EXPORTER_NODE_OPERATOR_ID=${LIDO_EXPORTER_NODE_OPERATOR_ID} + - LIDO_EXPORTER_REWARD_ADDRESS=${LIDO_EXPORTER_REWARD_ADDRESS} + - LIDO_EXPORTER_NETWORK=${LIDO_EXPORTER_NETWORK} + - LIDO_EXPORTER_RPC_ENDPOINTS=${LIDO_EXPORTER_RPC_ENDPOINTS} + - LIDO_EXPORTER_WS_ENDPOINTS=${LIDO_EXPORTER_WS_ENDPOINTS} + - LIDO_EXPORTER_SCRAPE_TIME=${LIDO_EXPORTER_SCRAPE_TIME} + - LIDO_EXPORTER_LOG_LEVEL=${LIDO_EXPORTER_LOG_LEVEL} + networks: + - sedge +{{ end }} \ No newline at end of file diff --git a/internal/monitoring/services/templates/templates.go b/internal/monitoring/services/templates/templates.go new file mode 100644 index 000000000..f62819e85 --- /dev/null +++ b/internal/monitoring/services/templates/templates.go @@ -0,0 +1,21 @@ +/* +Copyright 2022 Nethermind + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package templates + +import "embed" + +//go:embed services +var Services embed.FS \ No newline at end of file diff --git a/internal/monitoring/services/types/types.go b/internal/monitoring/services/types/types.go index 2939c6f8e..714be20dc 100644 --- a/internal/monitoring/services/types/types.go +++ b/internal/monitoring/services/types/types.go @@ -47,3 +47,8 @@ func (t MonitoringTarget) String() string { func (t MonitoringTarget) Endpoint() string { return t.Host + ":" + strconv.Itoa(int(t.Port)) } + +// ServiceTemplateData: Struct Data object to be applied to docker-compose script +type ServiceTemplateData struct { + LidoExporter bool +} \ No newline at end of file diff --git a/internal/pkg/services/gomock_reflect_1452349235/prog.go b/internal/pkg/services/gomock_reflect_1452349235/prog.go new file mode 100644 index 000000000..0a09b26ce --- /dev/null +++ b/internal/pkg/services/gomock_reflect_1452349235/prog.go @@ -0,0 +1,66 @@ + +package main + +import ( + "encoding/gob" + "flag" + "fmt" + "os" + "path" + "reflect" + + "github.com/golang/mock/mockgen/model" + + pkg_ "github.com/docker/docker/client" +) + +var output = flag.String("output", "", "The output file name, or empty to use stdout.") + +func main() { + flag.Parse() + + its := []struct{ + sym string + typ reflect.Type + }{ + + { "APIClient", reflect.TypeOf((*pkg_.APIClient)(nil)).Elem()}, + + } + pkg := &model.Package{ + // NOTE: This behaves contrary to documented behaviour if the + // package name is not the final component of the import path. + // The reflect package doesn't expose the package name, though. + Name: path.Base("github.com/docker/docker/client"), + } + + for _, it := range its { + intf, err := model.InterfaceFromInterfaceType(it.typ) + if err != nil { + fmt.Fprintf(os.Stderr, "Reflection: %v\n", err) + os.Exit(1) + } + intf.Name = it.sym + pkg.Interfaces = append(pkg.Interfaces, intf) + } + + outfile := os.Stdout + if len(*output) != 0 { + var err error + outfile, err = os.Create(*output) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to open output file %q", *output) + } + defer func() { + if err := outfile.Close(); err != nil { + fmt.Fprintf(os.Stderr, "failed to close output file %q", *output) + os.Exit(1) + } + }() + } + + if err := gob.NewEncoder(outfile).Encode(pkg); err != nil { + fmt.Fprintf(os.Stderr, "gob encode: %v\n", err) + os.Exit(1) + } +}