diff --git a/CHANGELOG.md b/CHANGELOG.md index d2daf9aeb6c..8a76065c943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Add F3GetCertificate & F3GetLatestCertificate to the gateway. ([filecoin-project/lotus#12778](https://github.com/filecoin-project/lotus/pull/12778)) - Add Magik's bootstrap node. ([filecoin-project/lotus#12792](https://github.com/filecoin-project/lotus/pull/12792)) - Lotus now reports the network name as a tag in most metrics. Some untagged metrics will be completed in a follow-up at a later date. ([filecoin-project/lotus#12733](https://github.com/filecoin-project/lotus/pull/12733)) +- Generate the cli docs directly from the code instead compiling and executing binaries' `help` output. ([filecoin-project/lotus#12717](https://github.com/filecoin-project/lotus/pull/12717)) # UNRELEASED v.1.32.0 diff --git a/Makefile b/Makefile index 8197dc2204c..bb179bae536 100644 --- a/Makefile +++ b/Makefile @@ -348,11 +348,11 @@ snap: lotus lotus-miner lotus-worker snapcraft # snapcraft upload ./lotus_*.snap -# separate from gen because it needs binaries -docsgen-cli: lotus lotus-miner lotus-worker +docsgen-cli:lotus lotus-miner lotus-worker $(GOCC) run ./scripts/docsgen-cli ./lotus config default > documentation/en/default-lotus-config.toml ./lotus-miner config default > documentation/en/default-lotus-miner-config.toml + .PHONY: docsgen-cli print-%: diff --git a/cli/cmd.go b/cli/cmd.go index 9ae8c14b75e..2c7f69f176d 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -57,10 +57,10 @@ var GetStorageMinerAPI = cliutil.GetStorageMinerAPI var GetWorkerAPI = cliutil.GetWorkerAPI var CommonCommands = []*cli.Command{ - AuthCmd, - LogCmd, - WaitApiCmd, - FetchParamCmd, + WithCategory("developer", AuthCmd), + WithCategory("developer", LogCmd), + WithCategory("developer", WaitApiCmd), + WithCategory("developer", FetchParamCmd), PprofCmd, VersionCmd, } diff --git a/cmd/lotus/backup.go b/cli/lotus/backup.go similarity index 99% rename from cmd/lotus/backup.go rename to cli/lotus/backup.go index 0fbdf29622c..ed664a370aa 100644 --- a/cmd/lotus/backup.go +++ b/cli/lotus/backup.go @@ -1,4 +1,4 @@ -package main +package lotus import ( "os" diff --git a/cmd/lotus/config.go b/cli/lotus/config.go similarity index 99% rename from cmd/lotus/config.go rename to cli/lotus/config.go index 4b323bfb227..9528cc05363 100644 --- a/cmd/lotus/config.go +++ b/cli/lotus/config.go @@ -1,4 +1,4 @@ -package main +package lotus import ( "fmt" diff --git a/cmd/lotus/daemon.go b/cli/lotus/daemon.go similarity index 99% rename from cmd/lotus/daemon.go rename to cli/lotus/daemon.go index 97d2294b79b..f4e2c4b7fa2 100644 --- a/cmd/lotus/daemon.go +++ b/cli/lotus/daemon.go @@ -1,7 +1,7 @@ //go:build !nodaemon // +build !nodaemon -package main +package lotus import ( "bufio" diff --git a/cmd/lotus/daemon_nodaemon.go b/cli/lotus/daemon_nodaemon.go similarity index 96% rename from cmd/lotus/daemon_nodaemon.go rename to cli/lotus/daemon_nodaemon.go index 7cf12dac41a..be3f8721e58 100644 --- a/cmd/lotus/daemon_nodaemon.go +++ b/cli/lotus/daemon_nodaemon.go @@ -1,7 +1,7 @@ //go:build nodaemon // +build nodaemon -package main +package lotus import ( "errors" diff --git a/cmd/lotus/debug_advance.go b/cli/lotus/debug_advance.go similarity index 99% rename from cmd/lotus/debug_advance.go rename to cli/lotus/debug_advance.go index 422f14e5a2e..915af49d668 100644 --- a/cmd/lotus/debug_advance.go +++ b/cli/lotus/debug_advance.go @@ -1,7 +1,7 @@ //go:build debug // +build debug -package main +package lotus import ( "encoding/binary" diff --git a/cli/lotus/lotus.go b/cli/lotus/lotus.go new file mode 100644 index 00000000000..1bdaf6fd217 --- /dev/null +++ b/cli/lotus/lotus.go @@ -0,0 +1,123 @@ +package lotus + +import ( + "context" + "os" + + "github.com/fatih/color" + logging "github.com/ipfs/go-log/v2" + "github.com/mattn/go-isatty" + "github.com/urfave/cli/v2" + "go.opencensus.io/trace" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/cli/clicommands" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/lib/tracing" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("lotus") + +var AdvanceBlockCmd *cli.Command + +func App() *cli.App { + api.RunningNodeType = api.NodeFull + + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + DaemonCmd, + backupCmd, + configCmd, + } + if AdvanceBlockCmd != nil { + local = append(local, AdvanceBlockCmd) + } + + jaeger := tracing.SetupJaegerTracing("lotus") + defer func() { + if jaeger != nil { + _ = jaeger.ForceFlush(context.Background()) + } + }() + + for _, cmd := range local { + cmd := cmd + originBefore := cmd.Before + cmd.Before = func(cctx *cli.Context) error { + if jaeger != nil { + _ = jaeger.Shutdown(cctx.Context) + } + jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) + + if cctx.IsSet("color") { + color.NoColor = !cctx.Bool("color") + } + + if originBefore != nil { + return originBefore(cctx) + } + return nil + } + } + ctx, span := trace.StartSpan(context.Background(), "/cli") + defer span.End() + + interactiveDef := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) + + app := &cli.App{ + Name: "lotus", + Usage: "Filecoin decentralized storage network client", + Version: string(build.NodeUserVersion()), + EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "panic-reports", + EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, + Hidden: true, + Value: "~/.lotus", // should follow --repo default + }, + &cli.BoolFlag{ + // examined in the Before above + Name: "color", + Usage: "use color in display output", + DefaultText: "depends on output being a TTY", + }, + &cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Hidden: true, + Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME + }, + &cli.BoolFlag{ + Name: "interactive", + Usage: "setting to false will disable interactive functionality of commands", + Value: interactiveDef, + }, + &cli.BoolFlag{ + Name: "force-send", + Usage: "if true, will ignore pre-send checks", + }, + cliutil.FlagVeryVerbose, + }, + After: func(c *cli.Context) error { + if r := recover(); r != nil { + // Generate report in LOTUS_PATH and re-raise panic + build.GenerateNodePanicReport(c.String("panic-reports"), c.String("repo"), c.App.Name) + panic(r) + } + return nil + }, + + Commands: append(local, clicommands.Commands...), + } + + app.Setup() + app.Metadata["traceContext"] = ctx + app.Metadata["repoType"] = repo.FullNode + + return app +} diff --git a/cmd/lotus-miner/actor.go b/cli/miner/actor.go similarity index 99% rename from cmd/lotus-miner/actor.go rename to cli/miner/actor.go index 1330d883924..89eae061304 100644 --- a/cmd/lotus-miner/actor.go +++ b/cli/miner/actor.go @@ -1,4 +1,4 @@ -package main +package miner import ( "fmt" diff --git a/cmd/lotus-miner/actor_test.go b/cli/miner/actor_test.go similarity index 99% rename from cmd/lotus-miner/actor_test.go rename to cli/miner/actor_test.go index 39d6a5ba25e..07e14b3c843 100644 --- a/cmd/lotus-miner/actor_test.go +++ b/cli/miner/actor_test.go @@ -1,4 +1,4 @@ -package main +package miner import ( "bytes" diff --git a/cmd/lotus-miner/allinfo_test.go b/cli/miner/allinfo_test.go similarity index 98% rename from cmd/lotus-miner/allinfo_test.go rename to cli/miner/allinfo_test.go index 001ffb3af3d..5141fc582a1 100644 --- a/cmd/lotus-miner/allinfo_test.go +++ b/cli/miner/allinfo_test.go @@ -1,4 +1,4 @@ -package main +package miner import ( "flag" diff --git a/cmd/lotus-miner/backup.go b/cli/miner/backup.go similarity index 96% rename from cmd/lotus-miner/backup.go rename to cli/miner/backup.go index cf8c9f9125c..b7c8cc9de50 100644 --- a/cmd/lotus-miner/backup.go +++ b/cli/miner/backup.go @@ -1,4 +1,4 @@ -package main +package miner import ( "github.com/urfave/cli/v2" diff --git a/cmd/lotus-miner/config.go b/cli/miner/config.go similarity index 99% rename from cmd/lotus-miner/config.go rename to cli/miner/config.go index b7af1b2e51e..16e3061ac0c 100644 --- a/cmd/lotus-miner/config.go +++ b/cli/miner/config.go @@ -1,4 +1,4 @@ -package main +package miner import ( "fmt" diff --git a/cmd/lotus-miner/info.go b/cli/miner/info.go similarity index 99% rename from cmd/lotus-miner/info.go rename to cli/miner/info.go index 9548a4a771f..992c1100ef3 100644 --- a/cmd/lotus-miner/info.go +++ b/cli/miner/info.go @@ -1,4 +1,4 @@ -package main +package miner import ( "context" diff --git a/cmd/lotus-miner/info_all.go b/cli/miner/info_all.go similarity index 98% rename from cmd/lotus-miner/info_all.go rename to cli/miner/info_all.go index 253d2befc67..c05cda49cbb 100644 --- a/cmd/lotus-miner/info_all.go +++ b/cli/miner/info_all.go @@ -1,4 +1,4 @@ -package main +package miner import ( "flag" @@ -194,7 +194,7 @@ var infoAllCmd = &cli.Command{ if !_test { fmt.Println("\n#: Goroutines") - if err := lcli.PprofGoroutines.Action(cctx); err != nil { + if err := lcli.PprofGoroutinesCmd.Action(cctx); err != nil { fmt.Println("ERROR: ", err) } } diff --git a/cmd/lotus-miner/init.go b/cli/miner/init.go similarity index 99% rename from cmd/lotus-miner/init.go rename to cli/miner/init.go index 6d6fe7362a5..4f84b9c1c84 100644 --- a/cmd/lotus-miner/init.go +++ b/cli/miner/init.go @@ -1,4 +1,4 @@ -package main +package miner import ( "bytes" diff --git a/cmd/lotus-miner/init_restore.go b/cli/miner/init_restore.go similarity index 99% rename from cmd/lotus-miner/init_restore.go rename to cli/miner/init_restore.go index 05721f6cf56..e8e8970a1e8 100644 --- a/cmd/lotus-miner/init_restore.go +++ b/cli/miner/init_restore.go @@ -1,4 +1,4 @@ -package main +package miner import ( "context" diff --git a/cli/miner/miner.go b/cli/miner/miner.go new file mode 100644 index 00000000000..3d432ec99b3 --- /dev/null +++ b/cli/miner/miner.go @@ -0,0 +1,168 @@ +package miner + +import ( + "context" + "fmt" + + "github.com/fatih/color" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/lib/tracing" + "github.com/filecoin-project/lotus/node/repo" +) + +var log = logging.Logger("lotus-miner") + +const ( + FlagMinerRepo = "miner-repo" +) + +// TODO remove after deprecation period +const FlagMinerRepoDeprecation = "storagerepo" + +func App() *cli.App { + api.RunningNodeType = api.NodeMiner + + lotuslog.SetupLogLevels() + + local := []*cli.Command{ + initCmd, + runCmd, + stopCmd, + configCmd, + backupCmd, + lcli.WithCategory("chain", actorCmd), + lcli.WithCategory("chain", infoCmd), + lcli.WithCategory("storage", sectorsCmd), + lcli.WithCategory("storage", provingCmd), + lcli.WithCategory("storage", storageCmd), + lcli.WithCategory("storage", sealingCmd), + } + + jaeger := tracing.SetupJaegerTracing("lotus") + defer func() { + if jaeger != nil { + _ = jaeger.ForceFlush(context.Background()) + } + }() + + for _, cmd := range local { + cmd := cmd + originBefore := cmd.Before + cmd.Before = func(cctx *cli.Context) error { + if jaeger != nil { + _ = jaeger.Shutdown(cctx.Context) + } + jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) + + if cctx.IsSet("color") { + color.NoColor = !cctx.Bool("color") + } + + if originBefore != nil { + return originBefore(cctx) + } + + return nil + } + } + + app := &cli.App{ + Name: "lotus-miner", + Usage: "Filecoin decentralized storage network miner", + Version: string(build.MinerUserVersion()), + EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "actor", + Value: "", + Usage: "specify other actor to query / manipulate", + Aliases: []string{"a"}, + }, + &cli.BoolFlag{ + // examined in the Before above + Name: "color", + Usage: "use color in display output", + DefaultText: "depends on output being a TTY", + }, + &cli.StringFlag{ + Name: "panic-reports", + EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, + Hidden: true, + Value: "~/.lotusminer", // should follow --repo default + }, + &cli.StringFlag{ + Name: "repo", + EnvVars: []string{"LOTUS_PATH"}, + Hidden: true, + Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME + }, + &cli.StringFlag{ + Name: FlagMinerRepo, + Aliases: []string{FlagMinerRepoDeprecation}, + EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, + Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME + Usage: fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation), + }, + cliutil.FlagVeryVerbose, + }, + Commands: append(local, lcli.CommonCommands...), + After: func(c *cli.Context) error { + if r := recover(); r != nil { + // Generate report in LOTUS_PATH and re-raise panic + build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagMinerRepo), c.App.Name) + panic(r) + } + return nil + }, + } + app.Setup() + app.Metadata["repoType"] = repo.StorageMiner + return app +} + +func getActorAddress(ctx context.Context, cctx *cli.Context) (maddr address.Address, err error) { + if cctx.IsSet("actor") { + maddr, err = address.NewFromString(cctx.String("actor")) + if err != nil { + return maddr, err + } + return + } + + minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return address.Undef, err + } + defer closer() + + maddr, err = minerApi.ActorAddress(ctx) + if err != nil { + return maddr, xerrors.Errorf("getting actor address: %w", err) + } + + return maddr, nil +} + +func LMActorOrEnvGetter(cctx *cli.Context) (address.Address, error) { + return getActorAddress(cctx.Context, cctx) +} + +func LMActorGetter(cctx *cli.Context) (address.Address, error) { + minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return address.Undef, err + } + defer closer() + + return minerApi.ActorAddress(cctx.Context) +} diff --git a/cmd/lotus-miner/proving.go b/cli/miner/proving.go similarity index 99% rename from cmd/lotus-miner/proving.go rename to cli/miner/proving.go index 79ad623d1ca..6b87e13ea17 100644 --- a/cmd/lotus-miner/proving.go +++ b/cli/miner/proving.go @@ -1,4 +1,4 @@ -package main +package miner import ( "encoding/json" @@ -244,7 +244,7 @@ It will not send any messages to the chain.`, return err } - //convert sector information into easily readable information + // convert sector information into easily readable information type PoStPartition struct { Index uint64 Skipped []uint64 diff --git a/cmd/lotus-miner/run.go b/cli/miner/run.go similarity index 99% rename from cmd/lotus-miner/run.go rename to cli/miner/run.go index be670287e1a..e84e098799b 100644 --- a/cmd/lotus-miner/run.go +++ b/cli/miner/run.go @@ -1,4 +1,4 @@ -package main +package miner import ( "fmt" diff --git a/cmd/lotus-miner/sealing.go b/cli/miner/sealing.go similarity index 99% rename from cmd/lotus-miner/sealing.go rename to cli/miner/sealing.go index ed2b2d2948a..0b807b31348 100644 --- a/cmd/lotus-miner/sealing.go +++ b/cli/miner/sealing.go @@ -1,4 +1,4 @@ -package main +package miner import ( "encoding/hex" diff --git a/cmd/lotus-miner/sectors.go b/cli/miner/sectors.go similarity index 99% rename from cmd/lotus-miner/sectors.go rename to cli/miner/sectors.go index fa23e5f268f..c9eb8e777e5 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cli/miner/sectors.go @@ -1,4 +1,4 @@ -package main +package miner import ( "bufio" diff --git a/cmd/lotus-miner/stop.go b/cli/miner/stop.go similarity index 96% rename from cmd/lotus-miner/stop.go rename to cli/miner/stop.go index 186a156169e..f5bfeaa4889 100644 --- a/cmd/lotus-miner/stop.go +++ b/cli/miner/stop.go @@ -1,4 +1,4 @@ -package main +package miner import ( _ "net/http/pprof" diff --git a/cmd/lotus-miner/storage.go b/cli/miner/storage.go similarity index 99% rename from cmd/lotus-miner/storage.go rename to cli/miner/storage.go index b39fe2bf750..2fcf60ff3db 100644 --- a/cmd/lotus-miner/storage.go +++ b/cli/miner/storage.go @@ -1,4 +1,4 @@ -package main +package miner import ( "context" diff --git a/cli/pprof.go b/cli/pprof.go index ae4016e1108..28b40dad170 100644 --- a/cli/pprof.go +++ b/cli/pprof.go @@ -15,11 +15,11 @@ var PprofCmd = &cli.Command{ Name: "pprof", Hidden: true, Subcommands: []*cli.Command{ - PprofGoroutines, + PprofGoroutinesCmd, }, } -var PprofGoroutines = &cli.Command{ +var PprofGoroutinesCmd = &cli.Command{ Name: "goroutines", Usage: "Get goroutine stacks", Action: func(cctx *cli.Context) error { diff --git a/cmd/lotus-worker/info.go b/cli/worker/info.go similarity index 99% rename from cmd/lotus-worker/info.go rename to cli/worker/info.go index 338345c2857..a8197141928 100644 --- a/cmd/lotus-worker/info.go +++ b/cli/worker/info.go @@ -1,4 +1,4 @@ -package main +package worker import ( "fmt" diff --git a/cmd/lotus-worker/resources.go b/cli/worker/resources.go similarity index 98% rename from cmd/lotus-worker/resources.go rename to cli/worker/resources.go index 45fd01e702b..6d79f9523be 100644 --- a/cmd/lotus-worker/resources.go +++ b/cli/worker/resources.go @@ -1,4 +1,4 @@ -package main +package worker import ( "fmt" diff --git a/cmd/lotus-worker/cli.go b/cli/worker/set.go similarity index 98% rename from cmd/lotus-worker/cli.go rename to cli/worker/set.go index 393ece18c86..bdbe15ef71c 100644 --- a/cmd/lotus-worker/cli.go +++ b/cli/worker/set.go @@ -1,4 +1,4 @@ -package main +package worker import ( "fmt" diff --git a/cmd/lotus-worker/storage.go b/cli/worker/storage.go similarity index 99% rename from cmd/lotus-worker/storage.go rename to cli/worker/storage.go index 8da35c94418..42e7b5ae3c1 100644 --- a/cmd/lotus-worker/storage.go +++ b/cli/worker/storage.go @@ -1,4 +1,4 @@ -package main +package worker import ( "encoding/json" diff --git a/cmd/lotus-worker/tasks.go b/cli/worker/tasks.go similarity index 99% rename from cmd/lotus-worker/tasks.go rename to cli/worker/tasks.go index 73ca5eb0a01..a67a53f0221 100644 --- a/cmd/lotus-worker/tasks.go +++ b/cli/worker/tasks.go @@ -1,4 +1,4 @@ -package main +package worker import ( "context" diff --git a/cli/worker/worker.go b/cli/worker/worker.go new file mode 100644 index 00000000000..64580821284 --- /dev/null +++ b/cli/worker/worker.go @@ -0,0 +1,874 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/google/uuid" + "github.com/ipfs/go-datastore/namespace" + logging "github.com/ipfs/go-log/v2" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + "github.com/urfave/cli/v2" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-jsonrpc/auth" + "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-statestore" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/build/buildconstants" + lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" + "github.com/filecoin-project/lotus/lib/ulimit" + "github.com/filecoin-project/lotus/metrics" + "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" + "github.com/filecoin-project/lotus/storage/sealer/sealtasks" + "github.com/filecoin-project/lotus/storage/sealer/storiface" +) + +const FlagWorkerRepo = "worker-repo" + +// TODO remove after deprecation period +const FlagWorkerRepoDeprecation = "workerrepo" + +var log = logging.Logger("worker") + +func App() *cli.App { + local := []*cli.Command{ + runCmd, + stopCmd, + infoCmd, + storageCmd, + setCmd, + waitQuietCmd, + resourcesCmd, + tasksCmd, + } + + app := &cli.App{ + Name: "lotus-worker", + Usage: "Remote miner worker", + Version: string(build.MinerUserVersion()), + EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: FlagWorkerRepo, + Aliases: []string{FlagWorkerRepoDeprecation}, + EnvVars: []string{"LOTUS_WORKER_PATH", "WORKER_PATH"}, + Value: "~/.lotusworker", // TODO: Consider XDG_DATA_HOME + Usage: fmt.Sprintf("Specify worker repo path. flag %s and env WORKER_PATH are DEPRECATION, will REMOVE SOON", FlagWorkerRepoDeprecation), + }, + &cli.StringFlag{ + Name: "panic-reports", + EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, + Hidden: true, + Value: "~/.lotusworker", // should follow --repo default + }, + &cli.StringFlag{ + Name: "miner-repo", + Aliases: []string{"storagerepo"}, + EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, + Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME + Usage: fmt.Sprintf("Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are DEPRECATION, will REMOVE SOON"), + }, + &cli.BoolFlag{ + Name: "enable-gpu-proving", + Usage: "enable use of GPU for mining operations", + Value: true, + EnvVars: []string{"LOTUS_WORKER_ENABLE_GPU_PROVING"}, + }, + }, + + After: func(c *cli.Context) error { + if r := recover(); r != nil { + // Generate report in LOTUS_PANIC_REPORT_PATH and re-raise panic + build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagWorkerRepo), c.App.Name) + panic(r) + } + return nil + }, + Commands: local, + } + + app.Setup() + app.Metadata["repoType"] = repo.Worker + return app +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "Start lotus worker", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Usage: "host address and port the worker api will listen on", + Value: "0.0.0.0:3456", + EnvVars: []string{"LOTUS_WORKER_LISTEN"}, + }, + &cli.StringFlag{ + Name: "address", + Hidden: true, + }, + &cli.BoolFlag{ + Name: "no-local-storage", + Usage: "don't use storageminer repo for sector storage", + EnvVars: []string{"LOTUS_WORKER_NO_LOCAL_STORAGE"}, + }, + &cli.BoolFlag{ + Name: "no-swap", + Usage: "don't use swap", + Value: false, + EnvVars: []string{"LOTUS_WORKER_NO_SWAP"}, + }, + &cli.StringFlag{ + Name: "name", + Usage: "custom worker name", + EnvVars: []string{"LOTUS_WORKER_NAME"}, + DefaultText: "hostname", + }, + &cli.BoolFlag{ + Name: "addpiece", + Usage: "enable addpiece", + Value: true, + EnvVars: []string{"LOTUS_WORKER_ADDPIECE"}, + }, + &cli.BoolFlag{ + Name: "precommit1", + Usage: "enable precommit1", + Value: true, + EnvVars: []string{"LOTUS_WORKER_PRECOMMIT1"}, + }, + &cli.BoolFlag{ + Name: "unseal", + Usage: "enable unsealing", + Value: true, + EnvVars: []string{"LOTUS_WORKER_UNSEAL"}, + }, + &cli.BoolFlag{ + Name: "precommit2", + Usage: "enable precommit2", + Value: true, + EnvVars: []string{"LOTUS_WORKER_PRECOMMIT2"}, + }, + &cli.BoolFlag{ + Name: "commit", + Usage: "enable commit", + Value: true, + EnvVars: []string{"LOTUS_WORKER_COMMIT"}, + }, + &cli.BoolFlag{ + Name: "replica-update", + Usage: "enable replica update", + Value: true, + EnvVars: []string{"LOTUS_WORKER_REPLICA_UPDATE"}, + }, + &cli.BoolFlag{ + Name: "prove-replica-update2", + Usage: "enable prove replica update 2", + Value: true, + EnvVars: []string{"LOTUS_WORKER_PROVE_REPLICA_UPDATE2"}, + }, + &cli.BoolFlag{ + Name: "regen-sector-key", + Usage: "enable regen sector key", + Value: true, + EnvVars: []string{"LOTUS_WORKER_REGEN_SECTOR_KEY"}, + }, + &cli.BoolFlag{ + Name: "sector-download", + Usage: "enable external sector data download", + Value: false, + EnvVars: []string{"LOTUS_WORKER_SECTOR_DOWNLOAD"}, + }, + &cli.BoolFlag{ + Name: "windowpost", + Usage: "enable window post", + Value: false, + EnvVars: []string{"LOTUS_WORKER_WINDOWPOST"}, + }, + &cli.BoolFlag{ + Name: "winningpost", + Usage: "enable winning post", + Value: false, + EnvVars: []string{"LOTUS_WORKER_WINNINGPOST"}, + }, + &cli.BoolFlag{ + Name: "no-default", + Usage: "disable all default compute tasks, use the worker for storage/fetching only", + Value: false, + EnvVars: []string{"LOTUS_WORKER_NO_DEFAULT"}, + }, + &cli.IntFlag{ + Name: "parallel-fetch-limit", + Usage: "maximum fetch operations to run in parallel", + Value: 5, + EnvVars: []string{"LOTUS_WORKER_PARALLEL_FETCH_LIMIT"}, + }, + &cli.IntFlag{ + Name: "post-parallel-reads", + Usage: "maximum number of parallel challenge reads (0 = no limit)", + Value: 32, + EnvVars: []string{"LOTUS_WORKER_POST_PARALLEL_READS"}, + }, + &cli.DurationFlag{ + Name: "post-read-timeout", + Usage: "time limit for reading PoSt challenges (0 = no limit)", + Value: 0, + EnvVars: []string{"LOTUS_WORKER_POST_READ_TIMEOUT"}, + }, + &cli.StringFlag{ + Name: "timeout", + Usage: "used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function", + Value: "30m", + EnvVars: []string{"LOTUS_WORKER_TIMEOUT"}, + }, + &cli.StringFlag{ + Name: "http-server-timeout", + Value: "30s", + }, + &cli.BoolFlag{ + Name: "data-cid", + Usage: "Run the data-cid task. true|false", + Value: true, + DefaultText: "inherits --addpiece", + }, + &cli.StringFlag{ + Name: "external-pc2", + Usage: "command for computing PC2 externally", + }, + }, + Description: `Run lotus-worker. + +--external-pc2 can be used to compute the PreCommit2 inputs externally. +The flag behaves similarly to the related lotus-worker flag, using it in +lotus-bench may be useful for testing if the external PreCommit2 command is +invoked correctly. + +The command will be called with a number of environment variables set: +* EXTSEAL_PC2_SECTOR_NUM: the sector number +* EXTSEAL_PC2_SECTOR_MINER: the miner id +* EXTSEAL_PC2_PROOF_TYPE: the proof type +* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes +* EXTSEAL_PC2_CACHE: the path to the cache directory +* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) +* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) + +The command is expected to: +* Create cache sc-02-data-tree-r* files +* Create cache sc-02-data-tree-c* files +* Create cache p_aux / t_aux files +* Transform the sealed file in place + +Example invocation of lotus-bench as external executor: +'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT' +`, + Before: func(cctx *cli.Context) error { + if cctx.IsSet("address") { + log.Warnf("The '--address' flag is deprecated, it has been replaced by '--listen'") + if err := cctx.Set("listen", cctx.String("address")); err != nil { + return err + } + } + + return nil + }, + Action: func(cctx *cli.Context) error { + log.Info("Starting lotus worker") + + if !cctx.Bool("enable-gpu-proving") { + if err := os.Setenv("BELLMAN_NO_GPU", "true"); err != nil { + return xerrors.Errorf("could not set no-gpu env: %+v", err) + } + } + + // ensure tmpdir exists + td := os.TempDir() + if err := os.MkdirAll(td, 0755); err != nil { + return xerrors.Errorf("ensuring temp dir %s exists: %w", td, err) + } + + // Check file descriptor limit + limit, _, err := ulimit.GetLimit() + switch { + case err == ulimit.ErrUnsupported: + log.Errorw("checking file descriptor limit failed", "error", err) + case err != nil: + return xerrors.Errorf("checking fd limit: %w", err) + default: + if limit < buildconstants.MinerFDLimit { + return xerrors.Errorf("soft file descriptor limit (ulimit -n) too low, want %d, current %d", buildconstants.MinerFDLimit, limit) + } + } + + // Check DC-environment variable + sectorSizes := []string{"2KiB", "8MiB", "512MiB", "32GiB", "64GiB"} + resourcesType := reflect.TypeOf(storiface.Resources{}) + + for _, sectorSize := range sectorSizes { + for i := 0; i < resourcesType.NumField(); i++ { + field := resourcesType.Field(i) + envName := field.Tag.Get("envname") + if envName != "" { + // Check if DC_[SectorSize]_[ResourceRestriction] is set + envVar, ok := os.LookupEnv("DC_" + sectorSize + "_" + envName) + if ok { + // If it is set, convert it to DC_[ResourceRestriction] + err := os.Setenv("DC_"+envName, envVar) + if err != nil { + log.Fatalf("Error setting environment variable: %v", err) + } + log.Warnf("Converted DC_%s_%s to DC_%s, because DC is a sector-size independent job", sectorSize, envName, envName) + } + } + } + } + + // Connect to storage-miner + ctx := lcli.ReqContext(cctx) + + // Create a new context with cancel function + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Listen for interrupt signals + go func() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + <-c + cancel() + }() + + var nodeApi api.StorageMiner + var closer func() + for { + nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) + if err == nil { + _, err = nodeApi.Version(ctx) + if err == nil { + break + } + } + fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err) + select { + case <-ctx.Done(): + return xerrors.New("Interrupted by user") + case <-time.After(time.Second): + } + } + defer closer() + // Register all metric views + if err := view.Register( + metrics.DefaultViews..., + ); err != nil { + log.Fatalf("Cannot register the view: %v", err) + } + + v, err := nodeApi.Version(ctx) + if err != nil { + return err + } + if v.APIVersion != api.MinerAPIVersion0 { + return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.APIVersion{APIVersion: api.MinerAPIVersion0}) + } + log.Infof("Remote version %s", v) + + // Check params + + act, err := nodeApi.ActorAddress(ctx) + if err != nil { + return err + } + ssize, err := nodeApi.ActorSectorSize(ctx, act) + if err != nil { + return err + } + + var taskTypes []sealtasks.TaskType + var workerType string + var needParams bool + + if cctx.Bool("windowpost") { + needParams = true + workerType = sealtasks.WorkerWindowPoSt + taskTypes = append(taskTypes, sealtasks.TTGenerateWindowPoSt) + } + if cctx.Bool("winningpost") { + needParams = true + workerType = sealtasks.WorkerWinningPoSt + taskTypes = append(taskTypes, sealtasks.TTGenerateWinningPoSt) + } + + if workerType == "" { + taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeUnsealed, sealtasks.TTFinalizeReplicaUpdate) + + if !cctx.Bool("no-default") { + workerType = sealtasks.WorkerSealing + } + } + + ttDataCidDefault := false + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") { + taskTypes = append(taskTypes, sealtasks.TTAddPiece) + ttDataCidDefault = true + } + if workerType == sealtasks.WorkerSealing { + if cctx.IsSet("data-cid") { + if cctx.Bool("data-cid") { + taskTypes = append(taskTypes, sealtasks.TTDataCid) + } + } else if ttDataCidDefault { + taskTypes = append(taskTypes, sealtasks.TTDataCid) + } + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("sector-download")) && cctx.Bool("sector-download") { + taskTypes = append(taskTypes, sealtasks.TTDownloadSector) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit1")) && cctx.Bool("precommit1") { + taskTypes = append(taskTypes, sealtasks.TTPreCommit1) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("unseal")) && cctx.Bool("unseal") { + taskTypes = append(taskTypes, sealtasks.TTUnseal) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit2")) && cctx.Bool("precommit2") { + taskTypes = append(taskTypes, sealtasks.TTPreCommit2) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("commit")) && cctx.Bool("commit") { + needParams = true + taskTypes = append(taskTypes, sealtasks.TTCommit2) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("replica-update")) && cctx.Bool("replica-update") { + taskTypes = append(taskTypes, sealtasks.TTReplicaUpdate) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("prove-replica-update2")) && cctx.Bool("prove-replica-update2") { + needParams = true + taskTypes = append(taskTypes, sealtasks.TTProveReplicaUpdate2) + } + if (workerType == sealtasks.WorkerSealing || cctx.IsSet("regen-sector-key")) && cctx.Bool("regen-sector-key") { + taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey) + } + + if cctx.Bool("no-default") && workerType == "" { + workerType = sealtasks.WorkerSealing + } + + if len(taskTypes) == 0 { + return xerrors.Errorf("no task types specified") + } + for _, taskType := range taskTypes { + if taskType.WorkerType() != workerType { + return xerrors.Errorf("expected all task types to be for %s worker, but task %s is for %s worker", workerType, taskType, taskType.WorkerType()) + } + } + + if needParams { + if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(ssize)); err != nil { + return xerrors.Errorf("get params: %w", err) + } + } + + // Open repo + + repoPath := cctx.String(FlagWorkerRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + if err := r.Init(repo.Worker); err != nil { + return err + } + + lr, err := r.Lock(repo.Worker) + if err != nil { + return err + } + + var localPaths []storiface.LocalPath + + if !cctx.Bool("no-local-storage") { + b, err := json.MarshalIndent(&storiface.LocalStorageMeta{ + ID: storiface.ID(uuid.New().String()), + Weight: 10, + CanSeal: true, + CanStore: false, + }, "", " ") + if err != nil { + return xerrors.Errorf("marshaling storage config: %w", err) + } + + if err := os.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { + return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) + } + + localPaths = append(localPaths, storiface.LocalPath{ + Path: lr.Path(), + }) + } + + if err := lr.SetStorage(func(sc *storiface.StorageConfig) { + sc.StoragePaths = append(sc.StoragePaths, localPaths...) + }); err != nil { + return xerrors.Errorf("set storage config: %w", err) + } + + { + // init datastore for r.Exists + _, err := lr.Datastore(context.Background(), "/metadata") + if err != nil { + return err + } + } + if err := lr.Close(); err != nil { + return xerrors.Errorf("close repo: %w", err) + } + } + + lr, err := r.Lock(repo.Worker) + if err != nil { + return err + } + defer func() { + if err := lr.Close(); err != nil { + log.Error("closing repo", err) + } + }() + ds, err := lr.Datastore(context.Background(), "/metadata") + if err != nil { + return err + } + + log.Info("Opening local storage; connecting to master") + const unspecifiedAddress = "0.0.0.0" + + address := cctx.String("listen") + host, port, err := net.SplitHostPort(address) + if err != nil { + return err + } + + if ip := net.ParseIP(host); ip != nil { + if ip.String() == unspecifiedAddress { + timeout, err := time.ParseDuration(cctx.String("timeout")) + if err != nil { + return err + } + rip, err := extractRoutableIP(timeout) + if err != nil { + return err + } + host = rip + } + } + + var newAddress string + + // Check if the IP address is IPv6 + ip := net.ParseIP(host) + if ip.To4() == nil && ip.To16() != nil { + newAddress = "[" + host + "]:" + port + } else { + newAddress = host + ":" + port + } + + localStore, err := paths.NewLocal(ctx, lr, nodeApi, []string{"http://" + newAddress + "/remote"}) + if err != nil { + return err + } + + // Setup remote sector store + sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + if err != nil { + return xerrors.Errorf("could not get api info: %w", err) + } + + remote := paths.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"), + &paths.DefaultPartialFileHandler{}) + + fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}} + remoteHandler := func(w http.ResponseWriter, r *http.Request) { + if !auth.HasPerm(r.Context(), nil, api.PermAdmin) { + w.WriteHeader(401) + _ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"}) + return + } + + fh.ServeHTTP(w, r) + } + + // Parse ffi executor flags + + var ffiOpts []ffiwrapper.FFIWrapperOpt + + if cctx.IsSet("external-pc2") { + extSeal := ffiwrapper.ExternalSealer{ + PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")), + } + + ffiOpts = append(ffiOpts, ffiwrapper.WithExternalSealCalls(extSeal)) + } + + // Create / expose the worker + + wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix)) + + workerApi := &sealworker.Worker{ + LocalWorker: sealer.NewLocalWorkerWithExecutor( + sealer.FFIExec(ffiOpts...), + sealer.WorkerConfig{ + TaskTypes: taskTypes, + NoSwap: cctx.Bool("no-swap"), + MaxParallelChallengeReads: cctx.Int("post-parallel-reads"), + ChallengeReadTimeout: cctx.Duration("post-read-timeout"), + Name: cctx.String("name"), + }, os.LookupEnv, remote, localStore, nodeApi, nodeApi, wsts), + LocalStore: localStore, + Storage: lr, + } + + log.Info("Setting up control endpoint at " + newAddress) + + timeout, err := time.ParseDuration(cctx.String("http-server-timeout")) + if err != nil { + return xerrors.Errorf("invalid time string %s: %x", cctx.String("http-server-timeout"), err) + } + + srv := &http.Server{ + Handler: sealworker.WorkerHandler(nodeApi.AuthVerify, remoteHandler, workerApi, true), + ReadHeaderTimeout: timeout, + BaseContext: func(listener net.Listener) context.Context { + ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker")) + return ctx + }, + } + + go func() { + <-ctx.Done() + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + + nl, err := net.Listen("tcp", newAddress) + if err != nil { + return err + } + + { + a, err := net.ResolveTCPAddr("tcp", newAddress) + if err != nil { + return xerrors.Errorf("parsing address: %w", err) + } + + ma, err := manet.FromNetAddr(a) + if err != nil { + return xerrors.Errorf("creating api multiaddress: %w", err) + } + + if err := lr.SetAPIEndpoint(ma); err != nil { + return xerrors.Errorf("setting api endpoint: %w", err) + } + + ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + if err != nil { + return xerrors.Errorf("could not get miner API info: %w", err) + } + + // TODO: ideally this would be a token with some permissions dropped + if err := lr.SetAPIToken(ainfo.Token); err != nil { + return xerrors.Errorf("setting api token: %w", err) + } + } + + minerSession, err := nodeApi.Session(ctx) + if err != nil { + return xerrors.Errorf("getting miner session: %w", err) + } + + waitQuietCh := func() chan struct{} { + out := make(chan struct{}) + go func() { + workerApi.LocalWorker.WaitQuiet() + close(out) + }() + return out + } + + go func() { + heartbeats := time.NewTicker(paths.HeartbeatInterval) + defer heartbeats.Stop() + + var redeclareStorage bool + var readyCh chan struct{} + for { + // If we're reconnecting, redeclare storage first + if redeclareStorage { + log.Info("Redeclaring local storage") + + if err := localStore.Redeclare(ctx, nil, false); err != nil { + log.Errorf("Redeclaring local storage failed: %+v", err) + + select { + case <-ctx.Done(): + return // graceful shutdown + case <-heartbeats.C: + } + continue + } + } + + // TODO: we could get rid of this, but that requires tracking resources for restarted tasks correctly + if readyCh == nil { + log.Info("Making sure no local tasks are running") + readyCh = waitQuietCh() + } + + for { + curSession, err := nodeApi.Session(ctx) + if err != nil { + log.Errorf("heartbeat: checking remote session failed: %+v", err) + } else { + if curSession != minerSession { + minerSession = curSession + break + } + } + + select { + case <-readyCh: + if err := nodeApi.WorkerConnect(ctx, "http://"+newAddress+"/rpc/v0"); err != nil { + log.Errorf("Registering worker failed: %+v", err) + cancel() + return + } + + log.Info("Worker registered successfully, waiting for tasks") + + readyCh = nil + case <-heartbeats.C: + case <-ctx.Done(): + return // graceful shutdown + } + } + + log.Errorf("LOTUS-MINER CONNECTION LOST") + + redeclareStorage = true + } + }() + + go func() { + <-workerApi.Done() + // Wait 20s to allow the miner to unregister the worker on next heartbeat + time.Sleep(20 * time.Second) + log.Warn("Shutting down...") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + + return srv.Serve(nl) + }, +} + +var stopCmd = &cli.Command{ + Name: "stop", + Usage: "Stop a running lotus worker", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetWorkerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := lcli.ReqContext(cctx) + + // Detach any storage associated with this worker + err = api.StorageDetachAll(ctx) + if err != nil { + return err + } + + err = api.Shutdown(ctx) + if err != nil { + return err + } + + return nil + }, +} + +func extractRoutableIP(timeout time.Duration) (string, error) { + minerMultiAddrKey := "MINER_API_INFO" + deprecatedMinerMultiAddrKey := "STORAGE_API_INFO" + env, ok := os.LookupEnv(minerMultiAddrKey) + if !ok { + _, ok = os.LookupEnv(deprecatedMinerMultiAddrKey) + if ok { + log.Warnf("Using a deprecated env(%s) value, please use env(%s) instead.", deprecatedMinerMultiAddrKey, minerMultiAddrKey) + } + return "", xerrors.New("MINER_API_INFO environment variable required to extract IP") + } + + // Splitting the env to separate the JWT from the multiaddress + splitEnv := strings.SplitN(env, ":", 2) + if len(splitEnv) < 2 { + return "", xerrors.Errorf("invalid MINER_API_INFO format") + } + // Only take the multiaddress part + maddrStr := splitEnv[1] + + maddr, err := multiaddr.NewMultiaddr(maddrStr) + if err != nil { + return "", err + } + + minerIP, _ := maddr.ValueForProtocol(multiaddr.P_IP6) + if minerIP == "" { + minerIP, _ = maddr.ValueForProtocol(multiaddr.P_IP4) + } + minerPort, _ := maddr.ValueForProtocol(multiaddr.P_TCP) + + // Format the address appropriately + addressToDial := net.JoinHostPort(minerIP, minerPort) + + conn, err := net.DialTimeout("tcp", addressToDial, timeout) + if err != nil { + return "", err + } + + defer func() { + if cerr := conn.Close(); cerr != nil { + log.Errorf("Error closing connection: %v", cerr) + } + }() + + localAddr := conn.LocalAddr().(*net.TCPAddr) + return localAddr.IP.String(), nil +} diff --git a/cmd/lotus-miner/main.go b/cmd/lotus-miner/main.go index 76b8c0deb05..55ab1e6a132 100644 --- a/cmd/lotus-miner/main.go +++ b/cmd/lotus-miner/main.go @@ -1,168 +1,11 @@ package main import ( - "context" - "fmt" - - "github.com/fatih/color" - logging "github.com/ipfs/go-log/v2" - "github.com/urfave/cli/v2" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" - cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/lib/lotuslog" - "github.com/filecoin-project/lotus/lib/tracing" - "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/cli/miner" ) -var log = logging.Logger("main") - -const ( - FlagMinerRepo = "miner-repo" -) - -// TODO remove after deprecation period -const FlagMinerRepoDeprecation = "storagerepo" - func main() { - api.RunningNodeType = api.NodeMiner - - lotuslog.SetupLogLevels() - - local := []*cli.Command{ - initCmd, - runCmd, - stopCmd, - configCmd, - backupCmd, - lcli.WithCategory("chain", actorCmd), - lcli.WithCategory("chain", infoCmd), - lcli.WithCategory("storage", sectorsCmd), - lcli.WithCategory("storage", provingCmd), - lcli.WithCategory("storage", storageCmd), - lcli.WithCategory("storage", sealingCmd), - } - - jaeger := tracing.SetupJaegerTracing("lotus") - defer func() { - if jaeger != nil { - _ = jaeger.ForceFlush(context.Background()) - } - }() - - for _, cmd := range local { - cmd := cmd - originBefore := cmd.Before - cmd.Before = func(cctx *cli.Context) error { - if jaeger != nil { - _ = jaeger.Shutdown(cctx.Context) - } - jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) - - if cctx.IsSet("color") { - color.NoColor = !cctx.Bool("color") - } - - if originBefore != nil { - return originBefore(cctx) - } - - return nil - } - } - - app := &cli.App{ - Name: "lotus-miner", - Usage: "Filecoin decentralized storage network miner", - Version: string(build.MinerUserVersion()), - EnableBashCompletion: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "actor", - Value: "", - Usage: "specify other actor to query / manipulate", - Aliases: []string{"a"}, - }, - &cli.BoolFlag{ - // examined in the Before above - Name: "color", - Usage: "use color in display output", - DefaultText: "depends on output being a TTY", - }, - &cli.StringFlag{ - Name: "panic-reports", - EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, - Hidden: true, - Value: "~/.lotusminer", // should follow --repo default - }, - &cli.StringFlag{ - Name: "repo", - EnvVars: []string{"LOTUS_PATH"}, - Hidden: true, - Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME - }, - &cli.StringFlag{ - Name: FlagMinerRepo, - Aliases: []string{FlagMinerRepoDeprecation}, - EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, - Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME - Usage: fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation), - }, - cliutil.FlagVeryVerbose, - }, - Commands: append(local, lcli.CommonCommands...), - After: func(c *cli.Context) error { - if r := recover(); r != nil { - // Generate report in LOTUS_PATH and re-raise panic - build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagMinerRepo), c.App.Name) - panic(r) - } - return nil - }, - } - app.Setup() - app.Metadata["repoType"] = repo.StorageMiner + app := miner.App() lcli.RunApp(app) } - -func getActorAddress(ctx context.Context, cctx *cli.Context) (maddr address.Address, err error) { - if cctx.IsSet("actor") { - maddr, err = address.NewFromString(cctx.String("actor")) - if err != nil { - return maddr, err - } - return - } - - minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return address.Undef, err - } - defer closer() - - maddr, err = minerApi.ActorAddress(ctx) - if err != nil { - return maddr, xerrors.Errorf("getting actor address: %w", err) - } - - return maddr, nil -} - -func LMActorOrEnvGetter(cctx *cli.Context) (address.Address, error) { - return getActorAddress(cctx.Context, cctx) -} - -func LMActorGetter(cctx *cli.Context) (address.Address, error) { - minerApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return address.Undef, err - } - defer closer() - - return minerApi.ActorAddress(cctx.Context) -} diff --git a/cmd/lotus-worker/main.go b/cmd/lotus-worker/main.go index 6d70b8240e8..7b120499a7d 100644 --- a/cmd/lotus-worker/main.go +++ b/cmd/lotus-worker/main.go @@ -1,882 +1,25 @@ package main import ( - "context" - "encoding/json" - "fmt" - "net" - "net/http" "os" - "os/signal" - "path/filepath" - "reflect" - "strings" - "time" - "github.com/google/uuid" - "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log/v2" - "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" - "github.com/urfave/cli/v2" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-jsonrpc/auth" - "github.com/filecoin-project/go-paramfetch" - "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/build/buildconstants" - lcli "github.com/filecoin-project/lotus/cli" - cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/cmd/lotus-worker/sealworker" + "github.com/filecoin-project/lotus/cli/worker" "github.com/filecoin-project/lotus/lib/lotuslog" - "github.com/filecoin-project/lotus/lib/ulimit" - "github.com/filecoin-project/lotus/metrics" - "github.com/filecoin-project/lotus/node/modules" - "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/paths" - "github.com/filecoin-project/lotus/storage/sealer" - "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" - "github.com/filecoin-project/lotus/storage/sealer/sealtasks" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("main") -const FlagWorkerRepo = "worker-repo" - -// TODO remove after deprecation period -const FlagWorkerRepoDeprecation = "workerrepo" - func main() { api.RunningNodeType = api.NodeWorker lotuslog.SetupLogLevels() - local := []*cli.Command{ - runCmd, - stopCmd, - infoCmd, - storageCmd, - setCmd, - waitQuietCmd, - resourcesCmd, - tasksCmd, - } - - app := &cli.App{ - Name: "lotus-worker", - Usage: "Remote miner worker", - Version: string(build.MinerUserVersion()), - EnableBashCompletion: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: FlagWorkerRepo, - Aliases: []string{FlagWorkerRepoDeprecation}, - EnvVars: []string{"LOTUS_WORKER_PATH", "WORKER_PATH"}, - Value: "~/.lotusworker", // TODO: Consider XDG_DATA_HOME - Usage: fmt.Sprintf("Specify worker repo path. flag %s and env WORKER_PATH are DEPRECATION, will REMOVE SOON", FlagWorkerRepoDeprecation), - }, - &cli.StringFlag{ - Name: "panic-reports", - EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, - Hidden: true, - Value: "~/.lotusworker", // should follow --repo default - }, - &cli.StringFlag{ - Name: "miner-repo", - Aliases: []string{"storagerepo"}, - EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, - Value: "~/.lotusminer", // TODO: Consider XDG_DATA_HOME - Usage: fmt.Sprintf("Specify miner repo path. flag storagerepo and env LOTUS_STORAGE_PATH are DEPRECATION, will REMOVE SOON"), - }, - &cli.BoolFlag{ - Name: "enable-gpu-proving", - Usage: "enable use of GPU for mining operations", - Value: true, - EnvVars: []string{"LOTUS_WORKER_ENABLE_GPU_PROVING"}, - }, - }, - - After: func(c *cli.Context) error { - if r := recover(); r != nil { - // Generate report in LOTUS_PANIC_REPORT_PATH and re-raise panic - build.GenerateMinerPanicReport(c.String("panic-reports"), c.String(FlagWorkerRepo), c.App.Name) - panic(r) - } - return nil - }, - Commands: local, - } - app.Setup() - app.Metadata["repoType"] = repo.Worker - + app := worker.App() if err := app.Run(os.Args); err != nil { log.Warnf("%+v", err) return } } - -var stopCmd = &cli.Command{ - Name: "stop", - Usage: "Stop a running lotus worker", - Flags: []cli.Flag{}, - Action: func(cctx *cli.Context) error { - api, closer, err := lcli.GetWorkerAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := lcli.ReqContext(cctx) - - // Detach any storage associated with this worker - err = api.StorageDetachAll(ctx) - if err != nil { - return err - } - - err = api.Shutdown(ctx) - if err != nil { - return err - } - - return nil - }, -} - -var runCmd = &cli.Command{ - Name: "run", - Usage: "Start lotus worker", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "listen", - Usage: "host address and port the worker api will listen on", - Value: "0.0.0.0:3456", - EnvVars: []string{"LOTUS_WORKER_LISTEN"}, - }, - &cli.StringFlag{ - Name: "address", - Hidden: true, - }, - &cli.BoolFlag{ - Name: "no-local-storage", - Usage: "don't use storageminer repo for sector storage", - EnvVars: []string{"LOTUS_WORKER_NO_LOCAL_STORAGE"}, - }, - &cli.BoolFlag{ - Name: "no-swap", - Usage: "don't use swap", - Value: false, - EnvVars: []string{"LOTUS_WORKER_NO_SWAP"}, - }, - &cli.StringFlag{ - Name: "name", - Usage: "custom worker name", - EnvVars: []string{"LOTUS_WORKER_NAME"}, - DefaultText: "hostname", - }, - &cli.BoolFlag{ - Name: "addpiece", - Usage: "enable addpiece", - Value: true, - EnvVars: []string{"LOTUS_WORKER_ADDPIECE"}, - }, - &cli.BoolFlag{ - Name: "precommit1", - Usage: "enable precommit1", - Value: true, - EnvVars: []string{"LOTUS_WORKER_PRECOMMIT1"}, - }, - &cli.BoolFlag{ - Name: "unseal", - Usage: "enable unsealing", - Value: true, - EnvVars: []string{"LOTUS_WORKER_UNSEAL"}, - }, - &cli.BoolFlag{ - Name: "precommit2", - Usage: "enable precommit2", - Value: true, - EnvVars: []string{"LOTUS_WORKER_PRECOMMIT2"}, - }, - &cli.BoolFlag{ - Name: "commit", - Usage: "enable commit", - Value: true, - EnvVars: []string{"LOTUS_WORKER_COMMIT"}, - }, - &cli.BoolFlag{ - Name: "replica-update", - Usage: "enable replica update", - Value: true, - EnvVars: []string{"LOTUS_WORKER_REPLICA_UPDATE"}, - }, - &cli.BoolFlag{ - Name: "prove-replica-update2", - Usage: "enable prove replica update 2", - Value: true, - EnvVars: []string{"LOTUS_WORKER_PROVE_REPLICA_UPDATE2"}, - }, - &cli.BoolFlag{ - Name: "regen-sector-key", - Usage: "enable regen sector key", - Value: true, - EnvVars: []string{"LOTUS_WORKER_REGEN_SECTOR_KEY"}, - }, - &cli.BoolFlag{ - Name: "sector-download", - Usage: "enable external sector data download", - Value: false, - EnvVars: []string{"LOTUS_WORKER_SECTOR_DOWNLOAD"}, - }, - &cli.BoolFlag{ - Name: "windowpost", - Usage: "enable window post", - Value: false, - EnvVars: []string{"LOTUS_WORKER_WINDOWPOST"}, - }, - &cli.BoolFlag{ - Name: "winningpost", - Usage: "enable winning post", - Value: false, - EnvVars: []string{"LOTUS_WORKER_WINNINGPOST"}, - }, - &cli.BoolFlag{ - Name: "no-default", - Usage: "disable all default compute tasks, use the worker for storage/fetching only", - Value: false, - EnvVars: []string{"LOTUS_WORKER_NO_DEFAULT"}, - }, - &cli.IntFlag{ - Name: "parallel-fetch-limit", - Usage: "maximum fetch operations to run in parallel", - Value: 5, - EnvVars: []string{"LOTUS_WORKER_PARALLEL_FETCH_LIMIT"}, - }, - &cli.IntFlag{ - Name: "post-parallel-reads", - Usage: "maximum number of parallel challenge reads (0 = no limit)", - Value: 32, - EnvVars: []string{"LOTUS_WORKER_POST_PARALLEL_READS"}, - }, - &cli.DurationFlag{ - Name: "post-read-timeout", - Usage: "time limit for reading PoSt challenges (0 = no limit)", - Value: 0, - EnvVars: []string{"LOTUS_WORKER_POST_READ_TIMEOUT"}, - }, - &cli.StringFlag{ - Name: "timeout", - Usage: "used when 'listen' is unspecified. must be a valid duration recognized by golang's time.ParseDuration function", - Value: "30m", - EnvVars: []string{"LOTUS_WORKER_TIMEOUT"}, - }, - &cli.StringFlag{ - Name: "http-server-timeout", - Value: "30s", - }, - &cli.BoolFlag{ - Name: "data-cid", - Usage: "Run the data-cid task. true|false", - Value: true, - DefaultText: "inherits --addpiece", - }, - &cli.StringFlag{ - Name: "external-pc2", - Usage: "command for computing PC2 externally", - }, - }, - Description: `Run lotus-worker. - ---external-pc2 can be used to compute the PreCommit2 inputs externally. -The flag behaves similarly to the related lotus-worker flag, using it in -lotus-bench may be useful for testing if the external PreCommit2 command is -invoked correctly. - -The command will be called with a number of environment variables set: -* EXTSEAL_PC2_SECTOR_NUM: the sector number -* EXTSEAL_PC2_SECTOR_MINER: the miner id -* EXTSEAL_PC2_PROOF_TYPE: the proof type -* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes -* EXTSEAL_PC2_CACHE: the path to the cache directory -* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) -* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) - -The command is expected to: -* Create cache sc-02-data-tree-r* files -* Create cache sc-02-data-tree-c* files -* Create cache p_aux / t_aux files -* Transform the sealed file in place - -Example invocation of lotus-bench as external executor: -'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT' -`, - Before: func(cctx *cli.Context) error { - if cctx.IsSet("address") { - log.Warnf("The '--address' flag is deprecated, it has been replaced by '--listen'") - if err := cctx.Set("listen", cctx.String("address")); err != nil { - return err - } - } - - return nil - }, - Action: func(cctx *cli.Context) error { - log.Info("Starting lotus worker") - - if !cctx.Bool("enable-gpu-proving") { - if err := os.Setenv("BELLMAN_NO_GPU", "true"); err != nil { - return xerrors.Errorf("could not set no-gpu env: %+v", err) - } - } - - // ensure tmpdir exists - td := os.TempDir() - if err := os.MkdirAll(td, 0755); err != nil { - return xerrors.Errorf("ensuring temp dir %s exists: %w", td, err) - } - - // Check file descriptor limit - limit, _, err := ulimit.GetLimit() - switch { - case err == ulimit.ErrUnsupported: - log.Errorw("checking file descriptor limit failed", "error", err) - case err != nil: - return xerrors.Errorf("checking fd limit: %w", err) - default: - if limit < buildconstants.MinerFDLimit { - return xerrors.Errorf("soft file descriptor limit (ulimit -n) too low, want %d, current %d", buildconstants.MinerFDLimit, limit) - } - } - - // Check DC-environment variable - sectorSizes := []string{"2KiB", "8MiB", "512MiB", "32GiB", "64GiB"} - resourcesType := reflect.TypeOf(storiface.Resources{}) - - for _, sectorSize := range sectorSizes { - for i := 0; i < resourcesType.NumField(); i++ { - field := resourcesType.Field(i) - envName := field.Tag.Get("envname") - if envName != "" { - // Check if DC_[SectorSize]_[ResourceRestriction] is set - envVar, ok := os.LookupEnv("DC_" + sectorSize + "_" + envName) - if ok { - // If it is set, convert it to DC_[ResourceRestriction] - err := os.Setenv("DC_"+envName, envVar) - if err != nil { - log.Fatalf("Error setting environment variable: %v", err) - } - log.Warnf("Converted DC_%s_%s to DC_%s, because DC is a sector-size independent job", sectorSize, envName, envName) - } - } - } - } - - // Connect to storage-miner - ctx := lcli.ReqContext(cctx) - - // Create a new context with cancel function - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - // Listen for interrupt signals - go func() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - <-c - cancel() - }() - - var nodeApi api.StorageMiner - var closer func() - for { - nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp) - if err == nil { - _, err = nodeApi.Version(ctx) - if err == nil { - break - } - } - fmt.Printf("\r\x1b[0KConnecting to miner API... (%s)", err) - select { - case <-ctx.Done(): - return xerrors.New("Interrupted by user") - case <-time.After(time.Second): - } - } - defer closer() - // Register all metric views - if err := view.Register( - metrics.DefaultViews..., - ); err != nil { - log.Fatalf("Cannot register the view: %v", err) - } - - v, err := nodeApi.Version(ctx) - if err != nil { - return err - } - if v.APIVersion != api.MinerAPIVersion0 { - return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.APIVersion{APIVersion: api.MinerAPIVersion0}) - } - log.Infof("Remote version %s", v) - - // Check params - - act, err := nodeApi.ActorAddress(ctx) - if err != nil { - return err - } - ssize, err := nodeApi.ActorSectorSize(ctx, act) - if err != nil { - return err - } - - var taskTypes []sealtasks.TaskType - var workerType string - var needParams bool - - if cctx.Bool("windowpost") { - needParams = true - workerType = sealtasks.WorkerWindowPoSt - taskTypes = append(taskTypes, sealtasks.TTGenerateWindowPoSt) - } - if cctx.Bool("winningpost") { - needParams = true - workerType = sealtasks.WorkerWinningPoSt - taskTypes = append(taskTypes, sealtasks.TTGenerateWinningPoSt) - } - - if workerType == "" { - taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeUnsealed, sealtasks.TTFinalizeReplicaUpdate) - - if !cctx.Bool("no-default") { - workerType = sealtasks.WorkerSealing - } - } - - ttDataCidDefault := false - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") { - taskTypes = append(taskTypes, sealtasks.TTAddPiece) - ttDataCidDefault = true - } - if workerType == sealtasks.WorkerSealing { - if cctx.IsSet("data-cid") { - if cctx.Bool("data-cid") { - taskTypes = append(taskTypes, sealtasks.TTDataCid) - } - } else if ttDataCidDefault { - taskTypes = append(taskTypes, sealtasks.TTDataCid) - } - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("sector-download")) && cctx.Bool("sector-download") { - taskTypes = append(taskTypes, sealtasks.TTDownloadSector) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit1")) && cctx.Bool("precommit1") { - taskTypes = append(taskTypes, sealtasks.TTPreCommit1) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("unseal")) && cctx.Bool("unseal") { - taskTypes = append(taskTypes, sealtasks.TTUnseal) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("precommit2")) && cctx.Bool("precommit2") { - taskTypes = append(taskTypes, sealtasks.TTPreCommit2) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("commit")) && cctx.Bool("commit") { - needParams = true - taskTypes = append(taskTypes, sealtasks.TTCommit2) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("replica-update")) && cctx.Bool("replica-update") { - taskTypes = append(taskTypes, sealtasks.TTReplicaUpdate) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("prove-replica-update2")) && cctx.Bool("prove-replica-update2") { - needParams = true - taskTypes = append(taskTypes, sealtasks.TTProveReplicaUpdate2) - } - if (workerType == sealtasks.WorkerSealing || cctx.IsSet("regen-sector-key")) && cctx.Bool("regen-sector-key") { - taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey) - } - - if cctx.Bool("no-default") && workerType == "" { - workerType = sealtasks.WorkerSealing - } - - if len(taskTypes) == 0 { - return xerrors.Errorf("no task types specified") - } - for _, taskType := range taskTypes { - if taskType.WorkerType() != workerType { - return xerrors.Errorf("expected all task types to be for %s worker, but task %s is for %s worker", workerType, taskType, taskType.WorkerType()) - } - } - - if needParams { - if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(ssize)); err != nil { - return xerrors.Errorf("get params: %w", err) - } - } - - // Open repo - - repoPath := cctx.String(FlagWorkerRepo) - r, err := repo.NewFS(repoPath) - if err != nil { - return err - } - - ok, err := r.Exists() - if err != nil { - return err - } - if !ok { - if err := r.Init(repo.Worker); err != nil { - return err - } - - lr, err := r.Lock(repo.Worker) - if err != nil { - return err - } - - var localPaths []storiface.LocalPath - - if !cctx.Bool("no-local-storage") { - b, err := json.MarshalIndent(&storiface.LocalStorageMeta{ - ID: storiface.ID(uuid.New().String()), - Weight: 10, - CanSeal: true, - CanStore: false, - }, "", " ") - if err != nil { - return xerrors.Errorf("marshaling storage config: %w", err) - } - - if err := os.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { - return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) - } - - localPaths = append(localPaths, storiface.LocalPath{ - Path: lr.Path(), - }) - } - - if err := lr.SetStorage(func(sc *storiface.StorageConfig) { - sc.StoragePaths = append(sc.StoragePaths, localPaths...) - }); err != nil { - return xerrors.Errorf("set storage config: %w", err) - } - - { - // init datastore for r.Exists - _, err := lr.Datastore(context.Background(), "/metadata") - if err != nil { - return err - } - } - if err := lr.Close(); err != nil { - return xerrors.Errorf("close repo: %w", err) - } - } - - lr, err := r.Lock(repo.Worker) - if err != nil { - return err - } - defer func() { - if err := lr.Close(); err != nil { - log.Error("closing repo", err) - } - }() - ds, err := lr.Datastore(context.Background(), "/metadata") - if err != nil { - return err - } - - log.Info("Opening local storage; connecting to master") - const unspecifiedAddress = "0.0.0.0" - - address := cctx.String("listen") - host, port, err := net.SplitHostPort(address) - if err != nil { - return err - } - - if ip := net.ParseIP(host); ip != nil { - if ip.String() == unspecifiedAddress { - timeout, err := time.ParseDuration(cctx.String("timeout")) - if err != nil { - return err - } - rip, err := extractRoutableIP(timeout) - if err != nil { - return err - } - host = rip - } - } - - var newAddress string - - // Check if the IP address is IPv6 - ip := net.ParseIP(host) - if ip.To4() == nil && ip.To16() != nil { - newAddress = "[" + host + "]:" + port - } else { - newAddress = host + ":" + port - } - - localStore, err := paths.NewLocal(ctx, lr, nodeApi, []string{"http://" + newAddress + "/remote"}) - if err != nil { - return err - } - - // Setup remote sector store - sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) - if err != nil { - return xerrors.Errorf("could not get api info: %w", err) - } - - remote := paths.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"), - &paths.DefaultPartialFileHandler{}) - - fh := &paths.FetchHandler{Local: localStore, PfHandler: &paths.DefaultPartialFileHandler{}} - remoteHandler := func(w http.ResponseWriter, r *http.Request) { - if !auth.HasPerm(r.Context(), nil, api.PermAdmin) { - w.WriteHeader(401) - _ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"}) - return - } - - fh.ServeHTTP(w, r) - } - - // Parse ffi executor flags - - var ffiOpts []ffiwrapper.FFIWrapperOpt - - if cctx.IsSet("external-pc2") { - extSeal := ffiwrapper.ExternalSealer{ - PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")), - } - - ffiOpts = append(ffiOpts, ffiwrapper.WithExternalSealCalls(extSeal)) - } - - // Create / expose the worker - - wsts := statestore.New(namespace.Wrap(ds, modules.WorkerCallsPrefix)) - - workerApi := &sealworker.Worker{ - LocalWorker: sealer.NewLocalWorkerWithExecutor( - sealer.FFIExec(ffiOpts...), - sealer.WorkerConfig{ - TaskTypes: taskTypes, - NoSwap: cctx.Bool("no-swap"), - MaxParallelChallengeReads: cctx.Int("post-parallel-reads"), - ChallengeReadTimeout: cctx.Duration("post-read-timeout"), - Name: cctx.String("name"), - }, os.LookupEnv, remote, localStore, nodeApi, nodeApi, wsts), - LocalStore: localStore, - Storage: lr, - } - - log.Info("Setting up control endpoint at " + newAddress) - - timeout, err := time.ParseDuration(cctx.String("http-server-timeout")) - if err != nil { - return xerrors.Errorf("invalid time string %s: %x", cctx.String("http-server-timeout"), err) - } - - srv := &http.Server{ - Handler: sealworker.WorkerHandler(nodeApi.AuthVerify, remoteHandler, workerApi, true), - ReadHeaderTimeout: timeout, - BaseContext: func(listener net.Listener) context.Context { - ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-worker")) - return ctx - }, - } - - go func() { - <-ctx.Done() - log.Warn("Shutting down...") - if err := srv.Shutdown(context.TODO()); err != nil { - log.Errorf("shutting down RPC server failed: %s", err) - } - log.Warn("Graceful shutdown successful") - }() - - nl, err := net.Listen("tcp", newAddress) - if err != nil { - return err - } - - { - a, err := net.ResolveTCPAddr("tcp", newAddress) - if err != nil { - return xerrors.Errorf("parsing address: %w", err) - } - - ma, err := manet.FromNetAddr(a) - if err != nil { - return xerrors.Errorf("creating api multiaddress: %w", err) - } - - if err := lr.SetAPIEndpoint(ma); err != nil { - return xerrors.Errorf("setting api endpoint: %w", err) - } - - ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) - if err != nil { - return xerrors.Errorf("could not get miner API info: %w", err) - } - - // TODO: ideally this would be a token with some permissions dropped - if err := lr.SetAPIToken(ainfo.Token); err != nil { - return xerrors.Errorf("setting api token: %w", err) - } - } - - minerSession, err := nodeApi.Session(ctx) - if err != nil { - return xerrors.Errorf("getting miner session: %w", err) - } - - waitQuietCh := func() chan struct{} { - out := make(chan struct{}) - go func() { - workerApi.LocalWorker.WaitQuiet() - close(out) - }() - return out - } - - go func() { - heartbeats := time.NewTicker(paths.HeartbeatInterval) - defer heartbeats.Stop() - - var redeclareStorage bool - var readyCh chan struct{} - for { - // If we're reconnecting, redeclare storage first - if redeclareStorage { - log.Info("Redeclaring local storage") - - if err := localStore.Redeclare(ctx, nil, false); err != nil { - log.Errorf("Redeclaring local storage failed: %+v", err) - - select { - case <-ctx.Done(): - return // graceful shutdown - case <-heartbeats.C: - } - continue - } - } - - // TODO: we could get rid of this, but that requires tracking resources for restarted tasks correctly - if readyCh == nil { - log.Info("Making sure no local tasks are running") - readyCh = waitQuietCh() - } - - for { - curSession, err := nodeApi.Session(ctx) - if err != nil { - log.Errorf("heartbeat: checking remote session failed: %+v", err) - } else { - if curSession != minerSession { - minerSession = curSession - break - } - } - - select { - case <-readyCh: - if err := nodeApi.WorkerConnect(ctx, "http://"+newAddress+"/rpc/v0"); err != nil { - log.Errorf("Registering worker failed: %+v", err) - cancel() - return - } - - log.Info("Worker registered successfully, waiting for tasks") - - readyCh = nil - case <-heartbeats.C: - case <-ctx.Done(): - return // graceful shutdown - } - } - - log.Errorf("LOTUS-MINER CONNECTION LOST") - - redeclareStorage = true - } - }() - - go func() { - <-workerApi.Done() - // Wait 20s to allow the miner to unregister the worker on next heartbeat - time.Sleep(20 * time.Second) - log.Warn("Shutting down...") - if err := srv.Shutdown(context.TODO()); err != nil { - log.Errorf("shutting down RPC server failed: %s", err) - } - log.Warn("Graceful shutdown successful") - }() - - return srv.Serve(nl) - }, -} - -func extractRoutableIP(timeout time.Duration) (string, error) { - minerMultiAddrKey := "MINER_API_INFO" - deprecatedMinerMultiAddrKey := "STORAGE_API_INFO" - env, ok := os.LookupEnv(minerMultiAddrKey) - if !ok { - _, ok = os.LookupEnv(deprecatedMinerMultiAddrKey) - if ok { - log.Warnf("Using a deprecated env(%s) value, please use env(%s) instead.", deprecatedMinerMultiAddrKey, minerMultiAddrKey) - } - return "", xerrors.New("MINER_API_INFO environment variable required to extract IP") - } - - // Splitting the env to separate the JWT from the multiaddress - splitEnv := strings.SplitN(env, ":", 2) - if len(splitEnv) < 2 { - return "", xerrors.Errorf("invalid MINER_API_INFO format") - } - // Only take the multiaddress part - maddrStr := splitEnv[1] - - maddr, err := multiaddr.NewMultiaddr(maddrStr) - if err != nil { - return "", err - } - - minerIP, _ := maddr.ValueForProtocol(multiaddr.P_IP6) - if minerIP == "" { - minerIP, _ = maddr.ValueForProtocol(multiaddr.P_IP4) - } - minerPort, _ := maddr.ValueForProtocol(multiaddr.P_TCP) - - // Format the address appropriately - addressToDial := net.JoinHostPort(minerIP, minerPort) - - conn, err := net.DialTimeout("tcp", addressToDial, timeout) - if err != nil { - return "", err - } - - defer func() { - if cerr := conn.Close(); cerr != nil { - log.Errorf("Error closing connection: %v", cerr) - } - }() - - localAddr := conn.LocalAddr().(*net.TCPAddr) - return localAddr.IP.String(), nil -} diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index 6b29f6cf83a..4778689d20e 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -1,124 +1,11 @@ package main import ( - "context" - "os" - - "github.com/fatih/color" - logging "github.com/ipfs/go-log/v2" - "github.com/mattn/go-isatty" - "github.com/urfave/cli/v2" - "go.opencensus.io/trace" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/cli/clicommands" - cliutil "github.com/filecoin-project/lotus/cli/util" - "github.com/filecoin-project/lotus/lib/lotuslog" - "github.com/filecoin-project/lotus/lib/tracing" - "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/cli/lotus" ) -var log = logging.Logger("main") - -var AdvanceBlockCmd *cli.Command - func main() { - api.RunningNodeType = api.NodeFull - - lotuslog.SetupLogLevels() - - local := []*cli.Command{ - DaemonCmd, - backupCmd, - configCmd, - } - if AdvanceBlockCmd != nil { - local = append(local, AdvanceBlockCmd) - } - - jaeger := tracing.SetupJaegerTracing("lotus") - defer func() { - if jaeger != nil { - _ = jaeger.ForceFlush(context.Background()) - } - }() - - for _, cmd := range local { - cmd := cmd - originBefore := cmd.Before - cmd.Before = func(cctx *cli.Context) error { - if jaeger != nil { - _ = jaeger.Shutdown(cctx.Context) - } - jaeger = tracing.SetupJaegerTracing("lotus/" + cmd.Name) - - if cctx.IsSet("color") { - color.NoColor = !cctx.Bool("color") - } - - if originBefore != nil { - return originBefore(cctx) - } - return nil - } - } - ctx, span := trace.StartSpan(context.Background(), "/cli") - defer span.End() - - interactiveDef := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) - - app := &cli.App{ - Name: "lotus", - Usage: "Filecoin decentralized storage network client", - Version: string(build.NodeUserVersion()), - EnableBashCompletion: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "panic-reports", - EnvVars: []string{"LOTUS_PANIC_REPORT_PATH"}, - Hidden: true, - Value: "~/.lotus", // should follow --repo default - }, - &cli.BoolFlag{ - // examined in the Before above - Name: "color", - Usage: "use color in display output", - DefaultText: "depends on output being a TTY", - }, - &cli.StringFlag{ - Name: "repo", - EnvVars: []string{"LOTUS_PATH"}, - Hidden: true, - Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME - }, - &cli.BoolFlag{ - Name: "interactive", - Usage: "setting to false will disable interactive functionality of commands", - Value: interactiveDef, - }, - &cli.BoolFlag{ - Name: "force-send", - Usage: "if true, will ignore pre-send checks", - }, - cliutil.FlagVeryVerbose, - }, - After: func(c *cli.Context) error { - if r := recover(); r != nil { - // Generate report in LOTUS_PATH and re-raise panic - build.GenerateNodePanicReport(c.String("panic-reports"), c.String("repo"), c.App.Name) - panic(r) - } - return nil - }, - - Commands: append(local, clicommands.Commands...), - } - - app.Setup() - app.Metadata["traceContext"] = ctx - app.Metadata["repoType"] = repo.FullNode - + app := lotus.App() lcli.RunApp(app) } diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index c483ac7b1b0..250df6b94e2 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -1,4 +1,5 @@ # lotus-miner + ``` NAME: lotus-miner - Filecoin decentralized storage network miner @@ -10,20 +11,21 @@ VERSION: 1.32.1-dev COMMANDS: - init Initialize a lotus miner repo - run Start a lotus miner process - stop Stop a running lotus miner - config Manage node config - backup Create node metadata backup - auth Manage RPC permissions - log Manage logging - wait-api Wait for lotus api to come online - fetch-params Fetch proving parameters - version Print version - help, h Shows a list of commands or help for one command + init Initialize a lotus miner repo + run Start a lotus miner process + stop Stop a running lotus miner + config Manage node config + backup Create node metadata backup + version Print version + help, h Shows a list of commands or help for one command CHAIN: actor manipulate the miner actor info Print miner info + DEVELOPER: + auth Manage RPC permissions + log Manage logging + wait-api Wait for lotus api to come online + fetch-params Fetch proving parameters STORAGE: sectors interact with sector store proving View proving information @@ -40,6 +42,7 @@ GLOBAL OPTIONS: ``` ## lotus-miner init + ``` NAME: lotus-miner init - Initialize a lotus miner repo @@ -69,6 +72,7 @@ OPTIONS: ``` ### lotus-miner init restore + ``` NAME: lotus-miner init restore - Initialize a lotus miner repo from a backup @@ -84,6 +88,7 @@ OPTIONS: ``` ## lotus-miner run + ``` NAME: lotus-miner run - Start a lotus miner process @@ -100,6 +105,7 @@ OPTIONS: ``` ## lotus-miner stop + ``` NAME: lotus-miner stop - Stop a running lotus miner @@ -112,6 +118,7 @@ OPTIONS: ``` ## lotus-miner config + ``` NAME: lotus-miner config - Manage node config @@ -129,6 +136,7 @@ OPTIONS: ``` ### lotus-miner config default + ``` NAME: lotus-miner config default - Print default node config @@ -142,6 +150,7 @@ OPTIONS: ``` ### lotus-miner config updated + ``` NAME: lotus-miner config updated - Print updated node config @@ -155,6 +164,7 @@ OPTIONS: ``` ## lotus-miner backup + ``` NAME: lotus-miner backup - Create node metadata backup @@ -175,151 +185,8 @@ OPTIONS: --help, -h show help ``` -## lotus-miner auth -``` -NAME: - lotus-miner auth - Manage RPC permissions - -USAGE: - lotus-miner auth command [command options] [arguments...] - -COMMANDS: - create-token Create token - api-info Get token with API info required to connect to this node - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help -``` - -### lotus-miner auth create-token -``` -NAME: - lotus-miner auth create-token - Create token - -USAGE: - lotus-miner auth create-token [command options] [arguments...] - -OPTIONS: - --perm value permission to assign to the token, one of: read, write, sign, admin - --help, -h show help -``` - -### lotus-miner auth api-info -``` -NAME: - lotus-miner auth api-info - Get token with API info required to connect to this node - -USAGE: - lotus-miner auth api-info [command options] [arguments...] - -OPTIONS: - --perm value permission to assign to the token, one of: read, write, sign, admin - --help, -h show help -``` - -## lotus-miner log -``` -NAME: - lotus-miner log - Manage logging - -USAGE: - lotus-miner log command [command options] [arguments...] - -COMMANDS: - list List log systems - set-level Set log level - alerts Get alert states - help, h Shows a list of commands or help for one command - -OPTIONS: - --help, -h show help -``` - -### lotus-miner log list -``` -NAME: - lotus-miner log list - List log systems - -USAGE: - lotus-miner log list [command options] [arguments...] - -OPTIONS: - --help, -h show help -``` - -### lotus-miner log set-level -``` -NAME: - lotus-miner log set-level - Set log level - -USAGE: - lotus-miner log set-level [command options] [level] - -DESCRIPTION: - Set the log level for logging systems: - - The system flag can be specified multiple times. - - eg) log set-level --system chain --system chainxchg debug - - Available Levels: - debug - info - warn - error - - Environment Variables: - GOLOG_LOG_LEVEL - Default log level for all log systems - GOLOG_LOG_FMT - Change output log format (json, nocolor) - GOLOG_FILE - Write logs to file - GOLOG_OUTPUT - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr - - -OPTIONS: - --system value [ --system value ] limit to log system - --help, -h show help -``` - -### lotus-miner log alerts -``` -NAME: - lotus-miner log alerts - Get alert states - -USAGE: - lotus-miner log alerts [command options] [arguments...] - -OPTIONS: - --all get all (active and inactive) alerts (default: false) - --help, -h show help -``` - -## lotus-miner wait-api -``` -NAME: - lotus-miner wait-api - Wait for lotus api to come online - -USAGE: - lotus-miner wait-api [command options] [arguments...] - -OPTIONS: - --timeout value duration to wait till fail (default: 30s) - --help, -h show help -``` - -## lotus-miner fetch-params -``` -NAME: - lotus-miner fetch-params - Fetch proving parameters - -USAGE: - lotus-miner fetch-params [command options] [sectorSize] - -OPTIONS: - --help, -h show help -``` - ## lotus-miner version + ``` NAME: lotus-miner version - Print version @@ -332,6 +199,7 @@ OPTIONS: ``` ## lotus-miner actor + ``` NAME: lotus-miner actor - manipulate the miner actor @@ -359,6 +227,7 @@ OPTIONS: ``` ### lotus-miner actor set-addresses + ``` NAME: lotus-miner actor set-addresses - set addresses that your miner can be publicly dialed on @@ -374,6 +243,7 @@ OPTIONS: ``` ### lotus-miner actor settle-deal + ``` NAME: lotus-miner actor settle-deal - Settle deals manually, if dealIds are not provided all deals will be settled @@ -387,6 +257,7 @@ OPTIONS: ``` ### lotus-miner actor withdraw + ``` NAME: lotus-miner actor withdraw - withdraw available balance to beneficiary @@ -401,6 +272,7 @@ OPTIONS: ``` ### lotus-miner actor repay-debt + ``` NAME: lotus-miner actor repay-debt - pay down a miner's debt @@ -414,6 +286,7 @@ OPTIONS: ``` ### lotus-miner actor set-peer-id + ``` NAME: lotus-miner actor set-peer-id - set the peer id of your miner @@ -427,6 +300,7 @@ OPTIONS: ``` ### lotus-miner actor set-owner + ``` NAME: lotus-miner actor set-owner - Set owner address (this command should be invoked twice, first with the old owner as the senderAddress, and then with the new owner) @@ -440,6 +314,7 @@ OPTIONS: ``` ### lotus-miner actor control + ``` NAME: lotus-miner actor control - Manage control addresses @@ -457,6 +332,7 @@ OPTIONS: ``` #### lotus-miner actor control list + ``` NAME: lotus-miner actor control list - Get currently set control addresses @@ -470,6 +346,7 @@ OPTIONS: ``` #### lotus-miner actor control set + ``` NAME: lotus-miner actor control set - Set control address(-es) @@ -483,6 +360,7 @@ OPTIONS: ``` ### lotus-miner actor propose-change-worker + ``` NAME: lotus-miner actor propose-change-worker - Propose a worker address change @@ -496,6 +374,7 @@ OPTIONS: ``` ### lotus-miner actor confirm-change-worker + ``` NAME: lotus-miner actor confirm-change-worker - Confirm a worker address change @@ -509,6 +388,7 @@ OPTIONS: ``` ### lotus-miner actor compact-allocated + ``` NAME: lotus-miner actor compact-allocated - compact allocated sectors bitfield @@ -524,6 +404,7 @@ OPTIONS: ``` ### lotus-miner actor propose-change-beneficiary + ``` NAME: lotus-miner actor propose-change-beneficiary - Propose a beneficiary address change @@ -539,6 +420,7 @@ OPTIONS: ``` ### lotus-miner actor confirm-change-beneficiary + ``` NAME: lotus-miner actor confirm-change-beneficiary - Confirm a beneficiary address change @@ -554,6 +436,7 @@ OPTIONS: ``` ## lotus-miner info + ``` NAME: lotus-miner info - Print miner info @@ -572,6 +455,7 @@ OPTIONS: ``` ### lotus-miner info all + ``` NAME: lotus-miner info all - dump all related miner info @@ -584,6 +468,7 @@ OPTIONS: ``` ## lotus-miner sectors + ``` NAME: lotus-miner sectors - interact with sector store @@ -620,6 +505,7 @@ OPTIONS: ``` ### lotus-miner sectors status + ``` NAME: lotus-miner sectors status - Get the seal status of a sector by its number @@ -636,6 +522,7 @@ OPTIONS: ``` ### lotus-miner sectors list + ``` NAME: lotus-miner sectors list - List sectors @@ -660,6 +547,7 @@ OPTIONS: ``` #### lotus-miner sectors list upgrade-bounds + ``` NAME: lotus-miner sectors list upgrade-bounds - Output upgrade bounds for available sectors @@ -675,6 +563,7 @@ OPTIONS: ``` ### lotus-miner sectors refs + ``` NAME: lotus-miner sectors refs - List References to sectors @@ -687,6 +576,7 @@ OPTIONS: ``` ### lotus-miner sectors update-state + ``` NAME: lotus-miner sectors update-state - ADVANCED: manually update the state of a sector, this may aid in error recovery @@ -700,6 +590,7 @@ OPTIONS: ``` ### lotus-miner sectors pledge + ``` NAME: lotus-miner sectors pledge - store random data in a sector @@ -712,6 +603,7 @@ OPTIONS: ``` ### lotus-miner sectors numbers + ``` NAME: lotus-miner sectors numbers - manage sector number assignments @@ -731,6 +623,7 @@ OPTIONS: ``` #### lotus-miner sectors numbers info + ``` NAME: lotus-miner sectors numbers info - view sector assigner state @@ -743,6 +636,7 @@ OPTIONS: ``` #### lotus-miner sectors numbers reservations + ``` NAME: lotus-miner sectors numbers reservations - list sector number reservations @@ -755,6 +649,7 @@ OPTIONS: ``` #### lotus-miner sectors numbers reserve + ``` NAME: lotus-miner sectors numbers reserve - create sector number reservations @@ -768,6 +663,7 @@ OPTIONS: ``` #### lotus-miner sectors numbers free + ``` NAME: lotus-miner sectors numbers free - remove sector number reservations @@ -780,6 +676,7 @@ OPTIONS: ``` ### lotus-miner sectors precommits + ``` NAME: lotus-miner sectors precommits - Print on-chain precommit info @@ -792,6 +689,7 @@ OPTIONS: ``` ### lotus-miner sectors check-expire + ``` NAME: lotus-miner sectors check-expire - Inspect expiring sectors @@ -805,6 +703,7 @@ OPTIONS: ``` ### lotus-miner sectors expired + ``` NAME: lotus-miner sectors expired - Get or cleanup expired sectors @@ -820,6 +719,7 @@ OPTIONS: ``` ### lotus-miner sectors extend + ``` NAME: lotus-miner sectors extend - Extend expiring sectors while not exceeding each sector's max life @@ -844,6 +744,7 @@ OPTIONS: ``` ### lotus-miner sectors terminate + ``` NAME: lotus-miner sectors terminate - Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector) @@ -862,6 +763,7 @@ OPTIONS: ``` #### lotus-miner sectors terminate flush + ``` NAME: lotus-miner sectors terminate flush - Send a terminate message if there are sectors queued for termination @@ -874,6 +776,7 @@ OPTIONS: ``` #### lotus-miner sectors terminate pending + ``` NAME: lotus-miner sectors terminate pending - List sector numbers of sectors pending termination @@ -886,6 +789,7 @@ OPTIONS: ``` ### lotus-miner sectors remove + ``` NAME: lotus-miner sectors remove - Forcefully remove a sector (WARNING: This means losing power and collateral for the removed sector (use 'terminate' for lower penalty)) @@ -899,6 +803,7 @@ OPTIONS: ``` ### lotus-miner sectors snap-up + ``` NAME: lotus-miner sectors snap-up - Mark a committed capacity sector to be filled with deals @@ -911,6 +816,7 @@ OPTIONS: ``` ### lotus-miner sectors abort-upgrade + ``` NAME: lotus-miner sectors abort-upgrade - Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before @@ -924,6 +830,7 @@ OPTIONS: ``` ### lotus-miner sectors seal + ``` NAME: lotus-miner sectors seal - Manually start sealing a sector (filling any unused space with junk) @@ -936,6 +843,7 @@ OPTIONS: ``` ### lotus-miner sectors set-seal-delay + ``` NAME: lotus-miner sectors set-seal-delay - Set the time (in minutes) that a new sector waits for deals before sealing starts @@ -949,6 +857,7 @@ OPTIONS: ``` ### lotus-miner sectors get-cc-collateral + ``` NAME: lotus-miner sectors get-cc-collateral - Get the collateral required to pledge a committed capacity sector @@ -962,6 +871,7 @@ OPTIONS: ``` ### lotus-miner sectors batching + ``` NAME: lotus-miner sectors batching - manage batch sector operations @@ -979,6 +889,7 @@ OPTIONS: ``` #### lotus-miner sectors batching commit + ``` NAME: lotus-miner sectors batching commit - list sectors waiting in commit batch queue @@ -992,6 +903,7 @@ OPTIONS: ``` #### lotus-miner sectors batching precommit + ``` NAME: lotus-miner sectors batching precommit - list sectors waiting in precommit batch queue @@ -1005,6 +917,7 @@ OPTIONS: ``` ### lotus-miner sectors match-pending-pieces + ``` NAME: lotus-miner sectors match-pending-pieces - force a refreshed match of pending pieces to open sectors without manually waiting for more deals @@ -1017,6 +930,7 @@ OPTIONS: ``` ### lotus-miner sectors compact-partitions + ``` NAME: lotus-miner sectors compact-partitions - removes dead sectors from partitions and reduces the number of partitions used if possible @@ -1032,6 +946,7 @@ OPTIONS: ``` ### lotus-miner sectors unseal + ``` NAME: lotus-miner sectors unseal - unseal a sector @@ -1044,6 +959,7 @@ OPTIONS: ``` ## lotus-miner proving + ``` NAME: lotus-miner proving - View proving information @@ -1067,6 +983,7 @@ OPTIONS: ``` ### lotus-miner proving info + ``` NAME: lotus-miner proving info - View current state information @@ -1079,6 +996,7 @@ OPTIONS: ``` ### lotus-miner proving deadlines + ``` NAME: lotus-miner proving deadlines - View the current proving period deadlines information @@ -1092,6 +1010,7 @@ OPTIONS: ``` ### lotus-miner proving deadline + ``` NAME: lotus-miner proving deadline - View the current proving period deadline information by its index @@ -1106,6 +1025,7 @@ OPTIONS: ``` ### lotus-miner proving faults + ``` NAME: lotus-miner proving faults - View the currently known proving faulty sectors information @@ -1118,6 +1038,7 @@ OPTIONS: ``` ### lotus-miner proving check + ``` NAME: lotus-miner proving check - Check sectors provable @@ -1134,6 +1055,7 @@ OPTIONS: ``` ### lotus-miner proving workers + ``` NAME: lotus-miner proving workers - list workers @@ -1146,6 +1068,7 @@ OPTIONS: ``` ### lotus-miner proving compute + ``` NAME: lotus-miner proving compute - Compute simulated proving tasks @@ -1162,6 +1085,7 @@ OPTIONS: ``` #### lotus-miner proving compute windowed-post + ``` NAME: lotus-miner proving compute windowed-post - Compute WindowPoSt for a specific deadline @@ -1178,6 +1102,7 @@ OPTIONS: ``` ### lotus-miner proving recover-faults + ``` NAME: lotus-miner proving recover-faults - Manually recovers faulty sectors on chain @@ -1191,6 +1116,7 @@ OPTIONS: ``` ## lotus-miner storage + ``` NAME: lotus-miner storage - manage sector storage @@ -1219,6 +1145,7 @@ OPTIONS: ``` ### lotus-miner storage attach + ``` NAME: lotus-miner storage attach - attach local storage path @@ -1259,6 +1186,7 @@ OPTIONS: ``` ### lotus-miner storage detach + ``` NAME: lotus-miner storage detach - detach local storage path @@ -1272,6 +1200,7 @@ OPTIONS: ``` ### lotus-miner storage redeclare + ``` NAME: lotus-miner storage redeclare - redeclare sectors in a local storage path @@ -1287,6 +1216,7 @@ OPTIONS: ``` ### lotus-miner storage list + ``` NAME: lotus-miner storage list - list local storage paths @@ -1303,6 +1233,7 @@ OPTIONS: ``` #### lotus-miner storage list sectors + ``` NAME: lotus-miner storage list sectors - get list of all sector files @@ -1315,6 +1246,7 @@ OPTIONS: ``` ### lotus-miner storage find + ``` NAME: lotus-miner storage find - find sector in the storage system @@ -1327,6 +1259,7 @@ OPTIONS: ``` ### lotus-miner storage cleanup + ``` NAME: lotus-miner storage cleanup - trigger cleanup actions @@ -1340,6 +1273,7 @@ OPTIONS: ``` ### lotus-miner storage locks + ``` NAME: lotus-miner storage locks - show active sector locks @@ -1352,6 +1286,7 @@ OPTIONS: ``` ## lotus-miner sealing + ``` NAME: lotus-miner sealing - interact with sealing pipeline @@ -1372,6 +1307,7 @@ OPTIONS: ``` ### lotus-miner sealing jobs + ``` NAME: lotus-miner sealing jobs - list running jobs @@ -1385,6 +1321,7 @@ OPTIONS: ``` ### lotus-miner sealing workers + ``` NAME: lotus-miner sealing workers - list workers @@ -1397,6 +1334,7 @@ OPTIONS: ``` ### lotus-miner sealing sched-diag + ``` NAME: lotus-miner sealing sched-diag - Dump internal scheduler state @@ -1410,6 +1348,7 @@ OPTIONS: ``` ### lotus-miner sealing abort + ``` NAME: lotus-miner sealing abort - Abort a running job @@ -1423,6 +1362,7 @@ OPTIONS: ``` ### lotus-miner sealing data-cid + ``` NAME: lotus-miner sealing data-cid - Compute data CID using workers @@ -1434,3 +1374,162 @@ OPTIONS: --file-size value real file size (default: 0) --help, -h show help ``` + +## lotus-miner auth + +``` +NAME: + lotus-miner auth - Manage RPC permissions + +USAGE: + lotus-miner auth command [command options] [arguments...] + +COMMANDS: + create-token Create token + api-info Get token with API info required to connect to this node + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help +``` + +### lotus-miner auth create-token + +``` +NAME: + lotus auth create-token - Create token + +USAGE: + lotus auth create-token [command options] [arguments...] + +OPTIONS: + --perm value permission to assign to the token, one of: read, write, sign, admin + --help, -h show help +``` + +### lotus-miner auth api-info + +``` +NAME: + lotus auth api-info - Get token with API info required to connect to this node + +USAGE: + lotus auth api-info [command options] [arguments...] + +OPTIONS: + --perm value permission to assign to the token, one of: read, write, sign, admin + --help, -h show help +``` + +## lotus-miner log + +``` +NAME: + lotus-miner log - Manage logging + +USAGE: + lotus-miner log command [command options] [arguments...] + +COMMANDS: + list List log systems + set-level Set log level + alerts Get alert states + help, h Shows a list of commands or help for one command + +OPTIONS: + --help, -h show help +``` + +### lotus-miner log list + +``` +NAME: + lotus log list - List log systems + +USAGE: + lotus log list [command options] [arguments...] + +OPTIONS: + --help, -h show help +``` + +### lotus-miner log set-level + +``` +NAME: + lotus log set-level - Set log level + +USAGE: + lotus log set-level [command options] [level] + +DESCRIPTION: + Set the log level for logging systems: + + The system flag can be specified multiple times. + + eg) log set-level --system chain --system chainxchg debug + + Available Levels: + debug + info + warn + error + + Environment Variables: + GOLOG_LOG_LEVEL - Default log level for all log systems + GOLOG_LOG_FMT - Change output log format (json, nocolor) + GOLOG_FILE - Write logs to file + GOLOG_OUTPUT - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr + + +OPTIONS: + --system value [ --system value ] limit to log system + --help, -h show help +``` + +### lotus-miner log alerts + +``` +NAME: + lotus log alerts - Get alert states + +USAGE: + lotus log alerts [command options] [arguments...] + +OPTIONS: + --all get all (active and inactive) alerts (default: false) + --help, -h show help +``` + +## lotus-miner wait-api + +``` +NAME: + lotus-miner wait-api - Wait for lotus api to come online + +USAGE: + lotus-miner wait-api [command options] [arguments...] + +CATEGORY: + DEVELOPER + +OPTIONS: + --timeout value duration to wait till fail (default: 30s) + --help, -h show help +``` + +## lotus-miner fetch-params + +``` +NAME: + lotus-miner fetch-params - Fetch proving parameters + +USAGE: + lotus-miner fetch-params [command options] [sectorSize] + +CATEGORY: + DEVELOPER + +OPTIONS: + --help, -h show help +``` diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index 81b639f728b..7e5cf8c4781 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -1,4 +1,5 @@ # lotus-worker + ``` NAME: lotus-worker - Remote miner worker @@ -27,6 +28,7 @@ GLOBAL OPTIONS: ``` ## lotus-worker run + ``` NAME: lotus-worker run - Start lotus worker @@ -89,6 +91,7 @@ OPTIONS: ``` ## lotus-worker stop + ``` NAME: lotus-worker stop - Stop a running lotus worker @@ -101,6 +104,7 @@ OPTIONS: ``` ## lotus-worker info + ``` NAME: lotus-worker info - Print worker info @@ -113,6 +117,7 @@ OPTIONS: ``` ## lotus-worker storage + ``` NAME: lotus-worker storage - manage sector storage @@ -131,6 +136,7 @@ OPTIONS: ``` ### lotus-worker storage attach + ``` NAME: lotus-worker storage attach - attach local storage path @@ -150,6 +156,7 @@ OPTIONS: ``` ### lotus-worker storage detach + ``` NAME: lotus-worker storage detach - detach local storage path @@ -163,6 +170,7 @@ OPTIONS: ``` ### lotus-worker storage redeclare + ``` NAME: lotus-worker storage redeclare - redeclare sectors in a local storage path @@ -178,6 +186,7 @@ OPTIONS: ``` ## lotus-worker resources + ``` NAME: lotus-worker resources - Manage resource table overrides @@ -192,6 +201,7 @@ OPTIONS: ``` ## lotus-worker tasks + ``` NAME: lotus-worker tasks - Manage task processing @@ -209,6 +219,7 @@ OPTIONS: ``` ### lotus-worker tasks enable + ``` NAME: lotus-worker tasks enable - Enable a task type @@ -222,6 +233,7 @@ OPTIONS: ``` ### lotus-worker tasks disable + ``` NAME: lotus-worker tasks disable - Disable a task type diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 1237c7e44ce..4b638b57d80 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -1,4 +1,5 @@ # lotus + ``` NAME: lotus - Filecoin decentralized storage network client @@ -41,7 +42,7 @@ COMMANDS: GLOBAL OPTIONS: --color use color in display output (default: depends on output being a TTY) - --interactive setting to false will disable interactive functionality of commands (default: false) + --interactive setting to false will disable interactive functionality of commands (default: true) --force-send if true, will ignore pre-send checks (default: false) --vv enables very verbose mode, useful for debugging the CLI (default: false) --help, -h show help @@ -49,6 +50,7 @@ GLOBAL OPTIONS: ``` ## lotus daemon + ``` NAME: lotus daemon - Start a lotus daemon process @@ -80,6 +82,7 @@ OPTIONS: ``` ### lotus daemon stop + ``` NAME: lotus daemon stop - Stop a running lotus daemon @@ -92,6 +95,7 @@ OPTIONS: ``` ## lotus backup + ``` NAME: lotus backup - Create node metadata backup @@ -113,6 +117,7 @@ OPTIONS: ``` ## lotus config + ``` NAME: lotus config - Manage node config @@ -130,6 +135,7 @@ OPTIONS: ``` ### lotus config default + ``` NAME: lotus config default - Print default node config @@ -143,6 +149,7 @@ OPTIONS: ``` ### lotus config updated + ``` NAME: lotus config updated - Print updated node config @@ -156,6 +163,7 @@ OPTIONS: ``` ## lotus version + ``` NAME: lotus version - Print version @@ -168,6 +176,7 @@ OPTIONS: ``` ## lotus send + ``` NAME: lotus send - Send funds between accounts @@ -193,6 +202,7 @@ OPTIONS: ``` ## lotus wallet + ``` NAME: lotus wallet - Manage wallet @@ -219,6 +229,7 @@ OPTIONS: ``` ### lotus wallet new + ``` NAME: lotus wallet new - Generate a new key of the given type @@ -231,6 +242,7 @@ OPTIONS: ``` ### lotus wallet list + ``` NAME: lotus wallet list - List wallet address @@ -246,6 +258,7 @@ OPTIONS: ``` ### lotus wallet balance + ``` NAME: lotus wallet balance - Get account balance @@ -258,6 +271,7 @@ OPTIONS: ``` ### lotus wallet export + ``` NAME: lotus wallet export - export keys @@ -270,6 +284,7 @@ OPTIONS: ``` ### lotus wallet import + ``` NAME: lotus wallet import - import keys @@ -284,6 +299,7 @@ OPTIONS: ``` ### lotus wallet default + ``` NAME: lotus wallet default - Get default wallet address @@ -296,6 +312,7 @@ OPTIONS: ``` ### lotus wallet set-default + ``` NAME: lotus wallet set-default - Set default wallet address @@ -308,6 +325,7 @@ OPTIONS: ``` ### lotus wallet sign + ``` NAME: lotus wallet sign - sign a message @@ -320,6 +338,7 @@ OPTIONS: ``` ### lotus wallet verify + ``` NAME: lotus wallet verify - verify the signature of a message @@ -332,6 +351,7 @@ OPTIONS: ``` ### lotus wallet delete + ``` NAME: lotus wallet delete - Soft delete an address from the wallet - hard deletion needed for permanent removal @@ -344,6 +364,7 @@ OPTIONS: ``` ### lotus wallet market + ``` NAME: lotus wallet market - Interact with market balances @@ -361,6 +382,7 @@ OPTIONS: ``` #### lotus wallet market withdraw + ``` NAME: lotus wallet market withdraw - Withdraw funds from the Storage Market Actor @@ -376,6 +398,7 @@ OPTIONS: ``` #### lotus wallet market add + ``` NAME: lotus wallet market add - Add funds to the Storage Market Actor @@ -390,6 +413,7 @@ OPTIONS: ``` ## lotus info + ``` NAME: lotus info - Print node info @@ -405,6 +429,7 @@ OPTIONS: ``` ## lotus msig + ``` NAME: lotus msig - Interact with a multisig wallet @@ -438,6 +463,7 @@ OPTIONS: ``` ### lotus msig create + ``` NAME: lotus msig create - Create a new multisig wallet @@ -454,6 +480,7 @@ OPTIONS: ``` ### lotus msig inspect + ``` NAME: lotus msig inspect - Inspect a multisig wallet @@ -468,6 +495,7 @@ OPTIONS: ``` ### lotus msig propose + ``` NAME: lotus msig propose - Propose a multisig transaction @@ -481,6 +509,7 @@ OPTIONS: ``` ### lotus msig propose-remove + ``` NAME: lotus msig propose-remove - Propose to remove a signer @@ -495,6 +524,7 @@ OPTIONS: ``` ### lotus msig approve + ``` NAME: lotus msig approve - Approve a multisig message @@ -508,6 +538,7 @@ OPTIONS: ``` ### lotus msig cancel + ``` NAME: lotus msig cancel - Cancel a multisig message @@ -521,6 +552,7 @@ OPTIONS: ``` ### lotus msig add-propose + ``` NAME: lotus msig add-propose - Propose to add a signer @@ -535,6 +567,7 @@ OPTIONS: ``` ### lotus msig add-approve + ``` NAME: lotus msig add-approve - Approve a message to add a signer @@ -548,6 +581,7 @@ OPTIONS: ``` ### lotus msig add-cancel + ``` NAME: lotus msig add-cancel - Cancel a message to add a signer @@ -561,6 +595,7 @@ OPTIONS: ``` ### lotus msig swap-propose + ``` NAME: lotus msig swap-propose - Propose to swap signers @@ -574,6 +609,7 @@ OPTIONS: ``` ### lotus msig swap-approve + ``` NAME: lotus msig swap-approve - Approve a message to swap signers @@ -587,6 +623,7 @@ OPTIONS: ``` ### lotus msig swap-cancel + ``` NAME: lotus msig swap-cancel - Cancel a message to swap signers @@ -600,6 +637,7 @@ OPTIONS: ``` ### lotus msig lock-propose + ``` NAME: lotus msig lock-propose - Propose to lock up some balance @@ -613,6 +651,7 @@ OPTIONS: ``` ### lotus msig lock-approve + ``` NAME: lotus msig lock-approve - Approve a message to lock up some balance @@ -626,6 +665,7 @@ OPTIONS: ``` ### lotus msig lock-cancel + ``` NAME: lotus msig lock-cancel - Cancel a message to lock up some balance @@ -639,6 +679,7 @@ OPTIONS: ``` ### lotus msig vested + ``` NAME: lotus msig vested - Gets the amount vested in an msig between two epochs @@ -653,6 +694,7 @@ OPTIONS: ``` ### lotus msig propose-threshold + ``` NAME: lotus msig propose-threshold - Propose setting a different signing threshold on the account @@ -666,6 +708,7 @@ OPTIONS: ``` ## lotus filplus + ``` NAME: lotus filplus - Interact with the verified registry actor used by Filplus @@ -692,6 +735,7 @@ OPTIONS: ``` ### lotus filplus grant-datacap + ``` NAME: lotus filplus grant-datacap - give allowance to the specified verified client address @@ -705,6 +749,7 @@ OPTIONS: ``` ### lotus filplus list-notaries + ``` NAME: lotus filplus list-notaries - list all notaries @@ -717,6 +762,7 @@ OPTIONS: ``` ### lotus filplus list-clients + ``` NAME: lotus filplus list-clients - list all verified clients @@ -729,6 +775,7 @@ OPTIONS: ``` ### lotus filplus check-client-datacap + ``` NAME: lotus filplus check-client-datacap - check verified client remaining bytes @@ -741,6 +788,7 @@ OPTIONS: ``` ### lotus filplus check-notary-datacap + ``` NAME: lotus filplus check-notary-datacap - check a notary's remaining bytes @@ -753,6 +801,7 @@ OPTIONS: ``` ### lotus filplus sign-remove-data-cap-proposal + ``` NAME: lotus filplus sign-remove-data-cap-proposal - allows a notary to sign a Remove Data Cap Proposal @@ -766,6 +815,7 @@ OPTIONS: ``` ### lotus filplus list-allocations + ``` NAME: lotus filplus list-allocations - List allocations available in verified registry actor or made by a client if specified @@ -780,6 +830,7 @@ OPTIONS: ``` ### lotus filplus list-claims + ``` NAME: lotus filplus list-claims - List claims available in verified registry actor or made by provider if specified @@ -794,6 +845,7 @@ OPTIONS: ``` ### lotus filplus remove-expired-allocations + ``` NAME: lotus filplus remove-expired-allocations - remove expired allocations (if no allocations are specified all eligible allocations are removed) @@ -807,6 +859,7 @@ OPTIONS: ``` ### lotus filplus remove-expired-claims + ``` NAME: lotus filplus remove-expired-claims - remove expired claims (if no claims are specified all eligible claims are removed) @@ -820,6 +873,7 @@ OPTIONS: ``` ### lotus filplus extend-claim + ``` NAME: lotus filplus extend-claim - extends claim expiration (TermMax) @@ -842,6 +896,7 @@ OPTIONS: ``` ## lotus paych + ``` NAME: lotus paych - Manage payment channels @@ -864,6 +919,7 @@ OPTIONS: ``` ### lotus paych add-funds + ``` NAME: lotus paych add-funds - Add funds to the payment channel between fromAddress and toAddress. Creates the payment channel if it doesn't already exist. @@ -877,6 +933,7 @@ OPTIONS: ``` ### lotus paych list + ``` NAME: lotus paych list - List all locally registered payment channels @@ -889,6 +946,7 @@ OPTIONS: ``` ### lotus paych voucher + ``` NAME: lotus paych voucher - Interact with payment channel vouchers @@ -910,6 +968,7 @@ OPTIONS: ``` #### lotus paych voucher create + ``` NAME: lotus paych voucher create - Create a signed payment channel voucher @@ -923,6 +982,7 @@ OPTIONS: ``` #### lotus paych voucher check + ``` NAME: lotus paych voucher check - Check validity of payment channel voucher @@ -935,6 +995,7 @@ OPTIONS: ``` #### lotus paych voucher add + ``` NAME: lotus paych voucher add - Add payment channel voucher to local datastore @@ -947,6 +1008,7 @@ OPTIONS: ``` #### lotus paych voucher list + ``` NAME: lotus paych voucher list - List stored vouchers for a given payment channel @@ -960,6 +1022,7 @@ OPTIONS: ``` #### lotus paych voucher best-spendable + ``` NAME: lotus paych voucher best-spendable - Print vouchers with highest value that is currently spendable for each lane @@ -973,6 +1036,7 @@ OPTIONS: ``` #### lotus paych voucher submit + ``` NAME: lotus paych voucher submit - Submit voucher to chain to update payment channel state @@ -985,6 +1049,7 @@ OPTIONS: ``` ### lotus paych settle + ``` NAME: lotus paych settle - Settle a payment channel @@ -997,6 +1062,7 @@ OPTIONS: ``` ### lotus paych status + ``` NAME: lotus paych status - Show the status of an outbound payment channel @@ -1009,6 +1075,7 @@ OPTIONS: ``` ### lotus paych status-by-from-to + ``` NAME: lotus paych status-by-from-to - Show the status of an active outbound payment channel by from/to addresses @@ -1021,6 +1088,7 @@ OPTIONS: ``` ### lotus paych collect + ``` NAME: lotus paych collect - Collect funds for a payment channel @@ -1033,6 +1101,7 @@ OPTIONS: ``` ## lotus auth + ``` NAME: lotus auth - Manage RPC permissions @@ -1050,6 +1119,7 @@ OPTIONS: ``` ### lotus auth create-token + ``` NAME: lotus auth create-token - Create token @@ -1063,6 +1133,7 @@ OPTIONS: ``` ### lotus auth api-info + ``` NAME: lotus auth api-info - Get token with API info required to connect to this node @@ -1076,6 +1147,7 @@ OPTIONS: ``` ## lotus mpool + ``` NAME: lotus mpool - Manage message pool @@ -1099,6 +1171,7 @@ OPTIONS: ``` ### lotus mpool pending + ``` NAME: lotus mpool pending - Get pending messages @@ -1115,6 +1188,7 @@ OPTIONS: ``` ### lotus mpool sub + ``` NAME: lotus mpool sub - Subscribe to mpool changes @@ -1127,6 +1201,7 @@ OPTIONS: ``` ### lotus mpool stat + ``` NAME: lotus mpool stat - print mempool stats @@ -1141,6 +1216,7 @@ OPTIONS: ``` ### lotus mpool replace + ``` NAME: lotus mpool replace - replace a message in the mempool @@ -1158,6 +1234,7 @@ OPTIONS: ``` ### lotus mpool find + ``` NAME: lotus mpool find - find a message in the mempool @@ -1173,6 +1250,7 @@ OPTIONS: ``` ### lotus mpool config + ``` NAME: lotus mpool config - get or set current mpool configuration @@ -1185,6 +1263,7 @@ OPTIONS: ``` ### lotus mpool gas-perf + ``` NAME: lotus mpool gas-perf - Check gas performance of messages in mempool @@ -1198,6 +1277,7 @@ OPTIONS: ``` ### lotus mpool manage + ``` NAME: lotus mpool manage @@ -1210,6 +1290,7 @@ OPTIONS: ``` ## lotus state + ``` NAME: lotus state - Interact with and query filecoin chain state @@ -1250,6 +1331,7 @@ OPTIONS: ``` ### lotus state power + ``` NAME: lotus state power - Query network or miner power @@ -1262,6 +1344,7 @@ OPTIONS: ``` ### lotus state sectors + ``` NAME: lotus state sectors - Query the sector set of a miner @@ -1274,6 +1357,7 @@ OPTIONS: ``` ### lotus state active-sectors + ``` NAME: lotus state active-sectors - Query the active sector set of a miner @@ -1286,6 +1370,7 @@ OPTIONS: ``` ### lotus state list-actors + ``` NAME: lotus state list-actors - list all actors in the network @@ -1298,6 +1383,7 @@ OPTIONS: ``` ### lotus state list-miners + ``` NAME: lotus state list-miners - list all miners in the network @@ -1311,6 +1397,7 @@ OPTIONS: ``` ### lotus state circulating-supply + ``` NAME: lotus state circulating-supply - Get the exact current circulating supply of Filecoin @@ -1324,6 +1411,7 @@ OPTIONS: ``` ### lotus state sector + ``` NAME: lotus state sector - Get miner sector info @@ -1336,6 +1424,7 @@ OPTIONS: ``` ### lotus state get-actor + ``` NAME: lotus state get-actor - Print actor information @@ -1348,6 +1437,7 @@ OPTIONS: ``` ### lotus state lookup + ``` NAME: lotus state lookup - Find corresponding ID address @@ -1361,6 +1451,7 @@ OPTIONS: ``` ### lotus state replay + ``` NAME: lotus state replay - Replay a particular message @@ -1375,6 +1466,7 @@ OPTIONS: ``` ### lotus state sector-size + ``` NAME: lotus state sector-size - Look up miners sector size @@ -1387,6 +1479,7 @@ OPTIONS: ``` ### lotus state read-state + ``` NAME: lotus state read-state - View a json representation of an actors state @@ -1399,6 +1492,7 @@ OPTIONS: ``` ### lotus state list-messages + ``` NAME: lotus state list-messages - list messages on chain matching given criteria @@ -1415,6 +1509,7 @@ OPTIONS: ``` ### lotus state compute-state + ``` NAME: lotus state compute-state - Perform state computations @@ -1434,6 +1529,7 @@ OPTIONS: ``` ### lotus state call + ``` NAME: lotus state call - Invoke a method on an actor locally @@ -1450,6 +1546,7 @@ OPTIONS: ``` ### lotus state get-deal + ``` NAME: lotus state get-deal - View on-chain deal info @@ -1462,6 +1559,7 @@ OPTIONS: ``` ### lotus state wait-msg + ``` NAME: lotus state wait-msg - Wait for a message to appear on chain @@ -1475,6 +1573,7 @@ OPTIONS: ``` ### lotus state search-msg + ``` NAME: lotus state search-msg - Search to see whether a message has appeared on chain @@ -1487,6 +1586,7 @@ OPTIONS: ``` ### lotus state miner-info + ``` NAME: lotus state miner-info - Retrieve miner information @@ -1499,6 +1599,7 @@ OPTIONS: ``` ### lotus state market + ``` NAME: lotus state market - Inspect the storage market actor @@ -1516,6 +1617,7 @@ OPTIONS: ``` #### lotus state market balance + ``` NAME: lotus state market balance - Get the market balance (locked and escrowed) for a given account @@ -1528,6 +1630,7 @@ OPTIONS: ``` #### lotus state market proposal-pending + ``` NAME: lotus state market proposal-pending - check if a given proposal CID is pending in the market actor @@ -1540,6 +1643,7 @@ OPTIONS: ``` ### lotus state exec-trace + ``` NAME: lotus state exec-trace - Get the execution trace of a given message @@ -1552,6 +1656,7 @@ OPTIONS: ``` ### lotus state network-version + ``` NAME: lotus state network-version - Returns the network version @@ -1564,6 +1669,7 @@ OPTIONS: ``` ### lotus state miner-proving-deadline + ``` NAME: lotus state miner-proving-deadline - Retrieve information about a given miner's proving deadline @@ -1576,6 +1682,7 @@ OPTIONS: ``` ### lotus state actor-cids + ``` NAME: lotus state actor-cids - Returns the built-in actor bundle manifest ID & system actor cids @@ -1589,6 +1696,7 @@ OPTIONS: ``` ## lotus chain + ``` NAME: lotus chain - Interact with filecoin blockchain @@ -1623,6 +1731,7 @@ OPTIONS: ``` ### lotus chain head + ``` NAME: lotus chain head - Print chain head @@ -1636,6 +1745,7 @@ OPTIONS: ``` ### lotus chain get-block + ``` NAME: lotus chain get-block - Get a block and print its details @@ -1649,6 +1759,7 @@ OPTIONS: ``` ### lotus chain read-obj + ``` NAME: lotus chain read-obj - Read the raw bytes of an object @@ -1661,6 +1772,7 @@ OPTIONS: ``` ### lotus chain delete-obj + ``` NAME: lotus chain delete-obj - Delete an object from the chain blockstore @@ -1677,6 +1789,7 @@ OPTIONS: ``` ### lotus chain stat-obj + ``` NAME: lotus chain stat-obj - Collect size and ipld link counts for objs @@ -1697,6 +1810,7 @@ OPTIONS: ``` ### lotus chain getmessage + ``` NAME: lotus chain getmessage - Get and print a message by its cid @@ -1709,6 +1823,7 @@ OPTIONS: ``` ### lotus chain sethead + ``` NAME: lotus chain sethead - manually set the local nodes head tipset (Caution: normally only used for recovery) @@ -1723,6 +1838,7 @@ OPTIONS: ``` ### lotus chain list + ``` NAME: lotus chain list - View a segment of the chain @@ -1740,6 +1856,7 @@ OPTIONS: ``` ### lotus chain get + ``` NAME: lotus chain get - Get chain DAG node by path @@ -1786,6 +1903,7 @@ OPTIONS: ``` ### lotus chain bisect + ``` NAME: lotus chain bisect - bisect chain for an event @@ -1813,6 +1931,7 @@ OPTIONS: ``` ### lotus chain export + ``` NAME: lotus chain export - export chain to a car file @@ -1828,6 +1947,7 @@ OPTIONS: ``` ### lotus chain export-range + ``` NAME: lotus chain export-range - export chain to a car file @@ -1847,6 +1967,7 @@ OPTIONS: ``` ### lotus chain slash-consensus + ``` NAME: lotus chain slash-consensus - Report consensus fault @@ -1861,6 +1982,7 @@ OPTIONS: ``` ### lotus chain gas-price + ``` NAME: lotus chain gas-price - Estimate gas prices @@ -1873,6 +1995,7 @@ OPTIONS: ``` ### lotus chain inspect-usage + ``` NAME: lotus chain inspect-usage - Inspect block space usage of a given tipset @@ -1888,6 +2011,7 @@ OPTIONS: ``` ### lotus chain decode + ``` NAME: lotus chain decode - decode various types @@ -1904,6 +2028,7 @@ OPTIONS: ``` #### lotus chain decode params + ``` NAME: lotus chain decode params - Decode message params @@ -1918,6 +2043,7 @@ OPTIONS: ``` ### lotus chain encode + ``` NAME: lotus chain encode - encode various types @@ -1934,6 +2060,7 @@ OPTIONS: ``` #### lotus chain encode params + ``` NAME: lotus chain encode params - Encodes the given JSON params @@ -1949,6 +2076,7 @@ OPTIONS: ``` ### lotus chain disputer + ``` NAME: lotus chain disputer - interact with the window post disputer @@ -1968,6 +2096,7 @@ OPTIONS: ``` #### lotus chain disputer start + ``` NAME: lotus chain disputer start - Start the window post disputer @@ -1981,6 +2110,7 @@ OPTIONS: ``` #### lotus chain disputer dispute + ``` NAME: lotus chain disputer dispute - Send a specific DisputeWindowedPoSt message @@ -1993,6 +2123,7 @@ OPTIONS: ``` ### lotus chain prune + ``` NAME: lotus chain prune - splitstore gc @@ -2011,6 +2142,7 @@ OPTIONS: ``` #### lotus chain prune compact-cold + ``` NAME: lotus chain prune compact-cold - force splitstore compaction on cold store state and run gc @@ -2026,6 +2158,7 @@ OPTIONS: ``` #### lotus chain prune hot + ``` NAME: lotus chain prune hot - run online (badger vlog) garbage collection on hotstore @@ -2040,6 +2173,7 @@ OPTIONS: ``` #### lotus chain prune hot-moving + ``` NAME: lotus chain prune hot-moving - run moving gc on hotstore @@ -2052,6 +2186,7 @@ OPTIONS: ``` ## lotus log + ``` NAME: lotus log - Manage logging @@ -2070,6 +2205,7 @@ OPTIONS: ``` ### lotus log list + ``` NAME: lotus log list - List log systems @@ -2082,6 +2218,7 @@ OPTIONS: ``` ### lotus log set-level + ``` NAME: lotus log set-level - Set log level @@ -2115,6 +2252,7 @@ OPTIONS: ``` ### lotus log alerts + ``` NAME: lotus log alerts - Get alert states @@ -2128,6 +2266,7 @@ OPTIONS: ``` ## lotus wait-api + ``` NAME: lotus wait-api - Wait for lotus api to come online @@ -2144,6 +2283,7 @@ OPTIONS: ``` ## lotus fetch-params + ``` NAME: lotus fetch-params - Fetch proving parameters @@ -2159,6 +2299,7 @@ OPTIONS: ``` ## lotus evm + ``` NAME: lotus evm - Commands related to the Filecoin EVM runtime @@ -2180,6 +2321,7 @@ OPTIONS: ``` ### lotus evm deploy + ``` NAME: lotus evm deploy - Deploy an EVM smart contract and return its address @@ -2194,6 +2336,7 @@ OPTIONS: ``` ### lotus evm invoke + ``` NAME: lotus evm invoke - Invoke an EVM smart contract using the specified CALLDATA @@ -2208,6 +2351,7 @@ OPTIONS: ``` ### lotus evm stat + ``` NAME: lotus evm stat - Print eth/filecoin addrs and code cid @@ -2220,6 +2364,7 @@ OPTIONS: ``` ### lotus evm call + ``` NAME: lotus evm call - Simulate an eth contract call @@ -2232,6 +2377,7 @@ OPTIONS: ``` ### lotus evm contract-address + ``` NAME: lotus evm contract-address - Generate contract address from smart contract code @@ -2244,6 +2390,7 @@ OPTIONS: ``` ### lotus evm bytecode + ``` NAME: lotus evm bytecode - Write the bytecode of a smart contract to a file @@ -2257,6 +2404,7 @@ OPTIONS: ``` ## lotus index + ``` NAME: lotus index - Commands related to managing the chainindex @@ -2273,6 +2421,7 @@ OPTIONS: ``` ### lotus index validate-backfill + ``` NAME: lotus index validate-backfill - Validates and optionally backfills the chainindex for a range of epochs @@ -2325,6 +2474,7 @@ OPTIONS: ``` ## lotus net + ``` NAME: lotus net - Manage P2P Network @@ -2356,6 +2506,7 @@ OPTIONS: ``` ### lotus net peers + ``` NAME: lotus net peers - Print peers @@ -2370,6 +2521,7 @@ OPTIONS: ``` ### lotus net ping + ``` NAME: lotus net ping - Ping peers @@ -2384,6 +2536,7 @@ OPTIONS: ``` ### lotus net connect + ``` NAME: lotus net connect - Connect to a peer @@ -2396,6 +2549,7 @@ OPTIONS: ``` ### lotus net disconnect + ``` NAME: lotus net disconnect - Disconnect from a peer @@ -2408,6 +2562,7 @@ OPTIONS: ``` ### lotus net listen + ``` NAME: lotus net listen - List listen addresses @@ -2420,6 +2575,7 @@ OPTIONS: ``` ### lotus net id + ``` NAME: lotus net id - Get node identity @@ -2432,6 +2588,7 @@ OPTIONS: ``` ### lotus net find-peer + ``` NAME: lotus net find-peer - Find the addresses of a given peerID @@ -2444,6 +2601,7 @@ OPTIONS: ``` ### lotus net scores + ``` NAME: lotus net scores - Print peers' pubsub scores @@ -2457,6 +2615,7 @@ OPTIONS: ``` ### lotus net reachability + ``` NAME: lotus net reachability - Print information about reachability from the internet @@ -2469,6 +2628,7 @@ OPTIONS: ``` ### lotus net bandwidth + ``` NAME: lotus net bandwidth - Print bandwidth usage information @@ -2483,6 +2643,7 @@ OPTIONS: ``` ### lotus net block + ``` NAME: lotus net block - Manage network connection gating rules @@ -2501,6 +2662,7 @@ OPTIONS: ``` #### lotus net block add + ``` NAME: lotus net block add - Add connection gating rules @@ -2519,6 +2681,7 @@ OPTIONS: ``` ##### lotus net block add peer + ``` NAME: lotus net block add peer - Block a peer @@ -2531,6 +2694,7 @@ OPTIONS: ``` ##### lotus net block add ip + ``` NAME: lotus net block add ip - Block an IP address @@ -2543,6 +2707,7 @@ OPTIONS: ``` ##### lotus net block add subnet + ``` NAME: lotus net block add subnet - Block an IP subnet @@ -2555,6 +2720,7 @@ OPTIONS: ``` #### lotus net block remove + ``` NAME: lotus net block remove - Remove connection gating rules @@ -2573,6 +2739,7 @@ OPTIONS: ``` ##### lotus net block remove peer + ``` NAME: lotus net block remove peer - Unblock a peer @@ -2585,6 +2752,7 @@ OPTIONS: ``` ##### lotus net block remove ip + ``` NAME: lotus net block remove ip - Unblock an IP address @@ -2597,6 +2765,7 @@ OPTIONS: ``` ##### lotus net block remove subnet + ``` NAME: lotus net block remove subnet - Unblock an IP subnet @@ -2609,6 +2778,7 @@ OPTIONS: ``` #### lotus net block list + ``` NAME: lotus net block list - list connection gating rules @@ -2621,6 +2791,7 @@ OPTIONS: ``` ### lotus net stat + ``` NAME: lotus net stat - Report resource usage for a scope @@ -2646,6 +2817,7 @@ OPTIONS: ``` ### lotus net limit + ``` NAME: lotus net limit - Get or set resource limits for a scope @@ -2672,6 +2844,7 @@ OPTIONS: ``` ### lotus net protect + ``` NAME: lotus net protect - Add one or more peer IDs to the list of protected peer connections @@ -2684,6 +2857,7 @@ OPTIONS: ``` ### lotus net unprotect + ``` NAME: lotus net unprotect - Remove one or more peer IDs from the list of protected peer connections. @@ -2696,6 +2870,7 @@ OPTIONS: ``` ### lotus net list-protected + ``` NAME: lotus net list-protected - List the peer IDs with protected connection. @@ -2708,6 +2883,7 @@ OPTIONS: ``` ## lotus sync + ``` NAME: lotus sync - Inspect or interact with the chain syncer @@ -2729,6 +2905,7 @@ OPTIONS: ``` ### lotus sync status + ``` NAME: lotus sync status - check sync status @@ -2741,6 +2918,7 @@ OPTIONS: ``` ### lotus sync wait + ``` NAME: lotus sync wait - Wait for sync to be complete @@ -2754,6 +2932,7 @@ OPTIONS: ``` ### lotus sync mark-bad + ``` NAME: lotus sync mark-bad - Mark the given block as bad, will prevent syncing to a chain that contains it @@ -2766,6 +2945,7 @@ OPTIONS: ``` ### lotus sync unmark-bad + ``` NAME: lotus sync unmark-bad - Unmark the given block as bad, makes it possible to sync to a chain containing it @@ -2779,6 +2959,7 @@ OPTIONS: ``` ### lotus sync check-bad + ``` NAME: lotus sync check-bad - check if the given block was marked bad, and for what reason @@ -2791,6 +2972,7 @@ OPTIONS: ``` ### lotus sync checkpoint + ``` NAME: lotus sync checkpoint - mark a certain tipset as checkpointed; the node will never fork away from this tipset @@ -2804,6 +2986,7 @@ OPTIONS: ``` ## lotus f3 + ``` NAME: lotus f3 - Manages Filecoin Fast Finality (F3) interactions @@ -2824,6 +3007,7 @@ OPTIONS: ``` ### lotus f3 list-miners + ``` NAME: lotus f3 list-miners - Lists the miners that currently participate in F3 via this node. @@ -2836,6 +3020,7 @@ OPTIONS: ``` ### lotus f3 powertable + ``` NAME: lotus f3 powertable @@ -2853,6 +3038,7 @@ OPTIONS: ``` #### lotus f3 powertable get + ``` NAME: lotus f3 powertable get - Get F3 power table at a specific instance ID or latest instance if none is specified. @@ -2866,6 +3052,7 @@ OPTIONS: ``` #### lotus f3 powertable get-proportion + ``` NAME: lotus f3 powertable get-proportion - Gets the total proportion of power for a list of actors at a given instance. @@ -2880,6 +3067,7 @@ OPTIONS: ``` ### lotus f3 certs + ``` NAME: lotus f3 certs - Manages interactions with F3 finality certificates. @@ -2932,6 +3120,7 @@ OPTIONS: ``` #### lotus f3 certs get + ``` NAME: lotus f3 certs get - Gets an F3 finality certificate to a given instance ID, or the latest certificate if no instance is specified. @@ -2945,6 +3134,7 @@ OPTIONS: ``` #### lotus f3 certs list + ``` NAME: lotus f3 certs list - Lists a range of F3 finality certificates. @@ -2995,6 +3185,7 @@ OPTIONS: ``` ### lotus f3 manifest + ``` NAME: lotus f3 manifest - Gets the current manifest used by F3. @@ -3008,6 +3199,7 @@ OPTIONS: ``` ### lotus f3 status + ``` NAME: lotus f3 status - Checks the F3 status. @@ -3020,6 +3212,7 @@ OPTIONS: ``` ## lotus status + ``` NAME: lotus status - Check node status diff --git a/scripts/docsgen-cli/doc_generator.go b/scripts/docsgen-cli/doc_generator.go new file mode 100644 index 00000000000..3638ac92cda --- /dev/null +++ b/scripts/docsgen-cli/doc_generator.go @@ -0,0 +1,156 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/urfave/cli/v2" +) + +// DocGenerator handles CLI documentation generation +type DocGenerator struct { + app *cli.App + outputDir string + writer io.Writer +} + +// NewDocGenerator creates a new documentation generator +func NewDocGenerator(outputDir string, app *cli.App) *DocGenerator { + return &DocGenerator{ + outputDir: outputDir, + app: app, + } +} + +// Generate generates documentation for the CLI app +func (g *DocGenerator) Generate(name string) error { + file, err := g.createMarkdownFile(name) + if err != nil { + return fmt.Errorf("failed to create markdown file: %w", err) + } + defer func() { + if err := file.Close(); err != nil { + fmt.Printf("failed to close markdown file: %v\n", err) + } + }() + + return g.generateContent(file, name) +} + +// createMarkdownFile creates a new markdown file for output. +func (g *DocGenerator) createMarkdownFile(name string) (*os.File, error) { + filePath := filepath.Join(g.outputDir, fmt.Sprintf("cli-%s.md", name)) + return os.Create(filePath) +} + +func (g *DocGenerator) generateContent(file *os.File, name string) error { + bufferedWriter := bufio.NewWriter(file) + g.writer = bufferedWriter + g.app.Writer = bufferedWriter + + if err := g.generateDocs(name); err != nil { + return fmt.Errorf("failed to generate documentation: %w", err) + } + + return bufferedWriter.Flush() +} + +// generateDocs orchestrates the documentation generation process +func (g *DocGenerator) generateDocs(name string) error { + if err := g.writeAppHeader(); err != nil { + return fmt.Errorf("failed to write app header: %w", err) + } + + return g.writeCommandDocs(g.app.Commands, name, 0) +} + +// writeAppHeader writes the application header documentation +func (g *DocGenerator) writeAppHeader() error { + if _, err := g.writer.Write([]byte(fmt.Sprintf("# %s\n\n```\n", g.app.Name))); err != nil { + return err + } + + if err := g.app.Run(getHelpArgs("", "")); err != nil { + return fmt.Errorf("failed to write command docs: %w", err) + } + + if _, err := g.writer.Write([]byte("```\n")); err != nil { + return err + } + + return nil +} + +func (g *DocGenerator) writeCommandDocs(commands cli.Commands, rootName string, depth int) error { + uncategorizedCmds, categorizedCmds := separateCommands(commands) + + // Write uncategorized commands first + if err := g.writeCommands(uncategorizedCmds, rootName, depth); err != nil { + return fmt.Errorf("failed to write uncategorized commands: %w", err) + } + + // Write categorized commands next + if err := g.writeCommands(categorizedCmds, rootName, depth); err != nil { + return fmt.Errorf("failed to write categorized commands: %w", err) + } + + return nil +} + +// separateCommands separates commands into uncategorized and categorized +func separateCommands(commands []*cli.Command) (uncategorized cli.Commands, categorized cli.Commands) { + for _, cmd := range commands { + if cmd.Category == "" { + uncategorized = append(uncategorized, cmd) + } else { + categorized = append(categorized, cmd) + } + } + + return uncategorized, categorized +} + +// writeCommands writes documentation for all commands recursively +func (g *DocGenerator) writeCommands(commands cli.Commands, rootName string, depth int) error { + for _, cmd := range commands { + if cmd.Name == "help" || cmd.Hidden { + continue + } + + cmdName := fmt.Sprintf("%s %s", rootName, cmd.Name) + + if _, err := g.writer.Write([]byte(fmt.Sprintf("\n%s %s\n\n```\n", strings.Repeat("#", depth+2), cmdName))); err != nil { + return err + } + + if err := g.app.Run(getHelpArgs(rootName, cmd.Name)); err != nil { + return fmt.Errorf("failed to write command docs: %w", err) + } + + if _, err := g.writer.Write([]byte("```\n")); err != nil { + return err + } + + if len(cmd.Subcommands) > 0 { + if err := g.writeCommands(cmd.Subcommands, rootName+" "+cmd.Name, depth+1); err != nil { + return err + } + } + } + return nil +} + +func getHelpArgs(rootName string, cmdName string) []string { + if rootName == "" && cmdName == "" { + return []string{"-h"} + } + + args := strings.Split(rootName, " ") + args = append(args, cmdName) + args = append(args, "-h") + return args +} diff --git a/scripts/docsgen-cli/main.go b/scripts/docsgen-cli/main.go index 95d3c5170d3..58478fc3e84 100644 --- a/scripts/docsgen-cli/main.go +++ b/scripts/docsgen-cli/main.go @@ -3,11 +3,12 @@ package main import ( "fmt" "os" - "os/exec" - "path/filepath" - "strings" - "golang.org/x/sync/errgroup" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/lotus/cli/lotus" + "github.com/filecoin-project/lotus/cli/miner" + "github.com/filecoin-project/lotus/cli/worker" ) const ( @@ -45,89 +46,25 @@ func main() { } fmt.Println("Generating CLI documentation...") - var eg errgroup.Group - for _, cmd := range []string{"lotus", "lotus-miner", "lotus-worker"} { - eg.Go(func() error { - err := generateMarkdownForCLI(cmd) - if err != nil { - fmt.Printf(" ❌ %s: %v\n", cmd, err) - } else { - fmt.Printf(" ✅ %s\n", cmd) - } - return err - }) - } - if err := eg.Wait(); err != nil { - fmt.Printf("Failed to generate CLI documentation: %v\n", err) - os.Exit(1) - } - fmt.Println("Documentation generation complete.") -} - -func generateMarkdownForCLI(cli string) error { - md := filepath.Join(outputDir, fmt.Sprintf("cli-%s.md", cli)) - out, err := os.Create(md) - if err != nil { - return err - } - defer func() { _ = out.Close() }() - return writeCommandDocs(out, cli, 0) -} - -func writeCommandDocs(file *os.File, command string, depth int) error { - // For sanity, fail fast if depth exceeds some arbitrarily large number. In which - // case, chances are there is a bug in this script. - if depth > depthRecursionLimit { - return fmt.Errorf("recursion exceeded limit of %d", depthRecursionLimit) - } - // Get usage from the command. - usage, err := exec.Command("sh", "-c", "./"+command+" -h").Output() - if err != nil { - return fmt.Errorf("failed to run '%s': %v", command, err) + cliApps := map[string]*cli.App{ + "lotus": lotus.App(), + "lotus-worker": worker.App(), + "lotus-miner": miner.App(), } - // Skip the first new line since the docs do not start with a newline at the very - // top. - if depth != 0 { - if _, err := file.WriteString("\n"); err != nil { - return err + for name, app := range cliApps { + for _, cmd := range app.Commands { + cmd.HelpName = fmt.Sprintf("%s %s", app.HelpName, cmd.Name) } - } - // Write out command header and usage. - header := fmt.Sprintf("%s# %s\n", strings.Repeat("#", depth), command) - if _, err := file.WriteString(header); err != nil { - return err - } else if _, err := file.WriteString("```\n"); err != nil { - return err - } else if _, err := file.Write(usage); err != nil { - return err - } else if _, err := file.WriteString("```\n"); err != nil { - return err - } - - // Recurse sub-commands. - commands := false - lines := strings.Split(string(usage), "\n") - for _, line := range lines { - switch line = strings.TrimSpace(line); { - case line == "": - commands = false - case line == "COMMANDS:": - commands = true - case strings.HasPrefix(line, "help, h"): - // Skip usage command. - case commands: - // Find the sub command and trim any potential comma in case of alias. - subCommand := strings.TrimSuffix(strings.Fields(line)[0], ",") - // Skip sections in usage that have no command. - if !strings.Contains(subCommand, ":") { - if err := writeCommandDocs(file, command+" "+subCommand, depth+1); err != nil { - return err - } - } + generator := NewDocGenerator(outputDir, app) + if err := generator.Generate(name); err != nil { + fmt.Printf(" ❌ %s: %v\n", name, err) + continue } + fmt.Printf(" ✅ %s\n", name) } - return nil + + fmt.Println("Documentation generation complete.") }