From ebc1c7d64eb345a5f44ed7670e642a4b6e33c2a0 Mon Sep 17 00:00:00 2001 From: Michael Kelly Date: Thu, 14 Nov 2024 16:46:11 -0800 Subject: [PATCH 1/4] Factor our rpmtree handlers --- cmd/rpmtree.go | 125 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 90 insertions(+), 35 deletions(-) diff --git a/cmd/rpmtree.go b/cmd/rpmtree.go index c34ffe0..da4f581 100644 --- a/cmd/rpmtree.go +++ b/cmd/rpmtree.go @@ -5,6 +5,7 @@ import ( "github.com/bazelbuild/buildtools/build" "github.com/rmohr/bazeldnf/cmd/template" + "github.com/rmohr/bazeldnf/pkg/api" "github.com/rmohr/bazeldnf/pkg/bazel" "github.com/rmohr/bazeldnf/pkg/reducer" "github.com/rmohr/bazeldnf/pkg/repo" @@ -29,6 +30,77 @@ type rpmtreeOpts struct { var rpmtreeopts = rpmtreeOpts{} +type Handler interface { + AddRPMs(pkgs []*api.Package, arch string) error + PruneRPMs(buildfile *build.File) + Write() error +} + +type MacroHandler struct { + bzl, defName string + bzlfile *build.File +} + +func NewMacroHandler(toMacro string) (Handler, error) { + bzl, defName, err := bazel.ParseMacro(rpmtreeopts.toMacro) + + if err != nil { + return nil, err + } + + bzlfile, err := bazel.LoadBzl(bzl) + if err != nil { + return nil, err + } + + return &MacroHandler{ + bzl: bzl, + bzlfile: bzlfile, + defName: defName, + }, nil +} + +func (h *MacroHandler) AddRPMs(pkgs []*api.Package, arch string) error { + return bazel.AddBzlfileRPMs(h.bzlfile, h.defName, pkgs, arch) +} + +func (h *MacroHandler) PruneRPMs(buildfile *build.File) { + bazel.PruneBzlfileRPMs(buildfile, h.bzlfile, h.defName) +} + +func (h *MacroHandler) Write() error { + return bazel.WriteBzl(false, h.bzlfile, h.bzl) +} + +type WorkspaceHandler struct { + workspace string + workspacefile *build.File +} + +func NewWorkspaceHandler(workspace string) (Handler, error) { + workspacefile, err := bazel.LoadWorkspace(workspace) + if err != nil { + return nil, err + } + + return &WorkspaceHandler{ + workspace: workspace, + workspacefile: workspacefile, + }, nil +} + +func (h *WorkspaceHandler) AddRPMs(pkgs []*api.Package, arch string) error { + return bazel.AddWorkspaceRPMs(h.workspacefile, pkgs, arch) +} + +func (h *WorkspaceHandler) PruneRPMs(buildfile *build.File) { + bazel.PruneWorkspaceRPMs(buildfile, h.workspacefile) +} + +func (h *WorkspaceHandler) Write() error { + return bazel.WriteWorkspace(false, h.workspacefile, h.workspace) +} + func NewRpmTreeCmd() *cobra.Command { rpmtreeCmd := &cobra.Command{ @@ -68,54 +140,37 @@ func NewRpmTreeCmd() *cobra.Command { if err != nil { return err } - workspace, err := bazel.LoadWorkspace(rpmtreeopts.workspace) + + var handler Handler + if writeToMacro { + handler, err = NewMacroHandler(rpmtreeopts.toMacro) + } else { + handler, err = NewWorkspaceHandler(rpmtreeopts.workspace) + } + if err != nil { return err } - var bzlfile *build.File - var bzl, defName string - if writeToMacro { - bzl, defName, err = bazel.ParseMacro(rpmtreeopts.toMacro) - if err != nil { - return err - } - bzlfile, err = bazel.LoadBzl(bzl) - if err != nil { - return err - } - } + build, err := bazel.LoadBuild(rpmtreeopts.buildfile) if err != nil { return err } - if writeToMacro { - err = bazel.AddBzlfileRPMs(bzlfile, defName, install, rpmtreeopts.arch) - if err != nil { - return err - } - } else { - err = bazel.AddWorkspaceRPMs(workspace, install, rpmtreeopts.arch) - if err != nil { - return err - } + + err = handler.AddRPMs(install, rpmtreeopts.arch) + if err != nil { + return err } + bazel.AddTree(rpmtreeopts.name, build, install, rpmtreeopts.arch, rpmtreeopts.public) - if writeToMacro { - bazel.PruneBzlfileRPMs(build, bzlfile, defName) - } else { - bazel.PruneWorkspaceRPMs(build, workspace) - } + + handler.PruneRPMs(build) logrus.Info("Writing bazel files.") - err = bazel.WriteWorkspace(false, workspace, rpmtreeopts.workspace) + err = handler.Write() if err != nil { return err } - if writeToMacro { - err = bazel.WriteBzl(false, bzlfile, bzl) - if err != nil { - return err - } - } + err = bazel.WriteBuild(false, build, rpmtreeopts.buildfile) if err != nil { return err From 3955774f6e75bbf3f582a742c6afcf2b58d73a8e Mon Sep 17 00:00:00 2001 From: Michael Kelly Date: Thu, 14 Nov 2024 17:33:03 -0800 Subject: [PATCH 2/4] Add support for generating lockfile with rpmtree This change introduces generation of the lockfile from the rpmtree command. Following the previous patterns with the other bazel files, this adds some helper methods to bazel.go and creates a handler object to wrap this up for rpmtree. Worth noting is that to make this work we also have to tweak AddTree() to take the configuration name because the generated repos are different when using the lockfile. This change leaves the MODULE.bazel configuration up to the user as there may be some additional configuration for bazeldnf that is beyond the scope of what rpmtree is doing and we don't want to interfere with that. --- cmd/rpmtree.go | 52 +++++++++++++++++++++++++---- pkg/api/bazeldnf/BUILD.bazel | 5 ++- pkg/api/bazeldnf/config.go | 12 +++++++ pkg/bazel/BUILD.bazel | 1 + pkg/bazel/bazel.go | 63 ++++++++++++++++++++++++++++++------ pkg/bazel/bazel_test.go | 2 +- 6 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 pkg/api/bazeldnf/config.go diff --git a/cmd/rpmtree.go b/cmd/rpmtree.go index da4f581..40583cd 100644 --- a/cmd/rpmtree.go +++ b/cmd/rpmtree.go @@ -1,11 +1,13 @@ package main import ( + "errors" "os" "github.com/bazelbuild/buildtools/build" "github.com/rmohr/bazeldnf/cmd/template" "github.com/rmohr/bazeldnf/pkg/api" + "github.com/rmohr/bazeldnf/pkg/api/bazeldnf" "github.com/rmohr/bazeldnf/pkg/bazel" "github.com/rmohr/bazeldnf/pkg/reducer" "github.com/rmohr/bazeldnf/pkg/repo" @@ -23,6 +25,8 @@ type rpmtreeOpts struct { workspace string toMacro string buildfile string + configname string + lockfile string name string public bool forceIgnoreRegex []string @@ -84,7 +88,7 @@ func NewWorkspaceHandler(workspace string) (Handler, error) { } return &WorkspaceHandler{ - workspace: workspace, + workspace: workspace, workspacefile: workspacefile, }, nil } @@ -101,6 +105,31 @@ func (h *WorkspaceHandler) Write() error { return bazel.WriteWorkspace(false, h.workspacefile, h.workspace) } +type LockFileHandler struct { + filename string + config *bazeldnf.Config +} + +func NewLockFileHandler(configname, filename string) (Handler, error) { + return &LockFileHandler{ + filename: filename, + config: &bazeldnf.Config{ + Name: configname, + RPMs: []bazeldnf.RPM{}, + }, + }, nil +} + +func (h *LockFileHandler) AddRPMs(pkgs []*api.Package, arch string) error { + return bazel.AddConfigRPMs(h.config, pkgs, arch) +} + +func (h *LockFileHandler) PruneRPMs(buildfile *build.File) {} + +func (h *LockFileHandler) Write() error { + return bazel.WriteLockFile(h.config, h.filename) +} + func NewRpmTreeCmd() *cobra.Command { rpmtreeCmd := &cobra.Command{ @@ -108,7 +137,9 @@ func NewRpmTreeCmd() *cobra.Command { Short: "Writes a rpmtree rule and its rpmdependencies to bazel files", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, required []string) error { - writeToMacro := rpmtreeopts.toMacro != "" + if rpmtreeopts.toMacro != "" && rpmtreeopts.lockfile != "" { + return errors.New("Must provide at most one of --lockfile --to-macro") + } repos, err := repo.LoadRepoFiles(rpmtreeopts.repofiles) if err != nil { @@ -142,16 +173,21 @@ func NewRpmTreeCmd() *cobra.Command { } var handler Handler - if writeToMacro { + if rpmtreeopts.toMacro != "" { handler, err = NewMacroHandler(rpmtreeopts.toMacro) + } else if rpmtreeopts.lockfile != "" { + handler, err = NewLockFileHandler( + rpmtreeopts.configname, + rpmtreeopts.lockfile, + ) } else { handler, err = NewWorkspaceHandler(rpmtreeopts.workspace) } - + if err != nil { return err } - + build, err := bazel.LoadBuild(rpmtreeopts.buildfile) if err != nil { return err @@ -162,8 +198,8 @@ func NewRpmTreeCmd() *cobra.Command { return err } - bazel.AddTree(rpmtreeopts.name, build, install, rpmtreeopts.arch, rpmtreeopts.public) - + bazel.AddTree(rpmtreeopts.name, rpmtreeopts.configname, build, install, rpmtreeopts.arch, rpmtreeopts.public) + handler.PruneRPMs(build) logrus.Info("Writing bazel files.") err = handler.Write() @@ -191,6 +227,8 @@ func NewRpmTreeCmd() *cobra.Command { rpmtreeCmd.Flags().StringVarP(&rpmtreeopts.workspace, "workspace", "w", "WORKSPACE", "Bazel workspace file") rpmtreeCmd.Flags().StringVarP(&rpmtreeopts.toMacro, "to-macro", "", "", "Tells bazeldnf to write the RPMs to a macro in the given bzl file instead of the WORKSPACE file. The expected format is: macroFile%defName") rpmtreeCmd.Flags().StringVarP(&rpmtreeopts.buildfile, "buildfile", "b", "rpm/BUILD.bazel", "Build file for RPMs") + rpmtreeCmd.Flags().StringVar(&rpmtreeopts.configname, "configname", "rpms", "config name to use in lockfile") + rpmtreeCmd.Flags().StringVar(&rpmtreeopts.lockfile, "lockfile", "", "lockfile for RPMs") rpmtreeCmd.Flags().StringVar(&rpmtreeopts.name, "name", "", "rpmtree rule name") rpmtreeCmd.Flags().StringArrayVar(&rpmtreeopts.forceIgnoreRegex, "force-ignore-with-dependencies", []string{}, "Packages matching these regex patterns will not be installed. Allows force-removing unwanted dependencies. Be careful, this can lead to hidden missing dependencies.") rpmtreeCmd.MarkFlagRequired("name") diff --git a/pkg/api/bazeldnf/BUILD.bazel b/pkg/api/bazeldnf/BUILD.bazel index 3176fe7..9b44a1f 100644 --- a/pkg/api/bazeldnf/BUILD.bazel +++ b/pkg/api/bazeldnf/BUILD.bazel @@ -2,7 +2,10 @@ load("@rules_go//go:def.bzl", "go_library") go_library( name = "bazeldnf", - srcs = ["repo.go"], + srcs = [ + "config.go", + "repo.go", + ], importpath = "github.com/rmohr/bazeldnf/pkg/api/bazeldnf", visibility = ["//visibility:public"], ) diff --git a/pkg/api/bazeldnf/config.go b/pkg/api/bazeldnf/config.go new file mode 100644 index 0000000..3372613 --- /dev/null +++ b/pkg/api/bazeldnf/config.go @@ -0,0 +1,12 @@ +package bazeldnf + +type RPM struct { + Name string `json:"name"` + SHA256 string `json:"sha256"` + URLs []string `json:"urls"` +} + +type Config struct { + Name string `json:"name"` + RPMs []RPM `json:"rpms"` +} diff --git a/pkg/bazel/BUILD.bazel b/pkg/bazel/BUILD.bazel index be333c8..c43e65a 100644 --- a/pkg/bazel/BUILD.bazel +++ b/pkg/bazel/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/api", + "//pkg/api/bazeldnf", "@com_github_bazelbuild_buildtools//build:go_default_library", "@com_github_bazelbuild_buildtools//edit:go_default_library", ], diff --git a/pkg/bazel/bazel.go b/pkg/bazel/bazel.go index 64580e8..157c9a7 100644 --- a/pkg/bazel/bazel.go +++ b/pkg/bazel/bazel.go @@ -1,9 +1,10 @@ package bazel import ( + "encoding/json" "fmt" - "io/ioutil" "net/url" + "os" "path/filepath" "sort" "strings" @@ -11,6 +12,7 @@ import ( "github.com/bazelbuild/buildtools/build" "github.com/bazelbuild/buildtools/edit" "github.com/rmohr/bazeldnf/pkg/api" + "github.com/rmohr/bazeldnf/pkg/api/bazeldnf" ) type Artifact struct { @@ -18,7 +20,7 @@ type Artifact struct { } func LoadWorkspace(path string) (*build.File, error) { - workspaceData, err := ioutil.ReadFile(path) + workspaceData, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("failed to parse WORSPACE orig: %v", err) } @@ -30,7 +32,7 @@ func LoadWorkspace(path string) (*build.File, error) { } func LoadBuild(path string) (*build.File, error) { - buildfileData, err := ioutil.ReadFile(path) + buildfileData, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("failed to parse BUILD.bazel orig: %v", err) } @@ -42,7 +44,7 @@ func LoadBuild(path string) (*build.File, error) { } func LoadBzl(path string) (*build.File, error) { - bzlData, err := ioutil.ReadFile(path) + bzlData, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("failed to parse bzl orig: %v", err) } @@ -58,7 +60,7 @@ func WriteBuild(dryRun bool, buildfile *build.File, path string) error { fmt.Println(build.FormatString(buildfile)) return nil } - return ioutil.WriteFile(path, build.Format(buildfile), 0666) + return os.WriteFile(path, build.Format(buildfile), 0666) } func WriteWorkspace(dryRun bool, workspace *build.File, path string) error { @@ -66,7 +68,7 @@ func WriteWorkspace(dryRun bool, workspace *build.File, path string) error { fmt.Println(build.FormatString(workspace)) return nil } - return ioutil.WriteFile(path, build.Format(workspace), 0666) + return os.WriteFile(path, build.Format(workspace), 0666) } func WriteBzl(dryRun bool, bzl *build.File, path string) error { @@ -74,7 +76,15 @@ func WriteBzl(dryRun bool, bzl *build.File, path string) error { fmt.Println(build.FormatString(bzl)) return nil } - return ioutil.WriteFile(path, build.Format(bzl), 0666) + return os.WriteFile(path, build.Format(bzl), 0666) +} + +func WriteLockFile(config *bazeldnf.Config, path string) error { + configJson, err := json.Marshal(config) + if err != nil { + return err + } + return os.WriteFile(path, configJson, 0666) } // ParseMacro parses a macro expression of the form macroFile%defName and returns the bzl file and the def name. @@ -258,7 +268,16 @@ func AddTar2Files(name string, rpmtree string, buildfile *build.File, files []st } } -func AddTree(name string, buildfile *build.File, pkgs []*api.Package, arch string, public bool) { +func AddTree(name, configname string, buildfile *build.File, pkgs []*api.Package, arch string, public bool) { + transform := func(n string) string { + return "@"+n+"//rpm" + } + if configname != "" { + transform = func(n string) string { + return "@" + configname + "//" + n + } + } + rpmtrees := map[string]*rpmTree{} for _, rule := range buildfile.Rules("rpmtree") { @@ -269,7 +288,7 @@ func AddTree(name string, buildfile *build.File, pkgs []*api.Package, arch strin rpms := []string{} for _, pkg := range pkgs { pkgName := sanitize(pkg.String() + "." + arch) - rpms = append(rpms, "@"+pkgName+"//rpm") + rpms = append(rpms, transform(pkgName)) } sort.SliceStable(rpms, func(i, j int) bool { return rpms[i] < rpms[j] @@ -486,6 +505,32 @@ func (r *tar2Files) SetFiles(dirs []string, fileMap map[string][]string) { r.Rule.SetAttr("files", filesMapExpr) } +func AddConfigRPMs(config *bazeldnf.Config, pkgs []*api.Package, arch string) error { + for _, pkg := range pkgs { + URLs := []string{} + + for _, mirror := range pkg.Repository.Mirrors { + u, err := url.Parse(mirror) + if err != nil { + return err + } + u = u.JoinPath(pkg.Location.Href) + URLs = append(URLs, u.String()) + } + + config.RPMs = append( + config.RPMs, + bazeldnf.RPM{ + Name: sanitize(pkg.String() + "." + arch), + SHA256: pkg.Checksum.Text, + URLs: URLs, + }, + ) + } + + return nil +} + func sanitize(name string) string { name = strings.ReplaceAll(name, ":", "__") name = strings.ReplaceAll(name, "+", "__plus__") diff --git a/pkg/bazel/bazel_test.go b/pkg/bazel/bazel_test.go index edf6a9a..bca03da 100644 --- a/pkg/bazel/bazel_test.go +++ b/pkg/bazel/bazel_test.go @@ -115,7 +115,7 @@ func TestBuildfileWithRPMs(t *testing.T) { defer os.Remove(tmpFile.Name()) file, err := LoadBuild(tt.orig) g.Expect(err).ToNot(HaveOccurred()) - AddTree("mytree", file, tt.pkgs, "myarch", false) + AddTree("mytree", "", file, tt.pkgs, "myarch", false) err = WriteBuild(false, file, tmpFile.Name()) g.Expect(err).ToNot(HaveOccurred()) From 0162120cff9fbdb8935d2911a0cd09392b4075de Mon Sep 17 00:00:00 2001 From: Michael Kelly Date: Tue, 3 Dec 2024 13:48:44 -0800 Subject: [PATCH 3/4] Minor fixes to rpmtree command (1) The error checking at the beginning isn't necessary because we also want to support WORKSPACE mode. (2) If we pass the default configname through to AddTree() then we'll generate the WORKSPACE file as if we had a lock file, which is not what we desire --- cmd/rpmtree.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/rpmtree.go b/cmd/rpmtree.go index 40583cd..ee01cb4 100644 --- a/cmd/rpmtree.go +++ b/cmd/rpmtree.go @@ -1,7 +1,6 @@ package main import ( - "errors" "os" "github.com/bazelbuild/buildtools/build" @@ -137,10 +136,6 @@ func NewRpmTreeCmd() *cobra.Command { Short: "Writes a rpmtree rule and its rpmdependencies to bazel files", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, required []string) error { - if rpmtreeopts.toMacro != "" && rpmtreeopts.lockfile != "" { - return errors.New("Must provide at most one of --lockfile --to-macro") - } - repos, err := repo.LoadRepoFiles(rpmtreeopts.repofiles) if err != nil { return err @@ -173,9 +168,12 @@ func NewRpmTreeCmd() *cobra.Command { } var handler Handler + var configname string + if rpmtreeopts.toMacro != "" { handler, err = NewMacroHandler(rpmtreeopts.toMacro) } else if rpmtreeopts.lockfile != "" { + configname = rpmtreeopts.configname handler, err = NewLockFileHandler( rpmtreeopts.configname, rpmtreeopts.lockfile, @@ -198,7 +196,7 @@ func NewRpmTreeCmd() *cobra.Command { return err } - bazel.AddTree(rpmtreeopts.name, rpmtreeopts.configname, build, install, rpmtreeopts.arch, rpmtreeopts.public) + bazel.AddTree(rpmtreeopts.name, configname, build, install, rpmtreeopts.arch, rpmtreeopts.public) handler.PruneRPMs(build) logrus.Info("Writing bazel files.") From f0105bc5e9df9cf84c34ddd849d7680575ec69c2 Mon Sep 17 00:00:00 2001 From: Michael Kelly Date: Thu, 2 Jan 2025 16:02:32 -0800 Subject: [PATCH 4/4] Add comments on no-op PruneRPMs --- cmd/rpmtree.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/rpmtree.go b/cmd/rpmtree.go index ee01cb4..e647bae 100644 --- a/cmd/rpmtree.go +++ b/cmd/rpmtree.go @@ -123,7 +123,9 @@ func (h *LockFileHandler) AddRPMs(pkgs []*api.Package, arch string) error { return bazel.AddConfigRPMs(h.config, pkgs, arch) } -func (h *LockFileHandler) PruneRPMs(buildfile *build.File) {} +func (h *LockFileHandler) PruneRPMs(buildfile *build.File) { + // we always generate from scratch and have nothing to prune +} func (h *LockFileHandler) Write() error { return bazel.WriteLockFile(h.config, h.filename)