estimate: show packages in NEW, but with a light blue color

It uses an additional query from the DAK Web API to get the packages
currently in NEW, but this one returns only source packages, so we have
to keep into memory the name of the source packages when we get the list
of all golang packages.

Author: n-peugnet

check_depends.go

@@ -91,7 +91,7 @@ func execCheckDepends(args []string) {
 // parseGoModDependencies parse ALL dependencies listed in go.mod
 // i.e. it returns the one defined in go.mod as well as the transitively ones
 // TODO: this may not be the best way of doing thing since it requires the package to be converted to go module
-func parseGoModDependencies(directory string, goBinaries map[string]string) ([]dependency, error) {
+func parseGoModDependencies(directory string, goBinaries map[string]debianPackage) ([]dependency, error) {
 	b, err := os.ReadFile(filepath.Join(directory, "go.mod"))
 	if err != nil {
 		return nil, err
@@ -115,7 +115,7 @@ func parseGoModDependencies(directory string, goBinaries map[string]string) ([]d
 			}
 
 			if val, exists := goBinaries[rr.Root]; exists {
-				packageName = val
+				packageName = val.binary
 			}
 
 			dependencies = append(dependencies, dependency{

check_depends_test.go

@@ -110,10 +110,10 @@ require (
 		t.Fatalf("Could not create dummy Debian package: %v", err)
 	}
 
-	deps, err := parseGoModDependencies(filepath.Join(tmpDir, "dummy-package"), map[string]string{
-		"github.com/charmbracelet/glamour": "golang-github-charmbracelet-glamour-dev",
-		"github.com/google/go-github":      "golang-github-google-go-github-dev",
-		"github.com/gregjones/httpcache":   "golang-github-gregjones-httpcache-dev",
+	deps, err := parseGoModDependencies(filepath.Join(tmpDir, "dummy-package"), map[string]debianPackage{
+		"github.com/charmbracelet/glamour": {binary: "golang-github-charmbracelet-glamour-dev", source: "golang-github-charmbracelet-glamour"},
+		"github.com/google/go-github":      {binary: "golang-github-google-go-github-dev", source: "golang-github-google-go-github"},
+		"github.com/gregjones/httpcache":   {binary: "golang-github-gregjones-httpcache-dev", source: "golang-github-gregjones-httpcache"},
 	})
 	if err != nil {
 		t.Fatalf("Could not parse go.mod dependencies: %v", err)

estimate.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -15,6 +17,10 @@ import (
 	"golang.org/x/tools/go/vcs"
 )
 
+const (
+	sourcesInNewURL = "https://api.ftp-master.debian.org/sources_in_suite/new"
+)
+
 // majorVersionRegexp checks if an import path contains a major version suffix.
 var majorVersionRegexp = regexp.MustCompile(`([/.])v([0-9]+)$`)
 
@@ -25,6 +31,29 @@ var moduleBlocklist = map[string]string{
 	"github.com/Microsoft/go-winio":     "Windows only",
 }
 
+func getSourcesInNew() (map[string]string, error) {
+	sourcesInNew := make(map[string]string)
+
+	resp, err := http.Get(sourcesInNewURL)
+	if err != nil {
+		return nil, fmt.Errorf("getting %q: %w", golangBinariesURL, err)
+	}
+	if got, want := resp.StatusCode, http.StatusOK; got != want {
+		return nil, fmt.Errorf("unexpected HTTP status code: got %d, want %d", got, want)
+	}
+	var pkgs []struct {
+		Source  string `json:"source"`
+		Version string `json:"version"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&pkgs); err != nil {
+		return nil, fmt.Errorf("decode: %w", err)
+	}
+	for _, pkg := range pkgs {
+		sourcesInNew[pkg.Source] = pkg.Version
+	}
+	return sourcesInNew, nil
+}
+
 func get(gopath, repodir, repo, rev string) error {
 	done := make(chan struct{})
 	defer close(done)
@@ -97,14 +126,14 @@ func otherVersions(mod string) (mods []string) {
 // findOtherVersion search in m for potential other versions of the given
 // module and returns the number of the major version found, 0 if not,
 // along with the corresponding package name.
-func findOtherVersion(m map[string]string, mod string) (int, string) {
+func findOtherVersion(m map[string]debianPackage, mod string) (int, debianPackage) {
 	versions := otherVersions(mod)
 	for i, version := range versions {
 		if pkg, ok := m[version]; ok {
 			return len(versions) - i, pkg
 		}
 	}
-	return 0, ""
+	return 0, debianPackage{}
 }
 
 // trackerLink generates an OSC 8 hyperlink to the tracker for the given Debian
@@ -170,7 +199,11 @@ func estimate(importpath, revision string) error {
 	// Retrieve already-packaged ones
 	golangBinaries, err := getGolangBinaries()
 	if err != nil {
-		return nil
+		return fmt.Errorf("get golang debian packages: %w", err)
+	}
+	sourcesInNew, err := getSourcesInNew()
+	if err != nil {
+		return fmt.Errorf("get packages in new: %w", err)
 	}
 
 	// Build a graph in memory from the output of go mod graph
@@ -233,7 +266,11 @@ func estimate(importpath, revision string) error {
 			if mod == "go" || mod == "toolchain" {
 				return
 			}
-			if _, ok := golangBinaries[mod]; ok {
+			if pkg, ok := golangBinaries[mod]; ok {
+				if _, ok := sourcesInNew[pkg.source]; ok {
+					line := fmt.Sprintf("%s\033[36m%s (in NEW)\033[0m", strings.Repeat("  ", indent), mod)
+					lines = append(lines, line)
+				}
 				return // already packaged in Debian
 			}
 			var repoRoot string
@@ -250,9 +287,13 @@ func estimate(importpath, revision string) error {
 				// Log info to indicate that it is an approximate match
 				// but consider that it is packaged and skip the children.
 				if v == 1 {
-					log.Printf("%s has no version string in Debian (%s)", mod, trackerLink(pkg))
+					log.Printf("%s has no version string in Debian (%s)", mod, trackerLink(pkg.source))
 				} else {
-					log.Printf("%s is v%d in Debian (%s)", mod, v, trackerLink(pkg))
+					log.Printf("%s is v%d in Debian (%s)", mod, v, trackerLink(pkg.source))
+				}
+				if _, ok := sourcesInNew[pkg.source]; ok {
+					line := fmt.Sprintf("%s\033[36m%s (in NEW)\033[0m", strings.Repeat("  ", indent), mod)
+					lines = append(lines, line)
 				}
 				return
 			}
@@ -262,7 +303,11 @@ func estimate(importpath, revision string) error {
 			if pkg, ok := golangBinaries[repoRoot]; ok {
 				// Log info to indicate that it is an approximate match
 				// but consider that it is packaged and skip the children.
-				log.Printf("%s is packaged as %s in Debian (%s)", mod, repoRoot, trackerLink(pkg))
+				log.Printf("%s is packaged as %s in Debian (%s)", mod, repoRoot, trackerLink(pkg.source))
+				if _, ok := sourcesInNew[pkg.source]; ok {
+					line := fmt.Sprintf("%s\033[36m%s (in NEW)\033[0m", strings.Repeat("  ", indent), mod)
+					lines = append(lines, line)
+				}
 				return
 			}
 			// Ignore modules from the blocklist.

make.go

@@ -932,7 +932,7 @@ func execMake(args []string, usage func()) {
 
 	var (
 		eg             errgroup.Group
-		golangBinaries map[string]string // map[goImportPath]debianBinaryPackage
+		golangBinaries map[string]debianPackage // map[goImportPath]debianPackage
 	)
 
 	// TODO: also check whether there already is a git repository on salsa.
@@ -965,9 +965,9 @@ func execMake(args []string, usage func()) {
 		log.Printf("Could not check for existing Go packages in Debian: %v", err)
 	}
 
-	if debbin, ok := golangBinaries[gopkg]; ok {
+	if debpkg, ok := golangBinaries[gopkg]; ok {
 		log.Printf("WARNING: A package called %q is already in Debian! See https://tracker.debian.org/pkg/%s\n",
-			debbin, debbin)
+			debpkg.binary, debpkg.source)
 	}
 
 	orig := fmt.Sprintf("%s_%s.orig.tar.%s", debsrc, u.version, u.compression)
@@ -995,12 +995,12 @@ func execMake(args []string, usage func()) {
 			debdependencies = append(debdependencies, debianNameFromGopkg(dep, typeLibrary, "", allowUnknownHoster)+"-dev")
 			continue
 		}
-		bin, ok := golangBinaries[dep]
+		pkg, ok := golangBinaries[dep]
 		if !ok {
 			log.Printf("Build-Dependency %q is not yet available in Debian, or has not yet been converted to use XS-Go-Import-Path in debian/control", dep)
 			continue
 		}
-		debdependencies = append(debdependencies, bin)
+		debdependencies = append(debdependencies, pkg.binary)
 	}
 
 	if err := writeTemplates(dir, gopkg, debsrc, debLib, debProg, debversion,

search.go

@@ -15,8 +15,13 @@ const (
 	golangBinariesURL = "https://api.ftp-master.debian.org/binary/by_metadata/Go-Import-Path"
 )
 
-func getGolangBinaries() (map[string]string, error) {
-	golangBinaries := make(map[string]string)
+type debianPackage struct {
+	binary string
+	source string
+}
+
+func getGolangBinaries() (map[string]debianPackage, error) {
+	golangBinaries := make(map[string]debianPackage)
 
 	resp, err := http.Get(golangBinariesURL)
 	if err != nil {
@@ -39,7 +44,10 @@ func getGolangBinaries() (map[string]string, error) {
 		}
 		for _, importPath := range strings.Split(pkg.XSGoImportPath, ",") {
 			// XS-Go-Import-Path can be comma-separated and contain spaces.
-			golangBinaries[strings.TrimSpace(importPath)] = pkg.Binary
+			golangBinaries[strings.TrimSpace(importPath)] = debianPackage{
+				binary: pkg.Binary,
+				source: pkg.Source,
+			}
 		}
 	}
 	return golangBinaries, nil
@@ -74,9 +82,9 @@ func execSearch(args []string) {
 		log.Fatal(err)
 	}
 
-	for importPath, binary := range golangBinaries {
+	for importPath, pkg := range golangBinaries {
 		if pattern.MatchString(importPath) {
-			fmt.Printf("%s: %s\n", binary, importPath)
+			fmt.Printf("%s: %s\n", pkg.binary, importPath)
 		}
 	}
 }