Include data streams summary in search responses (#1264)

This information is already included in responses to the /package endpoint. Add it to /search responses so it can be also used for discovery purposes, without having to request all packages individually.
elastic · Jan 8, 2025 · 3c06b39 · 3c06b39
1 parent 44a0b9c
commit 3c06b39
Show file tree

Hide file tree

Showing 39 changed files with 28,378 additions and 1,500 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* Include summary of data streams in search responses. [#1264](https://github.com/elastic/package-registry/pull/1264)
+
 ### Deprecated
 
 ### Known Issues

diff --git a/packages/datastream.go b/packages/datastream.go
@@ -43,7 +43,7 @@ type DataStream struct {
 	DatasetIsPrefix bool   `config:"dataset_is_prefix" json:"dataset_is_prefix,omitempty" yaml:"dataset_is_prefix,omitempty"`
 
 	Title   string `config:"title" json:"title" validate:"required"`
-	Release string `config:"release" json:"release"`
+	Release string `config:"release" json:"release,omitempty"`
 
 	// Deprecated: Replaced by elasticsearch.ingest_pipeline.name
 	IngestPipeline string                   `config:"ingest_pipeline,omitempty" json:"ingest_pipeline,omitempty" yaml:"ingest_pipeline,omitempty"`

diff --git a/search.go b/search.go
@@ -61,7 +61,7 @@ func searchHandlerWithProxyMode(logger *zap.Logger, indexer Indexer, proxyMode *
 			}
 		}
 
-		data, err := getPackageOutput(r.Context(), packages)
+		data, err := getSearchOutput(r.Context(), packages)
 		if err != nil {
 			notFoundError(w, err)
 			return
@@ -166,16 +166,16 @@ func getSpecVersion(version string) (*semver.Version, error) {
 	return specVersion, nil
 }
 
-func getPackageOutput(ctx context.Context, packageList packages.Packages) ([]byte, error) {
+func getSearchOutput(ctx context.Context, packageList packages.Packages) ([]byte, error) {
 	span, _ := apm.StartSpan(ctx, "GetPackageOutput", "app")
 	defer span.End()
 
 	// Packages need to be sorted to be always outputted in the same order
 	sort.Sort(packageList)
 
-	var output []packages.BasePackage
+	var output []packageSummary
 	for _, p := range packageList {
-		data := p.BasePackage
+		data := getPackageSummaryOutput(p)
 		output = append(output, data)
 	}
 
@@ -186,3 +186,28 @@ func getPackageOutput(ctx context.Context, packageList packages.Packages) ([]byt
 
 	return util.MarshalJSONPretty(output)
 }
+
+type packageSummary struct {
+	packages.BasePackage `json:",inline"`
+	DataStreams          []*packages.DataStream `json:"data_streams,omitempty"`
+}
+
+func getPackageSummaryOutput(index *packages.Package) packageSummary {
+	summary := packageSummary{
+		BasePackage: index.BasePackage,
+	}
+	if len(index.DataStreams) == 0 {
+		return summary
+	}
+
+	summary.DataStreams = make([]*packages.DataStream, len(index.DataStreams))
+	for i, datastream := range index.DataStreams {
+		summary.DataStreams[i] = &packages.DataStream{
+			Type:    datastream.Type,
+			Dataset: datastream.Dataset,
+			Title:   datastream.Title,
+		}
+	}
+
+	return summary
+}
diff --git a/testdata/generated/search-all-proxy.json b/testdata/generated/search-all-proxy.json
@@ -15,6 +15,13 @@
     },
     "categories": [
       "custom"
+    ],
+    "data_streams": [
+      {
+        "type": "metrics",
+        "dataset": "agent_privileges.agent_privileges",
+        "title": "Agent privileges data stream"
+      }
     ]
   },
   {
@@ -35,6 +42,23 @@
     ],
     "categories": [
       "custom"
+    ],
+    "data_streams": [
+      {
+        "type": "logs",
+        "dataset": "datasources.examplelog1",
+        "title": "Example dataset with inputs"
+      },
+      {
+        "type": "logs",
+        "dataset": "datasources.examplelog2",
+        "title": "Example dataset with inputs"
+      },
+      {
+        "type": "metrics",
+        "dataset": "datasources.examplemetric",
+        "title": "Example data stream with inputs"
+      }
     ]
   },
   {
@@ -53,6 +77,13 @@
     },
     "categories": [
       "custom"
+    ],
+    "data_streams": [
+      {
+        "type": "metrics",
+        "dataset": "elasticsearch_privileges.elasticsearch_privileges",
+        "title": "Elasticsearch privileges data stream"
+      }
     ]
   },
   {
@@ -82,6 +113,13 @@
     "categories": [
       "crm",
       "azure"
+    ],
+    "data_streams": [
+      {
+        "type": "logs",
+        "dataset": "nodirentries.foo",
+        "title": "Foo"
+      }
     ]
   },
   {
@@ -114,6 +152,13 @@
     "categories": [
       "crm",
       "azure"
+    ],
+    "data_streams": [
+      {
+        "type": "logs",
+        "dataset": "example.foo",
+        "title": "Foo"
+      }
     ]
   },
   {
@@ -144,7 +189,14 @@
       "crm",
       "azure"
     ],
-    "signature_path": "/epr/example/example-1.0.1.zip.sig"
+    "signature_path": "/epr/example/example-1.0.1.zip.sig",
+    "data_streams": [
+      {
+        "type": "logs",
+        "dataset": "example.foo",
+        "title": "Foo"
+      }
+    ]
   },
   {
     "name": "example",
@@ -179,6 +231,13 @@
     "categories": [
       "crm",
       "azure"
+    ],
+    "data_streams": [
+      {
+        "type": "logs",
+        "dataset": "example.foo",
+        "title": "Foo"
+      }
     ]
   },
   {
@@ -222,6 +281,13 @@
     },
     "categories": [
       "custom"
+    ],
+    "data_streams": [
+      {
+        "type": "metrics",
+        "dataset": "hidden.hidden",
+        "title": "Hidden data stream and ilm policy overrride"
+      }
     ]
   },
   {
@@ -240,6 +306,13 @@
     },
     "categories": [
       "custom"
+    ],
+    "data_streams": [
+      {
+        "type": "metrics",
+        "dataset": "ilm_policy.ilm_policy",
+        "title": "ILM policy overrride data stream"
+      }
     ]
   },
   {
@@ -333,6 +406,13 @@
     "categories": [
       "crm",
       "azure"
+    ],
+    "data_streams": [
+      {
+        "type": "logs",
+        "dataset": "integration_input.foo",
+        "title": "Foo"
+      }
     ]
   },
   {
@@ -512,6 +592,13 @@
     "path": "/package/no_stream_configs/1.0.0",
     "categories": [
       "custom"
+    ],
+    "data_streams": [
+      {
+        "type": "logs",
+        "dataset": "no_stream_configs.log",
+        "title": "Log Yaml pipeline"
+      }
     ]
   },
   {
@@ -550,6 +637,13 @@
     "categories": [
       "custom",
       "web"
+    ],
+    "data_streams": [
+      {
+        "type": "logs",
+        "dataset": "reference.reference",
+        "title": "Reference Logs Title"
+      }
     ]
   },
   {
@@ -563,6 +657,13 @@
     "path": "/package/yamlpipeline/1.0.0",
     "categories": [
       "custom"
+    ],
+    "data_streams": [
+      {
+        "type": "logs",
+        "dataset": "yamlpipeline.log",
+        "title": "Log Yaml pipeline"
+      }
     ]
   }
 ]