Merge branch 'main' of github.com:Bluetab/td-cache into feature/td-7401

Author: Jandro Oliva

.gitignore

@@ -1,5 +1,6 @@
 # The directory Mix will write compiled artifacts to.
 /_build/
+/.elixir_ls
 
 # If you run "mix test --cover", coverage assets end up here.
 /cover/

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+[7.11.0] 2025-09-24
+
+### Added
+
+- [TD-7301] Add batch retrieval for Concepts
+
 ## [7.7.0] 2025-06-20
 
 ### Added

lib/td_cache/concept_cache.ex

@@ -12,6 +12,7 @@ defmodule TdCache.ConceptCache do
   alias TdCache.Redix
   alias TdCache.RuleCache
   alias TdCache.TaxonomyCache
+  alias TdCache.Utils.MapHelpers
 
   require Logger
 
@@ -66,6 +67,10 @@ defmodule TdCache.ConceptCache do
     GenServer.call(__MODULE__, {:get, id, property, opts})
   end
 
+  def get_many(ids, opts \\ []) do
+    GenServer.call(__MODULE__, {:get_many, ids, opts})
+  end
+
   @doc """
   Reads a i18n of a concept for a given id from cache
   """
@@ -169,6 +174,16 @@ defmodule TdCache.ConceptCache do
     {:reply, {:ok, prop}, state}
   end
 
+  @impl true
+  def handle_call({:get_many, ids, opts}, _from, state) do
+    concepts =
+      ids
+      |> Enum.uniq()
+      |> read_concepts_batch(opts)
+
+    {:reply, {:ok, concepts}, state}
+  end
+
   @impl true
   def handle_call({:get_i18n, id}, _from, state) do
     prop =
@@ -243,6 +258,23 @@ defmodule TdCache.ConceptCache do
     end
   end
 
+  defp read_concepts_batch([], _opts), do: []
+
+  defp read_concepts_batch(ids, _opts) do
+    transform_fun = fn [key, value] -> {String.to_atom(key), value} end
+
+    ids
+    |> Enum.map(fn id -> ["HGETALL", "business_concept:#{id}"] end)
+    |> Redix.transaction_pipeline()
+    |> MapHelpers.zip_results_with_ids(ids)
+    |> Enum.map(fn {id, hash} ->
+      hash
+      |> Redix.hash_to_map(transform_fun)
+      |> Map.put(:id, id)
+      |> concept_entry_to_map()
+    end)
+  end
+
   defp read_concept_i18n(id) do
     concept_key = "business_concept:#{id}"
     {:ok, concept} = Redix.read_map(concept_key)

lib/td_cache/structure_cache.ex

@@ -6,6 +6,7 @@ defmodule TdCache.StructureCache do
   alias TdCache.LinkCache
   alias TdCache.Redix
   alias TdCache.SystemCache
+  alias TdCache.Utils.MapHelpers
 
   ## Client API
 
@@ -24,6 +25,15 @@ defmodule TdCache.StructureCache do
     {:ok, structure}
   end
 
+  def get_many(ids, opts \\ []) do
+    structures =
+      ids
+      |> Enum.uniq()
+      |> read_structures_batch(opts)
+
+    {:ok, structures}
+  end
+
   @doc """
   Deletes cache entries relating to a given structure id.
   """
@@ -88,6 +98,40 @@ defmodule TdCache.StructureCache do
     end
   end
 
+  defp read_structures_batch([], _opts), do: []
+
+  defp read_structures_batch(ids, _opts) do
+    transform_fun = fn [key, value] -> {String.to_atom(key), value} end
+
+    ids
+    |> Enum.map(fn id -> ["HGETALL", "data_structure:#{id}"] end)
+    |> Redix.transaction_pipeline()
+    |> MapHelpers.zip_results_with_ids(ids)
+    |> Enum.map(fn {id, hash} ->
+      hash
+      |> Redix.hash_to_map(transform_fun)
+      |> enrich_structure_map(id)
+    end)
+  end
+
+  defp enrich_structure_map(structure, id) do
+    {:ok, path} = Redix.read_list("data_structure:#{id}:path")
+    {:ok, system} = SystemCache.get(Map.get(structure, :system_id))
+
+    metadata =
+      case Map.get(structure, :metadata) do
+        nil -> %{}
+        metadata -> Jason.decode!(metadata)
+      end
+
+    structure
+    |> Map.update(:domain_ids, [], &Redix.to_integer_list!/1)
+    |> put_optional(:path, path)
+    |> put_optional(:system, system)
+    |> Map.put(:metadata, metadata)
+    |> Map.put(:id, id)
+  end
+
   def put_optional(map, _key, nil), do: map
   def put_optional(map, key, value), do: Map.put(map, key, value)
 

lib/td_cache/utils/map_helpers.ex

@@ -17,4 +17,12 @@ defmodule TdCache.Utils.MapHelpers do
   def parse_string(:string, value) when is_atom(value), do: Atom.to_string(value)
   # def parse_string(:datetime, value) when is_binary(value), do: DateTime.from_iso8601(value)
   def parse_string(_, value), do: value
+
+  def zip_results_with_ids({:ok, results}, ids) do
+    ids
+    |> Enum.zip(results)
+    |> Enum.filter(fn {_id, result} ->
+      not Enum.empty?(result)
+    end)
+  end
 end

mix.exs

@@ -4,7 +4,7 @@ defmodule TdCache.MixProject do
   def project do
     [
       app: :td_cache,
-      version: "7.7.0",
+      version: "7.11.0",
       elixir: "~> 1.18",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,

test/support/factory.ex

@@ -27,6 +27,30 @@ defmodule TdCache.Factory do
     }
   end
 
+  def system_factory do
+    %{
+      id: System.unique_integer([:positive]),
+      external_id: sequence("external_id_name"),
+      name: sequence("system_name")
+    }
+  end
+
+  def structure_factory do
+    %{
+      id: System.unique_integer([:positive]),
+      name: sequence("structure_name"),
+      external_id: sequence("ds_external_id"),
+      group: sequence("data_structure_version_group"),
+      type: "type",
+      path: ["foo", "bar"],
+      updated_at: DateTime.utc_now(),
+      metadata: %{"alias" => "source_alias"},
+      system_id: System.unique_integer([:positive]),
+      domain_ids: [1, 2],
+      deleted_at: DateTime.utc_now()
+    }
+  end
+
   def ingest_factory do
     %{
       id: unique_id(),

test/td_cache/concept_cache_test.exs

@@ -94,6 +94,41 @@ defmodule TdCache.ConceptCacheTest do
       assert {:ok, ^content} = ConceptCache.get(concept.id, :content)
     end
 
+    test "reads many concepts", %{domain: %{id: domain_id}} do
+      %{id: shared_domain_id} = build(:domain)
+
+      [%{id: id1}, %{id: id2}, %{id: id3}] =
+        inserted_concepts =
+        Enum.map(1..3, fn i ->
+          concept =
+            if rem(i, 2) == 0 do
+              build(:concept, domain_id: domain_id, shared_to_ids: [shared_domain_id])
+            else
+              build(:concept, domain_id: domain_id)
+            end
+
+          {:ok, _} = ConceptCache.put(concept)
+          concept
+        end)
+
+      ids = [id1, id2, id3, id1]
+
+      not_valid_id = Enum.max(ids) + 1
+
+      {:ok, cache_concepts} = ConceptCache.get_many(Enum.shuffle(ids ++ [not_valid_id]))
+
+      assert Enum.count(cache_concepts) == 3
+
+      assert Enum.all?(cache_concepts, fn %{id: concept_id, name: concept_name} ->
+               Enum.find(inserted_concepts, fn %{id: inserted_id, name: inserted_name} ->
+                 inserted_id == concept_id and inserted_name == concept_name
+               end)
+             end)
+
+      Redix.command!(["DEL"] ++ Redix.command!(["KEYS", "business_concept:*"]))
+      Redix.command!(["DEL", "domain:deleted_ids"])
+    end
+
     test "reads the content property of a concept with specific lang", %{
       concept: concept
     } do

test/td_cache/structure_cache_test.exs

@@ -2,6 +2,7 @@ defmodule TdCache.StructureCacheTest do
   use ExUnit.Case
 
   import Assertions
+  import TdCache.Factory
 
   alias TdCache.LinkCache
   alias TdCache.Redix
@@ -11,21 +12,8 @@ defmodule TdCache.StructureCacheTest do
   doctest TdCache.StructureCache
 
   setup do
-    system = %{id: System.unique_integer([:positive]), external_id: "foo", name: "bar"}
-
-    structure = %{
-      id: System.unique_integer([:positive]),
-      name: "name",
-      external_id: "ext_id",
-      group: "group",
-      type: "type",
-      path: ["foo", "bar"],
-      updated_at: DateTime.utc_now(),
-      metadata: %{"alias" => "source_alias"},
-      system_id: system.id,
-      domain_ids: [1, 2],
-      deleted_at: DateTime.utc_now()
-    }
+    system = build(:system)
+    structure = build(:structure, system_id: system.id)
 
     {:ok, _} = SystemCache.put(system)
 
@@ -68,6 +56,38 @@ defmodule TdCache.StructureCacheTest do
       assert s.system == system
     end
 
+    test "reads many structures", %{system: system} do
+      %{id: domain_id1} = build(:domain)
+      %{id: domain_id2} = build(:domain)
+
+      [%{id: id1}, %{id: id2}, %{id: id3}] =
+        inserted_structures =
+        Enum.map(1..3, fn _ ->
+          structure =
+            build(:structure, domain_ids: [domain_id1, domain_id2], system_id: system.id)
+
+          {:ok, _} = StructureCache.put(structure)
+          structure
+        end)
+
+      ids = [id1, id2, id3, id1]
+
+      not_valid_id = Enum.max(ids) + 1
+
+      {:ok, cache_structures} = StructureCache.get_many(Enum.shuffle(ids ++ [not_valid_id]))
+
+      assert Enum.count(cache_structures) == 3
+
+      assert Enum.all?(cache_structures, fn %{id: structure_id, name: structure_name} ->
+               Enum.find(inserted_structures, fn %{id: inserted_id, name: inserted_name} ->
+                 inserted_id == structure_id and inserted_name == structure_name
+               end)
+             end)
+
+      Redix.command!(["DEL"] ++ Redix.command!(["KEYS", "data_structure:*"]))
+      Redix.command!(["DEL", "domain:deleted_ids"])
+    end
+
     test "returns an empty map for metadata if not present", %{structure: structure} do
       structure = Map.delete(structure, :metadata)
       assert {:ok, _} = StructureCache.put(structure)
@@ -93,15 +113,15 @@ defmodule TdCache.StructureCacheTest do
     end
 
     test "does not update a structure already cached in redis having same update_at value", %{
-      structure: structure
+      structure: %{external_id: external_id} = structure
     } do
       {:ok, _} = StructureCache.put(structure)
       {:ok, s} = StructureCache.get(structure.id)
       assert s
       updated_structure = Map.put(structure, :external_id, "new_ext_id")
       {:ok, _} = StructureCache.put(updated_structure)
       {:ok, s} = StructureCache.get(structure.id)
-      assert s.external_id == "ext_id"
+      assert s.external_id == external_id
     end
 
     test "updates a structure already cached in redis when deleted_at has changed", %{