feat(transport): unified attribute request — fetch-all per scope + scope-separated gateway response

application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java

@@ -23,7 +23,6 @@
 import com.google.common.util.concurrent.MoreExecutors;
 import jakarta.annotation.Nullable;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.common.util.LinkedHashMapRemoveEldest;
 import org.thingsboard.server.actors.ActorSystemContext;
@@ -97,7 +96,6 @@
 import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -524,13 +522,11 @@ private void handleGetAttributesRequest(SessionInfoProto sessionInfo, GetAttribu
             Futures.addCallback(findAllAttributesByScope(AttributeScope.SHARED_SCOPE), new FutureCallback<>() {
                 @Override
                 public void onSuccess(@Nullable List<AttributeKvEntry> result) {
-                    GetAttributeResponseMsg responseMsg = GetAttributeResponseMsg.newBuilder()
+                    sendToTransport(GetAttributeResponseMsg.newBuilder()
                             .setRequestId(requestId)
                             .setSharedStateMsg(true)
                             .addAllSharedAttributeList(KvProtoUtil.attrToTsKvProtos(result))
-                            .setIsMultipleAttributesRequest(request.getSharedAttributeNamesCount() > 1)
-                            .build();
-                    sendToTransport(responseMsg, sessionInfo);
+                            .build(), sessionInfo);
                 }
 
                 @Override
@@ -545,15 +541,19 @@ public void onFailure(Throwable t) {
         } else {
             Futures.addCallback(getAttributesKvEntries(request), new FutureCallback<>() {
                 @Override
-                public void onSuccess(@Nullable List<List<AttributeKvEntry>> result) {
-                    GetAttributeResponseMsg responseMsg = GetAttributeResponseMsg.newBuilder()
+                public void onSuccess(ClientSharedAttributes result) {
+                    GetAttributeResponseMsg.Builder builder = GetAttributeResponseMsg.newBuilder()
                             .setRequestId(requestId)
-                            .addAllClientAttributeList(KvProtoUtil.attrToTsKvProtos(result.get(0)))
-                            .addAllSharedAttributeList(KvProtoUtil.attrToTsKvProtos(result.get(1)))
-                            .setIsMultipleAttributesRequest(
-                                    request.getSharedAttributeNamesCount() + request.getClientAttributeNamesCount() > 1)
-                            .build();
-                    sendToTransport(responseMsg, sessionInfo);
+                            .addAllClientAttributeList(KvProtoUtil.attrToTsKvProtos(result.client()))
+                            .addAllSharedAttributeList(KvProtoUtil.attrToTsKvProtos(result.shared()));
+                    // legacy gateway value/values response relies on the deprecated isMultipleAttributesRequest flag
+                    if (!request.getSeparateScopesResponse()) {
+                        builder.setIsMultipleAttributesRequest(
+                                request.getSharedAttributeNamesCount() + request.getClientAttributeNamesCount() > 1);
+                    } else {
+                        builder.setSeparateScopesResponse(true);
+                    }
+                    sendToTransport(builder.build(), sessionInfo);
                 }
 
                 @Override
@@ -567,23 +567,35 @@ public void onFailure(Throwable t) {
         }
     }
 
-    private ListenableFuture<List<List<AttributeKvEntry>>> getAttributesKvEntries(GetAttributeRequestMsg request) {
-        ListenableFuture<List<AttributeKvEntry>> clientAttributesFuture;
-        ListenableFuture<List<AttributeKvEntry>> sharedAttributesFuture;
-        if (CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) {
-            clientAttributesFuture = findAllAttributesByScope(AttributeScope.CLIENT_SCOPE);
-            sharedAttributesFuture = findAllAttributesByScope(AttributeScope.SHARED_SCOPE);
-        } else if (!CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && !CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) {
-            clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), AttributeScope.CLIENT_SCOPE);
-            sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), AttributeScope.SHARED_SCOPE);
-        } else if (CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && !CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) {
-            clientAttributesFuture = Futures.immediateFuture(Collections.emptyList());
-            sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), AttributeScope.SHARED_SCOPE);
+    private record ClientSharedAttributes(List<AttributeKvEntry> client, List<AttributeKvEntry> shared) {}
+
+    private ListenableFuture<ClientSharedAttributes> getAttributesKvEntries(GetAttributeRequestMsg request) {
+        boolean clientAll = request.getAllClientAttributes();
+        boolean sharedAll = request.getAllSharedAttributes();
+        List<String> clientNames = request.getClientAttributeNamesList();
+        List<String> sharedNames = request.getSharedAttributeNamesList();
+
+        // backward-compat: an empty request (no scope signalled at all) fetches everything from both scopes
+        boolean fetchAll = !clientAll && !sharedAll && clientNames.isEmpty() && sharedNames.isEmpty();
+
+        ListenableFuture<List<AttributeKvEntry>> clientFuture = resolveScopeFuture(clientAll || fetchAll, clientNames, AttributeScope.CLIENT_SCOPE);
+        ListenableFuture<List<AttributeKvEntry>> sharedFuture = resolveScopeFuture(sharedAll || fetchAll, sharedNames, AttributeScope.SHARED_SCOPE);
+
+        return Futures.whenAllSucceed(clientFuture, sharedFuture)
+                .call(() -> new ClientSharedAttributes(Futures.getDone(clientFuture), Futures.getDone(sharedFuture)),
+                        MoreExecutors.directExecutor());
+    }
+
+    // "all <scope>" wins over a specific key list for the same scope; no signal => empty result.
+    // Same precedence as JsonConverter.applyScope (the request-build side); keep the two in sync.
+    private ListenableFuture<List<AttributeKvEntry>> resolveScopeFuture(boolean all, List<String> names, AttributeScope scope) {
+        if (all) {
+            return findAllAttributesByScope(scope);
+        } else if (!names.isEmpty()) {
+            return findAttributesByScope(new HashSet<>(names), scope);
         } else {
-            sharedAttributesFuture = Futures.immediateFuture(Collections.emptyList());
-            clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), AttributeScope.CLIENT_SCOPE);
+            return Futures.immediateFuture(Collections.emptyList());
         }
-        return Futures.allAsList(Arrays.asList(clientAttributesFuture, sharedAttributesFuture));
     }
 
     private ListenableFuture<List<AttributeKvEntry>> findAllAttributesByScope(AttributeScope scope) {
@@ -594,10 +606,6 @@ private ListenableFuture<List<AttributeKvEntry>> findAttributesByScope(Set<Strin
         return systemContext.getAttributesService().find(tenantId, deviceId, scope, attributesSet);
     }
 
-    private Set<String> toSet(List<String> strings) {
-        return new HashSet<>(strings);
-    }
-
     private SessionType getSessionType(UUID sessionId) {
         return sessions.containsKey(sessionId) ? SessionType.ASYNC : SessionType.SYNC;
     }

application/src/test/java/org/thingsboard/server/system/BaseHttpDeviceApiTest.java

@@ -15,6 +15,8 @@
  */
 package org.thingsboard.server.system;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import org.awaitility.Awaitility;
 import org.junit.Before;
 import org.junit.Test;
 import org.springframework.test.context.TestPropertySource;
@@ -25,9 +27,11 @@
 import org.thingsboard.server.common.data.security.DeviceCredentials;
 import org.thingsboard.server.controller.AbstractControllerTest;
 
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -74,6 +78,60 @@ public void testGetAttributes() throws Exception {
         doGetAsync("/api/v1/" + deviceCredentials.getCredentialsId() + "/attributes?clientKeys=keyA,keyB,keyC").andExpect(status().isOk());
     }
 
+    @Test
+    public void testGetAllClientAttributesViaAllClientKeysParam() throws Exception {
+        String token = deviceCredentials.getCredentialsId();
+        Map<String, String> clientAttrs = new HashMap<>();
+        clientAttrs.put("clientA", "valueA");
+        clientAttrs.put("clientB", "valueB");
+        mockMvc.perform(
+                        asyncDispatch(doPost("/api/v1/" + token + "/attributes", clientAttrs, new String[]{}).andReturn()))
+                .andExpect(status().isOk());
+        String allClientKeysUrl = "/api/v1/" + token + "/attributes?allClientKeys=true";
+        Awaitility.await("client attributes are persisted and returned via allClientKeys=true")
+                .atMost(30, TimeUnit.SECONDS)
+                .pollInterval(Duration.ofMillis(100))
+                .ignoreExceptions()
+                .until(() -> {
+                    JsonNode r = JacksonUtil.toJsonNode(doGetAsync(allClientKeysUrl).andReturn().getResponse().getContentAsString());
+                    return r.has("client") && clientAttrs.keySet().stream().allMatch(r.get("client")::has);
+                });
+        String body = doGetAsync(allClientKeysUrl)
+                .andExpect(status().isOk()).andReturn().getResponse().getContentAsString();
+        JsonNode resp = JacksonUtil.toJsonNode(body);
+        assertThat(resp.has("client")).isTrue();
+        JsonNode client = resp.get("client");
+        clientAttrs.forEach((key, value) -> assertThat(client.get(key).asText()).isEqualTo(value));
+        assertThat(resp.has("shared")).isFalse();
+    }
+
+    @Test
+    public void testGetAllSharedAttributesViaAllSharedKeysParam() throws Exception {
+        String token = deviceCredentials.getCredentialsId();
+        Map<String, String> sharedAttrs = new HashMap<>();
+        sharedAttrs.put("sharedA", "valueA");
+        sharedAttrs.put("sharedB", "valueB");
+        mockMvc.perform(
+                        asyncDispatch(doPost("/api/plugins/telemetry/DEVICE/" + device.getId().getId() + "/attributes/SHARED_SCOPE", sharedAttrs).andReturn()))
+                .andExpect(status().isOk());
+        String allSharedKeysUrl = "/api/v1/" + token + "/attributes?allSharedKeys=true";
+        Awaitility.await("shared attributes are persisted and returned via allSharedKeys=true")
+                .atMost(30, TimeUnit.SECONDS)
+                .pollInterval(Duration.ofMillis(100))
+                .ignoreExceptions()
+                .until(() -> {
+                    JsonNode r = JacksonUtil.toJsonNode(doGetAsync(allSharedKeysUrl).andReturn().getResponse().getContentAsString());
+                    return r.has("shared") && sharedAttrs.keySet().stream().allMatch(r.get("shared")::has);
+                });
+        String body = doGetAsync(allSharedKeysUrl)
+                .andExpect(status().isOk()).andReturn().getResponse().getContentAsString();
+        JsonNode resp = JacksonUtil.toJsonNode(body);
+        assertThat(resp.has("shared")).isTrue();
+        JsonNode shared = resp.get("shared");
+        sharedAttrs.forEach((key, value) -> assertThat(shared.get(key).asText()).isEqualTo(value));
+        assertThat(resp.has("client")).isFalse();
+    }
+
     @Test
     public void testReplyToCommandWithLargeResponse() throws Exception {
         String errorResponse = doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc/5",

application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java

@@ -30,6 +30,9 @@ public abstract class AbstractTransportIntegrationTest extends AbstractControlle
 
     public static final int DEFAULT_WAIT_TIMEOUT_SECONDS = 30;
 
+    protected static final String CLIENT_ATTRIBUTE_KEYS = "clientStr,clientBool,clientDbl,clientLong,clientJson";
+    protected static final String SHARED_ATTRIBUTE_KEYS = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson";
+
     protected static final AtomicInteger atomicInteger = new AtomicInteger(2);
 
     public static final String DEVICE_TELEMETRY_PROTO_SCHEMA = "syntax =\"proto3\";\n" +

application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java

@@ -89,12 +89,13 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap
             "  }\n" +
             "}";
 
-    private static final String CLIENT_ATTRIBUTES_PAYLOAD = "{\"clientStr\":\"value1\",\"clientBool\":true,\"clientDbl\":42.0,\"clientLong\":73," +
+    protected static final String CLIENT_ATTRIBUTES_PAYLOAD = "{\"clientStr\":\"value1\",\"clientBool\":true,\"clientDbl\":42.0,\"clientLong\":73," +
             "\"clientJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}";
 
-    private static final String SHARED_ATTRIBUTES_PAYLOAD = "{\"sharedStr\":\"value1\",\"sharedBool\":true,\"sharedDbl\":42.0,\"sharedLong\":73," +
+    protected static final String SHARED_ATTRIBUTES_PAYLOAD = "{\"sharedStr\":\"value1\",\"sharedBool\":true,\"sharedDbl\":42.0,\"sharedLong\":73," +
             "\"sharedJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}";
 
+
     protected static final String SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION = "{\"sharedStr\":\"value\",\"sharedBool\":false,\"sharedDbl\":41.0,\"sharedLong\":72," +
             "\"sharedJson\":{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}}";
 
@@ -169,18 +170,28 @@ private byte[] getAttributesProtoPayloadBytes() {
     }
 
     protected void processJsonTestRequestAttributesValuesFromTheServer() throws Exception {
+        postAttributesAndAwaitWsUpdate();
+        String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + CLIENT_ATTRIBUTE_KEYS + "&sharedKeys=" + SHARED_ATTRIBUTE_KEYS;
+        client.setURI(featureTokenUrl);
+        validateJsonResponse(client.getMethod());
+    }
+
+    protected void processJsonTestRequestAttributesWithQuery(String querySuffix, String expectedResponse) throws Exception {
+        postAttributesAndAwaitWsUpdate();
+        String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + querySuffix;
+        client.setURI(featureTokenUrl);
+        CoapResponse response = client.getMethod();
+        assertEquals(CoAP.ResponseCode.CONTENT, response.getCode());
+        assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.fromBytes(response.getPayload()));
+    }
+
+    private void postAttributesAndAwaitWsUpdate() throws Exception {
         client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES);
         SingleEntityFilter dtf = new SingleEntityFilter();
         dtf.setSingleEntity(AliasEntityId.fromEntityId(savedDevice.getId()));
-        String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson";
-        String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson";
-        List<String> clientKeysList = List.of(clientKeysStr.split(","));
-        List<String> sharedKeysList = List.of(sharedKeysStr.split(","));
-        List<EntityKey> csKeys = getEntityKeys(clientKeysList, CLIENT_ATTRIBUTE);
-        List<EntityKey> shKeys = getEntityKeys(sharedKeysList, SHARED_ATTRIBUTE);
         List<EntityKey> keys = new ArrayList<>();
-        keys.addAll(csKeys);
-        keys.addAll(shKeys);
+        keys.addAll(getEntityKeys(List.of(CLIENT_ATTRIBUTE_KEYS.split(",")), CLIENT_ATTRIBUTE));
+        keys.addAll(getEntityKeys(List.of(SHARED_ATTRIBUTE_KEYS.split(",")), SHARED_ATTRIBUTE));
         getWsClient().subscribeLatestUpdate(keys, dtf);
         getWsClient().registerWaitForUpdate(2);
 
@@ -192,20 +203,14 @@ protected void processJsonTestRequestAttributesValuesFromTheServer() throws Exce
 
         String update = getWsClient().waitForUpdate();
         assertThat(update).as("ws update received").isNotBlank();
-
-        String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr;
-        client.setURI(featureTokenUrl);
-        validateJsonResponse(client.getMethod());
     }
 
     protected void processProtoTestRequestAttributesValuesFromTheServer() throws Exception {
         client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES);
         SingleEntityFilter dtf = new SingleEntityFilter();
         dtf.setSingleEntity(AliasEntityId.fromEntityId(savedDevice.getId()));
-        String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson";
-        String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson";
-        List<String> clientKeysList = List.of(clientKeysStr.split(","));
-        List<String> sharedKeysList = List.of(sharedKeysStr.split(","));
+        List<String> clientKeysList = List.of(CLIENT_ATTRIBUTE_KEYS.split(","));
+        List<String> sharedKeysList = List.of(SHARED_ATTRIBUTE_KEYS.split(","));
         List<EntityKey> csKeys = getEntityKeys(clientKeysList, CLIENT_ATTRIBUTE);
         List<EntityKey> shKeys = getEntityKeys(sharedKeysList, SHARED_ATTRIBUTE);
         List<EntityKey> keys = new ArrayList<>();
@@ -223,7 +228,7 @@ protected void processProtoTestRequestAttributesValuesFromTheServer() throws Exc
         String update = getWsClient().waitForUpdate();
         assertThat(update).as("ws update received").isNotBlank();
 
-        String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr;
+        String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + CLIENT_ATTRIBUTE_KEYS + "&sharedKeys=" + SHARED_ATTRIBUTE_KEYS;
         client.setURI(featureTokenUrl);
         validateProtoResponse(client.getMethod());
     }

application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java

@@ -47,4 +47,14 @@ public void afterTest() throws Exception {
     public void testRequestAttributesValuesFromTheServer() throws Exception {
         processJsonTestRequestAttributesValuesFromTheServer();
     }
+
+    @Test
+    public void testRequestAllSharedAttributes() throws Exception {
+        processJsonTestRequestAttributesWithQuery("?allSharedKeys=true", "{\"shared\":" + SHARED_ATTRIBUTES_PAYLOAD + "}");
+    }
+
+    @Test
+    public void testRequestAllClientAttributes() throws Exception {
+        processJsonTestRequestAttributesWithQuery("?allClientKeys=true", "{\"client\":" + CLIENT_ATTRIBUTES_PAYLOAD + "}");
+    }
 }