refactor: introduce $populate operator for relation loading and enforce separation of scalar and relation fields

Author: rogerpadilla

CHANGELOG.md

@@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file. Please add
 
 date format is [yyyy-mm-dd]
 
+## [0.8.4] - 2026-04-11
+
+### Features
+
+- Added first-class `$populate` (relations) and `$exclude` (subtractive scalar projection), while keeping backward compatibility for legacy relation keys in `$select`.
+
+### Breaking Changes (Public util surface)
+
+- Removed support for specifying relations in `$select`; use `$populate` instead.
+
+### Improvements
+
+- Populate-only relation loading now works consistently in SQL dialects.
+- Relation keys in `$select` now emit a deduplicated deprecation warning (use `$populate`).
+- `$select` and `$exclude` conflicts are validated recursively, including nested relation queries.
+- `findManyStream` now rejects unsupported relation loading early: **MongoDB** throws if any relation is requested in `$select` / `$populate` (streams use `find` only). **SQL** throws if **to-many** relations are requested (they are filled only after `findMany`, not while streaming).
+
+### Documentation
+
+- README and docs site clarify projection keys (`$select` / `$exclude` / `$populate`) and streaming behavior across SQL vs MongoDB.
+
 ## [0.8.3] - 2026-04-04
 
 ### Improvements

README.md

@@ -8,7 +8,8 @@
 
 ```ts
 const results = await querier.findMany(User, {
-  $select: { name: true, profile: { $select: { picture: true } } },
+  $select: { name: true },
+  $populate: { profile: { $select: { picture: true } } },
   $where: { name: { $istartsWith: 'a' }, posts: { tags: { name: 'typescript' } } },
   $sort: { createdAt: 'desc' },
   $limit: 10,
@@ -346,10 +347,8 @@ UQL provides a straightforward API to interact with your data. **Always ensure q
 const querier = await pool.getQuerier();
 try {
   const results = await querier.findMany(User, {
-    $select: {
-      name: true,
-      profile: { $select: { bio: true }, $required: true }, // INNER JOIN
-    },
+    $select: { name: true },
+    $populate: { profile: { $select: { bio: true }, $required: true } }, // INNER JOIN
     $where: {
       status: 'active',
       name: { $istartsWith: 'a' },
@@ -371,6 +370,34 @@ WHERE "User"."status" = 'active' AND "User"."name" ILIKE 'a%'
 LIMIT 10 OFFSET 0
 ```
 
+### Projection & Relation Loading
+
+Use these query keys with clear separation of concerns:
+
+- `$select`: scalar field whitelist (projection)
+- `$exclude`: scalar field subtraction from default eager set
+- `$populate`: relation loading (including nested relation query options)
+
+`$select` and `$exclude` are mutually exclusive when `$select` includes positive scalar keys.
+
+```ts
+const rows = await querier.findMany(User, {
+  $exclude: { password: true },
+  $populate: { profile: { $select: { picture: true } } },
+});
+```
+
+> **Migration note:** selecting relations inside `$select` is still supported for backward compatibility, but deprecated. Move relations to `$populate`.
+
+### Streaming (`findManyStream`) and relations
+
+`findManyStream` is optimized for **scalar** reads and a stable memory footprint. Relation loading differs by backend:
+
+- **SQL:** Joinable relations (e.g. many-to-one, one-to-one) are still included in the streamed `SELECT` / joins. **To-many** collections are not filled on the stream path (that uses extra queries in `findMany`); requesting them in `$select` / `$populate` throws a `TypeError` with a short explanation.
+- **MongoDB:** Streams use a plain `find` cursor, so **any** relation keys in `$select` or `$populate` throw a `TypeError`. Use `findMany` for relation loading (aggregation + fill).
+
+See [Cursor Streaming](https://uql-orm.dev/querying/streaming) on the docs site for details.
+
 ### Advanced Query Patterns
 
 ### Modern Indexing: Semantic Search

bun.lock

@@ -42,22 +42,22 @@
     }
   },
   "devDependencies": {
-    "@biomejs/biome": "2.4.10",
+    "@biomejs/biome": "2.4.11",
     "@commitlint/cli": "20.5.0",
     "@commitlint/config-conventional": "20.5.0",
     "@swc/core": "^1.15.24",
     "@swc/helpers": "^0.5.21",
-    "@types/bun": "^1.3.11",
+    "@types/bun": "^1.3.12",
     "@types/supertest": "^7.2.0",
-    "@vitest/coverage-v8": "^4.1.2",
+    "@vitest/coverage-v8": "^4.1.4",
     "lerna": "^9.0.7",
     "mongodb-memory-server": "11.0.1",
     "rimraf": "6.1.3",
     "supertest": "^7.2.2",
     "typescript": "^6.0.2",
     "unplugin-swc": "^1.5.9",
-    "vite": "^8.0.3",
-    "vitest": "^4.1.2"
+    "vite": "^8.0.8",
+    "vitest": "^4.1.4"
   },
   "author": "Roger Padilla"
 }

package.json

@@ -107,12 +107,12 @@
     "@types/express": "^5.0.6",
     "@types/pg": "^8.20.0",
     "@types/sqlstring": "^2.3.2",
-    "better-sqlite3": "^12.8.0",
-    "bunchee": "^6.9.4",
+    "better-sqlite3": "^12.9.0",
+    "bunchee": "^6.10.0",
     "express": "^5.2.1",
     "mariadb": "^3.5.2",
     "mongodb": "^7.1.1",
-    "mysql2": "^3.20.0",
+    "mysql2": "^3.22.0",
     "pg": "^8.20.0",
     "pg-query-stream": "^4.14.0"
   },

packages/uql-orm/package.json

@@ -23,13 +23,14 @@ it('stringifyQuery', () => {
   expect(stringifyQuery({})).toBe('');
   expect(stringifyQuery({ $sort: undefined })).toBe('');
   const source: Query<Item> = {
-    $select: { id: 1, name: 1, tax: true, measureUnit: { $select: { id: 1, name: 1, categoryId: 1 } } },
+    $select: { id: 1, name: 1 },
+    $populate: { tax: true, measureUnit: { $select: { id: 1, name: 1, categoryId: 1 } } },
     $where: { name: 'Batman', companyId: 38 },
     $sort: { companyId: 1, name: -1 },
     $limit: 5,
   };
   const result = stringifyQuery(source);
   const expected =
-    '?$select={"id":1,"name":1,"tax":true,"measureUnit":{"$select":{"id":1,"name":1,"categoryId":1}}}&$where={"name":"Batman","companyId":38}&$sort={"companyId":1,"name":-1}&$limit=5';
+    '?$select={"id":1,"name":1}&$populate={"tax":true,"measureUnit":{"$select":{"id":1,"name":1,"categoryId":1}}}&$where={"name":"Batman","companyId":38}&$sort={"companyId":1,"name":-1}&$limit=5';
   expect(result).toBe(expected);
 });

packages/uql-orm/src/browser/querier/querier.util.spec.ts

@@ -282,6 +282,22 @@ export abstract class AbstractSqlDialectSpec implements Spec {
     expect(res.values).toEqual([123, 'abc']);
   }
 
+  shouldFindWithPopulateOnly() {
+    const res = this.exec((ctx) =>
+      this.dialect.find(ctx, User, {
+        $populate: {
+          profile: {
+            $select: { picture: true },
+          },
+        },
+        $where: { id: 123 },
+      }),
+    );
+    expect(res.sql).toContain('LEFT JOIN `user_profile` `profile` ON `profile`.`creatorId` = `User`.`id`');
+    expect(res.sql).toContain('`profile`.`image` `profile.picture`');
+    expect(res.values).toEqual([123]);
+  }
+
   shouldBeSecure() {
     let res = this.exec((ctx) =>
       this.dialect.find(ctx, User, {
@@ -687,7 +703,9 @@ export abstract class AbstractSqlDialectSpec implements Spec {
   }
 
   shouldFind$selectFields() {
-    const { sql } = this.exec((ctx) => this.dialect.find(ctx, User, { $select: { id: true, company: true } }));
+    const { sql } = this.exec((ctx) =>
+      this.dialect.find(ctx, User, { $select: { id: true }, $populate: { company: true } }),
+    );
     expect(sql).toBe(
       'SELECT `User`.`id`, `company`.`id` `company.id`, `company`.`companyId` `company.companyId`, `company`.`creatorId` `company.creatorId`' +
         ', `company`.`createdAt` `company.createdAt`, `company`.`updatedAt` `company.updatedAt`' +
@@ -699,15 +717,16 @@ export abstract class AbstractSqlDialectSpec implements Spec {
   shouldFind$selectOneToOne() {
     let res = this.exec((ctx) =>
       this.dialect.find(ctx, User, {
-        $select: { id: true, name: true, profile: { $select: { id: true, picture: true } } },
+        $select: { id: true, name: true },
+        $populate: { profile: { $select: { id: true, picture: true } } },
       }),
     );
     expect(res.sql).toBe(
       'SELECT `User`.`id`, `User`.`name`, `profile`.`pk` `profile.pk`, `profile`.`image` `profile.picture` FROM `User`' +
         ' LEFT JOIN `user_profile` `profile` ON `profile`.`creatorId` = `User`.`id`',
     );
 
-    res = this.exec((ctx) => this.dialect.find(ctx, User, { $select: { profile: true } }));
+    res = this.exec((ctx) => this.dialect.find(ctx, User, { $populate: { profile: true } }));
     expect(res.sql).toBe(
       'SELECT `User`.`id`, `User`.`companyId`, `User`.`creatorId`, `User`.`createdAt`' +
         ', `User`.`updatedAt`, `User`.`name`, `User`.`email`' +
@@ -726,6 +745,8 @@ export abstract class AbstractSqlDialectSpec implements Spec {
           id: true,
           name: true,
           code: true,
+        },
+        $populate: {
           tax: { $select: { id: true, name: true }, $required: true },
           measureUnit: { $select: { id: true, name: true, categoryId: true } },
         },
@@ -749,8 +770,10 @@ export abstract class AbstractSqlDialectSpec implements Spec {
         $select: {
           id: true,
           name: true,
+        },
+        $populate: {
           measureUnit: { $select: { id: true, name: true }, $where: { name: { $ne: 'unidad' } }, $required: true },
-          tax: ['id', 'name'] as any,
+          tax: { $select: { id: true, name: true } },
         },
         $where: { salePrice: { $gte: 1000 }, name: { $istartsWith: 'A' } },
         $sort: { tax: { name: 1 }, measureUnit: { name: 1 }, createdAt: -1 },
@@ -793,8 +816,11 @@ export abstract class AbstractSqlDialectSpec implements Spec {
           name: 1,
           code: 1,
           tagsCount: 1,
+        },
+        $populate: {
           measureUnit: {
-            $select: { id: 1, name: 1, categoryId: 1, category: ['name'] },
+            $select: { id: 1, name: 1, categoryId: 1 },
+            $populate: { category: { $select: { name: 1 } } },
           },
         },
         $limit: 100,
@@ -818,8 +844,11 @@ export abstract class AbstractSqlDialectSpec implements Spec {
           id: 1,
           name: 1,
           code: 1,
+        },
+        $populate: {
           measureUnit: {
-            $select: { id: 1, name: 1, categoryId: 1, category: ['name'] },
+            $select: { id: 1, name: 1, categoryId: 1 },
+            $populate: { category: { $select: { name: 1 } } },
           },
         },
         $limit: 100,
@@ -841,8 +870,11 @@ export abstract class AbstractSqlDialectSpec implements Spec {
           id: true,
           name: true,
           code: true,
+        },
+        $populate: {
           measureUnit: {
-            $select: { id: true, name: true, category: { $select: { id: true, name: true } } },
+            $select: { id: true, name: true },
+            $populate: { category: { $select: { id: true, name: true } } },
           },
         },
         $limit: 100,
@@ -863,12 +895,17 @@ export abstract class AbstractSqlDialectSpec implements Spec {
           id: true,
           buyPrice: true,
           number: true,
+        },
+        $populate: {
           item: {
             $select: {
               id: true,
               name: true,
+            },
+            $populate: {
               measureUnit: {
-                $select: { id: true, name: true, category: ['id', 'name'] },
+                $select: { id: true, name: true },
+                $populate: { category: { $select: { id: true, name: true } } },
               },
             },
             $required: true,