chore: spec update to last openAPI spec

devenv.lock

@@ -19,10 +19,10 @@
     "flake-compat": {
       "flake": false,
       "locked": {
-        "lastModified": 1733328505,
+        "lastModified": 1747046372,
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
+        "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
         "type": "github"
       },
       "original": {
@@ -46,10 +46,31 @@
         "type": "github"
       }
     },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1760663237,
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "ca5b894d3e3e151ffc1db040b6ce4dcc75d31c37",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
     "gitignore": {
       "inputs": {
         "nixpkgs": [
-          "pre-commit-hooks",
+          "git-hooks",
           "nixpkgs"
         ]
       },
@@ -83,7 +104,7 @@
     },
     "nixpkgs-python": {
       "inputs": {
-        "flake-compat": "flake-compat",
+        "flake-compat": "flake-compat_2",
         "nixpkgs": [
           "nixpkgs"
         ]
@@ -101,49 +122,15 @@
         "type": "github"
       }
     },
-    "nixpkgs-stable": {
-      "locked": {
-        "lastModified": 1734202038,
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "bcba2fbf6963bf6bed3a749f9f4cf5bff4adb96d",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-24.05",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "pre-commit-hooks": {
-      "inputs": {
-        "flake-compat": "flake-compat_2",
-        "gitignore": "gitignore",
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "nixpkgs-stable": "nixpkgs-stable"
-      },
-      "locked": {
-        "lastModified": 1734279981,
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "devenv": "devenv",
+        "git-hooks": "git-hooks",
         "nixpkgs": "nixpkgs",
         "nixpkgs-python": "nixpkgs-python",
-        "pre-commit-hooks": "pre-commit-hooks"
+        "pre-commit-hooks": [
+          "git-hooks"
+        ]
       }
     }
   },

devenv.nix

@@ -21,7 +21,7 @@
   languages.python.version = "3.12";
 
   scripts.generate-model.exec = ''
-    ./source/model/generate_model.sh
+    $DEVENV_ROOT/dlt_source_affinity/model/generate_model.sh
   '';
 
   git-hooks.hooks = {
@@ -55,10 +55,11 @@
   '';
 
   scripts.sample-pipeline-run.exec = ''
-    python affinity_pipeline.py
+    python $DEVENV_ROOT/affinity_pipeline.py
   '';
 
   scripts.sample-pipeline-show.exec = ''
+    cd "$DEVENV_ROOT"
     dlt pipeline affinity_pipeline show
   '';
 }

dlt_source_affinity/__init__.py

@@ -405,9 +405,9 @@ def __entities(
                 "limit": len(ids),
                 "ids": ids,
                 "fieldTypes": [
-                    Type2.ENRICHED.value,
-                    Type2.GLOBAL_.value,
-                    Type2.RELATIONSHIP_INTELLIGENCE.value,
+                    Type3.ENRICHED.value,
+                    Type3.GLOBAL_.value,
+                    Type3.RELATIONSHIP_INTELLIGENCE.value,
                 ],
             },
             hooks=hooks,
@@ -471,10 +471,10 @@ def __list_entries() -> Iterable[TDataItem]:
                 params={
                     "limit": MAX_PAGE_LIMIT_V2,
                     "fieldTypes": [
-                        Type2.ENRICHED.value,
-                        Type2.GLOBAL_.value,
-                        Type2.RELATIONSHIP_INTELLIGENCE.value,
-                        Type2.LIST.value,
+                        Type3.ENRICHED.value,
+                        Type3.GLOBAL_.value,
+                        Type3.RELATIONSHIP_INTELLIGENCE.value,
+                        Type3.LIST.value,
                     ],
                 },
                 hooks=hooks,

dlt_source_affinity/model/.gitattributes

@@ -1,2 +1,2 @@
-v2.py linguist-generated=true
+v2/*.py linguist-generated=true
 *_spec.json linguist-vendored=true

dlt_source_affinity/model/README.md

@@ -6,4 +6,4 @@ You can find the latest version of the spec [here](https://developer.affinity.co
 
 1. Download the new spec from the link above.
 2. Put it into this folder.
-3. Run `./generate_model.sh`
+3. Run `generate-model` from project (requires direnv)

dlt_source_affinity/model/generate_model.sh

@@ -16,9 +16,19 @@ main() {
 
     log "Current Python version: ${python_version}"
 
+    log "Applying OpenAPI spec patch"
+    git apply \
+        --allow-empty \
+        ./v2_spec_patches.diff
+    log "Applied spec patch"
+
+    log "Generating code from OpenAPI spec"
+    rm -rf ./v2
+    mkdir -p ./v2
+    # we need to ignore extra fields because DLT adds extra fields to models like _dlt_id, etc.
     datamodel-codegen \
         --input v2_spec.json \
-        --output v2.py \
+        --output ./v2 \
         --output-model-type pydantic_v2.BaseModel \
         --use-annotated \
         --use-union-operator \
@@ -27,17 +37,22 @@ main() {
         --input-file-type openapi \
         --field-constraints \
         --use-double-quotes \
-        --base-class ..MyBaseModel \
+        --base-class ...MyBaseModel \
         --disable-timestamp \
-        --target-python-version "${python_version}"
-
+        --target-python-version "${python_version}" \
+        --extra-fields "ignore"
     log "Generated code"
+
+    log "Reverting OpenAPI spec patch"
+    git checkout \
+        ./v2_spec.json
+    log "Reverted spec patch"
 
+    log "Applying model patch"
     git apply \
         --allow-empty \
         ./v2_model_patches.diff
-
-    log "Applied patch"
+    log "Applied model patch"
 
     popd >/dev/null
 }