Add invocation_started_at (#11291)

.changes/unreleased/Breaking Changes-20250210-123306.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Add invocations_started_at field to artifact metadata
+time: 2025-02-10T12:33:06.722803-05:00
+custom:
+  Author: gshank
+  Issue: "11272"

core/dbt/artifacts/schemas/base.py

@@ -12,7 +12,7 @@
 from dbt_common.dataclass_schema import dbtClassMixin
 from dbt_common.events.functions import get_metadata_vars
 from dbt_common.exceptions import DbtInternalError, DbtRuntimeError
-from dbt_common.invocation import get_invocation_id
+from dbt_common.invocation import get_invocation_id, get_invocation_started_at
 
 BASE_SCHEMAS_URL = "https://schemas.getdbt.com/"
 SCHEMA_PATH = "dbt/{name}/v{version}.json"
@@ -57,6 +57,9 @@ class BaseArtifactMetadata(dbtClassMixin):
     dbt_version: str = __version__
     generated_at: datetime = dataclasses.field(default_factory=datetime.utcnow)
     invocation_id: Optional[str] = dataclasses.field(default_factory=get_invocation_id)
+    invocation_started_at: Optional[datetime] = dataclasses.field(
+        default_factory=get_invocation_started_at
+    )
     env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_vars)
 
     def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):

core/dbt/config/runtime.py

@@ -1,8 +1,7 @@
 import itertools
 import os
 from copy import deepcopy
-from dataclasses import dataclass, field
-from datetime import datetime
+from dataclasses import dataclass
 from pathlib import Path
 from typing import (
     Any,
@@ -16,8 +15,6 @@
     Type,
 )
 
-import pytz
-
 from dbt import tracking
 from dbt.adapters.contracts.connection import (
     AdapterRequiredConfig,
@@ -101,7 +98,6 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
     profile_name: str
     cli_vars: Dict[str, Any]
     dependencies: Optional[Mapping[str, "RuntimeConfig"]] = None
-    invoked_at: datetime = field(default_factory=lambda: datetime.now(pytz.UTC))
 
     def __post_init__(self):
         self.validate()

core/dbt/task/run.py

@@ -55,6 +55,7 @@
 from dbt_common.events.functions import fire_event, get_invocation_id
 from dbt_common.events.types import Formatting
 from dbt_common.exceptions import DbtValidationError
+from dbt_common.invocation import get_invocation_started_at
 
 
 @functools.total_ordering
@@ -572,7 +573,7 @@ def _execute_microbatch_materialization(
             is_incremental=self._is_incremental(model),
             event_time_start=event_time_start,
             event_time_end=event_time_end,
-            default_end_time=self.config.invoked_at,
+            default_end_time=get_invocation_started_at(),
         )
 
         if self.batch_idx is None:

core/setup.py

@@ -71,7 +71,7 @@
         "dbt-extractor>=0.5.0,<=0.6",
         "dbt-semantic-interfaces>=0.8.3,<0.9",
         # Minor versions for these are expected to be backwards-compatible
-        "dbt-common>=1.13.0,<2.0",
+        "dbt-common>=1.15.0,<2.0",
         "dbt-adapters>=1.13.0,<2.0",
         # ----
         # Expect compatibility with all new versions of these packages, so lower bounds only.

schemas/dbt/catalog/v1.json

@@ -27,6 +27,16 @@
             }
           ]
         },
+        "invocation_started_at": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
         "env": {
           "type": "object",
           "additionalProperties": {

schemas/dbt/manifest/v12.json

@@ -28,6 +28,16 @@
             }
           ]
         },
+        "invocation_started_at": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
         "env": {
           "type": "object",
           "additionalProperties": {

schemas/dbt/run-results/v6.json

@@ -27,6 +27,16 @@
             }
           ]
         },
+        "invocation_started_at": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
         "env": {
           "type": "object",
           "additionalProperties": {

schemas/dbt/sources/v3.json

@@ -27,6 +27,16 @@
             }
           ]
         },
+        "invocation_started_at": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
         "env": {
           "type": "object",
           "additionalProperties": {

tests/unit/contracts/graph/test_manifest.py

@@ -383,6 +383,7 @@ def test_no_nodes(self, mock_user):
         )
 
         invocation_id = dbt_common.invocation._INVOCATION_ID
+        invocation_started_at = dbt_common.invocation._INVOCATION_STARTED_AT
         mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf"
         set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None)
         self.assertEqual(
@@ -404,6 +405,7 @@ def test_no_nodes(self, mock_user):
                     "dbt_version": dbt.version.__version__,
                     "env": {ENV_KEY_NAME: "value"},
                     "invocation_id": invocation_id,
+                    "invocation_started_at": str(invocation_started_at).replace(" ", "T") + "Z",
                     "send_anonymous_usage_stats": False,
                     "user_id": "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf",
                 },
@@ -531,11 +533,13 @@ def test_build_flat_graph(self):
     def test_no_nodes_with_metadata(self, mock_user):
         mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf"
         dbt_common.invocation._INVOCATION_ID = "01234567-0123-0123-0123-0123456789ab"
+        dbt_common.invocation._INVOCATION_STARTED_AT = datetime.utcnow()
         set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None)
         metadata = ManifestMetadata(
             project_id="098f6bcd4621d373cade4e832627b4f6",
             adapter_type="postgres",
             generated_at=datetime.utcnow(),
+            invocation_started_at=dbt_common.invocation._INVOCATION_STARTED_AT,
             user_id="cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf",
             send_anonymous_usage_stats=False,
         )
@@ -553,6 +557,9 @@ def test_no_nodes_with_metadata(self, mock_user):
             saved_queries={},
         )
 
+        invocation_started_at = (
+            str(dbt_common.invocation._INVOCATION_STARTED_AT).replace(" ", "T") + "Z"
+        )
         self.assertEqual(
             manifest.writable_manifest().to_dict(omit_none=True),
             {
@@ -576,6 +583,7 @@ def test_no_nodes_with_metadata(self, mock_user):
                     "send_anonymous_usage_stats": False,
                     "adapter_type": "postgres",
                     "invocation_id": "01234567-0123-0123-0123-0123456789ab",
+                    "invocation_started_at": invocation_started_at,
                     "env": {ENV_KEY_NAME: "value"},
                 },
                 "disabled": {},
@@ -884,8 +892,11 @@ def tearDown(self):
     def test_no_nodes(self, mock_user):
         mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf"
         set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None)
+        invocation_started_at = datetime.utcnow()
         metadata = ManifestMetadata(
-            generated_at=datetime.utcnow(), invocation_id="01234567-0123-0123-0123-0123456789ab"
+            generated_at=datetime.utcnow(),
+            invocation_id="01234567-0123-0123-0123-0123456789ab",
+            invocation_started_at=invocation_started_at,
         )
         manifest = Manifest(
             nodes={},
@@ -918,6 +929,7 @@ def test_no_nodes(self, mock_user):
                     "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json",
                     "dbt_version": dbt.version.__version__,
                     "invocation_id": "01234567-0123-0123-0123-0123456789ab",
+                    "invocation_started_at": str(invocation_started_at).replace(" ", "T") + "Z",
                     "env": {ENV_KEY_NAME: "value"},
                     "send_anonymous_usage_stats": False,
                     "user_id": "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf",