Merge branch 'master' into support_312

.github/workflows/build.yml

@@ -34,14 +34,6 @@ jobs:
         with:
           options: "--check --diff --line-length 160"
           version: "23.3.0"
-      - name: Test with pytest
-        run: |
-          pytest -v . --junit-xml pytest.xml --cov=/home/runner/.local/lib/python${{ matrix.python-version }}/site-packages/azure/kusto --cov-report=xml
-        env:
-          APP_ID: ${{ secrets.APP_ID }}
-          APP_KEY: ${{ secrets.APP_KEY }}
-          AUTH_ID: ${{ secrets.AUTH_ID }}
-          ENGINE_CONNECTION_STRING: ${{ secrets.ENGINE_CONNECTION_STRING }}
       - name: EtoE Test with pytest
         env:
           APP_ID: ${{ secrets.APP_ID }}
@@ -53,7 +45,7 @@ jobs:
           APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING: https://ade.applicationinsights.io/subscriptions/12534eb3-8109-4d84-83ad-576c0d5e1d06/resourcegroups/clients_e2e_test/providers/microsoft.insights/components/kusto-e2e-app-insights
           APPLICATION_INSIGHTS_TEST_DATABASE: kusto-e2e-app-insights
         run: |
-          pytest -v ./azure-kusto-data/tests/e2e.py ./azure-kusto-ingest/tests/e2e.py --junit-xml pytest.xml --cov=/home/runner/.local/lib/python${{ matrix.python-version }}/site-packages/azure/kusto --cov-report=xml:coverage2.xml
+          pytest -v . --junit-xml pytest.xml --cov=/home/runner/.local/lib/python${{ matrix.python-version }}/site-packages/azure/kusto --cov-report=xml:coverage2.xml
       - name: Upload Unit Test Results
         if: always()
         uses: actions/upload-artifact@v2

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Fixed exception handling in web requests
 - Internal fixes for environment variables
+- Fixed documentation on E2E tests, and made it possible to test on a clean cluster
 ## [4.2.0] - 2023-04-16
 ### Added
 - Added Initial Catalog (Default Database) parameter to ConnectionStringBuilder

CONTRIBUTING.md

@@ -8,13 +8,41 @@ In order to run E2E tests, you need a Kusto database you have admin rights on.
 A Kusto free cluster is the easiest way to acquire one.
 You can cerate a free Kusto cluster here: https://dataexplorer.azure.com/home
 
+Make sure you set streaming ingestion to enabled in the cluster's configuration:  
+https://learn.microsoft.com/en-us/azure/data-explorer/ingest-data-streaming?tabs=azure-portal%2Ccsharp
+
 You should set then following environment vars where you run E2Es (in IDE run config, shell window, computer, etc).
 ```shell
 ENGINE_CONNECTION_STRING=<Your cluster URI>
-DM_CONNECTION_STRING=<Your ingest cluster URI>
+DM_CONNECTION_STRING=<Your ingest cluster URI> # Optional - if not set, will infer from ENGINE_CONNECTION_STRING
 TEST_DATABASE=<The name of the database>
 ```
 
+The E2E tests authenticate with DefaultAzureCredential, and will fall back to interactive login if needed.
+
+
+For more information on DefaultAzureCredential, see:  
+https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python
+
+To run the optional `token_provider` tests, you will need to set the booleans at the top of the file `test_token_providers.py` and the following environment variables in addition to the previous ones:
+```shell
+# app key provider:
+AZURE_CLIENT_ID=<Your client ID>
+AZURE_CLIENT_SECRET=<Your client secret>
+AZURE_TENANT_ID=<Your tenant ID>
+# certificate provider:
+CERT_THUMBPRINT=<Your certificate thumbprint>
+CERT_PUBLIC_CERT_PATH=<Your certificate public cert path>
+CERT_PEM_KEY_PATH=<Your certificate private key path>
+# managed identity provider:
+MSI_OBJECT_ID=<Your managed identity object ID>
+MSI_CLIENT_ID=<Your managed identity client ID>
+# user password provider:
+USER_NAME=<Your user name>
+USER_PASS=<Your user password>
+USER_AUTH_ID=<Your user auth ID> # optional
+```
+
 ## Requirements
 
 In order to work on this project, we recommend using the dev requirements:

azure-kusto-data/azure/kusto/data/client_base.py

@@ -81,7 +81,8 @@ def validate_endpoint(self):
         if not self._endpoint_validated and self._aad_helper is not None:
             if isinstance(self._aad_helper.token_provider, CloudInfoTokenProvider):
                 well_known_kusto_endpoints.validate_trusted_endpoint(
-                    self._kusto_cluster, CloudSettings.get_cloud_info_for_cluster(self._kusto_cluster).login_endpoint
+                    self._kusto_cluster,
+                    CloudSettings.get_cloud_info_for_cluster(self._kusto_cluster, self._aad_helper.token_provider._proxy_dict).login_endpoint,
                 )
             self._endpoint_validated = True
 

azure-kusto-data/azure/kusto/data/env_utils.py

@@ -1,4 +1,6 @@
 import os
+from dataclasses import dataclass, astuple
+from typing import Optional
 
 
 def get_env(*args, optional=False, default=None):
@@ -19,19 +21,42 @@ def set_env(key, value):
 def get_app_id(optional=False):
     """Return the app id."""
     result = get_env("APP_ID", "AZURE_CLIENT_ID", optional=optional)
-    os.environ["AZURE_CLIENT_ID"] = result
+    if result:
+        set_env("AZURE_CLIENT_ID", result)
     return result
 
 
 def get_auth_id(optional=False):
     """Return the auth id."""
     result = get_env("AUTH_ID", "APP_AUTH_ID", "AZURE_TENANT_ID", optional=optional)
-    os.environ["AZURE_TENANT_ID"] = result
+    if result:
+        set_env("AZURE_TENANT_ID", result)
     return result
 
 
 def get_app_key(optional=False):
     """Return the app key."""
     result = get_env("APP_KEY", "AZURE_CLIENT_SECRET", optional=optional)
-    os.environ["AZURE_CLIENT_SECRET"] = result
+    if result:
+        set_env("AZURE_CLIENT_SECRET", result)
     return result
+
+
+@dataclass(frozen=True)
+class AppKeyAuth:
+    app_id: str
+    app_key: str
+    auth_id: str
+
+    def __iter__(self):
+        return iter(astuple(self))
+
+
+def prepare_app_key_auth(optional=False) -> Optional[AppKeyAuth]:
+    """Gets app key auth information from the env, sets the correct values for azidentity, and returns the AppKeyAuth object."""
+    app_id = get_app_id(optional=optional)
+    app_key = get_app_key(optional=optional)
+    auth_id = get_auth_id(optional=optional)
+    if app_id and app_key and auth_id:
+        return AppKeyAuth(app_id, app_key, auth_id)
+    return None

azure-kusto-data/tests/aio/test_async_token_providers.py

@@ -5,7 +5,7 @@
 
 from azure.kusto.data._decorators import aio_documented_by
 from azure.kusto.data._token_providers import *
-from azure.kusto.data.env_utils import get_env, get_app_id, get_auth_id, get_app_key
+from azure.kusto.data.env_utils import get_env, get_app_id, get_auth_id, get_app_key, prepare_app_key_auth
 from .test_kusto_client import run_aio_tests
 from ..test_token_providers import KUSTO_URI, TOKEN_VALUE, TEST_AZ_AUTH, TEST_MSI_AUTH, TEST_DEVICE_AUTH, TokenProviderTests, MockProvider
 
@@ -142,8 +142,7 @@ async def fail_callback():
     @pytest.mark.asyncio
     async def test_az_provider(self):
         if not TEST_AZ_AUTH:
-            print(" *** Skipped Az-Cli Provider Test ***")
-            return
+            pytest.skip(" *** Skipped Az-Cli Provider Test ***")
 
         print("Note!\nThe test 'test_az_provider' will fail if 'az login' was not called.")
         with AzCliTokenProvider(KUSTO_URI, is_async=True) as provider:
@@ -158,8 +157,7 @@ async def test_az_provider(self):
     @pytest.mark.asyncio
     async def test_msi_provider(self):
         if not TEST_MSI_AUTH:
-            print(" *** Skipped MSI Provider Test ***")
-            return
+            pytest.skip(" *** Skipped MSI Provider Test ***")
 
         user_msi_object_id = get_env("MSI_OBJECT_ID", optional=True)
         user_msi_client_id = get_env("MSI_CLIENT_ID", optional=True)
@@ -175,15 +173,15 @@ async def test_msi_provider(self):
                     token = await provider.get_token_async()
                     assert self.get_token_value(token) is not None
             else:
-                print(" *** Skipped MSI Provider Client Id Test ***")
+                pytest.skip(" *** Skipped MSI Provider Client Id Test ***")
 
             if user_msi_client_id is not None:
                 args = {"client_id": user_msi_client_id}
                 with MsiTokenProvider(KUSTO_URI, args, is_async=True) as provider:
                     token = await provider.get_token_async()
                     assert self.get_token_value(token) is not None
             else:
-                print(" *** Skipped MSI Provider Object Id Test ***")
+                pytest.skip(" *** Skipped MSI Provider Object Id Test ***")
 
     @aio_documented_by(TokenProviderTests.test_user_pass_provider)
     @pytest.mark.asyncio
@@ -201,14 +199,13 @@ async def test_user_pass_provider(self):
                 token = provider._get_token_from_cache_impl()
                 assert self.get_token_value(token) is not None
         else:
-            print(" *** Skipped User & Pass Provider Test ***")
+            pytest.skip(" *** Skipped User & Pass Provider Test ***")
 
     @aio_documented_by(TokenProviderTests.test_device_auth_provider)
     @pytest.mark.asyncio
     async def test_device_auth_provider(self):
         if not TEST_DEVICE_AUTH:
-            print(" *** Skipped User Device Flow Test ***")
-            return
+            pytest.skip(" *** Skipped User Device Flow Test ***")
 
         def callback(x, x2, x3):
             # break here if you debug this test, and get the code from 'x'
@@ -225,18 +222,14 @@ def callback(x, x2, x3):
     @aio_documented_by(TokenProviderTests.test_app_key_provider)
     @pytest.mark.asyncio
     async def test_app_key_provider(self):
-        # default details are for kusto-client-e2e-test-app
-        # to run the test, get the key from Azure portal
-        app_id = get_app_id(optional=True)
-        auth_id = get_auth_id(optional=True)
-        app_key = get_app_key(optional=True)
+        app_auth = prepare_app_key_auth(optional=True)
 
-        if app_id and app_key and auth_id:
-            with ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key, is_async=True) as provider:
+        if app_auth:
+            with ApplicationKeyTokenProvider(KUSTO_URI, app_auth.auth_id, app_auth.app_id, app_auth.app_key, is_async=True) as provider:
                 token = await provider.get_token_async()
                 assert self.get_token_value(token) is not None
         else:
-            print(" *** Skipped App Id & Key Provider Test ***")
+            pytest.skip(" *** Skipped App Id & Key Provider Test ***")
 
     @aio_documented_by(TokenProviderTests.test_app_cert_provider)
     @pytest.mark.asyncio
@@ -269,10 +262,10 @@ async def test_app_cert_provider(self):
                         token = provider._get_token_from_cache_impl()
                         assert self.get_token_value(token) is not None
                 else:
-                    print(" *** Skipped App Cert SNI Provider Test ***")
+                    pytest.skip(" *** Skipped App Cert SNI Provider Test ***")
 
         else:
-            print(" *** Skipped App Cert Provider Test ***")
+            pytest.skip(" *** Skipped App Cert Provider Test ***")
 
     @aio_documented_by(TokenProviderTests.test_cloud_mfa_off)
     @pytest.mark.asyncio
@@ -342,9 +335,10 @@ async def inner():
     @aio_documented_by(TokenProviderTests.test_azure_identity_default_token_provider)
     @pytest.mark.asyncio
     async def test_azure_identity_token_provider(self):
-        app_id = get_app_id()
-        auth_id = get_auth_id()
-        app_key = get_app_key()
+        app_id, app_key, auth_id = prepare_app_key_auth(optional=True)
+        if not app_id or not app_key or not auth_id:
+            pytest.skip(" *** Skipped Azure Identity Provider Test ***")
+
         with AzureIdentityTokenCredentialProvider(KUSTO_URI, is_async=True, credential=AsyncDefaultAzureCredential()) as provider:
             token = await provider.get_token_async()
             assert TokenProviderTests.get_token_value(token) is not None