diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 7a0dca0b2..c4b0b63ba 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -54,7 +54,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] +# python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.12", "3.13"] steps: - uses: actions/checkout@v4 - name: Set up Python @@ -75,15 +76,16 @@ jobs: os: - image: ubuntu-latest id: manylinux_x86_64 - - image: ubuntu-latest - id: manylinux_aarch64 +# - image: ubuntu-latest +# id: manylinux_aarch64 - image: windows-latest id: win_amd64 - - image: macos-latest - id: macosx_x86_64 - - image: macos-latest - id: macosx_arm64 - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] +# - image: macos-latest +# id: macosx_x86_64 +# - image: macos-latest +# id: macosx_arm64 +# python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.12", "3.13"] name: Build ${{ matrix.os.id }}-py${{ matrix.python-version }} runs-on: ${{ matrix.os.image }} steps: @@ -125,23 +127,24 @@ jobs: fail-fast: false matrix: os: - - image_name: ubuntu-latest - download_name: manylinux_x86_64 - - image_name: macos-latest - download_name: macosx_x86_64 +# - image_name: ubuntu-latest +# download_name: manylinux_x86_64 +# - image_name: macos-latest +# download_name: macosx_x86_64 - image_name: windows-latest download_name: win_amd64 - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] +# python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.12", "3.13"] cloud-provider: [aws, azure, gcp] - # TODO: When there are prebuilt wheels accessible for our dependencies (i.e. numpy) + # TODO(SNOW-2043523): When there are prebuilt wheels accessible for our dependencies (i.e. numpy) # for Python 3.13 windows runs can be re-enabled. Currently, according to numpy: # "Numpy built with MINGW-W64 on Windows 64 bits is experimental, and only available for # testing. You are advised not to use it for production." - exclude: - - os: - image_name: windows-latest - download_name: win_amd64 - python-version: "3.13" +# exclude: +# - os: +# image_name: windows-latest +# download_name: win_amd64 +# python-version: "3.13" steps: - uses: actions/checkout@v4 - name: Set up Python @@ -179,7 +182,8 @@ jobs: - name: Install tox run: python -m pip install tox>=4 - name: Run tests - run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,unit,integ,pandas,sso}-ci | sed 's/ /,/g'` +# run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,unit,integ,pandas,sso}-ci | sed 's/ /,/g'` + run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,integ,pandas,sso}-ci | sed 's/ /,/g'` env: PYTHON_VERSION: ${{ matrix.python-version }} cloud_provider: ${{ matrix.cloud-provider }} @@ -311,6 +315,201 @@ jobs: .coverage coverage.xml + +# test: +# name: Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} +# needs: build +# runs-on: ${{ matrix.os.image_name }} +# strategy: +# fail-fast: false +# matrix: +# os: +# - image_name: ubuntu-latest +# download_name: manylinux_x86_64 +# - image_name: macos-latest +# download_name: macosx_x86_64 +# - image_name: windows-latest +# download_name: win_amd64 +# python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] +# cloud-provider: [aws, azure, gcp] +# # TODO: When there are prebuilt wheels accessible for our dependencies (i.e. numpy) +# # for Python 3.13 windows runs can be re-enabled. Currently, according to numpy: +# # "Numpy built with MINGW-W64 on Windows 64 bits is experimental, and only available for +# # testing. You are advised not to use it for production." +# exclude: +# - os: +# image_name: windows-latest +# download_name: win_amd64 +# python-version: "3.13" +# steps: +# - uses: actions/checkout@v4 +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Display Python version +# run: python -c "import sys; print(sys.version)" +# - name: Set up Java +# uses: actions/setup-java@v4 # for wiremock +# with: +# java-version: 11 +# distribution: 'temurin' +# java-package: 'jre' +# - name: Fetch Wiremock +# shell: bash +# run: curl https://repo1.maven.org/maven2/org/wiremock/wiremock-standalone/3.11.0/wiremock-standalone-3.11.0.jar --output .wiremock/wiremock-standalone.jar +# - name: Setup parameters file +# shell: bash +# env: +# PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} +# run: | +# gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ +# .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py +# - name: Download wheel(s) +# uses: actions/download-artifact@v4 +# with: +# name: ${{ matrix.os.download_name }}_py${{ matrix.python-version }} +# path: dist +# - name: Show wheels downloaded +# run: ls -lh dist +# shell: bash +# - name: Upgrade setuptools, pip and wheel +# run: python -m pip install -U setuptools pip wheel +# - name: Install tox +# run: python -m pip install tox>=4 +# - name: Run tests +# run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,unit,integ,pandas,sso}-ci | sed 's/ /,/g'` +# env: +# PYTHON_VERSION: ${{ matrix.python-version }} +# cloud_provider: ${{ matrix.cloud-provider }} +# PYTEST_ADDOPTS: --color=yes --tb=short +# TOX_PARALLEL_NO_SPINNER: 1 +# shell: bash +# - name: Combine coverages +# run: python -m tox run -e coverage --skip-missing-interpreters false +# shell: bash +# - uses: actions/upload-artifact@v4 +# with: +# include-hidden-files: true +# name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} +# path: | +# .tox/.coverage +# .tox/coverage.xml +# +# test-olddriver: +# name: Old Driver Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} +# needs: lint +# runs-on: ${{ matrix.os.image_name }} +# strategy: +# fail-fast: false +# matrix: +# os: +# # Because old the version 3.0.2 of snowflake-connector-python depends on oscrypto which causes conflicts with higher versions of libssl +# # TODO: It can be changed to ubuntu-latest, when python sf connector version in tox is above 3.4.0 +# - image_name: ubuntu-22.04 +# download_name: linux +# python-version: [3.9] +# cloud-provider: [aws] +# steps: +# - uses: actions/checkout@v4 +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Display Python version +# run: python -c "import sys; print(sys.version)" +# - name: Setup parameters file +# shell: bash +# env: +# PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} +# run: | +# gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ +# .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py +# - name: Upgrade setuptools, pip and wheel +# run: python -m pip install -U setuptools pip wheel +# - name: Install tox +# run: python -m pip install tox>=4 +# - name: Run tests +# run: python -m tox run -e olddriver +# env: +# PYTHON_VERSION: ${{ matrix.python-version }} +# cloud_provider: ${{ matrix.cloud-provider }} +# PYTEST_ADDOPTS: --color=yes --tb=short +# shell: bash +# +# test-noarrowextension: +# name: No Arrow Extension Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} +# needs: lint +# runs-on: ${{ matrix.os.image_name }} +# strategy: +# fail-fast: false +# matrix: +# os: +# - image_name: ubuntu-latest +# download_name: linux +# python-version: [3.9] +# cloud-provider: [aws] +# steps: +# - uses: actions/checkout@v4 +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Display Python version +# run: python -c "import sys; print(sys.version)" +# - name: Upgrade setuptools, pip and wheel +# run: python -m pip install -U setuptools pip wheel +# - name: Install tox +# run: python -m pip install tox>=4 +# - name: Run tests +# run: python -m tox run -e noarrowextension +# env: +# PYTHON_VERSION: ${{ matrix.python-version }} +# cloud_provider: ${{ matrix.cloud-provider }} +# PYTEST_ADDOPTS: --color=yes --tb=short +# shell: bash +# +# test-fips: +# name: Test FIPS linux-3.9-${{ matrix.cloud-provider }} +# needs: build +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# cloud-provider: [aws] +# steps: +# - uses: actions/checkout@v4 +# - name: Setup parameters file +# shell: bash +# env: +# PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} +# run: | +# gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ +# .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py +# - name: Download wheel(s) +# uses: actions/download-artifact@v4 +# with: +# name: manylinux_x86_64_py3.9 +# path: dist +# - name: Show wheels downloaded +# run: ls -lh dist +# shell: bash +# - name: Run tests +# run: ./ci/test_fips_docker.sh +# env: +# PYTHON_VERSION: 3.9 +# cloud_provider: ${{ matrix.cloud-provider }} +# PYTEST_ADDOPTS: --color=yes --tb=short +# TOX_PARALLEL_NO_SPINNER: 1 +# shell: bash +# - uses: actions/upload-artifact@v4 +# with: +# include-hidden-files: true +# name: coverage_linux-fips-3.9-${{ matrix.cloud-provider }} +# path: | +# .coverage +# coverage.xml + test-lambda: name: Test Lambda linux-${{ matrix.python-version }}-${{ matrix.cloud-provider }} needs: build @@ -358,6 +557,110 @@ jobs: .coverage.py${{ env.shortver }}-lambda-ci junit.py${{ env.shortver }}-lambda-ci-dev.xml +# combine-coverage: +# if: ${{ success() || failure() }} +# name: Combine coverage +# needs: [lint, test, test-fips, test-lambda] +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/download-artifact@v4 +# with: +# path: artifacts +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: '3.9' +# - name: Display Python version +# run: python -c "import sys; print(sys.version)" +# - name: Upgrade setuptools and pip +# run: python -m pip install -U setuptools pip wheel +# - name: Install tox +# run: python -m pip install tox>=4 +# - name: Collect all coverages to one dir +# run: | +# python -c ' +# from pathlib import Path +# import shutil +# +# src_dir = Path("artifacts") +# dst_dir = Path(".") / ".tox" +# dst_dir.mkdir() +# for src_file in src_dir.glob("*/.coverage"): +# dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:]) +# print("{} copy to {}".format(src_file, dst_file)) +# shutil.copy(str(src_file), str(dst_file))' +# - name: Combine coverages +# run: python -m tox run -e coverage +# - name: Publish html coverage +# uses: actions/upload-artifact@v4 +# with: +# include-hidden-files: true +# name: overall_cov_html +# path: .tox/htmlcov +# - name: Publish xml coverage +# uses: actions/upload-artifact@v4 +# with: +# include-hidden-files: true +# name: overall_cov_xml +# path: .tox/coverage.xml +# - uses: codecov/codecov-action@v4 +# with: +# files: .tox/coverage.xml +# token: ${{ secrets.CODECOV_TOKEN }} + +# combine-coverage: +# if: ${{ success() || failure() }} +# name: Combine coverage +# needs: [lint, test, test-fips, test-lambda] +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/download-artifact@v4 +# with: +# path: artifacts +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: '3.9' +# - name: Display Python version +# run: python -c "import sys; print(sys.version)" +# - name: Upgrade setuptools and pip +# run: python -m pip install -U setuptools pip wheel +# - name: Install tox +# run: python -m pip install tox>=4 +# - name: Collect all coverages to one dir +# run: | +# python -c ' +# from pathlib import Path +# import shutil +# +# src_dir = Path("artifacts") +# dst_dir = Path(".") / ".tox" +# dst_dir.mkdir() +# for src_file in src_dir.glob("*/.coverage"): +# dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:]) +# print("{} copy to {}".format(src_file, dst_file)) +# shutil.copy(str(src_file), str(dst_file))' +# - name: Combine coverages +# run: python -m tox run -e coverage +# - name: Publish html coverage +# uses: actions/upload-artifact@v4 +# with: +# include-hidden-files: true +# name: overall_cov_html +# path: .tox/htmlcov +# - name: Publish xml coverage +# uses: actions/upload-artifact@v4 +# with: +# include-hidden-files: true +# name: overall_cov_xml +# path: .tox/coverage.xml +# - uses: codecov/codecov-action@v4 +# with: +# files: .tox/coverage.xml +# token: ${{ secrets.CODECOV_TOKEN }} + combine-coverage: if: ${{ success() || failure() }} name: Combine coverage diff --git a/DESCRIPTION.md b/DESCRIPTION.md index 6fa0ef15d..17072cc4d 100644 --- a/DESCRIPTION.md +++ b/DESCRIPTION.md @@ -21,6 +21,8 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne - Added the `oauth_client_id`, `oauth_client_secret`, `oauth_token_request_url`, and `oauth_scope` parameters. - Added the `OAUTH_CLIENT_CREDENTIALS` value for the parameter authenticator. - For OAuth Token caching: Passing a username to driver configuration is required, and the `client_store_temporary_credential property` is to be set to `true`. + - Bumped numpy dependency from <2.1.0 to <2.2.4 + - Added Windows support for Python 3.13. - v3.14.1(April 21, 2025) - Added support for Python 3.13. diff --git a/ci/docker/connector_test/Dockerfile b/ci/docker/connector_test/Dockerfile index d90705038..1c8ca2edc 100644 --- a/ci/docker/connector_test/Dockerfile +++ b/ci/docker/connector_test/Dockerfile @@ -3,8 +3,7 @@ FROM $BASE_IMAGE RUN yum install -y java-11-openjdk -# TODO: When there are prebuilt wheels accessible for our dependencies (i.e. numpy) -# for Python 3.13 this rust cargo install command can be removed. +# Our dependencies rely on the Rust toolchain being available in the build-time environment RUN yum -y install rust cargo # This is to solve permission issue, read https://denibertovic.com/posts/handling-permissions-with-docker-volumes/ diff --git a/ci/docker/connector_test_lambda/Dockerfile313 b/ci/docker/connector_test_lambda/Dockerfile313 index 9b8d8d0f9..f761ef694 100644 --- a/ci/docker/connector_test_lambda/Dockerfile313 +++ b/ci/docker/connector_test_lambda/Dockerfile313 @@ -2,7 +2,8 @@ FROM public.ecr.aws/lambda/python:3.13-x86_64 WORKDIR /home/user/snowflake-connector-python -# TODO: When there are prebuilt wheels accessible for our dependencies (i.e. numpy) + +# TODO(SNOW-2043523): When there are prebuilt wheels accessible for our dependencies (i.e. numpy) # for Python 3.13 all dnf ... commands installing building kits can be removed. # Install necessary packages and compilers - we need to build numpy for newer version @@ -16,6 +17,7 @@ RUN dnf -y update && \ openblas-devel \ lapack-devel && \ dnf clean all +# Our dependencies rely on the Rust toolchain being available in the build-time environment RUN dnf -y install rust cargo RUN dnf -y upgrade diff --git a/setup.cfg b/setup.cfg index 2fecc08f4..2fd14a0ea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -83,7 +83,7 @@ development = Cython coverage more-itertools - numpy<2.1.0 + numpy<=2.2.4 pendulum!=2.1.1 pexpect pytest<7.5.0 diff --git a/src/snowflake/connector/file_transfer_agent.py b/src/snowflake/connector/file_transfer_agent.py index b3cdc5f9c..92d714fa7 100644 --- a/src/snowflake/connector/file_transfer_agent.py +++ b/src/snowflake/connector/file_transfer_agent.py @@ -56,6 +56,10 @@ from .s3_storage_client import SnowflakeS3RestClient from .storage_client import SnowflakeFileEncryptionMaterial, SnowflakeStorageClient +# from urllib.parse import urlparse +# from urllib.request import url2pathname + + if TYPE_CHECKING: # pragma: no cover from .connection import SnowflakeConnection from .cursor import SnowflakeCursor @@ -834,13 +838,50 @@ def result(self) -> dict[str, Any]: "rowset": sorted(rowset), } + # def _expand_filenames(self, locations: list[str]) -> list[str]: + # canonical_locations = [] + # for file_name in locations: + # if self._command_type == CMD_TYPE_UPLOAD: + # # TODO: Since python 3.13 os.path.isabs returns other values for URI ... comment to be finished + # parsed = urlparse(file_name) + # if parsed.scheme == "file": + # file_name = url2pathname(parsed.path) + # + # file_name = os.path.expanduser(file_name) + # if ( + # IS_WINDOWS + # and len(file_name) > 2 + # and file_name[0] == "/" + # and file_name[2] == ":" + # ): + # # Windows path: /C:/data/file1.txt where it starts with slash + # # followed by a drive letter and colon. + # file_name = file_name[1:] + # + # if not os.path.isabs(file_name): + # file_name = os.path.join(GET_CWD(), file_name) + # + # files = glob.glob(file_name) + # canonical_locations += files + # else: + # canonical_locations.append(file_name) + # + # return canonical_locations + def _expand_filenames(self, locations: list[str]) -> list[str]: canonical_locations = [] for file_name in locations: if self._command_type == CMD_TYPE_UPLOAD: + # # TODO: Since python 3.13 os.path.isabs returns other values for URI ... comment to be finished + # parsed = urlparse(file_name) + # if parsed.scheme == "file": + # file_name = url2pathname(parsed.path) + file_name = os.path.expanduser(file_name) + if not os.path.isabs(file_name): file_name = os.path.join(GET_CWD(), file_name) + if ( IS_WINDOWS and len(file_name) > 2 @@ -850,6 +891,7 @@ def _expand_filenames(self, locations: list[str]) -> list[str]: # Windows path: /C:/data/file1.txt where it starts with slash # followed by a drive letter and colon. file_name = file_name[1:] + files = glob.glob(file_name) canonical_locations += files else: diff --git a/src/snowflake/connector/log_configuration.py b/src/snowflake/connector/log_configuration.py index 476ab8961..1060af501 100644 --- a/src/snowflake/connector/log_configuration.py +++ b/src/snowflake/connector/log_configuration.py @@ -26,6 +26,8 @@ def parse_config_file(self): self.level = log.get("level", "INFO") self.path = log.get("path", os.path.join(DIRS.user_config_path, "logs")) + # TODO: wont work in 3.13? + # if not self.path.scheme and self.path.netloc and self.path.path.startswith("/"): if not os.path.isabs(self.path): raise FileNotFoundError( f"Log path must be an absolute file path: {self.path}" diff --git a/test/integ/test_put_windows_path.py b/test/integ/test_put_windows_path.py index f12b0d1a3..d24694122 100644 --- a/test/integ/test_put_windows_path.py +++ b/test/integ/test_put_windows_path.py @@ -14,6 +14,8 @@ def test_abc(conn_cnx, tmpdir, db_parameters): f.write("test1,test2") f.write("test3,test4") + assert os.path.exists(test_data), f"Test data {test_data} does not exist." + fileURI = pathlib.Path(test_data).as_uri() subdir = db_parameters["name"]