diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..09e3734
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+global-exclude *.py[cod] __pycache__ .DS_Store
+recursive-include deps/jars *.jar
+include README.md
+include LICENSE
+include NOTICE
diff --git a/dev/lint-python.sh b/dev/lint-python.sh
index 15a009e..2c7d440 100755
--- a/dev/lint-python.sh
+++ b/dev/lint-python.sh
@@ -577,9 +577,6 @@ function tox_check() {
# Ensure the permission of the scripts set correctly
chmod +x $PAIMON_PYTHON_DIR/dev/*
- # tox runs codes in virtual env, set var to avoid error
- export _PYPAIMON_TOX_TEST="true"
-
if [[ -n "$GITHUB_ACTION" ]]; then
# Run tests in all versions triggered by a Git push (tests aren't so many currently)
$TOX_PATH -vv -c $PAIMON_PYTHON_DIR/tox.ini --recreate 2>&1 | tee -a $LOG_FILE
diff --git a/pypaimon/py4j/paimon-python-java-bridge/copyright.txt b/paimon-python-java-bridge/copyright.txt
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/copyright.txt
rename to paimon-python-java-bridge/copyright.txt
diff --git a/pypaimon/py4j/paimon-python-java-bridge/pom.xml b/paimon-python-java-bridge/pom.xml
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/pom.xml
rename to paimon-python-java-bridge/pom.xml
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java b/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java
rename to paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java b/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java
rename to paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java b/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java
rename to paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java b/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java
rename to paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java b/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java
rename to paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PredicationUtil.java b/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PredicationUtil.java
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PredicationUtil.java
rename to paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PredicationUtil.java
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java b/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java
rename to paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java b/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java
rename to paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java b/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java
rename to paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/SchemaUtil.java b/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/SchemaUtil.java
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/SchemaUtil.java
rename to paimon-python-java-bridge/src/main/java/org/apache/paimon/python/SchemaUtil.java
diff --git a/pypaimon/py4j/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE b/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE
rename to paimon-python-java-bridge/src/main/resources/META-INF/NOTICE
diff --git a/pypaimon/py4j/paimon-python-java-bridge/tools/ci/log4j.properties b/paimon-python-java-bridge/tools/ci/log4j.properties
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/tools/ci/log4j.properties
rename to paimon-python-java-bridge/tools/ci/log4j.properties
diff --git a/pypaimon/py4j/paimon-python-java-bridge/tools/maven/checkstyle.xml b/paimon-python-java-bridge/tools/maven/checkstyle.xml
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/tools/maven/checkstyle.xml
rename to paimon-python-java-bridge/tools/maven/checkstyle.xml
diff --git a/pypaimon/py4j/paimon-python-java-bridge/tools/maven/suppressions.xml b/paimon-python-java-bridge/tools/maven/suppressions.xml
similarity index 100%
rename from pypaimon/py4j/paimon-python-java-bridge/tools/maven/suppressions.xml
rename to paimon-python-java-bridge/tools/maven/suppressions.xml
diff --git a/pypaimon/py4j/gateway_server.py b/pypaimon/py4j/gateway_server.py
index 9a259e0..f3a0fda 100644
--- a/pypaimon/py4j/gateway_server.py
+++ b/pypaimon/py4j/gateway_server.py
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-import importlib
+import importlib.resources
import os
import platform
import signal
@@ -74,17 +74,21 @@ def preexec_func():
stdin=PIPE, stderr=PIPE, preexec_fn=preexec_fn, env=env)
-_JAVA_IMPL_MODULE = 'pypaimon.py4j'
-_JAVA_DEPS = 'java_dependencies'
-_JAVA_BRIDGE = 'paimon-python-java-bridge'
+_JAVA_DEPS_PACKAGE = 'pypaimon.jars'
def _get_classpath(env):
classpath = []
- module = importlib.import_module(_JAVA_IMPL_MODULE)
- builtin_java_bridge = os.path.join(*module.__path__, _JAVA_DEPS, _JAVA_BRIDGE + '.jar')
- classpath.append(builtin_java_bridge)
+ # note that jars are not packaged in test
+ test_mode = os.environ.get(constants.PYPAIMON4J_TEST_MODE)
+ if not test_mode or test_mode.lower() != "true":
+ jars = importlib.resources.files(_JAVA_DEPS_PACKAGE)
+ one_jar = next(iter(jars.iterdir()), None)
+ if not one_jar:
+ raise ValueError("Haven't found necessary python-java-bridge jar, this is unexpected.")
+ builtin_java_classpath = os.path.join(os.path.dirname(str(one_jar)), '*')
+ classpath.append(builtin_java_classpath)
# user defined
if constants.PYPAIMON_JAVA_CLASSPATH in env:
diff --git a/pypaimon/py4j/tests/__init__.py b/pypaimon/py4j/tests/__init__.py
index 65b48d4..a9e2b63 100644
--- a/pypaimon/py4j/tests/__init__.py
+++ b/pypaimon/py4j/tests/__init__.py
@@ -15,3 +15,88 @@
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
+
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+import urllib.request
+
+from pypaimon.py4j import constants, Catalog
+from xml.etree import ElementTree
+
+
+def _setup_hadoop_bundle_jar(hadoop_dir):
+ url = 'https://repo.maven.apache.org/maven2/org/apache/flink/' \
+ 'flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar'
+
+ response = urllib.request.urlopen(url)
+ if not os.path.exists(hadoop_dir):
+ os.mkdir(hadoop_dir)
+
+ jar_path = os.path.join(hadoop_dir, "bundled-hadoop.jar")
+ with open(jar_path, 'wb') as file:
+ file.write(response.read())
+
+ os.environ[constants.PYPAIMON_HADOOP_CLASSPATH] = jar_path
+
+
+def _setup_bridge_jar(bridge_dir):
+ java_bridge_module = _find_java_bridge_module()
+ subprocess.run(
+ ["mvn", "clean", "package"],
+ cwd=java_bridge_module,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+
+ if not os.path.exists(bridge_dir):
+ os.mkdir(bridge_dir)
+
+ jar_path = os.path.join(bridge_dir, "paimon-python-java-bridge.jar")
+ shutil.copy(
+ os.path.join(java_bridge_module, 'target/{}-{}.jar'
+ .format('paimon-python-java-bridge', _extract_bridge_version())),
+ jar_path
+ )
+
+ os.environ[constants.PYPAIMON_JAVA_CLASSPATH] = jar_path
+
+
+def _extract_bridge_version():
+ pom_path = os.path.join(_find_java_bridge_module(), 'pom.xml')
+ return ElementTree.parse(pom_path).getroot().find(
+ 'POM:version',
+ namespaces={
+ 'POM': 'http://maven.apache.org/POM/4.0.0'
+ }).text
+
+
+def _find_java_bridge_module():
+ this_dir = os.path.abspath(os.path.dirname(__file__))
+ project_dir = os.path.dirname(os.path.dirname(os.path.dirname(this_dir)))
+ return os.path.join(project_dir, "paimon-python-java-bridge")
+
+
+class PypaimonTestBase(unittest.TestCase):
+ """
+ Base class for unit tests.
+ """
+
+ @classmethod
+ def setUpClass(cls):
+ os.environ[constants.PYPAIMON4J_TEST_MODE] = 'true'
+ cls.tempdir = tempfile.mkdtemp()
+
+ _setup_hadoop_bundle_jar(cls.tempdir)
+ _setup_bridge_jar(cls.tempdir)
+
+ cls.warehouse = os.path.join(cls.tempdir, 'warehouse')
+ cls.catalog = Catalog.create({'warehouse': cls.warehouse})
+ cls.catalog.create_database('default', False)
+
+ @classmethod
+ def tearDownClass(cls):
+ shutil.rmtree(cls.tempdir, ignore_errors=True)
+ del os.environ[constants.PYPAIMON4J_TEST_MODE]
diff --git a/pypaimon/py4j/tests/test_data_types.py b/pypaimon/py4j/tests/test_data_types.py
index 5fb809e..b0d0e41 100644
--- a/pypaimon/py4j/tests/test_data_types.py
+++ b/pypaimon/py4j/tests/test_data_types.py
@@ -16,43 +16,24 @@
# limitations under the License.
################################################################################
-import os
import random
-import shutil
import string
-import tempfile
import pyarrow as pa
-import unittest
from pypaimon import Schema
-from pypaimon.py4j import Catalog
-from pypaimon.py4j.tests import utils
+from pypaimon.py4j.tests import PypaimonTestBase
from pypaimon.py4j.util import java_utils
-from setup_utils import java_setuputils
-class DataTypesTest(unittest.TestCase):
+class DataTypesTest(PypaimonTestBase):
@classmethod
def setUpClass(cls):
- java_setuputils.setup_java_bridge()
- cls.hadoop_path = tempfile.mkdtemp()
- utils.setup_hadoop_bundle_jar(cls.hadoop_path)
- cls.warehouse = tempfile.mkdtemp()
+ super().setUpClass()
cls.simple_pa_schema = pa.schema([
('f0', pa.int32()),
('f1', pa.string())
])
- cls.catalog = Catalog.create({'warehouse': cls.warehouse})
- cls.catalog.create_database('default', False)
-
- @classmethod
- def tearDownClass(cls):
- java_setuputils.clean()
- if os.path.exists(cls.hadoop_path):
- shutil.rmtree(cls.hadoop_path)
- if os.path.exists(cls.warehouse):
- shutil.rmtree(cls.warehouse)
def test_int(self):
pa_schema = pa.schema([
diff --git a/pypaimon/py4j/tests/test_preicates.py b/pypaimon/py4j/tests/test_preicates.py
index 5b63759..f538c93 100644
--- a/pypaimon/py4j/tests/test_preicates.py
+++ b/pypaimon/py4j/tests/test_preicates.py
@@ -16,18 +16,12 @@
# limitations under the License.
################################################################################
-import os
-import shutil
-import tempfile
-import unittest
import random
import pandas as pd
import pyarrow as pa
from pypaimon import Schema
-from pypaimon.py4j import Catalog
-from pypaimon.py4j.tests import utils
-from setup_utils import java_setuputils
+from pypaimon.py4j.tests import PypaimonTestBase
def _check_filtered_result(read_builder, expected_df):
@@ -38,41 +32,34 @@ def _check_filtered_result(read_builder, expected_df):
actual_df.reset_index(drop=True), expected_df.reset_index(drop=True))
-# TODO: parquet has bug now
+# TODO: Parquet has bug now. Fixed in 1.0.
def _random_format():
return random.choice(['avro', 'orc'])
-class PredicateTest(unittest.TestCase):
+class PredicateTest(PypaimonTestBase):
@classmethod
def setUpClass(cls):
- java_setuputils.setup_java_bridge()
- cls.hadoop_path = tempfile.mkdtemp()
- utils.setup_hadoop_bundle_jar(cls.hadoop_path)
- cls.warehouse = tempfile.mkdtemp()
-
- catalog = Catalog.create({'warehouse': cls.warehouse})
- catalog.create_database('default', False)
-
+ super().setUpClass()
pa_schema = pa.schema([
('f0', pa.int64()),
('f1', pa.string()),
])
- catalog.create_table('default.test_append',
- Schema(pa_schema, options={'file.format': _random_format()}),
- False)
- catalog.create_table('default.test_pk',
- Schema(pa_schema, primary_keys=['f0'],
- options={'bucket': '1', 'file.format': _random_format()}),
- False)
+ cls.catalog.create_table('default.test_append',
+ Schema(pa_schema, options={'file.format': _random_format()}),
+ False)
+ cls.catalog.create_table('default.test_pk',
+ Schema(pa_schema, primary_keys=['f0'],
+ options={'bucket': '1', 'file.format': _random_format()}),
+ False)
df = pd.DataFrame({
'f0': [1, 2, 3, 4, 5],
'f1': ['abc', 'abbc', 'bc', 'd', None],
})
- append_table = catalog.get_table('default.test_append')
+ append_table = cls.catalog.get_table('default.test_append')
write_builder = append_table.new_batch_write_builder()
write = write_builder.new_write()
commit = write_builder.new_commit()
@@ -81,7 +68,7 @@ def setUpClass(cls):
write.close()
commit.close()
- pk_table = catalog.get_table('default.test_pk')
+ pk_table = cls.catalog.get_table('default.test_pk')
write_builder = pk_table.new_batch_write_builder()
write = write_builder.new_write()
commit = write_builder.new_commit()
@@ -90,17 +77,8 @@ def setUpClass(cls):
write.close()
commit.close()
- cls.catalog = catalog
cls.df = df
- @classmethod
- def tearDownClass(cls):
- java_setuputils.clean()
- if os.path.exists(cls.hadoop_path):
- shutil.rmtree(cls.hadoop_path)
- if os.path.exists(cls.warehouse):
- shutil.rmtree(cls.warehouse)
-
def testWrongFieldName(self):
table = self.catalog.get_table('default.test_append')
predicate_builder = table.new_read_builder().new_predicate_builder()
diff --git a/pypaimon/py4j/tests/test_write_and_read.py b/pypaimon/py4j/tests/test_write_and_read.py
index 27528d1..9b3e557 100644
--- a/pypaimon/py4j/tests/test_write_and_read.py
+++ b/pypaimon/py4j/tests/test_write_and_read.py
@@ -16,10 +16,6 @@
# limitations under the License.
################################################################################
-import os
-import shutil
-import tempfile
-import unittest
import pandas as pd
import pyarrow as pa
from py4j.protocol import Py4JJavaError
@@ -27,33 +23,20 @@
from pypaimon import Schema
from pypaimon.py4j import Catalog
from pypaimon.py4j.java_gateway import get_gateway
-from pypaimon.py4j.tests import utils
+from pypaimon.py4j.tests import PypaimonTestBase
from pypaimon.py4j.util import java_utils
-from setup_utils import java_setuputils
-class TableWriteReadTest(unittest.TestCase):
+class TableWriteReadTest(PypaimonTestBase):
@classmethod
def setUpClass(cls):
- java_setuputils.setup_java_bridge()
- cls.hadoop_path = tempfile.mkdtemp()
- utils.setup_hadoop_bundle_jar(cls.hadoop_path)
- cls.warehouse = tempfile.mkdtemp()
+ super().setUpClass()
cls.simple_pa_schema = pa.schema([
('f0', pa.int32()),
('f1', pa.string())
])
- cls.catalog = Catalog.create({'warehouse': cls.warehouse})
- cls.catalog.create_database('default', False)
- @classmethod
- def tearDownClass(cls):
- java_setuputils.clean()
- if os.path.exists(cls.hadoop_path):
- shutil.rmtree(cls.hadoop_path)
- if os.path.exists(cls.warehouse):
- shutil.rmtree(cls.warehouse)
def testReadEmptyAppendTable(self):
schema = Schema(self.simple_pa_schema)
diff --git a/pypaimon/py4j/util/constants.py b/pypaimon/py4j/util/constants.py
index 1039de2..f223309 100644
--- a/pypaimon/py4j/util/constants.py
+++ b/pypaimon/py4j/util/constants.py
@@ -26,3 +26,6 @@
# ------------------------ for catalog options ------------------------
MAX_WORKERS = "max-workers"
+
+# ------------------ for tests (Please don't use it) ------------------
+PYPAIMON4J_TEST_MODE = '_PYPAIMON4J_TEST_MODE'
diff --git a/setup_utils/version.py b/pypaimon/version.py
similarity index 100%
rename from setup_utils/version.py
rename to pypaimon/version.py
diff --git a/setup_utils/__init__.py b/setup.cfg
similarity index 93%
rename from setup_utils/__init__.py
rename to setup.cfg
index 27dc0ac..ecc3dda 100644
--- a/setup_utils/__init__.py
+++ b/setup.cfg
@@ -16,4 +16,5 @@
# limitations under the License.
################################################################################
-"""This module only contains utils for setup and won't be packaged."""
+[bdist_wheel]
+universal = 1
diff --git a/setup.py b/setup.py
index 628a6b8..0e632f2 100644
--- a/setup.py
+++ b/setup.py
@@ -16,84 +16,65 @@
# limitations under the License.
################################################################################
-import fnmatch
import os
-import shutil
-import setup_utils.java_setuputils as java_setuputils
-import setup_utils.version
+import sys
-from setuptools import Command, setup
-
-
-class CleanCommand(Command):
- description = 'Clean up temporary files and directories of last build.'
- user_options = []
-
- def initialize_options(self):
- pass
-
- def finalize_options(self):
- pass
-
- def run(self):
- directories_to_delete = ['build', 'dist', '*.egg-info']
-
- for directory in directories_to_delete:
- if '*' in directory:
- for matched_dir in filter(lambda x: fnmatch.fnmatch(x, directory), os.listdir('.')):
- if os.path.isdir(matched_dir):
- shutil.rmtree(matched_dir)
- else:
- if os.path.exists(directory):
- shutil.rmtree(directory)
+from setuptools import setup
+this_directory = os.path.abspath(os.path.dirname(__file__))
+version_file = os.path.join(this_directory, 'pypaimon/version.py')
try:
- PACKAGES = [
- 'pypaimon',
- 'pypaimon.api',
- 'pypaimon.py4j',
- 'pypaimon.py4j.util'
- ]
+ exec(open(version_file).read())
+except IOError:
+ print("Failed to load PyPaimon version file for packaging. " +
+ "'%s' not found!" % version_file,
+ file=sys.stderr)
+ sys.exit(-1)
+VERSION = __version__ # noqa
- PACKAGE_DATA = {
- 'pypaimon.py4j': java_setuputils.get_package_data()
- }
+PACKAGES = [
+ 'pypaimon',
+ 'pypaimon.api',
+ 'pypaimon.py4j',
+ 'pypaimon.py4j.util',
+ 'pypaimon.jars'
+]
- install_requires = [
- 'py4j==0.10.9.7',
- 'python-dateutil>=2.8.0,<3',
- 'pytz>=2018.3',
- 'numpy>=1.22.4',
- 'pandas>=1.3.0',
- 'pyarrow>=5.0.0'
- ]
+install_requires = [
+ 'py4j==0.10.9.7',
+ 'pandas>=1.3.0',
+ 'pyarrow>=5.0.0'
+]
- long_description = 'See Apache Paimon Python API \
+long_description = 'See Apache Paimon Python API \
[Doc](https://paimon.apache.org/docs/master/program-api/python-api/) for usage.'
- setup(
- name='pypaimon',
- version=setup_utils.version.__version__,
- packages=PACKAGES,
- include_package_data=True,
- package_data=PACKAGE_DATA,
- cmdclass={'clean': CleanCommand},
- install_requires=install_requires,
- description='Apache Paimon Python API',
- long_description=long_description,
- long_description_content_type='text/markdown',
- author='Apache Software Foundation',
- author_email='dev@paimon.apache.org',
- url='https://paimon.apache.org',
- classifiers=[
- 'Development Status :: 4 - Beta',
- 'License :: OSI Approved :: Apache Software License',
- 'Programming Language :: Python :: 3.8',
- 'Programming Language :: Python :: 3.9',
- 'Programming Language :: Python :: 3.10',
- 'Programming Language :: Python :: 3.11'],
- python_requires='>=3.8'
- )
-finally:
- java_setuputils.clean()
+setup(
+ name='pypaimon',
+ version=VERSION,
+ packages=PACKAGES,
+ include_package_data=True,
+ # releasing tool will generate deps
+ package_dir={
+ "pypaimon.jars": "deps/jars"
+ },
+ package_data={
+ "pypaimon.jars": ["*.jar"]
+ },
+ install_requires=install_requires,
+ description='Apache Paimon Python API',
+ long_description=long_description,
+ long_description_content_type='text/markdown',
+ author='Apache Software Foundation',
+ author_email='dev@paimon.apache.org',
+ url='https://paimon.apache.org',
+ classifiers=[
+ 'Development Status :: 4 - Beta',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Programming Language :: Python :: 3.8',
+ 'Programming Language :: Python :: 3.9',
+ 'Programming Language :: Python :: 3.10',
+ 'Programming Language :: Python :: 3.11'],
+ python_requires='>=3.8'
+)
diff --git a/setup_utils/java_setuputils.py b/setup_utils/java_setuputils.py
deleted file mode 100755
index 01b02e8..0000000
--- a/setup_utils/java_setuputils.py
+++ /dev/null
@@ -1,85 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-import os
-import shutil
-import subprocess
-
-from xml.etree import ElementTree
-
-_JAVA_IMPL_MODULE = 'pypaimon/py4j'
-_JAVA_DEPS = 'java_dependencies'
-_JAVA_BRIDGE = 'paimon-python-java-bridge'
-
-_PYPAIMON_TOX_TEST = '_PYPAIMON_TOX_TEST'
-
-
-def get_package_data():
- is_tox_test = os.environ.get(_PYPAIMON_TOX_TEST)
- if is_tox_test and is_tox_test.lower() == "true":
- return ['']
-
- setup_java_bridge()
- return [os.path.join(_JAVA_DEPS, '*')]
-
-
-def clean():
- java_deps_dir = os.path.join(_find_java_impl_dir(), _JAVA_DEPS)
- if os.path.exists(java_deps_dir):
- shutil.rmtree(java_deps_dir)
-
-
-def setup_java_bridge():
- java_impl_dir = _find_java_impl_dir()
-
- java_deps_dir = os.path.join(java_impl_dir, _JAVA_DEPS)
- if not os.path.exists(java_deps_dir):
- os.mkdir(java_deps_dir)
-
- java_bridge_dst = os.path.join(java_deps_dir, _JAVA_BRIDGE + '.jar')
- if os.path.exists(java_bridge_dst):
- return
-
- java_bridge_module = os.path.join(java_impl_dir, _JAVA_BRIDGE)
- subprocess.run(
- ["mvn", "clean", "package"],
- cwd=java_bridge_module,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
-
- shutil.copy(
- os.path.join(java_bridge_module, 'target/{}-{}.jar'
- .format(_JAVA_BRIDGE, _extract_bridge_version())),
- java_bridge_dst
- )
-
-
-def _extract_bridge_version():
- pom_path = os.path.join(_find_java_impl_dir(), _JAVA_BRIDGE, 'pom.xml')
- return ElementTree.parse(pom_path).getroot().find(
- 'POM:version',
- namespaces={
- 'POM': 'http://maven.apache.org/POM/4.0.0'
- }).text
-
-
-def _find_java_impl_dir():
- this_dir = os.path.abspath(os.path.dirname(__file__))
- paimon_python_dir = os.path.dirname(this_dir)
- return os.path.join(paimon_python_dir, _JAVA_IMPL_MODULE)
diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh
deleted file mode 100755
index 8188fb4..0000000
--- a/tools/releasing/create_binary_release.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-##
-## Required variables
-##
-RELEASE_VERSION=${RELEASE_VERSION}
-
-if [ -z "${RELEASE_VERSION}" ]; then
- echo "RELEASE_VERSION was not set"
- exit 1
-fi
-
-# fail immediately
-set -o errexit
-set -o nounset
-
-CURR_DIR=`pwd`
-BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-PROJECT_ROOT="${BASE_DIR}/../../"
-
-# Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root
-if [ ! -f ${PROJECT_ROOT}/LICENSE ]; then
- echo "Project root path ${PROJECT_ROOT} is not valid; script may be in the wrong directory."
- exit 1
-fi
-
-if [ "$(uname)" == "Darwin" ]; then
- SHASUM="shasum -a 512"
-else
- SHASUM="sha512sum"
-fi
-
-###########################
-
-RELEASE_DIR=${PROJECT_ROOT}/release/binary
-rm -rf ${RELEASE_DIR}
-mkdir -p ${RELEASE_DIR}
-
-# use lint-python.sh script to create a python environment.
-dev/lint-python.sh -s basic
-source dev/.conda/bin/activate
-
-# build
-dev/build-wheels.sh
-
-WHEEL_FILE_NAME="pypaimon-${RELEASE_VERSION}-py3-none-any.whl"
-cp "dist/${WHEEL_FILE_NAME}" "${RELEASE_DIR}/${WHEEL_FILE_NAME}"
-
-cd ${RELEASE_DIR}
-
-# Sign sha the wheel package
-gpg --armor --detach-sig ${WHEEL_FILE_NAME}
-$SHASUM ${WHEEL_FILE_NAME} > "${WHEEL_FILE_NAME}.sha512"
-
-cd ${CURR_DIR}
diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh
index 7d6a8a9..0554272 100755
--- a/tools/releasing/create_source_release.sh
+++ b/tools/releasing/create_source_release.sh
@@ -23,7 +23,7 @@
RELEASE_VERSION=${RELEASE_VERSION}
if [ -z "${RELEASE_VERSION}" ]; then
- echo "RELEASE_VERSION is unset"
+ echo "RELEASE_VERSION was not set"
exit 1
fi
@@ -33,7 +33,7 @@ set -o nounset
CURR_DIR=`pwd`
BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-PROJECT_ROOT="$( cd "$( dirname "${BASE_DIR}/../../../" )" >/dev/null && pwd )"
+PROJECT_ROOT="${BASE_DIR}/../../"
# Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root
if [ ! -f ${PROJECT_ROOT}/LICENSE ]; then
@@ -43,48 +43,52 @@ fi
if [ "$(uname)" == "Darwin" ]; then
SHASUM="shasum -a 512"
- TAR="tar --no-xattrs"
else
SHASUM="sha512sum"
- TAR="tar"
fi
###########################
-RELEASE_DIR=${PROJECT_ROOT}/release/source
-CLONE_DIR=${RELEASE_DIR}/paimon-tmp-clone
+# prepare bridge jar
+DEPS_DIR=${PROJECT_ROOT}/deps/jars
+rm -rf ${DEPS_DIR}
+mkdir -p ${DEPS_DIR}
+
+cd ${PROJECT_ROOT}/paimon-python-java-bridge
+
+# check there is no snapshot dependencies
+if grep -q ".*SNAPSHOT" "pom.xml"; then
+ echo "paimon-python-java-bridge is snapshot or contains snapshot dependencies"
+ exit 1
+
+# get version
+JAR_VERSION=$(grep -oP "\K[^<]+" "pom.xml" | head -n 1)
+
+mvn clean install -DskipTests
+cp "target/paimon-python-java-bridge-${JAR_VERSION}.jar" ${DEPS_DIR}
+
+cd ${CURR_DIR}
+
+# build source release
+
+RELEASE_DIR=${PROJECT_ROOT}/release
rm -rf ${RELEASE_DIR}
mkdir -p ${RELEASE_DIR}
-# delete the temporary release directory on error
-trap 'rm -rf ${RELEASE_DIR}' ERR
-
-echo "Creating source package"
-
-# create a temporary git clone to ensure that we have a pristine source release
-git clone ${PROJECT_ROOT} ${CLONE_DIR}
-
-cd ${CLONE_DIR}
-JAVA_ROOT="pypaimon/py4j/paimon-python-java-bridge"
-rsync -a \
- --exclude ".DS_Store" --exclude ".asf.yaml" --exclude ".git" \
- --exclude ".github" --exclude ".gitignore" --exclude ".idea" \
- --exclude ".mypy_cache" --exclude ".tox" --exclude "__pycache__" \
- --exclude "build" --exclude "dist" --exclude "*.egg-info" \
- --exclude "dev/.conda" --exclude "dev/.stage.txt" \
- --exclude "dev/download" --exclude "dev/log" --exclude "**/__pycache__" \
- --exclude "${JAVA_ROOT}/dependency-reduced-pom.xml" \
- --exclude "${JAVA_ROOT}/target" \
- . paimon-python-${RELEASE_VERSION}
-
-TAR czf ${RELEASE_DIR}/apache-paimon-python-${RELEASE_VERSION}-src.tgz paimon-python-${RELEASE_VERSION}
-gpg --armor --detach-sig ${RELEASE_DIR}/apache-paimon-python-${RELEASE_VERSION}-src.tgz
-cd ${RELEASE_DIR}
-${SHASUM} apache-paimon-python-${RELEASE_VERSION}-src.tgz > apache-paimon-python-${RELEASE_VERSION}-src.tgz.sha512
+# use lint-python.sh script to create a python environment.
+dev/lint-python.sh -s basic
+source dev/.conda/bin/activate
+
+python setup.py sdist
+conda deactivate
+WHEEL_FILE_NAME="pypaimon-${RELEASE_VERSION}.tar.gz"
+cp "dist/${WHEEL_FILE_NAME}" "${RELEASE_DIR}/${WHEEL_FILE_NAME}"
-rm -rf ${CLONE_DIR}
+cd ${RELEASE_DIR}
-echo "Done. Source release package and signatures created under ${RELEASE_DIR}/."
+# Sign sha the wheel package
+gpg --armor --detach-sig ${WHEEL_FILE_NAME}
+$SHASUM ${WHEEL_FILE_NAME} > "${WHEEL_FILE_NAME}.sha512"
cd ${CURR_DIR}