Clear side input caches of dofninvoker when finishing bundle. Add microbenchmarks for map_fn with side inputs. #37123
87 fail, 265 skipped, 3 871 pass in 17m 57s
4 files 4 suites 17m 57s ⏱️
4 223 tests 3 871 ✅ 265 💤 87 ❌
8 413 runs 7 346 ✅ 943 💤 124 ❌
Results for commit fc29a2c.
Annotations
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.TestWriteToBigQuery
github-actions / Test Results
1 out of 2 runs failed: test_streaming_triggering_frequency_with_auto_sharding (apache_beam.io.gcp.bigquery_test.TestWriteToBigQuery)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 1s]
Raw output
TypeError: 'NoneType' object is not iterable
self = <apache_beam.io.gcp.bigquery_test.TestWriteToBigQuery testMethod=test_streaming_triggering_frequency_with_auto_sharding>
def test_streaming_triggering_frequency_with_auto_sharding(self):
def noop(table, **kwargs):
return []
client = mock.Mock()
client.insert_rows_json = mock.Mock(side_effect=noop)
opt = StandardOptions()
opt.streaming = True
> with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1062:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f8e8dbdd0>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f8ef05b20>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f883e3f80>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_always_0 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 2s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_always_0>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1668: in test_insert_rows_json_errors_retry_always
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f8dcb9790>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f8c777290>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f8d673d10>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_always_1 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 1s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_always_1>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1668: in test_insert_rows_json_errors_retry_always
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f88408d10>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f8c8e49e0>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f883e0140>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_always_2 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 1s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_always_2>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1668: in test_insert_rows_json_errors_retry_always
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f882697f0>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f88140140>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f707a2bd0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_never_0 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 1s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_never_0>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1762: in test_insert_rows_json_errors_retry_never
with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f70447d10>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f88185730>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f70193f50>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_never_1 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_never_1>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1762: in test_insert_rows_json_errors_retry_never
with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f880ee270>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f884088c0>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f701e29f0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_never_2 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_never_2>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1762: in test_insert_rows_json_errors_retry_never
with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f7039ef30>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f706e17f0>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f694ffef0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_never_3 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_never_3>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1762: in test_insert_rows_json_errors_retry_never
with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f909aee40>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f70372000>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f7067b140>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_on_transient_error_0 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_on_transient_error_0>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1922: in test_insert_rows_json_errors_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f70673cb0>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f7039c830>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f88185b20>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_on_transient_error_1 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 1s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_on_transient_error_1>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1922: in test_insert_rows_json_errors_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f8ceda1b0>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f880ef5f0>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f8827aea0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_on_transient_error_2 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 1s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_on_transient_error_2>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1922: in test_insert_rows_json_errors_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f8cd1eb10>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f8dd075f0>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f8e2282f0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_on_transient_error_3 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 1s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_on_transient_error_3>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1922: in test_insert_rows_json_errors_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f8c675640>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f8c776780>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f707a0ad0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_on_transient_error_4 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_on_transient_error_4>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1922: in test_insert_rows_json_errors_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f8ced8470>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f8f528950>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f88269a30>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_errors_retry_on_transient_error_5 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_errors_retry_on_transient_error_5>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1922: in test_insert_rows_json_errors_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner', options=opt) as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f8845d250>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f707d53d0>
options = <apache_beam.options.pipeline_options.StandardOptions object at 0x788f88175580>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_always_0 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_always_0>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1302: in test_insert_rows_json_exception_retry_always
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f704bbc20>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f88186ab0>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f701ac200>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_always_1 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_always_1>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1302: in test_insert_rows_json_exception_retry_always
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f8852b1a0>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f70671910>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f8840a900>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_always_2 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_always_2>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1302: in test_insert_rows_json_exception_retry_always
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f8c6df9b0>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f8ee3a870>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f693f3a40>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_never_0 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_never_0>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1365: in test_insert_rows_json_exception_retry_never
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f693b0230>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f701afad0>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f7079bc20>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_never_1 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_never_1>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1365: in test_insert_rows_json_exception_retry_never
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f695e1a90>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f8844c6e0>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f69321070>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_on_transient_error_00 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 1s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_on_transient_error_00>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1479: in test_insert_rows_json_exception_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f6950ed50>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f69515f40>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f70475dc0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_on_transient_error_01 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 1s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_on_transient_error_01>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1479: in test_insert_rows_json_exception_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f7014c770>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f6950fd40>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f701e84d0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_on_transient_error_02 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_on_transient_error_02>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1479: in test_insert_rows_json_exception_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f70187bf0>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f707a4380>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f7067cb30>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_on_transient_error_03 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_on_transient_error_03>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1479: in test_insert_rows_json_exception_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f70221880>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f701ea390>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f70629fd0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_on_transient_error_04 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 0s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_on_transient_error_04>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1479: in test_insert_rows_json_exception_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f706c4530>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f7067df40>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f706b2cc0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError
Check warning on line 0 in apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling
github-actions / Test Results
1 out of 2 runs failed: test_insert_rows_json_exception_retry_on_transient_error_05 (apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling)
sdks/python/test-suites/tox/py312/build/srcs/sdks/python/pytest_py312-cloud.xml [took 1s]
Raw output
TypeError: 'NoneType' object is not iterable
a = (<apache_beam.io.gcp.bigquery_test.BigQueryStreamingInsertsErrorHandling testMethod=test_insert_rows_json_exception_retry_on_transient_error_05>,)
kw = {}
@wraps(func)
def standalone_func(*a, **kw):
> return func(*(a + p.args), **p.kwargs, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:620:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/parameterized/parameterized.py:93: in dummy_func
return orgfunc(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/mock/mock.py:1468: in patched
return func(*newargs, **newkeywargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/io/gcp/bigquery_test.py:1479: in test_insert_rows_json_exception_retry_on_transient_error
with beam.Pipeline(runner='BundleBasedDirectRunner') as p:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:646: in __exit__
self.result = self.run()
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:599: in run
self._options).run(False)
^^^^^^^^^^
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/pipeline.py:623: in run
return self.runner.run_pipeline(self, self._options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.runners.direct.direct_runner.BundleBasedDirectRunner object at 0x788f70628d10>
pipeline = <apache_beam.pipeline.Pipeline object at 0x788f702da540>
options = <apache_beam.options.pipeline_options.PipelineOptions object at 0x788f7067f2f0>
def run_pipeline(self, pipeline, options):
"""Execute the entire pipeline and returns an DirectPipelineResult."""
# TODO: Move imports to top. Pipeline <-> Runner dependency cause problems
# with resolving imports when they are at top.
# pylint: disable=wrong-import-position
from apache_beam.pipeline import PipelineVisitor
from apache_beam.runners.direct.consumer_tracking_pipeline_visitor import ConsumerTrackingPipelineVisitor
from apache_beam.runners.direct.evaluation_context import EvaluationContext
from apache_beam.runners.direct.executor import Executor
from apache_beam.runners.direct.transform_evaluator import TransformEvaluatorRegistry
from apache_beam.testing.test_stream import TestStream
from apache_beam.transforms.external import ExternalTransform
class VerifyNoCrossLanguageTransforms(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, ExternalTransform):
raise RuntimeError(
"Streaming Python direct runner "
"does not support cross-language pipelines."
"Please use other runners such as FlinkRunner, "
"DataflowRunner, or PrismRunner.")
pipeline.visit(VerifyNoCrossLanguageTransforms())
# If the TestStream I/O is used, use a mock test clock.
class TestStreamUsageVisitor(PipelineVisitor):
"""Visitor determining whether a Pipeline uses a TestStream."""
def __init__(self):
self.uses_test_stream = False
def visit_transform(self, applied_ptransform):
if isinstance(applied_ptransform.transform, TestStream):
self.uses_test_stream = True
visitor = TestStreamUsageVisitor()
pipeline.visit(visitor)
clock = TestClock() if visitor.uses_test_stream else RealClock()
# Performing configured PTransform overrides.
pipeline.replace_all(_get_transform_overrides(options))
_LOGGER.info('Running pipeline with DirectRunner.')
self.consumer_tracking_visitor = ConsumerTrackingPipelineVisitor()
pipeline.visit(self.consumer_tracking_visitor)
evaluation_context = EvaluationContext(
options,
BundleFactory(
stacked=options.view_as(
DirectOptions).direct_runner_use_stacked_bundle),
self.consumer_tracking_visitor.root_transforms,
self.consumer_tracking_visitor.value_to_consumers,
self.consumer_tracking_visitor.step_names,
self.consumer_tracking_visitor.views,
clock)
executor = Executor(
self.consumer_tracking_visitor.value_to_consumers,
TransformEvaluatorRegistry(evaluation_context),
evaluation_context)
# DirectRunner does not support injecting
# PipelineOptions values at runtime
RuntimeValueProvider.set_runtime_options(
> {'experiments': set(options.view_as(beam.options.pipeline_options.DebugOptions).experiments)}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
E TypeError: 'NoneType' object is not iterable
target/.tox-py312-cloud/py312-cloud/lib/python3.12/site-packages/apache_beam/runners/direct/direct_runner.py:617: TypeError