Nondeterministic failure in test_distribution_generic.py

[fritzo]

I've been seeing nondeterministic failures in tests that use random generators. This should not happen, since we set the seed in conftest.py. Is there a source of nondeterminism that I'm missing?

Example: #427 with log https://api.travis-ci.com/v3/job/474778699/log.txt

=================================== FAILURES ===================================
�[31m�[1m test_generic_log_prob[True-dist.TransformedDistribution( dist.Uniform(low=case.low, high=case.high), [dist.transforms.TanhTransform().inv]) (('low', '0.5*rand((2, 3))'), ('high', '0.5 + 0.5*rand((2, 3))'))] �[0m
[gw0] linux -- Python 3.6.7 /home/travis/virtualenv/python3.6.7/bin/python

case = <test.test_distribution_generic.DistTestCase object at 0x7f5b7dbf2cc0>
use_lazy = True

�[1m    @pytest.mark.parametrize("case", TEST_CASES, ids=str)�[0m
�[1m    @pytest.mark.parametrize("use_lazy", [True, False])�[0m
�[1m    def test_generic_log_prob(case, use_lazy):�[0m
�[1m        raw_dist = case.get_dist()�[0m
�[1m        expected_value_domain = case.expected_value_domain�[0m
�[1m    �[0m
�[1m        dim_to_name, name_to_dim = _default_dim_to_name(raw_dist.batch_shape)�[0m
�[1m        with interpretation(normalize_with_subs if use_lazy else eager):�[0m
�[1m            with xfail_if_not_implemented(match="try upgrading backend"):�[0m
�[1m                # some distributions have nontrivial eager patterns�[0m
�[1m                funsor_dist = to_funsor(raw_dist, output=funsor.Real, dim_to_name=dim_to_name)�[0m
�[1m        expected_inputs = {name: funsor.Bint[raw_dist.batch_shape[dim]] for dim, name in dim_to_name.items()}�[0m
�[1m        expected_inputs.update({"value": expected_value_domain})�[0m
�[1m    �[0m
�[1m        check_funsor(funsor_dist, expected_inputs, funsor.Real)�[0m
�[1m    �[0m
�[1m        if get_backend() == "jax":�[0m
�[1m            raw_value = raw_dist.sample(key=np.array([0, 0], dtype=np.uint32))�[0m
�[1m        else:�[0m
�[1m            raw_value = raw_dist.sample()�[0m
�[1m        expected_logprob = to_funsor(raw_dist.log_prob(raw_value), output=funsor.Real, dim_to_name=dim_to_name)�[0m
�[1m        funsor_value = to_funsor(raw_value, output=expected_value_domain, dim_to_name=dim_to_name)�[0m
�[1m>       assert_close(funsor_dist(value=funsor_value), expected_logprob, rtol=1e-3)�[0m

�[1m�[31mtest/test_distribution_generic.py�[0m:632: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
�[1m�[31mfunsor/testing.py�[0m:105: in assert_close
�[1m    assert_close(actual.data, expected.data, atol=atol, rtol=rtol)�[0m
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = tensor([[ 4.1991e-05, -7.2371e-01,  6.1929e-01],
        [ 2.5963e-01,  2.5250e-01,  7.6591e-01]])
expected = tensor([[ 4.2111e-05, -7.2371e-01,  6.1929e-01],
        [ 2.5963e-01,  2.5250e-01,  7.6591e-01]])
atol = 1e-06, rtol = 0.001

�[1m    def assert_close(actual, expected, atol=1e-6, rtol=1e-6):�[0m
�[1m        msg = ActualExpected(actual, expected)�[0m
�[1m        if is_array(actual):�[0m
�[1m            assert is_array(expected), msg�[0m
�[1m        elif isinstance(actual, Tensor) and is_array(actual.data):�[0m
�[1m            assert isinstance(expected, Tensor) and is_array(expected.data)�[0m
�[1m        elif isinstance(actual, Contraction) and isinstance(actual.terms[0], Tensor) \�[0m
�[1m                and is_array(actual.terms[0].data):�[0m
�[1m            assert isinstance(expected, Contraction) and is_array(expected.terms[0].data)�[0m
�[1m        elif isinstance(actual, Gaussian) and is_array(actual.info_vec):�[0m
�[1m            assert isinstance(expected, Gaussian) and is_array(expected.info_vec)�[0m
�[1m        else:�[0m
�[1m            assert type(actual) == type(expected), msg�[0m
�[1m    �[0m
�[1m        if isinstance(actual, Funsor):�[0m
�[1m            assert isinstance(actual, Funsor)�[0m
�[1m            assert isinstance(expected, Funsor)�[0m
�[1m            assert actual.inputs == expected.inputs, (actual.inputs, expected.inputs)�[0m
�[1m            assert actual.output == expected.output, (actual.output, expected.output)�[0m
�[1m    �[0m
�[1m        if isinstance(actual, (Number, Tensor)):�[0m
�[1m            assert_close(actual.data, expected.data, atol=atol, rtol=rtol)�[0m
�[1m        elif isinstance(actual, Delta):�[0m
�[1m            assert frozenset(n for n, p in actual.terms) == frozenset(n for n, p in expected.terms)�[0m
�[1m            actual = actual.align(tuple(n for n, p in expected.terms))�[0m
�[1m            for (actual_name, (actual_point, actual_log_density)), \�[0m
�[1m                    (expected_name, (expected_point, expected_log_density)) in \�[0m
�[1m                    zip(actual.terms, expected.terms):�[0m
�[1m                assert actual_name == expected_name�[0m
�[1m                assert_close(actual_point, expected_point, atol=atol, rtol=rtol)�[0m
�[1m                assert_close(actual_log_density, expected_log_density, atol=atol, rtol=rtol)�[0m
�[1m        elif isinstance(actual, Gaussian):�[0m
�[1m            assert_close(actual.info_vec, expected.info_vec, atol=atol, rtol=rtol)�[0m
�[1m            assert_close(actual.precision, expected.precision, atol=atol, rtol=rtol)�[0m
�[1m        elif isinstance(actual, Contraction):�[0m
�[1m            assert actual.red_op == expected.red_op�[0m
�[1m            assert actual.bin_op == expected.bin_op�[0m
�[1m            assert actual.reduced_vars == expected.reduced_vars�[0m
�[1m            assert len(actual.terms) == len(expected.terms)�[0m
�[1m            for ta, te in zip(actual.terms, expected.terms):�[0m
�[1m                assert_close(ta, te, atol, rtol)�[0m
�[1m        elif type(actual).__name__ == "Tensor":�[0m
�[1m            assert get_backend() == "torch"�[0m
�[1m            import torch�[0m
�[1m    �[0m
�[1m            assert actual.dtype == expected.dtype, msg�[0m
�[1m            assert actual.shape == expected.shape, msg�[0m
�[1m            if actual.dtype in (torch.long, torch.uint8, torch.bool):�[0m
�[1m                assert (actual == expected).all(), msg�[0m
�[1m            else:�[0m
�[1m                eq = (actual == expected)�[0m
�[1m                if eq.all():�[0m
�[1m                    return�[0m
�[1m                if eq.any():�[0m
�[1m                    actual = actual[~eq]�[0m
�[1m                    expected = expected[~eq]�[0m
�[1m                diff = (actual.detach() - expected.detach()).abs()�[0m
�[1m                if rtol is not None:�[0m
�[1m>                   assert (diff / (atol + expected.detach().abs())).max() < rtol, msg�[0m
�[1m�[31mE                   AssertionError: Expected:�[0m
�[1m�[31mE                   tensor([[ 4.2111e-05, -7.2371e-01,  6.1929e-01],�[0m
�[1m�[31mE                           [ 2.5963e-01,  2.5250e-01,  7.6591e-01]])�[0m
�[1m�[31mE                   Actual:�[0m
�[1m�[31mE                   tensor([[ 4.1991e-05, -7.2371e-01,  6.1929e-01],�[0m
�[1m�[31mE                           [ 2.5963e-01,  2.5250e-01,  7.6591e-01]])�[0m

�[1m�[31mfunsor/testing.py�[0m:142: AssertionError
�[31m�[1m test_generic_log_prob[False-dist.TransformedDistribution( dist.Uniform(low=case.low, high=case.high), [dist.transforms.TanhTransform().inv]) (('low', '0.5*rand((2, 3))'), ('high', '0.5 + 0.5*rand((2, 3))'))] �[0m
[gw0] linux -- Python 3.6.7 /home/travis/virtualenv/python3.6.7/bin/python

case = <test.test_distribution_generic.DistTestCase object at 0x7f5b7dbf2cc0>
use_lazy = False

�[1m    @pytest.mark.parametrize("case", TEST_CASES, ids=str)�[0m
�[1m    @pytest.mark.parametrize("use_lazy", [True, False])�[0m
�[1m    def test_generic_log_prob(case, use_lazy):�[0m
�[1m        raw_dist = case.get_dist()�[0m
�[1m        expected_value_domain = case.expected_value_domain�[0m
�[1m    �[0m
�[1m        dim_to_name, name_to_dim = _default_dim_to_name(raw_dist.batch_shape)�[0m
�[1m        with interpretation(normalize_with_subs if use_lazy else eager):�[0m
�[1m            with xfail_if_not_implemented(match="try upgrading backend"):�[0m
�[1m                # some distributions have nontrivial eager patterns�[0m
�[1m                funsor_dist = to_funsor(raw_dist, output=funsor.Real, dim_to_name=dim_to_name)�[0m
�[1m        expected_inputs = {name: funsor.Bint[raw_dist.batch_shape[dim]] for dim, name in dim_to_name.items()}�[0m
�[1m        expected_inputs.update({"value": expected_value_domain})�[0m
�[1m    �[0m
�[1m        check_funsor(funsor_dist, expected_inputs, funsor.Real)�[0m
�[1m    �[0m
�[1m        if get_backend() == "jax":�[0m
�[1m            raw_value = raw_dist.sample(key=np.array([0, 0], dtype=np.uint32))�[0m
�[1m        else:�[0m
�[1m            raw_value = raw_dist.sample()�[0m
�[1m        expected_logprob = to_funsor(raw_dist.log_prob(raw_value), output=funsor.Real, dim_to_name=dim_to_name)�[0m
�[1m        funsor_value = to_funsor(raw_value, output=expected_value_domain, dim_to_name=dim_to_name)�[0m
�[1m>       assert_close(funsor_dist(value=funsor_value), expected_logprob, rtol=1e-3)�[0m

�[1m�[31mtest/test_distribution_generic.py�[0m:632: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
�[1m�[31mfunsor/testing.py�[0m:105: in assert_close
�[1m    assert_close(actual.data, expected.data, atol=atol, rtol=rtol)�[0m
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

actual = tensor([[ 4.1991e-05, -7.2371e-01,  6.1929e-01],
        [ 2.5963e-01,  2.5250e-01,  7.6591e-01]])
expected = tensor([[ 4.2111e-05, -7.2371e-01,  6.1929e-01],
        [ 2.5963e-01,  2.5250e-01,  7.6591e-01]])
atol = 1e-06, rtol = 0.001

�[1m    def assert_close(actual, expected, atol=1e-6, rtol=1e-6):�[0m
�[1m        msg = ActualExpected(actual, expected)�[0m
�[1m        if is_array(actual):�[0m
�[1m            assert is_array(expected), msg�[0m
�[1m        elif isinstance(actual, Tensor) and is_array(actual.data):�[0m
�[1m            assert isinstance(expected, Tensor) and is_array(expected.data)�[0m
�[1m        elif isinstance(actual, Contraction) and isinstance(actual.terms[0], Tensor) \�[0m
�[1m                and is_array(actual.terms[0].data):�[0m
�[1m            assert isinstance(expected, Contraction) and is_array(expected.terms[0].data)�[0m
�[1m        elif isinstance(actual, Gaussian) and is_array(actual.info_vec):�[0m
�[1m            assert isinstance(expected, Gaussian) and is_array(expected.info_vec)�[0m
�[1m        else:�[0m
�[1m            assert type(actual) == type(expected), msg�[0m
�[1m    �[0m
�[1m        if isinstance(actual, Funsor):�[0m
�[1m            assert isinstance(actual, Funsor)�[0m
�[1m            assert isinstance(expected, Funsor)�[0m
�[1m            assert actual.inputs == expected.inputs, (actual.inputs, expected.inputs)�[0m
�[1m            assert actual.output == expected.output, (actual.output, expected.output)�[0m
�[1m    �[0m
�[1m        if isinstance(actual, (Number, Tensor)):�[0m
�[1m            assert_close(actual.data, expected.data, atol=atol, rtol=rtol)�[0m
�[1m        elif isinstance(actual, Delta):�[0m
�[1m            assert frozenset(n for n, p in actual.terms) == frozenset(n for n, p in expected.terms)�[0m
�[1m            actual = actual.align(tuple(n for n, p in expected.terms))�[0m
�[1m            for (actual_name, (actual_point, actual_log_density)), \�[0m
�[1m                    (expected_name, (expected_point, expected_log_density)) in \�[0m
�[1m                    zip(actual.terms, expected.terms):�[0m
�[1m                assert actual_name == expected_name�[0m
�[1m                assert_close(actual_point, expected_point, atol=atol, rtol=rtol)�[0m
�[1m                assert_close(actual_log_density, expected_log_density, atol=atol, rtol=rtol)�[0m
�[1m        elif isinstance(actual, Gaussian):�[0m
�[1m            assert_close(actual.info_vec, expected.info_vec, atol=atol, rtol=rtol)�[0m
�[1m            assert_close(actual.precision, expected.precision, atol=atol, rtol=rtol)�[0m
�[1m        elif isinstance(actual, Contraction):�[0m
�[1m            assert actual.red_op == expected.red_op�[0m
�[1m            assert actual.bin_op == expected.bin_op�[0m
�[1m            assert actual.reduced_vars == expected.reduced_vars�[0m
�[1m            assert len(actual.terms) == len(expected.terms)�[0m
�[1m            for ta, te in zip(actual.terms, expected.terms):�[0m
�[1m                assert_close(ta, te, atol, rtol)�[0m
�[1m        elif type(actual).__name__ == "Tensor":�[0m
�[1m            assert get_backend() == "torch"�[0m
�[1m            import torch�[0m
�[1m    �[0m
�[1m            assert actual.dtype == expected.dtype, msg�[0m
�[1m            assert actual.shape == expected.shape, msg�[0m
�[1m            if actual.dtype in (torch.long, torch.uint8, torch.bool):�[0m
�[1m                assert (actual == expected).all(), msg�[0m
�[1m            else:�[0m
�[1m                eq = (actual == expected)�[0m
�[1m                if eq.all():�[0m
�[1m                    return�[0m
�[1m                if eq.any():�[0m
�[1m                    actual = actual[~eq]�[0m
�[1m                    expected = expected[~eq]�[0m
�[1m                diff = (actual.detach() - expected.detach()).abs()�[0m
�[1m                if rtol is not None:�[0m
�[1m>                   assert (diff / (atol + expected.detach().abs())).max() < rtol, msg�[0m
�[1m�[31mE                   AssertionError: Expected:�[0m
�[1m�[31mE                   tensor([[ 4.2111e-05, -7.2371e-01,  6.1929e-01],�[0m
�[1m�[31mE                           [ 2.5963e-01,  2.5250e-01,  7.6591e-01]])�[0m
�[1m�[31mE                   Actual:�[0m
�[1m�[31mE                   tensor([[ 4.1991e-05, -7.2371e-01,  6.1929e-01],�[0m
�[1m�[31mE                           [ 2.5963e-01,  2.5250e-01,  7.6591e-01]])�[0m

�[1m�[31mfunsor/testing.py�[0m:142: AssertionError
�[33m=============================== warnings summary ===============================�[0m
funsor/tensor.py:879
funsor/tensor.py:879
funsor/tensor.py:879
funsor/tensor.py:879
  /home/travis/build/pyro-ppl/funsor/funsor/tensor.py:879: DeprecationWarning: tuple types like (Real, Reals[2]) are deprecated, use Tuple[Real, Reals[2]] instead
    DeprecationWarning)

-- Docs: https://docs.pytest.org/en/latest/warnings.html
�[31m�[1m 2 failed, 10155 passed, 19 skipped, 533 xfailed, 91 xpassed, 4 warnings in 398.66 seconds �[0m
Makefile:23: recipe for target 'test' failed
make: *** [test] Error 1
travis_time:end:088f56c3:start=1611533200567500419,finish=1611533609395635812,duration=408828135393,event=script
�[0K�[31;1mThe command "FUNSOR_BACKEND=torch make test" exited with 2.�[0m
travis_fold:start:cache.2
�[0Kstore build cache
travis_time:start:0130e024
�[0Ktravis_time:end:0130e024:start=1611533609400001839,finish=1611533609402740055,duration=2738216,event=cache
�[0Ktravis_time:start:01fc5838
�[0K�[32;1mnothing changed�[0m
travis_time:end:01fc5838:start=1611533609406249165,finish=1611533610249520444,duration=843271279,event=cache
�[0Ktravis_fold:end:cache.2
�[0K

Done. Your build exited with 1.

[fehiepsi]

saw this issue too. I guess some tests are flaky given bad input values.

[fehiepsi]

Okie, I understand why the non-deterministic issue happens. In test_distribution_generic, we invoke random statements when loading the test file before invoking set_rng_seed in conftest.py. I think we can add pyro.set_rng_seed(0) on the top of that file to resolve the issue.