Error when Running tests

1

When I ran the tests I get the following error , what could be the reason ?

[gw2] [ 67%] PASSED tests/infer/test_valid_models.py::test_iplate_iplate_swap_ok[TraceEnum_ELBO-full-0]
tests/infer/test_valid_models.py::test_iplate_iplate_swap_ok[TraceEnum_ELBO-full-inf]
[gw7] [ 67%] FAILED tests/infer/test_jit.py::test_masked_fill

======================================================================== FAILURES ========================================================================
____________________________________________________________________ test_masked_fill ____________________________________________________________________
[gw7] darwin – Python 3.6.3 /Users/hrishikesh/anaconda3/envs/pyro_env/bin/python

def test_masked_fill():

    def f(y, mask):
        return y.clone().masked_fill_(mask, 0.)

    x = torch.tensor([-float('inf'), -1., 0., 1., float('inf')])
    y = x / x.unsqueeze(-1)
    mask = ~(y == y)

  jit_f = torch.jit.trace(f, (y, mask))

tests/infer/test_jit.py:137:

…/…/…/anaconda3/envs/pyro_env/lib/python3.6/site-packages/torch/jit/init.py:604: in trace
_check_trace([example_inputs], func, executor_options, module, check_tolerance)
…/…/…/anaconda3/envs/pyro_env/lib/python3.6/site-packages/torch/autograd/grad_mode.py:43: in decorate_no_grad
return func(*args, **kwargs)
…/…/…/anaconda3/envs/pyro_env/lib/python3.6/site-packages/torch/jit/init.py:505: in _check_trace
traced_outs = run_mod_and_filter_tensor_outputs(module, inputs, ‘trace’)

mod = f()
inputs = (tensor([[nan, 0., -0., -0., nan],
[inf, 1., -0., -1., -inf],
[-inf, -inf, nan, inf, inf],
[… [0, 0, 0, 0, 0],
[0, 0, 1, 0, 0],
[0, 0, 0, 0, 0],
[1, 0, 0, 0, 1]], dtype=torch.uint8))
running_what = ‘trace’

def run_mod_and_filter_tensor_outputs(mod, inputs, running_what):
    try:
        outs = wrap_retval(mod(*_clone_inputs(inputs)))
        outs = [out for out in outs if isinstance(out, torch.Tensor)]
        return outs
    except Exception as e:
        raise TracingCheckError(*graph_diagnostic_info(),
                                extra_msg='Encountered an exception while running the ' + running_what +

                                        ' with test inputs.\nException:\n' + indent(str(e)))

E torch.jit.TracingCheckError: Tracing failed sanity checks!
E Encountered an exception while running the trace with test inputs.
E Exception:
E
E Schema not found for node. File a bug report.
E Node: %4 : Dynamic = aten::masked_fill(%2, %mask, %3)
E
E Input types:Dynamic, Byte(*, ), float
E candidates were:
E graph(%y : Double(, )
E %mask : Byte(, *)) {
E %2 : Dynamic = aten::clone(%y)
E %3 : float = prim::Constantvalue=0
E %4 : Dynamic = aten::masked_fill(%2, %mask, %3)
E return (%4);
E }
E
E :
E /Users/hrishikesh/Hrishikesh/Projects/pyro/tests/infer/test_jit.py(132): f
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/torch/jit/init.py(597): trace
E /Users/hrishikesh/Hrishikesh/Projects/pyro/tests/infer/test_jit.py(137): test_masked_fill
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/python.py(196): pytest_pyfunc_call
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/callers.py(187): _multicall
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/manager.py(61):
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/manager.py(67): _hookexec
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/hooks.py(284): call
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/python.py(1431): runtest
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/runner.py(111): pytest_runtest_call
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/callers.py(187): _multicall
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/manager.py(61):
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/manager.py(67): _hookexec
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/hooks.py(284): call
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/runner.py(183):
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/runner.py(201): init
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/runner.py(185): call_runtest_hook
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/runner.py(161): call_and_report
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/runner.py(81): runtestprotocol
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/runner.py(66): pytest_runtest_protocol
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/callers.py(187): _multicall
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/manager.py(61):
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/manager.py(67): _hookexec
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/hooks.py(284): call
E (83): run_one_test
E (66): pytest_runtestloop
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/callers.py(187): _multicall
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/manager.py(61):
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/manager.py(67): _hookexec
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/hooks.py(284): call
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/main.py(215): _main
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/main.py(178): wrap_session
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/_pytest/main.py(208): pytest_cmdline_main
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/callers.py(187): _multicall
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/manager.py(61):
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/manager.py(67): _hookexec
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/pluggy/hooks.py(284): call
E (270):
E (1): do_exec
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/execnet/gateway_base.py(1072): executetask
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/execnet/gateway_base.py(213): run
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/execnet/gateway_base.py(277): _perform_spawn
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/execnet/gateway_base.py(259): integrate_as_primary_thread
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/execnet/gateway_base.py(1047): serve
E /Users/hrishikesh/anaconda3/envs/pyro_env/lib/python3.6/site-packages/execnet/gateway_base.py(1534): serve
E (8):
E (1):

…/…/…/anaconda3/envs/pyro_env/lib/python3.6/site-packages/torch/jit/init.py:474: TracingCheckError

2

are you using pytorch nightly/master? the jit is not supported on release versions.

edit: for these types of issues please post your pyro and pytorch versions for easier debugging.

1 Like
3

Thank you for your response @jpchen . Yes , I am using PyTorch nightly version. What’s the ideal version to get tests running ? Further what exactly is jit?

4

Use the nightly build pinned to 20181128 as follows:

pip install torch_nightly==1.0.0.dev20181128 -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html

I would suggest that you run your specific test or a few tests specific to your features using something like:

pytest -vs tests/infer/{name_of_test}.py

or, you could also run all unit tests using make test. Running the entire suite might take too much time, so feel free to submit your PR after running a more limited set. The entire test suite will be kicked off on travis CI once you submit your PR.

1 Like
5

Thank you @neerajprad I will try using the version of Nightly.

6

I installed the version of torch nightly you mentioned. Now , I get these errors.

When I ran make test

[gw7] [ 98%] PASSED tests/infer/mcmc/test_mcmc.py::test_mcmc_interface 
tests/infer/mcmc/test_nuts.py::test_nuts_conjugate_gaussian[dim=10_chain-len=3_num_obs=1] 
[gw7] [ 98%] SKIPPED tests/infer/mcmc/test_nuts.py::test_nuts_conjugate_gaussian[dim=10_chain-len=3_num_obs=1] 
tests/infer/mcmc/test_nuts.py::test_gaussian_hmm[3-JIT=False] 
[gw7] [ 98%] PASSED tests/infer/mcmc/test_nuts.py::test_gaussian_hmm[3-JIT=False] 
tests/infer/mcmc/test_nuts.py::test_gaussian_hmm[3-JIT=True] 
[gw7] [ 98%] FAILED tests/infer/mcmc/test_nuts.py::test_gaussian_hmm[3-JIT=True] 

=================================== FAILURES ===================================
________________________ test_gaussian_hmm[3-JIT=True] _________________________
[gw7] darwin -- Python 3.6.3 /Users/hrishikesh/anaconda3/envs/pyro_env/bin/python

jit = True, num_steps = 3

    @pytest.mark.parametrize("jit", [False, mark_jit(True)], ids=jit_idfn)
    @pytest.mark.parametrize("num_steps", [2, 3, 30])
    def test_gaussian_hmm(jit, num_steps):
        dim = 4
    
        def model(data):
            initialize = pyro.sample("initialize", dist.Dirichlet(torch.ones(dim)))
            with pyro.plate("states", dim):
                transition = pyro.sample("transition", dist.Dirichlet(torch.ones(dim, dim)))
                emission_loc = pyro.sample("emission_loc", dist.Normal(torch.zeros(dim), torch.ones(dim)))
                emission_scale = pyro.sample("emission_scale", dist.LogNormal(torch.zeros(dim), torch.ones(dim)))
            x = None
            with ignore_jit_warnings([("Iterating over a tensor", RuntimeWarning)]):
                for t, y in pyro.markov(enumerate(data)):
                    x = pyro.sample("x_{}".format(t),
                                    dist.Categorical(initialize if x is None else transition[x]),
                                    infer={"enumerate": "parallel"})
                    pyro.sample("y_{}".format(t), dist.Normal(emission_loc[x], emission_scale[x]), obs=y)
    
        def _get_initial_trace():
            guide = AutoDelta(poutine.block(model, expose_fn=lambda msg: not msg["name"].startswith("x") and
                                            not msg["name"].startswith("y")))
            elbo = TraceEnum_ELBO(max_plate_nesting=1)
            svi = SVI(model, guide, optim.Adam({"lr": .01}), elbo, num_steps=100).run(data)
            return svi.exec_traces[-1]
    
        def _generate_data():
            transition_probs = torch.rand(dim, dim)
            emissions_loc = torch.arange(dim, dtype=torch.Tensor().dtype)
            emissions_scale = 1.
            state = torch.tensor(1)
            obs = [dist.Normal(emissions_loc[state], emissions_scale).sample()]
            for _ in range(num_steps):
                state = dist.Categorical(transition_probs[state]).sample()
                obs.append(dist.Normal(emissions_loc[state], emissions_scale).sample())
            return torch.stack(obs)
    
        data = _generate_data()
        nuts_kernel = NUTS(model, max_plate_nesting=1, jit_compile=jit, ignore_jit_warnings=True)
        if num_steps == 30:
            nuts_kernel.initial_trace = _get_initial_trace()
>       MCMC(nuts_kernel, num_samples=5, warmup_steps=5).run(data)

tests/infer/mcmc/test_nuts.py:327: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
pyro/infer/abstract_infer.py:154: in run
    for i, vals in enumerate(self._traces(*args, **kwargs)):
pyro/infer/mcmc/mcmc.py:254: in _traces
    for sample in self.sampler._traces(*args, **kwargs):
pyro/infer/mcmc/mcmc.py:197: in _traces
    self.kernel.setup(self.warmup_steps, *args, **kwargs)
pyro/infer/mcmc/hmc.py:363: in setup
    self._configure_adaptation()
pyro/infer/mcmc/hmc.py:275: in _configure_adaptation
    initial_step_size = self._find_reasonable_step_size(z)
pyro/infer/mcmc/hmc.py:225: in _find_reasonable_step_size
    energy_current = self._energy(z, r)
pyro/infer/mcmc/hmc.py:201: in _energy
    return self._kinetic_energy(r) + self._potential_energy(z)
pyro/infer/mcmc/hmc.py:161: in _potential_energy
    return self._potential_energy_jit(z)
pyro/infer/mcmc/hmc.py:197: in _potential_energy_jit
    self._compiled_potential_fn = torch.jit.trace(compiled, vals, check_trace=False)
../../../anaconda3/envs/pyro_env/lib/python3.6/site-packages/torch/jit/__init__.py:612: in trace
    var_lookup_fn, _force_outplace)
pyro/infer/mcmc/hmc.py:188: in compiled
    potential_energy = -self._compute_trace_log_prob(trace)
pyro/infer/mcmc/hmc.py:148: in _compute_trace_log_prob
    return self._trace_prob_evaluator.log_prob(model_trace)
pyro/infer/mcmc/util.py:210: in log_prob
    return contract_to_tensor(log_probs, self._enum_dims, cache=cache)
pyro/ops/contract.py:240: in contract_to_tensor
    ordinal, term = _contract_component(ring, component, dims, target_dims & dims)
pyro/ops/contract.py:116: in _contract_component
    term = ring.sumproduct(terms, dims - local_dims)
pyro/ops/rings.py:151: in sumproduct
    term = contract(equation, *terms, backend=self._backend)
pyro/ops/einsum/__init__.py:44: in contract
    expr = contract_expression(equation, *shapes)
pyro/ops/einsum/__init__.py:26: in contract_expression
    expr = opt_einsum.contract_expression(equation, *shapes, **kwargs)
../../../anaconda3/envs/pyro_env/lib/python3.6/site-packages/opt_einsum/contract.py:846: in contract_expression
    return contract(subscripts, *dummy_arrays, **kwargs)
../../../anaconda3/envs/pyro_env/lib/python3.6/site-packages/opt_einsum/contract.py:490: in contract
    *operands, optimize=optimize_arg, memory_limit=memory_limit, einsum_call=True, use_blas=use_blas)
../../../anaconda3/envs/pyro_env/lib/python3.6/site-packages/opt_einsum/contract.py:272: in contract_path
    path = paths.branch(input_sets, output_set, dimension_dict, memory_arg, nbranch=2)
../../../anaconda3/envs/pyro_env/lib/python3.6/site-packages/opt_einsum/paths.py:333: in branch
    flops=0)
../../../anaconda3/envs/pyro_env/lib/python3.6/site-packages/opt_einsum/paths.py:305: in _branch_iterate
    candidate = _assess_candidate(k1, k2, i, j)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

k1 = frozenset({'f', 'h'}), k2 = frozenset({'h'}), i = 0, j = 1

    def _assess_candidate(k1, k2, i, j):
        # find resulting indices and cost
        try:
            k12, flops12 = result_cache[k1, k2]
        except KeyError:
            k12, flops12 = result_cache[k1, k2] = _calc_k12_flops(inputs, output, remaining, i, j, size_dict)
    
        # sieve based on current best flops
        new_flops = flops + flops12
>       if new_flops >= best['flops']:
E       RuntimeError: value cannot be converted to type int64_t without overflow: inf

../../../anaconda3/envs/pyro_env/lib/python3.6/site-packages/opt_einsum/paths.py:264: RuntimeError
=============================== warnings summary ===============================
<undetermined location>
  Benchmarks are automatically disabled because xdist plugin is active.Benchmarks cannot be performed reliably in a parallelized environment.

-- Docs: http://doc.pytest.org/en/latest/warnings.html
!!!!!!!!!!!! xdist.dsession.Interrupted: stopping after 1 failures !!!!!!!!!!!!!
 1 failed, 4102 passed, 135 skipped, 118 xfailed, 1 warnings in 309.27 seconds =
make: *** [test] Error 2
7

That’s strange. What is the version of opt_einsum you are using? You can check that using pip freeze.

8

I agree, that’s weird. Can you try make install in the pyro directory to ensure all dependencies are up-to-date?

9

opt-einsum==2.3.1