Commit 878fb5b6 authored by Tiago Peixoto's avatar Tiago Peixoto
Browse files

inference: remove 'allow_empty' option

parent 67372fc8
Pipeline #520 failed with stage
in 622 minutes and 14 seconds
...@@ -38,7 +38,7 @@ rec_p = g.new_ep("double", random(g.num_edges())) ...@@ -38,7 +38,7 @@ rec_p = g.new_ep("double", random(g.num_edges()))
rec_s = g.new_ep("double", normal(0, 10, g.num_edges())) rec_s = g.new_ep("double", normal(0, 10, g.num_edges()))
def _gen_state(directed, deg_corr, layers, overlap, rec, rec_type, allow_empty): def _gen_state(directed, deg_corr, layers, overlap, rec, rec_type):
u = GraphView(g, directed=directed) u = GraphView(g, directed=directed)
if layers != False: if layers != False:
base_type = graph_tool.inference.LayeredBlockState base_type = graph_tool.inference.LayeredBlockState
...@@ -48,8 +48,7 @@ def _gen_state(directed, deg_corr, layers, overlap, rec, rec_type, allow_empty): ...@@ -48,8 +48,7 @@ def _gen_state(directed, deg_corr, layers, overlap, rec, rec_type, allow_empty):
recs=[rec] if rec is not None else [], recs=[rec] if rec is not None else [],
rec_types=[rec_type] if rec is not None else [], rec_types=[rec_type] if rec is not None else [],
overlap=overlap, overlap=overlap,
layers=layers == True, layers=layers == True)
allow_empty=allow_empty)
elif overlap: elif overlap:
base_type = graph_tool.inference.OverlapBlockState base_type = graph_tool.inference.OverlapBlockState
state_args = dict(B=2 * u.num_edges(), state_args = dict(B=2 * u.num_edges(),
...@@ -61,8 +60,7 @@ def _gen_state(directed, deg_corr, layers, overlap, rec, rec_type, allow_empty): ...@@ -61,8 +60,7 @@ def _gen_state(directed, deg_corr, layers, overlap, rec, rec_type, allow_empty):
state_args = dict(B=u.num_vertices(), state_args = dict(B=u.num_vertices(),
recs=[rec] if rec is not None else [], recs=[rec] if rec is not None else [],
rec_types=[rec_type] if rec is not None else [], rec_types=[rec_type] if rec is not None else [],
deg_corr=deg_corr, deg_corr=deg_corr)
allow_empty=allow_empty)
return u, base_type, state_args return u, base_type, state_args
...@@ -71,9 +69,8 @@ def gen_state(*args): ...@@ -71,9 +69,8 @@ def gen_state(*args):
return base_type(u, **state_args) return base_type(u, **state_args)
def gen_nested_state(*args): def gen_nested_state(*args):
u, base_type, state_args = _gen_state(*args, False) u, base_type, state_args = _gen_state(*args)
B = state_args.pop("B") B = state_args.pop("B")
state_args.pop("allow_empty", None)
return NestedBlockState(u, return NestedBlockState(u,
bs=[numpy.arange(B)] + [numpy.zeros(1)] * 6, bs=[numpy.arange(B)] + [numpy.zeros(1)] * 6,
base_type=base_type, state_args=state_args, base_type=base_type, state_args=state_args,
...@@ -87,7 +84,6 @@ pranges = [("directed", [False, True]), ...@@ -87,7 +84,6 @@ pranges = [("directed", [False, True]),
("deg_corr", [False, True]), ("deg_corr", [False, True]),
("dl", [False, True]), ("dl", [False, True]),
("degree_dl_kind", ["uniform", "distributed", "entropy"]), ("degree_dl_kind", ["uniform", "distributed", "entropy"]),
("allow_empty", [False, True]),
("exact", [True, False])] ("exact", [True, False])]
pranges = OrderedDict(pranges) pranges = OrderedDict(pranges)
...@@ -120,7 +116,7 @@ for pvals in iter_ranges(pranges): ...@@ -120,7 +116,7 @@ for pvals in iter_ranges(pranges):
rec_ = rec_s rec_ = rec_s
print("\t mcmc (unweighted)", file=out) print("\t mcmc (unweighted)", file=out)
state = gen_state(directed, deg_corr, layered, overlap, rec_, rec, allow_empty) state = gen_state(directed, deg_corr, layered, overlap, rec_, rec)
print("\t\t", print("\t\t",
state.mcmc_sweep(beta=0, state.mcmc_sweep(beta=0,
...@@ -138,7 +134,7 @@ for pvals in iter_ranges(pranges): ...@@ -138,7 +134,7 @@ for pvals in iter_ranges(pranges):
state.get_nonempty_B(), file=out) state.get_nonempty_B(), file=out)
print("\t mcmc (unweighted, multiflip)", file=out) print("\t mcmc (unweighted, multiflip)", file=out)
state = gen_state(directed, deg_corr, layered, overlap, rec_, rec, allow_empty) state = gen_state(directed, deg_corr, layered, overlap, rec_, rec)
print("\t\t", print("\t\t",
state.multiflip_mcmc_sweep(beta=0, state.multiflip_mcmc_sweep(beta=0,
entropy_args=dict(dl=dl, entropy_args=dict(dl=dl,
...@@ -147,7 +143,7 @@ for pvals in iter_ranges(pranges): ...@@ -147,7 +143,7 @@ for pvals in iter_ranges(pranges):
state.get_nonempty_B(), file=out) state.get_nonempty_B(), file=out)
print("\t gibbs (unweighted)", file=out) print("\t gibbs (unweighted)", file=out)
state = gen_state(directed, deg_corr, layered, overlap, rec_, rec, allow_empty) state = gen_state(directed, deg_corr, layered, overlap, rec_, rec)
print("\t\t", print("\t\t",
state.gibbs_sweep(beta=0, state.gibbs_sweep(beta=0,
...@@ -157,7 +153,7 @@ for pvals in iter_ranges(pranges): ...@@ -157,7 +153,7 @@ for pvals in iter_ranges(pranges):
state.get_nonempty_B(), file=out) state.get_nonempty_B(), file=out)
if not overlap: if not overlap:
state = gen_state(directed, deg_corr, layered, overlap, rec_, rec, allow_empty) state = gen_state(directed, deg_corr, layered, overlap, rec_, rec)
print("\t mcmc", file=out) print("\t mcmc", file=out)
bstate = state.get_block_state(vweight=True, deg_corr=deg_corr) bstate = state.get_block_state(vweight=True, deg_corr=deg_corr)
...@@ -185,7 +181,7 @@ for pvals in iter_ranges(pranges): ...@@ -185,7 +181,7 @@ for pvals in iter_ranges(pranges):
print("\t merge", file=out) print("\t merge", file=out)
state = gen_state(directed, deg_corr, layered, overlap, rec_, rec, allow_empty) state = gen_state(directed, deg_corr, layered, overlap, rec_, rec)
if not overlap: if not overlap:
bstate = state.get_block_state(vweight=True, deg_corr=deg_corr) bstate = state.get_block_state(vweight=True, deg_corr=deg_corr)
...@@ -223,7 +219,7 @@ for pvals in iter_ranges(pranges): ...@@ -223,7 +219,7 @@ for pvals in iter_ranges(pranges):
print("\t shrink", file=out) print("\t shrink", file=out)
state = gen_state(directed, deg_corr, layered, overlap, rec_, rec, allow_empty) state = gen_state(directed, deg_corr, layered, overlap, rec_, rec)
state = state.shrink(B=5, entropy_args=dict(dl=dl, state = state.shrink(B=5, entropy_args=dict(dl=dl,
degree_dl_kind=degree_dl_kind, degree_dl_kind=degree_dl_kind,
multigraph=False, multigraph=False,
......
...@@ -101,8 +101,7 @@ typedef mpl::vector1<std::false_type> rmap_tr; ...@@ -101,8 +101,7 @@ typedef mpl::vector1<std::false_type> rmap_tr;
((wparams, &, std::vector<std::vector<double>>&, 0)) \ ((wparams, &, std::vector<std::vector<double>>&, 0)) \
((recdx, &, std::vector<double>&, 0)) \ ((recdx, &, std::vector<double>&, 0)) \
((Lrecdx, &, std::vector<double>&, 0)) \ ((Lrecdx, &, std::vector<double>&, 0)) \
((epsilon, &, std::vector<double>&, 0)) \ ((epsilon, &, std::vector<double>&, 0))
((allow_empty,, bool, 0))
GEN_STATE_BASE(BlockStateBase, BLOCK_STATE_params) GEN_STATE_BASE(BlockStateBase, BLOCK_STATE_params)
...@@ -271,7 +270,7 @@ public: ...@@ -271,7 +270,7 @@ public:
BlockState::remove_partition_node(v, r); BlockState::remove_partition_node(v, r);
} }
bool allow_move(size_t v, size_t r, size_t nr, bool allow_empty = true) bool allow_move(size_t v, size_t r, size_t nr)
{ {
if (_coupled_state != nullptr && is_last(v)) if (_coupled_state != nullptr && is_last(v))
{ {
...@@ -280,10 +279,7 @@ public: ...@@ -280,10 +279,7 @@ public:
return false; return false;
} }
if (allow_empty) return _bclabel[r] == _bclabel[nr];
return ((_bclabel[r] == _bclabel[nr]) || (_wr[nr] == 0));
else
return _bclabel[r] == _bclabel[nr];
} }
template <class EFilt> template <class EFilt>
...@@ -1665,8 +1661,6 @@ public: ...@@ -1665,8 +1661,6 @@ public:
enable_partition_stats(); enable_partition_stats();
for (auto& ps : _partition_stats) for (auto& ps : _partition_stats)
actual_B += ps.get_actual_B(); actual_B += ps.get_actual_B();
if (_allow_empty)
actual_B = num_vertices(_bg);
size_t E = _partition_stats.front().get_E(); size_t E = _partition_stats.front().get_E();
dS -= get_edges_dl(actual_B, E, _g); dS -= get_edges_dl(actual_B, E, _g);
dS += get_edges_dl(actual_B + dr + ds, E, _g); dS += get_edges_dl(actual_B + dr + ds, E, _g);
...@@ -2051,8 +2045,6 @@ public: ...@@ -2051,8 +2045,6 @@ public:
size_t actual_B = 0; size_t actual_B = 0;
for (auto& ps : _partition_stats) for (auto& ps : _partition_stats)
actual_B += ps.get_actual_B(); actual_B += ps.get_actual_B();
if (_allow_empty)
actual_B = num_vertices(_bg);
S_dl += get_edges_dl(actual_B, _partition_stats.front().get_E(), _g); S_dl += get_edges_dl(actual_B, _partition_stats.front().get_E(), _g);
} }
...@@ -2381,7 +2373,7 @@ public: ...@@ -2381,7 +2373,7 @@ public:
for (size_t c = 0; c < C; ++c) for (size_t c = 0; c < C; ++c)
_partition_stats.emplace_back(_g, _b, vcs[c], E, B, _partition_stats.emplace_back(_g, _b, vcs[c], E, B,
_vweight, _eweight, _degs, _vweight, _eweight, _degs,
_bmap, _allow_empty); _bmap);
for (auto r : vertices_range(_bg)) for (auto r : vertices_range(_bg))
_partition_stats[rc[r]].get_r(r); _partition_stats[rc[r]].get_r(r);
......
...@@ -95,7 +95,7 @@ struct Gibbs ...@@ -95,7 +95,7 @@ struct Gibbs
{ {
if (nr == null_group) if (nr == null_group)
{ {
if (!_allow_new_group || _state._allow_empty) if (!_allow_new_group)
return numeric_limits<double>::infinity(); return numeric_limits<double>::infinity();
if (_state._empty_blocks.empty()) if (_state._empty_blocks.empty())
_state.add_block(); _state.add_block();
......
...@@ -113,7 +113,7 @@ struct Merge ...@@ -113,7 +113,7 @@ struct Merge
s = uniform_sample(_available, rng); s = uniform_sample(_available, rng);
} }
if (s == r || !_state.allow_move(v, r, s, false)) if (s == r || !_state.allow_move(v, r, s))
return _null_move; return _null_move;
return s; return s;
......
...@@ -71,8 +71,8 @@ public: ...@@ -71,8 +71,8 @@ public:
class Vlist> class Vlist>
partition_stats(Graph& g, Vprop& b, Vlist& vlist, size_t E, size_t B, partition_stats(Graph& g, Vprop& b, Vlist& vlist, size_t E, size_t B,
VWprop& vweight, Eprop& eweight, Degs& degs, VWprop& vweight, Eprop& eweight, Degs& degs,
std::vector<size_t>& bmap, bool allow_empty) std::vector<size_t>& bmap)
: _bmap(bmap), _N(0), _E(E), _total_B(B), _allow_empty(allow_empty) : _bmap(bmap), _N(0), _E(E), _total_B(B)
{ {
if (!use_rmap) if (!use_rmap)
{ {
...@@ -134,10 +134,7 @@ public: ...@@ -134,10 +134,7 @@ public:
double get_partition_dl() double get_partition_dl()
{ {
double S = 0; double S = 0;
if (_allow_empty) S += lbinom(_N - 1, _actual_B - 1);
S += lbinom(_total_B + _N - 1, _N);
else
S += lbinom(_N - 1, _actual_B - 1);
S += lgamma_fast(_N + 1); S += lgamma_fast(_N + 1);
for (auto nr : _total) for (auto nr : _total)
S -= lgamma_fast(nr + 1); S -= lgamma_fast(nr + 1);
...@@ -304,7 +301,7 @@ public: ...@@ -304,7 +301,7 @@ public:
if (nr != null_group && _total[nr] == 0) if (nr != null_group && _total[nr] == 0)
dB++; dB++;
if ((dN != 0 || dB != 0) && !_allow_empty) if ((dN != 0 || dB != 0))
{ {
S_b += lbinom_fast(_N - 1, _actual_B - 1); S_b += lbinom_fast(_N - 1, _actual_B - 1);
S_a += lbinom_fast(_N - 1 + dN, _actual_B + dB - 1); S_a += lbinom_fast(_N - 1 + dN, _actual_B + dB - 1);
...@@ -323,7 +320,7 @@ public: ...@@ -323,7 +320,7 @@ public:
double get_delta_edges_dl(size_t v, size_t r, size_t nr, VProp& vweight, double get_delta_edges_dl(size_t v, size_t r, size_t nr, VProp& vweight,
size_t actual_B, Graph& g) size_t actual_B, Graph& g)
{ {
if (r == nr || _allow_empty) if (r == nr)
return 0; return 0;
if (r != null_group) if (r != null_group)
...@@ -670,7 +667,6 @@ private: ...@@ -670,7 +667,6 @@ private:
size_t _E; size_t _E;
size_t _actual_B; size_t _actual_B;
size_t _total_B; size_t _total_B;
bool _allow_empty;
vector<map_t> _hist; vector<map_t> _hist;
vector<int> _total; vector<int> _total;
vector<int> _ep; vector<int> _ep;
......
...@@ -71,7 +71,7 @@ public: ...@@ -71,7 +71,7 @@ public:
entropy_args_t& ea) = 0; entropy_args_t& ea) = 0;
virtual vprop_map_t<int32_t>::type::unchecked_t& get_b() = 0; virtual vprop_map_t<int32_t>::type::unchecked_t& get_b() = 0;
virtual bool check_edge_counts(bool emat=true) = 0; virtual bool check_edge_counts(bool emat=true) = 0;
virtual bool allow_move(size_t v, size_t r, size_t nr, bool allow_empty = true) = 0; virtual bool allow_move(size_t v, size_t r, size_t nr) = 0;
}; };
} // graph_tool namespace } // graph_tool namespace
......
...@@ -95,13 +95,11 @@ struct Layers ...@@ -95,13 +95,11 @@ struct Layers
{ {
public: public:
LayerState(const BaseState& base_state, LayeredBlockState& lstate, LayerState(const BaseState& base_state, LayeredBlockState& lstate,
bmap_t& block_map, block_rmap_t block_rmap, bmap_t& block_map, block_rmap_t block_rmap, size_t l)
vector<size_t>& free_blocks, size_t l)
: BaseState(base_state), : BaseState(base_state),
_lstate(&lstate), _lstate(&lstate),
_block_map(block_map), _block_map(block_map),
_block_rmap(block_rmap), _block_rmap(block_rmap),
_free_blocks(free_blocks),
_l(l), _E(0) _l(l), _E(0)
{ {
for (auto e : edges_range(BaseState::_g)) for (auto e : edges_range(BaseState::_g))
...@@ -111,12 +109,12 @@ struct Layers ...@@ -111,12 +109,12 @@ struct Layers
LayeredBlockState* _lstate; LayeredBlockState* _lstate;
bmap_t& _block_map; bmap_t& _block_map;
block_rmap_t _block_rmap; block_rmap_t _block_rmap;
vector<size_t>& _free_blocks;
size_t _l; size_t _l;
size_t _E; size_t _E;
using BaseState::_bg; using BaseState::_bg;
using BaseState::_wr; using BaseState::_wr;
using BaseState::_empty_blocks;
using BaseState::add_block; using BaseState::add_block;
size_t get_block_map(size_t r, bool put_new=true) size_t get_block_map(size_t r, bool put_new=true)
...@@ -125,11 +123,19 @@ struct Layers ...@@ -125,11 +123,19 @@ struct Layers
auto iter = _block_map.find(r); auto iter = _block_map.find(r);
if (iter == _block_map.end()) if (iter == _block_map.end())
{ {
if (_free_blocks.empty()) r_u = null_group;
_free_blocks.push_back(_block_map.size()); for (auto s : _empty_blocks)
r_u = _free_blocks.back(); {
while (r_u >= num_vertices(_bg)) if (_block_rmap[s] != -1)
add_block(); continue;
r_u = s;
break;
}
if (r_u == null_group)
{
r_u = add_block();
_block_rmap[r_u] = -1;
}
assert(r_u < num_vertices(_bg)); assert(r_u < num_vertices(_bg));
if (put_new) if (put_new)
...@@ -147,7 +153,6 @@ struct Layers ...@@ -147,7 +153,6 @@ struct Layers
// _lstate->_lcoupled_state->set_block(_l, r_u, hb[r_u]); // _lstate->_lcoupled_state->set_block(_l, r_u, hb[r_u]);
// assert(_lstate->_lcoupled_state->get_block(_l, r_u) == size_t(hb[r_u])); // assert(_lstate->_lcoupled_state->get_block(_l, r_u) == size_t(hb[r_u]));
} }
_free_blocks.pop_back();
assert(_lstate->_lcoupled_state == nullptr || assert(_lstate->_lcoupled_state == nullptr ||
r_u == _lstate->_lcoupled_state->get_layer_node(_l, r)); r_u == _lstate->_lcoupled_state->get_layer_node(_l, r));
// assert(_lstate->_lcoupled_state == nullptr || // assert(_lstate->_lcoupled_state == nullptr ||
...@@ -204,10 +209,8 @@ struct Layers ...@@ -204,10 +209,8 @@ struct Layers
boost::python::object temp = ostate.attr("block_rmap").attr("_get_any")(); boost::python::object temp = ostate.attr("block_rmap").attr("_get_any")();
boost::any& a = python::extract<boost::any&>(temp); boost::any& a = python::extract<boost::any&>(temp);
block_rmap_t block_rmap = boost::any_cast<block_rmap_t>(a); block_rmap_t block_rmap = boost::any_cast<block_rmap_t>(a);
std::vector<size_t>& free_blocks =
python::extract<std::vector<size_t>&>(ostate.attr("free_blocks"));
bmap_t& block_map = _block_map[l]; bmap_t& block_map = _block_map[l];
_layers.emplace_back(state, *this, block_map, block_rmap, free_blocks, l); _layers.emplace_back(state, *this, block_map, block_rmap, l);
} }
for (auto r : vertices_range(BaseState::_bg)) for (auto r : vertices_range(BaseState::_bg))
{ {
...@@ -487,9 +490,9 @@ struct Layers ...@@ -487,9 +490,9 @@ struct Layers
set_partition(b.get_unchecked()); set_partition(b.get_unchecked());
} }
bool allow_move(size_t v, size_t r, size_t nr, bool allow_empty = true) bool allow_move(size_t v, size_t r, size_t nr)
{ {
return BaseState::allow_move(v, r, nr, allow_empty); return BaseState::allow_move(v, r, nr);
} }
template <class MEntries> template <class MEntries>
...@@ -695,11 +698,8 @@ struct Layers ...@@ -695,11 +698,8 @@ struct Layers
if (ea.edges_dl) if (ea.edges_dl)
{ {
size_t actual_B = _actual_B;
if (BaseState::_allow_empty)
actual_B = num_vertices(BaseState::_bg);
for (auto& state : _layers) for (auto& state : _layers)
S_dl += get_edges_dl(actual_B, state._E, _g); S_dl += get_edges_dl(_actual_B, state._E, _g);
} }
if (ea.recs) if (ea.recs)
...@@ -729,16 +729,9 @@ struct Layers ...@@ -729,16 +729,9 @@ struct Layers
for (auto& state : _layers) for (auto& state : _layers)
{ {
size_t actual_B = 0; size_t actual_B = 0;
if (BaseState::_allow_empty) for (auto r : vertices_range(state._bg))
{ if (state._wr[r] > 0)
actual_B = num_vertices(_bg); actual_B++;
}
else
{
for (auto r : vertices_range(state._bg))
if (state._wr[r] > 0)
actual_B++;
}
S_dl += get_edges_dl(actual_B, state._E, _g); S_dl += get_edges_dl(actual_B, state._E, _g);
} }
} }
...@@ -754,7 +747,7 @@ struct Layers ...@@ -754,7 +747,7 @@ struct Layers
double get_delta_edges_dl(size_t v, size_t r, size_t s) double get_delta_edges_dl(size_t v, size_t r, size_t s)
{ {
if (r == s || BaseState::_allow_empty) if (r == s)
return 0; return 0;
if (BaseState::_vweight[v] == 0) if (BaseState::_vweight[v] == 0)
return 0; return 0;
...@@ -768,7 +761,7 @@ struct Layers ...@@ -768,7 +761,7 @@ struct Layers
{ {
auto get_x = [](size_t B) auto get_x = [](size_t B)
{ {
if (is_directed_::apply<typename BaseState::g_t>::type::value) if constexpr (is_directed_::apply<typename BaseState::g_t>::type::value)
return B * B; return B * B;
else else
return (B * (B + 1)) / 2; return (B * (B + 1)) / 2;
...@@ -957,15 +950,15 @@ struct Layers ...@@ -957,15 +950,15 @@ struct Layers
size_t add_block() size_t add_block()
{ {
auto r = BaseState::add_block(); return BaseState::add_block();
for (size_t l = 0; l < _layers.size(); ++l) // for (size_t l = 0; l < _layers.size(); ++l)
{ // {
auto& state = _layers[l]; // auto& state = _layers[l];
size_t r_u = state.add_block(); // size_t r_u = state.add_block();
if (_lcoupled_state != nullptr) // if (_lcoupled_state != nullptr)
_lcoupled_state->get_layer(l).coupled_resize_vertex(r_u); // _lcoupled_state->get_layer(l).coupled_resize_vertex(r_u);
} // }
return r; // return r;
} }
void coupled_resize_vertex(size_t v) void coupled_resize_vertex(size_t v)
......
...@@ -50,7 +50,10 @@ typedef mpl::vector2<std::true_type, std::false_type> use_hash_tr; ...@@ -50,7 +50,10 @@ typedef mpl::vector2<std::true_type, std::false_type> use_hash_tr;
((mrm,, vmap_t, 0)) \ ((mrm,, vmap_t, 0)) \
((wr,, vmap_t, 0)) \ ((wr,, vmap_t, 0)) \
((b,, vmap_t, 0)) \ ((b,, vmap_t, 0)) \
((empty_blocks, & ,std::vector<size_t>&, 0)) \
((empty_pos,, vmap_t, 0)) \
((candidate_blocks, &, std::vector<size_t>&, 0)) \ ((candidate_blocks, &, std::vector<size_t>&, 0)) \
((candidate_pos,, vmap_t, 0)) \
((bclabel,, vmap_t, 0)) \ ((bclabel,, vmap_t, 0)) \
((pclabel,, vmap_t, 0)) \