If you want to fork this repository to propose merge requests, please send an email to tiago@skewed.de, and your project limit will be raised.

Commit d25f8d92 authored by Tiago Peixoto's avatar Tiago Peixoto

inference: Implement code for uncertain networks

parent 2d5051a9
......@@ -54,6 +54,10 @@ libgraph_tool_inference_la_SOURCES = \
layers/graph_blockmodel_layers_overlap_multicanonical_multiflip.cc \
layers/graph_blockmodel_layers_overlap_multiflip_mcmc.cc \
layers/graph_blockmodel_layers_overlap_vacate.cc \
uncertain/graph_blockmodel_measured.cc \
uncertain/graph_blockmodel_measured_mcmc.cc \
uncertain/graph_blockmodel_uncertain.cc \
uncertain/graph_blockmodel_uncertain_mcmc.cc \
support/cache.cc \
support/int_part.cc \
support/spence.cc \
......@@ -76,22 +80,25 @@ libgraph_tool_inference_la_include_HEADERS = \
blockmodel/graph_blockmodel_partition.hh \
blockmodel/graph_blockmodel_util.hh \
blockmodel/graph_blockmodel_weights.hh \
graph_modularity.hh \
overlap/graph_blockmodel_overlap.hh \
overlap/graph_blockmodel_overlap_mcmc_bundled.hh \
overlap/graph_blockmodel_overlap_util.hh \
overlap/graph_blockmodel_overlap_vacate.hh \
overlap/graph_blockmodel_overlap_partition.hh \
layers/graph_blockmodel_layers.hh \
layers/graph_blockmodel_layers_util.hh \
uncertain/graph_blockmodel_measured.hh \
uncertain/graph_blockmodel_uncertain.hh \
uncertain/graph_blockmodel_uncertain_mcmc.hh \
loops/bundled_vacate_loop.hh \
loops/exhaustive_loop.hh \
loops/gibbs_loop.hh \
loops/mcmc_loop.hh \
loops/merge_loop.hh \
overlap/graph_blockmodel_overlap.hh \
overlap/graph_blockmodel_overlap_mcmc_bundled.hh \
overlap/graph_blockmodel_overlap_util.hh \
overlap/graph_blockmodel_overlap_vacate.hh \
overlap/graph_blockmodel_overlap_partition.hh \
support/cache.hh \
support/graph_neighbor_sampler.hh \
support/graph_state.hh \
support/int_part.hh \
support/parallel_rng.hh \
support/util.hh
support/util.hh \
graph_modularity.hh
......@@ -54,15 +54,15 @@ class EGroups
{
public:
template <class Vprop, class Eprop, class BGraph>
void init(Vprop b, Eprop eweight, Graph& g, BGraph& bg)
void init(Vprop b, Eprop& eweight, Graph& g, BGraph& bg)
{
_egroups.clear();
_egroups.resize(num_vertices(bg));
for (auto e : edges_range(g))
{
_epos[e] = make_pair(numeric_limits<size_t>::max(),
numeric_limits<size_t>::max());
_epos[e] = {numeric_limits<size_t>::max(),
numeric_limits<size_t>::max()};
insert_edge(e, eweight[e], b, g);
}
}
......@@ -77,12 +77,31 @@ public:
return _egroups.empty();
}
template <class Vprop>
bool check(Vprop b, Graph& g)
template <class Vprop, class Eprop>
bool check(Vprop b, Eprop& eweight, Graph& g) const
{
for (auto e : edges_range(g))
{
if (eweight[e] == 0)
continue;
auto r = b[get_source(e, g)];
auto s = b[get_target(e, g)];
const auto& pos = _epos[e];
if (!(pos.first < _egroups[r].size() &&
is_valid(pos.first, _egroups[r]) &&
get<0>(_egroups[r][pos.first]) == e &&
pos.second < _egroups[s].size() &&
is_valid(pos.second, _egroups[s]) &&
get<0>(_egroups[s][pos.second]) == e))
{
assert(false);
return false;
}
}
for (size_t r = 0; r < _egroups.size(); ++r)
{
auto& edges = _egroups[r];
const auto& edges = _egroups[r];
for (size_t i = 0; i < edges.size(); ++i)
{
const auto& e = edges[i];
......@@ -100,13 +119,13 @@ public:
}
template <class Edge>
bool is_valid(size_t i, DynamicSampler<Edge>& elist)
bool is_valid(size_t i, const DynamicSampler<Edge>& elist) const
{
return elist.is_valid(i);
}
template <class Edge>
bool is_valid(size_t, vector<Edge>&)
bool is_valid(size_t, const vector<Edge>&) const
{
return true;
}
......
......@@ -79,6 +79,7 @@ struct MCMC
typename state_t::g_t& _g;
typename state_t::m_entries_t _m_entries;
size_t _null_move = null_group;
size_t node_state(size_t v)
{
......
......@@ -71,6 +71,7 @@ struct Multicanonical
int _i;
double _dS;
size_t _null_move = null_group;
int get_bin(double S)
{
......
......@@ -36,6 +36,7 @@ typedef vprop_map_t<std::vector<std::tuple<size_t, size_t, size_t>>>::type
struct simple_degs_t {};
template <class Graph, class Vprop, class Eprop, class F>
__attribute__((always_inline)) __attribute__((flatten)) inline
void degs_op(size_t v, Vprop& vweight, Eprop& eweight, const simple_degs_t&,
Graph& g, F&& f)
{
......@@ -43,6 +44,7 @@ void degs_op(size_t v, Vprop& vweight, Eprop& eweight, const simple_degs_t&,
}
template <class Graph, class Vprop, class Eprop, class F>
__attribute__((always_inline)) __attribute__((flatten)) inline
void degs_op(size_t v, Vprop& vweight, Eprop& eweight,
const typename degs_map_t::unchecked_t& degs, Graph& g, F&& f)
{
......@@ -139,67 +141,111 @@ public:
return S;
}
double get_deg_dl_ent()
double get_deg_dl_ent(auto&& rs, auto&& ks)
{
double S = 0;
for (size_t r = 0; r < _ep.size(); ++r)
for (auto r : rs)
{
r = get_r(r);
size_t total = 0;
for (auto& k_c : _hist[r])
if (ks.empty())
{
S -= xlogx_fast(k_c.second);
total += k_c.second;
for (auto& k_c : _hist[r])
{
S -= xlogx_fast(k_c.second);
total += k_c.second;
}
}
else
{
auto& h = _hist[r];
for (auto& k : ks)
{
auto iter = h.find(k);
auto k_c = (iter != h.end()) ? iter->second : 0;
S -= xlogx(k_c);
}
total = _total[r];
}
S += xlogx_fast(total);
}
return S;
}
double get_deg_dl_uniform()
double get_deg_dl_uniform(auto&& rs, auto&&)
{
double S = 0;
for (size_t r = 0; r < _ep.size(); ++r)
for (auto r : rs)
{
r = get_r(r);
S += lbinom(_total[r] + _ep[r] - 1, _ep[r]);
S += lbinom(_total[r] + _em[r] - 1, _em[r]);
}
return S;
}
double get_deg_dl_dist()
double get_deg_dl_dist(auto&& rs, auto&& ks)
{
double S = 0;
for (size_t r = 0; r < _ep.size(); ++r)
for (auto r : rs)
{
r = get_r(r);
S += log_q(_ep[r], _total[r]);
S += log_q(_em[r], _total[r]);
size_t total = 0;
for (auto& k_c : _hist[r])
if (ks.empty())
{
S -= lgamma_fast(k_c.second + 1);
total += k_c.second;
for (auto& k_c : _hist[r])
{
S -= lgamma_fast(k_c.second + 1);
total += k_c.second;
}
}
else
{
auto& h = _hist[r];
for (auto& k : ks)
{
auto iter = h.find(k);
auto k_c = (iter != h.end()) ? iter->second : 0;
S -= lgamma_fast(k_c + 1);
}
total = _total[r];
}
S += lgamma_fast(total + 1);
}
return S;
}
double get_deg_dl(int kind)
double get_deg_dl(int kind, auto&& rs, auto&& ks)
{
switch (kind)
{
case deg_dl_kind::ENT:
return get_deg_dl_ent();
return get_deg_dl_ent(rs, ks);
case deg_dl_kind::UNIFORM:
return get_deg_dl_uniform();
return get_deg_dl_uniform(rs, ks);
case deg_dl_kind::DIST:
return get_deg_dl_dist();
return get_deg_dl_dist(rs, ks);
default:
return numeric_limits<double>::quiet_NaN();
}
}
double get_deg_dl(int kind)
{
return get_deg_dl(kind, boost::counting_range(size_t(0), _total_B),
std::array<std::pair<size_t,size_t>,0>());
}
template <class Graph>
double get_edges_dl(size_t B, Graph& g)
{
size_t BB = (graph_tool::is_directed(g)) ? B * B : (B * (B + 1)) / 2;
return lbinom(BB + _E - 1, _E);
}
template <class VProp>
double get_delta_partition_dl(size_t v, size_t r, size_t nr, VProp& vweight)
{
......@@ -297,16 +343,8 @@ public:
if (dB != 0)
{
auto get_x = [&g](size_t B)
{
if (graph_tool::is_directed(g))
return B * B;
else
return (B * (B + 1)) / 2;
};
S_b += lbinom_fast<false>(get_x(actual_B) + _E - 1, _E);
S_a += lbinom_fast<false>(get_x(actual_B + dB) + _E - 1, _E);
S_b += get_edges_dl(actual_B, g);
S_a += get_edges_dl(actual_B + dB, g);
}
return S_a - S_b;
......@@ -558,7 +596,10 @@ public:
if (_ignore_degree[v] == 2)
kout = 0;
auto deg = make_pair(kin, kout);
_hist[r][deg] += diff * vweight;
auto iter = _hist[r].insert({deg, 0}).first;
iter->second += diff * vweight;
if (iter->second == 0)
_hist[r].erase(iter);
_em[r] += diff * deg.first * vweight;
_ep[r] += diff * deg.second * vweight;
}
......@@ -589,6 +630,50 @@ public:
_total_B++;
}
template <class Graph, class VProp, class VWeight, class EWeight, class Degs>
bool check_degs(Graph& g, VProp& b, VWeight& vweight, EWeight& eweight, Degs& degs)
{
vector<map_t> dhist;
for (auto v : vertices_range(g))
{
degs_op(v, vweight, eweight, degs, g,
[&](auto kin, auto kout, auto n)
{
auto r = get_r(b[v]);
if (r >= dhist.size())
dhist.resize(r + 1);
dhist[r][{kin, kout}] += n;
});
}
for (size_t r = 0; r < dhist.size(); ++r)
{
for (auto& kn : dhist[r])
{
auto count = (r >= _hist.size()) ? 0 : _hist[r][kn.first];
if (kn.second != count)
{
assert(false);
return false;
}
}
}
for (size_t r = 0; r < _hist.size(); ++r)
{
for (auto& kn : _hist[r])
{
auto count = (r >= dhist.size()) ? 0 : dhist[r][kn.first];
if (kn.second != count)
{
assert(false);
return false;
}
}
}
return true;
}
private:
vector<size_t>& _bmap;
......
......@@ -46,6 +46,11 @@ public:
virtual void remove_edge_rec(const GraphInterface::edge_t& e) = 0;
virtual void update_edge_rec(const GraphInterface::edge_t& e,
const std::vector<double>& delta) = 0;
virtual void add_edge(size_t u, size_t v, GraphInterface::edge_t& e,
const std::vector<double>& rec) = 0;
virtual void remove_edge(size_t u, size_t v, GraphInterface::edge_t& e,
const std::vector<double>& rec) = 0;
virtual double edge_entropy_term(size_t u, size_t v, entropy_args_t ea) = 0;
virtual double recs_dS(size_t, size_t,
const std::vector<std::tuple<size_t, size_t,
GraphInterface::edge_t, int,
......
......@@ -104,6 +104,10 @@ extern void export_blockmodel_exhaustive();
extern void export_overlap_blockmodel_exhaustive();
extern void export_layered_blockmodel_exhaustive();
extern void export_layered_overlap_blockmodel_exhaustive();
extern void export_uncertain_state();
extern void export_uncertain_mcmc();
extern void export_measured_state();
extern void export_measured_mcmc();
extern void export_marginals();
extern void export_modularity();
......@@ -145,6 +149,10 @@ BOOST_PYTHON_MODULE(libgraph_tool_inference)
export_overlap_blockmodel_exhaustive();
export_layered_blockmodel_exhaustive();
export_layered_overlap_blockmodel_exhaustive();
export_uncertain_state();
export_uncertain_mcmc();
export_measured_state();
export_measured_mcmc();
export_marginals();
export_modularity();
......
......@@ -763,6 +763,8 @@ struct Layers
}
}
double edge_entropy_term(size_t u, size_t v, entropy_args_t ea) { return 0; }
void enable_partition_stats()
{
if (!_is_partition_stats_enabled)
......@@ -969,6 +971,16 @@ struct Layers
BaseState::update_edge_rec(e, delta);
}
void add_edge(size_t u, size_t v, GraphInterface::edge_t& e,
const std::vector<double>& rec)
{
}
void remove_edge(size_t u, size_t v, GraphInterface::edge_t& e,
const std::vector<double>& rec)
{
}
double recs_dS(size_t, size_t,
const std::vector<std::tuple<size_t, size_t,
GraphInterface::edge_t, int,
......
......@@ -50,9 +50,8 @@ bool metropolis_accept(double dS, double mP, double beta, RNG& rng)
}
else
{
typedef std::uniform_real_distribution<> rdist_t;
double sample = rdist_t()(rng);
return sample < exp(a);
std::uniform_real_distribution<> sample;
return sample(rng) < exp(a);
}
}
}
......@@ -75,19 +74,17 @@ auto mcmc_sweep(MCMCState state, RNG& rng)
for (size_t vi = 0; vi < vlist.size(); ++vi)
{
size_t v;
if (state.is_sequential())
v = vlist[vi];
else
v = uniform_sample(vlist, rng);
auto&& v = (state.is_sequential()) ?
vlist[vi] : uniform_sample(vlist, rng);
if (state.skip_node(v))
continue;
auto r = state.node_state(v);
auto s = state.move_proposal(v, rng);
auto r = (state._verbose) ? state.node_state(v)
: decltype(state.node_state(v))();
auto&& s = state.move_proposal(v, rng);
if (s == null_group)
if (s == state._null_move)
continue;
double dS, mP;
......
......@@ -505,6 +505,16 @@ public:
move_vertices(vs, rs);
}
void add_edge(size_t, size_t, GraphInterface::edge_t&,
const std::vector<double>&)
{
}
void remove_edge(size_t, size_t, GraphInterface::edge_t&,
const std::vector<double>&)
{
}
template <class VMap>
void set_partition(VMap&& b)
{
......@@ -1325,6 +1335,8 @@ public:
return S;
}
double edge_entropy_term(size_t u, size_t v, entropy_args_t ea) { return 0; }
void enable_partition_stats()
{
if (_partition_stats.empty())
......@@ -1624,8 +1636,8 @@ public:
void remove_partition_node(size_t, size_t) { }
void set_vertex_weight(size_t, int) { }
void coupled_resize_vertex(size_t) { }
void update_edge(const GraphInterface::edge_t&,
const std::vector<double>&) { }
void update_block_edge(const GraphInterface::edge_t&,
const std::vector<double>&) { }
//private:
typedef typename
......
......@@ -105,6 +105,7 @@ struct MCMC
std::vector<std::vector<size_t>> _half_edges;
std::vector<std::vector<size_t>> _bundles;
bool _parallel;
size_t _null_move = null_group;
size_t node_state(size_t i)
{
......
......@@ -25,6 +25,8 @@
#include "cache.hh"
#include <boost/range/counting_range.hpp>
namespace graph_tool
{
using namespace boost;
......
// graph-tool -- a general graph modification and manipulation thingy
//
// Copyright (C) 2006-2017 Tiago de Paula Peixoto <tiago@skewed.de>
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 3
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#define BOOST_PYTHON_MAX_ARITY 40
#include <boost/python.hpp>
#include "graph_tool.hh"
#include "random.hh"
#include "../blockmodel/graph_blockmodel.hh"
#define BASE_STATE_params BLOCK_STATE_params
#include "graph_blockmodel_measured.hh"
#include "../support/graph_state.hh"
using namespace boost;
using namespace graph_tool;
GEN_DISPATCH(block_state, BlockState, BLOCK_STATE_params)
template <class BaseState>
GEN_DISPATCH(measured_state, Measured<BaseState>::template MeasuredState,
MEASURED_STATE_params)
python::object make_measured_state(boost::python::object oblock_state,
boost::python::object omeasured_state)
{
python::object state;
auto dispatch = [&](auto& block_state)
{
typedef typename std::remove_reference<decltype(block_state)>::type
state_t;
measured_state<state_t>::make_dispatch
(omeasured_state,
[&](auto& s)
{
state = python::object(s);
},
block_state);
};
block_state::dispatch(oblock_state, dispatch);
return state;
}
void export_measured_state()
{
using namespace boost::python;
def("make_measured_state", &make_measured_state);
block_state::dispatch
([&](auto* bs)
{
typedef typename std::remove_reference<decltype(*bs)>::type block_state_t;
measured_state<block_state_t>::dispatch
([&](auto* s)
{
typedef typename std::remove_reference<decltype(*s)>::type state_t;
class_<state_t>
c(name_demangle(typeid(state_t).name()).c_str(),
no_init);
c.def("remove_edge", &state_t::remove_edge)
.def("add_edge", &state_t::add_edge)
.def("remove_edge_dS", &state_t::remove_edge_dS)
.def("add_edge_dS", &state_t::add_edge_dS)
.def("entropy", &state_t::entropy)
.def("set_hparams", &state_t::set_hparams)
.def("get_N", &state_t::get_N)
.def("get_X", &state_t::get_X)
.def("get_T", &state_t::get_T)
.def("get_M", &state_t::get_M);
});
});
}
// graph-tool -- a general graph modification and manipulation thingy
//
// Copyright (C) 2006-2017 Tiago de Paula Peixoto <tiago@skewed.de>
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 3
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#ifndef GRAPH_BLOCKMODEL_MEASURED_HH
#define GRAPH_BLOCKMODEL_MEASURED_HH
#include "config.h"
#include <vector>
#include "../support/graph_state.hh"
#include "../blockmodel/graph_blockmodel_util.hh"
namespace graph_tool
{
using namespace boost;
using namespace std;
#define MEASURED_STATE_params \
((__class__,&, mpl::vector<python::object>, 1)) \
((g, &, all_graph_views, 1)) \
((n,, eprop_map_t<int>::type, 0)) \
((x,, eprop_map_t<int>::type, 0)) \
((n_default,, int, 0)) \
((x_default,, int, 0)) \
((alpha,, double, 0)) \
((beta,, double, 0)) \
((mu,, double, 0)) \
((nu,, double, 0)) \
((aE,, double, 0)) \
((self_loops,, bool, 0))
template <class BlockState>
struct Measured
{
GEN_STATE_BASE(MeasuredStateBase, MEASURED_STATE_params)
template <class... Ts>
class MeasuredState
: public MeasuredStateBase<Ts...>
{
public:
GET_PARAMS_USING(MeasuredStateBase<Ts...>,
MEASURED_STATE_params)
GET_PARAMS_TYPEDEF(Ts, MEASURED_STATE_params)
template <class... ATs,
typename std::enable_if_t<sizeof...(ATs) == sizeof...(Ts)>* = nullptr>
MeasuredState(BlockState& block_state, ATs&&... args)
: MeasuredStateBase<Ts...>(std::forward<ATs>(args)...),
_block_state(block_state)
{
_u_edges.resize(num_vertices(_u));
for (auto e : edges_range(_u))
{
get_u_edge