inference: Implement code for uncertain networks

src/graph/inference/Makefile.am

@@ -54,6 +54,10 @@ libgraph_tool_inference_la_SOURCES = \
     layers/graph_blockmodel_layers_overlap_multicanonical_multiflip.cc \
     layers/graph_blockmodel_layers_overlap_multiflip_mcmc.cc \
     layers/graph_blockmodel_layers_overlap_vacate.cc \
+    uncertain/graph_blockmodel_measured.cc \
+    uncertain/graph_blockmodel_measured_mcmc.cc \
+    uncertain/graph_blockmodel_uncertain.cc \
+    uncertain/graph_blockmodel_uncertain_mcmc.cc \
     support/cache.cc \
     support/int_part.cc \
     support/spence.cc \
@@ -76,22 +80,25 @@ libgraph_tool_inference_la_include_HEADERS = \
     blockmodel/graph_blockmodel_partition.hh \
     blockmodel/graph_blockmodel_util.hh \
     blockmodel/graph_blockmodel_weights.hh \
-    graph_modularity.hh \
+    overlap/graph_blockmodel_overlap.hh \
+    overlap/graph_blockmodel_overlap_mcmc_bundled.hh \
+    overlap/graph_blockmodel_overlap_util.hh \
+    overlap/graph_blockmodel_overlap_vacate.hh \
+    overlap/graph_blockmodel_overlap_partition.hh \
     layers/graph_blockmodel_layers.hh \
     layers/graph_blockmodel_layers_util.hh \
+    uncertain/graph_blockmodel_measured.hh \
+    uncertain/graph_blockmodel_uncertain.hh \
+    uncertain/graph_blockmodel_uncertain_mcmc.hh \
     loops/bundled_vacate_loop.hh \
     loops/exhaustive_loop.hh \
     loops/gibbs_loop.hh \
     loops/mcmc_loop.hh \
     loops/merge_loop.hh \
-    overlap/graph_blockmodel_overlap.hh \
-    overlap/graph_blockmodel_overlap_mcmc_bundled.hh \
-    overlap/graph_blockmodel_overlap_util.hh \
-    overlap/graph_blockmodel_overlap_vacate.hh \
-    overlap/graph_blockmodel_overlap_partition.hh \
     support/cache.hh \
     support/graph_neighbor_sampler.hh \
     support/graph_state.hh \
     support/int_part.hh \
     support/parallel_rng.hh \
-    support/util.hh
+    support/util.hh \
+    graph_modularity.hh

src/graph/inference/blockmodel/graph_blockmodel_elist.hh

@@ -54,15 +54,15 @@ class EGroups
 {
 public:
     template <class Vprop, class Eprop, class BGraph>
-    void init(Vprop b, Eprop eweight, Graph& g, BGraph& bg)
+    void init(Vprop b, Eprop& eweight, Graph& g, BGraph& bg)
     {
         _egroups.clear();
         _egroups.resize(num_vertices(bg));
 
         for (auto e : edges_range(g))
         {
-            _epos[e] = make_pair(numeric_limits<size_t>::max(),
-                                 numeric_limits<size_t>::max());
+            _epos[e] = {numeric_limits<size_t>::max(),
+                        numeric_limits<size_t>::max()};
             insert_edge(e, eweight[e], b, g);
         }
     }
@@ -77,12 +77,31 @@ public:
         return _egroups.empty();
     }
 
-    template <class Vprop>
-    bool check(Vprop b, Graph& g)
+    template <class Vprop, class Eprop>
+    bool check(Vprop b, Eprop& eweight, Graph& g) const
     {
+        for (auto e : edges_range(g))
+        {
+            if (eweight[e] == 0)
+                continue;
+            auto r = b[get_source(e, g)];
+            auto s = b[get_target(e, g)];
+            const auto& pos = _epos[e];
+            if (!(pos.first < _egroups[r].size() &&
+                  is_valid(pos.first, _egroups[r]) &&
+                  get<0>(_egroups[r][pos.first]) == e &&
+                  pos.second < _egroups[s].size() &&
+                  is_valid(pos.second, _egroups[s]) &&
+                  get<0>(_egroups[s][pos.second]) == e))
+            {
+                assert(false);
+                return false;
+            }
+        }
+
         for (size_t r = 0; r < _egroups.size(); ++r)
         {
-            auto& edges = _egroups[r];
+            const auto& edges = _egroups[r];
             for (size_t i = 0; i < edges.size(); ++i)
             {
                 const auto& e = edges[i];
@@ -100,13 +119,13 @@ public:
     }
 
     template <class Edge>
-    bool is_valid(size_t i, DynamicSampler<Edge>& elist)
+    bool is_valid(size_t i, const DynamicSampler<Edge>& elist) const
     {
         return elist.is_valid(i);
     }
 
     template <class Edge>
-    bool is_valid(size_t, vector<Edge>&)
+    bool is_valid(size_t, const vector<Edge>&) const
     {
         return true;
     }

src/graph/inference/blockmodel/graph_blockmodel_mcmc.hh

@@ -79,6 +79,7 @@ struct MCMC
 
         typename state_t::g_t& _g;
         typename state_t::m_entries_t _m_entries;
+        size_t _null_move = null_group;
 
         size_t node_state(size_t v)
         {

src/graph/inference/blockmodel/graph_blockmodel_multicanonical.hh

@@ -71,6 +71,7 @@ struct Multicanonical
 
         int _i;
         double _dS;
+        size_t _null_move = null_group;
 
         int get_bin(double S)
         {

src/graph/inference/blockmodel/graph_blockmodel_partition.hh

@@ -36,6 +36,7 @@ typedef vprop_map_t<std::vector<std::tuple<size_t, size_t, size_t>>>::type
 struct simple_degs_t {};
 
 template <class Graph, class Vprop, class Eprop, class F>
+__attribute__((always_inline)) __attribute__((flatten)) inline
 void degs_op(size_t v, Vprop& vweight, Eprop& eweight, const simple_degs_t&,
              Graph& g, F&& f)
 {
@@ -43,6 +44,7 @@ void degs_op(size_t v, Vprop& vweight, Eprop& eweight, const simple_degs_t&,
 }
 
 template <class Graph, class Vprop, class Eprop, class F>
+__attribute__((always_inline)) __attribute__((flatten)) inline
 void degs_op(size_t v, Vprop& vweight, Eprop& eweight,
              const typename degs_map_t::unchecked_t& degs, Graph& g, F&& f)
 {
@@ -139,67 +141,111 @@ public:
         return S;
     }
 
-    double get_deg_dl_ent()
+    double get_deg_dl_ent(auto&& rs, auto&& ks)
     {
         double S = 0;
-        for (size_t r = 0; r < _ep.size(); ++r)
+        for (auto r : rs)
         {
+            r = get_r(r);
             size_t total = 0;
-            for (auto& k_c : _hist[r])
+            if (ks.empty())
             {
-                S -= xlogx_fast(k_c.second);
-                total += k_c.second;
+                for (auto& k_c : _hist[r])
+                {
+                    S -= xlogx_fast(k_c.second);
+                    total += k_c.second;
+                }
+            }
+            else
+            {
+                auto& h = _hist[r];
+                for (auto& k : ks)
+                {
+                    auto iter = h.find(k);
+                    auto k_c = (iter != h.end()) ? iter->second : 0;
+                    S -= xlogx(k_c);
+                }
+                total = _total[r];
             }
             S += xlogx_fast(total);
         }
         return S;
     }
 
-    double get_deg_dl_uniform()
+    double get_deg_dl_uniform(auto&& rs, auto&&)
     {
         double S = 0;
-        for (size_t r = 0; r < _ep.size(); ++r)
+        for (auto r : rs)
         {
+            r = get_r(r);
             S += lbinom(_total[r] + _ep[r] - 1, _ep[r]);
             S += lbinom(_total[r] + _em[r] - 1, _em[r]);
         }
         return S;
     }
 
-    double get_deg_dl_dist()
+    double get_deg_dl_dist(auto&& rs, auto&& ks)
     {
         double S = 0;
-        for (size_t r = 0; r < _ep.size(); ++r)
+        for (auto r : rs)
         {
+            r = get_r(r);
             S += log_q(_ep[r], _total[r]);
             S += log_q(_em[r], _total[r]);
 
             size_t total = 0;
-            for (auto& k_c : _hist[r])
+            if (ks.empty())
             {
-                S -= lgamma_fast(k_c.second + 1);
-                total += k_c.second;
+                for (auto& k_c : _hist[r])
+                {
+                    S -= lgamma_fast(k_c.second + 1);
+                    total += k_c.second;
+                }
+            }
+            else
+            {
+                auto& h = _hist[r];
+                for (auto& k : ks)
+                {
+                    auto iter = h.find(k);
+                    auto k_c = (iter != h.end()) ? iter->second : 0;
+                    S -= lgamma_fast(k_c + 1);
+                }
+                total = _total[r];
             }
             S += lgamma_fast(total + 1);
         }
         return S;
     }
 
-    double get_deg_dl(int kind)
+    double get_deg_dl(int kind, auto&& rs, auto&& ks)
     {
         switch (kind)
         {
         case deg_dl_kind::ENT:
-            return get_deg_dl_ent();
+            return get_deg_dl_ent(rs, ks);
         case deg_dl_kind::UNIFORM:
-            return get_deg_dl_uniform();
+            return get_deg_dl_uniform(rs, ks);
         case deg_dl_kind::DIST:
-            return get_deg_dl_dist();
+            return get_deg_dl_dist(rs, ks);
         default:
             return numeric_limits<double>::quiet_NaN();
         }
     }
 
+    double get_deg_dl(int kind)
+    {
+        return get_deg_dl(kind, boost::counting_range(size_t(0), _total_B),
+                          std::array<std::pair<size_t,size_t>,0>());
+    }
+
+    template <class Graph>
+    double get_edges_dl(size_t B, Graph& g)
+    {
+        size_t BB = (graph_tool::is_directed(g)) ? B * B : (B * (B + 1)) / 2;
+        return lbinom(BB + _E - 1, _E);
+    }
+
     template <class VProp>
     double get_delta_partition_dl(size_t v, size_t r, size_t nr, VProp& vweight)
     {
@@ -297,16 +343,8 @@ public:
 
         if (dB != 0)
         {
-            auto get_x = [&g](size_t B)
-                {
-                    if (graph_tool::is_directed(g))
-                        return B * B;
-                    else
-                        return (B * (B + 1)) / 2;
-                };
-
-            S_b += lbinom_fast<false>(get_x(actual_B) + _E - 1, _E);
-            S_a += lbinom_fast<false>(get_x(actual_B + dB) + _E - 1, _E);
+            S_b += get_edges_dl(actual_B, g);
+            S_a += get_edges_dl(actual_B + dB, g);
         }
 
         return S_a - S_b;
@@ -558,7 +596,10 @@ public:
             if (_ignore_degree[v] == 2)
                 kout = 0;
             auto deg = make_pair(kin, kout);
-            _hist[r][deg] += diff * vweight;
+            auto iter = _hist[r].insert({deg, 0}).first;
+            iter->second += diff * vweight;
+            if (iter->second == 0)
+                _hist[r].erase(iter);
             _em[r] += diff * deg.first * vweight;
             _ep[r] += diff * deg.second * vweight;
         }
@@ -589,6 +630,50 @@ public:
         _total_B++;
     }
 
+    template <class Graph, class VProp, class VWeight, class EWeight, class Degs>
+    bool check_degs(Graph& g, VProp& b, VWeight& vweight, EWeight& eweight, Degs& degs)
+    {
+        vector<map_t> dhist;
+        for (auto v : vertices_range(g))
+        {
+            degs_op(v, vweight, eweight, degs, g,
+                    [&](auto kin, auto kout, auto n)
+                    {
+                        auto r = get_r(b[v]);
+                        if (r >= dhist.size())
+                            dhist.resize(r + 1);
+                        dhist[r][{kin, kout}] += n;
+                    });
+        }
+
+        for (size_t r = 0; r < dhist.size(); ++r)
+        {
+            for (auto& kn : dhist[r])
+            {
+                auto count = (r >= _hist.size()) ? 0 : _hist[r][kn.first];
+                if (kn.second != count)
+                {
+                    assert(false);
+                    return false;
+                }
+            }
+        }
+
+        for (size_t r = 0; r < _hist.size(); ++r)
+        {
+            for (auto& kn : _hist[r])
+            {
+                auto count = (r >= dhist.size()) ? 0 : dhist[r][kn.first];
+                if (kn.second != count)
+                {
+                    assert(false);
+                    return false;
+                }
+            }
+        }
+
+        return true;
+    }
 
 private:
     vector<size_t>& _bmap;

src/graph/inference/blockmodel/graph_blockmodel_util.hh

@@ -46,6 +46,11 @@ public:
     virtual void remove_edge_rec(const GraphInterface::edge_t& e) = 0;
     virtual void update_edge_rec(const GraphInterface::edge_t& e,
                                  const std::vector<double>& delta) = 0;
+    virtual void add_edge(size_t u, size_t v, GraphInterface::edge_t& e,
+                          const std::vector<double>& rec) = 0;
+    virtual void remove_edge(size_t u, size_t v, GraphInterface::edge_t& e,
+                             const std::vector<double>& rec) = 0;
+    virtual double edge_entropy_term(size_t u, size_t v, entropy_args_t ea) = 0;
     virtual double recs_dS(size_t, size_t,
                            const std::vector<std::tuple<size_t, size_t,
                                              GraphInterface::edge_t, int,

src/graph/inference/graph_inference.cc

@@ -104,6 +104,10 @@ extern void export_blockmodel_exhaustive();
 extern void export_overlap_blockmodel_exhaustive();
 extern void export_layered_blockmodel_exhaustive();
 extern void export_layered_overlap_blockmodel_exhaustive();
+extern void export_uncertain_state();
+extern void export_uncertain_mcmc();
+extern void export_measured_state();
+extern void export_measured_mcmc();
 extern void export_marginals();
 extern void export_modularity();
 
@@ -145,6 +149,10 @@ BOOST_PYTHON_MODULE(libgraph_tool_inference)
     export_overlap_blockmodel_exhaustive();
     export_layered_blockmodel_exhaustive();
     export_layered_overlap_blockmodel_exhaustive();
+    export_uncertain_state();
+    export_uncertain_mcmc();
+    export_measured_state();
+    export_measured_mcmc();
     export_marginals();
     export_modularity();
 

src/graph/inference/layers/graph_blockmodel_layers.hh

@@ -763,6 +763,8 @@ struct Layers
             }
         }
 
+        double edge_entropy_term(size_t u, size_t v, entropy_args_t ea) { return 0; }
+
         void enable_partition_stats()
         {
             if (!_is_partition_stats_enabled)
@@ -969,6 +971,16 @@ struct Layers
             BaseState::update_edge_rec(e, delta);
         }
 
+        void add_edge(size_t u, size_t v, GraphInterface::edge_t& e,
+                      const std::vector<double>& rec)
+        {
+        }
+
+        void remove_edge(size_t u, size_t v, GraphInterface::edge_t& e,
+                         const std::vector<double>& rec)
+        {
+        }
+
         double recs_dS(size_t, size_t,
                        const std::vector<std::tuple<size_t, size_t,
                                                     GraphInterface::edge_t, int,

src/graph/inference/loops/mcmc_loop.hh

@@ -50,9 +50,8 @@ bool metropolis_accept(double dS, double mP, double beta, RNG& rng)
         }
         else
         {
-            typedef std::uniform_real_distribution<> rdist_t;
-            double sample = rdist_t()(rng);
-            return sample < exp(a);
+            std::uniform_real_distribution<> sample;
+            return sample(rng) < exp(a);
         }
     }
 }
@@ -75,19 +74,17 @@ auto mcmc_sweep(MCMCState state, RNG& rng)
 
         for (size_t vi = 0; vi < vlist.size(); ++vi)
         {
-            size_t v;
-            if (state.is_sequential())
-                v = vlist[vi];
-            else
-                v = uniform_sample(vlist, rng);
+            auto&& v = (state.is_sequential()) ?
+                vlist[vi] : uniform_sample(vlist, rng);
 
             if (state.skip_node(v))
                 continue;
 
-            auto r = state.node_state(v);
-            auto s = state.move_proposal(v, rng);
+            auto r = (state._verbose) ? state.node_state(v)
+                : decltype(state.node_state(v))();
+            auto&& s = state.move_proposal(v, rng);
 
-            if (s == null_group)
+            if (s == state._null_move)
                 continue;
 
             double dS, mP;

src/graph/inference/overlap/graph_blockmodel_overlap.hh

@@ -505,6 +505,16 @@ public:
         move_vertices(vs, rs);
     }
 
+    void add_edge(size_t, size_t, GraphInterface::edge_t&,
+                  const std::vector<double>&)
+    {
+    }
+
+    void remove_edge(size_t, size_t, GraphInterface::edge_t&,
+                     const std::vector<double>&)
+    {
+    }
+
     template <class VMap>
     void set_partition(VMap&& b)
     {
@@ -1325,6 +1335,8 @@ public:
         return S;
     }
 
+    double edge_entropy_term(size_t u, size_t v, entropy_args_t ea) { return 0; }
+
     void enable_partition_stats()
     {
         if (_partition_stats.empty())
@@ -1624,8 +1636,8 @@ public:
     void remove_partition_node(size_t, size_t) { }
     void set_vertex_weight(size_t, int) { }
     void coupled_resize_vertex(size_t) { }
-    void update_edge(const GraphInterface::edge_t&,
-                     const std::vector<double>&) { }
+    void update_block_edge(const GraphInterface::edge_t&,
+                           const std::vector<double>&) { }
 
 //private:
     typedef typename

src/graph/inference/overlap/graph_blockmodel_overlap_mcmc_bundled.hh

@@ -105,6 +105,7 @@ struct MCMC
         std::vector<std::vector<size_t>> _half_edges;
         std::vector<std::vector<size_t>> _bundles;
         bool _parallel;
+        size_t _null_move = null_group;
 
         size_t node_state(size_t i)
         {

src/graph/inference/support/util.hh

@@ -25,6 +25,8 @@
 
 #include "cache.hh"
 
+#include <boost/range/counting_range.hpp>
+
 namespace graph_tool
 {
 using namespace boost;

src/graph/inference/uncertain/graph_blockmodel_measured.cc

+// graph-tool -- a general graph modification and manipulation thingy
+//
+// Copyright (C) 2006-2017 Tiago de Paula Peixoto <tiago@skewed.de>
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 3
+// of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#define BOOST_PYTHON_MAX_ARITY 40
+#include <boost/python.hpp>
+
+#include "graph_tool.hh"
+#include "random.hh"
+
+#include "../blockmodel/graph_blockmodel.hh"
+#define BASE_STATE_params BLOCK_STATE_params
+#include "graph_blockmodel_measured.hh"
+#include "../support/graph_state.hh"
+
+using namespace boost;
+using namespace graph_tool;
+
+GEN_DISPATCH(block_state, BlockState, BLOCK_STATE_params)
+
+template <class BaseState>
+GEN_DISPATCH(measured_state, Measured<BaseState>::template MeasuredState,
+             MEASURED_STATE_params)
+
+python::object make_measured_state(boost::python::object oblock_state,
+                                    boost::python::object omeasured_state)
+{
+    python::object state;
+    auto dispatch = [&](auto& block_state)
+        {
+            typedef typename std::remove_reference<decltype(block_state)>::type
+            state_t;
+
+            measured_state<state_t>::make_dispatch
+                (omeasured_state,
+                 [&](auto& s)
+                 {
+                     state = python::object(s);
+                 },
+                 block_state);
+        };
+    block_state::dispatch(oblock_state, dispatch);
+    return state;
+}
+
+
+void export_measured_state()
+{
+    using namespace boost::python;
+
+    def("make_measured_state", &make_measured_state);
+
+    block_state::dispatch
+        ([&](auto* bs)
+         {
+             typedef typename std::remove_reference<decltype(*bs)>::type block_state_t;
+
+             measured_state<block_state_t>::dispatch
+                 ([&](auto* s)
+                  {
+                      typedef typename std::remove_reference<decltype(*s)>::type state_t;
+
+                      class_<state_t>
+                          c(name_demangle(typeid(state_t).name()).c_str(),
+                            no_init);
+                      c.def("remove_edge", &state_t::remove_edge)
+                          .def("add_edge", &state_t::add_edge)
+                          .def("remove_edge_dS", &state_t::remove_edge_dS)
+                          .def("add_edge_dS", &state_t::add_edge_dS)
+                          .def("entropy", &state_t::entropy)
+                          .def("set_hparams", &state_t::set_hparams)
+                          .def("get_N", &state_t::get_N)
+                          .def("get_X", &state_t::get_X)
+                          .def("get_T", &state_t::get_T)
+                          .def("get_M", &state_t::get_M);
+                  });
+         });
+
+}

src/graph/inference/uncertain/graph_blockmodel_measured.hh

+// graph-tool -- a general graph modification and manipulation thingy
+//