Refactor 'community' module and rename to 'inference'

This is a significant refactoring of the statistical inference code,
with many cleanups and improvements, and is aimed towards future
extensibility.

The old community module is available as 'community_old' and will be
completely removed in the near future.

Makefile.am

@@ -18,7 +18,7 @@ nobase_dist_graphtooldoc_DATA = \
 	doc/draw.rst \
 	doc/index.rst \
 	doc/spectral.rst \
-	doc/community.rst  \
+	doc/inference.rst  \
 	doc/flow.rst \
 	doc/Makefile \
 	doc/quickstart.rst \
@@ -30,7 +30,6 @@ nobase_dist_graphtooldoc_DATA = \
 	doc/topology.rst \
 	doc/conf.py \
 	doc/pyenv.py \
-	doc/community.xml \
 	doc/search_example.xml \
 	doc/sphinxext/README.txt \
 	doc/sphinxext/LICENSE.txt \

configure.ac

@@ -457,11 +457,12 @@ src/Makefile
 src/graph/Makefile
 src/graph/centrality/Makefile
 src/graph/clustering/Makefile
-src/graph/community/Makefile
+src/graph/community_old/Makefile
 src/graph/correlations/Makefile
 src/graph/draw/Makefile
 src/graph/flow/Makefile
 src/graph/generation/Makefile
+src/graph/inference/Makefile
 src/graph/layout/Makefile
 src/graph/search/Makefile
 src/graph/spectral/Makefile

doc/community.rst

-.. automodule:: graph_tool.community
-    :no-members:
-    :no-undoc-members:
-
-    .. autofunction:: minimize_blockmodel_dl
-    .. autoclass:: BlockState
-    .. autoclass:: OverlapBlockState
-    .. autoclass:: CovariateBlockState
-    .. autofunction:: mcmc_sweep
-    .. autofunction:: multilevel_minimize
-    .. autofunction:: collect_edge_marginals
-    .. autofunction:: collect_vertex_marginals
-    .. autofunction:: mf_entropy
-    .. autofunction:: bethe_entropy
-    .. autofunction:: model_entropy
-    .. autofunction:: get_max_B
-    .. autofunction:: get_akc
-    .. autofunction:: condensation_graph
-    .. autofunction:: minimize_nested_blockmodel_dl
-    .. autoclass:: NestedBlockState
-    .. autofunction:: init_nested_state
-    .. autofunction:: nested_mcmc_sweep
-    .. autofunction:: nested_tree_sweep
-    .. autofunction:: get_hierarchy_tree
-    .. autofunction:: get_block_edge_gradient
-    .. autofunction:: community_structure
-    .. autofunction:: modularity

doc/graph_tool.rst

@@ -207,11 +207,11 @@ Available subpackages
    centrality
    clustering
    collection
-   community
    correlations
    draw
    flow
    generation
+   inference
    search_module
    spectral
    stats

doc/test_inference.py

+#!/bin/env python
+
+from graph_tool.all import *
+import numpy.random
+from numpy.random import randint
+
+seed_rng(42)
+numpy.random.seed(42)
+
+graph_tool.inference.set_test(True)
+
+g = collection.data["football"]
+ec = g.new_ep("int", randint(0, 10, g.num_edges()))
+
+def gen_state(directed, deg_corr, layers, overlap):
+    u = GraphView(g, directed=directed)
+    if layers != False:
+        state = graph_tool.inference.LayeredBlockState(u, B=u.num_vertices(),
+                                                       deg_corr=deg_corr,
+                                                       ec=ec.copy(),
+                                                       overlap=overlap,
+                                                       layers=layers == True)
+    elif overlap:
+        state = graph_tool.inference.OverlapBlockState(u, B=2 * u.num_edges(),
+                                                       deg_corr=deg_corr)
+    else:
+        state = graph_tool.inference.BlockState(u, B=u.num_vertices(),
+                                                deg_corr=deg_corr)
+    return state
+
+
+for directed in [True, False]:
+    for overlap in [False, True]:
+        for layered in [False, "covariates", True]:
+            for deg_corr in [False, True]:
+                for dl in [False, True]:
+
+                    print("\ndirected:", directed, "overlap:", overlap,
+                          "layered:", layered, "deg-corr:", deg_corr, "dl:", dl)
+
+
+                    print("\t mcmc (unweighted)")
+                    state = gen_state(directed, deg_corr, layered, overlap)
+
+                    print("\t\t", state.mcmc_sweep(beta=0, allow_empty=True,
+                                                   entropy_args=dict(dl=dl)),
+                          (state.wr.a > 0).sum())
+                    if overlap:
+                        print("\t\t", state.mcmc_sweep(beta=0, bundled=True,
+                                                       allow_empty=True,
+                                                       entropy_args=dict(dl=dl)),
+                              (state.wr.a > 0).sum())
+
+                    state = gen_state(directed, deg_corr, layered, overlap)
+
+                    if not overlap:
+                        print("\t mcmc")
+                        bstate = state.get_block_state(vweight=True,
+                                                       deg_corr=deg_corr)
+
+                        print("\t\t",
+                              bstate.mcmc_sweep(beta=0,
+                                                allow_empty=True,
+                                                entropy_args=dict(dl=dl,
+                                                                  multigraph=False)),
+                              (bstate.wr.a > 0).sum())
+
+                        print("\t\t",
+                              bstate.mcmc_sweep(beta=0, allow_empty=True,
+                                                entropy_args=dict(dl=dl,
+                                                                  multigraph=False)),
+                              (bstate.wr.a > 0).sum())
+
+                        print("\t\t",
+                              bstate.gibbs_sweep(beta=0, allow_empty=True,
+                                                 entropy_args=dict(dl=dl,
+                                                                   multigraph=False)),
+                              (bstate.wr.a > 0).sum())
+
+                    print("\t merge")
+
+                    state = gen_state(directed, deg_corr, layered, overlap)
+
+                    if not overlap:
+                        bstate = state.get_block_state(vweight=True,
+                                                       deg_corr=deg_corr)
+
+                        print("\t\t",
+                              bstate.merge_sweep(50,
+                                                 entropy_args=dict(dl=dl,
+                                                                   multigraph=False)))
+
+                        bstate = bstate.copy()
+
+                        print("\t\t",
+                              bstate.mcmc_sweep(beta=0, allow_empty=True,
+                                                entropy_args=dict(dl=dl,
+                                                                  multigraph=False)))
+                        print("\t\t",
+                              bstate.gibbs_sweep(beta=0, allow_empty=True,
+                                                 entropy_args=dict(dl=dl,
+                                                                multigraph=False)))
+                    else:
+                        print("\t\t",
+                              state.merge_sweep(50,
+                                                entropy_args=dict(dl=dl,
+                                                                  multigraph=False)))
+
+                    print("\t shrink")
+
+                    state = gen_state(directed, deg_corr, layered, overlap)
+                    state = state.shrink(B=5, entropy_args=dict(dl=dl,
+                                                                multigraph=False))
+                    print("\t\t", state.B)
+
+for directed in [True, False]:
+    for overlap in [False, True]:
+        for layered in [False, "covariates", True]:
+            for deg_corr in [False, True]:
+                print("\ndirected:", directed, "overlap:", overlap,
+                      "layered:", layered, "deg-corr:", deg_corr)
+
+                state = minimize_blockmodel_dl(GraphView(g, directed=directed),
+                                               verbose=(1, "\t"),
+                                               deg_corr=deg_corr,
+                                               overlap=overlap,
+                                               layers=layered != False,
+                                               state_args=dict(ec=ec,
+                                                               layers=(layered == True)))
+                print(state.B, state.entropy())
+
+                state = minimize_nested_blockmodel_dl(GraphView(g, directed=directed),
+                                                      verbose=(1, "\t"),
+                                                      deg_corr=deg_corr,
+                                                      overlap=overlap,
+                                                      layers=layered != False,
+                                                      state_args=dict(ec=ec,
+                                                                      layers=(layered == True)))
+                state.print_summary()
+                print(state.entropy())
+

src/boost-workaround/boost/graph/reverse_graph_alt.hpp

@@ -306,7 +306,8 @@ namespace detail {
     typedef typename property_traits<PM>::reference reference;
     typedef typename property_traits<PM>::category category;
 
-    explicit reverse_graph_edge_property_map(const PM& pm): underlying_pm(pm) {}
+    reverse_graph_edge_property_map(const PM& pm): underlying_pm(pm) {}
+    reverse_graph_edge_property_map() = default;
 
     friend reference
     get(const reverse_graph_edge_property_map& m,

src/graph/Makefile.am

 ## Process this file with automake to produce Makefile.in
 
-SUBDIRS = centrality clustering community correlations draw flow generation layout search spectral stats topology util
+SUBDIRS = centrality clustering community_old correlations draw flow generation inference layout search spectral stats topology util
 
 AM_CPPFLAGS =\
     -I$(srcdir)/.. \

src/graph/community/Makefile.am → src/graph/community_old/Makefile.am

@@ -4,11 +4,11 @@ AM_CPPFLAGS = $(MOD_CPPFLAGS)
 
 AM_CFLAGS = $(AM_CXXFLAGS)
 
-libgraph_tool_communitydir = $(MOD_DIR)/community
+libgraph_tool_communitydir = $(MOD_DIR)/community_old
 
 libgraph_tool_community_LTLIBRARIES = libgraph_tool_community.la
 
-libgraph_tool_community_la_includedir = $(MOD_DIR)/include/community
+libgraph_tool_community_la_includedir = $(MOD_DIR)/include/community_old
 
 libgraph_tool_community_la_LIBADD = $(MOD_LIBADD)
 
@@ -19,16 +19,10 @@ libgraph_tool_community_la_SOURCES = \
     graph_blockmodel_covariates.cc \
     graph_blockmodel_overlap.cc \
     graph_community.cc \
-    graph_community_network.cc \
-    graph_community_network_edges.cc \
-    graph_community_network_vavg.cc \
-    graph_community_network_eavg.cc \
-    graph_community_network_eavg_imp1.cc \
     spence.cc
 
 libgraph_tool_community_la_include_HEADERS = \
     graph_blockmodel.hh \
     graph_blockmodel_covariates.hh \
     graph_blockmodel_overlap.hh \
-    graph_community.hh \
-    graph_community_network.hh
+    graph_community.hh

src/graph/community/graph_blockmodel.cc → src/graph/community_old/graph_blockmodel.cc

@@ -58,58 +58,64 @@ using namespace graph_tool;
 namespace graph_tool
 {
 
-vector<double> __safelog_cache;
-vector<double> __xlogx_cache;
-vector<double> __lgamma_cache;
-
-void init_safelog(size_t x)
-{
-    size_t old_size = __safelog_cache.size();
-    if (x >= old_size)
-    {
-        __safelog_cache.resize(x + 1);
-        for (size_t i = old_size; i < __safelog_cache.size(); ++i)
-            __safelog_cache[i] = safelog(double(i));
-    }
-}
-
-void clear_safelog()
-{
-    vector<double>().swap(__safelog_cache);
-}
-
-
-void init_xlogx(size_t x)
-{
-    size_t old_size = __xlogx_cache.size();
-    if (x >= old_size)
-    {
-        __xlogx_cache.resize(x + 1);
-        for (size_t i = old_size; i < __xlogx_cache.size(); ++i)
-            __xlogx_cache[i] = i * safelog(i);
-    }
-}
-
-void clear_xlogx()
-{
-    vector<double>().swap(__xlogx_cache);
-}
-
-void init_lgamma(size_t x)
-{
-    size_t old_size = __lgamma_cache.size();
-    if (x >= old_size)
-    {
-        __lgamma_cache.resize(x + 1);
-        for (size_t i = old_size; i < __lgamma_cache.size(); ++i)
-            __lgamma_cache[i] = lgamma(i);
-    }
-}
-
-void clear_lgamma()
-{
-    vector<double>().swap(__lgamma_cache);
-}
+// vector<double> __safelog_cache;
+// vector<double> __xlogx_cache;
+// vector<double> __lgamma_cache;
+
+void init_safelog(size_t x);
+// void init_safelog(size_t x)
+// {
+//     size_t old_size = __safelog_cache.size();
+//     if (x >= old_size)
+//     {
+//         __safelog_cache.resize(x + 1);
+//         for (size_t i = old_size; i < __safelog_cache.size(); ++i)
+//             __safelog_cache[i] = safelog(double(i));
+//     }
+// }
+
+void clear_safelog();
+// void clear_safelog()
+// {
+//     vector<double>().swap(__safelog_cache);
+// }
+
+void init_xlogx(size_t x);
+
+// void init_xlogx(size_t x)
+// {
+//     size_t old_size = __xlogx_cache.size();
+//     if (x >= old_size)
+//     {
+//         __xlogx_cache.resize(x + 1);
+//         for (size_t i = old_size; i < __xlogx_cache.size(); ++i)
+//             __xlogx_cache[i] = i * safelog(i);
+//     }
+// }
+
+void clear_xlogx();
+// void clear_xlogx()
+// {
+//     vector<double>().swap(__xlogx_cache);
+// }
+
+void init_lgamma(size_t x);
+// void init_lgamma(size_t x)
+// {
+//     size_t old_size = __lgamma_cache.size();
+//     if (x >= old_size)
+//     {
+//         __lgamma_cache.resize(x + 1);
+//         for (size_t i = old_size; i < __lgamma_cache.size(); ++i)
+//             __lgamma_cache[i] = lgamma(i);
+//     }
+// }
+
+void clear_lgamma();
+// void clear_lgamma()
+// {
+//     vector<double>().swap(__lgamma_cache);
+// }
 
 
 }

src/graph/community/graph_blockmodel_covariates.cc → src/graph/community_old/graph_blockmodel_covariates.cc

@@ -690,55 +690,55 @@ void do_split_graph(GraphInterface& gi, boost::any& aec, boost::any& ab,
                                        std::ref(uvmap)))();
 }
 
-bool bmap_has(const bmap_t& bmap, size_t c, size_t r)
-{
-    if (c > bmap.size())
-        throw GraphException("invalid covariate value:" + lexical_cast<string>(c));
-    auto iter = bmap[c].find(r);
-    if (iter == bmap[c].end())
-        return false;
-    return true;
-}
+// bool bmap_has(const bmap_t& bmap, size_t c, size_t r)
+// {
+//     if (c > bmap.size())
+//         throw GraphException("invalid covariate value:" + lexical_cast<string>(c));
+//     auto iter = bmap[c].find(r);
+//     if (iter == bmap[c].end())
+//         return false;
+//     return true;
+// }
 
-size_t bmap_get(const bmap_t& bmap, size_t c, size_t r)
-{
-    if (c > bmap.size())
-        throw GraphException("invalid covariate value:" + lexical_cast<string>(c));
-    auto iter = bmap[c].find(r);
-    if (iter == bmap[c].end())
-        throw GraphException("no mapping for block " + lexical_cast<string>(r)
-                             + " in layer " + lexical_cast<string>(c));
-    return iter->second;
-}
+// size_t bmap_get(const bmap_t& bmap, size_t c, size_t r)
+// {
+//     if (c > bmap.size())
+//         throw GraphException("invalid covariate value:" + lexical_cast<string>(c));
+//     auto iter = bmap[c].find(r);
+//     if (iter == bmap[c].end())
+//         throw GraphException("no mapping for block " + lexical_cast<string>(r)
+//                              + " in layer " + lexical_cast<string>(c));
+//     return iter->second;
+// }
 
-void bmap_set(bmap_t& bmap, size_t c, size_t r, size_t r_u)
-{
-    if (c > bmap.size())
-        throw GraphException("invalid covariate value:" + lexical_cast<string>(c));
-    bmap[c][r] = r_u;
-}
+// void bmap_set(bmap_t& bmap, size_t c, size_t r, size_t r_u)
+// {
+//     if (c > bmap.size())
+//         throw GraphException("invalid covariate value:" + lexical_cast<string>(c));
+//     bmap[c][r] = r_u;
+// }
 
-void bmap_del_c(bmap_t& bmap, size_t c)
-{
-    if (c > bmap.size())
-        throw GraphException("invalid covariate value:" + lexical_cast<string>(c));
-    bmap.erase(bmap.begin() + c);
-}
+// void bmap_del_c(bmap_t& bmap, size_t c)
+// {
+//     if (c > bmap.size())
+//         throw GraphException("invalid covariate value:" + lexical_cast<string>(c));
+//     bmap.erase(bmap.begin() + c);
+// }
 
-bmap_t bmap_copy(const bmap_t& bmap)
-{
-    return bmap;
-}
+// bmap_t bmap_copy(const bmap_t& bmap)
+// {
+//     return bmap;
+// }
 
 
 void export_blockmodel_covariate()
 {
-    boost::python::class_<bmap_t>("bmap_t")
-        .def("has", bmap_has)
-        .def("get", bmap_get)
-        .def("set", bmap_set)
-        .def("del_c", bmap_del_c)
-        .def("copy", bmap_copy);
+    // boost::python::class_<bmap_t>("bmap_t")
+    //     .def("has", bmap_has)
+    //     .def("get", bmap_get)
+    //     .def("set", bmap_set)
+    //     .def("del_c", bmap_del_c)
+    //     .def("copy", bmap_copy);
 
     boost::python::def("cov_move_sweep", do_cov_move_sweep);
     boost::python::def("covariate_entropy", do_covariate_entropy);

src/graph/community/graph_community.cc → src/graph/community_old/graph_community.cc

@@ -97,27 +97,6 @@ double modularity(GraphInterface& g, boost::any weight, boost::any property)
 using namespace boost::python;
 
 
-extern void community_network(GraphInterface& gi, GraphInterface& cgi,
-                              boost::any community_property,
-                              boost::any condensed_community_property,
-                              boost::any vertex_count, boost::any edge_count,
-                              boost::any vweight, boost::any eweight,
-                              bool self_loops, bool parallel_edges);
-
-void community_network_vavg(GraphInterface& gi, GraphInterface& cgi,
-                            boost::any community_property,
-                            boost::any condensed_community_property,
-                            boost::any vweight,
-                            boost::python::list avprops);
-
-void community_network_eavg(GraphInterface& gi, GraphInterface& cgi,
-                            boost::any community_property,
-                            boost::any condensed_community_property,
-                            boost::any eweight,
-                            boost::python::list aeprops,
-                            bool self_loops);
-
-
 extern void export_blockmodel();
 extern void export_blockmodel_overlap();
 extern void export_blockmodel_covariate();
@@ -126,9 +105,6 @@ BOOST_PYTHON_MODULE(libgraph_tool_community)
 {
     def("community_structure", &community_structure);
     def("modularity", &modularity);
-    def("community_network", &community_network);
-    def("community_network_vavg", &community_network_vavg);
-    def("community_network_eavg", &community_network_eavg);
 
     export_blockmodel();
     export_blockmodel_overlap();

src/graph/draw/graph_cairo_draw.cc

@@ -1045,7 +1045,7 @@ public:
             _attrs.template get<vector<double> >(EDGE_CONTROL_POINTS);
         vector<double> gradient =
             _attrs.template get<vector<double> >(EDGE_GRADIENT);
-        if (gradient.size() == 2)
+        if (gradient.size() == 1)
         {
             auto e_color = _attrs.template get<color_t>(EDGE_COLOR);
             auto s_color = _s._attrs.template get<color_t>(VERTEX_FILL_COLOR);

src/graph/generation/Makefile.am

@@ -15,29 +15,35 @@ libgraph_tool_generation_la_LIBADD = $(MOD_LIBADD) $(CGAL_LIBADD)
 libgraph_tool_generation_la_LDFLAGS = $(MOD_LDFLAGS)
 
 libgraph_tool_generation_la_SOURCES = \
+    graph_community_network.cc \
+    graph_community_network_eavg.cc \
+    graph_community_network_eavg_imp1.cc \
+    graph_community_network_edges.cc \
+    graph_community_network_vavg.cc \
+    graph_complete.cc \
     graph_generation.cc \
-    graph_rewiring.cc \
-    graph_predecessor.cc \
+    graph_geometric.cc \
+    graph_lattice.cc \
     graph_line_graph.cc \
+    graph_predecessor.cc \
+    graph_price.cc \
+    graph_rewiring.cc \
+    graph_triangulation.cc \
     graph_union.cc \
-    graph_union_vprop.cc \
     graph_union_eprop.cc \
-    graph_triangulation.cc \
-    graph_lattice.cc \
-    graph_geometric.cc \
-    graph_complete.cc \
-    graph_price.cc
+    graph_union_vprop.cc
 
 
 libgraph_tool_generation_la_include_HEADERS = \
+    dynamic_sampler.hh \
+    graph_community_network.hh \
+    graph_complete.hh \
     graph_generation.hh \
-    graph_rewiring.hh \
-    graph_predecessor.hh \
-    graph_union.hh \
-    graph_triangulation.hh \
-    graph_lattice.hh \
     graph_geometric.hh \
-    graph_complete.hh \
+    graph_lattice.hh \
+    graph_predecessor.hh \
     graph_price.hh \
-    dynamic_sampler.hh \
+    graph_rewiring.hh \
+    graph_triangulation.hh \
+    graph_union.hh \
     sampler.hh

src/graph/generation/graph_generation.cc

@@ -48,9 +48,10 @@ private:
     boost::python::object _o;
 };
 
-void generate_graph(GraphInterface& gi, size_t N, boost::python::object deg_sample,
-                    bool no_parallel, bool no_self_loops, bool undirected,
-                    rng_t& rng, bool verbose, bool verify)
+void generate_graph(GraphInterface& gi, size_t N,
+                    boost::python::object deg_sample, bool no_parallel,
+                    bool no_self_loops, bool undirected, rng_t& rng,
+                    bool verbose, bool verify)
 {
     typedef graph_tool::detail::get_all_graph_views::apply<
     graph_tool::detail::filt_scalar_type, boost::mpl::bool_<false>,
@@ -84,15 +85,34 @@ void vertex_property_union(GraphInterface& ugi, GraphInterface& gi,
 void edge_property_union(GraphInterface& ugi, GraphInterface& gi,
                          boost::any p_vprop, boost::any p_eprop,
                          boost::any uprop, boost::any prop);
-void triangulation(GraphInterface& gi, boost::python::object points, boost::any pos,
-                   string type, bool periodic);
+void triangulation(GraphInterface& gi, boost::python::object points,
+                   boost::any pos, string type, bool periodic);
 void lattice(GraphInterface& gi, boost::python::object oshape, bool periodic);
 void geometric(GraphInterface& gi, boost::python::object opoints, double r,
                boost::python::object orange, bool periodic, boost::any pos);
 void price(GraphInterface& gi, size_t N, double gamma, double c, size_t m,
            rng_t& rng);
 void complete(GraphInterface& gi, size_t N, bool directed, bool self_loops);
-void circular(GraphInterface& gi, size_t N, size_t k, bool directed, bool self_loops);
+void circular(GraphInterface& gi, size_t N, size_t k, bool directed,
+              bool self_loops);
+
+void community_network(GraphInterface& gi, GraphInterface& cgi,
+                       boost::any community_property,
+                       boost::any condensed_community_property,
+                       boost::any vertex_count, boost::any edge_count,
+                       boost::any vweight, boost::any eweight, bool self_loops,
+                       bool parallel_edges);
+
+void community_network_vavg(GraphInterface& gi, GraphInterface& cgi,
+                            boost::any community_property,
+                            boost::any condensed_community_property,
+                            boost::any vweight, boost::python::list avprops);
+
+void community_network_eavg(GraphInterface& gi, GraphInterface& cgi,
+                            boost::any community_property,
+                            boost::any condensed_community_property,
+                            boost::any eweight, boost::python::list aeprops,
+                            bool self_loops);
 
 using namespace boost::python;
 
@@ -111,6 +131,9 @@ BOOST_PYTHON_MODULE(libgraph_tool_generation)
     def("price", &price);
     def("complete", &complete);
     def("circular", &circular);
+    def("community_network", &community_network);
+    def("community_network_vavg", &community_network_vavg);
+    def("community_network_eavg", &community_network_eavg);