Commit 8426d557 authored by Tiago Peixoto's avatar Tiago Peixoto
Browse files

Include SBM processing

parent 96a54c7a
......@@ -31,6 +31,9 @@ import dbm
import numpy
import scipy.sparse.linalg
import time
whales = process_entry.whales
@contextlib.contextmanager
def open_cache(name, flag="rf"):
......@@ -54,7 +57,7 @@ def cache_result(f):
name = f.__name__.split(".")[-1]
@wraps(f)
def wrap(entry, alt, g, cache, *args, cache_only=False, force=False,
**kwargs):
cont=False, **kwargs):
try:
if force:
raise KeyError()
......@@ -65,6 +68,9 @@ def cache_result(f):
ret = cache[alt][name]
else:
ret = cache[name]
if cont and not cache_only:
kwargs = dict(kwargs, prev_ret=ret)
raise KeyError()
return ret
except KeyError:
if cache_only:
......@@ -80,17 +86,23 @@ def cache_result(f):
return ret
return wrap
def restrict(N, exclude=[]):
"""Decorator that restricts the function call to networks with N nodes or less,
and that do not belong to the exclude list, and returns None otherwise.
def restrict(N, E=None, exclude=[]):
"""Decorator that restricts the function call to networks with N nodes (or E
edges) or less, and that do not belong to the exclude list, and returns None
otherwise.
"""
def rec(f):
@wraps(f)
def wrap(entry, alt, g, cache, *args, **kwargs):
N_e = get_N(entry, alt, g, cache, *args, **kwargs)
if N_e > N or entry.name in exclude:
return None
if E is None:
N_e = get_N(entry, alt, g, cache)
if N_e > N or entry.name in exclude:
return None
else:
E_e = get_E(entry, alt, g, cache)
if E_e > E or entry.name in exclude:
return None
return f(entry, alt, g, cache, *args, **kwargs)
return wrap
return rec
......@@ -100,7 +112,7 @@ def uses(names):
@wraps(f)
def wrap(*args, **kwargs):
global analyses
x = [analyses[name](*args, **kwargs) for name in names]
x = [analyses[name]["f"](*args, **kwargs) for name in names]
return f(*args, *x, **kwargs)
return wrap
return rec
......@@ -120,28 +132,20 @@ def meta_cache(f):
analyses = {}
meta_analyses = {}
titles = {}
scales = {}
def register(name=None, title=None, scale="linear", meta=False):
def register(name=None, title=None, scale="linear", cont=False, meta=False):
"""Decorator that registers the function to the global analyses list, with a
given name and title."""
global analyses
global titles
titles[name] = title
scales[name] = scale
def reg(f):
nonlocal name
nonlocal meta
if name is None:
name = f.__name__.split(".")[-1]
analyses[name] = f
if meta:
meta_analyses[name] = f
analyses[name] = dict(f=f, title=title, scale=scale, meta=meta,
cont=cont)
return f
return reg
......@@ -285,7 +289,7 @@ def get_vprops(g):
return vprops
@register("pos")
@restrict(N=1000000, exclude=["openstreetmap"])
@restrict(N=1000000, exclude=whales)
@cache_result
def get_pos(g):
if g.num_vertices() < 1000:
......@@ -296,21 +300,44 @@ def get_pos(g):
x, y = ungroup_vector_property(pos, [0, 1])
return [x.a, y.a]
@register("sbm_fit", cont=True)
@restrict(N=None, E=10000000, exclude=whales)
@cache_result
def get_sbm_fit(g, prev_ret=None):
if prev_ret is None:
state = NestedBlockState(g)
dS = []
Ss = []
Bs = []
else:
bs, dS, Ss, Bs = prev_ret
if len(dS) > 10 and abs(sum(dS[-10:])) < 1e-8:
return (bs, dS, Ss, Bs)
state = NestedBlockState(g, bs=bs)
begin = time.time()
while time.time() - begin < 600:
ret = state.multiflip_mcmc_sweep(beta=numpy.inf, niter=10)
dS.append(ret[0])
Ss.append(state.entropy())
Bs.append(state.levels[0].get_nonempty_B())
bs = state.get_bs()
bs = order_nested_partition_labels(bs)
return (bs, dS, Ss, Bs)
@register("knn_proj", meta=True)
@meta_cache
def get_knn_proj(entries):
global analyses
global meta_analyses
fs = []
for entry in entries:
if entry.name in ["openstreetmap"]:
if entry.name in whales:
continue
for alt, g in entry.parse(lazy=True, cache_only=True):
x = []
for a, f in analyses.items():
if a in ["num_vertices", "num_edges", "pos", "vertex_properties",
"edge_properties"] or a in meta_analyses:
"edge_properties"] or analyses[a]["meta"]:
continue
x.append(float(entry.analyses[alt][a]))
fs.append(x)
......@@ -330,7 +357,7 @@ def get_knn_proj(entries):
j = 0
for entry in entries:
if entry.name in ["openstreetmap"]:
if entry.name in whales:
continue
for alt, _ in entry.parse(lazy=True, cache_only=True):
res[(entry.name, alt)] = list(pos[j])
......@@ -353,7 +380,6 @@ def get_knn_proj_2(entries, ret):
def analyze_entries(entries, names=[], skip=[], force=[], cache_only=True,
global_cache=False):
global analyses
global meta_analyses
analyze_cache = {}
updated_global_cache = False
......@@ -382,12 +408,13 @@ def analyze_entries(entries, names=[], skip=[], force=[], cache_only=True,
Nmax = None
for alt, g in entry.parse(lazy=True, cache_only=True):
for a, f in analyses.items():
if a in skip or a in meta_analyses:
f, title, scale, meta, cont = f.values()
if a in skip or meta:
continue
if len(names) > 0 and a not in names:
continue
v = f(entry, alt, g, cache, force=a in force,
cache_only=cache_only)
cache_only=cache_only, cont=cont)
if isinstance(v, PropertyArray):
v = float(v)
entry.analyses[alt][a] = v
......@@ -418,12 +445,13 @@ def analyze_entries(entries, names=[], skip=[], force=[], cache_only=True,
if lock is None:
return
with open_cache("meta", "rf") as cache:
for a, f in meta_analyses.items():
if a in skip:
for a, f in analyses.items():
f, title, scale, meta, cont = f.values()
if a in skip or not meta:
continue
ret = f(entries, cache)
for entry in entries:
if entry.name in ["openstreetmap"]:
if entry.name in whales:
continue
for alt, g in entry.parse(lazy=True, cache_only=True):
try:
......@@ -450,5 +478,8 @@ if __name__ == "__main__":
with acquire_lock_file("./cache/meta.lock", block=True) as lock:
with open_cache("meta", "c") as cache:
for a, f in meta_analyses.items():
for a, f in analyses.items():
f, title, scale, meta, cont = f.values()
if not meta:
continue
ret = f(entries.values(), cache)
......@@ -54,7 +54,7 @@ analyze.analyze_entries(entries.values(), skip=["pos", "knn_proj"],
icon_ref = {entry.icon_hash : entry for entry in entries.values()}
whales = ["openstreetmap", "human_brains"]
whales = process_entry.whales
import markdown
......@@ -213,8 +213,9 @@ def stats_page():
n_undirected=n_undirected, n_bip=n_bip,
n_nets_s=n_nets_s, n_directed_s=n_directed_s,
n_undirected_s=n_undirected_s, n_bip_s=n_bip_s,
whales=whales, analyses=analyze.titles,
scales=analyze.scales)
whales=whales,
analyses={a:f["title"] for a, f in analyze.analyses.items()},
scales={a:f["scale"] for a, f in analyze.analyses.items()})
@app.route("/draw/<net>")
@app.route("/draw/<net>/<alt>")
......
......@@ -43,6 +43,8 @@ from parsers import *
from util import *
from gdrive import *
whales = ["openstreetmap", "human_brains"]
root = os.path.dirname(os.path.abspath(__file__))
def get_entries(names=None):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment