net_site.py 9.83 KB
Newer Older
Tiago Peixoto's avatar
Tiago Peixoto committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2020 Tiago de Paula Peixoto <tiago@skewed.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import os.path
import io
import subprocess
import re
from collections import defaultdict
import functools
import pickle
27

Tiago Peixoto's avatar
Tiago Peixoto committed
28
from flask import Flask, render_template, make_response, redirect, Markup, \
29
    send_file, abort, request, jsonify, url_for
Tiago Peixoto's avatar
Tiago Peixoto committed
30 31 32 33 34 35

import process_entry
import analyze
import draw

import numpy
36
import math
Tiago Peixoto's avatar
Tiago Peixoto committed
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54

app = Flask(__name__)

access_tokens = set()
if os.path.exists("access_tokens"):
    for line in open("access_tokens"):
        token = line.split("#")[0].strip()
        if token != "":
            access_tokens.add(token)

app.config['JSON_SORT_KEYS'] = False

base = os.path.dirname(__file__)

entries = process_entry.get_entries()
analyze.analyze_entries(entries.values(), skip=["pos"],
                        global_cache=True)

55 56
whales = ["openstreetmap"]

57 58
import markdown

Tiago Peixoto's avatar
Tiago Peixoto committed
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
@app.context_processor
def file_processor():
    def file_size(filename):
        try:
            size = os.stat(f"cache/network/{filename}").st_size
        except FileNotFoundError:
            filename = filename.split("/")
            filename[-1] = filename[-1].split(".")
            filename[-1][-3] = "network"
            filename[-1] = ".".join(filename[-1])
            filename = "/".join(filename)
            try:
                size = os.stat(f"cache/network/{filename}").st_size
            except FileNotFoundError:
                return "N/A"
        if size > 1024 ** 3:
Tiago Peixoto's avatar
Tiago Peixoto committed
75
            size = f"{(size/1024**3):.3f} GiB"
Tiago Peixoto's avatar
Tiago Peixoto committed
76
        elif size > 1024 ** 2:
Tiago Peixoto's avatar
Tiago Peixoto committed
77
            size = f"{(size/1024**2):.1f} MiB"
Tiago Peixoto's avatar
Tiago Peixoto committed
78
        elif size > 1024:
Tiago Peixoto's avatar
Tiago Peixoto committed
79
            size = f"{round(size//1024)} KiB"
Tiago Peixoto's avatar
Tiago Peixoto committed
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
        else:
            size = f"{size}B"
        return size
    return dict(fsize=file_size)

@app.route("/")
def main_page():
    global entries
    tags = request.args.get('tags', None)
    if tags is not None:
        tags = set(tags.split(","))
        fentries = [entry for entry in entries.values()
                    if tags is None or len(set(entry.tags) & tags) > 0]
    else:
        fentries = entries.values()
95 96 97 98 99 100 101 102 103 104
    search = request.args.get('search', "")
    search = search.strip()
    if search != "":
        match = []
        for s in re.split(r'(?<!\\)&', search):
            try:
                match.append(re.compile(s.strip(), re.IGNORECASE))
            except re.error:
                pass
        def score_match(m, vals):
Tiago Peixoto's avatar
Tiago Peixoto committed
105 106 107 108 109
            n = 0
            for v in vals:
                if v is None:
                    continue
                if isinstance(v, str):
110
                    n += len(m.findall(v))
Tiago Peixoto's avatar
Tiago Peixoto committed
111 112 113
                else:
                    n += score(v)
            return n
114 115 116 117 118 119 120
        def score(vals):
            n = 1
            for m in match:
                n *= score_match(m, vals)
                if n == 0:
                    break
            return n
Tiago Peixoto's avatar
Tiago Peixoto committed
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
        fentries = [(entry, score([entry.name, entry.title, entry.description,
                                   entry.url, entry.upstream_prefix,
                                   entry.citation, entry.upstream_license,
                                   entry.tags])) for entry in fentries]
        fentries = [entry for entry in fentries if entry[1] > 0]
        fentries = sorted(fentries, key=lambda entry: -entry[1])
        fentries = [entry[0] for entry in fentries]
    return render_template('main.html', entries=fentries, tags=tags,
                           search=search)

@app.route("/net/<net>")
def network_page(net):
    global entries
    try:
        entry = entries[net]
    except KeyError:
        abort(404)

    full = bool(request.args.get('full', False))
    page = int(request.args.get('page', 1))
    page_size = 1000 if not full else len(entry.files)

143 144 145
    desc = markdown.markdown(entry.description, extensions=['footnotes'])

    return render_template('network.html', entry=entry, description=desc,
Tiago Peixoto's avatar
Tiago Peixoto committed
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
                           page=page, page_size=page_size)


@app.route("/net/<net>/files/<filename>")
def network_download(net, filename):
    global entries
    try:
        entry = entries[net]
    except KeyError:
        abort(404)
    if entry.restricted:
        token = request.headers.get("WWW-Authenticate", "")
        token = token.replace("Authorization: Bearer", "").strip()
        if token not in access_tokens:
            abort(401)
    if filename.split(".")[0] == entry.name:
        filename = ".".join(["network"] + filename.split(".")[1:])
    try:
        return send_file(f"cache/network/{entry.name}/{filename}")
    except (FileNotFoundError, IOError):
        abort(404)

@app.route("/stats")
def stats_page():
    global entries
171 172
    if not request.script_root:
        request.script_root = url_for('main_page', _external=True)
Tiago Peixoto's avatar
Tiago Peixoto committed
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193

    n_data = len(entries)
    n_nets = sum(len(entry.files) for entry in entries.values())
    tags = defaultdict(int)
    n_directed = 0
    n_undirected = 0
    n_bip = 0
    for entry in entries.values():
        for tag in entry.tags:
            tags[tag] += 1
        for f, alt, fmt in entry.files:
            if entry.analyses[alt]["is_directed"]:
                n_directed += 1
            else:
                n_undirected += 1
            if entry.analyses[alt]["is_bipartite"]:
                n_bip += 1

    tags = list(tags.items())
    tags = sorted(tags, key=lambda x: -x[1])

194
    entries_s = {name: e for name, e in entries.items() if name not in whales}
Tiago Peixoto's avatar
Tiago Peixoto committed
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
    n_nets_s = sum(len(entry.files) for entry in entries_s.values())
    n_directed_s = 0
    n_undirected_s = 0
    n_bip_s = 0
    for entry in entries_s.values():
        for f, alt, fmt in entry.files:
            if entry.analyses[alt]["is_directed"]:
                n_directed_s += 1
            else:
                n_undirected_s += 1
            if entry.analyses[alt]["is_bipartite"]:
                n_bip_s += 1

    return render_template('stats.html', n_data=n_data, n_nets=n_nets,
                           tags=tags, n_directed=n_directed,
                           n_undirected=n_undirected, n_bip=n_bip,
                           n_nets_s=n_nets_s, n_directed_s=n_directed_s,
                           n_undirected_s=n_undirected_s, n_bip_s=n_bip_s,
213 214
                           whales=",".join(whales), analyses=analyze.titles,
                           scales=analyze.scales)
Tiago Peixoto's avatar
Tiago Peixoto committed
215 216 217 218 219 220 221 222 223

@app.route("/draw/<net>")
@app.route("/draw/<net>/<alt>")
def net_draw(net, alt=None):
    global entries

    try:
        entry = entries[net]

224 225 226
        thumb = bool(request.args.get('thumb', False))

        if not thumb and len(request.args) > 0:
Tiago Peixoto's avatar
Tiago Peixoto committed
227 228 229 230 231 232 233 234
            svg = bool(request.args.get('svg', False))
            size = min(int(request.args.get('size', 1000)), 3000)
            bg_color = request.args.get('bg_color', "#cdcdcd")
            edge_color = request.args.get('edge_color', None)

            buf = draw.draw_entry(entry, alt, svg=svg, size=size,
                                  bg_color=bg_color, edge_color=edge_color)
        else:
235
            buf = draw.draw_entry(entry, alt, thumb=thumb)
Tiago Peixoto's avatar
Tiago Peixoto committed
236 237 238 239 240 241 242 243 244 245 246 247

        if bool(request.args.get('svg', False)):
            return send_file(buf, mimetype="image/svg+xml")
        else:
            return send_file(buf, mimetype=f"image/png")
    except KeyError:
        abort(404)

@app.route("/api")
def api():
    return render_template('api.html')

248 249 250 251 252 253 254 255 256 257 258 259 260 261
def clean_floats(d):
    new_d = {}
    for k, v in d.items():
        if isinstance(v, dict):
            v = clean_floats(v)
        else:
            try:
                if math.isnan(v) or math.isinf(v):
                    v = None
            except TypeError:
                pass
        new_d[k] = v
    return new_d

Tiago Peixoto's avatar
Tiago Peixoto committed
262 263 264 265 266 267 268 269
def get_entry_json(entry):
    return dict(title=entry.title,
                description=entry.description,
                citation=entry.citation,
                url=entry.url,
                restricted=entry.restricted,
                nets=[(alt if alt is not None else entry.name)
                      for f,alt,fmt in entry.files],
270 271
                analyses=clean_floats(entry.analyses if len(entry.files) > 1
                                      else entry.analyses[None]))
Tiago Peixoto's avatar
Tiago Peixoto committed
272 273 274 275 276 277 278

@app.route("/api/nets")
def api_entries():
    global entries
    tags = request.args.get('tags', None)
    if tags is not None:
        tags = set(tags.split(","))
279 280 281 282
    exclude = request.args.get('exclude', None)
    if exclude is not None:
        exclude = set(exclude.split(","))
    full = bool(request.args.get('full', False))
Tiago Peixoto's avatar
Tiago Peixoto committed
283
    fentries = [entry for entry in entries.values()
284 285
                if ((tags is None or len(set(entry.tags) & tags) > 0) and
                    (exclude is None or entry.name not in exclude))]
Tiago Peixoto's avatar
Tiago Peixoto committed
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
    if full:
        return {entry.name: get_entry_json(entry) for entry in fentries}
    else:
        return jsonify([entry.name for entry in fentries])

@app.route("/api/net/<net>")
def api_entires(net):
    global entries
    try:
        entry = entries[net]
    except KeyError:
        abort(404)
    return get_entry_json(entry)

@app.route("/about")
def about_page():
    return render_template('about.html')

@app.route("/git")
def git_page():
    return redirect("//git.skewed.de/count0/netzschleuder/")

@app.route("/issues")
def issues_page():
    return redirect("//git.skewed.de/count0/netzschleuder/issues/")

if __name__ == "__main__":
313
    app.run(debug=True, host='0.0.0.0')