home/diploma/lws_root/lws/serve.py

# -*- coding: utf-8 -*-
#---Standard library imports
import string
import sys
import logging
import time
from datetime import datetime
from collections import namedtuple, defaultdict, OrderedDict
import json
import traceback
from functools import wraps
import Queue
import threading
import hashlib
from UserDict import UserDict
import random

#---Third-party imports
from utils.path import path
if sys.platform == 'win32':
    import win32api
    import win32con

from configobj import ConfigObj, flatten_errors
import validate
import cherrypy
from cherrypy.lib.static import serve_file
from cherrypy.lib.http import HTTPDate
from dateutil import parser
import scipy

from runphoton import BBox, Resolution

from utils.materials import materialsAsXml, Brdf, loadMaterialsFromString
from utils import daemonthread, loadLocations
from utils.xml import StringToTree, TreeToString, subelement

import evaluate
import render
import optimize as optimizer
import simulate as simulator
import plotter
import localize


#---Globals
CONFIG_SPEC = path(__file__).parent.joinpath('config/configspec.ini')
PATH_STATIC = path(__file__).parent.joinpath('static').abspath()

LOCALIZATION_ON = True
LOCALIZATION_ON = False

LOGFILE = path('.').joinpath('out.log')
    
MEASUREMENTS_STORAGE = path('.').joinpath('measurements.txt')
if not MEASUREMENTS_STORAGE.exists():
    MEASUREMENTS_STORAGE.touch()

# first substitution is station name, second is apid
OPTIMIZE_FILES = '/home/dirk/loco/maps/UMIC/experiment/measure_%s_%s.txt'

log = logging.getLogger('lws')
app = None

def _splitBaseurl(baseurl):
    if baseurl is None:
        return None, None
    parts = baseurl.split(':')
    if len(parts) == 1:
        parts.append(None)
    return parts[0], parts[1]

def getHTTPConfig():
    c = urlparse.urlsplit(cherrypy.request.base)
    host, port = _splitBaseurl(c[1])

    if app.config['behindProxy']:
        port = app.config['proxyHttpPort']
        host = _splitBaseurl(urlparse.urlsplit(app.config['proxyBase'])[1])[0]
    else:
        port = app.config['httpPort']

    return host, port


def setupLogging():
    log.handlers = []
    log.setLevel(logging.DEBUG)
    
    sh = logging.StreamHandler()
    formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%m-%d %H:%M:%S")
    sh.setFormatter(formatter)
    log.addHandler(sh)

    logfile = LOGFILE
    log.info('storing logfile to %s' % logfile.normpath())

    fh = logging.FileHandler(logfile, 'a')
    fh.setFormatter(logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s", '%m-%d %X'))
    log.addHandler(fh)
    logging.getLogger('cherrypy.error').addHandler(fh)

def setForceNoCache(expires_past=HTTPDate(1169942400.0)):
    headers = cherrypy.response.headers
    headers['Expires'] = expires_past
    headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'
    headers['Pragma'] = 'no-cache'

def nocache(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        setForceNoCache()
        return func(*args, **kwargs)
    return wrapper

Measurement = namedtuple('Measurement', 'x y z avgrssi rssis')
APInfo = namedtuple('AP', 'rssi remotets localts delta')
class MobileStation(object):
    def __init__(self, name):
        self.name = name
        self.data = defaultdict(list) # key: (bssid, ssid), values: list of apinfo
    
class LWS(object):
    def __init__(self, configfile, cfgoverlay=None):
        self.mstations = {} # key: station name, value: MobileStation()
        self.mac2cfg = {} # key: mac address, value: config section for corresponding access point id
        self.aliasedmac2apid = {} # key: mac, value corresponding apid
        
        self.activeScene = None # points to sctive scene section in config
        self.configfile = path(configfile)
        self.cfgoverlay = cfgoverlay # nested dict to overwrite config values
        self._loadConfig()
        
        self.measure_enabled = None # beocmes tuple (station name, locid, startts, duration)
        self.measurement_stopped = None
        
        self.pathtrack_enabled = None
        self.lastCollectedPath = None
        
        # becomes a dict with key: station name and value a dict with key: locid and value: {(bssid, ssid): APInfo()}
        self.tracked_measurements = defaultdict(dict) 
        # dict with key stationname, and value: dict with key pathid, and value: list (locid, timestamp)
        self.tracked_positions = defaultdict(dict) 
        
        # in memory storage of measurements (from data in MEASUREMENTS_STORAGE) - like self.tracked_measurements
        self.stored_measurements = defaultdict(lambda: defaultdict(list)) 
        self.loadMeasurementDump()
        if not self.activeScene['locationfile'].exists():
            print 'creating locfile at %s' % self.activeScene['locationfile'].abspath()
            self.activeScene['locationfile'].touch()
        
        self.known_locations = loadLocations(self.activeScene['locationfile']) # key: locid, value: measure.Location()
        
        daemonthread(target=self._checkConfig, name="configReloadLoop")
        
        
        self.simulator = simulator.Simulator(self.config['tmp'])
        self.hash2objfile = {}
        self.loadobjfiles()
        
        optimizer.app = self
        self.optimizer = optimizer.Optimizer()
        
        self.plotLock = threading.Lock()
        self.writeLock = threading.Lock()
        
        # measurements for different APs
        self._apid2measurements = {}
        
        self.localization_data = defaultdict(lambda: Queue.Queue())
        self.last_result = defaultdict(lambda: Queue.Queue())
        self.device2localizationenv = {}
        self.localizeSessions = defaultdict(set)
        self.localizeSessionInactiveAPs = defaultdict(set) # key: device, value: set of deactivated APs
        self.draw_aptiles_lock = threading.Lock()
        
    def loadobjfiles(self):
        objpath = self.config['tmp'].joinpath('obj')
        if not objpath.exists():
            objpath.mkdir()
            
        for f in objpath.files():
            h = hashlib.sha1(f.bytes()).hexdigest()
            self.hash2objfile[h] = f
            
    def _loadConfig(self):
        # check file exists and is readable
        f = open(self.configfile)
        f.read(1024)
        f.close()

        log.info('loading configfile %s' % self.configfile)
        configSpec = ConfigObj(CONFIG_SPEC, encoding='utf-8', list_values=False)
        validator = validate.Validator()
        self.config = ConfigObj(self.configfile, configspec=configSpec, interpolation='template', encoding='UTF-8')

        res = self.config.validate(validator, preserve_errors=True)
        #check for errors
        for entry in flatten_errors(self.config, res):
            # each entry is a tuple
            section_list, key, error = entry
            section_list.append('[missing section]') if key is None else section_list.append(key)
            section_string = ', '.join(section_list)
            if not error:
                error = 'Missing value or section.'
            raise Exception, '%s=%s' % (section_string, error)
        
        def _applyOverlay(section, d):
            for key, value in d.items():
                if isinstance(d[key], dict):
                    _applyOverlay(section[key], d[key])
                else:
                    section[key] = value
            
        if self.cfgoverlay:
            _applyOverlay(self.config, self.cfgoverlay)
            
                
        for k in 'tmp', 'tmp_apdata', 'tmp_optruns', 'tmp_tracked':
            self.config[k] = path(self.config[k]).abspath()
            if not self.config[k].exists():
                self.config[k].mkdir()
            
            
        # check objfile fs paths
        for secname in self.config['scenes'].sections:
                
            for fname in ('objfile', 'locationfile', 'measurementsfile'):
                if secname == 'onewall':
                    # special default lookup under ./HERE/static
                    f = path(__file__).parent.joinpath('setups', secname, self.config['scenes'][secname][fname])
                else:
                    f = path(self.config['scenes'][secname][fname])
                    
                self.config['scenes'][secname][fname] = f
                
                if not fname in ('locationfile', 'measurementsfile') and not f.exists():
                    log.warn('configured %s %s for scene %s does not exist' % (fname, f.abspath(), secname))
                    
                    sys.exit()
                    
        active = self.config['scenes']['active']
        self.activeScene = self.config['scenes'][active]
        
        if active == 'onewall' and len(self.config['aps'].sections) == 0:
            # add a default AP for onewall scene
            class FAKE(UserDict): pass
            for i, (name, x, y, z) in enumerate([('ap1', -6.0, -1.0, -2.0), ('ap2', 0.0, 0.0, 0.0)]):
                faked_ap = FAKE()
                self.config['aps'][name] = faked_ap
                faked_ap.name = name
                faked_ap['mac'] = '00:00:00:00:00:0' + str(i)
                faked_ap['powerid'] = 'default'
                faked_ap['power'] = 0.1
                faked_ap['x'] = x
                faked_ap['y'] = y
                faked_ap['z'] = z
                faked_ap['on'] = True
                faked_ap['optimize'] = True
                
                self.config['aps'].sections.append(name)
        
        self.aliasedmac2apid.clear()
        apcfgs = self.config['aps']
        for apid in apcfgs.sections:
            assert apcfgs[apid]['mac'] is not None, 'invalid cfg for %s: %r' % (apid, apcfgs[apid])
                
            self.mac2cfg[apcfgs[apid]['mac']] = apcfgs[apid]
            for mac in apcfgs[apid]['aliases']:
                self.aliasedmac2apid[mac] = apid
            
            #~ if apid != '107':
                #~ self.config['aps'][apid]['optimize'] = False
        
        # add reverse paths
        for pathid in list(self.activeScene['paths'].scalars):
            if not pathid.startswith('still_'):
                self.activeScene['paths']['%s_r' % pathid] = list(reversed(self.activeScene['paths'][pathid]))
        
    def _checkConfig(self):
        last_mtime = self.configfile.mtime
        while True:
            try:
                if self.configfile.mtime > last_mtime:
                    self._loadConfig()
                last_mtime = self.configfile.mtime
            except Exception:
                log.exception('error during loading config from %s' % self.configfile.normpath())
            time.sleep(2)

    
    
    @cherrypy.expose
    def index(self, **kwargs):
        reload(render)
        return render.renderIndex(self)
        
        
    @cherrypy.expose
    def localize(self, name, **kwargs):
        reload(render)
        return render.renderLocalize(self, name, refresh='refresh' in kwargs)
        
    @cherrypy.expose
    def view(self, name):
        if not name in self.mstations:
            return ''' unknown station: "%s"''' % name
        reload(render)
        return render.renderView(self, self.mstations[name])
    
    @cherrypy.expose
    def measure(self, name):
        ''' collect measurements for training raytracer params '''
        reload(render)
        return render.renderMeasure(self, self.mstations.get(name) )
    
    @cherrypy.expose
    def locate(self, name):
        if not name in self.mstations:
            return ''' unknown station: "%s"''' % name
        reload(render)
        return render.renderLocate(self, self.mstations[name])
    
    @cherrypy.expose
    def cluster(self):
        reload(render)
        return render.renderCluster(self)
    
    @cherrypy.expose
    def measurements(self, view=None, stations=None):
        reload(render)
        if stations is None:
            stations = self.config['knownStations']
        else:
            stations = [e.strip() for e in stations.split(',') if e.strip()]
        if view is not None:
            view = view.split(',')
        else:
            view = ['mean', 'median', 'stddev']
            
        return render.renderMeasurements(self, stations=stations, view=view)
    
    @cherrypy.expose
    def optimize(self):
        reload(render)
        return render.renderOptimizer(self)
        
            
    
    @cherrypy.expose
    def optruns(self, optruns='', **kwargs):
        reload(render)
        return render.renderEvaluteOptruns(self, optruns, compact='compact' in kwargs, **kwargs)
    
    @cherrypy.expose
    def plotHistogram(self, **kwargs):
        plotfile = self.config['tmp'].joinpath('_hist.png')
        reload(plotter)
        with self.plotLock:
            plotter.plotHistogram(plotfile, **kwargs)
        return serve_file(plotfile)
        
    @cherrypy.expose
    def collect(self, station):
        ''' collect a path for using in locatization'''
        reload(render)
        return render.renderCollectPath(self, station)
        
    @cherrypy.expose
    def evaluate(self, station, optrun=None, cubewidth=3, errtype='2d', filter=None, algo='hmm', **kwargs):
        ''' evalute collected measurements'''
        reload(render)
        return render.renderEvaluate(self, optrun, station, int(cubewidth), errtype=errtype, 
                                    refresh='refresh' in kwargs and not kwargs['refresh'] == False,
                                    filter=filter, algo=algo)
    
    @cherrypy.expose
    def evaluatePath(self, station, optrun, pathid, runid, setting, cubewidth=3, algo='hmm', **kwargs):
        ''' evalute collected measurements'''
        reload(render)
        if pathid == 'LASTCOLLECTED':
            pathid = self.lastCollectedPath[1]
        if runid == 'LASTCOLLECTED':
            runid = self.lastCollectedPath[2]
            
        return render.renderEvaluatePath(self, optrun, station, pathid, runid, setting,
                                    int(cubewidth), algo, refresh='refresh' in kwargs and not kwargs['refresh'] == False)
    
    @cherrypy.expose
    def startOptimizer(self, optsession, station, power, density, numphotons, resolution, bbox, scene, 
                            startpop, childcount, childcull, mutprob=0.1, mutamt=0.1, resume='', 
                            neworgs=0, timelimit=0, genlimit=0, **kwds):
        
        # forbid these two chars in name as they have some internal semantics
        optsession = optsession.replace('_', '.').replace(',', '.')
        
        power = float(power)
        density =  float(density)
        numphotons = int(numphotons)
        
        bbox = [float(e) for e in bbox.split(',') if e.strip()]
        bbox = BBox(x1=bbox[0], x2=bbox[1], y1=bbox[2], y2=bbox[3], z1=bbox[4], z2=bbox[5])
        if ',' in resolution: # its a triple
            res = [int(e) for e in resolution.split(',')]
            resolution = Resolution(res[0], res[1], res[2])
        else: # its only the x value - calculate the rest to ensure the other two dims have the same pixel/m ratio
            xres = int(resolution)
            xr, yr, zr = bbox.x2 - bbox.x1, bbox.y2 - bbox.y1, bbox.z2 - bbox.z1
            resolution = Resolution(x=xres, y=int(float(xres) * yr / xr), z=int(float(xres) * zr / xr))
        
        startpop = int(startpop)
        childcount = int(childcount)
        childcull = int(childcull)
        
        self.optimizer.startOptimizeRun(optsession, station, power, density, numphotons, resolution, bbox, scene,
                                        startpop, childcount, childcull,  float(mutprob), 
                                        float(mutamt), resume=resume.strip(), neworgs=int(neworgs), 
                                        timelimit=timelimit, genlimit=genlimit)
                                        
        raise cherrypy.HTTPRedirect(cherrypy.request.headers['Referer'])
    
    @cherrypy.expose
    def stopOptimizer(self):
        self.optimizer.stopOptimizeRun()
        raise cherrypy.HTTPRedirect(cherrypy.request.headers['Referer'])
        
    @cherrypy.expose
    def getAPInfo(self, name, jsid, filter=None):
        cherrypy.response.headers['Content-Type'] = 'application/json'
        if filter is not None:
            filter = {e for e in filter.split(',') if e.strip()}
                
        if not name in self.mstations:
            return json.dumps({})
        
        if not 'lastdelivered' in cherrypy.session:
            cherrypy.session['lastdelivered'] = {} #key: jsid, value: last delivered ts
        
        lastdelivered = cherrypy.session['lastdelivered']
        
        
        ld = lastdelivered.get(jsid)
        lastdelivered[jsid] = datetime.now()
        
        resdata = {}
        for (mac, apname), infolist in self.mstations[name].data.items():
            if filter is not None:
                if not mac in self.mac2cfg:
                    continue
                apid = self.mac2cfg[mac].name
                if not apid in filter:
                    continue
                    
            
            if ld is not None:
                _infolist = [apinfo for apinfo in infolist if apinfo.localts > ld]
            else:
                _infolist = infolist[-5:]
                
            resdata[mac.replace(':', '_')] = _infolist
        
        
        dthandler = lambda obj: obj.strftime('%H:%M:%S') if isinstance(obj, datetime) else None
        return json.dumps(resdata, default=dthandler)
    
    @cherrypy.expose
    def startMeasureLoc(self, name, locid, duration):
        cherrypy.response.headers['Content-Type'] = 'application/json'
        
        if not locid.isdigit() or not int(locid) in self.known_locations:
            valid = ','.join(str(e) for e in self.known_locations)
            return json.dumps({'started': '', 'error': 'invalid location %s [valid: %s]' % (locid, valid)})
            
        duration = float(duration)
        startts = time.time()
        with self.writeLock, open(MEASUREMENTS_STORAGE, 'a') as f:
            f.write('start: %s %s %s\n' % (name, locid, datetime.fromtimestamp(startts)))
        self.measure_enabled = (name, locid, startts, duration)
        # setup list
        self.tracked_measurements[name][locid] = defaultdict(list)
            
        def waitForEnd():
            while time.time() < startts + duration:
                time.sleep(0.1)
            self.measure_enabled = None
            self.measurement_stopped = datetime.now()
            
            with self.writeLock, open(MEASUREMENTS_STORAGE, 'a') as f:
                aps = self.tracked_measurements[name][locid].items()
                log.info('storing measurements for %s access points' % len(aps))
                
                run = {}
                for (bssid, ssid), data in aps:
                    if any(not c in string.printable for c in ssid):
                        log.warn('got corrupt ssid: %s' % repr(ssid))
                        continue
                    
                    apid = '???' if not bssid in self.mac2cfg else self.mac2cfg[bssid].name
                    s = '%s %s %s %s %s rssi: ' % (name, locid, apid, bssid, ssid)
                    f.write(s)
                    f.write(','.join(str(apinfo.rssi) for apinfo in data) + '\n')
                    run[bssid] = [apinfo.rssi for apinfo in data]
                        
                f.write('end: %s %s %s\n' % (name, locid, self.measurement_stopped))
                
                self.stored_measurements[name][int(locid)].append((self.measurement_stopped, run))
                
        daemonthread(target=waitForEnd, name="waitForEnd_%s" % name)
        
        return json.dumps({'started': str(datetime.now()).split()[1]})
    
    def getMeasurements(self, station, apid, fromMeasurementStore=True):
        if fromMeasurementStore:
            if not self.activeScene['measurementsfile']:
                return None
            
            key = (station, apid)
            measurements, ts = self._apid2measurements.get(key, (None, None))
            f = path(self.activeScene['measurementsfile'] % (station, apid))
            if measurements is None or (f.exists() and f.mtime > ts):
                measurements = {}
                try:
                    if not f.exists():
                        log.warn('no measurement data found at %s' % f.abspath())
                        self._apid2measurements[key] = (None, None)
                        return None
                        
                    else:
                        log.info('loading measurement data from %s' % f.abspath())
                        
                    for line in f.lines():
                        if not line.strip():
                            continue
                        
                        first, all_rssis = line.split(':')
                        locid, x, y, z, avgrssi = first.split()
                        rssis = [float(e) for e in all_rssis.split(',')]
                        measurements[locid] = Measurement(float(x), float(y), float(z), float(avgrssi), rssis)
                    self._apid2measurements[key] = (measurements, f.mtime)
                except Exception:
                    log.error('cannot load measurements from %s' % f.abspath())
                    measurements = {}
                    
            return measurements
        else:
            measurements = {}
            MIN_DIST = 0.2 # only use measurements that are near a location
            
            # from tracked path
            pathids2runids = evaluate.getCollectedPathids2runids(station, self.activeScene['paths'], 
                                                    self.config['tmp_tracked'])
            
            for pathid, runs in pathids2runids.items():
                log.debug('extracting measurements from %s/%s/%s' % (apid, pathid, ','.join(runs)))
                
                locid2rssis = defaultdict(list)
                locids = [int(e) for e in self.activeScene['paths'][pathid]]
                for runid in runs:
                    
                    try:
                        ms = localize.Measurements.fromTrackedMeasurements(self.known_locations, self.config['tmp_tracked'], 
                                                                station, pathid, runid)
                    except Exception:
                        log.error('cannot read path %s/%s' % (pathid, runid))
                        continue
                        
                    nearestMeasurements = defaultdict(lambda : (None, None))
                    for m in ms:
                        for locid in locids:
                            loc = self.known_locations[locid]
                            _m, distance = nearestMeasurements[locid]
                            curr_dist = ((m.pos.x - loc.x)**2 + (m.pos.y - loc.y)**2 + + (m.pos.z - loc.z)**2)**0.5
                            if distance is None or curr_dist < distance:
                                nearestMeasurements[locid] = (m, curr_dist)
                
                    for locid in locids:
                        m, distance = nearestMeasurements[locid]
                        rssi = m.apdata.get(apid)
                        if distance < MIN_DIST and rssi is not None:
                            locid2rssis[locid].append(rssi)
                
                # now proces all collected rssis for the given path
                for locid, rssis in locid2rssis.items():
                    loc = self.known_locations[locid]
                    if len(rssis) > 0:
                        avgrssi = sum(rssis) / len(rssis)
                        measurements[str(locid)] = Measurement(float(loc.x), float(loc.y), float(loc.z), float(avgrssi), rssis)
                
            return measurements
    def loadMeasurementDump(self):
        ''' load measurements from file that includes all stations '''
        self.stored_measurements.clear()
        run = {}
        for i, l in enumerate(MEASUREMENTS_STORAGE.lines()):
            try:
                if l.startswith('start:'):
                    l2 = l.split()
                    
                    _, station, locid, d1, d2 = l2
                    run[(station, locid)] = {}
                elif l.startswith('end:'):
                    l2 = l.split()
                    if len(l2) == 5:
                        _, station, locid, d1, d2 = l2
                        if (station, locid) in run:
                            dt = parser.parse('%s %s' % (d1, d2))
                            self.stored_measurements[station][int(locid)].append((dt, run[(station, locid)]))
                            #~ self.stored_measurements[self.active_station]
                        run.pop((station, locid), None)
                else:
                    if 'rssi:' in l:
                        meta, rssis = l.split('rssi:')
                        meta = meta.split()
                        station, locid, apid, mac = meta[:4]
                        
                        rssis = [float(e) for e in rssis.split(',') if e.strip()]
                        
                        if (station, locid) in run:
                            run[(station, locid)][apid] = rssis
                
            except Exception:
                log.exception('error during parsing line %s: %s]' % (i+1, l))
    @cherrypy.expose
    def getMeasurementResults(self, name, locid):
        cherrypy.response.headers['Content-Type'] = 'application/json'
        
        apdata = {}
        if self.measure_enabled is None:
            aps = self.tracked_measurements[name][locid].items()
            for (bssid, ssid), data in aps:
                rssis = [apinfo.rssi for apinfo in data]
                
                avg_rssi = sum(rssis) / float(len(rssis)) if len(rssis) > 0 else 0
                
                apdata[bssid.replace(':', '_')] = {'rssis': rssis, 'avg': '%.1f' % avg_rssi}
        
        # signal still running measurement with endts = ""
        endts = str(self.measurement_stopped).split()[1] if self.measurement_stopped is not None else ''
            
        return json.dumps({'data': apdata,
                            'stopped': endts,
                            })
    
    @nocache
    @cherrypy.expose
    def buildOptimizefile(self, station):
        cherrypy.response.headers['Content-Type'] = 'text/plain'
        self.loadMeasurementDump()
        
        log.info('known locids: %s' % ','.join(str(e) for e in self.known_locations.keys()))
        
        apid2locs = defaultdict(lambda: defaultdict(list))
        for locid in self.stored_measurements[station]:
            for dt, runs in self.stored_measurements[station][locid]:
                for apid in runs:
                    # ignore all mac addresses, that are not configured
                    if not apid in self.config['aps']:
                        continue
                    
                    apid2locs[apid][locid].extend(runs[apid])
        res = ''
        for apid, locs in apid2locs.items():
            outfile = OPTIMIZE_FILES % (station, apid)
            res += 'write to %s\n' % path(outfile).abspath()
            with open(outfile, 'w') as f:
                for locid, rssis in locs.items():
                    if not locid in self.known_locations:
                        log.warn('skipping unknown location %s' % locid)
                        continue
                    loc = self.known_locations[locid]
                    avg_sig = sum(rssis) / float(len(rssis))
                    
                    _str_rssis = ','.join(str(e) for e in rssis)
                    f.write('%s %s %s %s %.2f : %s\n' % (locid, loc.x, loc.y, loc.z, avg_sig, _str_rssis))
            res += '%s\n' % path(outfile).text()
            
        return res
    
    
    @cherrypy.expose
    def initDevice(self, name):
        cherrypy.response.headers['Content-Type'] = 'text/plain'
        return 'true 0.5'
    
    
    def _localize(self, device):
        SHUTDOWN_TIMEOUT = 10 #seconds
        CUBE_WIDTH = 3
        COUNT = 2000
        try:
            
            # init stuff
            env = self.getLocalizationEnv(device)
            
            last_ts = time.time()
            lmse = localize.LMSELocalizer(env, cubewidth=CUBE_WIDTH, verbose=False, with_history=True)
            hmm = localize.HMMLocalizer(env, cubewidth=CUBE_WIDTH, verbose=False)
            #~ localizers = [('lmse', lmse), ('hmm', hmm)]
            localizers = [('hmm', hmm)]
            #~ localizers = [('lmse', lmse)]
            #~ localizers = []
            
            while True:
                try:
                    # get from Queue and raises Queue.empty exception
                    data = self.localization_data[device].get(timeout=0.03)
                    
                    ts2apdata = defaultdict(dict)
                    for info in data:
                        cfg = self.mac2cfg.get(info['bssid'])
                        if not cfg:
                            continue
                        apid = cfg.name
                        
                        if apid in self.localizeSessionInactiveAPs[device]:
                            continue
                            
                        ts = round(info['ts'], 1) # round to 100ms granularity
                        if 'pos' in info:
                            pos = tuple(info['pos'])
                        else:
                            pos = (0,0,0)
                        
                        ts2apdata[(pos, ts)][apid] = info['rssi']
                        
                    
                    measurements = []
                    for (pos, ts), apdata in sorted(ts2apdata.items()):
                        remotets = datetime.fromtimestamp(ts)
                        measurements.append(localize.Measurement(remotets, apdata, pos))
                    
                    tt = time.time()
                    for curr_m in measurements:
                        for algo, localizer in localizers:
                            paths, measurements_modified = localizer.evaluateNext(localize.Measurements([curr_m]))
                            measurements_a = measurements_modified.asArray(localizer.apids)
                            #~ xyz = paths['seq'][0]
                            for mm, ma, i in zip(measurements_modified, measurements_a, reversed(range(1, len(paths.values()[0])+1))):
                                xyzs = [(env.cube2meter(x, y ,z , cubewidth=CUBE_WIDTH), costs) for x, y, z, costs  in localizer.decoder.history[-i][-COUNT:]]
                                res = [(x, y, z, costs, ma, mm) for (x, y, z), costs in xyzs]
                                for sessionid in list(self.localizeSessions[device]):
                                    self.last_result[(device, sessionid, algo)].put(res)
                                    
                    #~ log.debug('both in %.3f secs' % (time.time() - tt)
                            
                except Queue.Empty:
                    if time.time() - last_ts > SHUTDOWN_TIMEOUT:
                        log.info('shutdown localizer worker')
                        break
                    time.sleep(0.02)
                    continue
                else:
                    last_ts = time.time()
        finally:
            self.localization_data.pop(device)
            self.localizeSessions[device].clear()
            self.localizeSessionInactiveAPs.clear()
            
        
    def getLocalizationEnv(self, device):
        if not device in self.device2localizationenv:
            locfile = self.activeScene['locationfile']
            objfile = self.activeScene['objfile']
            
            aps = []
            for apid in self.config['aps'].sections:
                apcfg = self.config['aps'][apid]
                aps.append((apid, apcfg['x'], apcfg['y'], apcfg['z']))
            
            # HARDCODE
            #optrun = 'iconia.basic5mat_1'
            #optrun = 'optsession_1_swaped'
            #optrun = 'optnetwork4_1'
            optrun = self.config['serve_localize_apdata']
            if not optrun: 
                print "Error: No optrun"
            else: 
                print "optrun: ", optrun

            optrundir = self.config['tmp_apdata'].joinpath(optrun)
            vip = optrundir.joinpath(self.activeScene.name + '_%s.dat')
                    
            env = localize.Environment(objfile=objfile, aps=aps, locationfile=locfile, tmpdir=self.config['tmp'], 
                                        vi_path=vip, bbox2d=self.activeScene['bbox'][:4],
                                        davalues=[])
            
            self.device2localizationenv[device] = env
        return self.device2localizationenv[device]
    
    @cherrypy.expose
    def newData(self, name, data):
        ''' receive new data from a measuring device '''
        try:
            localts = datetime.now()
            
            if isinstance(data, basestring):
                #TODO: use json?
                data = eval(data)
            
            if not name in self.mstations:
                self.mstations[name] = MobileStation(name)
            
            unique_aps = len({e['bssid'] for e in data})
            unique_ts = len({e['ts'] for e in data})
            
            log.info('receiving data from %s with %s aps [%s unique ts]' % (name, unique_aps, unique_ts))
            station = self.mstations[name]
            
            # this is probably not threadsafe
            if not name in self.localization_data:
                queue = self.localization_data[name]
                daemonthread(target=self._localize, name="localize_%s" % name, args=(name,))
            else:
                queue = self.localization_data[name]
            queue.put(data)
            
            ## check aliases
            rssids_from_aliased = defaultdict(list)
            for info in data:
                mac = info['bssid']
                apid = self.aliasedmac2apid.get(mac)
                if apid is not None:
                    found = False
                    # first check if alias target is in data
                    for info2 in data:
                        if self.mac2cfg.get(info2['bssid']) == apid:
                            found = True
                    
                    if not found:
                        # change the mac of this info record
                        info['bssid'] = self.config['aps'][apid]['mac']
                    else:
                        rssids_from_aliased[apid].append(info['rssi'])
                    
                
            last_rssi = {}
            
            for info in data:
                # (mac, ssid)
                key = (info['bssid'], info['ssid'])
                    
                if not key in last_rssi and key in station.data:
                    last_rssi[key] = station.data[key][-1].rssi
                    
                if not key in last_rssi:
                    delta = 0
                else:
                    delta = last_rssi[key] - info['rssi']
                
                
                rssi = info['rssi']
                if info['bssid'] in self.mac2cfg and self.mac2cfg[info['bssid']].name in rssids_from_aliased:
                    rssis = rssids_from_aliased[self.mac2cfg[info['bssid']].name]
                    rssi = sum(rssis + [rssi]) / (len(rssis) + 1)
                    
                last_rssi[key] = rssi
                
                api = APInfo(rssi=rssi, 
                            remotets=datetime.fromtimestamp(info['ts']), 
                            localts=localts,
                            delta=delta)
                #~ print api.localts - api.remotets 
                
                if self.measure_enabled is not None:
                    stationname, locid, _, _ = self.measure_enabled
                    
                    measurement_info = self.tracked_measurements[stationname][locid]
                    measurement_info[key].append(api)
                
                elif self.pathtrack_enabled is not None:
                    stationname, pathid, _ = self.pathtrack_enabled
                
                    measurement_info = self.tracked_measurements[stationname][pathid]
                    measurement_info[key].append(api)
                                
                
                station.data[key].append(api)
        except Exception:
            log.exception('error during receiving data')
            
        
    @cherrypy.expose
    def uploadobjfile(self, expectedhash, objfile):
        ''' remote guis can upload their locally modified obj files over this interface'''
        objpath = self.config['tmp'].joinpath('obj')
        s = objfile.file.read()
        h = hashlib.sha1(s).hexdigest()
        assert h == expectedhash
        
        name = path(objfile.filename.replace('\\', '/').split('/')[-1]).namebase
        f = objpath.joinpath('%s_%s.obj' % (h, name))
        
        log.info('storing objfile [%.1f kb] to %s...' % (len(s) / 1024.0, f.abspath()))
        f.write_bytes(s)
        self.hash2objfile[h] = f
            
    @cherrypy.expose
    def queueJob(self, scenename, objhash, materials, ap, bbox, resolution, density, numphotons, disabledObjects=''):
        ''' queue a rendering job from a remote gui'''
        cherrypy.response.headers['Content-Type'] = 'text/xml'
        
        if not objhash in self.hash2objfile:
            log.warn('need new obj file')
            return '<error>unknown objfile</error>'
            
        objfile = self.hash2objfile[objhash]
        
        # eval is quite unsecure  - see http://stackoverflow.com/questions/661084/security-of-pythons-eval-on-untrusted-strings
        resolution = Resolution(**eval(resolution, {'__builtins__':[]}, {}))
        bbox = BBox(**eval(bbox, {'__builtins__':[]}, {}))
        ap = simulator.AccessPoint(**eval(ap, {'__builtins__':[]}, {}))
        materials = loadMaterialsFromString(materials)
        density = float(density)
        numphotons = int(numphotons)
        disabledObjects = disabledObjects.split(',')
        
        run = simulator.Run(scenename, objfile, materials, ap, bbox, resolution, 
                            density, numphotons, code='transfer_result', disabledObjects=disabledObjects)
        
        self.simulator.queue(run, simulator.HIGH_PRIO)
        log.info('queued run %s' % run.id)
        
        return '<success state="queued" id="%s"/>' % run.id
        
    @cherrypy.expose
    def getJob(self, worker):
        cherrypy.response.headers['Content-Type'] = 'text/xml'
        ctrlfile = self.simulator.getWork(worker)
        if ctrlfile is None:
            return '<empty/>'
        return ctrlfile.text()
    
    @cherrypy.expose
    def transferFullResult(self, worker, runid, zippedfile):
        ''' called by worker_code_transfer_result - sends back .raw and .dat file'''
        zippeddata = zippedfile.file.read()
        self.simulator.storeTransferedResult(worker, runid, zippeddata)
    
    @cherrypy.expose
    def transferResultAtLocIDs(self, worker, runid, data):
        ''' called by worker_code_transfer_result - sends back .raw and .dat file
        data is a dict with key: locid, value: rssi
        '''
        data = eval(data, {'__builtins__':[]}, {})
        #~ t = time.time()
        self.simulator.storeResultAtLocIDs(worker, runid, data)
        #~ print time.time() - t
        
    @cherrypy.expose
    def queryJobs(self, runids):
        cherrypy.response.headers['Content-Type'] = 'text/xml'
        waiting, running, finished = self.simulator.queryRuns(runids.split(','))
        tree = StringToTree('<result/>')
        tree.getroot().attrib['waiting'] = ','.join(waiting)
        tree.getroot().attrib['running'] = ','.join(running)
        tree.getroot().attrib['finished'] = ','.join(finished)
        
        return TreeToString(tree)
    
    @cherrypy.expose
    def getJobZipData(self, runid):
        for run in self.simulator.finished_jobs:
            if runid == run.id:
                for f in run.resultfiles:
                    if f.endswith('.zip'):
                        return f.bytes()
    
    @cherrypy.expose
    def getObjData(self, worker, objfile, runid):
        cherrypy.response.headers['Content-Type'] = 'text/plain'
        #~ run = simulator.Run.get(runid)
        log.info('serving objfile %s from run %s to worker %s' % (objfile, runid, worker))
        of = self.simulator.tmpdir.joinpath(runid, objfile)
        return of.text()
    
    @cherrypy.expose
    def brokenJob(self, worker, runid):
        log.warn('signaling broken job [%s]' % cherrypy.request.headers.get('user-agent', ''))
        
        self.simulator.brokenRun(worker, runid)
        if 'Referer' in cherrypy.request.headers:
            # redirect to referer
            raise cherrypy.HTTPRedirect(cherrypy.request.headers['Referer'])
    
    @cherrypy.expose
    def optimizePlot(self, session, onlymins='false'):
        reload(plotter)
        sdir = self.optimizer.tmpdir.joinpath(session)
        
        if onlymins == 'true':
            lines =  sdir.joinpath('mins.txt').lines()[:-1]
            plotfile = sdir.joinpath('_plot_mins.png')
            highlightmins = False
        else:
            lines = []
            for f in list(sorted(sdir.files('population_*.txt'), key=lambda x: x.mtime)):
                lines.extend(f.lines()[:-1])
            
            plotfile = sdir.joinpath('_plot_all.png')
            highlightmins = True
        
        max_mt = max([f.mtime for f in sdir.files('*.txt')] + [0])
        
        if not plotfile.exists() or max_mt > plotfile.mtime:
            with self.plotLock:
                plotter.plotOptimizeRun(lines, plotfile, highlightmins=highlightmins)
            
        return serve_file(plotfile)
    
    @cherrypy.expose
    def simulatorStatsPlot(self):
        reload(plotter)
        tmpdir = self.simulator.tmpdir
        statsfile = tmpdir.parent.joinpath('simulator.stats')
        assert statsfile.exists()
        
        plotfile = tmpdir.joinpath('simulatorStats.png')
        lines = statsfile.lines()
        with self.plotLock:
            plotter.plotSimulatorStats(lines, plotfile)
        return serve_file(plotfile)
    
    
    @cherrypy.expose
    def getAPConfig(self):
        cherrypy.response.headers['Content-Type'] = 'text/xml'
        tree = StringToTree('<aps/>')
        for apid in self.config['aps'].sections:
            cfg = self.config['aps'][apid]
            
            attribs = {'mac': cfg['mac'], 'x': cfg['x'], 'y': cfg['y'], 'z': cfg['z'],
            'on': str(cfg['on']).lower(), 'powerid': cfg['powerid'], 'id': apid, 'power': cfg['power']}
            subelement(tree.getroot(), 'ap', attribs=attribs)
        
        return TreeToString(tree)
    
    @cherrypy.expose
    def getObjFile(self):
        cherrypy.response.headers['Content-Type'] = 'text/plain'
        return self.activeScene['objfile'].text()
    
        
    @cherrypy.expose
    def getMaterialsFile(self):
        cherrypy.response.headers['Content-Type'] = 'text/xml'
        materials = {}
        for matname in self.activeScene['materials'].scalars:
            reflect, alpha = self.activeScene['materials'][matname]
            materials[matname] = Brdf(reflect, alpha)
        
        return TreeToString(materialsAsXml(materials))
        
    
    @cherrypy.expose
    def getSceneConfig(self):
        cherrypy.response.headers['Content-Type'] = 'text/xml'
        tree = StringToTree('<scene/>')
        e = tree.getroot()
        e.attrib['bbox'] = ','.join(str(e) for e in self.activeScene['bbox'])
        e.attrib['resolution'] = ','.join(str(e) for e in self.activeScene['resolution'])
        e.attrib['numphotons'] = str(self.activeScene['numphotons'])
        e.attrib['density'] = str(self.activeScene['density'])
        e.attrib['name'] = self.activeScene.name
        e.attrib['device'] = self.config['defaultDevice']
        e.attrib['alldevices'] = ','.join(self.config['knownStations'])
        return TreeToString(tree)
        
    @cherrypy.expose
    def getOptimizeSessions(self):
        cherrypy.response.headers['Content-Type'] = 'text/xml'
        tree = StringToTree('<sessions/>')
        _key = lambda x: x.files('*.txt')[0].mtime if len(x.files('*.txt')) > 0 else 0
        for i, d in enumerate(reversed(sorted(self.optimizer.tmpdir.dirs(), key=_key))):
            f = d.joinpath('init.txt')
            if not f.exists():
                continue
            initparams = dict([(e.split(':')[0], e.split(':')[1].strip()) 
                            for e in f.text().split('\n') if e.strip()])
            attribs = {'name': d.name, 'dt': str(datetime.fromtimestamp(d.mtime)).split('.')[0]}
            attribs.update(initparams)
            sessionEl = subelement(tree.getroot(), 'session', attribs=attribs)
        
            mindelta = -1
            f = d.joinpath('mins.txt')
            if f.exists():
                mindata = f.text().strip().split('\n')[-1].split()
                e = subelement(sessionEl, 'minimum')
                for s in mindata[1:]:
                    name, value = s.split(':')
                    e.attrib[name.replace('/', '-')] = value
            
        return TreeToString(tree)
    
    
    def _plot2dmap(self, level, refresh, customdraw=None):
        reload(plotter)
        tmpdir = self.simulator.tmpdir
        plotfile = tmpdir.joinpath('2dcut_%s.png' % level)
        aps = {e.name: e for e in self.mac2cfg.values()}
        plotter.plot2DMap(self.activeScene, level, self.known_locations, aps, plotfile, refresh, customdraw)
        return plotfile
        
    @cherrypy.expose
    def plot2DMap(self, level, **kwds):
        plotfile = self._plot2dmap(level, 'refresh' in kwds)
        return serve_file(plotfile)
    
    @cherrypy.expose
    def plotMeasurementCoverage(self, level, apfilter=None, **kwds):
        plotfile = self.simulator.tmpdir.joinpath('2dcut_%s_fancy.png' % level)    
        def customdraw(img, draw, m2d, z_cut, aps, locations):
            for apid, apcfg in aps.items():
                if apfilter is not None and not apfilter in apid:
                    continue
                for locid, loc in locations.items():
                    if abs(loc.z - z_cut) < 2 and abs(apcfg['z'] - z_cut) < 2:
                        x1, y1 = m2d.meter2pixel(apcfg['x'], apcfg['y'])
                        x2, y2 = m2d.meter2pixel(loc.x, loc.y)
                        draw.line((x1, y1, x2, y2), fill=(64,64,64))
            
            img.save(plotfile)

        self._plot2dmap(level, 'refresh' in kwds, customdraw)
        
        return serve_file(plotfile)
    
    @cherrypy.expose
    def getLocationsAndMeasurements(self, station, overlay_tracked='false'):
        cherrypy.response.headers['Content-Type'] = 'text/xml'
        tree = StringToTree('<root/>')
        locationsEl = subelement(tree.getroot(), 'locations')
        for locid, loc in self.known_locations.items():
            attribs = {'id': locid, 'x': str(loc.x), 'y': str(loc.y), 'z': str(loc.z)}
            subelement(locationsEl, 'location', attribs=attribs)
        
        measurementsEl = subelement(tree.getroot(), 'measurements')
        for mac, apcfg in self.mac2cfg.items():
            attribs = {'mac': mac, 'id': apcfg.name}
            apEl = subelement(measurementsEl, 'ap', attribs=attribs)
            measurements = self.getMeasurements(station, apcfg.name, fromMeasurementStore=overlay_tracked != 'true')            
            if measurements is not None:
                for locid, measurement in measurements.items():
                    attribs = {'id': locid, 'avgrssi': measurement.avgrssi}
                    subelement(apEl, 'location', attribs=attribs).text = ','.join(str(e) for e in measurement.rssis)
        
        
        return TreeToString(tree)
    
    @cherrypy.expose
    def startCollectPath(self, station, pathid):
        log.info('starting collecting path for station: %s and path: %s' % (station, pathid))
        self.pathtrack_enabled = (station, pathid, time.time())
        self.tracked_measurements[station][pathid] = defaultdict(list)
        self.tracked_positions[station][pathid] = []
        
    def _getPathByRemoteTS(self, tracked_measurements):
        by_remotets = defaultdict(list)
        for (mac, ssid), values in tracked_measurements.items():
            for apinfo in values:
                
                #!!! HACK for eduroam multi personalities
                aps = [('eduegcorr1', '00:17:df:a8:19:e'),
                    ('edueg031', '00:17:df:a8:1b:4'),
                    ('eduegcorr2' ,'00:17:df:a7:e8:0'),
                    ('eduegdemo', '00:17:df:a8:19:9'),
                    ('eduog1108', '00:17:df:a8:03:2'),
                    ('eduog1server', '00:17:df:a7:e8:e'),
                    ('eduog2206', '00:17:df:a8:a4:f'),
                    ('eduog2corr', '00:17:df:a7:ea:f'),
                    ('eduog2kitchen', '00:17:df:a7:e9:3'),
                    ]
                for apid, mac_prefix in aps:
                    if mac.startswith(mac_prefix):
                        mac = self.config['aps'][apid]['mac']
                        break
                        
                # kill last 5 microseconds digits
                ts = apinfo.remotets.replace(microsecond = apinfo.remotets.microsecond / 10**5 * 10**5)
                by_remotets[ts].append((mac, apinfo))
        return by_remotets
        
    @cherrypy.expose
    def stopCollectPath(self, station, pathid, **kwds):
        self.pathtrack_enabled = None
        if 'broken' in kwds:
            log.warn('stopping broken measurement')
            return
            
        
        d = self.config['tmp_tracked'].joinpath(station, pathid)
        if not d.exists():
            d.makedirs()
        
        for idx in range(1, 100):
            runid = '%02d' % idx
            if not d.joinpath('measurements_%s.txt' % runid).exists():
                break
        
        self.lastCollectedPath = (station, pathid, runid)
        
        outfile = d.joinpath('measurements_%s.txt' % runid)
        outfileLocations = d.joinpath('locations_%s.txt' % runid)
        
        tracked_measurements = self.tracked_measurements[station][pathid]
        
        tracked_positions = self.tracked_positions[station][pathid]
        log.info('collecting path for station: %s and pathid: %s finished' % (station, pathid))
        log.info('got %s measurements and %s location infos' % (len(tracked_measurements), len(tracked_positions)))
        
        by_remotets = self._getPathByRemoteTS(tracked_measurements)
        
        _mean = lambda l: float(sum(l)) / len(l)
            
        with open(outfile, 'w') as f:
            for ts, values in sorted(by_remotets.items()):
                mac2rssis = defaultdict(list)
                for mac, apinfo in values:
                    if mac in self.mac2cfg:
                        mac2rssis[mac].append(apinfo.rssi)
                        
                s = ' '.join('%s:%s' % (self.mac2cfg[mac].name, '%.1f' % _mean(rssis)) for mac, rssis in mac2rssis.items() )
                    
                if not s:
                    continue
                
                f.write('%s ts:%s\n' % (s, str(ts).replace(' ', 'T')))
        
        with open(outfileLocations, 'w') as f:
            for locid, ts in tracked_positions:
                f.write('locid:%s ts:%s\n' % (locid, str(ts).replace(' ', 'T')))
                
    
    @cherrypy.expose
    def newCollectPathPosition(self, station, pathid, locid, remotets):
        cherrypy.response.headers['Content-Type'] = 'text/xml'
        log.info('got locid: %s for station: %s and pathid: %s' % (locid, station, pathid))
        
        self.tracked_positions[station][pathid].append((locid, datetime.fromtimestamp(float(remotets))))
        return '<root value="%s"/>' % len(self._getPathByRemoteTS(self.tracked_measurements[station][pathid]))
    
    def _getCollectedPathids2runids(self, station, filter=None):
        tree = self._getCollectedPaths(station)
        pathids2runids = defaultdict(list)
        for pathEl in tree.getroot().xpath('path'):
            if filter is not None and filter.endswith('_r'):
                filter = filter[:-2]
                
            if filter is not None and not filter in pathEl.attrib['id']:
                continue
            for runEl in pathEl.xpath('run'):
                pathids2runids[pathEl.attrib['id']].append(runEl.attrib['id'])
        return pathids2runids
        
    def _getCollectedPaths(self, station):
        tree = StringToTree('<paths/>')
        for pathid in self.activeScene['paths'].scalars:
            locs = ','.join(str(e) for e in self.activeScene['paths'][pathid])
            pel = subelement(tree.getroot(), 'path', attribs={'id': pathid, 'locations': locs})
            d = self.config['tmp_tracked'].joinpath(station, pathid)
            if not d.exists():
                continue
                
            fnames = sorted(d.files('measurements_*.txt'))
            for fname in fnames:
                id = fname.name[len('measurements_'):-4]
                rel = subelement(pel, 'run', attribs = {'id': id})
                subelement(rel, 'measurements').text = fname.text()
                locfile = d.joinpath('locations_%s.txt' % id)
                subelement(rel, 'locations').text = locfile.text() if locfile.exists() else ''
        return tree
        
    @cherrypy.expose
    def getCollectedPaths(self, station):
        cherrypy.response.headers['Content-Type'] = 'text/xml'
        tree = evaluate.getCollectedPaths(station, self.activeScene['paths'], self.config['tmp_tracked'])
        return TreeToString(tree)
    
    @cherrypy.expose
    def plotEvaluatedPath(self, optrun, station, pathid, runid, setting, cubewidth, algo):
        plotfile = self.config['tmp'].joinpath('evaluator', optrun, '%s_%s_%s_%s_%s_%s.png' % (station, cubewidth, algo, setting, pathid, runid))
        return serve_file(plotfile)
    @cherrypy.expose
    def interestingOptrun(self, optrun):
        f = self.optimizer.tmpdir.joinpath(optrun, '.interested')
        if f.exists():
            f.remove()
        else:
            f.touch()
        raise cherrypy.HTTPRedirect(cherrypy.request.headers['Referer'])
    
    @cherrypy.expose
    def buildOptrun(self, optrun):
        reload(evaluate)
        evaluate.buildOptrun(self, optrun)
        raise cherrypy.HTTPRedirect(cherrypy.request.headers['Referer'])
        
    @cherrypy.expose
    def tiles(self, dummy, level, zoom, x, y):
        f = self.config['tmp'].joinpath('tiles_%s' % level, zoom, x, y)
        return serve_file(f, content_type="image/jpeg")
    
    @cherrypy.expose
    def apdatatiles(self, dummy, device_apid, zoom, x, y):
        device, apid = device_apid.split(',')
        
        with self.draw_aptiles_lock:
            d = self.config['tmp'].joinpath('apdatatiles_%s' % apid)
            
            if not d.exists():
                env = self.getLocalizationEnv(device)
                reload(plotter)
                plotter.plot2DMapAPData(self, self.activeScene, env, apid)
                
        f = d.joinpath(zoom, x, y)
        return serve_file(f, content_type="image/jpeg")
    
    def intoMapSpace(self, x, y, z):
        ''' from real space (m) into map space (px)'''
        pixel_per_meter = self.activeScene['2dpixelpermeter']
        x1, x2, y1, y2, z1, z2 = self.activeScene['bbox']
        res_x = int(x *  pixel_per_meter * 8 - x1 * pixel_per_meter * 8)
        res_y = int(y *  pixel_per_meter * 8 - y1 * pixel_per_meter * 8)
        res_z = int(z *  pixel_per_meter * 8 - z1 * pixel_per_meter * 8)
        return res_x, res_y, res_z
    
    def fromMapSpace(self, x, y, z):
        ''' from map (px) into real space (m)'''
        pixel_per_meter = self.activeScene['2dpixelpermeter']
        x1, x2, y1, y2, z1, z2 = self.activeScene['bbox']
        res_x = (x + x1 * pixel_per_meter * 8) /  (pixel_per_meter * 8)
        res_y = (y + y1 * pixel_per_meter * 8) /  (pixel_per_meter * 8)
        res_z = (z + z1 * pixel_per_meter * 8) /  (pixel_per_meter * 8)
        return res_x, res_y, res_z
        
    @cherrypy.expose
    def locinfo(self, device, sessionid, algo, limit=100, skip=0):
        limit = int(limit)
        skip = int(skip)
        
        cherrypy.response.headers['Content-Type'] = 'application/json'
        
        env = self.getLocalizationEnv(device)
        
        queue = self.last_result[(device, sessionid, algo)]
        while queue.qsize() > 100:
            try:
                queue.get(False)
            except Queue.Empty:
                continue
        
        results = []
        for i in range(100):
            try:
                results.append(queue.get(False))
            except Queue.Empty:
                break
        
        #~ print len(results), (device, algo)
        
        frames = []
        for j, result in enumerate(results):
            if skip > 0 and j % skip != 0:
                continue
            
            frame = {}
            z_count = defaultdict(int)
            
            x_best, y_best, z_best, costs, ma_best, mm_best = result[-1]
            hight = z_best 
            m_best_dict = {}
            for apid, aprssi in zip(env.aps.keys(), ma_best):
                if aprssi < 0:
                    m_best_dict[apid] = aprssi
                
            x_best, y_best, z_best = env.translateToVoxel(x_best, y_best, z_best)
            
            for i, (x, y, z, costs, ma, mm) in enumerate(result):
                x, y, z = self.intoMapSpace(x, y, z)
                z_count[z] += 1
                frame[(x, y, z)] = (costs, i)
        
            top_z = list(sorted([(v, k) for k, v in z_count.items()], reverse=True))[0][1]
            
            topn_z = {}
            for (x, y, z), v in frame.items():
                if top_z != z:
                    continue
                topn_z[(x, y)] = (v[0], v[1])
        
            topn = list(sorted(topn_z.items(), key=lambda e: e[1][1]))[-limit:]
            topn = [(k[0], k[1], int(v[1]))  for k, v in topn]
            
            aps = env.aps.keys()
            
            apdata = [(e, env.aps[e].dbdata[x_best, y_best, z_best], m_best_dict.get(e, -100)) for e in aps]
            pos_x, pos_y, pos_z = self.intoMapSpace(*mm_best.pos)
            frames.append({'pos': (pos_x, pos_y, pos_z), 'z': top_z, 'topn': topn, 'aps': apdata, 'hight':hight})
                    
        #~ print frames
        
        return json.dumps({'data': frames, 'meta': {'queuesize': queue.qsize()}})
    
    @cherrypy.expose
    def spawnSyntheticSignal(self, device, lon, lat):
        x, y, _ = self.fromMapSpace(float(lon), float(lat), 0)
        env = self.getLocalizationEnv(device)
        
        ts = time.time()
        
        _x, _y, _z = env.translateToVoxel(x, y, 1)
        
        _scandata = []
        for apid, ap in env.aps.items():
            bssid = self.config['aps'][apid]['mac']
            rssi = ap.dbdata[_x, _y, _z]
            _scandata.append({'bssid': bssid, 'rssi': rssi, 'ts': ts, 'ssid': apid})
        self.newData(device, _scandata)
    
    @cherrypy.expose
    def startLocalizeSession(self, device, sessionid):
        self.localizeSessions[device].add(sessionid)
        
        #TODO: need clean up strategy for self.last_result
    
    @cherrypy.expose
    def stopLocalizeSession(self, device, sessionid):
        self.localizeSessions[device].discard(sessionid)
        for (_device, _sessionid, algo) in self.last_result.keys():
            if _device == device and _sessionid == sessionid:
                queue = self.last_result[(_device, _sessionid, algo)]
                while queue.qsize() > 0:
                    try:
                        queue.get(False)
                    except Queue.Empty:
                        continue
    
    @cherrypy.expose
    def useAPforLocalizeDevice(self, device, apid, use):
        if use == 'false':
            self.localizeSessionInactiveAPs[device].add(apid)
        else:
            self.localizeSessionInactiveAPs[device].discard(apid)
        
def handleError():
    '''cherrypy error handling function'''
    
    cherrypy.response.headers['Content-Type'] = 'text/plain'

    h = sorted("  %s: %s" % (k, v) for k, v in cherrypy.request.header_list)
    headers = '\nRequest Headers:\n' + '\n'.join(h)
    url = cherrypy.url(qs=cherrypy.request.query_string)

    formattedparams = ', '.join('%s: %s' % 
        (k, v.encode('utf-8') if isinstance(v, basestring) else '?') 
            for k, v in cherrypy.request.params.items())

    params = '\nRequest Parameters:\n  ' + (formattedparams if formattedparams.strip() else '<no parameters>')
    msg = '\nRequest Url:\n  %s\n%s\n %s\n\n' % (url, params, headers)
    
    log.error(msg, exc_info=True)

    stackTrace = "".join(traceback.format_exc())
    cherrypy.response.body = [''.join([msg, stackTrace, '\n'])]
    

def check_shutdown(sig):
    if sig == win32con.CTRL_C_EVENT:
        log.warn('received CTRL-C, inititating shutdown')
    elif sig == win32con.CTRL_CLOSE_EVENT:
        log.warn('received close, inititating shutdown')
    elif sig in (win32con.CTRL_SHUTDOWN_EVENT, win32con.CTRL_LOGOFF_EVENT):
        log.warn('received logoff/shutdown, inititating shutdown')

    cherrypy.engine.stop()
    cherrypy.engine.exit()
    print 'LWS SHUTDOWN DONE'

def startServer(configfile=None, blocking=True):
    setupLogging()
    
    if sys.platform == 'win32':
        win32api.SetConsoleCtrlHandler(check_shutdown, True)

    
    if configfile is None:
        configfile = path('./lws.ini').abspath()
        if not path('./lws.ini').exists():
            print 'no configfile given - autocreate at %s' % configfile.abspath()
            configfile.touch()
    else:
        configfile = path(configfile).abspath()


    global app
    app = LWS(configfile)
    
    
    globalconf = {
    'server.thread_pool'            : 50,
    'server.socket_host'            : app.config['httpHost'],
    'server.socket_port'            : app.config['httpPort'],

    'engine.autoreload_on'          : app.config['cpAutoRestart'],
    'log.screen'                    : app.config['cpLogScreen'],

    'tools.sessions.on'             : True,
    'tools.sessions.storage_type'   : 'ram',
    'checker.on'                    : False, # disable cherrypy checker
    'tools.decode.on'               : True, # decode with utf-8 - fallback to latin-1

    'tools.gzip.on'                 : True,
    'tools.gzip.compress_level'     : 6,
     #gzip these mimetypes 
    'tools.gzip.mime_types'         : ['text/html', 'text/plain',
                                       'text/css', 'application/x-javascript',
                                       'text/javascript', 'text/xml', 'application/javascript',
                                       'application/json',
                                       ],
    'tools.proxy.on'                : app.config['behindProxy'],
    'tools.proxy.base'              : app.config['proxyBase'],
    
    'tools.encode.on'               : True,
    'tools.encode.encoding'         : 'utf-8',
        
    'request.error_response'        : handleError,
    }

    cherrypy.config.update(globalconf)

    appconf = {}

    appconf['/static'] = {
        'tools.etags.on'         : True,
        'tools.etags.autotags'   : True,
        'tools.staticdir.on'     : True,
        'tools.staticdir.dir'    : PATH_STATIC,
        'tools.sessions.on'      : False,
        }

    cptree = cherrypy.tree.mount(app, '/', config=appconf)
    log.info('mounting server at %s:%s' % (app.config['httpHost'], app.config['httpPort']))
    cherrypy.engine.start()
    if blocking:
        cherrypy.engine.block()

def stopServer():
    cherrypy.engine.exit()