''' % ('cluster',
len(worker_trs),
'\n'.join(worker_trs),
len(waiting_trs),
'\n'.join(waiting_trs),
len(running_trs),
'\n'.join(running_trs),
len(finished),
'\n'.join(finished_trs),
)
return render(body=body)
def getInterestingOptruns(lws):
return [d.name for d in lws.optimizer.tmpdir.dirs() if d.joinpath('.interested').exists()]
def getAllOptruns(lws, filter=None):
''' return optruns that have reached at least a minimum'''
dirs = [d for d in lws.optimizer.tmpdir.dirs() if d.joinpath('mins.txt').exists()]
if filter is not None:
_dirs = []
for d in dirs:
data = [s for s in d.joinpath('mins.txt').lines()[-1].split()]
data_init = [s for s in d.joinpath('init.txt').lines() if s.strip()]
allowed = []
if 'gen' in filter:
for s in data:
if s.startswith('gen:'):
allowed.append(int(s[4:]) > filter['gen'])
if filter.get('station'):
for s in data_init:
if s.startswith('station:'):
allowed.append(any(e in s[8:] for e in filter['station'].split(',') if e.strip()))
if filter.get('ts'):
allowed.append(data[0] >= filter['ts'])
if all(allowed):
_dirs.append(d)
dirs = set(_dirs)
return list(sorted(d.name for d in dirs))
def renderOptimizer(lws):
active_session = lws.optimizer.optsession if lws.optimizer.running else None
session_trs = []
lastinit = {}
files = sorted(lws.optimizer.tmpdir.dirs(),
key=lambda x: x.files('*.txt')[0].mtime if len(x.files('*.txt')) > 0 else 0)
for i, d in enumerate(reversed(files)):
f = d.joinpath('init.txt')
if not f.exists():
continue
initparams = dict([(e.split(':')[0], e.split(':')[1].strip())
for e in f.text().split('\n') if e.strip()])
if i == 0:
lastinit = initparams
trdata = defaultdict(str)
trdata.update({'name': d.name,
'ts': datetime.fromtimestamp(d.mtime).strftime('%d.%m %H:%M')})
trdata.update(initparams)
trdata['min_avg_delta'] = ''
trdata['min_generation'] = ''
trdata['curr_generation'] = ''
f = d.joinpath('mins.txt')
if f.exists():
lastmin = f.text().strip().split('\n')[-1].split()
generation, orgid, mindelta = lastmin[1].split(':')[1], lastmin[2].split(':')[1], lastmin[3].split(':')[1]
trdata['min_avg_delta'] = '%s@%s' % (mindelta, generation)
cmp = lambda x: int(x.namebase.split('_')[1]) if x.namebase.startswith('population_') else -1
popfiles = list(sorted(d.files('*.txt'), key=cmp))
# and again get the last line
lastsim = popfiles[-1].text().strip().split('\n')[-1].split()
generation, orgid = lastsim[1].split(':')[1], lastsim[2].split(':')[1]
trdata['curr_generation'] = '%s' % generation
trdata['isinterest'] = 'yes' if d.joinpath('.interested').exists() else 'no'
station_evaluate = []
for station in initparams['station'].split(','):
a = '%s' % (station, d.name, station)
station_evaluate.append(a)
trdata['stations'] = '\n'.join(station_evaluate)
restarturl = '/'
tr = '''
''' % {'pagename': 'optimize',
'active_session': active_session,
'stoppable': 'none' if active_session is None else 'inline',
'lastsessions': '\n'.join(session_trs),
'power': lastinit.get('power', '2.0e-6'),
'session': lastinit.get('session', 'optsession'),
'station': lastinit.get('station', lws.config['defaultDevice']),
'timelimit': lastinit.get('timelimit', '0'),
'genlimit': lastinit.get('genlimit', '0'),
'density': lastinit.get('density', str(lws.activeScene['density'])),
'resolution': lastinit.get('resolution', ','.join(str(e) for e in lws.activeScene['resolution'])),
'numphotons': lastinit.get('numphotons', str(lws.activeScene['numphotons'])),
'bbox': lastinit.get('bbox', ','.join(str(e) for e in lws.activeScene['bbox'])),
'scene': lastinit.get('scene', lws.activeScene.name),
'startpop': lastinit.get('startpop', '6'),
'childcount': lastinit.get('childcount', '8'),
'childcull': lastinit.get('childcull', '20'),
'mutprob': lastinit.get('mutprob', '0.1'),
'mutamt': lastinit.get('mutamt', '0.1'),
}
css = '''
td.genparam, td.raytracerparam {
display:none;
}
tr.optrun:hover { background:#DDF; }
table{
border-collapse:collapse;
border: 1px solid #ccc;
font-size: 14;
float:left;
};
'''
return render(body=body, css=css)
def renderEvaluteOptruns(lws, optruns, compact=False, **kwargs):
optruns = [e for e in optruns.split(',') if e]
if 'all' in optruns:
optruns = getAllOptruns(lws, filter={'gen': 20,
'station': kwargs.get('station', ''),
'ts': kwargs.get('ts')})
optruns_a = []
for o in sorted(getInterestingOptruns(lws)):
_optruns = set(optruns)
if o in _optruns:
if len(_optruns) > 1:
_optruns.discard(o)
style = 'style="font-weight:bold"'
else:
_optruns.add(o)
style = 'style="font-weight:normal"'
_compact = '' if not compact else '?compact'
a = '%s' % (style, ','.join(sorted(_optruns)), _compact, o)
optruns_a.append(a)
optrun_containerdata = defaultdict(list)
for optrun in optruns:
odir = lws.optimizer.tmpdir.joinpath(optrun)
if not compact:
ww = 'width="%s%%"' % int(100 / len(optruns))
else:
ww = ''
td = '''
%(optrun)s
''' % {'optrun': optrun, 'ww': ww}
optrun_containerdata[optrun].append(td)
trs = []
f = odir.joinpath('mins.txt')
if not f.exists():
continue
lastmin = f.lines()[-1]
for i, s in enumerate(lastmin.split()):
if i == 0:
name = 'timestamp'
value = s
else:
name, value = s.split(':')
name_css = name.replace('.', '_').replace('/', '_')
tr = '''
''' % {'optrun': optrun}
optrun_containerdata[optrun].append(td)
num_tds = len(optrun_containerdata.values()[0]) if optrun_containerdata else 0
optrun_containers = []
for i in range(num_tds):
tds = [tds[i] for optrun, tds in optrun_containerdata.items()]
tr = '''
%s
''' % '\n'.join(tds)
optrun_containers.append(tr)
if not compact:
ww = 'width="100%"'
else:
ww = ''
body = '''
''' % ('\n'.join(apheader), '\n'.join(trs))
tables.insert(0, table)
## aggregated by location
trs = []
for station in stations:
locid_aggregated = []
for locid in lws.known_locations:
v = stats[(station, 'locid_%s' % locid)]
style = 'background-color:#FFCCCC' if v == 0 else ''
locid_aggregated.append('
%s
' % (style, v))
tr = '''
%s
%s
%s
%s
''' % (station, 0, 0, '\n'.join(locid_aggregated))
trs.append(tr)
locidheader = []
for locid in lws.known_locations:
locidheader.append('
%s
' % locid)
table = '''
device
avail
miss
%s
%s
''' % ('\n'.join(locidheader), '\n'.join(trs))
tables.insert(1, table)
## the rest
cutplanes = ['cut %s' % (s, s) for s in lws.activeScene['levels2d'].scalars]
body = '''
' % (pathid, pathid, len(runrows), ''.join(tds))
rows.append(tr)
rows.extend(runrows)
tds = []
for s in settings:
if len(all_avg_errors[s]) > 0:
tds.append('
''' % ( optrun,
station,
'device: %s with cubewidth: %s and errtype: %s' % (station, cubewidth, errtype),
'\n'.join(['
%s
' % s for s in settings]),
'\n'.join(rows),
)
tables.append(table)
station_divs.append('
%s
' % '\n'.join(tables))
optruns_a = []
for o in sorted(getInterestingOptruns(lws)):
_optruns = set(e for e in optruns.split(',') if e)
if o in _optruns:
if len(_optruns) > 1:
_optruns.discard(o)
style = 'style="font-weight:bold"'
else:
_optruns.add(o)
style = 'style="font-weight:normal"'
qs = '?algo=' + algo
qs += '&filter=%s' % filter if filter else ''
a = '%s' % (style, stations, ','.join(sorted(_optruns)), qs, o)
optruns_a.append(a)
stations_a = []
for s in sorted(lws.config['knownStations']):
_stations = set(stations.split(','))
if s in _stations:
if len(_stations) > 1:
_stations.discard(s)
style = 'style="font-weight:bold"'
else:
_stations.add(s)
style = 'style="font-weight:normal"'
qs = '?algo=' + algo
qs += '&filter=%s' % filter if filter else ''
a = '%s' % (style, ','.join(sorted(_stations)), optruns, qs, s)
stations_a.append(a)
body = '''
''' % ('locate', station.name, '\n'.join(trs))
return render(body=body)
def renderCluster(lws):
worker_trs = []
for wname, worker in sorted(lws.simulator.workers.items()):
total = sum(worker.durations)
avg = total / len(worker.durations) if total > 0 else 0
num_per_hour = worker.jobcount / (total / 3600) if total > 0 else 0
color = 'green' if worker.active else 'red'
available_hours = (time.time() - worker.firstseen) / 3600
active_hours = worker.jobcount * avg / 3600
tr = '''