worker_code_transfer_result.py 1.5 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647
  1. def after_simulation(ctrltree, datfile, opener, mpart_opener, options):
  2. import time
  3. from zipfile import ZipFile, ZIP_DEFLATED
  4. from utils.xml import TreeToFile
  5. runid = ctrltree.getroot().attrib['runid']
  6. url = options.url + '/transferFullResult?worker=%s&runid=%s' % (WORKER_ID, runid)
  7. # try to open - sometimes the raw file seems not to be fully flushed to disk
  8. vi = VolumeImage()
  9. for i in range(5):
  10. try:
  11. vi.read(datfile)
  12. except Exception:
  13. time.sleep(0.5)
  14. if i == 4:
  15. raise IOError('cannot read datfile')
  16. else:
  17. break
  18. rawfile = datfile.parent.joinpath(datfile.namebase + '.raw')
  19. zfile = datfile.parent.joinpath(datfile.namebase + '.raw.zip')
  20. ctrlfile = datfile.parent.joinpath(datfile.namebase + '.xml')
  21. TreeToFile(ctrltree, ctrlfile)
  22. log.info('zipping %s and %s' % (rawfile, datfile))
  23. with ZipFile(zfile, mode='w', compression=ZIP_DEFLATED) as zf:
  24. zf.write(rawfile)
  25. zf.write(datfile)
  26. zf.write(ctrlfile)
  27. log.info('sending %.1f kb zipped rawdata...' % (zfile.size / 1024.0))
  28. t = time.time()
  29. try:
  30. mpart_opener.open(url, data={'zippedfile': open(zfile, "rb")})
  31. except Exception:
  32. log.exception('cannot send data to %s' % url)
  33. else:
  34. log.info('transmitted with %.2f kb/sec' % (zfile.size / 1024.0 / (time.time() - t)))