From 610fb59cce450f09f9f7d585e7a36c6fe9e5a01a Mon Sep 17 00:00:00 2001 From: Javier Moldon Date: Thu, 2 May 2019 12:37:49 +0100 Subject: [PATCH] Clean code and organize files for v1.0. Eliminate all GUI references, and remove some old code and unused imports --- default_params.json | 2 +- documentation/docs.md | 21 +- eMERLIN_CASA_pipeline.py | 74 +++-- functions/dflux.py | 149 --------- ...IN_CASA_functions.py => eMCP_functions.py} | 58 ++-- .../{eMERLIN_CASA_plots.py => eMCP_plots.py} | 9 +- functions/{weblog.py => eMCP_weblog.py} | 47 +-- functions/eMERLIN_CASA_GUI.py | 296 ------------------ eMCP.css => utils/eMCP.css | 0 emerlin-2.gif => utils/emerlin-2.gif | Bin 10 files changed, 115 insertions(+), 541 deletions(-) delete mode 100644 functions/dflux.py rename functions/{eMERLIN_CASA_functions.py => eMCP_functions.py} (99%) rename functions/{eMERLIN_CASA_plots.py => eMCP_plots.py} (99%) rename functions/{weblog.py => eMCP_weblog.py} (93%) delete mode 100644 functions/eMERLIN_CASA_GUI.py rename eMCP.css => utils/eMCP.css (100%) rename emerlin-2.gif => utils/emerlin-2.gif (100%) diff --git a/default_params.json b/default_params.json index e399e07..67b6600 100644 --- a/default_params.json +++ b/default_params.json @@ -240,7 +240,7 @@ "p_scan_minsnr" : 2, "ap_scan_tablename" : "phscal_ap_scan.G3", "ap_scan_prev_cal" : ["bpcal.BP2", "allcal_d.K1", "allcal_p.G3"], - "ap_scan_calibrator" : "default", + "ap_scan_calibrator" : "phscals", "ap_scan_solint" : "inf", "ap_scan_spw" : ["*", "innerchan"], "ap_scan_combine" : "", diff --git a/documentation/docs.md b/documentation/docs.md index a2a3e14..900444e 100644 --- a/documentation/docs.md +++ b/documentation/docs.md @@ -1,10 +1,5 @@ - - # e-MERLIN CASA pipeline -### Documentation for v1.0.0 +### Documentation for v1.0 --- # Table of contents @@ -52,17 +47,16 @@ You are ready to execute the pipeline. If `casa` points to casa 5.4 version: - Create the working path where you will work, and copy the `inputs.txt` to your working path. - Edit the inputs.txt file: fill the `fits_path` (where the fits files are located) and the `inbase` (any name you want to give to your project). - Fill in the inputs.txt file the names of your fields: `targets`, `phscals`, `fluxcal`,`bpcal`,`ptcal`. -- Leave all other parameters as default. -- A MS `.ms` will be produced -- Data will be converted to MS and prepared. Open in a web browser the file `./weblog/index.html`. -- The weblog and the plots will be updated every time a step is finished. +- Leave all other parameters as default so all steps will be executed. +- Data will be converted to MS and prepared. Once data is loaded you can open in a web browser the file `./weblog/index.html`. +- The weblog and the plots will be continuosly updated every time a step is finished. - If you have a list of manual flags to apply, write them in `inputfg.flags` to flag data before it is averaged in the next step. - If `average = 1` a new MS will be produced with name `_avg.ms`. ### Data calibration - You may want to include additional manual flags in `./inputfg_avg.flags` - Run all the calibration steps by setting them to 1. You may prefer to run each of them one by one and check the output plots. -- It is a good practice to redo the calibration once you are happy with your flags and you are sure of all the steps. +- It is a good practice to redo the calibration once you are happy with your flags and parameters. - The whole calibration process can be repeat from scratch by setting all the steps in the Calibration section to 1, including `restore_flags` which will restore the flag status when the data was averaged. @@ -128,6 +122,11 @@ This inputs just select which of the processing steps will be executed: Additionally, steps that produce calibration tables can be set to also apply the calibration up to that point, so also modifying the corrected column. A value of 2 means run the step and apply the calibration up to that step. The standard procedure is to apply everything at the end of the calibration with the step `applycal_all = 1`, so setting 2 to any previous step is useful just to check the calibration up to each step. +### Default parameters + +All the parameters used by the pipeline are controlled by the file `default_param.json` that will be loaded as a python dictionary. That file can be either at your working directory, or the one in the directory `eMERLIN_CASA_pipeline` will be used. + +The parameters are divided in sections according to which pipeline steps they are controlling. Some parameters control or select what needs to be run, but in most cases a parameter translates directly to the CASA parameter that will be used. For example the parameter `defaults['gaincal_final']['ap_solint']`` will set the solution interval for the `ap` calibration in the step `gaincal_final`. --- diff --git a/eMERLIN_CASA_pipeline.py b/eMERLIN_CASA_pipeline.py index 29c7782..c33f516 100644 --- a/eMERLIN_CASA_pipeline.py +++ b/eMERLIN_CASA_pipeline.py @@ -2,7 +2,6 @@ import os,sys,math import numpy as np import pickle -from Tkinter import * import getopt import logging import collections @@ -14,7 +13,7 @@ import casadef -current_version = 'v0.10.24' +current_version = 'v0.10.25' # Find path of pipeline to find external files (like aoflagger strategies or emerlin-2.gif) try: @@ -27,10 +26,9 @@ pipeline_path = pipeline_path + '/' sys.path.append(pipeline_path) -import functions.eMERLIN_CASA_functions as em -import functions.weblog as emwlog -import functions.eMERLIN_CASA_plots as emplt -#from default_params import defaults +import functions.eMCP_functions as em +import functions.eMCP_weblog as emwlog +import functions.eMCP_plots as emplt casalog.setlogfile('casa_eMCP.log') @@ -65,7 +63,7 @@ def get_pipeline_version(pipeline_path): return branch, short_commit -def run_pipeline(inputs=None, inputs_path=''): +def create_dir_structure(pipeline_path): # Paths to use weblog_dir = './weblog/' info_dir = './weblog/info/' @@ -82,37 +80,67 @@ def run_pipeline(inputs=None, inputs_path=''): em.makedir(images_dir) em.makedir(logs_dir) em.makedir(plots_dir+'caltables') - os.system('cp -p {0}/emerlin-2.gif {1}'.format(pipeline_path, weblog_dir)) - os.system('cp -p {0}/eMCP.css {1}'.format(pipeline_path, weblog_dir)) - pipeline_version = current_version + os.system('cp -p {0}/utils/emerlin-2.gif {1}'.format(pipeline_path, weblog_dir)) + os.system('cp -p {0}/utils/eMCP.css {1}'.format(pipeline_path, weblog_dir)) + return calib_dir, info_dir - # Continue with previous pipeline configuration if possible: +def start_eMCP_dict(info_dir): try: eMCP = load_obj(info_dir + 'eMCP_info.pkl') except: eMCP = collections.OrderedDict() eMCP['steps'] = em.eMCP_info_start_steps() eMCP['img_stats'] = collections.OrderedDict() + return eMCP - eMCP['inputs'] = inputs +def get_logger( + LOG_FORMAT = '%(asctime)s | %(levelname)s | %(message)s', + DATE_FORMAT = '%Y-%m-%d %H:%M:%S', + LOG_NAME = 'logger', + LOG_FILE_INFO = 'eMCP.log', + LOG_FILE_ERROR = 'eMCP_errors.log'): - # Setup logger - logger = logging.getLogger('logger') + log = logging.getLogger(LOG_NAME) + log_formatter = logging.Formatter(fmt=LOG_FORMAT, datefmt=DATE_FORMAT) logging.Formatter.converter = time.gmtime - logger.setLevel(logging.INFO) - #logger.setLevel(logging.DEBUG) - handler = logging.FileHandler('eMCP.log', mode = 'a') # create a file handler - formatter = logging.Formatter(fmt='%(asctime)s | %(levelname)s | %(message)s',datefmt='%Y-%m-%d %H:%M:%S') - handler.setFormatter(formatter) - logger.addHandler(handler) # add the handlers to the logger - consoleHandler = logging.StreamHandler() # Stream errors to terminal also - consoleHandler.setFormatter(formatter) - logger.addHandler(consoleHandler) + + # comment this to suppress console output + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(log_formatter) + log.addHandler(stream_handler) + + # Normal eMCP log with all information + file_handler_info = logging.FileHandler(LOG_FILE_INFO, mode='a') + file_handler_info.setFormatter(log_formatter) + file_handler_info.setLevel(logging.INFO) + log.addHandler(file_handler_info) + + # eMCP_error log containing only warnings or worse + file_handler_error = logging.FileHandler(LOG_FILE_ERROR, mode='a') + file_handler_error.setFormatter(log_formatter) + file_handler_error.setLevel(logging.WARNING) + log.addHandler(file_handler_error) + + log.setLevel(logging.INFO) + return log + + +def run_pipeline(inputs=None, inputs_path=''): + #Create directory structure + calib_dir, info_dir = create_dir_structure(pipeline_path) + + # Setup logger + logger = get_logger() + + # Initialize eMCP dictionary, or continue with previous pipeline configuration if possible: + eMCP = start_eMCP_dict(info_dir) + eMCP['inputs'] = inputs try: branch, short_commit = get_pipeline_version(pipeline_path) except: branch, short_commit = 'unknown', 'unknown' + pipeline_version = current_version logger.info('Starting pipeline') logger.info('Running pipeline from:') logger.info('{}'.format(pipeline_path)) diff --git a/functions/dflux.py b/functions/dflux.py deleted file mode 100644 index d949347..0000000 --- a/functions/dflux.py +++ /dev/null @@ -1,149 +0,0 @@ -import numpy -import math -import sys - - -####### -# Python version of 3C286 flux calculation program (original author unknown) -# .............................. -# Author DMF 20/10/2011 -# .............................. -# -# Update to use Perley & Butler 2012 coefficients -# 10/04/2013 -# DMF -######## - -# Reworked to use the 1999 VLA flux formula, and a 2nd formula to give a continuous estimate of the resolved fraction, by Ian Stewart, JBO, 8 Aug 2007. -# Minor changes by amsr, 8 Aug 2007 - -# my $program_name = 'dflux'; # $0 returns the './' prefix if this is used. - - -def dfluxpy(freq,baseline): - - - lowest_freq = 300.0; - highest_freq = 50000.0; - if (freq < lowest_freq or freq > highest_freq): - print "Frequency must be between $lowest_freq and $highest_freq MHz. \n" - - # Old values for 3C286 - # A = 1.23734 - # B = -0.43276 - # C = -0.14223 - # D = 0.00345 - - # New values taken from AIPS SETJY 31DEC11 - # Values as of 2010 - - # A = 1.2361 - # B = -0.4127 - # C = -0.1864 - # D = 0.0294 - - # Perley & Butler 2012 values - A = 1.2515 - B = -0.4605 - C = -0.1715 - D = 0.0336 - - log10f = (math.log(freq)/2.3025851) - 3.0; # Why the -3? Because freq has to be GHz for the formula to work. - log_flux = A + B*log10f + C*log10f*log10f + D*log10f*log10f*log10f - vlaflux = math.pow(10.0,log_flux) - - - - - # The VLA flux must now be corrected to account for the higher resolving power of merlin. The formula used was obtained with the help of Peter Thomasson. If we assume that 3C286 is represented by a gaussian of angular size theta_s, and represent the resolving power of the given baseline as a function of frequency f and antenna separation L by theta_b(f,L), then the reduction in central flux density A(0) due to the finite theta_s is given by - # - # 1 - # ----------------------------------- - # A'(0) 2 pi (theta_b(f,L)^2 + theta_s^2) - # ------- = --------------------------------------- , - # A(0) 1 - # --------------------- - # 2 pi theta_b(f,L)^2 - # - # 1 - # = -------------- , - # 1 + rho(f,L) - # - # where the resolved fraction rho(f,L) is given by - # - # theta_s^2 - # rho(f,L) = ---------------- . - # theta_b(f,L)^2 - # - # Use of theta_b(f,L) = k/(fL) allows this to be written - # - # ( f*L )^2 - # rho(f,L) = (-----------) * rho_ref . - # (f_ref*L_ref) - # - # The reference value of rho is fixed at 0.04 for the MK-TA baseline at 5 GHz (Peter Thomasson). - - ref_bl_length = 11236.79 # MK-TA separation in metres. - ref_freq = 5000.0 - ref_rho = 0.04 - thisbl = "this baseline (Mk-Ta)" - - bl_length = baseline - # bl_str = sprintf "%8.2f", $ref_bl_length; - - - frac = (freq / ref_freq) * (bl_length / ref_bl_length) - rho = frac * frac * ref_rho - merlinflux = vlaflux / (1.0 + rho) - - # Another useful quantity is the resolved percentage: - # - resolved_percent = 100.0 * rho / (1.0 + rho) - caution_res_pc = 10.0 - - return vlaflux, merlinflux, resolved_percent, caution_res_pc, thisbl - - - - - -##################################################################### -''' - -if __name__ == '__main__': - - helptext = "This command prints the flux density of the flux calibrator 3C286\nat a given frequency along with advice for use with MERLIN. \n\nUsage: dfluxpy [-h] []\n Examples:\n dflupy 4994 assumes MK-TA baseline.\n dfluxpy 6034.8 15923 ie, work out the MERLIN correction for the DA-TA baseline instead.\n Some short MERLIN baselines (m):\n MK-TA: 11236.79 \n DA-TA: 15923.55\n MK-DA: 17737.45\n CA-MK is about 200000 m depending on hour angle (not recommended).\n" - - - if (len(sys.argv)<2): - print helptext - # raise 'notEnoughInfoOnCmdLine' - sys.exit() - - - - elif (len(sys.argv)>2): - bl_length = float(sys.argv[2]) - if (bl_length <= 0.0): - print "Second command-line argument does not seem to be non-zero baseline distance." - thisbl = "this baseline" - freq = float(sys.argv[1]) - - else: - freq = float(sys.argv[1]) - bl_length = 11236.79 - - - print freq, bl_length - vlaflux, merlinflux, resolved_percent, caution_res_pc, thisbl = dfluxpy(freq,bl_length) - - - - print '\nAt frequency',freq, 'MHz:' - print 'Flux of 3C286 =',vlaflux, 'Jy using the VLA formula with 1999.2 coefficients.' - print '3C286 is resolved by',resolved_percent, 'percent on a baseline length of ',str(bl_length),' m.' - if (resolved_percent >= caution_res_pc): - print' ->(CAUTION! Resolved values higher than about',caution_res_pc, 'percent become increasingly inaccurate.)' - print 'Suggested flux density for MERLIN on', thisbl, 'is' ,merlinflux, 'Jy. \n' - -''' diff --git a/functions/eMERLIN_CASA_functions.py b/functions/eMCP_functions.py similarity index 99% rename from functions/eMERLIN_CASA_functions.py rename to functions/eMCP_functions.py index 531791d..2c4c73d 100644 --- a/functions/eMERLIN_CASA_functions.py +++ b/functions/eMCP_functions.py @@ -4,7 +4,7 @@ import socket import pickle import glob -from Tkinter import * +import re import tkMessageBox import sys, shutil import copy @@ -12,12 +12,11 @@ import datetime import dateutil import collections -from eMERLIN_CASA_GUI import GUI_pipeline from scipy import stats import logging -import functions.weblog as emwlog -import functions.eMERLIN_CASA_plots as emplt +import functions.eMCP_weblog as emwlog +import functions.eMCP_plots as emplt # CASA imports from taskinit import * @@ -57,17 +56,14 @@ def check_in(pipeline_path): if o in ('-i','--input'): logger.info('Inputs from file: {}'.format(a)) inputs = headless(a) ## read input file -# inputs['quit'] = 0 ##needed to add to be compatible with GUI - elif o in ('-g','--gui'): - inputs = GUI_pipeline(pipeline_path).confirm_parameters() ## read input file - logger.info('inputs from GUI: {}'.format(inputs)) elif o in ('-h','--help'): - logger.debug('help will be written soon') - sys.exit('Closing pipeline eMCP') + print('Usage:') + print('casa -c /path/to/pipeline/eMERLIN_CASA_pipeline.py -i ') + sys.exit() elif o == '-c': logger.debug('Executing!') else: - assert False, "rerun with either headless -i or gui" #if none are specifed run GUI + assert False, "inputs.txt file required" return inputs def backslash_check(directory): @@ -76,7 +72,6 @@ def backslash_check(directory): else: return directory - def headless(inputfile): ''' Parse the list of inputs given in the specified file. (Modified from evn_funcs.py)''' logger.info('Parameters in inputs file:') @@ -110,25 +105,6 @@ def headless(inputfile): logger.info('{0:16s}: {1}'.format(param, valueout)) return control -def Tkinter_select(): - root = Tkinter.Tk() - root.withdraw() - file = tkFileDialog.askdirectory(parent=root,mode='rb',title='Choose a file') - if file != None: - print file - return file - -def check_history(vis): - tb.open(vis+'/HISTORY') - x = tb.getcol('MESSAGE') - y = [i for i, item in enumerate(x) if 'eMER_CASA_Pipeline:' in item] - if len(y) == 0: - print 'Measurement set has not been processed \n' - else: - print 'WARNING: Some pipeline processes have already been run' - for i in range(len(y)): - print x[y[i]] - def makedir(pathdir): try: os.mkdir(pathdir) @@ -166,6 +142,9 @@ def mvdir(pathdir, outpudir): def exit_pipeline(eMCP=''): + os.system('cp eMCP.log {}eMCP.log.txt'.format(info_dir)) + os.system('cp eMCP_errors.log {}eMCP_errors.log.txt'.format(info_dir)) + os.system('cp casa_eMCP.log {}casa_eMCP.log.txt'.format(info_dir)) if eMCP != '': logger.info('Something went wrong. Producing weblog and quiting') emwlog.start_weblog(eMCP) @@ -216,6 +195,7 @@ def add_step_time(step, eMCP, msg, t0, doweblog=True): eMCP['steps'][step] = [timestamp, delta_t_min, msg] save_obj(eMCP, info_dir + 'eMCP_info.pkl') os.system('cp eMCP.log {}eMCP.log.txt'.format(info_dir)) + os.system('cp eMCP_errors.log {}eMCP_errors.log.txt'.format(info_dir)) os.system('cp casa_eMCP.log {}casa_eMCP.log.txt'.format(info_dir)) if doweblog: emwlog.start_weblog(eMCP) @@ -2362,7 +2342,10 @@ def select_calibrator(param, eMCP, add_main=True): # Default is using all the calsources. User can specify sources in params msinfo = eMCP['msinfo'] if param == 'default': - calsources = eMCP['msinfo']['sources']['calsources'] + calsources = msinfo['sources']['calsources'] + field = ','.join(np.unique(calsources.split(','))) + elif param == 'phscals': + calsources = msinfo['sources']['phscals'] field = ','.join(np.unique(calsources.split(','))) elif param != '': field = param @@ -2699,11 +2682,14 @@ def eM_fluxscale(eMCP, caltables): cals_to_scale = select_calibrator(eMCP['defaults']['initial_gaincal']['ap_calibrator'], eMCP) fluxcal = sources['fluxcal'] caltable_name = flux['tablename'] + # Remove previous results: + rmfile(calib_dir + 'allcal_ap.G1_fluxes.txt') + rmfile(calib_dir+ '{0}_fluxscale.png'.format(msinfo['msfilename'])) # Check if table allcal_ap.G1 exists: check_table_exists(caltables, ampcal_table) caltables[caltable_name] = copy.copy(caltables[ampcal_table]) caltables[caltable_name]['table']=caltables[ampcal_table]['table']+'_fluxscaled' - fluxes_txt = info_dir+ caltables[caltable_name]['name']+'_fluxes.txt' + fluxes_txt = calib_dir+ caltables[caltable_name]['name']+'_fluxes.txt' logger.info('Flux density scale from: {0}'.format(fluxcal)) logger.info('Transfered to: {0}'.format(cals_to_scale)) logger.info('Input caltable: {0}'.format(caltables[ampcal_table]['table'])) @@ -2716,13 +2702,13 @@ def eM_fluxscale(eMCP, caltables): listfile = fluxes_txt) logger.info('Modified caltable: {0}'.format(caltables[caltable_name]['table'])) logger.info('Spectrum information: {0}'.format(caltables[caltable_name]['table']+'_fluxes.txt')) - save_obj(calfluxes, info_dir + 'calfluxes.pkl') if calfluxes == None: logger.critical('Something went wrong with fluxscale') logger.warning('This probably means that necessary data are missing:') logger.warning('Required sources: {0}'.format(cals_to_scale)) logger.warning('Required antennas: {0}'.format(','.join(anten_for_flux))) exit_pipeline(eMCP) + save_obj(calfluxes, calib_dir + 'calfluxes.pkl') # Compute correction to scale the flux density of 3C286 according to # resolution provided by the shortest available baseline of e-MERLIN eMfactor = calc_eMfactor(msfile, field=fluxcal) @@ -3071,7 +3057,7 @@ def gaincal_final_scan(eMCP, caltables): caltables[caltable_name]['previous_cal'] = gain_final['ap_scan_prev_cal'] caltables[caltable_name]['gaintype'] = 'G' caltables[caltable_name]['calmode'] = 'ap' - caltables[caltable_name]['field'] = select_calibrator(gain_final['ap_calibrator'], eMCP, add_main=False) + caltables[caltable_name]['field'] = select_calibrator(gain_final['ap_scan_calibrator'], eMCP, add_main=False) caltables[caltable_name]['solint'] = gain_final['ap_scan_solint'] caltables[caltable_name]['combine'] = gain_final['ap_scan_combine'] caltables[caltable_name]['spw'] = make_spw(msinfo, gain_final['ap_scan_spw']) @@ -3122,6 +3108,8 @@ def applycal_all(eMCP, caltables): msfile_sp = eMCP['msinfo']['msfile_sp'] logger.info('Running statwt on narrow data {}'.format(msfile_sp)) statwt(vis=msfile_sp, timebin=timebin) + else: + logger.info('statwt not selected') msg = '' eMCP = add_step_time('applycal_all', eMCP, msg, t0) return eMCP diff --git a/functions/eMERLIN_CASA_plots.py b/functions/eMCP_plots.py similarity index 99% rename from functions/eMERLIN_CASA_plots.py rename to functions/eMCP_plots.py index 4ad2b90..2a112d1 100644 --- a/functions/eMERLIN_CASA_plots.py +++ b/functions/eMCP_plots.py @@ -11,9 +11,7 @@ import shutil import glob -plt.ioff() - -import functions.weblog as emwlog +import functions.eMCP_weblog as emwlog # CASA imports from taskinit import * @@ -21,9 +19,12 @@ from casac import casac msmd = casac.msmetadata() +plt.ioff() + import logging logger = logging.getLogger('logger') + weblog_dir = './weblog/' info_dir = './weblog/info/' calib_dir = './weblog/calib/' @@ -721,6 +722,6 @@ def fluxscale_models(calfluxes, eMfactor, msinfo): ax1.set_yscale('log') ax1.yaxis.set_major_formatter(ScalarFormatter()) - plots_obs_dir = './weblog/info/' + plots_obs_dir = calib_dir plot_file = plots_obs_dir+'{0}_fluxscale.png'.format(msinfo['msfilename']) fig.savefig(plot_file, bbox_inches='tight') diff --git a/functions/weblog.py b/functions/eMCP_weblog.py similarity index 93% rename from functions/weblog.py rename to functions/eMCP_weblog.py index 8c18046..a319faa 100644 --- a/functions/weblog.py +++ b/functions/eMCP_weblog.py @@ -100,8 +100,8 @@ def weblog_foot(wlog): wlog.write('\n') wlog.write('\n') -def write_link_txt(wlog, infile, intext): - wlog.write('{0}: txt
\n'.format(intext, infile)) +def write_link_txt(wlog, infile, intext, text='txt'): + wlog.write('{0}: {2}
\n'.format(intext, infile, text)) def weblog_index(msinfo): wlog = open("./weblog/index.html","w") @@ -353,30 +353,31 @@ def weblog_pipelineinfo(eMCP): wlog.write('Green = executed
') wlog.write('Red = executed but outdated by a previous step
') wlog.write('

Relevant log files:

\n') - write_link_txt(wlog, info_link + 'eMCP.log.txt', 'eMCP.log') - write_link_txt(wlog, info_link + 'casa_eMCP.log.txt', 'casa_eMCP.log') + write_link_txt(wlog, info_link + 'eMCP.log.txt', 'Pipeline log', text='eMCP.log') + write_link_txt(wlog, info_link + 'eMCP_errors.log.txt', 'Pipeline log errors only', text='eMCP_errors.log') + write_link_txt(wlog, info_link + 'casa_eMCP.log.txt', 'CASA log', text='casa_eMCP.log') # eMCP_info dictionary as text prt_dict_tofile(eMCP, tofilename=info_dir + 'eMCP_info.txt', pre=' ') eMCP_txtfile = info_link + 'eMCP_info.txt' wlog.write('

Relevant parameter files:

\n') - write_link_txt(wlog, eMCP_txtfile, 'Detailed eMCP_info dictionary') + write_link_txt(wlog, eMCP_txtfile, 'Pipeline info (dict)', text='eMCP_info.txt') # caltables dictionary as text try: caltables = load_obj(calib_dir+'caltables.pkl') prt_dict_tofile(caltables, tofilename=info_dir + 'caltables.txt', pre=' ') caltables_txtfile = info_link + 'caltables.txt' - write_link_txt(wlog, caltables_txtfile, 'Calibration tables txt') + write_link_txt(wlog, caltables_txtfile, 'Calibration info (dict)', text='caltables.txt') except: pass #------------------------------------------ weblog_foot(wlog) wlog.close() -def write_fluxscale(wlog): - p = glob.glob('./weblog/info/*fluxscale.png')[0] +def write_fluxscale(wlog, msinfo): + p = glob.glob(calib_dir + '{0}_fluxscale.png'.format(msinfo['msfilename']))[0] wlog.write('\n'.format(p)) - with open(info_dir + 'allcal_ap.G1_fluxes.txt') as f: + with open(calib_dir + 'allcal_ap.G1_fluxes.txt') as f: lines = f.readlines() wlog.write('\n

\n') wlog.write('CASA fluxscale output (not corrected by eMfactor):') @@ -429,9 +430,9 @@ def weblog_calibration(eMCP): for calstep in all_calsteps: logger.debug('calstep {}'.format(calstep)) try: - if calstep == 'fluxscale' and os.path.isfile(info_dir + 'allcal_ap.G1_fluxes.txt'): + if calstep == 'fluxscale' and os.path.isfile(calib_dir + 'allcal_ap.G1_fluxes.txt'): wlog.write('

{}

\n'.format('fluxscale')) - write_fluxscale(wlog) + write_fluxscale(wlog, msinfo) wlog.write('\n
\n') else: wlog.write('

{}

\n'.format(caltables[calstep]['name'])) @@ -577,26 +578,28 @@ def show_image(eMCP, wlog, i): wlog.write('{0} (Target image) ' \ 'Peak: {1:3.3f} mJy ' \ '(scaling: {2:4.1f})' \ - '(Target residual) rms: {3:5.3f} mJy\n'.format(target, - peak_target*1000., - scaling_target, - noise_target*1000.)) - wlog.write(''.format(img_target+'.image'+ext+'.png')) + '(Target residual) rms: {3:5.3f} mJy' \ + 'zoom imagezoom residual\n'.format(target, + peak_target*1000., + scaling_target, + noise_target*1000.)) + wlog.write('') + wlog.write(''.format(img_target+'.image'+ext+'.png')) wlog.write('\n'.format(img_target+'.residual'+ext+'.png')) - wlog.write('{0} image{0} residual\n'.format('zoom')) - wlog.write(''.format(img_target+'.image'+ext+'_zoom.png')) + wlog.write(''.format(img_target+'.image'+ext+'_zoom.png')) wlog.write('\n'.format(img_target+'.residual'+ext+'_zoom.png')) + wlog.write('') wlog.write('{0} (Phasecal image) ' \ 'Peak: {1:5.1f} mJy ' \ '(scaling: {2:4.1f})' \ - '(Phasecal residual) rms: {3:5.3f} mJy\n'.format(phscal, + '(Phasecal residual) rms: {3:5.3f} mJy' \ + 'zoom imagezoom residual\n'.format(phscal, peak_phscal*1000., scaling_phscal, noise_phscal*1000.)) - wlog.write(''.format(img_phscal+'.image'+ext+'.png')) + wlog.write(''.format(img_phscal+'.image'+ext+'.png')) wlog.write('\n'.format(img_phscal+'.residual'+ext+'.png')) - wlog.write('{0} image{0} residual\n'.format('zoom')) - wlog.write(''.format(img_phscal+'.image'+ext+'_zoom.png')) + wlog.write(''.format(img_phscal+'.image'+ext+'_zoom.png')) wlog.write('\n'.format(img_phscal+'.residual'+ext+'_zoom.png')) wlog.write('
\n') diff --git a/functions/eMERLIN_CASA_GUI.py b/functions/eMERLIN_CASA_GUI.py deleted file mode 100644 index 861dd79..0000000 --- a/functions/eMERLIN_CASA_GUI.py +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/local/python -import os - -# CASA imports -from taskinit import * -from tasks import * - -from Tkinter import * -import tkMessageBox - -class GUI_pipeline: - def __init__(self, pipeline_path): - self.root = Toplevel() - self.root.title('eMERLIN CASA Pipeline') - logo = PhotoImage(file=pipeline_path+'emerlin-2.gif') - self.w1 = Label(self.root,image=logo) - self.w1.grid(row=0,column=2,columnspan=3,rowspan=2,sticky='w,e,n,s') - self.w2 = Label(self.root,justify='left',padx=10,text="This is the eMERLIN pipeline") - self.w2.grid(row=0,column=0,columnspan=1,rowspan=2) - self.subtitle = Label(self.root,justify='center',padx=10,pady=10,text="------ Inputs ------") - self.subtitle.grid(row=3,column=0,columnspan=3,rowspan=1) - - ##Set directories for plots, calibration and data - ## Data dir ## - self.data_dir = StringVar() - self.data_dir.set('./data') - self.data_dir_label = Label(self.root,text='Data directory:',justify='right') - self.data_dir_label.grid(row=5,column=3,columnspan=1,sticky='e') - self.data_dir_entry = Entry(self.root,textvariable=self.data_dir) - self.data_dir_entry.grid(row=5,column=4,columnspan=3) - ## Plots dir ## - self.plots_dir = StringVar() - self.plots_dir.set('./plots') - self.plots_dir_label = Label(self.root,text='Plots directory:',justify='right') - self.plots_dir_label.grid(row=6,column=3,columnspan=1,sticky='e') - self.plots_dir_entry = Entry(self.root,textvariable=self.plots_dir) - self.plots_dir_entry.grid(row=6,column=4,columnspan=3) - ## Calibration directory ## - self.calib_dir = StringVar() - self.calib_dir.set('./calibration') - self.calib_dir_label = Label(self.root,text='Calibration directory:',justify='right') - self.calib_dir_label.grid(row=7,column=3,columnspan=1,sticky='e') - self.calib_dir_entry = Entry(self.root,textvariable=self.calib_dir) - self.calib_dir_entry.grid(row=7,column=4,columnspan=3) - - ##file names ### - self.inbase = StringVar() - ################################################ - ##set default to be the last instance with fits# - ################################################ - x=[] - for file in os.listdir('./data'): - if file.endswith('.ms') or file.endswith('.mms'): - x=x+[file] - if len(x) > 3: - self.inbase.set('Multiple ms, click check history for cwd') - else: - if len(x)!=0: - x=x[0] - if x.endswith('.ms'): - x=x[:-3] - if x.endswith('.mms'): - x=x[:-4] - if x.endswith('.fits'): - x=x[:-5] - self.inbase.set(x) - self.history = StringVar() - self.inbase_label = Label(self.root,text='UV file (without .fits/.ms/.mms):',justify='right') - self.inbase_label.grid(row=4,column=0,columnspan=1,sticky='e') - self.inbase_entry = Entry(self.root,textvariable=self.inbase) - self.inbase_entry.grid(row=4,column=1,columnspan=2) - self.inbase_button_his = Button(self.root,text='Check history',command=self.printhistory) - self.inbase_button_his.grid(row=4,column=3) - - - ################################################## - ##### Set defaults to be set from targets ######## - ################################################## - fields = [] - if len(x) != 0: - if os.path.isdir('./'+x+'.mms') == True: - fields = vishead(x+'.mms',mode='list',listitems='field')['field'][0] - elif os.path.isdir('./'+x+'.ms') == True: - fields = vishead(x+'.ms',mode='list',listitems='field')['field'][0] - ## Targets ### - self.targets = StringVar() - self.targets_label = Label(self.root,text='Targets:',justify='right') - self.targets_label.grid(row=5,column=0,columnspan=1,sticky='e') - self.targets_entry = Entry(self.root,textvariable=self.targets) - self.targets_entry.grid(row=5,column=1,columnspan=2) - - ## Phase calibrators ### - self.phscals = StringVar() - self.phscals_label = Label(self.root,text='Phase calibrators:',justify='right') - self.phscals_label.grid(row=6,column=0,columnspan=1,sticky='e') - self.phscals_entry = Entry(self.root,textvariable=self.phscals) - self.phscals_entry.grid(row=6,column=1,columnspan=2) - - ## Flux calibrators ### - x = '' - if len(fields) != 0: - if '1407+284' in fields: - x = x+'1407+284' - if '1331+305' in fields: - if len(x)!=0: - x=x+',' - x=x+'1331+305' - self.fluxcal = StringVar() - self.fluxcal.set(x) - self.fluxcal_label = Label(self.root,text='Flux calibrators:',justify='right') - self.fluxcal_label.grid(row=7,column=0,columnspan=1,sticky='e') - self.fluxcal_entry = Entry(self.root,textvariable=self.fluxcal) - self.fluxcal_entry.grid(row=7,column=1,columnspan=2) - - ## Bandpass calibrators ### - x='' - if len(fields) != 0: - if '1407+284' in fields: - x = x+'1407+284' - self.bpcal = StringVar() - self.bpcal.set(x) - self.bpcal_label = Label(self.root,text='Bandpass calibrators:',justify='right') - self.bpcal_label.grid(row=8,column=0,columnspan=1,sticky='e') - self.bpcal_entry = Entry(self.root,textvariable=self.bpcal) - self.bpcal_entry.grid(row=8,column=1,columnspan=2) - - ## Point calibrators ### - x='' - if len(fields) != 0: - if '1407+284' in fields: - x = x+'1407+284' - self.ptcal = StringVar() - self.ptcal.set(x) - self.ptcal_label = Label(self.root,text='Point calibrators:',justify='right') - self.ptcal_label.grid(row=9,column=0,columnspan=1,sticky='e') - self.ptcal_entry = Entry(self.root,textvariable=self.ptcal) - self.ptcal_entry.grid(row=9,column=1,columnspan=2) - - ## Refant ### - self.refant = StringVar() - self.refant.set(x) - self.refant_label = Label(self.root,text='Reference antennas:',justify='right') - self.refant_label.grid(row=10,column=0,columnspan=1,sticky='e') - self.refant_entry = Entry(self.root,textvariable=self.refant) - self.refant_entry.grid(row=10,column=1,columnspan=2) - - #-----------------------------------------------------------------# - ##############Processes############################################ - #-----------------------------------------------------------------# - self.subtitle2 = Label(self.root,justify='center',padx=10,pady=10,text="----- Processes ------") - self.subtitle2.grid(row=11,column=0,columnspan=3,rowspan=1) - - ## Convert fits to ms ## - self.run_importfits = IntVar() - self.run_importfits_check = Checkbutton(self.root,text='Convert fits-IDI to ms',variable =self.run_importfits,onvalue=1,offvalue=0,justify='left') - self.run_importfits_check.grid(row=12,column=0,sticky='w') - #---------------------## - - ## Hanning smoothing and flag autocorrelations ## - self.hanning = IntVar() - self.hanning_check = Checkbutton(self.root,text='Hanning smoothing',variable =self.hanning,onvalue=1,offvalue=0) - self.hanning_check.grid(row=13,column=0,sticky='w') - # ---------------------------------------------## - - # Auto flagging # - self.autoflag = IntVar() - self.autoflag_check = Checkbutton(self.root,text='Autoflagging?',variable =self.autoflag,onvalue=1,offvalue=0,pady=5) - self.autoflag_check.grid(row=14,column=0,sticky='w') - - ## Rfigui to set strategies ## - self.rfigui = IntVar() - self.rfigui_check = Checkbutton(self.root,text='Set strategies per source?',variable =self.rfigui,onvalue=1,offvalue=0) - self.rfigui_check.grid(row=14,column=1,sticky='w') - # -------------------------------------------## - - ## Convert to mms ## - self.ms2mms = IntVar() - self.ms2mms_check = Checkbutton(self.root,text='Convert to MMS',variable =self.ms2mms,onvalue=1,offvalue=0,pady=5) - self.ms2mms_check.grid(row=15,column=0,sticky='w') - ## ---------------## - - self.do_prediag = IntVar() - self.do_prediag_check = Checkbutton(self.root,text='Pre-diagnostics',variable =self.do_prediag,onvalue=1,offvalue=0,pady=5) - self.do_prediag_check.grid(row=16,column=0,sticky='w') - - self.solve_delays = IntVar() - self.solve_delays_check = Checkbutton(self.root,text='Solve delays',variable =self.solve_delays,onvalue=1,offvalue=0,pady=5) - self.solve_delays_check.grid(row=17,column=0,sticky='w') - - self.do_initial_bandpass = IntVar() - self.do_initial_bandpass_check = Checkbutton(self.root,text='Initial bandpass',variable =self.do_initial_bandpass,onvalue=1,offvalue=0,pady=5) - self.do_initial_bandpass_check.grid(row=18,column=0,sticky='w') - - ## Set parameters ## - self.w6 = Button(self.root,text='Confirm?',command=self.confirm_parameters) - self.w6.grid(row=100,column=2,sticky='e') - - ### Run button ### - self.run = Button(self.root,text='Run',command=self.root.quit) - self.run.grid(row=100,column=1,sticky='e') - - self.run = Button(self.root,text='Summary',command=self.check_inputs) - self.run.grid(row=100,column=0,sticky='e') - - self.quit_var = IntVar() - self.quit = Button(self.root,text='Quit',command=self.quit) - self.quit.grid(row=100,column=3,sticky='e') - ################### - ### Destroy GUI ### - self.root.update_idletasks() - self.root.mainloop() - self.root.destroy() - ################### - - def quit(self): - self.quit_var.set(1) - self.root.quit() - - def confirm_parameters(self): - self.inputs = {'quit':self.quit_var.get(),'data_dir':self.data_dir.get(),\ - 'plots_dir':self.plots_dir.get(),'calib_dir':self.calib_dir.get(),\ - 'inbase':self.inbase.get(),'targets':self.targets.get(),\ - 'phscals':self.phscals.get(),'fluxcal':self.fluxcal.get(),\ - 'bpcal':self.bpcal.get(),'ptcal':self.ptcal.get(),'refant':self.refant.get(),\ - 'run_importfits':self.run_importfits.get(),'hanning':self.hanning.get(),\ - 'autoflag':self.autoflag.get(),'rfigui':self.rfigui.get(),\ - 'ms2mms':self.ms2mms.get(),'do_prediag':self.do_prediag.get(),\ - 'do_delay':self.solve_delays.get(),'do_initial_bandpass':self.do_initial_bandpass.get()} - return self.inputs - - def printhistory(self): - def check_his(msname): - lines = [] - tb.open(msname+'/HISTORY') - x = tb.getcol('MESSAGE') - y = [i for i, item in enumerate(x) if 'eMER_CASA_Pipeline:' in item] - if len(y) == 0: - lines=lines+['Measurement set: '+msname+' has not been processed'] - else: - lines=lines+['Measurement set: '+msname+' has these steps conducted:'] - print 'Measurement set: '+msname+' has these steps conducted:' - for i in range(len(y)): - lines = lines+[x[y[i]][20:]] +["\n"] - print x[y[i]][20:] - return lines - if os.path.isdir('./'+self.inbase_entry.get()+'.ms') == True: - msname = self.inbase_entry.get()+'.ms' - lines = check_his(msname) - tkMessageBox.showinfo('History of'+msname,"\n".join(lines)) - elif os.path.isdir('./'+self.inbase_entry.get()+'.mms') == True: - msname = self.inbase_entry.get()+'.mms' - lines = check_his(msname) - tkMessageBox.showinfo('History of'+msname,"\n".join(lines)) - else: - lines = [] - lines = lines+['Data set: '+self.inbase_entry.get()+'.fits/.ms/.mms does not exist','Current working directory:'] - print 'Current working directory:' - for file in os.listdir('./'): - if file.endswith('.ms') or file.endswith('.mms') or file.endswith('.fits'): - lines=lines+[file] - print file - tkMessageBox.showinfo('Wrong ms',"\n".join(lines)) - - def default_inbase(self): - x = [] - for file in os.listdir('./'): - if file.endswith('.ms') or file.endswith('.mms') or file.endswith('.fits'): - x=x+[file] - if len(x) > 3: - return 'Multiple ms, click check history for cwd' - else: - x=x[0] - if x.endswith('.ms'): - return x[:-3] - if x.endswith('.mms'): - return x[:-4] - if x.endswith('.fits'): - return x[:-5] - - def check_inputs(self): - self.inputs = {'quit':self.quit_var.get(),'data_dir':self.data_dir.get(),\ - 'plots_dir':self.plots_dir.get(),'calib_dir':self.calib_dir.get(),\ - 'inbase':self.inbase.get(),'targets':self.targets.get(),\ - 'phscals':self.phscals.get(),'fluxcal':self.fluxcal.get(),\ - 'bpcal':self.bpcal.get(),'ptcal':self.ptcal.get(),'refant':self.refant.get(),\ - 'run_importfits':self.run_importfits.get(),'hanning':self.hanning.get(),\ - 'autoflag':self.autoflag.get(),'rfigui':self.rfigui.get(),\ - 'ms2mms':self.ms2mms.get(),'do_prediag':self.do_prediag.get(),\ - 'do_delay':self.solve_delays.get(),'do_initial_bandpass':self.do_initial_bandpass.get()} - print self.inputs - input_key=self.inputs.keys() - input_values = self.inputs.values() - lines = [] - for i in range(len(input_key)): - lines = lines + [str(input_key[i])+':'+str(input_values[i])] - tkMessageBox.showinfo('Summary',"\n".join(lines)) diff --git a/eMCP.css b/utils/eMCP.css similarity index 100% rename from eMCP.css rename to utils/eMCP.css diff --git a/emerlin-2.gif b/utils/emerlin-2.gif similarity index 100% rename from emerlin-2.gif rename to utils/emerlin-2.gif