-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathmake_output_checklist.py
143 lines (99 loc) · 6.37 KB
/
make_output_checklist.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
# ???? HAVE NOT DONE THIS YET: Changing method so that there are only two options in the result column:
# DEM_exists or DEM_DNE
# if DEM_DNE, there may be a reason in the reason column
# first check if the DEM exists. if it does, record and continue
# if it doesn't, first check to see if data was missing (qresult == 'missingData' or 'missingData-ADAPT')
# original: script that takes a batch as inputs, reads the query output summary and:
# 1. writes an output checklist with a 1:1 input row to output row, where:
# if query result was not 'processing' (ie sent to DISCOVER for processing) the outResult is the same as the inResult
# if 'TIME LIMIT' is in slurm.out file, outResult is 'timedOut' and status is 'rerun'...*TD will eventually also write an input csv to rerun the timedOut pairs from query (with extended time)
# if neither of these two are the case, outResult is DEM_DNE. *TD may narrow this down to other causes but for now DEM_DNE
# 2. if DEM_exists (no time outs, no missing Data errors, etc.): write to a ADAPT_completedPairs_batch$name.txt so list can be used to delete unneeded inputs and outputs from DISCOVER
# *TD = to-do
import os
import glob
import sys
import datetime
import subprocess as sp
batch = sys.argv[1] # get batch id from command line
start = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
aspdir = "/att/pubrepo/DEM/hrsi_dsm/"
incsv = "/att/gpfsfs/briskfs01/ppl/mwooten3/Paul_TTE/batch_summary_csvs/batch{}_output_summary.csv".format(batch)
outcsv = "/att/gpfsfs/briskfs01/ppl/mwooten3/Paul_TTE/checklists/batch{}_checklist__{}.csv".format(batch, start)
slurmdir = os.path.join(aspdir, 'outSlurm')
logdir = os.path.join(aspdir, 'logs') # might change later
compPairsFile = "/att/gpfsfs/briskfs01/ppl/mwooten3/Paul_TTE/ADAPT_completedPairs/ADAPT_completedPairs_batch{}.txt".format(batch)
if os.path.isfile(compPairsFile):
os.remove(compPairsFile) # first delete the old completed pairs file if it exists
with open(outcsv, 'w') as o:
o.write('batchID,pairname,catID_1,catID_1_found,catID_2,catID_2_found,result,reason (optional)\n')
with open(incsv, 'r') as c:
csvlist = [f.strip() for f in c.readlines()]
requery_pairnames = [] # *TD what are we doing with this?
cnt=0
for cline in csvlist:
cnt+=1
if cnt==1:
continue # skip hdr
clist = cline.split(',')
# get pairname from csvlist
pairname = clist[1].strip()
qresult = clist[9]
cat1 = clist[2]
cat1_found = clist[3]
cat2 = clist[4]
cat2_found = clist[5]
if qresult != 'processing': # if the pair was not sent to processing, put the reason (aka qresult) in the output csv
if qresult == 'alreadyProcessed': #*TD might change this in the batch query script so that it writes DEM_exists instead of alreadyProcessed, in which case erase this block and just write qresult to output
qresult_write = 'DEM_exists'
else:
qresult_write = qresult
# we still do not need to write this DEM_exists/alreadyProcessed pair to the completed pairs list. if it was alreadyProcessed, pair will not be sent to DISCOVER in the batch
outline = '{},{},{},{},{},{},{}, '.format(batch, pairname, cat1, cat1_found, cat2, cat2_found, qresult_write)
with open(outcsv, 'a') as oc:
oc.write('{}\n'.format(outline))
continue
# now check pair sent for processing to be sure output exist but only if query result = processing
checkOut1 = os.path.join(aspdir, "{}/out-strip-DEM.txt".format(pairname))
checkOut2 = os.path.join(aspdir, "{}/out-strip-DEM.tif".format(pairname))
if os.path.isfile(checkOut1) and os.path.isfile(checkOut2):
outline = '{},{},{},{},{},{},{}, '.format(batch, pairname, cat1, cat1_found, cat2, cat2_found, 'DEM_exists')
with open(outcsv, 'a') as oc:
oc.write('{}\n'.format(outline))
with open(compPairsFile, 'a') as cp:
cp.write('{}\n'.format(pairname))
continue
# at this point we know there were no query issues and we know the DEM does not exist for the pair...but why?
## job time out
## IOerror / disk quota error
## can not find (xml error) ?
# get the slurm if it is available
if len(glob.glob(os.path.join(slurmdir, 'batch{}__{}__slurm*out'.format(batch, pairname)))) > 0:
slurmFile = glob.glob(os.path.join(slurmdir, 'batch{}__{}__slurm*out'.format(batch, pairname)))[0]
# first check for time outs
with open(slurmFile, 'r') as sf:
slurm = sf.read() # dump contents of slurm file into var
if 'TIME LIMIT' in slurm:
# print '{} timed out'.format(pairname)
requery_pairnames.append(pairname)
outline = '{},{},{},{},{},{},{},{}'.format(batch, pairname, cat1, cat1_found, cat2, cat2_found, 'DEM_DNE', 'timedOut')
with open(outcsv, 'a') as oc:
oc.write('{}\n'.format(outline))
# *TD run function to put pairname inputs into a new query where the csv file is called like reQuery_batch{}-try2.csv -- or rerunTime, something. might do timed out pairs separately
# # prob create separate function that will access the batches original input list and extract the pair's line. function could take the output hrsi list name as input (so function can be reused when dealing with pairs that failed but didnt time out)
continue
else:
print " Slurm file(s) not yet available. Disregard output spreadsheet for this batch"
print os.path.join(slurmdir, 'batch{}__{}__slurm*out'.format(batch, pairname)) #*d
# try to get log file. what for? idk
if len (glob.glob(os.path.join(logdir, 'run_asp_LOG_{}__batch{}*txt'.format(pairname, batch)))) > 0:
logFile = glob.glob(os.path.join(logdir, 'run_asp_LOG_{}__batch{}*txt'.format(pairname, batch)))[0]
else:
print " Log file(s) not yet available. Disregard output spreadsheet for this batch"
# at this point we know no query errors, no time outs, but DEM DoesNotExist
# *TD for now just do DEM_DNE, we may narrow this down later to further errors
# after going through known errors, we still dont know why it wont work ??do we wanna rerun figure out later
outline = '{},{},{},{},{},{},{}, '.format(batch, pairname, cat1, cat1_found, cat2, cat2_found, 'DEM_DNE') # *TD what about status column?
# write output line to csv:
with open(outcsv, 'a') as oc:
oc.write('{}\n'.format(outline))