From 2e6d52ff0005d119942133ef4f65750b127f9d2f Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sat, 17 Jun 2017 18:57:24 +0200 Subject: [PATCH 1/4] fixes #285 --- .../MSMF/ClassImageDeconvMachineMSMF.py | 30 ++++++++++++------- DDFacet/Parset/DefaultParset.cfg | 6 +++- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py b/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py index f26f343d..4fc030cc 100644 --- a/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py +++ b/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py @@ -88,7 +88,7 @@ def __init__(self, Gain=0.3, if self.ModelMachine.DicoSMStacked["Type"] not in ("MSMF", "HMP"): raise ValueError("ModelMachine Type should be HMP") self.PSFHasChanged=False - # reset overall iteration counter + self._previous_initial_peak = None self.maincache = MainCache # reset overall iteration counter self._niter = 0 @@ -551,7 +551,7 @@ def updateRMS(self): def setMask(self,Mask): self.CurrentNegMask=Mask - def Deconvolve(self, ch=0,UpdateRMS=True): + def Deconvolve(self, ch=0, UpdateRMS=True): """ Runs minor cycle over image channel 'ch'. initMinor is number of minor iteration (keeps continuous count through major iterations) @@ -560,7 +560,7 @@ def Deconvolve(self, ch=0,UpdateRMS=True): Returns tuple of: return_code,continue,updated where return_code is a status string; continue is True if another cycle should be executed; - update is True if model has been updated (note that1-sparse1.model01.fitst update=False implies continue=False) + update is True if model has been updated (note update=False implies continue=False) """ if self._niter >= self.MaxMinorIter: return "MaxIter", False, False @@ -610,6 +610,13 @@ def Deconvolve(self, ch=0,UpdateRMS=True): # in weighted or noisemap mode, look up the true max as well trueMaxDirty = MaxDirty if self._peakMode is "normal" else ThisFlux + if self._previous_initial_peak is None: + self._previous_initial_peak = ThisFlux + elif ThisFlux > self.GD["HMP"]["MajorStallThreshold"]*self._previous_initial_peak: + print>>log,ModColor.Str("STALL! dirty image peak %10.6g Jy, was %10.6g at previous major cycle." + % (ThisFlux, self._previous_initial_peak), col="red") + print>>log,ModColor.Str(" will stop cleaning now") + return "Stall", False, False #x,y,MaxDirty=NpParallel.A_whereMax(self._MeanDirty.copy(),NCPU=1,DoAbs=DoAbs,Mask=self._MaskArray.copy()) #A=self._MeanDirty.copy() #A.flat[:]=np.arange(A.size)[:] @@ -697,6 +704,8 @@ def Deconvolve(self, ch=0,UpdateRMS=True): # pBAR.render(0,"g=%3.3f"%self.GainMachine.GiveGain()) PreviousFlux=ThisFlux + divergence_factor = 1 + max(self.GD["HMP"]["AllowResidIncrease"],0) + def GivePercentDone(ThisMaxFlux): fracDone = 1.-(ThisMaxFlux-StopFlux)/(MaxDirty-StopFlux) return max(int(round(100*fracDone)), 100) @@ -725,14 +734,13 @@ def GivePercentDone(ThisMaxFlux): # stop T.timeit("max0") - if not self.GD["HMP"]["AllowResidIncrease"]: - if np.abs(ThisFlux)>1.1*np.abs(PreviousFlux): - print>>log, ModColor.Str( - " [iter=%i] peak of %.3g Jy higher than previous one of %.3g Jy " % - (i, ThisFlux, PreviousFlux), col="red") - return "Diverging", True, True - elif np.abs(ThisFlux) divergence_factor*np.abs(PreviousFlux): + print>>log, ModColor.Str( + " [iter=%i] peak of %.3g Jy diverging w.r.t. floor of %.3g Jy " % + (i, ThisFlux, PreviousFlux), col="red") + return "Diverging", False, True + if np.abs(ThisFlux) < np.abs(PreviousFlux): + PreviousFlux = ThisFlux ThisPNR=ThisFlux/rms diff --git a/DDFacet/Parset/DefaultParset.cfg b/DDFacet/Parset/DefaultParset.cfg index 10528264..3265e16b 100755 --- a/DDFacet/Parset/DefaultParset.cfg +++ b/DDFacet/Parset/DefaultParset.cfg @@ -234,7 +234,11 @@ Scales = [0] # List of scales to use. #metavar:LIST Ratios = [] # @cyriltasse please document NTheta = 6 # Number of PA steps to use. #metavar:N SolverMode = PI # Solver mode: pseudoinverse, or non-negative least squares. #options:PI|NNLS -AllowResidIncrease = True # Allow the maximum residual to increase within the minor cycle +AllowResidIncrease = 0.1 # Allow the maximum residual to increase by at most this much relative to + the lowest residual, before bailing out due to divergence.#metavar:FACTOR +MajorStallThreshold = 0.8 # Major cycle stall threshold. If the residual at the beginning of a major cycle + is above X*residual at the beginning of the previous major cycle, then we consider the deconvolution stalled + and bail out. #metavar:X #type:float Taper = 0 # Weighting taper size for HMP fit. If 0, determined automatically. #type:int Support = 0 # Basis function support size. If 0, determined automatically. #type:int PeakWeightImage = None # weigh the peak finding by given image From 05a88578d415ffd389dbd91e1c7331590232a6af Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sat, 17 Jun 2017 19:26:34 +0200 Subject: [PATCH 2/4] removing unnecessary set_num_threads call --- DDFacet/Imager/ClassDeconvMachine.py | 2 +- DDFacet/Imager/ClassFacetMachine.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DDFacet/Imager/ClassDeconvMachine.py b/DDFacet/Imager/ClassDeconvMachine.py index 575856b0..022bd51d 100644 --- a/DDFacet/Imager/ClassDeconvMachine.py +++ b/DDFacet/Imager/ClassDeconvMachine.py @@ -164,7 +164,7 @@ def Init(self): del(self.GD["Deconv"]["MaxMajorIter"]) # If we do the deconvolution construct a model according to what is in MinorCycleConfig MinorCycleConfig=dict(self.GD["Deconv"]) - MinorCycleConfig["NCPU"]=self.GD["Parallel"]["NCPU"] + MinorCycleConfig["NCPU"] = self.GD["Parallel"]["NCPU"] MinorCycleConfig["NBand"]=MinorCycleConfig["NFreqBands"]=self.VS.NFreqBands MinorCycleConfig["GD"] = self.GD MinorCycleConfig["ImagePolDescriptor"] = self.VS.StokesConverter.RequiredStokesProducts() diff --git a/DDFacet/Imager/ClassFacetMachine.py b/DDFacet/Imager/ClassFacetMachine.py index ecaab1e5..c0d386f1 100644 --- a/DDFacet/Imager/ClassFacetMachine.py +++ b/DDFacet/Imager/ClassFacetMachine.py @@ -1187,7 +1187,7 @@ def FacetsToIm_Channel(self, kind="Dirty",ChanSel=None): NFacets=len(self.DicoImager.keys()) pBAR.render(0, NFacets) - numexpr.set_num_threads(self.GD["Parallel"]["NCPU"]) + # numexpr.set_num_threads(self.GD["Parallel"]["NCPU"]) # done in DDF.py for iFacet in self.DicoImager.keys(): From e7cc061eef5ef0b2614cade31d3b0998b8732c4a Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sat, 17 Jun 2017 19:30:32 +0200 Subject: [PATCH 3/4] fixes #285 properly. I think. --- DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py b/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py index 4fc030cc..8175de27 100644 --- a/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py +++ b/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py @@ -589,16 +589,16 @@ def Deconvolve(self, ch=0, UpdateRMS=True): if self.CurrentNegMask is not None: - print>>log,"Using externally defined Mask (self.CurrentNegMask)" + print>>log," using externally defined Mask (self.CurrentNegMask)" CurrentNegMask=self.CurrentNegMask elif self.MaskMachine: - print>>log,"Using MaskMachine Mask" + print>>log," using MaskMachine Mask" CurrentNegMask=self.MaskMachine.CurrentNegMask elif self._MaskArray is not None: - print>>log,"Using externally defined Mask (self._MaskArray)" + print>>log," using externally defined Mask (self._MaskArray)" CurrentNegMask=self._MaskArray else: - print>>log,"Not using a mask" + print>>log," not using a mask" CurrentNegMask=None x,y,MaxDirty = NpParallel.A_whereMax(self._PeakSearchImage,NCPU=self.NCPU,DoAbs=DoAbs,Mask=CurrentNegMask) @@ -610,13 +610,12 @@ def Deconvolve(self, ch=0, UpdateRMS=True): # in weighted or noisemap mode, look up the true max as well trueMaxDirty = MaxDirty if self._peakMode is "normal" else ThisFlux - if self._previous_initial_peak is None: - self._previous_initial_peak = ThisFlux - elif ThisFlux > self.GD["HMP"]["MajorStallThreshold"]*self._previous_initial_peak: + if self._previous_initial_peak is not None and abs(ThisFlux) > self.GD["HMP"]["MajorStallThreshold"]*self._previous_initial_peak: print>>log,ModColor.Str("STALL! dirty image peak %10.6g Jy, was %10.6g at previous major cycle." % (ThisFlux, self._previous_initial_peak), col="red") print>>log,ModColor.Str(" will stop cleaning now") return "Stall", False, False + self._previous_initial_peak = abs(ThisFlux) #x,y,MaxDirty=NpParallel.A_whereMax(self._MeanDirty.copy(),NCPU=1,DoAbs=DoAbs,Mask=self._MaskArray.copy()) #A=self._MeanDirty.copy() #A.flat[:]=np.arange(A.size)[:] From d96b8789be5deb7f08e1ce171b6618e9c08f3ba3 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Thu, 22 Jun 2017 22:09:07 +0200 Subject: [PATCH 4/4] implemented stall checks --- DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py b/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py index 8175de27..40ba393d 100644 --- a/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py +++ b/DDFacet/Imager/MSMF/ClassImageDeconvMachineMSMF.py @@ -609,12 +609,15 @@ def Deconvolve(self, ch=0, UpdateRMS=True): ThisFlux = abs(ThisFlux) # in weighted or noisemap mode, look up the true max as well trueMaxDirty = MaxDirty if self._peakMode is "normal" else ThisFlux + # return condition indicating cleaning is to be continued + cont = True if self._previous_initial_peak is not None and abs(ThisFlux) > self.GD["HMP"]["MajorStallThreshold"]*self._previous_initial_peak: print>>log,ModColor.Str("STALL! dirty image peak %10.6g Jy, was %10.6g at previous major cycle." % (ThisFlux, self._previous_initial_peak), col="red") - print>>log,ModColor.Str(" will stop cleaning now") - return "Stall", False, False + print>>log,ModColor.Str("This will be the last major cycle") + cont = False + self._previous_initial_peak = abs(ThisFlux) #x,y,MaxDirty=NpParallel.A_whereMax(self._MeanDirty.copy(),NCPU=1,DoAbs=DoAbs,Mask=self._MaskArray.copy()) #A=self._MeanDirty.copy() @@ -756,7 +759,7 @@ def GivePercentDone(ThisMaxFlux): (i, ThisPNR, ThisFlux), col="green") - cont = ThisFlux > self.FluxThreshold + cont = cont and ThisFlux > self.FluxThreshold if not cont: print>>log, ModColor.Str( " [iter=%i] absolute flux threshold of %.3g Jy has been reached, PNR %.3g" %