From 852a37537f881f07abb46e6624e9a8d66713e2a0 Mon Sep 17 00:00:00 2001 From: Peter Teuben Date: Tue, 19 Jun 2018 13:18:10 -0400 Subject: [PATCH 01/34] Felix patch to make pipeline work --- admit/Summary.py | 2 ++ admit/util/AbstractPlot.py | 8 +++++--- admit/util/Image.py | 7 +++++-- admit/util/casautil.py | 11 +++++++---- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/admit/Summary.py b/admit/Summary.py index ee3f727..8a2c32c 100644 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1020,6 +1020,8 @@ def html(self,outdir,flowmanager,dotdiagram="",editor=True): f.close() def _imageIsSVG(self,image): + if not image: return False + if image[len(image)-3:len(image)+1] == 'svg': return True else: diff --git a/admit/util/AbstractPlot.py b/admit/util/AbstractPlot.py index de25cd5..fb756f8 100644 --- a/admit/util/AbstractPlot.py +++ b/admit/util/AbstractPlot.py @@ -158,8 +158,9 @@ def getThumbnail(self,figno,relative): return self._thumbnailfiles[figno].replace(self._abspath,"") else: return self._thumbnailfiles[figno] - except KeyError: - raise Exception, "Thumbnail for figure %d was not created by this %s ." % (figno,self.__class__.__name__) + except KeyError: + if self._plot_mode != PlotControl.NOPLOT: + raise Exception, "Thumbnail for figure %d was not created by this %s ." % (figno,self.__class__.__name__) def getFigure(self,figno,relative): """Get the name of the figure file for given figure number @@ -184,7 +185,8 @@ def getFigure(self,figno,relative): else: return self._figurefiles[figno] except KeyError: - raise Exception, "Figure %d was not created by this %s." % (figno, self.__class__.__name__ ) + if self._plot_mode != PlotControl.NOPLOT: + raise Exception, "Figure %d was not created by this %s." % (figno, self.__class__.__name__ ) def figure(self,figno=1): """set the figure number. diff --git a/admit/util/Image.py b/admit/util/Image.py index 67f3f04..7b972ec 100644 --- a/admit/util/Image.py +++ b/admit/util/Image.py @@ -140,11 +140,14 @@ def setkey(self, name="", value=""): for k1, v1 in v.iteritems(): if k1 == bt.THUMB or k1 == bt.AUX: raise Exception("Thumnails and auxiliary files cannot be added as part of the images item, they must be added under the thumbnail or auxiliary item.") - if hasattr(self, k): + if hasattr(self, k): + if v==None: + print('WARNING: v was None, set to ""') + v='' if type(v) == type(getattr(self, k)): setattr(self, k, v) else: - raise Exception("Cannot change data type for %s, expected %s but got %s" % (k, str(type(getattr(self, k))), str(type(v)))) + raise Exception("Cannot change data type for %s, expected %s but got %s" % (k, str(type(getattr(self, k))), str(type(v)))) else: raise Exception("Invalid key given to Image class: %s" % (k)) elif not name == "": diff --git a/admit/util/casautil.py b/admit/util/casautil.py index 9af0379..fdde402 100644 --- a/admit/util/casautil.py +++ b/admit/util/casautil.py @@ -16,8 +16,6 @@ except: print "WARNING: No CASA; casautil can't function" -# imview was left out of the casa namespace in CASA5 -from imview import imview as casa_imview import PlotControl @@ -73,10 +71,15 @@ def implot(rasterfile, figname, contourfile=None, plottype=PlotControl.PNG, #can't support this until imview out= is fixed! (see below) #orientation=PlotControl.LANDSCAPE - - + if plotmode == PlotControl.NOPLOT: return + # imview creates a casa viewer which in turn can result in errors when run in parallel mode. Therefore + # although not standard practice, the import is only done in the function when needed. + + # imview was left out of the casa namespace in CASA5 + from imview import imview as casa_imview + if contourfile==None and rasterfile==None: raise Exception, "You must provide rasterfile and/or contourfile" From ea1804c00778371b8e9d490a5657b4b90b44bf73 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Tue, 18 Sep 2018 10:21:00 -0400 Subject: [PATCH 02/34] for getFigure/Thumbnail, return None if figure doesn't exist AND plotmode=NOPLOT --- admit/util/AbstractPlot.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/admit/util/AbstractPlot.py b/admit/util/AbstractPlot.py index fb756f8..c26261f 100644 --- a/admit/util/AbstractPlot.py +++ b/admit/util/AbstractPlot.py @@ -152,15 +152,23 @@ def getThumbnail(self,figno,relative): ------- str Thumbnail file name + None + If figno doesn't exist and plotmode is PlotControl.NOPLOT + Raises exception + If figno doesn't exist and plotmode is not PlotControl.NOPLOT """ try: if relative: return self._thumbnailfiles[figno].replace(self._abspath,"") else: return self._thumbnailfiles[figno] - except KeyError: - if self._plot_mode != PlotControl.NOPLOT: - raise Exception, "Thumbnail for figure %d was not created by this %s ." % (figno,self.__class__.__name__) + except KeyError: + # note: we don't put this return at the top of the method + # because a figure may have been created and then plotmode + # changed, in which case a figure for figno may exist. + if self._plot_mode == PlotControl.NOPLOT: + return None + raise Exception, "Thumbnail for figure %d was not created by this %s ." % (figno,self.__class__.__name__) def getFigure(self,figno,relative): """Get the name of the figure file for given figure number @@ -178,6 +186,10 @@ def getFigure(self,figno,relative): ------- str Figure file name + None + If figno doesn't exist and plotmode is PlotControl.NOPLOT + Raises exception + If figno doesn't exist and plotmode is not PlotControl.NOPLOT """ try: if relative: @@ -185,8 +197,12 @@ def getFigure(self,figno,relative): else: return self._figurefiles[figno] except KeyError: - if self._plot_mode != PlotControl.NOPLOT: - raise Exception, "Figure %d was not created by this %s." % (figno, self.__class__.__name__ ) + # note: we don't put this return at the top of the method + # because a figure may have been created and then plotmode + # changed, in which case a figure for figno may exist. + if self._plot_mode == PlotControl.NOPLOT: + return None + raise Exception, "Figure %d was not created by this %s." % (figno, self.__class__.__name__ ) def figure(self,figno=1): """set the figure number. @@ -221,6 +237,8 @@ def makeThumbnail(self,figno=None, scale=0.33, fig=None): None """ + if self._plot_mode == PlotControl.NOPLOT: return + if figno: fno = figno else: From 2f4a969a4a7d44e0d5db20884f3544031846ddc8 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 10:26:43 -0400 Subject: [PATCH 03/34] changing over to use new noplot member variable --- admit/Summary.py | 59 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 15 deletions(-) mode change 100644 => 100755 admit/Summary.py diff --git a/admit/Summary.py b/admit/Summary.py old mode 100644 new mode 100755 index 8a2c32c..8853079 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1156,7 +1156,11 @@ def _process(self,taskname,tid,titems,thetask,outdir): if tlower == "cubespectrum_at": spectra = titems.get('spectra',None) - if (spectra) != None: + if spectra == None: + allspecs = "

%s produced no output for image %s

" % (taskname, casaimage) + elif spectra.getNoPlot(): + allspecs = "

%s created output for image %s but was told to create no PNGs for display.

" % (taskname, casaimage) + else: count = 0 # task arguments are the same in all entries. taskargs = spectra.taskargs @@ -1183,8 +1187,6 @@ def _process(self,taskname,tid,titems,thetask,outdir): banner = '

%s output for image %s

' % (taskname, casaimage) allspecs = banner + allspecs - else: - allspecs = "

%s produced no output for image %s

" % (taskname, casaimage) retval = header % (taskclass, tid,thetask.statusicons(),taskname,tid,taskargs,tid,allspecs,tid) @@ -1666,7 +1668,11 @@ def _process(self,taskname,tid,titems,thetask,outdir): if tlower == "generatespectrum_at": spectra = titems.get('spectra',None) - if (spectra) != None: + if spectra == None: + allspecs = "

%s produced no output

" % (taskname) + elif spectra.getNoPlot(): + allspecs = "

%s created output but was told to create no PNGs for display.

" % (taskname) + else: count = 0 # task arguments are the same in all entries. taskargs = spectra.taskargs @@ -1690,8 +1696,6 @@ def _process(self,taskname,tid,titems,thetask,outdir): banner = '

%s output

' % (taskname) allspecs = banner + allspecs - else: - allspecs = "

%s produced no output

" % (taskname) retval = header % (taskclass, tid,thetask.statusicons(),taskname,tid,taskargs,tid,allspecs,tid) @@ -1863,10 +1867,9 @@ def _test(self): class SummaryEntry: """ Defines a single 'row' of a Summary data entry. A Summary key can refer to a list of SummaryEntry. - This class makes management of complicated data entries easier. It was getting tough - to slice and unzip all those lists! + This class makes management of complicated data entries easier. """ - def __init__(self,value=[],taskname="",taskid=-1,taskargs=""): + def __init__(self,value=[],taskname="",taskid=-1,taskargs="",noplot=False): if isinstance(value,list): self._value = value else: @@ -1875,6 +1878,8 @@ def __init__(self,value=[],taskname="",taskid=-1,taskargs=""): self._taskid = taskid self._taskargs = taskargs self._type = bt.SUMMARYENTRY + # True if the task that created this entry did NOT create plots + self._noplot = noplot def getValue(self): """Get the underlying data value from this SummaryEntry. Value will be list @@ -1932,8 +1937,6 @@ def getTaskArgs(self): The string task arguments""" return self._taskargs; - - def setTaskname(self,name): """Set the name of the task that created this SummaryEntry. @@ -1976,6 +1979,31 @@ def setTaskArgs(self,args): """ self._taskargs = args; + def getNoPlot(self): + """Discover if the task that created this SummaryEntry made plots. + + Returns + ------- + True if no plots were created + False otherwise + """ + return self._noplot + + def setNoPlot(self,noplot): + """Indicate if the task that created this SummaryEntry made plots. + + Parameters + ---------- + noplot: boolean + True if no plots were created + + Returns + ------- + None + """ + self._noplot = noplot + + #------------------------------- # Make the internal data be well-behaved properties. # See e.g. http://www.programiz.com/python-programming/property @@ -1983,6 +2011,7 @@ def setTaskArgs(self,args): taskname = property(getTaskname, setTaskname, None, 'The name of the task that created this SummaryEntry') taskid = property(getTaskID, setTaskID, None, 'The integer task ID that created this SummaryEntry') taskargs = property(getTaskArgs, setTaskArgs, None, 'The arguments of the task to display in the data browser web page.') + noplot = property(getNoPlot,setNoPlot,None,'True if the task that created this SummaryEntry did NOT create plots') #------------------------------- def unset(self): @@ -2012,7 +2041,7 @@ def write(self,root): """ snode = et.SubElement(root,"summaryEntry") snode.set("type",bt.SUMMARYENTRY) - writer = XmlWriter.XmlWriter(self,["_value","_taskname","_taskid","_taskargs"],{"_value":bt.LIST,"_taskname":bt.STRING,"_taskid":bt.INT,"_taskargs":bt.STRING},snode,None) + writer = XmlWriter.XmlWriter(self,["_value","_taskname","_taskid","_taskargs","_noplot"],{"_value":bt.LIST,"_taskname":bt.STRING,"_taskid":bt.INT,"_taskargs":bt.STRING,"_noplot":bt.BOOL},snode,None) # Two SummaryEntrys are equal if their taskids are equal def __eq__(self,other): @@ -2034,8 +2063,8 @@ def __hash__(self): return self._taskid def __str__(self): - return "SummaryEntry(value=%s, taskname=%s, taskid=%d, taskargs=%s)" % \ - (str(self._value), self._taskname, self._taskid,self._taskargs) + return "SummaryEntry(value=%s, taskname=%s, taskid=%d, taskargs=%s, noplot=%s)" % \ + (str(self._value), self._taskname, self._taskid,self._taskargs,str(self._noplot)) # ensure that printing a list of SummaryEntry will print the # individual values and not just addresses. @@ -2049,7 +2078,7 @@ def __str__(self): qq = SummaryEntry("Hqwl","AT1",1,"blah") rr = SummaryEntry("Hrwl","AT1",1,"blah") ss = SummaryEntry("FOOS","AT2",1,"blah") - tt = SummaryEntry("FOOT","AT3",2,"blah") + tt = SummaryEntry("FOOT","AT3",2,"blah",noplot=True) print "qq == rr (True?):" + str(qq == rr) print "qq == ss (True?):" + str(qq == ss) print "ss == tt (False?):" + str(qq == tt) From e881a046ff7917a6dfa21a5fc987439212ae2b93 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 10:27:02 -0400 Subject: [PATCH 04/34] changing over to use new noplot member variable --- admit/at/GenerateSpectrum_AT.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/admit/at/GenerateSpectrum_AT.py b/admit/at/GenerateSpectrum_AT.py index 5312250..f4f7458 100644 --- a/admit/at/GenerateSpectrum_AT.py +++ b/admit/at/GenerateSpectrum_AT.py @@ -208,6 +208,7 @@ def run(self): nspectra = self.getkey("nspectra") taskargs = " contin=%f freq=%f delta=%f nspectra=%f " % (contin,f0,df,nspectra) spec = range(nspectra) + noplot = True dt.tag("start") if self.getkey("file") != "": print "READING spectrum from",self.getkey("file") @@ -274,7 +275,7 @@ def run(self): images = {} # png's accumulated for i in range(nspectra): sd = [] - caption = "Generated Spectrum %d" % i + imcaption = "Generated Spectrum %d" % i # construct the Table for CubeSpectrum_BDP # @todo note data needs to be a tuple, later to be column_stack'd labels = ["channel" ,"frequency" ,"flux" ] @@ -291,20 +292,28 @@ def run(self): xlab = 'Channel' y = [spec[i]] sd.append(xlab) + if self._plot_mode == PlotControl.NOPLOT: + figname = "not created" + thumbname = "not created" + imcaption = "not created" + noplot = True + else: + + myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode, abspath=self.dir()) + ylab = 'Flux' + p1 = "%s_%d" % (bdp_name,i) + myplot.plotter(x,y,"",p1,xlab=xlab,ylab=ylab,thumbnail=True) + # Why not use p1 as the key? + figname = images["pos%d" % i] = myplot.getFigure(figno=myplot.figno,relative=True) + thumbname = myplot.getThumbnail(figno=myplot.figno,relative=True) - myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode, abspath=self.dir()) - ylab = 'Flux' - p1 = "%s_%d" % (bdp_name,i) - myplot.plotter(x,y,"",p1,xlab=xlab,ylab=ylab,thumbnail=True) - # Why not use p1 as the key? - ii = images["pos%d" % i] = myplot.getFigure(figno=myplot.figno,relative=True) - thumbname = myplot.getThumbnail(figno=myplot.figno,relative=True) + image = Image(images=images, description="CubeSpectrum") + noplot = False - image = Image(images=images, description="CubeSpectrum") - sd.extend([ii, thumbname, caption]) + sd.extend([figname, thumbname, imcaption]) self.spec_description.append(sd) - self._summary["spectra"] = SummaryEntry(self.spec_description,"GenerateSpectrum_AT",self.id(True), taskargs) + self._summary["spectra"] = SummaryEntry(self.spec_description,"GenerateSpectrum_AT",self.id(True), taskargs, noplot=noplot) dt.tag("table") From 32476bbdb295c34297899a573f5874b8ae3011bd Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 10:27:12 -0400 Subject: [PATCH 05/34] changing over to use new noplot member variable --- admit/at/ContinuumSub_AT.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/admit/at/ContinuumSub_AT.py b/admit/at/ContinuumSub_AT.py index fb19808..ce62766 100644 --- a/admit/at/ContinuumSub_AT.py +++ b/admit/at/ContinuumSub_AT.py @@ -207,19 +207,28 @@ def run(self): rdata = casautil.getdata(self.dir(f3)).data logging.regression("CSUB: %f %f" % (rdata.min(),rdata.max())) - # Create two output images for html and their thumbnails, too - implot = ImPlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) - implot.plotter(rasterfile=f3,figname=f3,colorwedge=True) - figname = implot.getFigure(figno=implot.figno,relative=True) - thumbname = implot.getThumbnail(figno=implot.figno,relative=True) b2.setkey("image", Image(images={bt.CASA:f2})) - b3.setkey("image", Image(images={bt.CASA:f3, bt.PNG : figname})) + + # Create two output images for html and their thumbnails, too + if self._plot_mode == PlotControl.NOPLOT: + figname = "not created" + thumbname = "not created" + imcaption = "not created" + b3.setkey("image", Image(images={bt.CASA:f3})) + noplot = True + else: + implot = ImPlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) + implot.plotter(rasterfile=f3,figname=f3,colorwedge=True) + figname = implot.getFigure(figno=implot.figno,relative=True) + thumbname = implot.getThumbnail(figno=implot.figno,relative=True) + b3.setkey("image", Image(images={bt.CASA:f3, bt.PNG : figname})) + noplot = False dt.tag("implot") if len(ch) > 0: taskargs = "pad=%d fitorder=%d contsub=%s" % (pad,fitorder,str(contsub)) imcaption = "Continuum map" - self._summary["continuumsub"] = SummaryEntry([figname,thumbname,imcaption],"ContinuumSub_AT",self.id(True),taskargs) + self._summary["continuumsub"] = SummaryEntry([figname,thumbname,imcaption],"ContinuumSub_AT",self.id(True),taskargs,noplot=noplot) dt.tag("done") dt.end() From 38afae67640f0a20ba806eccd949e9e2354afcea Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 10:27:21 -0400 Subject: [PATCH 06/34] changing over to use new noplot member variable --- admit/at/CubeSpectrum_AT.py | 39 ++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/admit/at/CubeSpectrum_AT.py b/admit/at/CubeSpectrum_AT.py index 05ce041..603989f 100644 --- a/admit/at/CubeSpectrum_AT.py +++ b/admit/at/CubeSpectrum_AT.py @@ -267,9 +267,10 @@ def run(self): planes = range(npos) # labels for the tables (placeholder) images = {} # png's accumulated + noplot = True for i in range(npos): # loop over pos, they can have mixed types now sd = [] - caption = "Spectrum" + imcaption = "Spectrum" xpos = pos[i][0] ypos = pos[i][1] if type(xpos) != type(ypos): @@ -282,7 +283,7 @@ def run(self): cbox = '(%d,%d,%d,%d)' % (xpos,ypos,xpos,ypos) # use extend here, not append, we want individual values in a list sd.extend([xpos,ypos,cbox]) - caption = "Average Spectrum at %s" % cbox + imcaption = "Average Spectrum at %s" % cbox if False: # this will fail on 3D cubes (see CAS-7648) imval[i] = casa.imval(self.dir(fin),box=box) @@ -291,12 +292,12 @@ def run(self): # another approach is the ia.getprofile(), see CubeStats, this will # also integrate over regions, imval will not (!!!) region = 'centerbox[[%dpix,%dpix],[1pix,1pix]]' % (xpos,ypos) - caption = "Average Spectrum at %s" % region + imcaption = "Average Spectrum at %s" % region imval[i] = casa.imval(self.dir(fin),region=region) elif type(xpos)==str: # this is tricky, to stay under 1 pixel , or you get a 2x2 back. region = 'centerbox[[%s,%s],[1pix,1pix]]' % (xpos,ypos) - caption = "Average Spectrum at %s" % region + imcaption = "Average Spectrum at %s" % region sd.extend([xpos,ypos,region]) imval[i] = casa.imval(self.dir(fin),region=region) else: @@ -354,14 +355,22 @@ def run(self): else: title = '%s %d @ %s,%s' % (bdp_name,i,xpos,ypos) # or use box, once we allow non-points - myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode, abspath=self.dir()) - ylab = 'Flux (%s)' % unit - p1 = "%s_%d" % (bdp_name,i) - myplot.plotter(x,y,title,p1,xlab=xlab,ylab=ylab,thumbnail=True) - # Why not use p1 as the key? - ii = images["pos%d" % i] = myplot.getFigure(figno=myplot.figno,relative=True) - thumbname = myplot.getThumbnail(figno=myplot.figno,relative=True) - sd.extend([ii, thumbname, caption, fin]) + if self._plot_mode == PlotControl.NOPLOT: + figname = "not created" + thumbname = "not created" + imcaption = "not created" + noplot = True + else: + myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode, abspath=self.dir()) + ylab = 'Flux (%s)' % unit + p1 = "%s_%d" % (bdp_name,i) + myplot.plotter(x,y,title,p1,xlab=xlab,ylab=ylab,thumbnail=True) + # Why not use p1 as the key? + figname = images["pos%d" % i] = myplot.getFigure(figno=myplot.figno,relative=True) + thumbname = myplot.getThumbnail(figno=myplot.figno,relative=True) + noplot = False + + sd.extend([figname, thumbname, imcaption, fin]) self.spec_description.append(sd) logging.regression("CSP: %s" % str(smax)) @@ -385,7 +394,11 @@ def run(self): # SummaryEntry([[data for spec1]], "CubeSpectrum_AT",taskid) # For multiple spectra this is # SummaryEntry([[data for spec1],[data for spec2],...], "CubeSpectrum_AT",taskid) - self._summary["spectra"] = SummaryEntry(self.spec_description,"CubeSpectrum_AT",self.id(True)) + + # @todo if range(npos) is [] don't create a summary entry + # so that check against None in Summary.py does the right thing, + # although len(npos) == 0 is trapped earlier so perhaps not necessary + self._summary["spectra"] = SummaryEntry(self.spec_description, "CubeSpectrum_AT", self.id(True), noplot=noplot) taskargs = "pos="+str(pos) taskargs += '            ' + fin.split('/')[0] + ' ' for v in self._summary: From d0943b2b323cd1eac96b11a9074629cb9a25932b Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 11:06:43 -0400 Subject: [PATCH 07/34] changing over to use new noplot member variable --- admit/at/CubeSum_AT.py | 96 ++++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/admit/at/CubeSum_AT.py b/admit/at/CubeSum_AT.py index 5054d9d..b72379a 100644 --- a/admit/at/CubeSum_AT.py +++ b/admit/at/CubeSum_AT.py @@ -347,45 +347,59 @@ def run(self): logging.info("Sum: %f (beam parameters missing)" % (st['sum'])) logging.regression("CSM: %s" % str(rdata)) - # Create two output images for html and their thumbnails, too - implot = ImPlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) - implot.plotter(rasterfile=bdp_name,figname=bdp_name, - colorwedge=True,zoom=self.getkey("zoom")) - figname = implot.getFigure(figno=implot.figno,relative=True) - thumbname = implot.getThumbnail(figno=implot.figno,relative=True) - - dt.tag("implot") - - thumbtype = bt.PNG # really should be correlated with self._plot_type!! - - # 2. Create a histogram of the map data - # get the data for a histogram - data = casautil.getdata(image_out,zeromask=True).compressed() - dt.tag("getdata") - - # get the label for the x axis - bunit = casa.imhead(imagename=image_out, mode="get", hdkey="bunit") - - # Make the histogram plot - # Since we give abspath in the constructor, figname should be relative - myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) - auxname = bdp_name + "_histo" - auxtype = bt.PNG # really should be correlated with self._plot_type!! - myplot.histogram(columns = data, - figname = auxname, - xlab = bunit, - ylab = "Count", - title = "Histogram of CubeSum: %s" % (bdp_name), - thumbnail=True) - auxname = myplot.getFigure(figno=myplot.figno,relative=True) - auxthumb = myplot.getThumbnail(figno=myplot.figno,relative=True) - - images = {bt.CASA : bdp_name, bt.PNG : figname} - casaimage = Image(images = images, - auxiliary = auxname, - auxtype = auxtype, - thumbnail = thumbname, - thumbnailtype = thumbtype) + if self._plot_mode == PlotControl.NOPLOT: + figname = "not created" + thumbname = "not created" + imcaption = "not created" + auxcaption = "not created" + auxthumb = "not created" + images = {bt.CASA : bdp_name} + casaimage = Image(images = images) + noplot = True + else: + + # Create two output images for html and their thumbnails, too + implot = ImPlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) + implot.plotter(rasterfile=bdp_name,figname=bdp_name, + colorwedge=True,zoom=self.getkey("zoom")) + figname = implot.getFigure(figno=implot.figno,relative=True) + thumbname = implot.getThumbnail(figno=implot.figno,relative=True) + + dt.tag("implot") + + thumbtype = bt.PNG # really should be correlated with self._plot_type!! + + # 2. Create a histogram of the map data + # get the data for a histogram + data = casautil.getdata(image_out,zeromask=True).compressed() + dt.tag("getdata") + + # get the label for the x axis + bunit = casa.imhead(imagename=image_out, mode="get", hdkey="bunit") + + # Make the histogram plot + # Since we give abspath in the constructor, figname should be relative + myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) + auxname = bdp_name + "_histo" + auxtype = bt.PNG # really should be correlated with self._plot_type!! + myplot.histogram(columns = data, + figname = auxname, + xlab = bunit, + ylab = "Count", + title = "Histogram of CubeSum: %s" % (bdp_name), + thumbnail=True) + auxname = myplot.getFigure(figno=myplot.figno,relative=True) + auxthumb = myplot.getThumbnail(figno=myplot.figno,relative=True) + + images = {bt.CASA : bdp_name, bt.PNG : figname} + casaimage = Image(images = images, + auxiliary = auxname, + auxtype = auxtype, + thumbnail = thumbname, + thumbnailtype = thumbtype) + imcaption = "Integral (moment 0) of all emission in image cube" + auxcaption = "Histogram of cube sum for image cube" + noplot = False if hasattr(b1,"line"): # SpwCube doesn't have Line line = deepcopy(getattr(b1,"line")) @@ -395,10 +409,8 @@ def run(self): line = Line(name="Undetermined") # fake a Line if there wasn't one self.addoutput(Moment_BDP(xmlFile=bdp_name,moment=0,image=deepcopy(casaimage),line=line)) - imcaption = "Integral (moment 0) of all emission in image cube" - auxcaption = "Histogram of cube sum for image cube" taskargs = "numsigma=%.1f sigma=%g smooth=%s" % (numsigma, sigma, str(smooth)) - self._summary["cubesum"] = SummaryEntry([figname,thumbname,imcaption,auxname,auxthumb,auxcaption,bdp_name,infile],"CubeSum_AT",self.id(True),taskargs) + self._summary["cubesum"] = SummaryEntry([figname,thumbname,imcaption,auxname,auxthumb,auxcaption,bdp_name,infile],"CubeSum_AT",self.id(True),taskargs,noplot=noplot) ia.done() From f7f125dbef0b4bcdd0033b82bbdf32304b5119ac Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 11:09:23 -0400 Subject: [PATCH 08/34] changing over to use new noplot member variable --- admit/at/Flow11_AT.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/admit/at/Flow11_AT.py b/admit/at/Flow11_AT.py index a0ffc20..1f62346 100755 --- a/admit/at/Flow11_AT.py +++ b/admit/at/Flow11_AT.py @@ -191,9 +191,9 @@ def summarize(self): # finally, set some parameters for the ADMIT summary abc_list= ['robust', 'chauvenet', 21.3, 123] - self._summary['datamin'] = SummaryEntry(3.14159,"Flow11_AT",taskid=self.id(True)) - self._summary['datamax'] = SummaryEntry(2.71828,"Flow11_AT",taskid=self.id(True)) - self._summary['rmsmethd'] = SummaryEntry(abc_list,"Flow11_AT",taskid=self.id(True)) + self._summary['datamin'] = SummaryEntry(3.14159,"Flow11_AT",taskid=self.id(True),noplot=True) + self._summary['datamax'] = SummaryEntry(2.71828,"Flow11_AT",taskid=self.id(True),noplot=True) + self._summary['rmsmethd'] = SummaryEntry(abc_list,"Flow11_AT",taskid=self.id(True),noplot=True) print "Flow11_AT taskid = %d" % self.id(True) From 0490baefb23fad9d545ddb10d4c857b1f33e27b1 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 11:09:41 -0400 Subject: [PATCH 09/34] changing over to use new noplot member variable --- admit/at/BDPIngest_AT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admit/at/BDPIngest_AT.py b/admit/at/BDPIngest_AT.py index cb1993f..f7eb3e2 100644 --- a/admit/at/BDPIngest_AT.py +++ b/admit/at/BDPIngest_AT.py @@ -110,4 +110,4 @@ def run(self): table.addRow(["Associated File",f]) table.description = "Information about the ingested BDP" taskargs = "file=%s" % self.getkey('file') - self._summary["bdpingest"] = SummaryEntry(table.serialize(),"BDPIngest_AT",self.id(True),taskargs) + self._summary["bdpingest"] = SummaryEntry(table.serialize(),"BDPIngest_AT",self.id(True),taskargs,noplot=True) From 8f3673e61a037776949a1137a8f9e4475d8e5c79 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 11:12:49 -0400 Subject: [PATCH 10/34] changing over to use new noplot member variable --- admit/at/Ingest_AT.py | 1 + 1 file changed, 1 insertion(+) diff --git a/admit/at/Ingest_AT.py b/admit/at/Ingest_AT.py index df6dc5c..579f255 100644 --- a/admit/at/Ingest_AT.py +++ b/admit/at/Ingest_AT.py @@ -867,3 +867,4 @@ def _summarize(self, fitsname, casaname, header, shape, taskargs): self._summary[k].setTaskname("Ingest_AT") self._summary[k].setTaskID(self.id(True)) self._summary[k].setTaskArgs(taskargs) + self._summary[k].setNoPlot(True) From a6064369fca065a0453158e56b6101169a7e2e0f Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 11:27:04 -0400 Subject: [PATCH 11/34] changing over to use new noplot member variable --- admit/at/Template_AT.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/admit/at/Template_AT.py b/admit/at/Template_AT.py index eb6ea8c..2167f3c 100644 --- a/admit/at/Template_AT.py +++ b/admit/at/Template_AT.py @@ -251,9 +251,9 @@ def run(self): # # Output data product (spectrum table). obdp3 = admit.Table_BDP(ospecstem) - obdp3.table.description = "Template 1-D Spectrum" - obdp3.table.columns = ["Channel", "Spectrum @ (%d, %d)" % specpos] - obdp3.table.setData(np.transpose(np.vstack([ochan, ospec]))) + obdp3.table.description = "Template 1-D Spectrum" + obdp3.table.columns = ["Channel", "Spectrum @ (%d, %d)" % specpos] + obdp3.table.setData(np.transpose(np.vstack([ochan, ospec]))) self.addoutput(obdp3) @@ -286,6 +286,7 @@ def run(self): oimgstem+".png", oimgstem+"_thumb.png", oimgtitle, ocubeim], [specpos[0], specpos[1], "", "Channel", ospecstem+".png", ospecstem+"_thumb.png", ospectitle, ocubeim]] + # If no plots are produced by your task, change noplot to True self._summary["spectra"] = admit.SummaryEntry(spec_description, "Template_AT", - self.id(True), keys) + self.id(True), keys, noplot=False) From 64878e4ec767c64f6caa615621908475bafd4ed2 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 11:39:59 -0400 Subject: [PATCH 12/34] changing over to use new noplot member variable --- admit/at/SFind2D_AT.py | 85 +++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/admit/at/SFind2D_AT.py b/admit/at/SFind2D_AT.py index 7a8fbbe..68b326f 100644 --- a/admit/at/SFind2D_AT.py +++ b/admit/at/SFind2D_AT.py @@ -394,48 +394,55 @@ def run(self): logging.info(" Restoring Beam: Major axis: %10.3g %s , Minor axis: %10.3g %s , PA: %5.1f %s" % (beammaj, beamunit, beammin, beamunit, beamang, angunit)) # form into a xml table + slbdp.table.description="Table of source locations and sizes (not deconvolved)" # output is a table_bdp self.addoutput(slbdp) - # instantiate a plotter for all plots made herein - myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) - - # make output png with circles marking sources found - if mpl: - circles=[] - nx = data.shape[1] # data[] array was already flipud(rot90)'d - ny = data.shape[0] # - for (x,y) in zip(xtab,ytab): - circles.append([x,y,1]) - # @todo variable name - if logscale: - logging.warning("LogScaling applied") - data = data/sigma - data = np.where(data<0,-np.log10(1-data),+np.log10(1+data)) - if nsources == 0: - title = "SFind2D: 0 sources above S/N=%.1f" % (nsigma) - elif nsources == 1: - title = "SFind2D: 1 source (%.1f < S/N < %.1f)" % (nsigma,sn0) - else: - title = "SFind2D: %d sources (%.1f < S/N < %.1f)" % (nsources,nsigma,sn0) - myplot.map1(data,title,slbase,thumbnail=True,circles=circles, - zoom=self.getkey("zoom")) - - #--------------------------------------------------------- - # Get the figure and thumbmail names and create a caption - #--------------------------------------------------------- - imname = myplot.getFigure(figno=myplot.figno,relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno,relative=True) - caption = "Image of input map with sources found by SFind2D overlayed in green." - slbdp.table.description="Table of source locations and sizes (not deconvolved)" + if self._plot_mode == PlotControl.NOPLOT: + figname = "not created" + thumbname = "not created" + imcaption = "not created" + noplot = True + else: + # instantiate a plotter for all plots made herein + myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) + + # make output png with circles marking sources found + if mpl: + circles=[] + nx = data.shape[1] # data[] array was already flipud(rot90)'d + ny = data.shape[0] # + for (x,y) in zip(xtab,ytab): + circles.append([x,y,1]) + # @todo variable name + if logscale: + logging.warning("LogScaling applied") + data = data/sigma + data = np.where(data<0,-np.log10(1-data),+np.log10(1+data)) + if nsources == 0: + title = "SFind2D: 0 sources above S/N=%.1f" % (nsigma) + elif nsources == 1: + title = "SFind2D: 1 source (%.1f < S/N < %.1f)" % (nsigma,sn0) + else: + title = "SFind2D: %d sources (%.1f < S/N < %.1f)" % (nsources,nsigma,sn0) + myplot.map1(data,title,slbase,thumbnail=True,circles=circles, + zoom=self.getkey("zoom")) + + #--------------------------------------------------------- + # Get the figure and thumbmail names and create a caption + #--------------------------------------------------------- + figname = myplot.getFigure(figno=myplot.figno,relative=True) + thumbname = myplot.getThumbnail(figno=myplot.figno,relative=True) + imcaption = "Image of input map with sources found by SFind2D overlayed in green." - #--------------------------------------------------------- - # Add finder image to the BDP - #--------------------------------------------------------- - image = Image(images={bt.PNG: imname}, - thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=caption) - slbdp.image.addimage(image, "finderimage") + #--------------------------------------------------------- + # Add finder image to the BDP + #--------------------------------------------------------- + image = Image(images={bt.PNG: figname}, + thumbnail=thumbname, + thumbnailtype=bt.PNG, description=imcaption) + slbdp.image.addimage(image, "finderimage") + noplot=False #------------------------------------------------------------- # Create the summary entry for the table and image @@ -443,7 +450,7 @@ def run(self): self._summary["sources"] = SummaryEntry([slbdp.table.serialize(), slbdp.image.serialize()], "SFind2D_AT", - self.id(True), taskargs) + self.id(True), taskargs, noplot=noplot) dt.tag("done") dt.end() From 13a153d681d6466e056e3e821cf265dd28067a86 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 11:40:16 -0400 Subject: [PATCH 13/34] a few more tasks using noplot --- admit/Summary.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/admit/Summary.py b/admit/Summary.py index 8853079..59ffb3f 100755 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1208,7 +1208,11 @@ def _process(self,taskname,tid,titems,thetask,outdir): if tlower == "cubesum_at": cubesum = titems.get('cubesum',None) - if cubesum != None: + if cubesum == None: + allspecs = "

%s produced no output

" % (taskname) + elif cubesum.getNoPlot(): + allspecs = "

%s created output but was told to create no PNGs for display.

" % (taskname) + else: allspecs = "" taskargs = cubesum.taskargs val = cubesum.getValue() @@ -1226,8 +1230,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): specval = SPAN4VALB % (image, thumb, caption, caption, caption,button,button2) banner = "

%s output for %s

" % (taskname, casaimage) allspecs = banner + allspecs + "\n" + specval - else: - allspecs = "

%s computed nothing for the input image

" % taskname + retval = header % (taskclass, tid,thetask.statusicons(),taskname,tid,taskargs,tid,allspecs,tid) if tlower == "continuumsub_at": @@ -1487,7 +1490,11 @@ def _process(self,taskname,tid,titems,thetask,outdir): if tlower == "sfind2d_at": the_item = titems.get('sources',None) #SummaryEntry - if the_item != None: + if the_item == None: + tablestr = "

%s identified no sources

" % taskname + elif the_item.getNoPlot(): # probably will never happen + tablestr = "

%s created output for but was told to create no table for display.

" % (taskname) + else: summarydata = the_item.getValue() if summarydata == None or len(summarydata) == 0: tablestr = "

%s identified no sources

" % taskname @@ -1503,8 +1510,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): tablestr = "

%s identified no sources


%s" % (taskname,imstr) else: tablestr = STARTROW + imstr + (SPANXVAL % ("8",atable.html('class="table table-admit table-bordered table-striped"'))) + ENDROW - else: - tablestr = "

%s identified no sources

" % taskname + retval = header % (taskclass, tid,thetask.statusicons(),taskname,tid,the_item.taskargs,tid,tablestr,tid) if tlower == "overlapintegral_at": @@ -1555,11 +1561,15 @@ def _process(self,taskname,tid,titems,thetask,outdir): if tlower == "template_at": # Template returns one table (in a list) and two plots. the_item = titems.get('template',None) #SummaryEntry - tablestr = '' - if the_item != None: + if the_item == None: + tablestr = "

%s produced no table output

" % (taskname) + elif the_item.getNoPlot(): # probably will never happen + tablestr = "

%s created output for but was told to create no table for display.

" % (taskname) + else: + tablestr = '' summarydata = the_item.getValue() - if summarydata == None or len(summarydata) == 0: - tablestr = "

%s produced no output

" % taskname + if summarydata == None or len(summarydata) == 0: + tablestr = "

%s produced no table output

" % taskname else: atable = admit.util.Table() atable.deserialize(summarydata[0]) @@ -1567,13 +1577,15 @@ def _process(self,taskname,tid,titems,thetask,outdir): tablestr = tablestr + "

No data available for this summary.

" else: tablestr = tablestr + atable.html('class="table table-admit table-bordered table-striped"')+os.linesep+os.linesep - else: - tablestr = "

%s produced no output

" % taskname # Output plots. allspecs = tablestr + "\n" spectra = titems.get('spectra',None) - if (spectra) != None: + if spectra == None: + allspecs = allspecs + "

%s produced no spectral output

" % (taskname) + elif the_item.getNoPlot(): + allspecs = allspecs + "

%s created output for but was told to create no PNGs for display.

" % (taskname) + else: count = 0 # task arguments are the same in all entries. taskargs = spectra.taskargs @@ -1599,8 +1611,6 @@ def _process(self,taskname,tid,titems,thetask,outdir): banner = '

%s output for %s

' % (taskname, casaimage) allspecs = banner + allspecs - else: - allspecs = allspecs + "

No spectra were produced by %s

" % taskname retval = header % (taskclass, tid,thetask.statusicons(),taskname,tid,taskargs,tid,allspecs,tid) From 83504cc46c213ff8052ebe10b9104ffee44e1957 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 11:58:27 -0400 Subject: [PATCH 14/34] this task does not create plots but inserted the noplot into SummaryEntry as a reminder to future changes which may include plots --- admit/at/Regrid_AT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admit/at/Regrid_AT.py b/admit/at/Regrid_AT.py index 1f1d55a..cb7d181 100644 --- a/admit/at/Regrid_AT.py +++ b/admit/at/Regrid_AT.py @@ -258,6 +258,6 @@ def run(self): #keys = "pixsize=%.4g naxis1=%d naxis2=%d mean_ra=%0.4f mean_dec=%0.4f reffreq=%g chan_width=%g" % (pix_scale/(4.848137E-6), npix_ra, npix_dec, mean_ra,mean_dec,min_nu,chan_width) taskargs = "pix_scale = " + str(self.getkey("pix_scale")) + " chan_width = "+str(self.getkey("chan_width")) - self._summary["regrid"] = SummaryEntry(atable.serialize(),"Regrid_AT",self.id(True),taskargs) + self._summary["regrid"] = SummaryEntry(atable.serialize(),"Regrid_AT",self.id(True),taskargs,noplot=True) dt.tag("done") dt.end() From ce5b58e5b06e0ee55dd10e13a82226aaf6851841 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 12:21:51 -0400 Subject: [PATCH 15/34] changing over to use new noplot member variable, and remove unused variables --- admit/at/PVSlice_AT.py | 168 ++++++++++++++++++++++------------------- 1 file changed, 89 insertions(+), 79 deletions(-) diff --git a/admit/at/PVSlice_AT.py b/admit/at/PVSlice_AT.py index 5185b19..ba323a9 100644 --- a/admit/at/PVSlice_AT.py +++ b/admit/at/PVSlice_AT.py @@ -272,87 +272,97 @@ def run(self): logging.info("PV stats: mean/std/max %f %f %f" % (r_mean, r_std, r_max)) logging.regression("PVSLICE: %f %f %f" % (r_mean, r_std, r_max)) - myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) - - # hack to get a slice on a mom0map - # @todo if pmode is not png, can viewer handle this? - figname = pvname + ".png" - slicename = self.dir(figname) - overlay = pvname+"_overlay" - if b11 != None: - f11 = b11.getimagefile(bt.CASA) - tb = taskinit.tbtool() - tb.open(self.dir(f11)) - data = tb.getcol('map') - nx = data.shape[0] - ny = data.shape[1] - tb.close() - d1 = np.flipud(np.rot90 (data.reshape((nx,ny)))) - if len(pvslice) == 4: - segm = [[pvslice[0],pvslice[2],pvslice[1],pvslice[3]]] - pa = np.arctan2(pvslice[2]-pvslice[0],pvslice[1]-pvslice[3])*180.0/np.pi - title = "PV Slice location : slice PA=%.1f" % pa - xcen = (pvslice[0]+pvslice[2])/2.0 - ycen = (pvslice[1]+pvslice[3])/2.0 - elif len(pvslit) == 4: - # can only do this now if using pixel coordinates - xcen = pvslit[0] - ycen = pvslit[1] - slen = pvslit[2] - pard = pvslit[3]*np.pi/180.0 - cosp = np.cos(pard) - sinp = np.sin(pard) - halflen = 0.5*slen - segm = [[xcen-halflen*sinp,xcen+halflen*sinp,ycen+halflen*cosp,ycen-halflen*cosp]] - pa = pvslit[3] - title = "PV Slice location : slit PA=%g" % pa - else: - # bogus, some error in pvslice - logging.warning("bogus segm since pvslice=%s" % str(pvslice)) - segm = [[10,20,10,20]] - pa = -999.999 - title = "PV Slice location - bad PA" - logging.info("MAP1 segm %s %s" % (str(segm),str(pvslice))) - if d1.max() < clip: - logging.warning("datamax=%g, clip=%g" % (d1.max(), clip)) - title = title + ' (no signal over %g?)' % clip - myplot.map1(d1,title,overlay,segments=segm,thumbnail=True, - zoom=self.getkey("zoom"),star=[xcen,ycen]) - else: - myplot.map1(d1,title,overlay,segments=segm,range=[clip],thumbnail=True, - zoom=self.getkey("zoom"),star=[xcen,ycen]) - dt.tag("plot") - overlayname = myplot.getFigure(figno=myplot.figno,relative=True) - overlaythumbname = myplot.getThumbnail(figno=myplot.figno,relative=True) - Qover = True + pvcaption = "Position-velocity diagram through emission centroid" + if self._plot_mode == PlotControl.NOPLOT: + figname = "not created" + overlayname = "not created" + overlaythumbname = "not created" + overlaycaption = "not created" + thumbname = "not created" + noplot = True + pvimage = Image(images={bt.CASA : pvname}, description=pvcaption) else: - Qover = False - - implot = ImPlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir()) - implot.plotter(rasterfile=pvname, figname=pvname, colorwedge=True) - thumbname = implot.getThumbnail(figno=implot.figno,relative=True) - figname = implot.getFigure(figno=implot.figno,relative=True) - if False: - # debug: - # - # @todo tmp1 is ok, tmp2 is not displaying the whole thing - # use casa_imview, not casa.imview - if this is enabled. - # old style: viewer() seems to plot full image, but imview() wants square pixels? - casa.viewer(infile=self.dir(pvname), outfile=self.dir('tmp1.pv.png'), gui=False, outformat="png") - casa.imview(raster={'file':self.dir(pvname), 'colorwedge' : True, 'scaling':-1}, - axes={'y':'Declination'}, - out=self.dir('tmp2.pv.png')) - # - # -> this one works, axes= should be correct - # imview(raster={'file':'x.pv', 'colorwedge' : True, 'scaling':-1},axes={'y':'Frequency'}) - # - # @TODO big fixme, we're going to reuse 'tmp1.pv.png' because implot give a broken view - figname = 'tmp1.pv.png' + noplot = False + myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) + + # hack to get a slice on a mom0map + # @todo if pmode is not png, can viewer handle this? + overlay = pvname+"_overlay" + + if b11 != None: + f11 = b11.getimagefile(bt.CASA) + tb = taskinit.tbtool() + tb.open(self.dir(f11)) + data = tb.getcol('map') + nx = data.shape[0] + ny = data.shape[1] + tb.close() + d1 = np.flipud(np.rot90 (data.reshape((nx,ny)))) + if len(pvslice) == 4: + segm = [[pvslice[0],pvslice[2],pvslice[1],pvslice[3]]] + pa = np.arctan2(pvslice[2]-pvslice[0],pvslice[1]-pvslice[3])*180.0/np.pi + title = "PV Slice location : slice PA=%.1f" % pa + xcen = (pvslice[0]+pvslice[2])/2.0 + ycen = (pvslice[1]+pvslice[3])/2.0 + elif len(pvslit) == 4: + # can only do this now if using pixel coordinates + xcen = pvslit[0] + ycen = pvslit[1] + slen = pvslit[2] + pard = pvslit[3]*np.pi/180.0 + cosp = np.cos(pard) + sinp = np.sin(pard) + halflen = 0.5*slen + segm = [[xcen-halflen*sinp,xcen+halflen*sinp,ycen+halflen*cosp,ycen-halflen*cosp]] + pa = pvslit[3] + title = "PV Slice location : slit PA=%g" % pa + else: + # bogus, some error in pvslice + logging.warning("bogus segm since pvslice=%s" % str(pvslice)) + segm = [[10,20,10,20]] + pa = -999.999 + title = "PV Slice location - bad PA" + logging.info("MAP1 segm %s %s" % (str(segm),str(pvslice))) + if d1.max() < clip: + logging.warning("datamax=%g, clip=%g" % (d1.max(), clip)) + title = title + ' (no signal over %g?)' % clip + myplot.map1(d1,title,overlay,segments=segm,thumbnail=True, + zoom=self.getkey("zoom"),star=[xcen,ycen]) + else: + myplot.map1(d1,title,overlay,segments=segm,range=[clip],thumbnail=True, + zoom=self.getkey("zoom"),star=[xcen,ycen]) + dt.tag("plot") + overlayname = myplot.getFigure(figno=myplot.figno,relative=True) + overlaythumbname = myplot.getThumbnail(figno=myplot.figno,relative=True) + Qover = True + else: + Qover = False + + implot = ImPlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir()) + implot.plotter(rasterfile=pvname, figname=pvname, colorwedge=True) + thumbname = implot.getThumbnail(figno=implot.figno,relative=True) + figname = implot.getFigure(figno=implot.figno,relative=True) + if False: + # debug: + # + # @todo tmp1 is ok, tmp2 is not displaying the whole thing + # use casa_imview, not casa.imview - if this is enabled. + # old style: viewer() seems to plot full image, but imview() wants square pixels? + casa.viewer(infile=self.dir(pvname), outfile=self.dir('tmp1.pv.png'), gui=False, outformat="png") + casa.imview(raster={'file':self.dir(pvname), 'colorwedge' : True, 'scaling':-1}, + axes={'y':'Declination'}, + out=self.dir('tmp2.pv.png')) + # + # -> this one works, axes= should be correct + # imview(raster={'file':'x.pv', 'colorwedge' : True, 'scaling':-1},axes={'y':'Frequency'}) + # + # @TODO big fixme, we're going to reuse 'tmp1.pv.png' because implot give a broken view + figname = 'tmp1.pv.png' # @todo technically we don't know what map it was overlay'd on.... CubeSum/Moment0 - overlaycaption = "Location of position-velocity slice overlaid on a CubeSum map" - pvcaption = "Position-velocity diagram through emission centroid" - pvimage = Image(images={bt.CASA : pvname, bt.PNG : figname},thumbnail=thumbname,thumbnailtype=bt.PNG, description=pvcaption) + overlaycaption = "Location of position-velocity slice overlaid on a CubeSum map" + pvimage = Image(images={bt.CASA : pvname, bt.PNG : figname},thumbnail=thumbname,thumbnailtype=bt.PNG, description=pvcaption) + b2.setkey("image",pvimage) b2.setkey("mean",float(r_mean)) b2.setkey("sigma",float(r_std)) @@ -364,7 +374,7 @@ def run(self): # Yes, this is a nested list. Against the day when PVSLICE can # compute multiple slices per map. pvslicesummary.append(thispvsummary) - self._summary["pvslices"] = SummaryEntry(pvslicesummary,"PVSlice_AT",self.id(True),taskargs) + self._summary["pvslices"] = SummaryEntry(pvslicesummary,"PVSlice_AT",self.id(True),taskargs,noplot=noplot) dt.tag("done") dt.end() From 28bfd8ca30e19db0d2305ed57440fa98f77bce2d Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 12:25:22 -0400 Subject: [PATCH 16/34] pvslice_at and pvcorr_at using noplot --- admit/Summary.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/admit/Summary.py b/admit/Summary.py index 59ffb3f..10c8aae 100755 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1397,7 +1397,11 @@ def _process(self,taskname,tid,titems,thetask,outdir): if tlower == "pvslice_at": pvslices = titems.get('pvslices',None) - if pvslices != None: + if pvslices == None: + specval = "

No PV slices were computed for this cube

" + elif pvslices.getNoPlot(): + specval = "

%s created output but was told to create no PNGs for display.

" % (taskname) + else: for val in pvslices.value: specval = STARTROW slicetype = val[0] @@ -1423,8 +1427,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): banner = "

%s output for %s

" % (taskname, slicename) specval = banner + specval - else: - specval = "

No PV slices were computed for this cube

" + retval = header % (taskclass, tid,thetask.statusicons(),taskname,tid,pvslices.taskargs,tid,specval,tid) if tlower == "linecube_at": @@ -1458,7 +1461,11 @@ def _process(self,taskname,tid,titems,thetask,outdir): if tlower == "pvcorr_at": the_item = titems.get('pvcorr',None) - if the_item != None: + if the_item == None: + specval = "

No PV correlation diagrams were computed from the input cube

" + elif pvslices.getNoPlot(): + specval = "

%s created output but was told to create no PNGs for display.

" % (taskname) + else: val = the_item.getValue() image = val[0] thumb = val[1] @@ -1467,8 +1474,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): specval = STARTROW + (SPAN4VAL % ( image, thumb, caption, caption, caption)) + ENDROW banner = "

%s output for %s

" % (taskname, pvcorrname) specval = banner + specval - else: - specval = "

No PV correlation diagrams were computed from the input cube

" + retval = header % (taskclass, tid,thetask.statusicons(),taskname,tid,the_item.taskargs,tid,specval,tid) # @todo move table formatting to here. @@ -1493,7 +1499,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if the_item == None: tablestr = "

%s identified no sources

" % taskname elif the_item.getNoPlot(): # probably will never happen - tablestr = "

%s created output for but was told to create no table for display.

" % (taskname) + tablestr = "

%s created output but was told to create no table for display.

" % (taskname) else: summarydata = the_item.getValue() if summarydata == None or len(summarydata) == 0: @@ -1564,7 +1570,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if the_item == None: tablestr = "

%s produced no table output

" % (taskname) elif the_item.getNoPlot(): # probably will never happen - tablestr = "

%s created output for but was told to create no table for display.

" % (taskname) + tablestr = "

%s created output but was told to create no table for display.

" % (taskname) else: tablestr = '' summarydata = the_item.getValue() @@ -1584,7 +1590,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if spectra == None: allspecs = allspecs + "

%s produced no spectral output

" % (taskname) elif the_item.getNoPlot(): - allspecs = allspecs + "

%s created output for but was told to create no PNGs for display.

" % (taskname) + allspecs = allspecs + "

%s created output but was told to create no PNGs for display.

" % (taskname) else: count = 0 # task arguments are the same in all entries. From 22102059419962195cd44e80a4ec8029f7759716 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 13:32:44 -0400 Subject: [PATCH 17/34] changing over to use new noplot member variable --- admit/at/PVCorr_AT.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/admit/at/PVCorr_AT.py b/admit/at/PVCorr_AT.py index c3a61c1..b958c44 100644 --- a/admit/at/PVCorr_AT.py +++ b/admit/at/PVCorr_AT.py @@ -136,7 +136,10 @@ def run(self): b1 = self._bdp_in[0] # PVSlice_BDP fin = b1.getimagefile(bt.CASA) # CASA image data = casautil.getdata_raw(self.dir(fin)) # grab the data as a numpy array - self.myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) + bdp_name = self.mkext(fin,"pvc") # output PVCorr_BDP + b3 = PVCorr_BDP(bdp_name) + self.addoutput(b3) + #print 'DATA[0,0]:',data[0,0] #print 'pv shape: ',data.shape npos = data.shape[0] @@ -189,9 +192,6 @@ def run(self): dt.tag("imstat") - bdp_name = self.mkext(fin,"pvc") # output PVCorr_BDP - b3 = PVCorr_BDP(bdp_name) - self.addoutput(b3) if ch0<0 or ch1>=nvel: # this probably only happens to small cubes (problematic for PVCorr) @@ -255,22 +255,32 @@ def run(self): segp = [] segp.append( [0,len(ch),0.0,0.0] ) segp.append( [0,len(ch),3.0*rms, 3.0*rms] ) + # @todo: in principle we know with given noise and size of box, what the sigma in pvcorr should be - self.myplot.plotter(x,y,title,figname=p1,xlab=xlab,ylab=ylab,segments=segp, thumbnail=True) + + taskargs = "numsigma=%.1f range=[%d,%d]" % (numsigma,ch0,ch1) + + if self._plot_mode == PlotControl.NOPLOT: + thumbname = "not created" + figname = "not created" + imcaption = "not created" + image = Image(description=imcaption) + else: + self.myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) + self.myplot.plotter(x,y,title,figname=p1,xlab=xlab,ylab=ylab,segments=segp, thumbnail=True) #out1 = np.rot90 (data.reshape((nvel,npos)) ) if mode > 1: self.myplot.map1(data=out,title="testing PVCorr_AT: mode%d"%mode,figname='testPVCorr', thumbnail=True) - taskargs = "numsigma=%.1f range=[%d,%d]" % (numsigma,ch0,ch1) - caption = "Position-velocity correlation plot" - thumbname = self.myplot.getThumbnail(figno=self.myplot.figno,relative=True) - figname = self.myplot.getFigure(figno=self.myplot.figno,relative=True) - image = Image(images={bt.PNG: figname}, thumbnail=thumbname, thumbnailtype=bt.PNG, - description=caption) + imcaption = "Position-velocity correlation plot" + thumbname = self.myplot.getThumbnail(figno=self.myplot.figno,relative=True) + figname = self.myplot.getFigure(figno=self.myplot.figno,relative=True) + image = Image(images={bt.PNG: figname}, thumbnail=thumbname, thumbnailtype=bt.PNG, description=imcaption) + b3.image.addimage(image, "pvcorr") - self._summary["pvcorr"] = SummaryEntry([figname,thumbname,caption,fin],"PVCorr_AT",self.id(True),taskargs) + self._summary["pvcorr"] = SummaryEntry([figname,thumbname,imcaption,fin],"PVCorr_AT",self.id(True),taskargs,noplot=noplot) else: self._summary["pvcorr"] = None logging.warning("No summary") @@ -304,7 +314,8 @@ def mode3(self, data, v0, v1, dmin=0.0): fmax = f.max() print "PVCorr mode3:",f1.sum(),'/',f0.sum(),'min/max',smin,fmax out = scipy.signal.correlate2d(data,f,mode='same') - self.myplot.map1(data=f,title="PVCorr 2D Kernel",figname='PVCorrKernel', thumbnail=True) + if self._plot_mode != PlotControl.NOPLOT: + self.myplot.map1(data=f,title="PVCorr 2D Kernel",figname='PVCorrKernel', thumbnail=True) print 'PVCorr min/max:',out.min(),out.max() n1,m1,s1,n2,m2,s2 = stats.mystats(out.flatten()) From ea55d3fdfa67f4cee1a6547a795ab05cbcdc43e8 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 14:42:13 -0400 Subject: [PATCH 18/34] changing over to use new noplot member variable --- admit/at/PrincipalComponent_AT.py | 268 +++++++++++++++--------------- 1 file changed, 136 insertions(+), 132 deletions(-) diff --git a/admit/at/PrincipalComponent_AT.py b/admit/at/PrincipalComponent_AT.py index 63018f9..0fbb93a 100644 --- a/admit/at/PrincipalComponent_AT.py +++ b/admit/at/PrincipalComponent_AT.py @@ -40,7 +40,7 @@ class PrincipalComponent_AT(admit.Task): the number of input maps; the default empty list implies all zeroes. **covarmin**: float - Minimum pairwise covariance for summary output (only); default: 0.90. + Minimum pairwise covariance for summary output (only); default: 0.90. **Input BDPs** @@ -52,7 +52,7 @@ class PrincipalComponent_AT(admit.Task): **Output BDPs** **Table_BDP**: count: 3 - First table contains the input mean and standard deviation plus output + First table contains the input mean and standard deviation plus output eigenimage filename and variance fractions. Second table is the calculated projection matrix transforming (mean-subtracted and variance-normalized) input data to principal component space. Third @@ -148,151 +148,155 @@ def run(self): ------- None """ - self._summary = {} + self._summary = {} - # BDP output uses the alias name if provided, else a flow-unique one. - stem = self._alias - if not stem: stem = "pca%d" % (self.id()) + # BDP output uses the alias name if provided, else a flow-unique one. + stem = self._alias + if not stem: stem = "pca%d" % (self.id()) - inum = 0 - data = [] - icols = [] + inum = 0 + data = [] + icols = [] for ibdp in self._bdp_in: - # Convert input CASA images to numpy arrays. - istem = ibdp.getimagefile(bt.CASA) - ifile = ibdp.baseDir() + istem - icols.append(os.path.splitext(istem)[0]) - if os.path.dirname(icols[-1]): - icols[-1] = os.path.dirname(icols[-1]) # Typical line cube case. - img = admit.casautil.getdata(ifile, zeromask=True).data - data.append(img) - admit.logging.info("%s shape=%s min=%g max=%g" % - (icols[-1], str(img.shape), np.amin(img), np.amax(img))) - assert len(data[0].shape) == 2, "Only 2-D input images supported" - assert data[0].shape == data[inum].shape, "Input shapes must match" - inum += 1 - - # At least two inputs required for meaningful PCA! - assert inum >= 2, "At least two input images required" - - # Each 2-D input image is a plane in a single multi-color image. - # Each color multiplet (one per pixel) is an observation. - # For PCA we collate the input images into a vector of observations. - shape = data[0].shape - npix = shape[0] * shape[1] - clip = self.getkey('clipvals') - if not clip: clip = [0 for i in range(inum)] - assert len(clip) >= inum, "Too few clipvals provided" - - # Clip input values and stack into a vector of observations. - pca_data = [] + # Convert input CASA images to numpy arrays. + istem = ibdp.getimagefile(bt.CASA) + ifile = ibdp.baseDir() + istem + icols.append(os.path.splitext(istem)[0]) + if os.path.dirname(icols[-1]): + icols[-1] = os.path.dirname(icols[-1]) # Typical line cube case. + img = admit.casautil.getdata(ifile, zeromask=True).data + data.append(img) + admit.logging.info("%s shape=%s min=%g max=%g" % + (icols[-1], str(img.shape), np.amin(img), np.amax(img))) + assert len(data[0].shape) == 2, "Only 2-D input images supported" + assert data[0].shape == data[inum].shape, "Input shapes must match" + inum += 1 + + # At least two inputs required for meaningful PCA! + assert inum >= 2, "At least two input images required" + + # Each 2-D input image is a plane in a single multi-color image. + # Each color multiplet (one per pixel) is an observation. + # For PCA we collate the input images into a vector of observations. + shape = data[0].shape + npix = shape[0] * shape[1] + clip = self.getkey('clipvals') + if not clip: clip = [0 for i in range(inum)] + assert len(clip) >= inum, "Too few clipvals provided" + + # Clip input values and stack into a vector of observations. + pca_data = [] for i in range(inum): nd = data[i] - nd[nd < clip[i]] = 0.0 - pca_data.append(np.reshape(nd, (npix,1))) - pca_in = np.hstack(pca_data) - pca = mlab.PCA(pca_in) - - # Input statistics and output variance fractions. - #print "fracs:", pca.fracs - #print "mean:", pca.mu - #print "sdev:", pca.sigma - obdp = admit.Table_BDP(stem+"_stats") - obdp.table.setData(np.vstack([pca.mu, pca.sigma,pca.fracs]).T) - obdp.table.columns = ["Input mean", "Input deviation", - "Eigenimage variance fraction"] - obdp.table.description = "PCA Image Statistics" - self.addoutput(obdp) - - # Pre-format columns for summary output. - # This is required when mixing strings and numbers in a table. - # (NumPy will output the array as all strings.) - table1 = admit.Table() - table1.setData(np.vstack([[i for i in range(inum)], - icols, - ["%.3e" % x for x in pca.mu], - ["%.3e" % x for x in pca.sigma], - ["%s_eigen/%d.im" % (stem, i) - for i in range(inum)], - ["%.4f" % x for x in pca.fracs]]).T) - table1.columns = ["Index", "Input", "Input mean", - "Input deviation", - "Eigenimage", - "Eigenimage variance fraction"] - table1.description = "PCA Image Statistics" - - # Projection matrix (eigenvectors). - #print "projection:", pca.Wt - obdp = admit.Table_BDP(stem + "_proj") - obdp.table.setData(pca.Wt) - obdp.table.columns = icols - obdp.table.description = \ - "PCA Projection Matrix (normalized input to output)" - self.addoutput(obdp) - - # Covariance matrix. - covar = np.cov(pca.a, rowvar=0, bias=1) - #print "covariance:", covar - obdp = admit.Table_BDP(stem + "_covar") - obdp.table.setData(covar) - obdp.table.columns = icols - obdp.table.description = "PCA Covariance Matrix" - self.addoutput(obdp) - - # Collate projected observations into eigenimages and save output. - os.mkdir(self.baseDir()+stem+"_eigen") - pca_out = np.hsplit(pca.Y, inum) - odata = [] + nd[nd < clip[i]] = 0.0 + pca_data.append(np.reshape(nd, (npix,1))) + pca_in = np.hstack(pca_data) + pca = mlab.PCA(pca_in) + + # Input statistics and output variance fractions. + #print "fracs:", pca.fracs + #print "mean:", pca.mu + #print "sdev:", pca.sigma + obdp = admit.Table_BDP(stem+"_stats") + obdp.table.setData(np.vstack([pca.mu, pca.sigma,pca.fracs]).T) + obdp.table.columns = ["Input mean", "Input deviation", + "Eigenimage variance fraction"] + obdp.table.description = "PCA Image Statistics" + self.addoutput(obdp) + + # Pre-format columns for summary output. + # This is required when mixing strings and numbers in a table. + # (NumPy will output the array as all strings.) + table1 = admit.Table() + table1.setData(np.vstack([[i for i in range(inum)], + icols, + ["%.3e" % x for x in pca.mu], + ["%.3e" % x for x in pca.sigma], + ["%s_eigen/%d.im" % (stem, i) + for i in range(inum)], + ["%.4f" % x for x in pca.fracs]]).T) + table1.columns = ["Index", "Input", "Input mean", + "Input deviation", + "Eigenimage", + "Eigenimage variance fraction"] + table1.description = "PCA Image Statistics" + + # Projection matrix (eigenvectors). + #print "projection:", pca.Wt + obdp = admit.Table_BDP(stem + "_proj") + obdp.table.setData(pca.Wt) + obdp.table.columns = icols + obdp.table.description = \ + "PCA Projection Matrix (normalized input to output)" + self.addoutput(obdp) + + # Covariance matrix. + covar = np.cov(pca.a, rowvar=0, bias=1) + #print "covariance:", covar + obdp = admit.Table_BDP(stem + "_covar") + obdp.table.setData(covar) + obdp.table.columns = icols + obdp.table.description = "PCA Covariance Matrix" + self.addoutput(obdp) + + # Collate projected observations into eigenimages and save output. + os.mkdir(self.baseDir()+stem+"_eigen") + pca_out = np.hsplit(pca.Y, inum) + odata = [] for i in range(inum): - ofile = "%s_eigen/%d" % (stem, i) - img = np.reshape(pca_out[i], shape) + ofile = "%s_eigen/%d" % (stem, i) + img = np.reshape(pca_out[i], shape) odata.append(img) - #print ofile, "shape, min, max:", img.shape, np.amin(img), np.amax(img) - - aplot = admit.util.APlot(figno=inum, abspath=self.baseDir(), - ptype=admit.util.PlotControl.PNG) - aplot.map1(np.rot90(img), title=ofile, figname=ofile) - aplot.final() - - # Currently the output eigenimages are stored as PNG files only. - admit.casautil.putdata_raw(self.baseDir()+ofile+".im", img, ifile) - oimg = admit.Image() - oimg.addimage(admit.imagedescriptor(ofile+".im", format=bt.CASA)) - oimg.addimage(admit.imagedescriptor(ofile+".png", format=bt.PNG)) + #print ofile, "shape, min, max:", img.shape, np.amin(img), np.amax(img) + + + # Currently the output eigenimages are stored as PNG files only. + admit.casautil.putdata_raw(self.baseDir()+ofile+".im", img, ifile) + oimg = admit.Image() + oimg.addimage(admit.imagedescriptor(ofile+".im", format=bt.CASA)) + if self._plot_mode != admit.PlotControl.NOPLOT: + noplot = False + aplot = admit.util.APlot(figno=inum, abspath=self.baseDir(), + ptype=admit.util.PlotControl.PNG) + aplot.map1(np.rot90(img), title=ofile, figname=ofile) + aplot.final() + oimg.addimage(admit.imagedescriptor(ofile+".png", format=bt.PNG)) + else: + noplot = True obdp = admit.Image_BDP(ofile) - obdp.addimage(oimg) - self.addoutput(obdp) + obdp.addimage(oimg) + self.addoutput(obdp) - # As a cross-check, reconstruct input images and compute differences. + # As a cross-check, reconstruct input images and compute differences. for k in range(inum): - ximg = pca.Wt[0][k]*odata[0] - for l in range(1,inum): - ximg += pca.Wt[l][k]*odata[l] + ximg = pca.Wt[0][k]*odata[0] + for l in range(1,inum): + ximg += pca.Wt[l][k]*odata[l] - ximg = pca.mu[k] + pca.sigma[k]*ximg + ximg = pca.mu[k] + pca.sigma[k]*ximg admit.logging.regression("PCA: %s residual: " % icols[k] + - str(np.linalg.norm(ximg - data[k]))) + str(np.linalg.norm(ximg - data[k]))) - # Collect large covariance values for summary. - cvmin = self.getkey('covarmin') - cvsum = [] + # Collect large covariance values for summary. + cvmin = self.getkey('covarmin') + cvsum = [] cvmax = 0.0 - for i in range(inum): - for j in range(i+1, inum): - if abs(covar[i][j]) >= cvmax: - cvmax = abs(covar[i][j]) - if abs(covar[i][j]) >= cvmin: - cvsum.append([icols[i], icols[j], "%.4f" % (covar[i][j])]) + for i in range(inum): + for j in range(i+1, inum): + if abs(covar[i][j]) >= cvmax: + cvmax = abs(covar[i][j]) + if abs(covar[i][j]) >= cvmin: + cvsum.append([icols[i], icols[j], "%.4f" % (covar[i][j])]) admit.logging.regression("PCA: Covariances > %.4f: %s (max: %.4f)" % (cvmin,str(cvsum),cvmax)) - table2 = admit.Table() - table2.columns = ["Input1", "Input2", "Covariance"] - table2.setData(cvsum) - table2.description = "PCA High Covariance Summary" + table2 = admit.Table() + table2.columns = ["Input1", "Input2", "Covariance"] + table2.setData(cvsum) + table2.description = "PCA High Covariance Summary" - keys = "covarmin=%.4f clipvals=%s" % (cvmin, str(clip)) + keys = "covarmin=%.4f clipvals=%s" % (cvmin, str(clip)) self._summary["pca"] = admit.SummaryEntry([table1.serialize(), - table2.serialize() - ], - "PrincipalComponent_AT", - self.id(True), keys) + table2.serialize() + ], + "PrincipalComponent_AT", + self.id(True), keys,noplot=noplot) From 7a79ad6fc920b26a6785c842987d84e05903fc33 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 15:09:49 -0400 Subject: [PATCH 19/34] changing over to use new noplot member variable --- admit/at/OverlapIntegral_AT.py | 38 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/admit/at/OverlapIntegral_AT.py b/admit/at/OverlapIntegral_AT.py index 5f33b0c..f9fba58 100644 --- a/admit/at/OverlapIntegral_AT.py +++ b/admit/at/OverlapIntegral_AT.py @@ -33,7 +33,7 @@ import casa import taskinit except: - print "WARNING: No scipy or casa; OverlapIntegral task cannot function." + print("WARNING: No scipy or casa; OverlapIntegral task cannot function.") class OverlapIntegral_AT(AT): @@ -140,7 +140,6 @@ def run(self): normalize = self.getkey("normalize") doCross = True doCross = False - myplot = APlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir()) dt.tag("start") @@ -236,23 +235,28 @@ def run(self): pdata = np.rot90(out.squeeze()) logging.info("PDATA: %s" % str(pdata.shape)) - myplot.map1(pdata,title,"testOI",thumbnail=True,cmap=cmap) + if self._plot_mode == PlotControl.NOPLOT: + figname = "not created" + thumbnailname = "not created" + imcaption = "not created" + noplot = True + else: + myplot = APlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir()) + myplot.map1(pdata,title,"testOI",thumbnail=True,cmap=cmap) + figname = myplot.getFigure(figno=myplot.figno,relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno,relative=True) + #@todo fill in imcaption with more info - line names, etc. + imcaption = "Need descriptive imcaption here" + noplot = False #----------------------------- # Populate summary information #----------------------------- taskargs = "chans=%s cmap=%s" % (chans, cmap) - imname = "" - thumbnailname = "" - # uncomment when ready. - imname = myplot.getFigure(figno=myplot.figno,relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno,relative=True) - #@todo fill in caption with more info - line names, etc. - caption = "Need descriptive caption here" - summaryinfo = [summarytable.serialize(),imname,thumbnailname,caption] + summaryinfo = [summarytable.serialize(),figname,thumbnailname,imcaption] self._summary["overlap"] = SummaryEntry(summaryinfo, "OverlapIntegral_AT", - self.id(True),taskargs) + self.id(True),taskargs,noplot=noplot) #----------------------------- dt.tag("done") dt.end() @@ -262,6 +266,10 @@ def crossn(data, myplot): CAUTION: Expensive routine @todo divide cross(i,j)/sqrt(auto(i)*auto(j)) """ + # even if self._plot_mode = PlotControl.NOPLOT, run through the + # computation because a) the user asked for it by doCross=True, + # b) it would be more consistent with any regression testing, + # and c) if this computation ever gets put into a BDP we'd want it that way. n = len(data) nx = data[0].shape[0] ny = data[0].shape[1] @@ -270,7 +278,8 @@ def crossn(data, myplot): idata = data[i].data.squeeze() out = scipy.signal.correlate2d(idata,idata,mode='same') auto[i] = np.rot90(out.reshape((nx,ny))) - myplot.map1(auto[i],"autocorr-%d" % i,"testOI-%d-%d" % (i,i),thumbnail=False) + if self._plot_mode != PlotControl.NOPLOT: + myplot.map1(auto[i],"autocorr-%d" % i,"testOI-%d-%d" % (i,i),thumbnail=False) for i in range(n): idata = data[i].data.squeeze() for j in range(i+1,n): @@ -278,7 +287,8 @@ def crossn(data, myplot): out = scipy.signal.correlate2d(idata,jdata,mode='same') #outd = np.rot90(out.reshape((nx,ny))) / np.sqrt(auto[i]*auto[j]) outd = np.rot90(out.reshape((nx,ny))) - myplot.map1(outd,"crosscorr-%d-%d" % (i,j),"testOI-%d-%d" % (i,j),thumbnail=False) + if self._plot_mode != PlotControl.NOPLOT: + myplot.map1(outd,"crosscorr-%d-%d" % (i,j),"testOI-%d-%d" % (i,j),thumbnail=False) def rgb1(r,g,b, normalize=False): From 0a9b24f0ae48b602ffd17e41dfbdfe4bdb3d0b6c Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 15:10:06 -0400 Subject: [PATCH 20/34] pca and overlapintegral using noplot --- admit/Summary.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/admit/Summary.py b/admit/Summary.py index 10c8aae..7b4f997 100755 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1159,7 +1159,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if spectra == None: allspecs = "

%s produced no output for image %s

" % (taskname, casaimage) elif spectra.getNoPlot(): - allspecs = "

%s created output for image %s but was told to create no PNGs for display.

" % (taskname, casaimage) + allspecs = "

%s created output for image %s but was told to create no images for display.

" % (taskname, casaimage) else: count = 0 # task arguments are the same in all entries. @@ -1211,7 +1211,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if cubesum == None: allspecs = "

%s produced no output

" % (taskname) elif cubesum.getNoPlot(): - allspecs = "

%s created output but was told to create no PNGs for display.

" % (taskname) + allspecs = "

%s created output but was told to create no images for display.

" % (taskname) else: allspecs = "" taskargs = cubesum.taskargs @@ -1400,7 +1400,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if pvslices == None: specval = "

No PV slices were computed for this cube

" elif pvslices.getNoPlot(): - specval = "

%s created output but was told to create no PNGs for display.

" % (taskname) + specval = "

%s created output but was told to create no images for display.

" % (taskname) else: for val in pvslices.value: specval = STARTROW @@ -1464,7 +1464,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if the_item == None: specval = "

No PV correlation diagrams were computed from the input cube

" elif pvslices.getNoPlot(): - specval = "

%s created output but was told to create no PNGs for display.

" % (taskname) + specval = "

%s created output but was told to create no images for display.

" % (taskname) else: val = the_item.getValue() image = val[0] @@ -1521,7 +1521,9 @@ def _process(self,taskname,tid,titems,thetask,outdir): if tlower == "overlapintegral_at": the_item = titems.get('overlap',None) - if the_item != None: + if the_item == None: + tablestr = "

%s produced no output.

" % taskname + else: # summary info format: #[table,image,thumb,caption] summarydata = the_item.getValue() @@ -1533,21 +1535,25 @@ def _process(self,taskname,tid,titems,thetask,outdir): if len(atable) == 0: tablestr = "

%s produced no output.

" % taskname else: + tablestr = SPANXVAL % ("8",atable.html('class="table table-admit table-bordered table-striped"')) + if the_item.getNoPlot(): + imstr = "

%s was told not to create any images for display

" % taskname + else: image = summarydata[1] thumb = summarydata[2] caption = summarydata[3] - tablestr = SPANXVAL % ("8",atable.html('class="table table-admit table-bordered table-striped"')) imstr = SPAN4VAL % (image, thumb, caption, caption, caption) - tablestr = STARTROW + tablestr + imstr + ENDROW - else: - tablestr = "

%s produced no output.

" % taskname + tablestr = STARTROW + tablestr + imstr + ENDROW + retval = header % (taskclass, tid,thetask.statusicons(),taskname,tid,the_item.taskargs,tid,tablestr,tid) if tlower == "principalcomponent_at": # PCA returns two two tables in a list [[table1 h,u,d],[table2 h,u,d]]. tablestr = '' the_item = titems.get('pca',None) #SummaryEntry - if the_item != None: + if the_item == None: + tablestr = "

%s produced no output

" % taskname + else: summarydata = the_item.getValue() if summarydata == None or len(summarydata) == 0: tablestr = "

%s produced no output

" % taskname @@ -1559,9 +1565,9 @@ def _process(self,taskname,tid,titems,thetask,outdir): tablestr = tablestr + "

No covariance data available for this summary. Try lowering covarmin in the PrincipalComponent_AT task arguments.

" else: tablestr = tablestr + atable.html('class="table table-admit table-bordered table-striped"')+os.linesep+os.linesep + if the_item.getNoPlot(): + tablestr = tablestr + "

%s was told not to create images for display

" % taskname - else: - tablestr = "

%s produced no output

" % taskname retval = header % (taskclass, tid,thetask.statusicons(),taskname,tid,the_item.taskargs,tid,tablestr,tid) if tlower == "template_at": @@ -1590,7 +1596,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if spectra == None: allspecs = allspecs + "

%s produced no spectral output

" % (taskname) elif the_item.getNoPlot(): - allspecs = allspecs + "

%s created output but was told to create no PNGs for display.

" % (taskname) + allspecs = allspecs + "

%s created output but was told not to create images for display.

" % (taskname) else: count = 0 # task arguments are the same in all entries. @@ -1687,7 +1693,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if spectra == None: allspecs = "

%s produced no output

" % (taskname) elif spectra.getNoPlot(): - allspecs = "

%s created output but was told to create no PNGs for display.

" % (taskname) + allspecs = "

%s created output but was told not to create images for display.

" % (taskname) else: count = 0 # task arguments are the same in all entries. From 51506427bfc989f594bd8800e7035b7e66707f91 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 15:16:37 -0400 Subject: [PATCH 21/34] moment_at using noplot --- admit/Summary.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/admit/Summary.py b/admit/Summary.py index 7b4f997..805a30b 100755 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1358,8 +1358,12 @@ def _process(self,taskname,tid,titems,thetask,outdir): if tlower == "moment_at": moments = titems.get('moments',None) - allspecs = "" - if moments != None: + if moments == None: + allspecs = "

No moments were computed for this cube

" + elif moments.getNoPlot(): + allspecs = "

%s computed moments but was told not to create any images for display

" % taskname + else: + allspecs = "" count = 0 auximage = [] auxthumb = [] @@ -1379,19 +1383,20 @@ def _process(self,taskname,tid,titems,thetask,outdir): auximage.append(val[5]) auxthumb.append(val[6]) auxcaption.append(val[7]) + + # @todo Put the buttons on even if no PNGs were created casaimage = val[8] casamoment = image[:-4] # remove '.png' to get name of moment CASA format image button = utils.getButton(casamoment,"viewimage","View in CASA") # can't have two buttons with same html ID, so add ".fits" button2 = utils.getButton(casamoment+".fits","exportimage","Export to FITS") + specval = specval + (SPAN4VALB % ( image, thumb, caption, caption, caption, button,button2)) allspecs = allspecs + "\n" + specval count = count + 1 - banner = "

%s output for %s

" % (taskname, casaimage) - allspecs = banner + allspecs - else: - allspecs = "

No moments were computed for this cube

" + banner = "

%s output for %s

" % (taskname, casaimage) + allspecs = banner + allspecs retval = header % (taskclass, tid,thetask.statusicons(),taskname,tid,taskargs,tid,allspecs,tid) From 554f0fac9998ceddedff9da851ac804e9b404052 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 15:48:12 -0400 Subject: [PATCH 22/34] changing over to use new noplot member variable --- admit/at/Moment_AT.py | 81 +++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/admit/at/Moment_AT.py b/admit/at/Moment_AT.py index b08fdc2..7a8bd44 100644 --- a/admit/at/Moment_AT.py +++ b/admit/at/Moment_AT.py @@ -283,8 +283,12 @@ def run(self): # loop over moments to rename them to _0, _1, _2 etc. # apply a mask as well for proper histogram creation map = {} - myplot = APlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir()) - implot = ImPlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir()) + if self._plot_mode != PlotControl.NOPLOT: + myplot = APlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir()) + implot = ImPlot(pmode=self._plot_mode,ptype=self._plot_type,abspath=self.dir()) + noplot = False + else: + noplot = True for mom in moments: figname = imagename = "%s_%i" % (basename, mom) @@ -305,13 +309,20 @@ def run(self): dt.tag("makemask") if mom == 0: beamarea = nppb(self.dir(imagename)) - implot.plotter(rasterfile=imagename,figname=figname, - colorwedge=True,zoom=self.getkey("zoom")) - imagepng = implot.getFigure(figno=implot.figno,relative=True) - thumbname = implot.getThumbnail(figno=implot.figno,relative=True) - images = {bt.CASA : imagename, bt.PNG : imagepng} - thumbtype=bt.PNG - dt.tag("implot") + if self._plot_mode == PlotControl.NOPLOT: + figname = "not created" + imagepng = "not created" + thumbname = "not created" + imcaption = "not created" + images = {bt.CASA : imagename} + else: + implot.plotter(rasterfile=imagename,figname=figname, + colorwedge=True,zoom=self.getkey("zoom")) + imagepng = implot.getFigure(figno=implot.figno,relative=True) + thumbname = implot.getThumbnail(figno=implot.figno,relative=True) + images = {bt.CASA : imagename, bt.PNG : imagepng} + thumbtype=bt.PNG + dt.tag("implot") # get the data for a histogram (ia access is about 1000-2000 faster than imval()) map[mom] = casautil.getdata(self.dir(imagename)) @@ -327,21 +338,29 @@ def run(self): # Make the histogram plot # Since we give abspath in the constructor, figname should be relative - auxname = imagename + '_histo' - auxtype = bt.PNG - myplot.histogram(columns = data, - figname = auxname, - xlab = bunit, - ylab = "Count", - title = "Histogram of Moment %d: %s" % (mom, imagename), thumbnail=True) - - casaimage = Image(images = images, - auxiliary = auxname, - auxtype = auxtype, - thumbnail = thumbname, - thumbnailtype = thumbtype) - auxname = myplot.getFigure(figno=myplot.figno,relative=True) - auxthumb = myplot.getThumbnail(figno=myplot.figno,relative=True) + if self._plot_mode == PlotControl.NOPLOT: + auxname = "not created" + auxthumb = "not created" + auxcaption = "not created" + casaimage = Image(images = images) + else: + auxname = imagename + '_histo' + auxtype = bt.PNG + myplot.histogram(columns = data, + figname = auxname, + xlab = bunit, + ylab = "Count", + title = "Histogram of Moment %d: %s" % (mom, imagename), thumbnail=True) + auxname = myplot.getFigure(figno=myplot.figno,relative=True) + auxthumb = myplot.getThumbnail(figno=myplot.figno,relative=True) + imcaption = "%s Moment %d map of Source %s" % (line.name, mom, objectname) + auxcaption = "Histogram of %s Moment %d of Source %s" % (line.name, mom, objectname) + + casaimage = Image(images = images, + auxiliary = auxname, + auxtype = auxtype, + thumbnail = thumbname, + thumbnailtype = thumbtype) if hasattr(self._bdp_in[0], "line"): # SpwCube doesn't have Line line = deepcopy(getattr(self._bdp_in[0], "line")) @@ -355,8 +374,6 @@ def run(self): image=deepcopy(casaimage), line=line)) dt.tag("ren+mask_%d" % mom) - imcaption = "%s Moment %d map of Source %s" % (line.name, mom, objectname) - auxcaption = "Histogram of %s Moment %d of Source %s" % (line.name, mom, objectname) thismomentsummary = [line.name, mom, imagepng, thumbname, imcaption, auxname, auxthumb, auxcaption, infile] momentsummary.append(thismomentsummary) @@ -408,6 +425,9 @@ def run(self): # create a histogram of flux per channel + # NOTE THERE IS NO BDP ASSOCIATED WITH THIS! + # Run it anyway even if noplot==True. + # grab the X coordinates for the histogram, we want them in km/s # restfreq should also be in summary restfreq = casa.imhead(self.dir(infile),mode="get",hdkey="restfreq")['value']/1e9 # in GHz @@ -425,10 +445,11 @@ def run(self): # @todo make a flux1 with fluxes derived from a good mask flux1 = flux0 # construct histogram - title = 'Flux Spectrum (%g)' % flux0sum - xlab = 'VLSR (km/s)' - ylab = 'Flux (Jy)' - myplot.plotter(x,[flux0,flux1],title=title,figname=fluxname,xlab=xlab,ylab=ylab,histo=True) + if self._plot_mode != PlotControl.NOPLOT: + title = 'Flux Spectrum (%g)' % flux0sum + xlab = 'VLSR (km/s)' + ylab = 'Flux (Jy)' + myplot.plotter(x,[flux0,flux1],title=title,figname=fluxname,xlab=xlab,ylab=ylab,histo=True) dt.tag("flux-spectrum") self._summary["moments"] = SummaryEntry(momentsummary, "Moment_AT", From 0802ab0801d14279a1c3dc27351e106a2b95714f Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 16:33:16 -0400 Subject: [PATCH 23/34] changing over to use new noplot member variable --- admit/at/LineSegment_AT.py | 72 ++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/admit/at/LineSegment_AT.py b/admit/at/LineSegment_AT.py index e583703..732e8b9 100644 --- a/admit/at/LineSegment_AT.py +++ b/admit/at/LineSegment_AT.py @@ -210,7 +210,11 @@ def run(self): # instantiate a plotter for all plots made herein self._plot_type = admit.util.PlotControl.SVG - myplot = APlot(ptype=self._plot_type, pmode=self._plot_mode, abspath=self.dir()) + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + else: + noplot = False + myplot = APlot(ptype=self._plot_type, pmode=self._plot_mode, abspath=self.dir()) dt.tag("start") ############################################################################ @@ -331,49 +335,63 @@ def run(self): ylab=label[i], figname=imbase + "_statspec%i" % i, segments=freqs, cutoff= (spec.contin() + mult*(spec.noise() * self.getkey("numsigma"))), continuum=spec.contin(), thumbnail=True) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=caption[i]) - lsbdp.image.addimage(image, "statspec%i" % i) + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + # leave captions unchanged for now + else: + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=caption[i]) + lsbdp.image.addimage(image, "statspec%i" % i) spec_description.append([lsbdp.ra, lsbdp.dec, "", xlabel, imname, thumbnailname, caption[i], infile]) for i in range(len(specs)): freqs = [] + caption = "Detected line segments from input spectrum #%i." % (i) for ch in specseg[i]: frq = [min(specs[i].freq()[ch[0]], specs[i].freq()[ch[1]]), max(specs[i].freq()[ch[0]], specs[i].freq()[ch[1]])] freqs.append(frq) rdata.append(frq) - myplot.segplotter(specs[i].freq(), specs[i].spec(csub=False), - title="Detected Line Segments", xlab=xlabel, - ylab="Intensity", figname=imbase + "_spec%03d" % i, - segments=freqs, cutoff=specs[i].contin() + (specs[i].noise() * self.getkey("numsigma")), - continuum=specs[i].contin(), thumbnail=True) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, - relative=True) - caption = "Detected line segments from input spectrum #%i." % (i) - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=caption) - lsbdp.image.addimage(image, "spec%03d" % i) + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + else: + myplot.segplotter(specs[i].freq(), specs[i].spec(csub=False), + title="Detected Line Segments", xlab=xlabel, + ylab="Intensity", figname=imbase + "_spec%03d" % i, + segments=freqs, cutoff=specs[i].contin() + (specs[i].noise() * self.getkey("numsigma")), + continuum=specs[i].contin(), thumbnail=True) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, + relative=True) + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=caption) + lsbdp.image.addimage(image, "spec%03d" % i) + spec_description.append([lsbdp.ra, lsbdp.dec, "", xlabel, imname, thumbnailname, caption, infile]) caption = "Merged segments overlaid on CubeStats spectrum" + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + else: + myplot.summaryspec(statspec, specs, None, imbase + "_summary", llist) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) + caption = "Identified segments overlaid on Signal/Noise plot of all spectra." - myplot.summaryspec(statspec, specs, None, imbase + "_summary", llist) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) - caption = "Identified segments overlaid on Signal/Noise plot of all spectra." + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=caption) - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=caption) + lsbdp.image.addimage(image, "summary") - lsbdp.image.addimage(image, "summary") spec_description.append([lsbdp.ra, lsbdp.dec, "", "Signal/Noise", imname, thumbnailname, caption, infile]) @@ -381,10 +399,10 @@ def run(self): self._summary["segments"] = SummaryEntry(lsbdp.table.serialize(), "LineSegment_AT", - self.id(True), taskargs) + self.id(True), taskargs,noplot=noplot) self._summary["spectra"] = [SummaryEntry(spec_description, "LineSegment_AT", - self.id(True), taskargs)] + self.id(True), taskargs,noplot=noplot)] self.addoutput(lsbdp) logging.regression("LINESEG: %s" % str(rdata)) From 503508151f359478ec36a701386f6ab3de5aaba7 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 16:33:25 -0400 Subject: [PATCH 24/34] linesegment_at using noplot --- admit/Summary.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/admit/Summary.py b/admit/Summary.py index 805a30b..7601e8e 100755 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1646,7 +1646,11 @@ def _process(self,taskname,tid,titems,thetask,outdir): bigstr = '
' + STARTROW + tablestr + ENDROW spectra = titems.get('spectra',None) specval="" - if spectra != None: + if spectra == None: + allspecs = allspecs + "

%s created no spectral output.

" % (taskname) + elif the_item.getNoPlot(): + allspecs = allspecs + "

%s created spectral output but was told not to create images for display.

" % (taskname) + else: allspecs = '' count = 0 for val in spectra.value: @@ -1673,8 +1677,9 @@ def _process(self,taskname,tid,titems,thetask,outdir): allspecs = allspecs + "\n" + specval count = count + 1 - bigstr = bigstr + STARTROW + allspecs + ENDROW - retval = header % (taskclass, tid, thetask.statusicons(),taskname, tid, the_item.taskargs, tid, bigstr, tid) + + bigstr = bigstr + STARTROW + allspecs + ENDROW + retval = header % (taskclass, tid, thetask.statusicons(),taskname, tid, the_item.taskargs, tid, bigstr, tid) if tlower == "bdpingest_at": the_item = titems.get('bdpingest',None) #SummaryEntry From eba7ff6da4ac8e52ed8a7a3478ad7224630be771 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 16:41:56 -0400 Subject: [PATCH 25/34] lineid_at using noplot --- admit/Summary.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/admit/Summary.py b/admit/Summary.py index 7601e8e..189d364 100755 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1020,8 +1020,8 @@ def html(self,outdir,flowmanager,dotdiagram="",editor=True): f.close() def _imageIsSVG(self,image): - if not image: return False - + if not image: return False + if image[len(image)-3:len(image)+1] == 'svg': return True else: @@ -1326,6 +1326,8 @@ def _process(self,taskname,tid,titems,thetask,outdir): spectra = titems.get('spectra',None) if spectra == None: retval = header % (taskclass, tid, thetask.statusicons(),taskname, tid, the_item.taskargs, tid, bigstr, tid) + elif spectra.getNoPlot(): + allspecs = "

%s identified spectral lines but was told not to create images for display.

" % taskname else: allspecs = '' count = 0 @@ -1353,8 +1355,10 @@ def _process(self,taskname,tid,titems,thetask,outdir): specval = specval + (SPAN4VAL % ( image, thumb, caption, caption, caption)) allspecs = allspecs + "\n" + specval count = count + 1 - bigstr = bigstr + STARTROW + allspecs + ENDROW - retval = header % (taskclass, tid, thetask.statusicons(),taskname, tid, the_item.taskargs, tid, tid, bigstr, tid) + + bigstr = bigstr + STARTROW + allspecs + ENDROW + + retval = header % (taskclass, tid, thetask.statusicons(),taskname, tid, the_item.taskargs, tid, tid, bigstr, tid) if tlower == "moment_at": moments = titems.get('moments',None) From 046e2d4fffea4359b780a96f88f13a7b3455d9b4 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Wed, 19 Sep 2018 16:42:03 -0400 Subject: [PATCH 26/34] changing over to use new noplot member variable --- admit/at/LineID_AT.py | 93 ++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/admit/at/LineID_AT.py b/admit/at/LineID_AT.py index 88856ef..a3297e4 100644 --- a/admit/at/LineID_AT.py +++ b/admit/at/LineID_AT.py @@ -3101,7 +3101,11 @@ def run(self): self._plot_type = admit.util.PlotControl.SVG # instantiate a plotter for all plots made herein - myplot = APlot(ptype=self._plot_type, pmode=self._plot_mode, abspath=self.dir()) + if self._plot_mode != PlotControl.NOPLOT: + noplot = False + myplot = APlot(ptype=self._plot_type, pmode=self._plot_mode, abspath=self.dir()) + else: + noplot = True ############################################################################ # Smoothing and continuum (baseline) subtraction of input spectra # @@ -3254,65 +3258,80 @@ def run(self): if i == 1: mult = -1. # print("MWP plot cutoff[%d] = %f, contin=%f" % (i, (spec.contin() + mult*(spec.noise() * self.getkey("numsigma")))[0], spec.contin()[0] ) ) - myplot.segplotter(x=spec.freq(), y=spec.spec(csub=False), - title="Potential Line Locations", xlab=xlabel, - ylab=label[i], figname=imbase + "_statspec%i" % i, segments=freqs, - cutoff=(spec.contin() + mult * (spec.noise() * self.getkey("numsigma"))), - continuum=spec.contin(), thumbnail=True) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=caption[i]) - llbdp.image.addimage(image, "statspec%i" % i) + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + # leave captions unchanged for now + else: + myplot.segplotter(x=spec.freq(), y=spec.spec(csub=False), + title="Potential Line Locations", xlab=xlabel, + ylab=label[i], figname=imbase + "_statspec%i" % i, segments=freqs, + cutoff=(spec.contin() + mult * (spec.noise() * self.getkey("numsigma"))), + continuum=spec.contin(), thumbnail=True) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=caption[i]) + llbdp.image.addimage(image, "statspec%i" % i) self.spec_description.append([llbdp.ra, llbdp.dec, "", xlabel, imname, thumbnailname, caption[i], self.infile]) for i, spec in enumerate(self.specs): freqs = [] + _caption = "Potential lines overlaid on input spectrum #%i." % (i) for ch in self.specseg[i]: freqs.append([min(spec.freq()[ch[0]], spec.freq()[ch[1]]), max(spec.freq()[ch[0]], spec.freq()[ch[1]])]) - myplot.segplotter(x=spec.freq(), y=spec.spec(csub=False), - title="Potential Line Locations", xlab=xlabel, - ylab="Intensity", figname=imbase + "_spec%03d" % i, segments=freqs, - cutoff=spec.contin() + (spec.noise() * self.getkey("numsigma")), - continuum=spec.contin(), thumbnail=True) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, - relative=True) - _caption = "Potential lines overlaid on input spectrum #%i." % (i) - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=_caption) - llbdp.image.addimage(image, "spec%03d" % i) + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + else: + myplot.segplotter(x=spec.freq(), y=spec.spec(csub=False), + title="Potential Line Locations", xlab=xlabel, + ylab="Intensity", figname=imbase + "_spec%03d" % i, segments=freqs, + cutoff=spec.contin() + (spec.noise() * self.getkey("numsigma")), + continuum=spec.contin(), thumbnail=True) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, + relative=True) + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=_caption) + llbdp.image.addimage(image, "spec%03d" % i) + self.spec_description.append([llbdp.ra, llbdp.dec, "", xlabel, imname, thumbnailname, _caption, self.infile]) if self.pvspec is not None: freqs = [] + _caption = "Potential lines overlaid on Correlation plot from PVCorr_BDP." for ch in self.pvseg: freqs.append([min(self.pvspec.freq()[ch[0]], self.pvspec.freq()[ch[1]]), max(self.pvspec.freq()[ch[0]], self.pvspec.freq()[ch[1]])]) - myplot.segplotter(x=self.pvspec.freq(), y=self.pvspec.spec(csub=False), - title="Potential Line Locations", xlab=xlabel, - ylab="Corr. Coef.", figname=imbase + "_pvspec", - segments=freqs, cutoff=self.pvspec.noise() * self.getkey("numsigma"), - thumbnail=True) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, - relative=True) - _caption = "Potential lines overlaid on Correlation plot from PVCorr_BDP." - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=_caption) - llbdp.image.addimage(image, "pvspec") + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + else: + myplot.segplotter(x=self.pvspec.freq(), y=self.pvspec.spec(csub=False), + title="Potential Line Locations", xlab=xlabel, + ylab="Corr. Coef.", figname=imbase + "_pvspec", + segments=freqs, cutoff=self.pvspec.noise() * self.getkey("numsigma"), + thumbnail=True) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, + relative=True) + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=_caption) + llbdp.image.addimage(image, "pvspec") + self.spec_description.append([llbdp.ra, llbdp.dec, "", xlabel, imname, thumbnailname, _caption, self.infile]) self._summary["linelist"] = SummaryEntry(llbdp.table.serialize(), "LineID_AT", - self.id(True), taskargs) + self.id(True), taskargs,noplot=noplot) self._summary["spectra"] = [SummaryEntry(self.spec_description, "LineID_AT", - self.id(True), taskargs)] + self.id(True), taskargs,noplot=noplot)] self.addoutput(llbdp) self.dt.tag("done") From d45071ec06d327ebda4adbc7bf54e1aff0b6fedb Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Thu, 20 Sep 2018 14:24:52 -0400 Subject: [PATCH 27/34] small bug fixes after runa1 testing --- admit/Summary.py | 2 +- admit/at/BDPIngest_AT.py | 1 + admit/at/ContinuumSub_AT.py | 1 + admit/at/CubeSpectrum_AT.py | 1 + admit/at/CubeStats_AT.py | 1 + admit/at/CubeSum_AT.py | 1 + admit/at/Export_AT.py | 1 + admit/at/FeatureList_AT.py | 1 + admit/at/GenerateSpectrum_AT.py | 1 + admit/at/Ingest_AT.py | 1 + admit/at/LineCube_AT.py | 1 + admit/at/LineID_AT.py | 1 + admit/at/LineSegment_AT.py | 11 ++++++----- admit/at/Moment_AT.py | 16 +++++++++------- admit/at/OverlapIntegral_AT.py | 1 + admit/at/PVCorr_AT.py | 9 ++++++--- admit/at/PVSlice_AT.py | 1 + admit/at/PrincipalComponent_AT.py | 5 +++-- admit/at/Regrid_AT.py | 1 + admit/at/SFind2D_AT.py | 1 + admit/at/Smooth_AT.py | 1 + admit/at/Template_AT.py | 7 ++++--- 22 files changed, 45 insertions(+), 21 deletions(-) diff --git a/admit/Summary.py b/admit/Summary.py index 189d364..6677784 100755 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1472,7 +1472,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): the_item = titems.get('pvcorr',None) if the_item == None: specval = "

No PV correlation diagrams were computed from the input cube

" - elif pvslices.getNoPlot(): + elif the_item.getNoPlot(): specval = "

%s created output but was told to create no images for display.

" % (taskname) else: val = the_item.getValue() diff --git a/admit/at/BDPIngest_AT.py b/admit/at/BDPIngest_AT.py index f7eb3e2..831208b 100644 --- a/admit/at/BDPIngest_AT.py +++ b/admit/at/BDPIngest_AT.py @@ -12,6 +12,7 @@ import admit.util.utils as utils from admit.util.Table import Table from admit.Summary import SummaryEntry +#import admit.util.PlotControl as PlotControl class BDPIngest_AT(AT): diff --git a/admit/at/ContinuumSub_AT.py b/admit/at/ContinuumSub_AT.py index ce62766..1a6a2aa 100644 --- a/admit/at/ContinuumSub_AT.py +++ b/admit/at/ContinuumSub_AT.py @@ -18,6 +18,7 @@ from admit.bdp.LineList_BDP import LineList_BDP from admit.bdp.LineSegment_BDP import LineSegment_BDP import admit.util.utils as utils +import admit.util.PlotControl as PlotControl import admit.util.filter.Filter1D as Filter1D from admit.util.AdmitLogging import AdmitLogging as logging import numpy as np diff --git a/admit/at/CubeSpectrum_AT.py b/admit/at/CubeSpectrum_AT.py index 603989f..02a671e 100644 --- a/admit/at/CubeSpectrum_AT.py +++ b/admit/at/CubeSpectrum_AT.py @@ -20,6 +20,7 @@ import admit.util.Image as Image from admit.util import APlot import admit.util.utils as utils +import admit.util.PlotControl as PlotControl from admit.util.AdmitLogging import AdmitLogging as logging from copy import deepcopy diff --git a/admit/at/CubeStats_AT.py b/admit/at/CubeStats_AT.py index 18196aa..7352fff 100644 --- a/admit/at/CubeStats_AT.py +++ b/admit/at/CubeStats_AT.py @@ -19,6 +19,7 @@ from admit.util.segmentfinder import ADMITSegmentFinder from admit.Summary import SummaryEntry import admit.util.casautil as casautil +import admit.util.PlotControl as PlotControl from admit.util.AdmitLogging import AdmitLogging as logging from copy import deepcopy diff --git a/admit/at/CubeSum_AT.py b/admit/at/CubeSum_AT.py index b72379a..1bc6c82 100644 --- a/admit/at/CubeSum_AT.py +++ b/admit/at/CubeSum_AT.py @@ -19,6 +19,7 @@ from admit.bdp.LineList_BDP import LineList_BDP from admit.bdp.Moment_BDP import Moment_BDP import admit.util.utils as utils +import admit.util.PlotControl as PlotControl import admit.util.filter.Filter1D as Filter1D from admit.util.AdmitLogging import AdmitLogging as logging import numpy as np diff --git a/admit/at/Export_AT.py b/admit/at/Export_AT.py index b340abb..0d120c7 100644 --- a/admit/at/Export_AT.py +++ b/admit/at/Export_AT.py @@ -17,6 +17,7 @@ from admit.bdp.LineList_BDP import LineList_BDP from admit.bdp.Moment_BDP import Moment_BDP import admit.util.utils as utils +import admit.util.PlotControl as PlotControl import admit.util.filter.Filter1D as Filter1D from admit.util.AdmitLogging import AdmitLogging as logging import numpy as np diff --git a/admit/at/FeatureList_AT.py b/admit/at/FeatureList_AT.py index 5b1e60a..1046608 100644 --- a/admit/at/FeatureList_AT.py +++ b/admit/at/FeatureList_AT.py @@ -11,6 +11,7 @@ import admit.util.bdp_types as bt from admit.util.Image import Image from admit.util.utils import Dtime +import admit.util.PlotControl as PlotControl from admit.bdp.CubeStats_BDP import CubeStats_BDP from admit.bdp.Image_BDP import Image_BDP diff --git a/admit/at/GenerateSpectrum_AT.py b/admit/at/GenerateSpectrum_AT.py index f4f7458..7c7af5e 100644 --- a/admit/at/GenerateSpectrum_AT.py +++ b/admit/at/GenerateSpectrum_AT.py @@ -15,6 +15,7 @@ from admit.util import APlot import admit.util.Image as Image from admit.util import SpectralLineSearch +import admit.util.PlotControl as PlotControl from admit.Summary import SummaryEntry import os diff --git a/admit/at/Ingest_AT.py b/admit/at/Ingest_AT.py index 579f255..fa4dc34 100644 --- a/admit/at/Ingest_AT.py +++ b/admit/at/Ingest_AT.py @@ -15,6 +15,7 @@ import admit.util.utils as utils import admit.util.casautil as casautil import admit.util.ImPlot as ImPlot +import admit.util.PlotControl as PlotControl from admit.bdp.SpwCube_BDP import SpwCube_BDP from admit.bdp.Image_BDP import Image_BDP from admit.util.AdmitLogging import AdmitLogging as logging diff --git a/admit/at/LineCube_AT.py b/admit/at/LineCube_AT.py index fdfdb27..4977953 100644 --- a/admit/at/LineCube_AT.py +++ b/admit/at/LineCube_AT.py @@ -16,6 +16,7 @@ from admit.util.Image import Image import admit.util.Table import admit.util.utils as utils +import admit.util.PlotControl as PlotControl from admit.util.AdmitLogging import AdmitLogging as logging diff --git a/admit/at/LineID_AT.py b/admit/at/LineID_AT.py index a3297e4..ffb2e37 100644 --- a/admit/at/LineID_AT.py +++ b/admit/at/LineID_AT.py @@ -22,6 +22,7 @@ from admit.bdp.CubeSpectrum_BDP import CubeSpectrum_BDP from admit.bdp.CubeStats_BDP import CubeStats_BDP from admit.bdp.PVCorr_BDP import PVCorr_BDP +import admit.util.PlotControl as PlotControl import admit.util.filter.Filter1D as Filter1D from admit.util import APlot from admit.util.Image import Image diff --git a/admit/at/LineSegment_AT.py b/admit/at/LineSegment_AT.py index 732e8b9..0ca84a4 100644 --- a/admit/at/LineSegment_AT.py +++ b/admit/at/LineSegment_AT.py @@ -18,6 +18,7 @@ from admit.bdp.LineSegment_BDP import LineSegment_BDP from admit.bdp.CubeSpectrum_BDP import CubeSpectrum_BDP from admit.bdp.CubeStats_BDP import CubeStats_BDP +import admit.util.PlotControl as PlotControl from admit.util import APlot from admit.util.Image import Image from admit.util.AdmitLogging import AdmitLogging as logging @@ -330,16 +331,16 @@ def run(self): if i == 1: mult = -1. # print("MWP statspec plot cutoff[%d] = %f, contin=%f" % (i, (statspec[i].contin() + mult*(statspec[i].noise() * self.getkey("numsigma")))[0], statspec[i].contin()[0] ) ) - myplot.segplotter(spec.freq(), spec.spec(csub=False), - title="Detected Line Segments", xlab=xlabel, - ylab=label[i], figname=imbase + "_statspec%i" % i, - segments=freqs, cutoff= (spec.contin() + mult*(spec.noise() * self.getkey("numsigma"))), - continuum=spec.contin(), thumbnail=True) if self._plot_mode == PlotControl.NOPLOT: imname = "not created" thumbnailname = "not created" # leave captions unchanged for now else: + myplot.segplotter(spec.freq(), spec.spec(csub=False), + title="Detected Line Segments", xlab=xlabel, + ylab=label[i], figname=imbase + "_statspec%i" % i, + segments=freqs, cutoff= (spec.contin() + mult*(spec.noise() * self.getkey("numsigma"))), + continuum=spec.contin(), thumbnail=True) imname = myplot.getFigure(figno=myplot.figno, relative=True) thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, diff --git a/admit/at/Moment_AT.py b/admit/at/Moment_AT.py index 7a8bd44..abbfb2d 100644 --- a/admit/at/Moment_AT.py +++ b/admit/at/Moment_AT.py @@ -16,6 +16,7 @@ from admit.bdp.CubeStats_BDP import CubeStats_BDP import admit.util.Image as Image import admit.util.Line as Line +import admit.util.PlotControl as PlotControl import admit.util.ImPlot as ImPlot import admit.util.utils as utils import admit.util.casautil as casautil @@ -336,6 +337,14 @@ def run(self): # object for the caption objectname = casa.imhead(imagename=self.dir(imagename), mode="get", hdkey="object") + if hasattr(self._bdp_in[0], "line"): # SpwCube doesn't have Line + line = deepcopy(getattr(self._bdp_in[0], "line")) + if not isinstance(line, Line): + line = Line(name="Unidentified") + else: + # fake a Line if there wasn't one + line = Line(name="Unidentified") + # Make the histogram plot # Since we give abspath in the constructor, figname should be relative if self._plot_mode == PlotControl.NOPLOT: @@ -362,13 +371,6 @@ def run(self): thumbnail = thumbname, thumbnailtype = thumbtype) - if hasattr(self._bdp_in[0], "line"): # SpwCube doesn't have Line - line = deepcopy(getattr(self._bdp_in[0], "line")) - if not isinstance(line, Line): - line = Line(name="Unidentified") - else: - # fake a Line if there wasn't one - line = Line(name="Unidentified") # add the BDP to the output array self.addoutput(Moment_BDP(xmlFile=imagename, moment=mom, image=deepcopy(casaimage), line=line)) diff --git a/admit/at/OverlapIntegral_AT.py b/admit/at/OverlapIntegral_AT.py index f9fba58..f35cde0 100644 --- a/admit/at/OverlapIntegral_AT.py +++ b/admit/at/OverlapIntegral_AT.py @@ -21,6 +21,7 @@ import admit.util.utils as utils import admit.util.APlot as APlot import admit.util.Line as Line +import admit.util.PlotControl as PlotControl import admit.util.ImPlot as ImPlot import admit.util.Table from admit.bdp.Image_BDP import Image_BDP diff --git a/admit/at/PVCorr_AT.py b/admit/at/PVCorr_AT.py index b958c44..13191b2 100644 --- a/admit/at/PVCorr_AT.py +++ b/admit/at/PVCorr_AT.py @@ -17,6 +17,7 @@ from admit.bdp.CubeStats_BDP import CubeStats_BDP from admit.bdp.PVCorr_BDP import PVCorr_BDP from admit.bdp.Image_BDP import Image_BDP +import admit.util.PlotControl as PlotControl from admit.util import APlot from admit.util import stats from admit.util.AdmitLogging import AdmitLogging as logging @@ -261,17 +262,19 @@ def run(self): taskargs = "numsigma=%.1f range=[%d,%d]" % (numsigma,ch0,ch1) if self._plot_mode == PlotControl.NOPLOT: + noplot = True thumbname = "not created" figname = "not created" imcaption = "not created" image = Image(description=imcaption) else: + noplot = False self.myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) self.myplot.plotter(x,y,title,figname=p1,xlab=xlab,ylab=ylab,segments=segp, thumbnail=True) - #out1 = np.rot90 (data.reshape((nvel,npos)) ) - if mode > 1: - self.myplot.map1(data=out,title="testing PVCorr_AT: mode%d"%mode,figname='testPVCorr', thumbnail=True) + #out1 = np.rot90 (data.reshape((nvel,npos)) ) + if mode > 1: + self.myplot.map1(data=out,title="testing PVCorr_AT: mode%d"%mode,figname='testPVCorr', thumbnail=True) imcaption = "Position-velocity correlation plot" thumbname = self.myplot.getThumbnail(figno=self.myplot.figno,relative=True) diff --git a/admit/at/PVSlice_AT.py b/admit/at/PVSlice_AT.py index ba323a9..ab8c4af 100644 --- a/admit/at/PVSlice_AT.py +++ b/admit/at/PVSlice_AT.py @@ -15,6 +15,7 @@ from admit.util.Image import Image +import admit.util.PlotControl as PlotControl from admit.util import APlot from admit.util import utils from admit.util import stats diff --git a/admit/at/PrincipalComponent_AT.py b/admit/at/PrincipalComponent_AT.py index 0fbb93a..e3bc060 100644 --- a/admit/at/PrincipalComponent_AT.py +++ b/admit/at/PrincipalComponent_AT.py @@ -9,6 +9,7 @@ import admit import admit.util.bdp_types as bt +import admit.util.PlotControl as PlotControl class PrincipalComponent_AT(admit.Task): """ @@ -254,10 +255,10 @@ def run(self): admit.casautil.putdata_raw(self.baseDir()+ofile+".im", img, ifile) oimg = admit.Image() oimg.addimage(admit.imagedescriptor(ofile+".im", format=bt.CASA)) - if self._plot_mode != admit.PlotControl.NOPLOT: + if self._plot_mode != PlotControl.NOPLOT: noplot = False aplot = admit.util.APlot(figno=inum, abspath=self.baseDir(), - ptype=admit.util.PlotControl.PNG) + ptype=PlotControl.PNG) aplot.map1(np.rot90(img), title=ofile, figname=ofile) aplot.final() oimg.addimage(admit.imagedescriptor(ofile+".png", format=bt.PNG)) diff --git a/admit/at/Regrid_AT.py b/admit/at/Regrid_AT.py index cb7d181..e715347 100644 --- a/admit/at/Regrid_AT.py +++ b/admit/at/Regrid_AT.py @@ -11,6 +11,7 @@ import admit.util.bdp_types as bt import admit.util.Table import admit.util.utils as utils +import admit.util.PlotControl as PlotControl from admit.bdp.SpwCube_BDP import SpwCube_BDP import numpy as np import os as os diff --git a/admit/at/SFind2D_AT.py b/admit/at/SFind2D_AT.py index 68b326f..ff099cf 100644 --- a/admit/at/SFind2D_AT.py +++ b/admit/at/SFind2D_AT.py @@ -13,6 +13,7 @@ import admit.util.casautil as casautil import admit.util.Image as Image import admit.util.Line as Line +import admit.util.PlotControl as PlotControl import admit.util.ImPlot as ImPlot from admit.bdp.Image_BDP import Image_BDP from admit.bdp.CubeStats_BDP import CubeStats_BDP diff --git a/admit/at/Smooth_AT.py b/admit/at/Smooth_AT.py index b41e032..8bb4ae3 100644 --- a/admit/at/Smooth_AT.py +++ b/admit/at/Smooth_AT.py @@ -10,6 +10,7 @@ import admit.util.bdp_types as bt import admit.util.Image as Image import admit.util.utils as utils +import admit.util.PlotControl as PlotControl import admit.util.Line as Line from admit.bdp.SpwCube_BDP import SpwCube_BDP from admit.util.AdmitLogging import AdmitLogging as logging diff --git a/admit/at/Template_AT.py b/admit/at/Template_AT.py index 2167f3c..e9c79a2 100644 --- a/admit/at/Template_AT.py +++ b/admit/at/Template_AT.py @@ -10,6 +10,7 @@ import admit import admit.util.bdp_types as bt +import admit.util.PlotControl as PlotControl class Template_AT(admit.Task): """ @@ -211,8 +212,8 @@ def run(self): # Image data must be rotated to match matplotlib axis order. oimgtitle="Template Image Slice @ channel %d" % imgslice aplot = admit.APlot(figno=1, abspath=self.baseDir(), - pmode=admit.PlotControl.BATCH, - ptype=admit.PlotControl.PNG) + pmode=PlotControl.BATCH, + ptype=PlotControl.PNG) aplot.map1(np.rot90(oimg), xlab="R.A. (pixels)", ylab="Dec (pixels)", title=oimgtitle, figname=oimgstem) @@ -242,7 +243,7 @@ def run(self): # Create a PNG plot (standalone). ospectitle = "Template Spectrum @ position %s" % str(specpos) aplot = admit.APlot(figno=2, abspath=self.baseDir(), - pmode=admit.PlotControl.BATCH, + pmode=PlotControl.BATCH, ptype=admit.PlotControl.PNG) aplot.plotter(x=ochan, y=[ospec], xlab="Channel", ylab="Intensity", From acf32ecf9fea3be2d65bf1f532ca7d295cb499b3 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Thu, 20 Sep 2018 17:13:04 -0400 Subject: [PATCH 28/34] more bugfixes from runa1 --- admit/Summary.py | 4 +- admit/at/CubeSum_AT.py | 1 + admit/at/LineID_AT.py | 203 ++++++++++++++++++++++++----------------- 3 files changed, 122 insertions(+), 86 deletions(-) diff --git a/admit/Summary.py b/admit/Summary.py index 6677784..5cd31ec 100755 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1159,7 +1159,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if spectra == None: allspecs = "

%s produced no output for image %s

" % (taskname, casaimage) elif spectra.getNoPlot(): - allspecs = "

%s created output for image %s but was told to create no images for display.

" % (taskname, casaimage) + allspecs = "

%s created output but was told to create no images for display.

" % (taskname) else: count = 0 # task arguments are the same in all entries. @@ -1650,12 +1650,12 @@ def _process(self,taskname,tid,titems,thetask,outdir): bigstr = '
' + STARTROW + tablestr + ENDROW spectra = titems.get('spectra',None) specval="" + allspecs = '' if spectra == None: allspecs = allspecs + "

%s created no spectral output.

" % (taskname) elif the_item.getNoPlot(): allspecs = allspecs + "

%s created spectral output but was told not to create images for display.

" % (taskname) else: - allspecs = '' count = 0 for val in spectra.value: # default bootstrap width is 12 columns. We are using 'span4' so diff --git a/admit/at/CubeSum_AT.py b/admit/at/CubeSum_AT.py index 1bc6c82..74650a6 100644 --- a/admit/at/CubeSum_AT.py +++ b/admit/at/CubeSum_AT.py @@ -352,6 +352,7 @@ def run(self): figname = "not created" thumbname = "not created" imcaption = "not created" + auxname = "not created" auxcaption = "not created" auxthumb = "not created" images = {bt.CASA : bdp_name} diff --git a/admit/at/LineID_AT.py b/admit/at/LineID_AT.py index ffb2e37..d5282f7 100644 --- a/admit/at/LineID_AT.py +++ b/admit/at/LineID_AT.py @@ -3482,54 +3482,69 @@ def run(self): mult = 1. if i == 1: mult = -1. - myplot.makespec(x=spec.freq(), y=spec.spec(csub=False), chan=spec.chans(), - cutoff=(spec.contin() + mult * (spec.noise() * self.getkey("numsigma"))), - figname=imbase +"_statspec%i" % i, title="Line ID (vlsr=%.2f)" % self.vlsr, - xlabel=xlabel, lines={}, force=self.force, blends=[], - continuum=spec.contin(), ylabel=label[i], - thumbnail=True, references=line_ref) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + # leave captions unchanged for now + else: + myplot.makespec(x=spec.freq(), y=spec.spec(csub=False), chan=spec.chans(), + cutoff=(spec.contin() + mult * (spec.noise() * self.getkey("numsigma"))), + figname=imbase +"_statspec%i" % i, title="Line ID (vlsr=%.2f)" % self.vlsr, + xlabel=xlabel, lines={}, force=self.force, blends=[], + continuum=spec.contin(), ylabel=label[i], + thumbnail=True, references=line_ref) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=caption[i]) - llbdp.image.addimage(image, "statspec%i" % i) + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=caption[i]) + llbdp.image.addimage(image, "statspec%i" % i) self.spec_description.append([llbdp.ra, llbdp.dec, "", xlabel, imname, thumbnailname, caption[i], self.infile]) # cubespec output (1 for each input spectra, there could be many from a single BDP) for i, spec in enumerate(self.specs): - myplot.makespec(x=spec.freq(), y=spec.spec(csub=False), chan=spec.chans(), - cutoff=spec.contin() + spnoise[i], - figname=imbase +"_spec%03d" % i, - title="Line ID (vlsr=%.2f)" % self.vlsr, xlabel=xlabel, - lines={}, force=self.force, blends=[], - continuum=spec.contin(), thumbnail=True, - references=line_ref) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, - relative=True) _caption = "Identified lines overlaid on input spectrum #%i." % (i) - image = Image(images={bt.SVG: imname}, - thumbnail=thumbnailname, thumbnailtype=bt.PNG, - description=_caption) - llbdp.image.addimage(image, "spec%03d" % i) + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + # leave captions unchanged for now + else: + myplot.makespec(x=spec.freq(), y=spec.spec(csub=False), chan=spec.chans(), + cutoff=spec.contin() + spnoise[i], + figname=imbase +"_spec%03d" % i, + title="Line ID (vlsr=%.2f)" % self.vlsr, xlabel=xlabel, + lines={}, force=self.force, blends=[], + continuum=spec.contin(), thumbnail=True, + references=line_ref) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, + relative=True) + image = Image(images={bt.SVG: imname}, + thumbnail=thumbnailname, thumbnailtype=bt.PNG, + description=_caption) + llbdp.image.addimage(image, "spec%03d" % i) self.spec_description.append([llbdp.ra, llbdp.dec, "", xlabel, imname, thumbnailname, _caption, self.infile]) if self.pvspec is not None: - myplot.makespec(x=self.pvspec.freq(), y=self.pvspec.spec(csub=False), chan=self.pvspec.chans(), - cutoff=self.pvcutoff, - figname=imbase + "_pvspec", title="Line ID (vlsr=%.2f)" % self.vlsr, - xlabel=xlabel, lines={}, force=self.force, blends=[], - continuum=[0.0] * len(self.pvspec), ylabel="Corr. Coeff.", - thumbnail=True, references=line_ref) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) - _caption = "Identified lines overlaid on Correlation Coefficient plot from PVCorr_BDP." - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=_caption) - llbdp.image.addimage(image, "pvspec") + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + # leave captions unchanged for now + else: + myplot.makespec(x=self.pvspec.freq(), y=self.pvspec.spec(csub=False), chan=self.pvspec.chans(), + cutoff=self.pvcutoff, + figname=imbase + "_pvspec", title="Line ID (vlsr=%.2f)" % self.vlsr, + xlabel=xlabel, lines={}, force=self.force, blends=[], + continuum=[0.0] * len(self.pvspec), ylabel="Corr. Coeff.", + thumbnail=True, references=line_ref) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) + + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=_caption) + llbdp.image.addimage(image, "pvspec") self.spec_description.append([llbdp.ra, llbdp.dec, "", xlabel, imname, thumbnailname, _caption, self.infile]) @@ -4008,21 +4023,26 @@ def run(self): mult = 1. if i == 1: mult = -1. - myplot.makespec(x=self.statspec[i].freq(), y=self.statspec[i].spec(csub=False), - chan=self.statspec[i].chans(), - cutoff=(self.statspec[i].contin() + mult * (self.statspec[i].noise() * - self.getkey("numsigma"))), - figname=imbase + "_statspec%i" % i, title="Line ID (vlsr=%.2f)" % self.vlsr, - xlabel=xlabel, lines=mlist, force=self.force, - blends=peaks["stats"][i].blends, continuum=self.statspec[i].contin(), - ylabel=label[i], thumbnail=True, references=line_ref) - imname = myplot.getFigure(figno=myplot.figno, relative=True) + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + # leave captions unchanged for now + else: + myplot.makespec(x=self.statspec[i].freq(), y=self.statspec[i].spec(csub=False), + chan=self.statspec[i].chans(), + cutoff=(self.statspec[i].contin() + mult * (self.statspec[i].noise() * + self.getkey("numsigma"))), + figname=imbase + "_statspec%i" % i, title="Line ID (vlsr=%.2f)" % self.vlsr, + xlabel=xlabel, lines=mlist, force=self.force, + blends=peaks["stats"][i].blends, continuum=self.statspec[i].contin(), + ylabel=label[i], thumbnail=True, references=line_ref) + imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=caption[i]) - llbdp.image.addimage(image, "statspec%i" % i) + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=caption[i]) + llbdp.image.addimage(image, "statspec%i" % i) self.spec_description.append([llbdp.ra, llbdp.dec, "", xlabel, imname, thumbnailname, caption[i], self.infile]) @@ -4066,22 +4086,27 @@ def run(self): mlist += ulist xlabel = "%s Frequency (GHz)" % (t) - myplot.makespec(x=self.specs[i].freq(), y=self.specs[i].spec(csub=False), - chan=self.specs[i].chans(), - cutoff=self.specs[i].contin() + (self.specs[i].noise() * - self.getkey("numsigma")), - figname=imbase + "_spec%03d" % i, - title="Line ID (vlsr=%.2f)" % self.vlsr, xlabel=xlabel, lines=mlist, - force=self.force, blends=peaks["specs"][i].blends, - continuum=self.specs[i].contin(), thumbnail=True, references=line_ref) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, - relative=True) _caption = "Identified lines overlaid on input spectrum #%i." % (i) + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + # leave captions unchanged for now + else: + myplot.makespec(x=self.specs[i].freq(), y=self.specs[i].spec(csub=False), + chan=self.specs[i].chans(), + cutoff=self.specs[i].contin() + (self.specs[i].noise() * + self.getkey("numsigma")), + figname=imbase + "_spec%03d" % i, + title="Line ID (vlsr=%.2f)" % self.vlsr, xlabel=xlabel, lines=mlist, + force=self.force, blends=peaks["specs"][i].blends, + continuum=self.specs[i].contin(), thumbnail=True, references=line_ref) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, + relative=True) - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=_caption) - llbdp.image.addimage(image, "spec%03d" % i) + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=_caption) + llbdp.image.addimage(image, "spec%03d" % i) self.spec_description.append([llbdp.ra, llbdp.dec, "", xlabel, imname, thumbnailname, _caption, self.infile]) @@ -4124,19 +4149,24 @@ def run(self): xlabel = "%s Frequency (GHz)" % (t) - myplot.makespec(x=self.pvspec.freq(), y=self.pvspec.spec(csub=False), chan=self.pvspec.chans(), - cutoff=self.pvspec.noise() * self.getkey("numsigma"), - figname=imbase + "_pvspec", title="Line ID (vlsr=%.2f)" % self.vlsr, - xlabel=xlabel, lines=mlist, force=self.force, blends=peaks["pvc"].blends, - continuum=[0.0] * len(self.pvspec), ylabel="Correlation", - thumbnail=True, references=line_ref) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) _caption = "Identified lines overlaid on correlation coefficient plot from PVCorr_BDP." + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + # leave captions unchanged for now + else: + myplot.makespec(x=self.pvspec.freq(), y=self.pvspec.spec(csub=False), chan=self.pvspec.chans(), + cutoff=self.pvspec.noise() * self.getkey("numsigma"), + figname=imbase + "_pvspec", title="Line ID (vlsr=%.2f)" % self.vlsr, + xlabel=xlabel, lines=mlist, force=self.force, blends=peaks["pvc"].blends, + continuum=[0.0] * len(self.pvspec), ylabel="Correlation", + thumbnail=True, references=line_ref) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) - image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=_caption) - llbdp.image.addimage(image, "pvspec") + image = Image(images={bt.SVG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=_caption) + llbdp.image.addimage(image, "pvspec") self.spec_description.append([llbdp.ra, llbdp.dec, "", xlabel, imname, thumbnailname, _caption, self.infile]) @@ -4427,18 +4457,23 @@ def run(self): llbdp.addRow(m) logging.regression("LINEID: %s %.5f %d %d" % (m.getkey("formula"), m.getkey("frequency"), m.getstart(), m.getend())) - # Need to adjust plot DPI = 72 for SVG; stick to PNG for now... - myplot._plot_type = admit.util.PlotControl.PNG - myplot._plot_mode = admit.util.PlotControl.NONE - myplot.summaryspec(self.statspec, self.specs, self.pvspec, imbase + "_summary", llist, force=self.force) - imname = myplot.getFigure(figno=myplot.figno, relative=True) - thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) + _caption = "Identified lines overlaid on Signal/Noise plot of all spectra." + # Need to adjust plot DPI = 72 for SVG; stick to PNG for now... + if self._plot_mode == PlotControl.NOPLOT: + imname = "not created" + thumbnailname = "not created" + # leave captions unchanged for now + else: + myplot._plot_type = admit.util.PlotControl.PNG + myplot.summaryspec(self.statspec, self.specs, self.pvspec, imbase + "_summary", llist, force=self.force) + imname = myplot.getFigure(figno=myplot.figno, relative=True) + thumbnailname = myplot.getThumbnail(figno=myplot.figno, relative=True) - image = Image(images={bt.PNG: imname}, thumbnail=thumbnailname, - thumbnailtype=bt.PNG, description=_caption) + image = Image(images={bt.PNG: imname}, thumbnail=thumbnailname, + thumbnailtype=bt.PNG, description=_caption) - llbdp.image.addimage(image, "summary") + llbdp.image.addimage(image, "summary") self.spec_description.append([llbdp.ra, llbdp.dec, "", "Signal/Noise", imname, thumbnailname, _caption, self.infile]) From 6cca839c5ad9f7ca37a158306ad66828bc87d53e Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Fri, 21 Sep 2018 10:52:46 -0400 Subject: [PATCH 29/34] more bugfixes from runa1 --- admit/Summary.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/admit/Summary.py b/admit/Summary.py index 5cd31ec..6ff15a2 100755 --- a/admit/Summary.py +++ b/admit/Summary.py @@ -1159,7 +1159,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if spectra == None: allspecs = "

%s produced no output for image %s

" % (taskname, casaimage) elif spectra.getNoPlot(): - allspecs = "

%s created output but was told to create no images for display.

" % (taskname) + allspecs = "

%s created output but was told not to create images for display.

" % (taskname) else: count = 0 # task arguments are the same in all entries. @@ -1211,7 +1211,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if cubesum == None: allspecs = "

%s produced no output

" % (taskname) elif cubesum.getNoPlot(): - allspecs = "

%s created output but was told to create no images for display.

" % (taskname) + allspecs = "

%s created output but was told not to create images for display.

" % (taskname) else: allspecs = "" taskargs = cubesum.taskargs @@ -1235,7 +1235,12 @@ def _process(self,taskname,tid,titems,thetask,outdir): if tlower == "continuumsub_at": continuumsub = titems.get('continuumsub',None) - if continuumsub != None: + if continuumsub == None: + allspecs = "

%s produced no continuum subtraction for the input image

" % taskname + + elif continuumsub.getNoPlot(): + allspecs = "

%s subtracted continuum from the input image but was told not to create any images for display

" % taskname + else: allspecs = "" taskargs = continuumsub.taskargs val = continuumsub.getValue() @@ -1251,8 +1256,6 @@ def _process(self,taskname,tid,titems,thetask,outdir): banner = "

%s output for %s

" % (taskname, "casaimage") #banner = "

%s output for %s

" % (taskname, casaimage) allspecs = banner + allspecs + "\n" + specval - else: - allspecs = "

%s no continuum subtracted for the input image

" % taskname retval = header % (taskclass,tid,thetask.statusicons(),taskname,tid,taskargs,tid,allspecs,tid) if tlower == "cubestats_at": @@ -1285,8 +1288,14 @@ def _process(self,taskname,tid,titems,thetask,outdir): else: datamean = '%.3E' % sumentry.value[0] + specval = "" + sumentry = titems.get('spectra',None) - if sumentry != None: + if sumentry == None: + specval = specval + "

%s did not create an emission summary for this cube

" % taskname + elif sumentry.getNoPlot(): + specval = specval + "

%s created output but was told not to create an emission summary for this cube

" % taskname + else: val = sumentry.getValue() #@todo do something with position and box? position = "(%s,%s)" % ( str(val[0]),str(val[1]) ) @@ -1298,7 +1307,11 @@ def _process(self,taskname,tid,titems,thetask,outdir): specval = specval + (SPAN4VAL % ( image, thumb, caption, caption, caption)) sumentry = titems.get('peakpnt',None) - if sumentry != None: + if sumentry == None: + specval = specval + "

%s did not create a peak point plot for this cube

" % taskname + elif sumentry.getNoPlot(): + specval = specval + "

%s created output but was told not to create a peak point plot for this cube

" % taskname + else: val = sumentry.getValue() image = val[0] thumb = val[1] @@ -1327,7 +1340,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if spectra == None: retval = header % (taskclass, tid, thetask.statusicons(),taskname, tid, the_item.taskargs, tid, bigstr, tid) elif spectra.getNoPlot(): - allspecs = "

%s identified spectral lines but was told not to create images for display.

" % taskname + allspecs = "

%s identified spectral lines but was told to no create images for display.

" % taskname else: allspecs = '' count = 0 @@ -1365,7 +1378,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if moments == None: allspecs = "

No moments were computed for this cube

" elif moments.getNoPlot(): - allspecs = "

%s computed moments but was told not to create any images for display

" % taskname + allspecs = "

%s computed moments but was told not to create images for display

" % taskname else: allspecs = "" count = 0 @@ -1409,7 +1422,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if pvslices == None: specval = "

No PV slices were computed for this cube

" elif pvslices.getNoPlot(): - specval = "

%s created output but was told to create no images for display.

" % (taskname) + specval = "

%s created output but was told not to create images for display.

" % (taskname) else: for val in pvslices.value: specval = STARTROW @@ -1473,7 +1486,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if the_item == None: specval = "

No PV correlation diagrams were computed from the input cube

" elif the_item.getNoPlot(): - specval = "

%s created output but was told to create no images for display.

" % (taskname) + specval = "

%s created output but was told not to create images for display.

" % (taskname) else: val = the_item.getValue() image = val[0] @@ -1508,7 +1521,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if the_item == None: tablestr = "

%s identified no sources

" % taskname elif the_item.getNoPlot(): # probably will never happen - tablestr = "

%s created output but was told to create no table for display.

" % (taskname) + tablestr = "

%s created output but was told not to create a table for display.

" % (taskname) else: summarydata = the_item.getValue() if summarydata == None or len(summarydata) == 0: @@ -1585,7 +1598,7 @@ def _process(self,taskname,tid,titems,thetask,outdir): if the_item == None: tablestr = "

%s produced no table output

" % (taskname) elif the_item.getNoPlot(): # probably will never happen - tablestr = "

%s created output but was told to create no table for display.

" % (taskname) + tablestr = "

%s created output but was told not to create a table for display.

" % (taskname) else: tablestr = '' summarydata = the_item.getValue() From adaabd35fba4b4e62639dcc0309956b7f5eb038d Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Fri, 21 Sep 2018 10:52:57 -0400 Subject: [PATCH 30/34] changing over to use new noplot member variable --- admit/at/CubeStats_AT.py | 121 +++++++++++++++++++++++++++------------ 1 file changed, 83 insertions(+), 38 deletions(-) diff --git a/admit/at/CubeStats_AT.py b/admit/at/CubeStats_AT.py index 7352fff..c1f0c2e 100644 --- a/admit/at/CubeStats_AT.py +++ b/admit/at/CubeStats_AT.py @@ -401,24 +401,34 @@ def run(self): y4 = np.zeros(len(minval)) y5 = y1-y4 y = [y1,y2,y3,y4] - title = 'CubeStats: ' + bdp_name+'_0' - xlab = 'Channel' - ylab = 'log(Peak,Noise,Peak/Noise)' - labels = ['log(peak)','log(rms noise)','log(peak/noise)','log(|minval|)'] - myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) - segp = [[chans[0],chans[nchan-1],math.log10(sigma0),math.log10(sigma0)]] - myplot.plotter(chans,y,title,bdp_name+"_0",xlab=xlab,ylab=ylab,segments=segp,labels=labels,thumbnail=True) - imfile = myplot.getFigure(figno=myplot.figno,relative=True) - thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True) - - image0 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_0") - b2.addimage(image0,"im0") + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + imfile = "not created" + thumbfile = "not created" + else: + noplot = False + title = 'CubeStats: ' + bdp_name+'_0' + xlab = 'Channel' + ylab = 'log(Peak,Noise,Peak/Noise)' + labels = ['log(peak)','log(rms noise)','log(peak/noise)','log(|minval|)'] + myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir()) + segp = [[chans[0],chans[nchan-1],math.log10(sigma0),math.log10(sigma0)]] + myplot.plotter(chans,y,title,bdp_name+"_0",xlab=xlab,ylab=ylab,segments=segp,labels=labels,thumbnail=True) + imfile = myplot.getFigure(figno=myplot.figno,relative=True) + thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True) + + image0 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_0") + b2.addimage(image0,"im0") if use_ppp: - # new trial for Lee - title = 'PeakSum: (numsigma=%.1f)' % (numsigma) - ylab = 'Jy*N_ppb' - myplot.plotter(chans,[peaksum],title,bdp_name+"_00",xlab=xlab,ylab=ylab,thumbnail=False) + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + else: + noplot = False + # new trial for Lee + title = 'PeakSum: (numsigma=%.1f)' % (numsigma) + ylab = 'Jy*N_ppb' + myplot.plotter(chans,[peaksum],title,bdp_name+"_00",xlab=xlab,ylab=ylab,thumbnail=False) if True: # hack ascii table @@ -442,24 +452,29 @@ def run(self): caption += " green: noise per channel," caption += " blue: peak value per channel," caption += " red: peak/noise per channel)." - self._summary["spectra"] = SummaryEntry([0, 0, str(specbox), 'Channel', imfile, thumbfile , caption, fin], "CubeStats_AT", self.id(True)) - self._summary["chanrms"] = SummaryEntry([float(sigma0), fin], "CubeStats_AT", self.id(True)) + self._summary["spectra"] = SummaryEntry([0, 0, str(specbox), 'Channel', imfile, thumbfile , caption, fin], "CubeStats_AT", self.id(True),noplot=noplot) + + self._summary["chanrms"] = SummaryEntry([float(sigma0), fin], "CubeStats_AT", self.id(True),noplot=noplot) # @todo Will imstat["max"][0] always be equal to s['datamax']? If not, why not? if 'datamax' in s: - self._summary["dynrange"] = SummaryEntry([float(s['datamax']/sigma0), fin], "CubeStats_AT", self.id(True)) + self._summary["dynrange"] = SummaryEntry([float(s['datamax']/sigma0), fin], "CubeStats_AT", self.id(True),noplot=noplot) else: self._summary["dynrange"] = SummaryEntry([float(imstat0["max"][0]/sigma0), fin], "CubeStats_AT", self.id(True)) - self._summary["datamean"] = SummaryEntry([imstat0["mean"][0], fin], "CubeStats_AT", self.id(True)) + self._summary["datamean"] = SummaryEntry([imstat0["mean"][0], fin], "CubeStats_AT", self.id(True),noplot=noplot) - title = bdp_name + "_1" - xlab = 'log(Peak,Noise,P/N)' - myplot.histogram([y1,y2,y3],title,bdp_name+"_1",xlab=xlab,thumbnail=True) + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + else: + noplot = False + title = bdp_name + "_1" + xlab = 'log(Peak,Noise,P/N)' + myplot.histogram([y1,y2,y3],title,bdp_name+"_1",xlab=xlab,thumbnail=True) - imfile = myplot.getFigure(figno=myplot.figno,relative=True) - thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True) - image1 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_1") - b2.addimage(image1,"im1") + imfile = myplot.getFigure(figno=myplot.figno,relative=True) + thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True) + image1 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_1") + b2.addimage(image1,"im1") # note that the 'y2' can have been clipped, which can throw off stats.robust() # @todo should set a mask for those. @@ -471,7 +486,11 @@ def run(self): y2_mean = ry2.mean() y2_std = ry2.std() if n>9: logging.debug("NORMALTEST2: %s" % str(scipy.stats.normaltest(ry2))) - myplot.hisplot(y2,title,bdp_name+"_2",xlab=xlab,gauss=[y2_mean,y2_std],thumbnail=True) + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + else: + noplot = False + myplot.hisplot(y2,title,bdp_name+"_2",xlab=xlab,gauss=[y2_mean,y2_std],thumbnail=True) title = bdp_name + "_3" xlab = 'log(diff[Noise])' @@ -482,7 +501,11 @@ def run(self): dy2_mean = rdy2.mean() dy2_std = rdy2.std() if n>9: logging.debug("NORMALTEST3: %s" % str(scipy.stats.normaltest(rdy2))) - myplot.hisplot(dy2,title,bdp_name+"_3",xlab=xlab,gauss=[dy2_mean,dy2_std],thumbnail=True) + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + else: + noplot = False + myplot.hisplot(dy2,title,bdp_name+"_3",xlab=xlab,gauss=[dy2_mean,dy2_std],thumbnail=True) title = bdp_name + "_4" @@ -492,7 +515,11 @@ def run(self): y3_mean = ry3.mean() y3_std = ry3.std() if n>9: logging.debug("NORMALTEST4: %s" % str(scipy.stats.normaltest(ry3))) - myplot.hisplot(y3,title,bdp_name+"_4",xlab=xlab,gauss=[y3_mean,y3_std],thumbnail=True) + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + else: + noplot = False + myplot.hisplot(y3,title,bdp_name+"_4",xlab=xlab,gauss=[y3_mean,y3_std],thumbnail=True) title = bdp_name + "_5" xlab = 'log(diff[Signal/Noise)])' @@ -502,7 +529,11 @@ def run(self): dy3_mean = rdy3.mean() dy3_std = rdy3.std() if n>9: logging.debug("NORMALTEST5: %s" % str(scipy.stats.normaltest(rdy3))) - myplot.hisplot(dy3,title,bdp_name+"_5",xlab=xlab,gauss=[dy3_mean,dy3_std],thumbnail=True) + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + else: + noplot = False + myplot.hisplot(dy3,title,bdp_name+"_5",xlab=xlab,gauss=[dy3_mean,dy3_std],thumbnail=True) title = bdp_name + "_6" @@ -512,7 +543,11 @@ def run(self): y5_mean = ry5.mean() y5_std = ry5.std() if n>9: logging.debug("NORMALTEST6: %s" % str(scipy.stats.normaltest(ry5))) - myplot.hisplot(y5,title,bdp_name+"_6",xlab=xlab,gauss=[y5_mean,y5_std],thumbnail=True) + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + else: + noplot = False + myplot.hisplot(y5,title,bdp_name+"_6",xlab=xlab,gauss=[y5_mean,y5_std],thumbnail=True) logging.debug("LogPeak: m,s= %f %f min/max %f %f" % (y1.mean(),y1.std(),y1.min(),y1.max())) logging.debug("LogNoise: m,s= %f %f %f %f min/max %f %f" % (y2.mean(),y2.std(),y2_mean,y2_std,y2.min(),y2.max())) @@ -535,12 +570,18 @@ def run(self): s = np.pi * ( smax * (z0**gamma) )**2 cmds = ["grid", "axis equal"] title = "Peak Points per channel" - pppimage = bdp_name + '_ppp' - myplot.scatter(xpos,ypos,title=title,figname=pppimage,size=s,color=chans,cmds=cmds,thumbnail=True) - pppimage = myplot.getFigure(figno=myplot.figno,relative=True) - pppthumbnail = myplot.getThumbnail(figno=myplot.figno,relative=True) caption = "Peak point plot: Locations of per-channel peaks in the image cube " + fin - self._summary["peakpnt"] = SummaryEntry([pppimage, pppthumbnail, caption, fin], "CubeStats_AT", self.id(True)) + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + pppimage = "not created" + pppthumbnail = "not created" + else: + noplot = False + pppimage = bdp_name + '_ppp' + myplot.scatter(xpos,ypos,title=title,figname=pppimage,size=s,color=chans,cmds=cmds,thumbnail=True) + pppimage = myplot.getFigure(figno=myplot.figno,relative=True) + pppthumbnail = myplot.getThumbnail(figno=myplot.figno,relative=True) + self._summary["peakpnt"] = SummaryEntry([pppimage, pppthumbnail, caption, fin], "CubeStats_AT", self.id(True),noplot=noplot) dt.tag("plotting") # making PeakStats plot @@ -553,7 +594,11 @@ def run(self): ylab = 'FWHM (channels)' pppimage = bdp_name + '_peakstats' cval = mval - myplot.scatter(pval,wval,title=title,xlab=xlab,ylab=ylab,color=cval,figname=pppimage,thumbnail=False) + if self._plot_mode == PlotControl.NOPLOT: + noplot = True + pppimage = "not created" + else: + myplot.scatter(pval,wval,title=title,xlab=xlab,ylab=ylab,color=cval,figname=pppimage,thumbnail=False) dt.tag("peakstats") From 5d15e88f16250ba30f6590eee158b06197a2cff7 Mon Sep 17 00:00:00 2001 From: Peter Teuben Date: Tue, 2 Oct 2018 11:07:30 -0400 Subject: [PATCH 31/34] _noplot forgotten for admit.xml --- admit/xmlio/dtd/admit.dtd | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/admit/xmlio/dtd/admit.dtd b/admit/xmlio/dtd/admit.dtd index f3626e1..608fefa 100644 --- a/admit/xmlio/dtd/admit.dtd +++ b/admit/xmlio/dtd/admit.dtd @@ -16,7 +16,7 @@ - + @@ -28,6 +28,8 @@ + + From ac5aa216b5c4823304dd974ca38fd813d10ca956 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Fri, 5 Oct 2018 14:59:47 -0400 Subject: [PATCH 32/34] fix getimagefile() in case where lower level return value is None, i.e. getimagefile() was queried with a bt.type not contained in this BDP --- admit/bdp/Image_BDP.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/admit/bdp/Image_BDP.py b/admit/bdp/Image_BDP.py index efee9c3..de9a473 100644 --- a/admit/bdp/Image_BDP.py +++ b/admit/bdp/Image_BDP.py @@ -105,4 +105,8 @@ def getimagefile(self, imtype=bt.CASA, name=""): String containing the image name, or None if it does not exist """ - return self.image.getimage(imtype, name).file + retimage = self.image.getimage(imtype, name) + if retimage != None: + return retimage.file + else: + return None From 3e400a926a5dfd02bf51017fd2a2586d3801dd04 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Fri, 5 Oct 2018 15:01:02 -0400 Subject: [PATCH 33/34] show() must also check for MultiImages, since an Image_BDP does not actually contain admit.util.Image but rather admit.util.Multimage --- admit/bdp/BDP.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/admit/bdp/BDP.py b/admit/bdp/BDP.py index 8fa4136..d0e0f73 100644 --- a/admit/bdp/BDP.py +++ b/admit/bdp/BDP.py @@ -126,10 +126,11 @@ def getfiles(self): files = [] for i in self.__dict__: if isinstance(getattr(self, i), Image): - #print getattr(self, i).images for key in getattr(self, i).images: files.append(getattr(self, i).images[key]) - #files.append(getattr(self, i).fileName) + if isinstance(getattr(self, i), MultiImage): + for key in getattr(self, i).mimages: + files.append(getattr(self, i).mimages[key]) return files def show(self): From 21da053a1ac8ccde8297c78b249af33b4587e312 Mon Sep 17 00:00:00 2001 From: Marc Pound <22331890+mpound@users.noreply.github.com> Date: Fri, 5 Oct 2018 15:50:54 -0400 Subject: [PATCH 34/34] catch a keyerror --- admit/bdp/Image_BDP.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/admit/bdp/Image_BDP.py b/admit/bdp/Image_BDP.py index de9a473..e6feb28 100644 --- a/admit/bdp/Image_BDP.py +++ b/admit/bdp/Image_BDP.py @@ -105,7 +105,10 @@ def getimagefile(self, imtype=bt.CASA, name=""): String containing the image name, or None if it does not exist """ - retimage = self.image.getimage(imtype, name) + try: + retimage = self.image.getimage(imtype, name) + except KeyError: + return None if retimage != None: return retimage.file else: