/[CCFs]/bin/mlovccf
ViewVC logotype

Diff of /bin/mlovccf

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.15 by ccflib, Wed Apr 13 10:26:13 2016 UTC revision 1.24 by ccflib, Fri Nov 13 16:09:22 2020 UTC
# Line 18  Line 18 
18  # This is obtained from the report of all observations that the XSA makes to CDS,  # This is obtained from the report of all observations that the XSA makes to CDS,
19  # xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt).  # xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt).
20  #  #
21  # $Id: mlovccf,v 1.14 2016/04/04 09:33:19 ccflib Exp $  # $Id: mlovccf,v 1.23 2019/11/12 16:00:29 ccflib Exp $
22    
23  # Next two lines allow to submit this script to the grid  # Next two lines allow to submit this script to the grid
24  # request Bourne shell as shell for job  # request Bourne shell as shell for job
25  #$ -S /bin/bash  #$ -S /bin/bash
26    
27  adminmail="eduardo.ojero@esa.int"  export DEBUG="T"
28    
29  VALID_CCF="$HOME/valid_ccf"  adminmail="eduardo.ojero@sciops.esa.int"
30    
31    VALID_CCF="$HOME/processing_valid_ccf"
32    
33  mailfile="${VALID_CCF}/mailfile"  mailfile="${VALID_CCF}/mailfile"
34    
# Line 55  bodyfile=$2 Line 57  bodyfile=$2
57    
58  echo ; echo "Sending message to ${adminmail} with subject ${subject}" ; echo  echo ; echo "Sending message to ${adminmail} with subject ${subject}" ; echo
59    
60  mailx -s "${subject}" "${adminmail}" < ${bodyfile}  if [ "$DEBUG" ] ; then
61    
62            echo "DEBUG: mlovccf: mailnotify: Going to send mail as "
63            echo "mailx -v -s ${subject} ${adminmail} < ${bodyfile}"
64    fi
65    
66    mailx -v -s "${subject}" "${adminmail}" < ${bodyfile}
67    
68    if [ "$DEBUG" ] ; then
69            echo "DEBUG: mlovccf: mailnotify: Mail sent!"
70    fi
71  }  }
72    
73    
# Line 77  XSAOBSURL="http://nxsa.esac.esa.int/ftp_ Line 88  XSAOBSURL="http://nxsa.esac.esa.int/ftp_
88    
89  # Get the list latest list of ODFs available from nXSA server.  # Get the list latest list of ODFs available from nXSA server.
90    
91  wget -q ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt  wget --quiet ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt
92    
93  # Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference  # Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference
94  # of which list of ODFs was used to compute the list of valid CCFs.  # of which list of ODFs was used to compute the list of valid CCFs.
95    
96  mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt  mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt
97    
98  }  if [ "$DEBUG" ] ; then
99            echo "DEBUG: mlovccf: getxsaobslog: XSA Obs. log file obtained successfully and renamed to ${VALID_CCF}/xsaobslog_${now}.txt"
100    fi
   
 # +++ Function find_latest_mif to get the latest XMM_CALINDEX from given directory  
   
 find_latest_mif()  
 {  
   
 [ -z "$1" ] && return  
   
 CCFPATH="$1"  
   
 maxissue="0"  
 for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF`  
 do  
         mifbase=`basename $mif`  
         issue=`echo $mifbase | awk -F"." '{print $1}' | awk -F"_" '{print $3}'`  
         [ "$issue" -ge "$maxissue" ] && maxissue="$issue"  
 done  
   
 MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF  
101    
102  }  }
103    
# Line 136  fi Line 128  fi
128  fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \  fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
129                          columns="SCOPE TYPEID ISSUE"            \                          columns="SCOPE TYPEID ISSUE"            \
130                          rows=-  STDOUT                            rows=-  STDOUT  
131    
132  }  }
133    
134    
# Line 157  fillvalidccfdir() Line 150  fillvalidccfdir()
150    
151  validccflist=$1  validccflist=$1
152    
153  if [ ! -f "${VALID_CCF}/${validccflist}"  ] ; then  if [ ! -f "${VALID_CCF}/${validccflist}" -o ! -s "${VALID_CCF}/${validccflist}"  ] ; then
154          echo "Error: ${VALID_CCF}/${validccflist} not found - Abort!"          echo "Error: ${VALID_CCF}/${validccflist} not found or empty - Abort!"
155          [ -f "${mailfile}" ] && rm -rf ${mailfile}          [ -f "${mailfile}" ] && rm -rf ${mailfile}
156          echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found" >> $mailfile          echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found or it is empty" >> $mailfile
157          mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found - Abort!" $mailfile          mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found or it is empty - Abort!" $mailfile
158          rm ${mailfile}          rm ${mailfile}
159          exit          exit
160  fi  fi
# Line 169  fi Line 162  fi
162  VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents"  VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents"
163  rm -rf ${VALIDCONSTITUENTSDIR}/*  rm -rf ${VALIDCONSTITUENTSDIR}/*
164    
165    # This operation can only succeed because the subdir valid has enable the write permission
166    # for the group "sas" (g=rwx), provided that CCFs belong to sasbuild(5153):sas.
167  VALIDCCFDIRINTRANET="/ccf/valid"  VALIDCCFDIRINTRANET="/ccf/valid"
168  rm -rf ${VALIDCCFDIRINTRANET}/*  rm -rf ${VALIDCCFDIRINTRANET}/*
169    
# Line 183  do Line 178  do
178          cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/          cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/
179  done < ${VALID_CCF}/${validccflist}  done < ${VALID_CCF}/${validccflist}
180    
181    if [ "$DEBUG" ] ; then
182            echo "DEBUG: mlovccf: fillvalidccfdir: Processed $validccflist"
183    fi
184  }  }
185    
186    
# Line 203  if [ "$xsaobslogsize"  = "0" ] ; then Line 201  if [ "$xsaobslogsize"  = "0" ] ; then
201          [ -f "${mailfile}" ] && rm -rf ${mailfile}          [ -f "${mailfile}" ] && rm -rf ${mailfile}
202          echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile          echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile
203          echo "." >> $mailfile          echo "." >> $mailfile
204            if [ "$DEBUG" ] ; then
205                    echo "DEBUG: mlovccf: main: xsaobslogsize has 0 size. Mail notification being sent!"
206            fi
207          mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile"          mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile"
208          rm ${mailfile}          rm ${mailfile}
209          exit          exit
210  fi  fi
211    
212  # 2. Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host  #
213    # 2. Initialises HEADAS and SAS
214    #
215    
216    # Defines SASBUILD_LOCAL depending on host.
217    # Then uses it to build up HEADAS and setup LD_LIBRARY_PATH accordingly.
218    
219  case "$host" in  case "$host" in
220          xvsoc01|xmm)          sasbld01n|sasbld02n)
221                  export SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"                  export SASBUILD_LOCAL="/sasbuild/local/${host}/GNU_CC_CXX_9.2.0"
                 export SAS_DIR=/data/xmm/ccflib/sas  
                 export SAS_PATH=$SAS_DIR  
                 source $SAS_DIR/sas-setup.sh  
222                  ;;                  ;;
223          sasbld01|sasbld02)          sciggw7|scigvn7-*)
224                  export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"                  export SASBUILD_LOCAL="/sasbuild/local/sasbld04n/GNU_CC_CXX_9.2.0"
                 /sas/bin/confsas  
                 export HEADAS=/sasbuild/local/${host}/headas/architecture  
                 . $HEADAS/headas-init.sh  
                 # To be able to mix several jobs on the same ccflib account  
                 [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}  
                 export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"  
                 echo -n "HEADAS version: "  
                 fversion  
                 source $HOME/setsas.sh 1> /dev/null  
                 echo "SAS Version: "  
                 sasversion -V 1  
                 export SAS_VERBOSITY=0  
225                  ;;                  ;;
226          scigrid6|cn-*)          *) echo "Error: Unsupported host $host" ; exit 1 ;;
                 export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"  
                 /sas/bin/confsas  
                 export HEADAS=/sasbuild/local/sasbld02/headas/architecture  
                 . $HEADAS/headas-init.sh  
                 # To be able to mix several jobs on the same ccflib account  
                 [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}  
                 export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"  
                 echo -n "HEADAS version: "  
                 fversion  
                 source $HOME/setsas.sh 1> /dev/null  
                 echo "SAS Version: "  
                 sasversion -V 1  
                 export SAS_VERBOSITY=0  
                 ;;  
         *) echo "Error: Do not know how to do it in host $host" ; exit ;;  
227  esac  esac
228    
229  # 3. Finds the latest MIF issue  export HEADAS=${SASBUILD_LOCAL}/headas/architecture
230    
231    headas_output="${VALID_CCF}/headas_output"
232    source $HEADAS/headas-init.sh > ${headas_output} 2>&1
233    
234    [ "$DEBUG" ] && echo "DEBUG: mlovccf: main: Executed Headas initialisation script for $HEADAS"
235    
236  find_latest_mif "${SAS_CCFPATH}"  # The next statement should not be necessary because the SAS initialisation
237    # provides under $SAS_DIR/libextra all the necessary libraries from GNU C/C++
238    # which were required to build Heasoft. But, in order to make the HEADAS
239    # initialisation independent of the SAS initialisation, we set the
240    # LD_LIBRARY_PATH as it has to be to run Headas by itself without SAS been
241    # ready.
242    
243    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${SASBUILD_LOCAL}/gcc/lib64
244    
245    
246    headas_ok=`cat ${headas_output}`
247    
248    if [ -n "${headas_ok}" ] ; then
249            echo "mlovccf: HEADAS initialisation failed with error" >> ${mailfile}
250            cat ${headas_output}                                    >> ${mailfile}
251            mailnotify "mlovccf: HEADAS initilisation failed" ${mailfile}
252            if [ "$DEBUG" ] ; then
253                    echo "DEBUG: mlovccf: main: Headas initialization failed. Notification mail sent!"
254            fi
255            rm -v ${mailfile}
256            rm -v ${headas_output}
257            exit 1
258    fi
259    
260    # To be able to mix several jobs on the same ccflib account
261    [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
262    export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
263    if [ "$DEBUG" ] ; then
264            echo -n "DEBUG: mlovccf: main: HEADAS version: "`fversion`
265    fi
266    
267    #
268    #        W a r n i n g   o n   S A S   i n i t i a l i s a t i o n  
269    #
270    # The SAS initialisation produced by setsas.sh sets SAS_CCFPATH to /ccf/pub.
271    # Hence, if we set SAS_CCFPATH before running setsas.sh, we will always
272    # use /ccf/pub as repository for CCF. This is right as long as /ccf/pub is in
273    # strict sync with /home/ccflib/ccf_ftp/constituents. But if this is not true,
274    # the resulting valid CCFs might be wrong. Therefore, we set it here explicitly.
275    #
276    
277    confsas_output="${VALID_CCF}/confsas_output"
278    source /sas/bin/confsas > ${confsas_output}
279    confsas_ok=`cat ${confsas_output}`
280    
281    if [ "$DEBUG" ] ; then
282            echo "DEBUG: mlovccf: main: Executed /sas/bin/confsas with output: ${confsas_ok}"
283    fi
284    
285    if [ -n "${confsas_ok}" ] ; then
286            echo "mlovccf: SAS initialization failed with error" >> ${mailfile}
287            cat ${confsas_output}                                >> ${mailfile}
288            mailnotify "mlovccf: SAS initialisation failed" ${mailfile}
289            if [ "$DEBUG" ] ; then
290                    echo "DEBUG: mlovccf: main: SAS initialization failed. Notification mail sent!"
291            fi
292            rm -v ${mailfile}
293            rm -v ${confsas_output}
294            exit 1
295    fi
296    
297    source $HOME/setsas.sh 1> /dev/null
298    
299    if [ "$DEBUG" ] ; then
300            echo "DEBUG: mlovccf: main: SAS setsas.sh script executed"
301            echo
302            echo "DEBUG: mlovccf: main: SAS Version: "`sasversion -V 1`
303    fi
304    
305    
306    # Set SAS vervbosity to 0 from now onwards
307    
308    export SAS_VERBOSITY=0
309    
310    
311    # Re-sets SAS_CCFPATH
312    
313    export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
314    
315  # 4. Scans the list of OBDS ID registered in the previous list to get the start time  
316  #    Then runs cifbuild to get the respective CIF.  
317  #    all_ccfs_${now}.txt is filled recursively on each iteration. Therefore we start  # 3. XMM_CALINDEX file from SAS_CCFPATH
318  #    with an empty file.  
319    MIF=`ls -1 ${SAS_CCFPATH}/XMM_CALINDEX_*.CCF`
320    
321    # 4. For each OBDS ID registered in the XSA observations list, gets the start
322    #    time of the observation. Then runs cifbuild to get the respective CIF.
323    #    all_ccfs_${now}.txt is filled recursively on each iteration.
324    #    Therefore we start with an empty file. The final list will be this one.
325    
326  #    Output file :          ${VALID_CCF}/all_ccfs_${now}.txt  #    Output file :          ${VALID_CCF}/all_ccfs_${now}.txt
327    
# Line 306  do Line 369  do
369    
370          cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt          cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt
371    
372            comm -1 -3 --nocheck-order ${VALID_CCF}/all_ccfs_${now}.tmp ${VALID_CCF}/all_ccfs_${now}.txt > ${VALID_CCF}/diff_iteration_${n}.txt
373            
374            diffsize=`stat --format=%s ${VALID_CCF}/diff_iteration_${n}.txt`
375            
376            if [ "$diffsize" != "0" ] ; then
377                    echo "CCF added in this iteration: "
378                    cat ${VALID_CCF}/diff_iteration_${n}.txt
379            fi      
380            
381            echo
382            echo "Number of valid CCFs found so far: "`cat ${VALID_CCF}/all_ccfs_${now}.txt | wc -l` ; echo
383    
384            rm ${VALID_CCF}/diff_iteration_${n}.txt
385          rm ${cif_file}          rm ${cif_file}
386          rm ${VALID_CCF}/${obsid}_ccfs.txt          rm ${VALID_CCF}/${obsid}_ccfs.txt
387          rm ${VALID_CCF}/all_ccfs_${now}.tmp          rm ${VALID_CCF}/all_ccfs_${now}.tmp
# Line 404  rm ${VALID_CCF}/ccf_issues_for*.txt Line 480  rm ${VALID_CCF}/ccf_issues_for*.txt
480    
481  # 6. Fill in the Valid CCF Set dir with the proper CCFs  # 6. Fill in the Valid CCF Set dir with the proper CCFs
482    
483  fillvalidccfdir "all_ccfs_${now}.txt"  #fillvalidccfdir "all_ccfs_${now}.txt"
484    
485    
486  [ -f "${mailfile}" ] && rm -rf ${mailfile}  #[ -f "${mailfile}" ] && rm -rf ${mailfile}
487  echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile  #echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile
488  echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}"  >> $mailfile  #echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}"  >> $mailfile
489  mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile  #mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile
490  rm ${mailfile}  #rm ${mailfile}
Legend:
Removed from v.1.15  
changed lines
  Added in v.1.24
  ViewVC Help
Powered by ViewVC 1.1.27