/[CCFs]/bin/mlovccf
ViewVC logotype

Diff of /bin/mlovccf

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.9 by ccflib, Fri Mar 18 09:18:29 2016 UTC revision 1.24 by ccflib, Fri Nov 13 16:09:22 2020 UTC
# Line 7  Line 7 
7  # typically the current date. This list will change with time as far as new  # typically the current date. This list will change with time as far as new
8  # CCFs are issued to deal with new calibration data and/or replace obsolete ones.  # CCFs are issued to deal with new calibration data and/or replace obsolete ones.
9  #  #
10  # To identify the set of CCFs required to process a given ODF is the ijob of task cifbuild.  # To identify the set of CCFs required to process a given ODF is the job of task cifbuild.
11  # To make such identification, cifbuild needs only the start time of the observation.  # To make such identification, cifbuild needs only the start time of the observation.
12  # Therefore, if we provide a list of all observed ODFs at a given date together with their  # Therefore, if we provide a list of all observed ODFs at a given date together with their
13  # respective start time, we could compute all the CCFs required to process them.  # respective start time, we could compute all the CCFs required to process them.
# Line 16  Line 16 
16  #  #
17  # We need as input the list of all observed ODFs at a given date.  # We need as input the list of all observed ODFs at a given date.
18  # This is obtained from the report of all observations that the XSA makes to CDS,  # This is obtained from the report of all observations that the XSA makes to CDS,
19  # xsaobslog.txt (ftp://nxsa.esac.esa.int/pub/cfs_obslog/xsaobslog.txt).  # xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt).
20  #  #
21  # $Id: mlovccf,v 1.8 2015/09/07 13:39:15 ccflib Exp $  # $Id: mlovccf,v 1.23 2019/11/12 16:00:29 ccflib Exp $
22    
23  # Next two lines allow to submit this script to the grid  # Next two lines allow to submit this script to the grid
24  # request Bourne shell as shell for job  # request Bourne shell as shell for job
25  #$ -S /bin/bash  #$ -S /bin/bash
26    
27    export DEBUG="T"
28    
29    adminmail="eduardo.ojero@sciops.esa.int"
30    
31    VALID_CCF="$HOME/processing_valid_ccf"
32    
33    mailfile="${VALID_CCF}/mailfile"
34    
35  host=`hostname | cut -d. -f1`  host=`hostname | cut -d. -f1`
36    
37  now=`date +'%Y%m%d_%H%M'`  now=`date +'%Y%m%d_%H%M'`
38  now_table=`date +'%Y-%m-%dT%H:%M:%S'`  now_table=`date +'%Y-%m-%dT%H:%M:%S'`
39    
 # Function find_latest_mif to get the latest XMM_CALINDEX from given directory  
40    
41  find_latest_mif()  # +++ Function mailnotify
42    #
43    # This function is used to notify adminmail on any issue while runnning this procedure
44    #
45    # Needs a subject as $1 and body as local file to be send.
46    
47    mailnotify ()
48  {  {
49    
50  [ -z "$1" ] && return  [ "$#" -ne "2" ] && return
51    
52  CCFPATH="$1"  subject=$1
53    
54  maxissue="0"  bodyfile=$2
55  for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF`  
56  do  [ ! -f "${bodyfile}" ] && return
         mifbase=`basename $mif`  
         issue=`echo $mifbase | awk -F"." '{print $1}' | awk -F"_" '{print $3}'`  
         [ "$issue" -ge "$maxissue" ] && maxissue="$issue"  
 done  
57    
58  MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF  echo ; echo "Sending message to ${adminmail} with subject ${subject}" ; echo
59    
60    if [ "$DEBUG" ] ; then
61    
62            echo "DEBUG: mlovccf: mailnotify: Going to send mail as "
63            echo "mailx -v -s ${subject} ${adminmail} < ${bodyfile}"
64    fi
65    
66    mailx -v -s "${subject}" "${adminmail}" < ${bodyfile}
67    
68    if [ "$DEBUG" ] ; then
69            echo "DEBUG: mlovccf: mailnotify: Mail sent!"
70    fi
71  }  }
72    
73  # Function get_ccf_list to list on stdout the table of CCFs in a given CIF  
74    
75    # +++ Function getxsaobslog
76    #
77    # We need to get the list of all observed ODFs at the current date.
78    # This is obtained from http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt.
79    
80    getxsaobslog ()
81    {
82    
83    XSAOBSURL="http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt"
84    
85    # Remove any xsaobslog.txt previously downloaded.
86    
87    [ -f "${VALID_CCF}/xsaobslog.txt" ] && rm -rf ${VALID_CCF}/xsaobslog.txt
88    
89    # Get the list latest list of ODFs available from nXSA server.
90    
91    wget --quiet ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt
92    
93    # Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference
94    # of which list of ODFs was used to compute the list of valid CCFs.
95    
96    mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt
97    
98    if [ "$DEBUG" ] ; then
99            echo "DEBUG: mlovccf: getxsaobslog: XSA Obs. log file obtained successfully and renamed to ${VALID_CCF}/xsaobslog_${now}.txt"
100    fi
101    
102    }
103    
104    
105    
106    # +++Function get_ccf_list to list on stdout the table of CCFs in a given CIF.
107  #  #
108  # Requires that Heasoft is initialised. Otherwise exits with error.  # Requires that Heasoft is initialised. Otherwise exits with error.
109    
# Line 65  noversion=`which fversion | grep -c no` Line 117  noversion=`which fversion | grep -c no`
117    
118  if [ "${noversion}" != "0" ] ; then  if [ "${noversion}" != "0" ] ; then
119          echo "Error: Heasoft not initialised !"          echo "Error: Heasoft not initialised !"
120            [ -f "${mailfile}" ] && rm -rf ${mailfile}
121            echo "Running of fversion produced the following output:" >> $mailfile
122            echo "$noversion" >> $mailfile
123            mailnotify "mvloccf: Error: Heasoft not initialised!" $mailfile
124            rm ${mailfile}
125          exit          exit
126  fi  fi
127    
128  fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \  fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
129                          columns="SCOPE TYPEID ISSUE"            \                          columns="SCOPE TYPEID ISSUE"            \
130                          rows=-  STDOUT                            rows=-  STDOUT  
131    
132  }  }
133    
134    
 # We need to get the list of all observed ODFs at the current date.  
 # This is obtained from ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt.  
135    
 VALID_CCF="$HOME/valid_ccf"  
136    
137  cd ${VALID_CCF}  # +++ Fill in the valid_constituents directory with the Valid CCF set
138    #
139    # The directory to be filled in is ${VALIDCONSTITUENTSDISR}.
140    # This directory is synchronized with the similar directory in the
141    # external ftp server.
142    #
143    # In addition to that, we also use this function to fill in the
144    # /ccf/valid directory available for users in the intranet.
145    
146  # Remove any xsaobslog.txt previously downloaded.  fillvalidccfdir()
147    {
148    
149  [ -f "xsaobslog.txt" ] && rm -rf xsaobslog.txt  [ -z "$1" ] && return
150    
151  # Get the list latest list of ODFs available from nXSA server.  validccflist=$1
   
 wget -q http://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt  
152    
153  # Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference  if [ ! -f "${VALID_CCF}/${validccflist}" -o ! -s "${VALID_CCF}/${validccflist}"  ] ; then
154  # of which list of ODFs was used to compute the list of valid CCFs.          echo "Error: ${VALID_CCF}/${validccflist} not found or empty - Abort!"
155            [ -f "${mailfile}" ] && rm -rf ${mailfile}
156            echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found or it is empty" >> $mailfile
157            mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found or it is empty - Abort!" $mailfile
158            rm ${mailfile}
159            exit
160    fi
161    
162    VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents"
163    rm -rf ${VALIDCONSTITUENTSDIR}/*
164    
165    # This operation can only succeed because the subdir valid has enable the write permission
166    # for the group "sas" (g=rwx), provided that CCFs belong to sasbuild(5153):sas.
167    VALIDCCFDIRINTRANET="/ccf/valid"
168    rm -rf ${VALIDCCFDIRINTRANET}/*
169    
170    CONSTITUENTSDIR="/home/ccflib/ftp-area/constituents"
171    n=0
172    while read ccf
173    do
174            n=$((n + 1))
175            echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/"
176            cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/
177            echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/"
178            cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/
179    done < ${VALID_CCF}/${validccflist}
180    
181    if [ "$DEBUG" ] ; then
182            echo "DEBUG: mlovccf: fillvalidccfdir: Processed $validccflist"
183    fi
184    }
185    
186    
187    #
188    #   M a i n    p r o g r a m
189    #
190    # 1. Get the list of all ODFs
191    
192    
193    xsaobslogsize="0"
194    
195    getxsaobslog
196    
197    xsaobslogsize=`stat --format=%s ${VALID_CCF}/xsaobslog_${now}.txt`
198    
199    if [ "$xsaobslogsize"  = "0" ] ; then
200            echo "Error: Failure to download the XSA Obs. Log file - Abort"
201            [ -f "${mailfile}" ] && rm -rf ${mailfile}
202            echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile
203            echo "." >> $mailfile
204            if [ "$DEBUG" ] ; then
205                    echo "DEBUG: mlovccf: main: xsaobslogsize has 0 size. Mail notification being sent!"
206            fi
207            mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile"
208            rm ${mailfile}
209            exit
210    fi
211    
212  mv xsaobslog.txt xsaobslog_${now}.txt  #
213    # 2. Initialises HEADAS and SAS
214    #
215    
216  # Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host  # Defines SASBUILD_LOCAL depending on host.
217    # Then uses it to build up HEADAS and setup LD_LIBRARY_PATH accordingly.
218    
219  case "$host" in  case "$host" in
220          xvsoc01|xmm)          sasbld01n|sasbld02n)
221                  export SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"                  export SASBUILD_LOCAL="/sasbuild/local/${host}/GNU_CC_CXX_9.2.0"
                 export SAS_DIR=/data/xmm/ccflib/sas  
                 export SAS_PATH=$SAS_DIR  
                 source $SAS_DIR/sas-setup.sh  
222                  ;;                  ;;
223          sasbld01|sasbld02)          sciggw7|scigvn7-*)
224                  export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"                  export SASBUILD_LOCAL="/sasbuild/local/sasbld04n/GNU_CC_CXX_9.2.0"
                 /sas/bin/confsas  
                 export HEADAS=/sasbuild/local/${host}/headas/architecture  
                 . $HEADAS/headas-init.sh  
                 # To be able to mix several jobs on the same ccflib account  
                 [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}  
                 export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"  
                 echo -n "HEADAS version: "  
                 fversion  
                 source $HOME/setsas.sh 1> /dev/null  
                 echo "SAS Version: "  
                 sasversion -V 1  
                 export SAS_VERBOSITY=0  
225                  ;;                  ;;
226          scigrid6|cn-*)          *) echo "Error: Unsupported host $host" ; exit 1 ;;
                 export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"  
                 /sas/bin/confsas  
                 export HEADAS=/sasbuild/local/sasbld02/headas/architecture  
                 . $HEADAS/headas-init.sh  
                 # To be able to mix several jobs on the same ccflib account  
                 [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}  
                 export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"  
                 echo -n "HEADAS version: "  
                 fversion  
                 source $HOME/setsas.sh 1> /dev/null  
                 echo "SAS Version: "  
                 sasversion -V 1  
                 export SAS_VERBOSITY=0  
                 ;;  
         *) echo "Error: Do not know how to do it in host $host" ; exit ;;  
227  esac  esac
228    
229  # Finds the latest MIF issue  export HEADAS=${SASBUILD_LOCAL}/headas/architecture
230    
231    headas_output="${VALID_CCF}/headas_output"
232    source $HEADAS/headas-init.sh > ${headas_output} 2>&1
233    
234    [ "$DEBUG" ] && echo "DEBUG: mlovccf: main: Executed Headas initialisation script for $HEADAS"
235    
236    # The next statement should not be necessary because the SAS initialisation
237    # provides under $SAS_DIR/libextra all the necessary libraries from GNU C/C++
238    # which were required to build Heasoft. But, in order to make the HEADAS
239    # initialisation independent of the SAS initialisation, we set the
240    # LD_LIBRARY_PATH as it has to be to run Headas by itself without SAS been
241    # ready.
242    
243    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${SASBUILD_LOCAL}/gcc/lib64
244    
245    
246    headas_ok=`cat ${headas_output}`
247    
248    if [ -n "${headas_ok}" ] ; then
249            echo "mlovccf: HEADAS initialisation failed with error" >> ${mailfile}
250            cat ${headas_output}                                    >> ${mailfile}
251            mailnotify "mlovccf: HEADAS initilisation failed" ${mailfile}
252            if [ "$DEBUG" ] ; then
253                    echo "DEBUG: mlovccf: main: Headas initialization failed. Notification mail sent!"
254            fi
255            rm -v ${mailfile}
256            rm -v ${headas_output}
257            exit 1
258    fi
259    
260    # To be able to mix several jobs on the same ccflib account
261    [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
262    export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
263    if [ "$DEBUG" ] ; then
264            echo -n "DEBUG: mlovccf: main: HEADAS version: "`fversion`
265    fi
266    
267  find_latest_mif "${SAS_CCFPATH}"  #
268    #        W a r n i n g   o n   S A S   i n i t i a l i s a t i o n  
269    #
270    # The SAS initialisation produced by setsas.sh sets SAS_CCFPATH to /ccf/pub.
271    # Hence, if we set SAS_CCFPATH before running setsas.sh, we will always
272    # use /ccf/pub as repository for CCF. This is right as long as /ccf/pub is in
273    # strict sync with /home/ccflib/ccf_ftp/constituents. But if this is not true,
274    # the resulting valid CCFs might be wrong. Therefore, we set it here explicitly.
275    #
276    
277    confsas_output="${VALID_CCF}/confsas_output"
278    source /sas/bin/confsas > ${confsas_output}
279    confsas_ok=`cat ${confsas_output}`
280    
281  # Now scans the list of OBDS ID registered in the previous list to get the start time  if [ "$DEBUG" ] ; then
282  # and then runs cifbuild to get the respective CIF          echo "DEBUG: mlovccf: main: Executed /sas/bin/confsas with output: ${confsas_ok}"
283    fi
284    
285    if [ -n "${confsas_ok}" ] ; then
286            echo "mlovccf: SAS initialization failed with error" >> ${mailfile}
287            cat ${confsas_output}                                >> ${mailfile}
288            mailnotify "mlovccf: SAS initialisation failed" ${mailfile}
289            if [ "$DEBUG" ] ; then
290                    echo "DEBUG: mlovccf: main: SAS initialization failed. Notification mail sent!"
291            fi
292            rm -v ${mailfile}
293            rm -v ${confsas_output}
294            exit 1
295    fi
296    
297    source $HOME/setsas.sh 1> /dev/null
298    
299    if [ "$DEBUG" ] ; then
300            echo "DEBUG: mlovccf: main: SAS setsas.sh script executed"
301            echo
302            echo "DEBUG: mlovccf: main: SAS Version: "`sasversion -V 1`
303    fi
304    
305    
306    # Set SAS vervbosity to 0 from now onwards
307    
308    export SAS_VERBOSITY=0
309    
310    
311    # Re-sets SAS_CCFPATH
312    
313    export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
314    
315    
316    
317    # 3. XMM_CALINDEX file from SAS_CCFPATH
318    
319    MIF=`ls -1 ${SAS_CCFPATH}/XMM_CALINDEX_*.CCF`
320    
321    # 4. For each OBDS ID registered in the XSA observations list, gets the start
322    #    time of the observation. Then runs cifbuild to get the respective CIF.
323    #    all_ccfs_${now}.txt is filled recursively on each iteration.
324    #    Therefore we start with an empty file. The final list will be this one.
325    
326    #    Output file :          ${VALID_CCF}/all_ccfs_${now}.txt
327    
 # Output file  
328  touch ${VALID_CCF}/all_ccfs_${now}.txt  touch ${VALID_CCF}/all_ccfs_${now}.txt
329    
330  n=0  n=0
# Line 189  do Line 369  do
369    
370          cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt          cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt
371    
372            comm -1 -3 --nocheck-order ${VALID_CCF}/all_ccfs_${now}.tmp ${VALID_CCF}/all_ccfs_${now}.txt > ${VALID_CCF}/diff_iteration_${n}.txt
373            
374            diffsize=`stat --format=%s ${VALID_CCF}/diff_iteration_${n}.txt`
375            
376            if [ "$diffsize" != "0" ] ; then
377                    echo "CCF added in this iteration: "
378                    cat ${VALID_CCF}/diff_iteration_${n}.txt
379            fi      
380            
381            echo
382            echo "Number of valid CCFs found so far: "`cat ${VALID_CCF}/all_ccfs_${now}.txt | wc -l` ; echo
383    
384            rm ${VALID_CCF}/diff_iteration_${n}.txt
385          rm ${cif_file}          rm ${cif_file}
386          rm ${VALID_CCF}/${obsid}_ccfs.txt          rm ${VALID_CCF}/${obsid}_ccfs.txt
387          rm ${VALID_CCF}/all_ccfs_${now}.tmp          rm ${VALID_CCF}/all_ccfs_${now}.tmp
388    
389  done < ${VALID_CCF}/xsaobslog_${now}.txt  done < ${VALID_CCF}/xsaobslog_${now}.txt
390    
391  # Make a table of classes and issues type "Pipeline Release Notes"  # 5. Make a table of classes and issues with the style "Pipeline Release Notes"
392    #    To get a classification, we need first to obtain all available classes.
393    #
394    
395    
396  # Lists all CCF classes  # Lists all CCF classes
397    
# Line 210  done < ${VALID_CCF}/all_ccfs_${now}.txt Line 406  done < ${VALID_CCF}/all_ccfs_${now}.txt
406  cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt  cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt
407  mv ${VALID_CCF}/kk.txt ${ccf_classes}  mv ${VALID_CCF}/kk.txt ${ccf_classes}
408    
409    
410    
411  # Output final file is named all_ccfs_${now}_table.txt  # Output final file is named all_ccfs_${now}_table.txt
412    
413  total_number_of_ccfs=0  total_number_of_ccfs=0
# Line 277  echo "" Line 475  echo ""
475  echo "Total number of valid CCFs required : ${total_number_of_ccfs}"                                            >> ${all_ccfs_table}  echo "Total number of valid CCFs required : ${total_number_of_ccfs}"                                            >> ${all_ccfs_table}
476  rm ${VALID_CCF}/ccf_classes.txt  rm ${VALID_CCF}/ccf_classes.txt
477  rm ${VALID_CCF}/ccf_issues_for*.txt  rm ${VALID_CCF}/ccf_issues_for*.txt
478    
479    
480    
481    # 6. Fill in the Valid CCF Set dir with the proper CCFs
482    
483    #fillvalidccfdir "all_ccfs_${now}.txt"
484    
485    
486    #[ -f "${mailfile}" ] && rm -rf ${mailfile}
487    #echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile
488    #echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}"  >> $mailfile
489    #mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile
490    #rm ${mailfile}
Legend:
Removed from v.1.9  
changed lines
  Added in v.1.24
  ViewVC Help
Powered by ViewVC 1.1.27