/[CCFs]/bin/mlovccf
ViewVC logotype

Diff of /bin/mlovccf

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.2 by ccflib, Tue Mar 10 13:27:33 2015 UTC revision 1.21 by ccflib, Wed Jul 25 12:06:24 2018 UTC
# Line 2  Line 2 
2    
3  # mlovccf (make list of valid ccfs)  # mlovccf (make list of valid ccfs)
4  #  #
5  # Makes the list of valid CCFs that are all CCFs, among those published since the  # Makes the list of valid CCFs. These are all CCFs, among those published since the
6  # beginning of the project, that are required to process any ODF at a given date,  # beginning of the project, that are required to process any ODF at a given date,
7  # typically the current date. Thus, this list will change with time as far as new  # typically the current date. This list will change with time as far as new
8  # CCFs are issued to deal with new calibration data and/or replace obsolete ones.  # CCFs are issued to deal with new calibration data and/or replace obsolete ones.
9  #  #
10  # To identify the set of CCFs required to process a given ODF is the task of cifbuild.  # To identify the set of CCFs required to process a given ODF is the job of task cifbuild.
11  # To make such identification, cifbuild needs only the start time of the observation.  # To make such identification, cifbuild needs only the start time of the observation.
12    # Therefore, if we provide a list of all observed ODFs at a given date together with their
13    # respective start time, we could compute all the CCFs required to process them.
14    # For such a moment, that list is the list of valid CCFs required to process
15    # any observed ODF.
16  #  #
17  # If we get the list of required CCFs for all observed ODFs at a given date,  # We need as input the list of all observed ODFs at a given date.
18  # we will obtain for such a moment, the list of valid CCFs required to process  # This is obtained from the report of all observations that the XSA makes to CDS,
19  # any observed ODF. Thus, we need as input the list of all observed ODFs at a given  # xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt).
 # date. This is obtained from the report of all observations that the XSA makes to CDS,  
 # xsaobslog.txt (ftp://nxsa.esac.esa.int/pub/cfs_obslog/xsaobslog.txt).  
20  #  #
21  # $Id: mlovccf,v 1.1 2015/03/09 17:06:20 ccflib Exp $  # $Id: mlovccf,v 1.20 2018/07/25 10:24:15 ccflib Exp $
22    
23    # Next two lines allow to submit this script to the grid
24    # request Bourne shell as shell for job
25    #$ -S /bin/bash
26    
27    adminmail="eduardo.ojero@sciops.esa.int"
28    
29    VALID_CCF="$HOME/processing_valid_ccf"
30    
31    mailfile="${VALID_CCF}/mailfile"
32    
33  host=`hostname | cut -d. -f1`  host=`hostname | cut -d. -f1`
34    
35  # Function find_latest_mif to get the latest XMM_CALINDEX from given directory  now=`date +'%Y%m%d_%H%M'`
36    now_table=`date +'%Y-%m-%dT%H:%M:%S'`
37    
38    
39    # +++ Function mailnotify
40    #
41    # This function is used to notify adminmail on any issue while runnning this procedure
42    #
43    # Needs a subject as $1 and body as local file to be send.
44    
45  find_latest_mif()  mailnotify ()
46  {  {
47    
48  [ -z "$1" ] && return  [ "$#" -ne "2" ] && return
49    
50  CCFPATH="$1"  subject=$1
51    
52  maxissue="0"  bodyfile=$2
53  for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF`  
54  do  [ ! -f "${bodyfile}" ] && return
55          mifbase=`basename $mif`  
56          issue=`echo $mifbase | awk -F"." '{print $1}' | awk -F"_" '{print $3}'`  echo ; echo "Sending message to ${adminmail} with subject ${subject}" ; echo
57          [ "$issue" -ge "$maxissue" ] && maxissue="$issue"  
58  done  mailx -v -s "${subject}" "${adminmail}" < ${bodyfile}
59    
60    }
61    
62    
63    
64    # +++ Function getxsaobslog
65    #
66    # We need to get the list of all observed ODFs at the current date.
67    # This is obtained from http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt.
68    
69    getxsaobslog ()
70    {
71    
72    XSAOBSURL="http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt"
73    
74    # Remove any xsaobslog.txt previously downloaded.
75    
76  MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF  [ -f "${VALID_CCF}/xsaobslog.txt" ] && rm -rf ${VALID_CCF}/xsaobslog.txt
77    
78    # Get the list latest list of ODFs available from nXSA server.
79    
80    wget --quiet ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt
81    
82    # Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference
83    # of which list of ODFs was used to compute the list of valid CCFs.
84    
85    mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt
86    
87  }  }
88    
89  # Function get_ccf_list to list on stdout the table of CCFs in a given CIF  
90    
91    # +++Function get_ccf_list to list on stdout the table of CCFs in a given CIF.
92  #  #
93  # Requires that Heasoft is initialised. Otherwise exits with error.  # Requires that Heasoft is initialised. Otherwise exits with error.
94    
# Line 55  noversion=`which fversion | grep -c no` Line 102  noversion=`which fversion | grep -c no`
102    
103  if [ "${noversion}" != "0" ] ; then  if [ "${noversion}" != "0" ] ; then
104          echo "Error: Heasoft not initialised !"          echo "Error: Heasoft not initialised !"
105            [ -f "${mailfile}" ] && rm -rf ${mailfile}
106            echo "Running of fversion produced the following output:" >> $mailfile
107            echo "$noversion" >> $mailfile
108            mailnotify "mvloccf: Error: Heasoft not initialised!" $mailfile
109            rm ${mailfile}
110          exit          exit
111  fi  fi
112    
# Line 64  fdump ${cif_to_process} prhead=no showco Line 116  fdump ${cif_to_process} prhead=no showco
116  }  }
117    
118    
 # We need to get the list of all observed ODFs at the current date.  
 # This is obtained from ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt.  
119    
 VALID_CCF="$HOME/valid_ccf"  
120    
121  cd ${VALID_CCF}  # +++ Fill in the valid_constituents directory with the Valid CCF set
122  wget -nc -q ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt  #
123    # The directory to be filled in is ${VALIDCONSTITUENTSDISR}.
124    # This directory is synchronized with the similar directory in the
125    # external ftp server.
126    #
127    # In addition to that, we also use this function to fill in the
128    # /ccf/valid directory available for users in the intranet.
129    
130    fillvalidccfdir()
131    {
132    
133    [ -z "$1" ] && return
134    
135    validccflist=$1
136    
137    if [ ! -f "${VALID_CCF}/${validccflist}" -o ! -s "${VALID_CCF}/${validccflist}"  ] ; then
138            echo "Error: ${VALID_CCF}/${validccflist} not found or empty - Abort!"
139            [ -f "${mailfile}" ] && rm -rf ${mailfile}
140            echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found or it is empty" >> $mailfile
141            mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found or it is empty - Abort!" $mailfile
142            rm ${mailfile}
143            exit
144    fi
145    
146    VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents"
147    rm -rf ${VALIDCONSTITUENTSDIR}/*
148    
149    # This operation can only succeed because the subdir valid has enable the write permission
150    # for the group "sas" (g=rwx), provided that CCFs belong to sasbuild(5153):sas.
151    VALIDCCFDIRINTRANET="/ccf/valid"
152    rm -rf ${VALIDCCFDIRINTRANET}/*
153    
154    CONSTITUENTSDIR="/home/ccflib/ftp-area/constituents"
155    n=0
156    while read ccf
157    do
158            n=$((n + 1))
159            echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/"
160            cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/
161            echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/"
162            cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/
163    done < ${VALID_CCF}/${validccflist}
164    
165    }
166    
167    
168    #
169    #   M a i n    p r o g r a m
170    #
171    # 1. Get the list of all ODFs
172    
173    
174    xsaobslogsize="0"
175    
176    getxsaobslog
177    
178    xsaobslogsize=`stat --format=%s ${VALID_CCF}/xsaobslog_${now}.txt`
179    
180  # Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host  if [ "$xsaobslogsize"  = "0" ] ; then
181            echo "Error: Failure to download the XSA Obs. Log file - Abort"
182            [ -f "${mailfile}" ] && rm -rf ${mailfile}
183            echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile
184            echo "." >> $mailfile
185            mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile"
186            rm ${mailfile}
187            exit
188    fi
189    
190    #
191    # 2. Initialises HEADAS and SAS
192    #
193    
194    # Defines SASBUILD_LOCAL depending on host.
195    # Then uses it to build up HEADAS and setup LD_LIBRARY_PATH accordingly.
196    
197  case "$host" in  case "$host" in
198          xvsoc01|xmm)          sasbld01n|sasbld02n)
199                  SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"                  export SASBUILD_LOCAL="/sasbuild/local/${host}/GNU_CC_CXX_7.1"
                 SAS_DIR=/data/xmm/ccflib/sas  
                 SAS_PATH=$SAS_DIR  
                 source $SAS_DIR/sas-setup.sh  
200                  ;;                  ;;
201          sasbld01|sasbld02)          scigrid6|cn-*)
202                  SAS_CCFPATH="/home/ccflib/ftp-area/constituents"                  export SASBUILD_LOCAL="/sasbuild/local/sasbld02n/GNU_CC_CXX_7.1"
                 /sas/bin/confsas  
                 export HEADAS=/sasbuild/local/${host}/headas/architecture  
                 . $HEADAS/headas-init.sh  
                 source $HOME/setsas.sh 1> /dev/null  
                 export SAS_VERBOSITY=0  
203                  ;;                  ;;
204            *) echo "Error: Unsupported host $host" ; exit 1 ;;
205  esac  esac
206    
207  # Finds the latest MIF issue  export HEADAS=${SASBUILD_LOCAL}/headas/architecture
208    
209    headas_output="${VALID_CCF}/headas_output"
210    source $HEADAS/headas-init.sh > ${headas_output} 2>&1
211    
212    # The next statement should not be necessary because the SAS initialisation
213    # provides under $SAS_DIR/libextra all the necessary libraries from GNU C/C++
214    # which were required to build Heasoft. But, in order to make the HEADAS
215    # initialisation independent of the SAS initialisation, we set the
216    # LD_LIBRARY_PATH as it has to be to run Headas by itself without SAS been
217    # ready.
218    
219    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${SASBUILD_LOCAL}/gcc/lib64
220    
221    
222    headas_ok=`cat ${headas_output}`
223    
224    if [ -n "${headas_ok}" ] ; then
225            echo "mlovccf: HEADAS initialisation failed with error" >> ${mailfile}
226            cat ${headas_output}                                    >> ${mailfile}
227            mailnotify "mlovccf: HEADAS initilisation failed" ${mailfile}
228            rm ${headas_output} ${mailfile}
229            exit 1
230    fi
231    
232    # To be able to mix several jobs on the same ccflib account
233    [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
234    export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
235    echo -n "HEADAS version: "
236    fversion
237    
238    #
239    #        W a r n i n g   o n   S A S   i n i t i a l i s a t i o n  
240    #
241    # The SAS initialisation produced by setsas.sh sets SAS_CCFPATH to /ccf/pub.
242    # Hence, if we set SAS_CCFPATH before running setsas.sh, we will always
243    # use /ccf/pub as repository for CCF. This is right as long as /ccf/pub is in
244    # strict sync with /home/ccflib/ccf_ftp/constituents. But if this is not true,
245    # the resulting valid CCFs might be wrong. Therefore, we set it here explicitly.
246    #
247    
248    confsas_output="${VALID_CCF}/confsas_output"
249    /sas/bin/confsas > ${confsas_output} 2>&1
250    confsas_ok=`cat ${confsas_output}`
251    
252    if [ -n "${confsas_ok}" ] ; then
253            echo "mlovccf: SAS initialization failed with error" >> ${mailfile}
254            cat ${confsas_output}                                >> ${mailfile}
255            mailnotify "mlovccf: SAS initialisation failed" ${mailfile}
256            rm ${confsas_output} ${mailfile}
257            exit 1
258    fi
259    
260    source $HOME/setsas.sh 1> /dev/null
261    
262    # Re-sets SAS_CCFPATH
263    
264    export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
265    
266    echo "SAS Version: "
267    sasversion -V 1
268    export SAS_VERBOSITY=0
269    
270    
271  find_latest_mif "${SAS_CCFPATH}"  # 3. XMM_CALINDEX file from SAS_CCFPATH
272    
273    MIF=`ls -1 ${SAS_CCFPATH}/XMM_CALINDEX_*.CCF`
274    
275  # Now scans the list of OBDS ID registered in the previous list to get the start time  # 4. For each OBDS ID registered in the XSA observations list, gets the start
276  # and then runs cifbuild to get the respective CIF  #    time of the observation. Then runs cifbuild to get the respective CIF.
277    #    all_ccfs_${now}.txt is filled recursively on each iteration.
278    #    Therefore we start with an empty file. The final list will be this one.
279    
280  # Output file  #    Output file :          ${VALID_CCF}/all_ccfs_${now}.txt
281  touch ${VALID_CCF}/all_ccfs.txt  
282    touch ${VALID_CCF}/all_ccfs_${now}.txt
283    
284  n=0  n=0
285    
# Line 141  do Line 319  do
319    
320          rm ${VALID_CCF}/${obsid}_ccfs.tmp          rm ${VALID_CCF}/${obsid}_ccfs.tmp
321    
322          mv ${VALID_CCF}/all_ccfs.txt ${VALID_CCF}/all_ccfs.tmp          mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp
323    
324            cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt
325    
326          cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs.tmp | sort -u  >  ${VALID_CCF}/all_ccfs.txt          comm -1 -3 --nocheck-order ${VALID_CCF}/all_ccfs_${now}.tmp ${VALID_CCF}/all_ccfs_${now}.txt > ${VALID_CCF}/diff_iteration_${n}.txt
327            
328            diffsize=`stat --format=%s ${VALID_CCF}/diff_iteration_${n}.txt`
329            
330            if [ "$diffsize" != "0" ] ; then
331                    echo "CCF added in this iteration: "
332                    cat ${VALID_CCF}/diff_iteration_${n}.txt
333            fi      
334            
335            echo
336            echo "Number of valid CCFs found so far: "`cat ${VALID_CCF}/all_ccfs_${now}.txt | wc -l` ; echo
337    
338            rm ${VALID_CCF}/diff_iteration_${n}.txt
339          rm ${cif_file}          rm ${cif_file}
340          rm ${VALID_CCF}/${obsid}_ccfs.txt          rm ${VALID_CCF}/${obsid}_ccfs.txt
341          rm ${VALID_CCF}/all_ccfs.tmp          rm ${VALID_CCF}/all_ccfs_${now}.tmp
342    
343    done < ${VALID_CCF}/xsaobslog_${now}.txt
344    
345  done < ${VALID_CCF}/xsaobslog.txt  # 5. Make a table of classes and issues with the style "Pipeline Release Notes"
346    #    To get a classification, we need first to obtain all available classes.
347    #
348    
 # Make a table of classes and issues type "Pipeline Release Notes"  
349    
350  # Lists all CCF classes  # Lists all CCF classes
351    
# Line 161  while read line Line 355  while read line
355  do  do
356          class=`echo $line | awk -F"_" '{print $1"_"$2}'`          class=`echo $line | awk -F"_" '{print $1"_"$2}'`
357          echo $class >> ${ccf_classes}          echo $class >> ${ccf_classes}
358  done < ${VALID_CCF}/all_ccfs.txt  done < ${VALID_CCF}/all_ccfs_${now}.txt
359    
360  cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt  cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt
361  mv ${VALID_CCF}/kk.txt ${ccf_classes}  mv ${VALID_CCF}/kk.txt ${ccf_classes}
362    
 # Output final file is named all_ccfs_table.txt  
363    
364  all_ccfs_table="{VALID_CCF}/all_ccfs_table.txt  
365    # Output final file is named all_ccfs_${now}_table.txt
366    
367    total_number_of_ccfs=0
368    
369    all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt"
370    
371  [ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}  [ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}
372    
373  echo "|================================|=============|" >> ${all_ccfs_table}  echo "Table of valid CCFs at $now_table"                        >> ${all_ccfs_table}
374  echo "| Calibration File               | Issue range |" >> ${all_ccfs_table}  echo                                                            >> ${all_ccfs_table}
375  echo "|================================|=============|" >> ${all_ccfs_table}  echo "|================================|=============|=======|" >> ${all_ccfs_table}
376    echo "| Calibration File               | Issue range | Count |" >> ${all_ccfs_table}
377    echo "|================================|=============|=======|" >> ${all_ccfs_table}
378    
379  while read ccf_class  while read ccf_class
380  do  do
381            echo -n "Processing class $ccf_class ..."
382          issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"          issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
383          [ -f "${issues_file}" ] && rm ${issues_file}          [ -f "${issues_file}" ] && rm ${issues_file}
384          while read line          while read line
# Line 186  do Line 387  do
387                  [ "$class" != "$ccf_class" ] && continue                  [ "$class" != "$ccf_class" ] && continue
388                  issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`                  issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`
389                  echo $issue >> ${issues_file}                  echo $issue >> ${issues_file}
390          done < ${VALID_CCF}/all_ccfs.txt          done < ${VALID_CCF}/all_ccfs_${now}.txt
391    
392          cat ${issues_file} | sort -u > kk.txt          cat ${issues_file} | sort -u > kk.txt
393          mv kk.txt ${issues_file}          mv kk.txt ${issues_file}
394            real_number_of_issues_for_this_class=`wc -l ${issues_file} | awk -F" " '{print $1}'`
395    
396          first_issue=`head -1 ${issues_file}`          first_issue=`head -1 ${issues_file}`
397          last_issue=`tail -1 ${issues_file}`          last_issue=`tail -1 ${issues_file}`
398    
399          if [ "${first_issue}" != "${last_issue}" ] ; then          li=$((10#$last_issue))
400                  printf "| %-30s | %4s-%4s   |\n" ${ccf_class} ${first_issue} ${last_issue} >> ${all_ccfs_table}          fi=$((10#$first_issue))
401          else  
402                  printf "| %-30s | %4s        |\n" ${ccf_class} ${first_issue}              >> ${all_ccfs_table}          sequential_number_of_issues_for_this_class=$((li - fi + 1))
403          fi  
404              asterisk="   "
405            [ "${real_number_of_issues_for_this_class}"  -lt "${sequential_number_of_issues_for_this_class}" ] &&  asterisk="(*)"
406    
407            total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs))
408    
409            rn=$real_number_of_issues_for_this_class
410    
411            if [ "${first_issue}" != "${last_issue}" ] ; then
412               if [ "$asterisk" = "(*)" ] ; then
413                  printf "| %-30s | %4s-%4s(*)|  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
414               else
415                  printf "| %-30s | %4s-%4s   |  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
416               fi
417            else
418               printf "| %-30s | %4s        |  %4s |\n" ${ccf_class} ${first_issue} ${rn}                           >> ${all_ccfs_table}
419            fi
420    
421            
422            echo "done"
423  done < ${ccf_classes}    done < ${ccf_classes}  
424    
425  echo "|================================|=============|"                                    >> ${all_ccfs_table}  echo "|================================|=============|=======|"                                                 >> ${all_ccfs_table}
426    echo                                                                                                            >> ${all_ccfs_table}
427    echo "(*): Incomplete range."                                                                                   >> ${all_ccfs_table}
428    echo ""                                                                                                         >> ${all_ccfs_table}
429    echo "Total number of valid CCFs required : ${total_number_of_ccfs}"                                            >> ${all_ccfs_table}
430    rm ${VALID_CCF}/ccf_classes.txt
431    rm ${VALID_CCF}/ccf_issues_for*.txt
432    
433    
434    
435    # 6. Fill in the Valid CCF Set dir with the proper CCFs
436    
437    fillvalidccfdir "all_ccfs_${now}.txt"
438    
439    [ -f "${mailfile}" ] && rm -rf ${mailfile}
440    echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile
441    echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}"  >> $mailfile
442    mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile
443    rm ${mailfile}
Legend:
Removed from v.1.2  
changed lines
  Added in v.1.21
  ViewVC Help
Powered by ViewVC 1.1.27