/[CCFs]/bin/mlovccf
ViewVC logotype

Diff of /bin/mlovccf

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.2 by ccflib, Tue Mar 10 13:27:33 2015 UTC revision 1.14 by ccflib, Mon Apr 4 09:33:19 2016 UTC
# Line 2  Line 2 
2    
3  # mlovccf (make list of valid ccfs)  # mlovccf (make list of valid ccfs)
4  #  #
5  # Makes the list of valid CCFs that are all CCFs, among those published since the  # Makes the list of valid CCFs. These are all CCFs, among those published since the
6  # beginning of the project, that are required to process any ODF at a given date,  # beginning of the project, that are required to process any ODF at a given date,
7  # typically the current date. Thus, this list will change with time as far as new  # typically the current date. This list will change with time as far as new
8  # CCFs are issued to deal with new calibration data and/or replace obsolete ones.  # CCFs are issued to deal with new calibration data and/or replace obsolete ones.
9  #  #
10  # To identify the set of CCFs required to process a given ODF is the task of cifbuild.  # To identify the set of CCFs required to process a given ODF is the job of task cifbuild.
11  # To make such identification, cifbuild needs only the start time of the observation.  # To make such identification, cifbuild needs only the start time of the observation.
12    # Therefore, if we provide a list of all observed ODFs at a given date together with their
13    # respective start time, we could compute all the CCFs required to process them.
14    # For such a moment, that list is the list of valid CCFs required to process
15    # any observed ODF.
16  #  #
17  # If we get the list of required CCFs for all observed ODFs at a given date,  # We need as input the list of all observed ODFs at a given date.
18  # we will obtain for such a moment, the list of valid CCFs required to process  # This is obtained from the report of all observations that the XSA makes to CDS,
19  # any observed ODF. Thus, we need as input the list of all observed ODFs at a given  # xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt).
 # date. This is obtained from the report of all observations that the XSA makes to CDS,  
 # xsaobslog.txt (ftp://nxsa.esac.esa.int/pub/cfs_obslog/xsaobslog.txt).  
20  #  #
21  # $Id: mlovccf,v 1.1 2015/03/09 17:06:20 ccflib Exp $  # $Id: mlovccf,v 1.13 2016/04/01 12:22:28 ccflib Exp $
22    
23    # Next two lines allow to submit this script to the grid
24    # request Bourne shell as shell for job
25    #$ -S /bin/bash
26    
27    adminmail="eduardo.ojero@esa.int"
28    
29    VALID_CCF="$HOME/valid_ccf"
30    
31    mailfile="${VALID_CCF}/mailfile"
32    
33  host=`hostname | cut -d. -f1`  host=`hostname | cut -d. -f1`
34    
35  # Function find_latest_mif to get the latest XMM_CALINDEX from given directory  now=`date +'%Y%m%d_%H%M'`
36    now_table=`date +'%Y-%m-%dT%H:%M:%S'`
37    
38    
39    # +++ Function mailnotify
40    #
41    # This function is used to notify adminmail on any issue while runnning this procedure
42    #
43    # Needs a subject as $1 and body as local file to be send.
44    
45    mailnotify ()
46    {
47    
48    [ "$#" -ne "2" ] && return
49    
50    subject=$1
51    
52    bodyfile=$2
53    
54    [ ! -f "${bodyfile}" ] && return
55    
56    mailx -s "${subject}" "${adminmail}" < ${bodyfile}
57    
58    }
59    
60    
61    
62    # +++ Function getxsaobslog
63    #
64    # We need to get the list of all observed ODFs at the current date.
65    # This is obtained from http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt.
66    
67    getxsaobslog ()
68    {
69    
70    XSAOBSURL="http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt"
71    
72    # Remove any xsaobslog.txt previously downloaded.
73    
74    [ -f "${VALID_CCF}/xsaobslog.txt" ] && rm -rf ${VALID_CCF}/xsaobslog.txt
75    
76    # Get the list latest list of ODFs available from nXSA server.
77    
78    wget -q ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt
79    
80    # Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference
81    # of which list of ODFs was used to compute the list of valid CCFs.
82    
83    mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt
84    
85    }
86    
87    
88    
89    # +++ Function find_latest_mif to get the latest XMM_CALINDEX from given directory
90    
91  find_latest_mif()  find_latest_mif()
92  {  {
# Line 41  MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}. Line 107  MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.
107    
108  }  }
109    
110  # Function get_ccf_list to list on stdout the table of CCFs in a given CIF  
111    
112    # +++Function get_ccf_list to list on stdout the table of CCFs in a given CIF.
113  #  #
114  # Requires that Heasoft is initialised. Otherwise exits with error.  # Requires that Heasoft is initialised. Otherwise exits with error.
115    
# Line 55  noversion=`which fversion | grep -c no` Line 123  noversion=`which fversion | grep -c no`
123    
124  if [ "${noversion}" != "0" ] ; then  if [ "${noversion}" != "0" ] ; then
125          echo "Error: Heasoft not initialised !"          echo "Error: Heasoft not initialised !"
126            [ -f "${mailfile}" ] && rm -rf ${mailfile}
127            echo "Running of fversion produced the following output:" >> $mailfile
128            echo "$noversion" >> $mailfile
129            mailnotify "mvloccf: Error: Heasoft not initialised!" $mailfile
130            rm ${mailfile}
131          exit          exit
132  fi  fi
133    
# Line 64  fdump ${cif_to_process} prhead=no showco Line 137  fdump ${cif_to_process} prhead=no showco
137  }  }
138    
139    
 # We need to get the list of all observed ODFs at the current date.  
 # This is obtained from ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt.  
140    
 VALID_CCF="$HOME/valid_ccf"  
141    
142  cd ${VALID_CCF}  # +++ Fill in the valid_constituents directory with the Valid CCF set
143  wget -nc -q ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt  #
144    # The directory to be filled in is ${VALIDCONSTITUENTSDISR}.
145    # This directory is synchronized with the similar directory in the
146    # external ftp server.
147    #
148    # In addition to that, we also use this function to fill in the
149    # /ccf/valid directory available for users in the intranet.
150    
151    fillvalidccfdir()
152    {
153    
154  # Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host  [ -z "$1" ] && return
155    
156    validccflist=$1
157    
158    if [ ! -f "${VALID_CCF}/${validccflist}"  ] ; then
159            echo "Error: ${VALID_CCF}/${validccflist} not found - Abort!"
160            [ -f "${mailfile}" ] && rm -rf ${mailfile}
161            echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found" >> $mailfile
162            mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found - Abort!" $mailfile
163            rm ${mailfile}
164            exit
165    fi
166    
167    VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents"
168    rm -rf ${VALIDCONSTITUENTSDIR}/*
169    
170    VALIDCCFDIRINTRANET="/ccf/valid"
171    rm -rf ${VALIDCCFDIRINTRANET}/*
172    
173    CONSTITUENTSDIR="/home/ccflib/ftp-area/constituents"
174    n=0
175    while read ccf
176    do
177            n=$((n + 1))
178            echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/"
179            cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/
180            echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/"
181            cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/
182    done < ${VALID_CCF}/${validccflist}
183    
184    }
185    
186    
187    #
188    #   M a i n    p r o g r a m
189    #
190    # 1. Get the list of all ODFs
191    
192    
193    xsaobslogsize="0"
194    
195    getxsaobslog
196    
197    xsaobslogsize=`stat --format=%s ${VALID_CCF}/xsaobslog_${now}.txt`
198    
199    if [ "$xsaobslogsize"  = "0" ] ; then
200            echo "Error: Failure to download the XSA Obs. Log file - Abort"
201            [ -f "${mailfile}" ] && rm -rf ${mailfile}
202            echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile
203            echo "." >> $mailfile
204            mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile"
205            rm ${mailfile}
206            exit
207    fi
208    
209    # 2. Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host
210    
211  case "$host" in  case "$host" in
212          xvsoc01|xmm)          xvsoc01|xmm)
213                  SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"                  export SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"
214                  SAS_DIR=/data/xmm/ccflib/sas                  export SAS_DIR=/data/xmm/ccflib/sas
215                  SAS_PATH=$SAS_DIR                  export SAS_PATH=$SAS_DIR
216                  source $SAS_DIR/sas-setup.sh                  source $SAS_DIR/sas-setup.sh
217                  ;;                  ;;
218          sasbld01|sasbld02)          sasbld01|sasbld02)
219                  SAS_CCFPATH="/home/ccflib/ftp-area/constituents"                  export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
220                  /sas/bin/confsas                  /sas/bin/confsas
221                  export HEADAS=/sasbuild/local/${host}/headas/architecture                  export HEADAS=/sasbuild/local/${host}/headas/architecture
222                  . $HEADAS/headas-init.sh                  . $HEADAS/headas-init.sh
223                    # To be able to mix several jobs on the same ccflib account
224                    [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
225                    export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
226                    echo -n "HEADAS version: "
227                    fversion
228                  source $HOME/setsas.sh 1> /dev/null                  source $HOME/setsas.sh 1> /dev/null
229                    echo "SAS Version: "
230                    sasversion -V 1
231                  export SAS_VERBOSITY=0                  export SAS_VERBOSITY=0
232                  ;;                  ;;
233            scigrid6|cn-*)
234                    export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
235                    /sas/bin/confsas
236                    export HEADAS=/sasbuild/local/sasbld02/headas/architecture
237                    . $HEADAS/headas-init.sh
238                    # To be able to mix several jobs on the same ccflib account
239                    [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
240                    export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
241                    echo -n "HEADAS version: "
242                    fversion
243                    source $HOME/setsas.sh 1> /dev/null
244                    echo "SAS Version: "
245                    sasversion -V 1
246                    export SAS_VERBOSITY=0
247                    ;;
248            *) echo "Error: Do not know how to do it in host $host" ; exit ;;
249  esac  esac
250    
251  # Finds the latest MIF issue  # 3. Finds the latest MIF issue
252    
253  find_latest_mif "${SAS_CCFPATH}"  find_latest_mif "${SAS_CCFPATH}"
254    
255    
256  # Now scans the list of OBDS ID registered in the previous list to get the start time  # 4. Scans the list of OBDS ID registered in the previous list to get the start time
257  # and then runs cifbuild to get the respective CIF  #    Then runs cifbuild to get the respective CIF.
258    #    all_ccfs_${now}.txt is filled recursively on each iteration. Therefore we start
259    #    with an empty file.
260    
261    #    Output file :          ${VALID_CCF}/all_ccfs_${now}.txt
262    
263  # Output file  touch ${VALID_CCF}/all_ccfs_${now}.txt
 touch ${VALID_CCF}/all_ccfs.txt  
264    
265  n=0  n=0
266    
# Line 141  do Line 300  do
300    
301          rm ${VALID_CCF}/${obsid}_ccfs.tmp          rm ${VALID_CCF}/${obsid}_ccfs.tmp
302    
303          mv ${VALID_CCF}/all_ccfs.txt ${VALID_CCF}/all_ccfs.tmp          mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp
304    
305          cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs.tmp | sort -u  >  ${VALID_CCF}/all_ccfs.txt          cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt
306    
307          rm ${cif_file}          rm ${cif_file}
308          rm ${VALID_CCF}/${obsid}_ccfs.txt          rm ${VALID_CCF}/${obsid}_ccfs.txt
309          rm ${VALID_CCF}/all_ccfs.tmp          rm ${VALID_CCF}/all_ccfs_${now}.tmp
310    
311  done < ${VALID_CCF}/xsaobslog.txt  done < ${VALID_CCF}/xsaobslog_${now}.txt
312    
313    # 5. Make a table of classes and issues with the style "Pipeline Release Notes"
314    #    To get a classification, we need first to obtain all available classes.
315    #
316    
 # Make a table of classes and issues type "Pipeline Release Notes"  
317    
318  # Lists all CCF classes  # Lists all CCF classes
319    
# Line 161  while read line Line 323  while read line
323  do  do
324          class=`echo $line | awk -F"_" '{print $1"_"$2}'`          class=`echo $line | awk -F"_" '{print $1"_"$2}'`
325          echo $class >> ${ccf_classes}          echo $class >> ${ccf_classes}
326  done < ${VALID_CCF}/all_ccfs.txt  done < ${VALID_CCF}/all_ccfs_${now}.txt
327    
328  cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt  cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt
329  mv ${VALID_CCF}/kk.txt ${ccf_classes}  mv ${VALID_CCF}/kk.txt ${ccf_classes}
330    
 # Output final file is named all_ccfs_table.txt  
331    
332  all_ccfs_table="{VALID_CCF}/all_ccfs_table.txt  
333    # Output final file is named all_ccfs_${now}_table.txt
334    
335    total_number_of_ccfs=0
336    
337    all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt"
338    
339  [ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}  [ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}
340    
341  echo "|================================|=============|" >> ${all_ccfs_table}  echo "Table of valid CCFs at $now_table"                        >> ${all_ccfs_table}
342  echo "| Calibration File               | Issue range |" >> ${all_ccfs_table}  echo                                                            >> ${all_ccfs_table}
343  echo "|================================|=============|" >> ${all_ccfs_table}  echo "|================================|=============|=======|" >> ${all_ccfs_table}
344    echo "| Calibration File               | Issue range | Count |" >> ${all_ccfs_table}
345    echo "|================================|=============|=======|" >> ${all_ccfs_table}
346    
347  while read ccf_class  while read ccf_class
348  do  do
349            echo -n "Processing class $ccf_class ..."
350          issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"          issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
351          [ -f "${issues_file}" ] && rm ${issues_file}          [ -f "${issues_file}" ] && rm ${issues_file}
352          while read line          while read line
# Line 186  do Line 355  do
355                  [ "$class" != "$ccf_class" ] && continue                  [ "$class" != "$ccf_class" ] && continue
356                  issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`                  issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`
357                  echo $issue >> ${issues_file}                  echo $issue >> ${issues_file}
358          done < ${VALID_CCF}/all_ccfs.txt          done < ${VALID_CCF}/all_ccfs_${now}.txt
359    
360          cat ${issues_file} | sort -u > kk.txt          cat ${issues_file} | sort -u > kk.txt
361          mv kk.txt ${issues_file}          mv kk.txt ${issues_file}
362            real_number_of_issues_for_this_class=`wc -l ${issues_file} | awk -F" " '{print $1}'`
363    
364          first_issue=`head -1 ${issues_file}`          first_issue=`head -1 ${issues_file}`
365          last_issue=`tail -1 ${issues_file}`          last_issue=`tail -1 ${issues_file}`
366    
367          if [ "${first_issue}" != "${last_issue}" ] ; then          li=$((10#$last_issue))
368                  printf "| %-30s | %4s-%4s   |\n" ${ccf_class} ${first_issue} ${last_issue} >> ${all_ccfs_table}          fi=$((10#$first_issue))
369          else  
370                  printf "| %-30s | %4s        |\n" ${ccf_class} ${first_issue}              >> ${all_ccfs_table}          sequential_number_of_issues_for_this_class=$((li - fi + 1))
371          fi  
372              asterisk="   "
373            [ "${real_number_of_issues_for_this_class}"  -lt "${sequential_number_of_issues_for_this_class}" ] &&  asterisk="(*)"
374    
375            total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs))
376    
377            rn=$real_number_of_issues_for_this_class
378    
379            if [ "${first_issue}" != "${last_issue}" ] ; then
380               if [ "$asterisk" = "(*)" ] ; then
381                  printf "| %-30s | %4s-%4s(*)|  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
382               else
383                  printf "| %-30s | %4s-%4s   |  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
384               fi
385            else
386               printf "| %-30s | %4s        |  %4s |\n" ${ccf_class} ${first_issue} ${rn}                           >> ${all_ccfs_table}
387            fi
388    
389            
390            echo "done"
391  done < ${ccf_classes}    done < ${ccf_classes}  
392    
393  echo "|================================|=============|"                                    >> ${all_ccfs_table}  echo "|================================|=============|=======|"                                                 >> ${all_ccfs_table}
394    echo                                                                                                            >> ${all_ccfs_table}
395    echo "(*): Incomplete range."                                                                                   >> ${all_ccfs_table}
396    echo ""                                                                                                         >> ${all_ccfs_table}
397    echo "Total number of valid CCFs required : ${total_number_of_ccfs}"                                            >> ${all_ccfs_table}
398    rm ${VALID_CCF}/ccf_classes.txt
399    rm ${VALID_CCF}/ccf_issues_for*.txt
400    
401    
402    
403    # 6. Fill in the Valid CCF Set dir with the proper CCFs
404    
405    fillvalidccfdir "all_ccfs_${now}.txt"
406    
407    [ -f "${mailfile}" ] && rm -rf ${mailfile}
408    echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile
409    echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}"  >> $mailfile
410    mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile
411    rm ${mailfile}
Legend:
Removed from v.1.2  
changed lines
  Added in v.1.14
  ViewVC Help
Powered by ViewVC 1.1.27