/[CCFs]/bin/mlovccf
ViewVC logotype

Diff of /bin/mlovccf

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.2 by ccflib, Tue Mar 10 13:27:33 2015 UTC revision 1.7 by ccflib, Thu Aug 27 16:47:16 2015 UTC
# Line 2  Line 2 
2    
3  # mlovccf (make list of valid ccfs)  # mlovccf (make list of valid ccfs)
4  #  #
5  # Makes the list of valid CCFs that are all CCFs, among those published since the  # Makes the list of valid CCFs. These are all CCFs, among those published since the
6  # beginning of the project, that are required to process any ODF at a given date,  # beginning of the project, that are required to process any ODF at a given date,
7  # typically the current date. Thus, this list will change with time as far as new  # typically the current date. This list will change with time as far as new
8  # CCFs are issued to deal with new calibration data and/or replace obsolete ones.  # CCFs are issued to deal with new calibration data and/or replace obsolete ones.
9  #  #
10  # To identify the set of CCFs required to process a given ODF is the task of cifbuild.  # To identify the set of CCFs required to process a given ODF is the ijob of task cifbuild.
11  # To make such identification, cifbuild needs only the start time of the observation.  # To make such identification, cifbuild needs only the start time of the observation.
12    # Therefore, if we provide a list of all observed ODFs at a given date together with their
13    # respective start time, we could compute all the CCFs required to process them.
14    # For such a moment, that list is the list of valid CCFs required to process
15    # any observed ODF.
16  #  #
17  # If we get the list of required CCFs for all observed ODFs at a given date,  # We need as input the list of all observed ODFs at a given date.
18  # we will obtain for such a moment, the list of valid CCFs required to process  # This is obtained from the report of all observations that the XSA makes to CDS,
 # any observed ODF. Thus, we need as input the list of all observed ODFs at a given  
 # date. This is obtained from the report of all observations that the XSA makes to CDS,  
19  # xsaobslog.txt (ftp://nxsa.esac.esa.int/pub/cfs_obslog/xsaobslog.txt).  # xsaobslog.txt (ftp://nxsa.esac.esa.int/pub/cfs_obslog/xsaobslog.txt).
20  #  #
21  # $Id: mlovccf,v 1.1 2015/03/09 17:06:20 ccflib Exp $  # $Id: mlovccf,v 1.6 2015/03/12 16:02:52 ccflib Exp $
22    
23    # Next two lines allow to submit this script to the grid
24    # request Bourne shell as shell for job
25    #$ -S /bin/bash
26    
27    
28  host=`hostname | cut -d. -f1`  host=`hostname | cut -d. -f1`
29    
30    now=`date +'%Y%m%d_%H%M'`
31    now_table=`date +'%Y-%m-%dT%H:%M:%S'`
32    
33  # Function find_latest_mif to get the latest XMM_CALINDEX from given directory  # Function find_latest_mif to get the latest XMM_CALINDEX from given directory
34    
35  find_latest_mif()  find_latest_mif()
# Line 70  fdump ${cif_to_process} prhead=no showco Line 80  fdump ${cif_to_process} prhead=no showco
80  VALID_CCF="$HOME/valid_ccf"  VALID_CCF="$HOME/valid_ccf"
81    
82  cd ${VALID_CCF}  cd ${VALID_CCF}
 wget -nc -q ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt  
83    
84    # Remove any xsaobslog.txt previously downloaded.
85    
86    [ -f "xsaobslog.txt" ] && rm -rf xsaobslog.txt
87    
88    # Get the list latest list of ODFs available from nXSA server.
89    
90    wget -q ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt
91    
92    # Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference
93    # of which list of ODFs was used to compute the list of valid CCFs.
94    
95    mv xsaobslog.txt xsaobslog_${now}.txt
96    
97  # Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host  # Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host
98    
99  case "$host" in  case "$host" in
100          xvsoc01|xmm)          xvsoc01|xmm)
101                  SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"                  export SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"
102                  SAS_DIR=/data/xmm/ccflib/sas                  export SAS_DIR=/data/xmm/ccflib/sas
103                  SAS_PATH=$SAS_DIR                  export SAS_PATH=$SAS_DIR
104                  source $SAS_DIR/sas-setup.sh                  source $SAS_DIR/sas-setup.sh
105                  ;;                  ;;
106          sasbld01|sasbld02)          sasbld01|sasbld02)
107                  SAS_CCFPATH="/home/ccflib/ftp-area/constituents"                  export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
108                  /sas/bin/confsas                  /sas/bin/confsas
109                  export HEADAS=/sasbuild/local/${host}/headas/architecture                  export HEADAS=/sasbuild/local/${host}/headas/architecture
110                  . $HEADAS/headas-init.sh                  . $HEADAS/headas-init.sh
111                    # To be able to mix several jobs on the same ccflib account
112                    [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
113                    export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
114                    echo -n "HEADAS version: "
115                    fversion
116                  source $HOME/setsas.sh 1> /dev/null                  source $HOME/setsas.sh 1> /dev/null
117                    echo "SAS Version: "
118                    sasversion -V 1
119                  export SAS_VERBOSITY=0                  export SAS_VERBOSITY=0
120                  ;;                  ;;
121            scigrid6|cn-*)
122                    export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
123                    /sas/bin/confsas
124                    export HEADAS=/sasbuild/local/sasbld02/headas/architecture
125                    . $HEADAS/headas-init.sh
126                    # To be able to mix several jobs on the same ccflib account
127                    [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
128                    export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
129                    echo -n "HEADAS version: "
130                    fversion
131                    source $HOME/setsas.sh 1> /dev/null
132                    echo "SAS Version: "
133                    sasversion -V 1
134                    export SAS_VERBOSITY=0
135                    ;;
136            *) echo "Error: Do not know how to do it in host $host" ; exit ;;
137  esac  esac
138    
139  # Finds the latest MIF issue  # Finds the latest MIF issue
# Line 101  find_latest_mif "${SAS_CCFPATH}" Line 145  find_latest_mif "${SAS_CCFPATH}"
145  # and then runs cifbuild to get the respective CIF  # and then runs cifbuild to get the respective CIF
146    
147  # Output file  # Output file
148  touch ${VALID_CCF}/all_ccfs.txt  touch ${VALID_CCF}/all_ccfs_${now}.txt
149    
150  n=0  n=0
151    
# Line 141  do Line 185  do
185    
186          rm ${VALID_CCF}/${obsid}_ccfs.tmp          rm ${VALID_CCF}/${obsid}_ccfs.tmp
187    
188          mv ${VALID_CCF}/all_ccfs.txt ${VALID_CCF}/all_ccfs.tmp          mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp
189    
190          cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs.tmp | sort -u  >  ${VALID_CCF}/all_ccfs.txt          cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt
191    
192          rm ${cif_file}          rm ${cif_file}
193          rm ${VALID_CCF}/${obsid}_ccfs.txt          rm ${VALID_CCF}/${obsid}_ccfs.txt
194          rm ${VALID_CCF}/all_ccfs.tmp          rm ${VALID_CCF}/all_ccfs_${now}.tmp
195    
196  done < ${VALID_CCF}/xsaobslog.txt  done < ${VALID_CCF}/xsaobslog_${now}.txt
197    
198  # Make a table of classes and issues type "Pipeline Release Notes"  # Make a table of classes and issues type "Pipeline Release Notes"
199    
# Line 161  while read line Line 205  while read line
205  do  do
206          class=`echo $line | awk -F"_" '{print $1"_"$2}'`          class=`echo $line | awk -F"_" '{print $1"_"$2}'`
207          echo $class >> ${ccf_classes}          echo $class >> ${ccf_classes}
208  done < ${VALID_CCF}/all_ccfs.txt  done < ${VALID_CCF}/all_ccfs_${now}.txt
209    
210  cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt  cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt
211  mv ${VALID_CCF}/kk.txt ${ccf_classes}  mv ${VALID_CCF}/kk.txt ${ccf_classes}
212    
213  # Output final file is named all_ccfs_table.txt  # Output final file is named all_ccfs_${now}_table.txt
214    
215    total_number_of_ccfs=0
216    
217  all_ccfs_table="{VALID_CCF}/all_ccfs_table.txt  all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt"
218    
219  [ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}  [ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}
220    
221  echo "|================================|=============|" >> ${all_ccfs_table}  echo "Table of valid CCFs at $now_table"                        >> ${all_ccfs_table}
222  echo "| Calibration File               | Issue range |" >> ${all_ccfs_table}  echo                                                            >> ${all_ccfs_table}
223  echo "|================================|=============|" >> ${all_ccfs_table}  echo "|================================|=============|=======|" >> ${all_ccfs_table}
224    echo "| Calibration File               | Issue range | Count |" >> ${all_ccfs_table}
225    echo "|================================|=============|=======|" >> ${all_ccfs_table}
226    
227  while read ccf_class  while read ccf_class
228  do  do
229            echo -n "Processing class $ccf_class ..."
230          issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"          issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
231          [ -f "${issues_file}" ] && rm ${issues_file}          [ -f "${issues_file}" ] && rm ${issues_file}
232          while read line          while read line
# Line 186  do Line 235  do
235                  [ "$class" != "$ccf_class" ] && continue                  [ "$class" != "$ccf_class" ] && continue
236                  issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`                  issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`
237                  echo $issue >> ${issues_file}                  echo $issue >> ${issues_file}
238          done < ${VALID_CCF}/all_ccfs.txt          done < ${VALID_CCF}/all_ccfs_${now}.txt
239    
240          cat ${issues_file} | sort -u > kk.txt          cat ${issues_file} | sort -u > kk.txt
241          mv kk.txt ${issues_file}          mv kk.txt ${issues_file}
242            real_number_of_issues_for_this_class=`wc -l ${issues_file} | awk -F" " '{print $1}'`
243    
244          first_issue=`head -1 ${issues_file}`          first_issue=`head -1 ${issues_file}`
245          last_issue=`tail -1 ${issues_file}`          last_issue=`tail -1 ${issues_file}`
246    
247            li=$((10#$last_issue))
248            fi=$((10#$first_issue))
249    
250            sequential_number_of_issues_for_this_class=$((li - fi + 1))
251            if [ "${real_number_of_issues_for_this_class}"  -lt "${sequential_number_of_issues_for_this_class}" ] ; then
252                    asterisk="(*)"
253            else
254                    asterisk="   "
255            fi
256            total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs))
257    
258            rn=$real_number_of_issues_for_this_class
259    
260          if [ "${first_issue}" != "${last_issue}" ] ; then          if [ "${first_issue}" != "${last_issue}" ] ; then
261                  printf "| %-30s | %4s-%4s   |\n" ${ccf_class} ${first_issue} ${last_issue} >> ${all_ccfs_table}             printf "| %-30s | %4s-%4s%3s|  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${asterisk} ${rn} >> ${all_ccfs_table}
262          else          else
263                  printf "| %-30s | %4s        |\n" ${ccf_class} ${first_issue}              >> ${all_ccfs_table}             printf "| %-30s | %4s        |  %4s |\n" ${ccf_class} ${first_issue} ${rn}                           >> ${all_ccfs_table}
264          fi          fi
265              
266            echo "done"
267  done < ${ccf_classes}    done < ${ccf_classes}  
268    
269  echo "|================================|=============|"                                    >> ${all_ccfs_table}  echo "|================================|=============|=======|"                                                 >> ${all_ccfs_table}
270    echo                                                                                                            >> ${all_ccfs_table}
271    echo "(*): Incomplete range."                                                                                   >> ${all_ccfs_table}
272    echo ""                                                                                                         >> ${all_ccfs_table}
273    echo "Total number of valid CCFs required : ${total_number_of_ccfs}"                                            >> ${all_ccfs_table}
274    rm ${VALID_CCF}/ccf_classes.txt
275    rm ${VALID_CCF}/ccf_issues_for*.txt
Legend:
Removed from v.1.2  
changed lines
  Added in v.1.7
  ViewVC Help
Powered by ViewVC 1.1.27