/[CCFs]/bin/mlovccf

Diff of /bin/mlovccf

Parent Directory | Revision Log | View Patch Patch

-revision 1.2 by ccflib,
Tue Mar 10 13:27:33 2015 UTC
+revision 1.24 by ccflib,
Fri Nov 13 16:09:22 2020 UTC
 Line 2
  # mlovccf (make list of valid ccfs)
  #
- # Makes the list of valid CCFs that are all CCFs, among those published since the
+ # Makes the list of valid CCFs. These are all CCFs, among those published since the
  # beginning of the project, that are required to process any ODF at a given date,
- # typically the current date. Thus, this list will change with time as far as new
+ # typically the current date. This list will change with time as far as new
  # CCFs are issued to deal with new calibration data and/or replace obsolete ones.
  #
- # To identify the set of CCFs required to process a given ODF is the task of cifbuild.
+ # To identify the set of CCFs required to process a given ODF is the job of task cifbuild.
  # To make such identification, cifbuild needs only the start time of the observation.
+ # Therefore, if we provide a list of all observed ODFs at a given date together with their
+ # respective start time, we could compute all the CCFs required to process them.
+ # For such a moment, that list is the list of valid CCFs required to process
+ # any observed ODF.
  #
- # If we get the list of required CCFs for all observed ODFs at a given date,
+ # We need as input the list of all observed ODFs at a given date.
- # we will obtain for such a moment, the list of valid CCFs required to process
+ # This is obtained from the report of all observations that the XSA makes to CDS,
- # any observed ODF. Thus, we need as input the list of all observed ODFs at a given
+ # xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt).
- # date. This is obtained from the report of all observations that the XSA makes to CDS,
- # xsaobslog.txt (ftp://nxsa.esac.esa.int/pub/cfs_obslog/xsaobslog.txt).
  #
- # $Id: mlovccf,v 1.1 2015/03/09 17:06:20 ccflib Exp $
+ # $Id: mlovccf,v 1.23 2019/11/12 16:00:29 ccflib Exp $
+ # Next two lines allow to submit this script to the grid
+ # request Bourne shell as shell for job
+ #$ -S /bin/bash
+ export DEBUG="T"
+ adminmail="eduardo.ojero@sciops.esa.int"
+ VALID_CCF="$HOME/processing_valid_ccf"
+ mailfile="${VALID_CCF}/mailfile"
  host=`hostname | cut -d. -f1`
- # Function find_latest_mif to get the latest XMM_CALINDEX from given directory
+ now=`date +'%Y%m%d_%H%M'`
+ now_table=`date +'%Y-%m-%dT%H:%M:%S'`
+ # +++ Function mailnotify
+ #
+ # This function is used to notify adminmail on any issue while runnning this procedure
+ #
+ # Needs a subject as $1 and body as local file to be send.
- find_latest_mif()
+ mailnotify ()
  {
- [ -z "$1" ] && return
+ [ "$#" -ne "2" ] && return
- CCFPATH="$1"
+ subject=$1
- maxissue="0"
+ bodyfile=$2
- for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF`
- do
+ [ ! -f "${bodyfile}" ] && return
-         mifbase=`basename $mif`
-         issue=`echo $mifbase | awk -F"." '{print $1}' | awk -F"_" '{print $3}'`
+ echo ; echo "Sending message to ${adminmail} with subject ${subject}" ; echo
-         [ "$issue" -ge "$maxissue" ] && maxissue="$issue"
- done
+ if [ "$DEBUG" ] ; then
+         echo "DEBUG: mlovccf: mailnotify: Going to send mail as "
+         echo "mailx -v -s ${subject} ${adminmail} < ${bodyfile}"
+ fi
- MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF
+ mailx -v -s "${subject}" "${adminmail}" < ${bodyfile}
+ if [ "$DEBUG" ] ; then
+         echo "DEBUG: mlovccf: mailnotify: Mail sent!"
+ fi
  }
- # Function get_ccf_list to list on stdout the table of CCFs in a given CIF
+ # +++ Function getxsaobslog
+ #
+ # We need to get the list of all observed ODFs at the current date.
+ # This is obtained from http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt.
+ getxsaobslog ()
+ {
+ XSAOBSURL="http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt"
+ # Remove any xsaobslog.txt previously downloaded.
+ [ -f "${VALID_CCF}/xsaobslog.txt" ] && rm -rf ${VALID_CCF}/xsaobslog.txt
+ # Get the list latest list of ODFs available from nXSA server.
+ wget --quiet ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt
+ # Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference
+ # of which list of ODFs was used to compute the list of valid CCFs.
+ mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt
+ if [ "$DEBUG" ] ; then
+         echo "DEBUG: mlovccf: getxsaobslog: XSA Obs. log file obtained successfully and renamed to ${VALID_CCF}/xsaobslog_${now}.txt"
+ fi
+ }
+ # +++Function get_ccf_list to list on stdout the table of CCFs in a given CIF.
  #
  # Requires that Heasoft is initialised. Otherwise exits with error.
-Line 55 
 noversion=`which fversion | grep -c no`
+Line 117 
 noversion=`which fversion | grep -c no`
  if [ "${noversion}" != "0" ] ; then
          echo "Error: Heasoft not initialised !"
+         [ -f "${mailfile}" ] && rm -rf ${mailfile}
+         echo "Running of fversion produced the following output:" >> $mailfile
+         echo "$noversion" >> $mailfile
+         mailnotify "mvloccf: Error: Heasoft not initialised!" $mailfile
+         rm ${mailfile}
          exit
  fi
  fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
                          columns="SCOPE TYPEID ISSUE"            \
                          rows=-  STDOUT
  }
- # We need to get the list of all observed ODFs at the current date.
- # This is obtained from ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt.
- VALID_CCF="$HOME/valid_ccf"
- cd ${VALID_CCF}
+ # +++ Fill in the valid_constituents directory with the Valid CCF set
- wget -nc -q ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt
+ #
+ # The directory to be filled in is ${VALIDCONSTITUENTSDISR}.
+ # This directory is synchronized with the similar directory in the
+ # external ftp server.
+ #
+ # In addition to that, we also use this function to fill in the
+ # /ccf/valid directory available for users in the intranet.
+ fillvalidccfdir()
+ {
+ [ -z "$1" ] && return
+ validccflist=$1
+ if [ ! -f "${VALID_CCF}/${validccflist}" -o ! -s "${VALID_CCF}/${validccflist}"  ] ; then
+         echo "Error: ${VALID_CCF}/${validccflist} not found or empty - Abort!"
+         [ -f "${mailfile}" ] && rm -rf ${mailfile}
+         echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found or it is empty" >> $mailfile
+         mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found or it is empty - Abort!" $mailfile
+         rm ${mailfile}
+         exit
+ fi
+ VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents"
+ rm -rf ${VALIDCONSTITUENTSDIR}/*
+ # This operation can only succeed because the subdir valid has enable the write permission
+ # for the group "sas" (g=rwx), provided that CCFs belong to sasbuild(5153):sas.
+ VALIDCCFDIRINTRANET="/ccf/valid"
+ rm -rf ${VALIDCCFDIRINTRANET}/*
+ CONSTITUENTSDIR="/home/ccflib/ftp-area/constituents"
+ n=0
+ while read ccf
+ do
+         n=$((n + 1))
+         echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/"
+         cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/
+         echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/"
+         cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/
+ done < ${VALID_CCF}/${validccflist}
+ if [ "$DEBUG" ] ; then
+         echo "DEBUG: mlovccf: fillvalidccfdir: Processed $validccflist"
+ fi
+ }
+ #
+ #   M a i n    p r o g r a m
+ #
+ # 1. Get the list of all ODFs
+ xsaobslogsize="0"
- # Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host
+ getxsaobslog
+ xsaobslogsize=`stat --format=%s ${VALID_CCF}/xsaobslog_${now}.txt`
+ if [ "$xsaobslogsize"  = "0" ] ; then
+         echo "Error: Failure to download the XSA Obs. Log file - Abort"
+         [ -f "${mailfile}" ] && rm -rf ${mailfile}
+         echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile
+         echo "." >> $mailfile
+         if [ "$DEBUG" ] ; then
+                 echo "DEBUG: mlovccf: main: xsaobslogsize has 0 size. Mail notification being sent!"
+         fi
+         mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile"
+         rm ${mailfile}
+         exit
+ fi
+ #
+ # 2. Initialises HEADAS and SAS
+ #
+ # Defines SASBUILD_LOCAL depending on host.
+ # Then uses it to build up HEADAS and setup LD_LIBRARY_PATH accordingly.
  case "$host" in
-         xvsoc01|xmm)
+         sasbld01n|sasbld02n)
-                 SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"
+                 export SASBUILD_LOCAL="/sasbuild/local/${host}/GNU_CC_CXX_9.2.0"
-                 SAS_DIR=/data/xmm/ccflib/sas
-                 SAS_PATH=$SAS_DIR
-                 source $SAS_DIR/sas-setup.sh
                  ;;
-         sasbld01|sasbld02)
+         sciggw7|scigvn7-*)
-                 SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
+                 export SASBUILD_LOCAL="/sasbuild/local/sasbld04n/GNU_CC_CXX_9.2.0"
-                 /sas/bin/confsas
-                 export HEADAS=/sasbuild/local/${host}/headas/architecture
-                 . $HEADAS/headas-init.sh
-                 source $HOME/setsas.sh 1> /dev/null
-                 export SAS_VERBOSITY=0
                  ;;
+         *) echo "Error: Unsupported host $host" ; exit 1 ;;
  esac
- # Finds the latest MIF issue
+ export HEADAS=${SASBUILD_LOCAL}/headas/architecture
+ headas_output="${VALID_CCF}/headas_output"
+ source $HEADAS/headas-init.sh > ${headas_output} 2>&1
+ [ "$DEBUG" ] && echo "DEBUG: mlovccf: main: Executed Headas initialisation script for $HEADAS"
+ # The next statement should not be necessary because the SAS initialisation
+ # provides under $SAS_DIR/libextra all the necessary libraries from GNU C/C++
+ # which were required to build Heasoft. But, in order to make the HEADAS
+ # initialisation independent of the SAS initialisation, we set the
+ # LD_LIBRARY_PATH as it has to be to run Headas by itself without SAS been
+ # ready.
+ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${SASBUILD_LOCAL}/gcc/lib64
+ headas_ok=`cat ${headas_output}`
+ if [ -n "${headas_ok}" ] ; then
+         echo "mlovccf: HEADAS initialisation failed with error" >> ${mailfile}
+         cat ${headas_output}                                    >> ${mailfile}
+         mailnotify "mlovccf: HEADAS initilisation failed" ${mailfile}
+         if [ "$DEBUG" ] ; then
+                 echo "DEBUG: mlovccf: main: Headas initialization failed. Notification mail sent!"
+         fi
+         rm -v ${mailfile}
+         rm -v ${headas_output}
+         exit 1
+ fi
+ # To be able to mix several jobs on the same ccflib account
+ [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
+ export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
+ if [ "$DEBUG" ] ; then
+         echo -n "DEBUG: mlovccf: main: HEADAS version: "`fversion`
+ fi
+ #
+ #        W a r n i n g   o n   S A S   i n i t i a l i s a t i o n
+ #
+ # The SAS initialisation produced by setsas.sh sets SAS_CCFPATH to /ccf/pub.
+ # Hence, if we set SAS_CCFPATH before running setsas.sh, we will always
+ # use /ccf/pub as repository for CCF. This is right as long as /ccf/pub is in
+ # strict sync with /home/ccflib/ccf_ftp/constituents. But if this is not true,
+ # the resulting valid CCFs might be wrong. Therefore, we set it here explicitly.
+ #
+ confsas_output="${VALID_CCF}/confsas_output"
+ source /sas/bin/confsas > ${confsas_output}
+ confsas_ok=`cat ${confsas_output}`
+ if [ "$DEBUG" ] ; then
+         echo "DEBUG: mlovccf: main: Executed /sas/bin/confsas with output: ${confsas_ok}"
+ fi
+ if [ -n "${confsas_ok}" ] ; then
+         echo "mlovccf: SAS initialization failed with error" >> ${mailfile}
+         cat ${confsas_output}                                >> ${mailfile}
+         mailnotify "mlovccf: SAS initialisation failed" ${mailfile}
+         if [ "$DEBUG" ] ; then
+                 echo "DEBUG: mlovccf: main: SAS initialization failed. Notification mail sent!"
+         fi
+         rm -v ${mailfile}
+         rm -v ${confsas_output}
+         exit 1
+ fi
+ source $HOME/setsas.sh 1> /dev/null
+ if [ "$DEBUG" ] ; then
+         echo "DEBUG: mlovccf: main: SAS setsas.sh script executed"
+         echo
+         echo "DEBUG: mlovccf: main: SAS Version: "`sasversion -V 1`
+ fi
+ # Set SAS vervbosity to 0 from now onwards
+ export SAS_VERBOSITY=0
+ # Re-sets SAS_CCFPATH
+ export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
- find_latest_mif "${SAS_CCFPATH}"
- # Now scans the list of OBDS ID registered in the previous list to get the start time
+ # 3. XMM_CALINDEX file from SAS_CCFPATH
- # and then runs cifbuild to get the respective CIF
- # Output file
+ MIF=`ls -1 ${SAS_CCFPATH}/XMM_CALINDEX_*.CCF`
- touch ${VALID_CCF}/all_ccfs.txt
+ # 4. For each OBDS ID registered in the XSA observations list, gets the start
+ #    time of the observation. Then runs cifbuild to get the respective CIF.
+ #    all_ccfs_${now}.txt is filled recursively on each iteration.
+ #    Therefore we start with an empty file. The final list will be this one.
+ #    Output file :          ${VALID_CCF}/all_ccfs_${now}.txt
+ touch ${VALID_CCF}/all_ccfs_${now}.txt
  n=0
-Line 141 
 do
+Line 365 
 do
          rm ${VALID_CCF}/${obsid}_ccfs.tmp
-         mv ${VALID_CCF}/all_ccfs.txt ${VALID_CCF}/all_ccfs.tmp
+         mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp
+         cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt
-         cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs.tmp | sort -u  >  ${VALID_CCF}/all_ccfs.txt
+         comm -1 -3 --nocheck-order ${VALID_CCF}/all_ccfs_${now}.tmp ${VALID_CCF}/all_ccfs_${now}.txt > ${VALID_CCF}/diff_iteration_${n}.txt
+         diffsize=`stat --format=%s ${VALID_CCF}/diff_iteration_${n}.txt`
+         if [ "$diffsize" != "0" ] ; then
+                 echo "CCF added in this iteration: "
+                 cat ${VALID_CCF}/diff_iteration_${n}.txt
+         fi
+         echo
+         echo "Number of valid CCFs found so far: "`cat ${VALID_CCF}/all_ccfs_${now}.txt | wc -l` ; echo
+         rm ${VALID_CCF}/diff_iteration_${n}.txt
          rm ${cif_file}
          rm ${VALID_CCF}/${obsid}_ccfs.txt
-         rm ${VALID_CCF}/all_ccfs.tmp
+         rm ${VALID_CCF}/all_ccfs_${now}.tmp
- done < ${VALID_CCF}/xsaobslog.txt
+ done < ${VALID_CCF}/xsaobslog_${now}.txt
+ # 5. Make a table of classes and issues with the style "Pipeline Release Notes"
+ #    To get a classification, we need first to obtain all available classes.
+ #
- # Make a table of classes and issues type "Pipeline Release Notes"
  # Lists all CCF classes
-Line 161 
 while read line
+Line 401 
 while read line
  do
          class=`echo $line | awk -F"_" '{print $1"_"$2}'`
          echo $class >> ${ccf_classes}
- done < ${VALID_CCF}/all_ccfs.txt
+ done < ${VALID_CCF}/all_ccfs_${now}.txt
  cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt
  mv ${VALID_CCF}/kk.txt ${ccf_classes}
- # Output final file is named all_ccfs_table.txt
- all_ccfs_table="{VALID_CCF}/all_ccfs_table.txt
+ # Output final file is named all_ccfs_${now}_table.txt
+ total_number_of_ccfs=0
+ all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt"
  [ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}
- echo "|================================|=============|" >> ${all_ccfs_table}
+ echo "Table of valid CCFs at $now_table"                        >> ${all_ccfs_table}
- echo "| Calibration File               | Issue range |" >> ${all_ccfs_table}
+ echo                                                            >> ${all_ccfs_table}
- echo "|================================|=============|" >> ${all_ccfs_table}
+ echo "|================================|=============|=======|" >> ${all_ccfs_table}
+ echo "| Calibration File               | Issue range | Count |" >> ${all_ccfs_table}
+ echo "|================================|=============|=======|" >> ${all_ccfs_table}
  while read ccf_class
  do
+         echo -n "Processing class $ccf_class ..."
          issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
          [ -f "${issues_file}" ] && rm ${issues_file}
          while read line
-Line 186 
 do
+Line 433 
 do
                  [ "$class" != "$ccf_class" ] && continue
                  issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`
                  echo $issue >> ${issues_file}
-         done < ${VALID_CCF}/all_ccfs.txt
+         done < ${VALID_CCF}/all_ccfs_${now}.txt
          cat ${issues_file} | sort -u > kk.txt
          mv kk.txt ${issues_file}
+         real_number_of_issues_for_this_class=`wc -l ${issues_file} | awk -F" " '{print $1}'`
          first_issue=`head -1 ${issues_file}`
          last_issue=`tail -1 ${issues_file}`
-         if [ "${first_issue}" != "${last_issue}" ] ; then
+         li=$((10#$last_issue))
-                 printf "| %-30s | %4s-%4s   |\n" ${ccf_class} ${first_issue} ${last_issue} >> ${all_ccfs_table}
+         fi=$((10#$first_issue))
-         else
-                 printf "| %-30s | %4s        |\n" ${ccf_class} ${first_issue}              >> ${all_ccfs_table}
+         sequential_number_of_issues_for_this_class=$((li - fi + 1))
-         fi
+         asterisk="   "
+         [ "${real_number_of_issues_for_this_class}"  -lt "${sequential_number_of_issues_for_this_class}" ] &&  asterisk="(*)"
+         total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs))
+         rn=$real_number_of_issues_for_this_class
+         if [ "${first_issue}" != "${last_issue}" ] ; then
+            if [ "$asterisk" = "(*)" ] ; then
+               printf "| %-30s | %4s-%4s(*)|  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
+            else
+               printf "| %-30s | %4s-%4s   |  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
+            fi
+         else
+            printf "| %-30s | %4s        |  %4s |\n" ${ccf_class} ${first_issue} ${rn}                           >> ${all_ccfs_table}
+         fi
+         echo "done"
  done < ${ccf_classes}
- echo "|================================|=============|"                                    >> ${all_ccfs_table}
+ echo "|================================|=============|=======|"                                                 >> ${all_ccfs_table}
+ echo                                                                                                            >> ${all_ccfs_table}
+ echo "(*): Incomplete range."                                                                                   >> ${all_ccfs_table}
+ echo ""                                                                                                         >> ${all_ccfs_table}
+ echo "Total number of valid CCFs required : ${total_number_of_ccfs}"                                            >> ${all_ccfs_table}
+ rm ${VALID_CCF}/ccf_classes.txt
+ rm ${VALID_CCF}/ccf_issues_for*.txt
+ # 6. Fill in the Valid CCF Set dir with the proper CCFs
+ #fillvalidccfdir "all_ccfs_${now}.txt"
+ #[ -f "${mailfile}" ] && rm -rf ${mailfile}
+ #echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile
+ #echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}"  >> $mailfile
+ #mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile
+ #rm ${mailfile}

 Legend:



Removed from v.1.2
 


changed lines


 
Added in v.1.24
 Legend:



Removed from v.1.2
 


changed lines


 
Added in v.1.24
-Removed from v.1.2
+Added in v.1.24

	ViewVC Help
Powered by ViewVC 1.1.27