Annotation of /bin/mlovccf

#! /bin/bash

# mlovccf (make list of valid ccfs)
#
# Makes the list of valid CCFs. These are all CCFs, among those published since the 
# beginning of the project, that are required to process any ODF at a given date,
# typically the current date. This list will change with time as far as new
# CCFs are issued to deal with new calibration data and/or replace obsolete ones. 
#
# To identify the set of CCFs required to process a given ODF is the job of task cifbuild. 
# To make such identification, cifbuild needs only the start time of the observation. 
# Therefore, if we provide a list of all observed ODFs at a given date together with their
# respective start time, we could compute all the CCFs required to process them.
# For such a moment, that list is the list of valid CCFs required to process 
# any observed ODF. 
#
# We need as input the list of all observed ODFs at a given date.
# This is obtained from the report of all observations that the XSA makes to CDS,
# xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt). 
#
# $Id: mlovccf,v 1.25 2021/11/08 13:23:09 ccflib Exp $

# Next two lines allow to submit this script to the grid
# request Bourne shell as shell for job
#$ -S /bin/bash

export DEBUG="T"

adminmail="eduardo.ojero@sciops.esa.int"

VALID_CCF="$HOME/processing_valid_ccf"

mailfile="${VALID_CCF}/mailfile"

host=`hostname | cut -d. -f1`

now=`date +'%Y%m%d_%H%M'`
now_table=`date +'%Y-%m-%dT%H:%M:%S'`


# +++ Function mailnotify
#
# This function is used to notify adminmail on any issue while runnning this procedure
#
# Needs a subject as $1 and body as local file to be send.

mailnotify ()
{

[ "$#" -ne "2" ] && return

subject=$1 

bodyfile=$2

[ ! -f "${bodyfile}" ] && return 

echo ; echo "Sending message to ${adminmail} with subject ${subject}" ; echo

if [ "$DEBUG" ] ; then

        echo "DEBUG: mlovccf: mailnotify: Going to send mail as "
        echo "mailx -v -s ${subject} ${adminmail} < ${bodyfile}"
fi

mailx -v -s "${subject}" "${adminmail}" < ${bodyfile}

if [ "$DEBUG" ] ; then
        echo "DEBUG: mlovccf: mailnotify: Mail sent!"
fi
}


# +++ Function getxsaobslog
#
# We need to get the list of all observed ODFs at the current date. 
# This is obtained from http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt.

getxsaobslog ()
{

XSAOBSURL="http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt"

# Remove any xsaobslog.txt previously downloaded. 

[ -f "${VALID_CCF}/xsaobslog.txt" ] && rm -rf ${VALID_CCF}/xsaobslog.txt

# Get the list latest list of ODFs available from nXSA server.
 
wget --quiet ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt 

# Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference 
# of which list of ODFs was used to compute the list of valid CCFs.

mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt

if [ "$DEBUG" ] ; then
        echo "DEBUG: mlovccf: getxsaobslog: XSA Obs. log file obtained successfully and renamed to ${VALID_CCF}/xsaobslog_${now}.txt"
fi

}


# +++Function get_ccf_list to list on stdout the table of CCFs in a given CIF.
#
# Requires that Heasoft is initialised. Otherwise exits with error.

get_ccf_list()
{
[ -z "$1" ] && return

cif_to_process="$1"

noversion=`which fversion | grep -c no`

if [ "${noversion}" != "0" ] ; then 
        echo "Error: Heasoft not initialised !"
        [ -f "${mailfile}" ] && rm -rf ${mailfile}
        echo "Running of fversion produced the following output:" >> $mailfile
        echo "$noversion" >> $mailfile
        mailnotify "mvloccf: Error: Heasoft not initialised!" $mailfile
        rm ${mailfile}
        exit
fi

fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
                        columns="SCOPE TYPEID ISSUE"            \
                        rows=-  STDOUT  

}


# +++ Fill in the valid_constituents directory with the Valid CCF set
#
# The directory to be filled in is ${VALIDCONSTITUENTSDISR}. 
# This directory is synchronized with the similar directory in the 
# external ftp server.
#
# In addition to that, we also use this function to fill in the 
# /ccf/valid directory available for users in the intranet.

fillvalidccfdir()
{

[ -z "$1" ] && return

validccflist=$1

if [ ! -f "${VALID_CCF}/${validccflist}" -o ! -s "${VALID_CCF}/${validccflist}"  ] ; then
        echo "Error: ${VALID_CCF}/${validccflist} not found or empty - Abort!" 
        [ -f "${mailfile}" ] && rm -rf ${mailfile}
        echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found or it is empty" >> $mailfile
        mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found or it is empty - Abort!" $mailfile
        rm ${mailfile}
        exit
fi

VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents"
# Checks that the VALIDCONSTITUENTSDIR directory is writable by me
touch ${VALIDCONSTITUENTSDIR}/test 2> /dev/null
if [ "$?" != "0" ] ; then 
        [ -f "${mailfile}" ] && rm -rf ${mailfile}
        echo "${VALIDCONSTITUENTSDIR} is not writable. Can not work" >> $mailfile
        mailnotify "mvloccf: Error: ${VALIDCONSTITUENTSDIR} is not writable - Abort!" $mailfile
        rm ${mailfile}
        exit
fi
rm -rf ${VALIDCONSTITUENTSDIR}/*

# This operation can only succeed because the subdir valid has enable the write permission 
# for the group "sas" (g=rwx), provided that CCFs belong to sasbuild(5153):sas.
VALIDCCFDIRINTRANET="/ccf/valid"
# Checks that the VALIDCCFDIRINTRANET directory is writable by me
touch ${VALIDCCFDIRINTRANET}/test 2> /dev/null
if [ "$?" != "0" ] ; then
        [ -f "${mailfile}" ] && rm -rf ${mailfile}
        echo "${VALIDCCFDIRINTRANET} is not writable. Can not work" >> $mailfile
        mailnotify "mvloccf: Error: ${VALIDCCFDIRINTRANET} is not writable - Abort!" $mailfile
        rm ${mailfile}
        exit
fi

rm -rf ${VALIDCCFDIRINTRANET}/*

CONSTITUENTSDIR="/home/ccflib/ftp-area/constituents"
n=0
while read ccf
do
        n=$((n + 1))
        echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/"
        cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/
        echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/"
        cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/
done < ${VALID_CCF}/${validccflist}

if [ "$DEBUG" ] ; then
        echo "DEBUG: mlovccf: fillvalidccfdir: Processed $validccflist"
fi
}


#
#   M a i n    p r o g r a m 
#
# 1. Get the list of all ODFs


xsaobslogsize="0" 

getxsaobslog 

xsaobslogsize=`stat --format=%s ${VALID_CCF}/xsaobslog_${now}.txt` 

if [ "$xsaobslogsize"  = "0" ] ; then
        echo "Error: Failure to download the XSA Obs. Log file - Abort" 
        [ -f "${mailfile}" ] && rm -rf ${mailfile}
        echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile
        echo "." >> $mailfile
        if [ "$DEBUG" ] ; then 
                echo "DEBUG: mlovccf: main: xsaobslogsize has 0 size. Mail notification being sent!"
        fi
        mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile"
        rm ${mailfile}
        exit
fi 

#
# 2. Initialises HEADAS and SAS
#

# Defines SASBUILD_LOCAL depending on host. 
# Then uses it to build up HEADAS and setup LD_LIBRARY_PATH accordingly.

case "$host" in
        sasbld01n|sasbld02n)
                export SASBUILD_LOCAL="/sasbuild/local/${host}/GNU_CC_CXX_9.2.0"
                ;;
        sciggw|scigrid-*)
                export SASBUILD_LOCAL="/sasbuild/local/sasbld04n/GNU_CC_CXX_9.2.0"
                ;;
        *) echo "Error: Unsupported host $host" ; exit 1 ;;
esac

export HEADAS=${SASBUILD_LOCAL}/headas/architecture

headas_output="${VALID_CCF}/headas_output"
source $HEADAS/headas-init.sh > ${headas_output} 2>&1

[ "$DEBUG" ] && echo "DEBUG: mlovccf: main: Executed Headas initialisation script for $HEADAS"

# The next statement should not be necessary because the SAS initialisation 
# provides under $SAS_DIR/libextra all the necessary libraries from GNU C/C++
# which were required to build Heasoft. But, in order to make the HEADAS
# initialisation independent of the SAS initialisation, we set the 
# LD_LIBRARY_PATH as it has to be to run Headas by itself without SAS been
# ready.

export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${SASBUILD_LOCAL}/gcc/lib64


headas_ok=`cat ${headas_output}`

if [ -n "${headas_ok}" ] ; then
        echo "mlovccf: HEADAS initialisation failed with error" >> ${mailfile}
        cat ${headas_output}                                    >> ${mailfile}
        mailnotify "mlovccf: HEADAS initilisation failed" ${mailfile}
        if [ "$DEBUG" ] ; then 
                echo "DEBUG: mlovccf: main: Headas initialization failed. Notification mail sent!"
        fi
        rm -v ${mailfile}
        rm -v ${headas_output}
        exit 1
fi

# To be able to mix several jobs on the same ccflib account
[ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
if [ "$DEBUG" ] ; then 
        echo -n "DEBUG: mlovccf: main: HEADAS version: "`fversion` 
fi

#
#        W a r n i n g   o n   S A S   i n i t i a l i s a t i o n  
#
# The SAS initialisation produced by setsas.sh sets SAS_CCFPATH to /ccf/pub.
# Hence, if we set SAS_CCFPATH before running setsas.sh, we will always 
# use /ccf/pub as repository for CCF. This is right as long as /ccf/pub is in 
# strict sync with /home/ccflib/ccf_ftp/constituents. But if this is not true, 
# the resulting valid CCFs might be wrong. Therefore, we set it here explicitly. 
#

confsas_output="${VALID_CCF}/confsas_output"
source /sas/bin/confsas > ${confsas_output} 2>&1 
confsas_error=`cat ${confsas_output} | grep -c Error`

if [ "$DEBUG" ] ; then 
        echo "DEBUG: mlovccf: main: Executed /sas/bin/confsas with output: ${confsas_ok}"
fi

if [ "${confsas_error}" != "0" ] ; then
        echo "mlovccf: SAS initialization failed with error" >> ${mailfile}
        cat ${confsas_output}                                >> ${mailfile}
        mailnotify "mlovccf: SAS initialisation failed" ${mailfile}
        if [ "$DEBUG" ] ; then
                echo "DEBUG: mlovccf: main: SAS initialization failed. Notification mail sent!"
        fi
        exit 1
fi

rm -v ${mailfile} 
rm -v ${confsas_output} 

source $HOME/setsas.sh 1> /dev/null

if [ "$DEBUG" ] ; then
        echo "DEBUG: mlovccf: main: SAS setsas.sh script executed" 
        echo 
        echo "DEBUG: mlovccf: main: SAS Version: "`sasversion -V 1`
fi


# Set SAS vervbosity to 0 from now onwards

export SAS_VERBOSITY=0


# Re-sets SAS_CCFPATH 

export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"


# 3. XMM_CALINDEX file from SAS_CCFPATH

MIF=`ls -1 ${SAS_CCFPATH}/XMM_CALINDEX_*.CCF`

# 4. For each OBDS ID registered in the XSA observations list, gets the start 
#    time of the observation. Then runs cifbuild to get the respective CIF.
#    all_ccfs_${now}.txt is filled recursively on each iteration. 
#    Therefore we start with an empty file. The final list will be this one.

#    Output file :          ${VALID_CCF}/all_ccfs_${now}.txt

touch ${VALID_CCF}/all_ccfs_${now}.txt

n=0

while read line 
do
        obsid=`echo $line | awk -F"|" '{print $2}'` 
        stime=`echo $line | awk -F"|" '{print $9}'`
        stime=`echo $stime | tr " " "T"`
        n=$((n+1))
        echo "($n) Processing $obsid $stime" ; echo

        # Run cifbuild

        cif_file=${VALID_CCF}/${obsid}.cif      

        cifbuild --withmasterindexset='yes'  \
                 --masterindexset=${MIF}     \
                 --withobservationdate='yes' \
                 --observationdate=${stime}  \
                 --analysisdate='now'        \
                 --calindexset=${cif_file} 


        # Gets the list of CCFs included in the CIF

        get_ccf_list "${cif_file}" > ${VALID_CCF}/${obsid}_ccfs.tmp

        while read line 
        do
                [ "$line" = "" ] && continue
                class=`echo $line | awk -F" " '{print $1"_"$2}'`
                issue=`echo $line | awk -F" " '{print $3}'` 
                issue=`printf "%04d" ${issue}`
                echo ${class}_${issue}.CCF >> ${VALID_CCF}/${obsid}_ccfs.txt

        done < ${VALID_CCF}/${obsid}_ccfs.tmp

        rm ${VALID_CCF}/${obsid}_ccfs.tmp

        mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp

        cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt

        comm -1 -3 --nocheck-order ${VALID_CCF}/all_ccfs_${now}.tmp ${VALID_CCF}/all_ccfs_${now}.txt > ${VALID_CCF}/diff_iteration_${n}.txt
        
        diffsize=`stat --format=%s ${VALID_CCF}/diff_iteration_${n}.txt` 
        
        if [ "$diffsize" != "0" ] ; then
                echo "CCF added in this iteration: " 
                cat ${VALID_CCF}/diff_iteration_${n}.txt
        fi      
        
        echo 
        echo "Number of valid CCFs found so far: "`cat ${VALID_CCF}/all_ccfs_${now}.txt | wc -l` ; echo

        rm ${VALID_CCF}/diff_iteration_${n}.txt
        rm ${cif_file}
        rm ${VALID_CCF}/${obsid}_ccfs.txt
        rm ${VALID_CCF}/all_ccfs_${now}.tmp

done < ${VALID_CCF}/xsaobslog_${now}.txt

# 5. Make a table of classes and issues with the style "Pipeline Release Notes"
#    To get a classification, we need first to obtain all available classes.
#


# Lists all CCF classes

ccf_classes="${VALID_CCF}/ccf_classes.txt"

while read line
do 
        class=`echo $line | awk -F"_" '{print $1"_"$2}'`
        echo $class >> ${ccf_classes}
done < ${VALID_CCF}/all_ccfs_${now}.txt

cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt
mv ${VALID_CCF}/kk.txt ${ccf_classes}


# Output final file is named all_ccfs_${now}_table.txt

total_number_of_ccfs=0

all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt"

[ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}

echo "Table of valid CCFs at $now_table"                        >> ${all_ccfs_table}
echo                                                            >> ${all_ccfs_table}
echo "|================================|=============|=======|" >> ${all_ccfs_table} 
echo "| Calibration File               | Issue range | Count |" >> ${all_ccfs_table}
echo "|================================|=============|=======|" >> ${all_ccfs_table}

while read ccf_class
do
        echo -n "Processing class $ccf_class ..." 
        issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
        [ -f "${issues_file}" ] && rm ${issues_file}
        while read line
        do
                class=`echo $line | awk -F"_" '{print $1"_"$2}'`
                [ "$class" != "$ccf_class" ] && continue
                issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`
                echo $issue >> ${issues_file}
        done < ${VALID_CCF}/all_ccfs_${now}.txt

        cat ${issues_file} | sort -u > kk.txt
        mv kk.txt ${issues_file}
        real_number_of_issues_for_this_class=`wc -l ${issues_file} | awk -F" " '{print $1}'`

        first_issue=`head -1 ${issues_file}`
        last_issue=`tail -1 ${issues_file}`

        li=$((10#$last_issue))
        fi=$((10#$first_issue))

        sequential_number_of_issues_for_this_class=$((li - fi + 1))

        asterisk="   "
        [ "${real_number_of_issues_for_this_class}"  -lt "${sequential_number_of_issues_for_this_class}" ] &&  asterisk="(*)"

        total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs))

        rn=$real_number_of_issues_for_this_class

        if [ "${first_issue}" != "${last_issue}" ] ; then
           if [ "$asterisk" = "(*)" ] ; then
              printf "| %-30s | %4s-%4s(*)|  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
           else
              printf "| %-30s | %4s-%4s   |  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
           fi
        else
           printf "| %-30s | %4s        |  %4s |\n" ${ccf_class} ${first_issue} ${rn}                           >> ${all_ccfs_table}
        fi

        
        echo "done" 
done < ${ccf_classes}  

echo "|================================|=============|=======|"                                                 >> ${all_ccfs_table}
echo                                                                                                            >> ${all_ccfs_table}
echo "(*): Incomplete range."                                                                                   >> ${all_ccfs_table} 
echo ""                                                                                                         >> ${all_ccfs_table}
echo "Total number of valid CCFs required : ${total_number_of_ccfs}"                                            >> ${all_ccfs_table}
rm ${VALID_CCF}/ccf_classes.txt
rm ${VALID_CCF}/ccf_issues_for*.txt


# 6. Fill in the Valid CCF Set dir with the proper CCFs

fillvalidccfdir "all_ccfs_${now}.txt"


[ -f "${mailfile}" ] && rm -rf ${mailfile}
echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile
echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}"  >> $mailfile 
mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile
rm ${mailfile}
1	ccflib	1.1	#! /bin/bash
2
3			# mlovccf (make list of valid ccfs)
4			#
5	ccflib	1.7	# Makes the list of valid CCFs. These are all CCFs, among those published since the
6	ccflib	1.2	# beginning of the project, that are required to process any ODF at a given date,
7	ccflib	1.7	# typically the current date. This list will change with time as far as new
8	ccflib	1.2	# CCFs are issued to deal with new calibration data and/or replace obsolete ones.
9	ccflib	1.1	#
10	ccflib	1.10	# To identify the set of CCFs required to process a given ODF is the job of task cifbuild.
11	ccflib	1.2	# To make such identification, cifbuild needs only the start time of the observation.
12	ccflib	1.7	# Therefore, if we provide a list of all observed ODFs at a given date together with their
13			# respective start time, we could compute all the CCFs required to process them.
14			# For such a moment, that list is the list of valid CCFs required to process
15			# any observed ODF.
16	ccflib	1.1	#
17	ccflib	1.7	# We need as input the list of all observed ODFs at a given date.
18			# This is obtained from the report of all observations that the XSA makes to CDS,
19	ccflib	1.10	# xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt).
20	ccflib	1.2	#
21	ccflib	1.26	# $Id: mlovccf,v 1.25 2021/11/08 13:23:09 ccflib Exp $
22	ccflib	1.4
23			# Next two lines allow to submit this script to the grid
24			# request Bourne shell as shell for job
25	ccflib	1.6	#$ -S /bin/bash
26	ccflib	1.1
27	ccflib	1.24	export DEBUG="T"
28
29	ccflib	1.16	adminmail="eduardo.ojero@sciops.esa.int"
30	ccflib	1.5
31	ccflib	1.17	VALID_CCF="$HOME/processing_valid_ccf"
32	ccflib	1.10
33	ccflib	1.13	mailfile="${VALID_CCF}/mailfile"
34
35	ccflib	1.1	host=`hostname \| cut -d. -f1`
36
37	ccflib	1.5	now=`date +'%Y%m%d_%H%M'`
38	ccflib	1.7	now_table=`date +'%Y-%m-%dT%H:%M:%S'`
39	ccflib	1.3
40	ccflib	1.10
41	ccflib	1.13	# +++ Function mailnotify
42			#
43			# This function is used to notify adminmail on any issue while runnning this procedure
44			#
45			# Needs a subject as $1 and body as local file to be send.
46
47			mailnotify ()
48			{
49
50			[ "$#" -ne "2" ] && return
51
52			subject=$1
53
54			bodyfile=$2
55
56			[ ! -f "${bodyfile}" ] && return
57
58	ccflib	1.15	echo ; echo "Sending message to ${adminmail} with subject ${subject}" ; echo
59
60	ccflib	1.24	if [ "$DEBUG" ] ; then
61
62			echo "DEBUG: mlovccf: mailnotify: Going to send mail as "
63			echo "mailx -v -s ${subject} ${adminmail} < ${bodyfile}"
64			fi
65
66	ccflib	1.16	mailx -v -s "${subject}" "${adminmail}" < ${bodyfile}
67	ccflib	1.13
68	ccflib	1.24	if [ "$DEBUG" ] ; then
69			echo "DEBUG: mlovccf: mailnotify: Mail sent!"
70			fi
71	ccflib	1.13	}
72
73
74
75			# +++ Function getxsaobslog
76			#
77	ccflib	1.10	# We need to get the list of all observed ODFs at the current date.
78			# This is obtained from http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt.
79
80			getxsaobslog ()
81			{
82
83			XSAOBSURL="http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt"
84
85			# Remove any xsaobslog.txt previously downloaded.
86
87			[ -f "${VALID_CCF}/xsaobslog.txt" ] && rm -rf ${VALID_CCF}/xsaobslog.txt
88
89			# Get the list latest list of ODFs available from nXSA server.
90
91	ccflib	1.20	wget --quiet ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt
92	ccflib	1.10
93			# Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference
94			# of which list of ODFs was used to compute the list of valid CCFs.
95
96			mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt
97
98	ccflib	1.24	if [ "$DEBUG" ] ; then
99			echo "DEBUG: mlovccf: getxsaobslog: XSA Obs. log file obtained successfully and renamed to ${VALID_CCF}/xsaobslog_${now}.txt"
100			fi
101
102	ccflib	1.10	}
103
104
105	ccflib	1.13
106			# +++Function get_ccf_list to list on stdout the table of CCFs in a given CIF.
107			#
108	ccflib	1.2	# Requires that Heasoft is initialised. Otherwise exits with error.
109	ccflib	1.1
110			get_ccf_list()
111			{
112			[ -z "$1" ] && return
113
114			cif_to_process="$1"
115
116			noversion=`which fversion \| grep -c no`
117
118			if [ "${noversion}" != "0" ] ; then
119			echo "Error: Heasoft not initialised !"
120	ccflib	1.13	[ -f "${mailfile}" ] && rm -rf ${mailfile}
121			echo "Running of fversion produced the following output:" >> $mailfile
122			echo "$noversion" >> $mailfile
123			mailnotify "mvloccf: Error: Heasoft not initialised!" $mailfile
124			rm ${mailfile}
125	ccflib	1.1	exit
126			fi
127
128			fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
129			columns="SCOPE TYPEID ISSUE" \
130			rows=- STDOUT
131	ccflib	1.24
132	ccflib	1.1	}
133
134	ccflib	1.13
135
136
137			# +++ Fill in the valid_constituents directory with the Valid CCF set
138			#
139			# The directory to be filled in is ${VALIDCONSTITUENTSDISR}.
140			# This directory is synchronized with the similar directory in the
141			# external ftp server.
142			#
143			# In addition to that, we also use this function to fill in the
144			# /ccf/valid directory available for users in the intranet.
145	ccflib	1.11
146			fillvalidccfdir()
147			{
148
149			[ -z "$1" ] && return
150
151			validccflist=$1
152
153	ccflib	1.21	if [ ! -f "${VALID_CCF}/${validccflist}" -o ! -s "${VALID_CCF}/${validccflist}" ] ; then
154			echo "Error: ${VALID_CCF}/${validccflist} not found or empty - Abort!"
155	ccflib	1.13	[ -f "${mailfile}" ] && rm -rf ${mailfile}
156	ccflib	1.21	echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found or it is empty" >> $mailfile
157			mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found or it is empty - Abort!" $mailfile
158	ccflib	1.13	rm ${mailfile}
159	ccflib	1.11	exit
160			fi
161
162	ccflib	1.13	VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents"
163	ccflib	1.25	# Checks that the VALIDCONSTITUENTSDIR directory is writable by me
164			touch ${VALIDCONSTITUENTSDIR}/test 2> /dev/null
165			if [ "$?" != "0" ] ; then
166			[ -f "${mailfile}" ] && rm -rf ${mailfile}
167			echo "${VALIDCONSTITUENTSDIR} is not writable. Can not work" >> $mailfile
168			mailnotify "mvloccf: Error: ${VALIDCONSTITUENTSDIR} is not writable - Abort!" $mailfile
169			rm ${mailfile}
170			exit
171			fi
172	ccflib	1.11	rm -rf ${VALIDCONSTITUENTSDIR}/*
173
174	ccflib	1.19	# This operation can only succeed because the subdir valid has enable the write permission
175			# for the group "sas" (g=rwx), provided that CCFs belong to sasbuild(5153):sas.
176	ccflib	1.13	VALIDCCFDIRINTRANET="/ccf/valid"
177	ccflib	1.25	# Checks that the VALIDCCFDIRINTRANET directory is writable by me
178			touch ${VALIDCCFDIRINTRANET}/test 2> /dev/null
179			if [ "$?" != "0" ] ; then
180			[ -f "${mailfile}" ] && rm -rf ${mailfile}
181			echo "${VALIDCCFDIRINTRANET} is not writable. Can not work" >> $mailfile
182			mailnotify "mvloccf: Error: ${VALIDCCFDIRINTRANET} is not writable - Abort!" $mailfile
183			rm ${mailfile}
184			exit
185			fi
186
187	ccflib	1.13	rm -rf ${VALIDCCFDIRINTRANET}/*
188
189	ccflib	1.11	CONSTITUENTSDIR="/home/ccflib/ftp-area/constituents"
190	ccflib	1.12	n=0
191	ccflib	1.11	while read ccf
192			do
193	ccflib	1.12	n=$((n + 1))
194			echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/"
195	ccflib	1.11	cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/
196	ccflib	1.13	echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/"
197			cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/
198	ccflib	1.12	done < ${VALID_CCF}/${validccflist}
199	ccflib	1.11
200	ccflib	1.24	if [ "$DEBUG" ] ; then
201			echo "DEBUG: mlovccf: fillvalidccfdir: Processed $validccflist"
202			fi
203	ccflib	1.11	}
204
205
206	ccflib	1.13	#
207			# M a i n p r o g r a m
208			#
209			# 1. Get the list of all ODFs
210
211	ccflib	1.1
212	ccflib	1.10	xsaobslogsize="0"
213	ccflib	1.1
214	ccflib	1.10	getxsaobslog
215	ccflib	1.1
216	ccflib	1.10	xsaobslogsize=`stat --format=%s ${VALID_CCF}/xsaobslog_${now}.txt`
217	ccflib	1.1
218	ccflib	1.10	if [ "$xsaobslogsize" = "0" ] ; then
219	ccflib	1.13	echo "Error: Failure to download the XSA Obs. Log file - Abort"
220			[ -f "${mailfile}" ] && rm -rf ${mailfile}
221			echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile
222			echo "." >> $mailfile
223	ccflib	1.24	if [ "$DEBUG" ] ; then
224			echo "DEBUG: mlovccf: main: xsaobslogsize has 0 size. Mail notification being sent!"
225			fi
226	ccflib	1.13	mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile"
227			rm ${mailfile}
228	ccflib	1.10	exit
229			fi
230	ccflib	1.1
231	ccflib	1.17	#
232	ccflib	1.18	# 2. Initialises HEADAS and SAS
233			#
234
235	ccflib	1.20	# Defines SASBUILD_LOCAL depending on host.
236			# Then uses it to build up HEADAS and setup LD_LIBRARY_PATH accordingly.
237	ccflib	1.1
238			case "$host" in
239	ccflib	1.20	sasbld01n\|sasbld02n)
240	ccflib	1.24	export SASBUILD_LOCAL="/sasbuild/local/${host}/GNU_CC_CXX_9.2.0"
241	ccflib	1.1	;;
242	ccflib	1.25	sciggw\|scigrid-*)
243	ccflib	1.24	export SASBUILD_LOCAL="/sasbuild/local/sasbld04n/GNU_CC_CXX_9.2.0"
244	ccflib	1.5	;;
245	ccflib	1.17	*) echo "Error: Unsupported host $host" ; exit 1 ;;
246	ccflib	1.1	esac
247
248	ccflib	1.20	export HEADAS=${SASBUILD_LOCAL}/headas/architecture
249	ccflib	1.1
250	ccflib	1.18	headas_output="${VALID_CCF}/headas_output"
251			source $HEADAS/headas-init.sh > ${headas_output} 2>&1
252	ccflib	1.20
253	ccflib	1.24	[ "$DEBUG" ] && echo "DEBUG: mlovccf: main: Executed Headas initialisation script for $HEADAS"
254
255	ccflib	1.20	# The next statement should not be necessary because the SAS initialisation
256			# provides under $SAS_DIR/libextra all the necessary libraries from GNU C/C++
257			# which were required to build Heasoft. But, in order to make the HEADAS
258			# initialisation independent of the SAS initialisation, we set the
259			# LD_LIBRARY_PATH as it has to be to run Headas by itself without SAS been
260			# ready.
261
262			export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${SASBUILD_LOCAL}/gcc/lib64
263
264
265	ccflib	1.18	headas_ok=`cat ${headas_output}`
266
267			if [ -n "${headas_ok}" ] ; then
268			echo "mlovccf: HEADAS initialisation failed with error" >> ${mailfile}
269			cat ${headas_output} >> ${mailfile}
270	ccflib	1.17	mailnotify "mlovccf: HEADAS initilisation failed" ${mailfile}
271	ccflib	1.24	if [ "$DEBUG" ] ; then
272			echo "DEBUG: mlovccf: main: Headas initialization failed. Notification mail sent!"
273			fi
274			rm -v ${mailfile}
275			rm -v ${headas_output}
276	ccflib	1.17	exit 1
277			fi
278	ccflib	1.1
279	ccflib	1.17	# To be able to mix several jobs on the same ccflib account
280			[ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
281			export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
282	ccflib	1.24	if [ "$DEBUG" ] ; then
283			echo -n "DEBUG: mlovccf: main: HEADAS version: "`fversion`
284			fi
285	ccflib	1.17
286	ccflib	1.18	#
287			# W a r n i n g o n S A S i n i t i a l i s a t i o n
288			#
289			# The SAS initialisation produced by setsas.sh sets SAS_CCFPATH to /ccf/pub.
290			# Hence, if we set SAS_CCFPATH before running setsas.sh, we will always
291			# use /ccf/pub as repository for CCF. This is right as long as /ccf/pub is in
292			# strict sync with /home/ccflib/ccf_ftp/constituents. But if this is not true,
293			# the resulting valid CCFs might be wrong. Therefore, we set it here explicitly.
294			#
295
296			confsas_output="${VALID_CCF}/confsas_output"
297	ccflib	1.26	source /sas/bin/confsas > ${confsas_output} 2>&1
298			confsas_error=`cat ${confsas_output} \| grep -c Error`
299	ccflib	1.18
300	ccflib	1.24	if [ "$DEBUG" ] ; then
301			echo "DEBUG: mlovccf: main: Executed /sas/bin/confsas with output: ${confsas_ok}"
302			fi
303
304	ccflib	1.26	if [ "${confsas_error}" != "0" ] ; then
305	ccflib	1.18	echo "mlovccf: SAS initialization failed with error" >> ${mailfile}
306			cat ${confsas_output} >> ${mailfile}
307			mailnotify "mlovccf: SAS initialisation failed" ${mailfile}
308	ccflib	1.24	if [ "$DEBUG" ] ; then
309			echo "DEBUG: mlovccf: main: SAS initialization failed. Notification mail sent!"
310			fi
311	ccflib	1.18	exit 1
312	ccflib	1.17	fi
313
314	ccflib	1.26	rm -v ${mailfile}
315			rm -v ${confsas_output}
316
317	ccflib	1.17	source $HOME/setsas.sh 1> /dev/null
318	ccflib	1.18
319	ccflib	1.24	if [ "$DEBUG" ] ; then
320			echo "DEBUG: mlovccf: main: SAS setsas.sh script executed"
321			echo
322			echo "DEBUG: mlovccf: main: SAS Version: "`sasversion -V 1`
323			fi
324
325
326			# Set SAS vervbosity to 0 from now onwards
327
328			export SAS_VERBOSITY=0
329
330
331	ccflib	1.18	# Re-sets SAS_CCFPATH
332
333	ccflib	1.17	export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
334	ccflib	1.18
335	ccflib	1.17
336	ccflib	1.18
337	ccflib	1.17	# 3. XMM_CALINDEX file from SAS_CCFPATH
338
339			MIF=`ls -1 ${SAS_CCFPATH}/XMM_CALINDEX_*.CCF`
340
341			# 4. For each OBDS ID registered in the XSA observations list, gets the start
342			# time of the observation. Then runs cifbuild to get the respective CIF.
343			# all_ccfs_${now}.txt is filled recursively on each iteration.
344			# Therefore we start with an empty file. The final list will be this one.
345	ccflib	1.13
346			# Output file : ${VALID_CCF}/all_ccfs_${now}.txt
347	ccflib	1.1
348	ccflib	1.3	touch ${VALID_CCF}/all_ccfs_${now}.txt
349	ccflib	1.2
350	ccflib	1.1	n=0
351	ccflib	1.2
352	ccflib	1.1	while read line
353			do
354			obsid=`echo $line \| awk -F"\|" '{print $2}'`
355			stime=`echo $line \| awk -F"\|" '{print $9}'`
356			stime=`echo $stime \| tr " " "T"`
357			n=$((n+1))
358			echo "($n) Processing $obsid $stime" ; echo
359
360			# Run cifbuild
361
362			cif_file=${VALID_CCF}/${obsid}.cif
363
364			cifbuild --withmasterindexset='yes' \
365			--masterindexset=${MIF} \
366			--withobservationdate='yes' \
367			--observationdate=${stime} \
368			--analysisdate='now' \
369			--calindexset=${cif_file}
370
371
372			# Gets the list of CCFs included in the CIF
373
374			get_ccf_list "${cif_file}" > ${VALID_CCF}/${obsid}_ccfs.tmp
375
376			while read line
377			do
378			[ "$line" = "" ] && continue
379			class=`echo $line \| awk -F" " '{print $1"_"$2}'`
380			issue=`echo $line \| awk -F" " '{print $3}'`
381			issue=`printf "%04d" ${issue}`
382			echo ${class}_${issue}.CCF >> ${VALID_CCF}/${obsid}_ccfs.txt
383	ccflib	1.2
384	ccflib	1.1	done < ${VALID_CCF}/${obsid}_ccfs.tmp
385
386			rm ${VALID_CCF}/${obsid}_ccfs.tmp
387
388	ccflib	1.3	mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp
389	ccflib	1.2
390	ccflib	1.3	cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp \| sort -u > ${VALID_CCF}/all_ccfs_${now}.txt
391	ccflib	1.1
392	ccflib	1.17	comm -1 -3 --nocheck-order ${VALID_CCF}/all_ccfs_${now}.tmp ${VALID_CCF}/all_ccfs_${now}.txt > ${VALID_CCF}/diff_iteration_${n}.txt
393
394			diffsize=`stat --format=%s ${VALID_CCF}/diff_iteration_${n}.txt`
395
396			if [ "$diffsize" != "0" ] ; then
397			echo "CCF added in this iteration: "
398			cat ${VALID_CCF}/diff_iteration_${n}.txt
399			fi
400
401			echo
402			echo "Number of valid CCFs found so far: "`cat ${VALID_CCF}/all_ccfs_${now}.txt \| wc -l` ; echo
403
404			rm ${VALID_CCF}/diff_iteration_${n}.txt
405	ccflib	1.1	rm ${cif_file}
406			rm ${VALID_CCF}/${obsid}_ccfs.txt
407	ccflib	1.3	rm ${VALID_CCF}/all_ccfs_${now}.tmp
408	ccflib	1.1
409	ccflib	1.7	done < ${VALID_CCF}/xsaobslog_${now}.txt
410	ccflib	1.2
411	ccflib	1.13	# 5. Make a table of classes and issues with the style "Pipeline Release Notes"
412			# To get a classification, we need first to obtain all available classes.
413			#
414
415	ccflib	1.2
416			# Lists all CCF classes
417
418			ccf_classes="${VALID_CCF}/ccf_classes.txt"
419
420			while read line
421			do
422			class=`echo $line \| awk -F"_" '{print $1"_"$2}'`
423			echo $class >> ${ccf_classes}
424	ccflib	1.3	done < ${VALID_CCF}/all_ccfs_${now}.txt
425	ccflib	1.2
426			cat ${ccf_classes} \| sort -u > ${VALID_CCF}/kk.txt
427			mv ${VALID_CCF}/kk.txt ${ccf_classes}
428
429	ccflib	1.13
430
431	ccflib	1.3	# Output final file is named all_ccfs_${now}_table.txt
432
433			total_number_of_ccfs=0
434	ccflib	1.2
435	ccflib	1.7	all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt"
436	ccflib	1.2
437			[ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}
438
439	ccflib	1.7	echo "Table of valid CCFs at $now_table" >> ${all_ccfs_table}
440			echo >> ${all_ccfs_table}
441			echo "\|================================\|=============\|=======\|" >> ${all_ccfs_table}
442			echo "\| Calibration File \| Issue range \| Count \|" >> ${all_ccfs_table}
443			echo "\|================================\|=============\|=======\|" >> ${all_ccfs_table}
444	ccflib	1.2
445			while read ccf_class
446	ccflib	1.3	do
447			echo -n "Processing class $ccf_class ..."
448	ccflib	1.2	issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
449			[ -f "${issues_file}" ] && rm ${issues_file}
450			while read line
451			do
452			class=`echo $line \| awk -F"_" '{print $1"_"$2}'`
453			[ "$class" != "$ccf_class" ] && continue
454			issue=`echo $line \| awk -F"_" '{print $3}' \| cut -d. -f1`
455			echo $issue >> ${issues_file}
456	ccflib	1.3	done < ${VALID_CCF}/all_ccfs_${now}.txt
457	ccflib	1.2
458			cat ${issues_file} \| sort -u > kk.txt
459			mv kk.txt ${issues_file}
460	ccflib	1.3	real_number_of_issues_for_this_class=`wc -l ${issues_file} \| awk -F" " '{print $1}'`
461	ccflib	1.2
462			first_issue=`head -1 ${issues_file}`
463			last_issue=`tail -1 ${issues_file}`
464
465	ccflib	1.3	li=$((10#$last_issue))
466			fi=$((10#$first_issue))
467
468	ccflib	1.7	sequential_number_of_issues_for_this_class=$((li - fi + 1))
469	ccflib	1.8
470			asterisk=" "
471			[ "${real_number_of_issues_for_this_class}" -lt "${sequential_number_of_issues_for_this_class}" ] && asterisk="(*)"
472
473	ccflib	1.7	total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs))
474
475			rn=$real_number_of_issues_for_this_class
476	ccflib	1.3
477	ccflib	1.8	if [ "${first_issue}" != "${last_issue}" ] ; then
478			if [ "$asterisk" = "(*)" ] ; then
479			printf "\| %-30s \| %4s-%4s(*)\| %4s \|\n" ${ccf_class} ${first_issue} ${last_issue} ${rn} >> ${all_ccfs_table}
480			else
481			printf "\| %-30s \| %4s-%4s \| %4s \|\n" ${ccf_class} ${first_issue} ${last_issue} ${rn} >> ${all_ccfs_table}
482			fi
483			else
484			printf "\| %-30s \| %4s \| %4s \|\n" ${ccf_class} ${first_issue} ${rn} >> ${all_ccfs_table}
485			fi
486
487	ccflib	1.7
488			echo "done"
489	ccflib	1.2	done < ${ccf_classes}
490
491	ccflib	1.7	echo "\|================================\|=============\|=======\|" >> ${all_ccfs_table}
492			echo >> ${all_ccfs_table}
493			echo "(*): Incomplete range." >> ${all_ccfs_table}
494			echo "" >> ${all_ccfs_table}
495			echo "Total number of valid CCFs required : ${total_number_of_ccfs}" >> ${all_ccfs_table}
496	ccflib	1.3	rm ${VALID_CCF}/ccf_classes.txt
497			rm ${VALID_CCF}/ccf_issues_for*.txt
498	ccflib	1.11
499	ccflib	1.13
500
501			# 6. Fill in the Valid CCF Set dir with the proper CCFs
502	ccflib	1.11
503	ccflib	1.25	fillvalidccfdir "all_ccfs_${now}.txt"
504	ccflib	1.13
505	ccflib	1.22
506	ccflib	1.25	[ -f "${mailfile}" ] && rm -rf ${mailfile}
507			echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile
508			echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}" >> $mailfile
509			mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile
510			rm ${mailfile}