Annotation of /bin/mlovccf

#! /bin/bash

# mlovccf (make list of valid ccfs)
#
# Makes the list of valid CCFs that are all CCFs, among those published since the 
# beginning of the project, that are required to process any ODF at a given date,
# typically the current date. Thus, this list will change with time as far as new
# CCFs are issued to deal with new calibration data and/or replace obsolete ones. 
#
# To identify the set of CCFs required to process a given ODF is the task of cifbuild. 
# To make such identification, cifbuild needs only the start time of the observation. 
#
# If we get the list of required CCFs for all observed ODFs at a given date, 
# we will obtain for such a moment, the list of valid CCFs required to process 
# any observed ODF. Thus, we need as input the list of all observed ODFs at a given
# date. This is obtained from the report of all observations that the XSA makes to CDS,
# xsaobslog.txt (ftp://nxsa.esac.esa.int/pub/cfs_obslog/xsaobslog.txt). 
#
# $Id: mlovccf,v 1.5 2015/03/11 16:12:16 ccflib Exp $

# Next two lines allow to submit this script to the grid
# request Bourne shell as shell for job
#$ -S /bin/bash


host=`hostname | cut -d. -f1`

now=`date +'%Y%m%d_%H%M'`

# Function find_latest_mif to get the latest XMM_CALINDEX from given directory

find_latest_mif()
{

[ -z "$1" ] && return

CCFPATH="$1"

maxissue="0"
for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF`
do 
        mifbase=`basename $mif`
        issue=`echo $mifbase | awk -F"." '{print $1}' | awk -F"_" '{print $3}'`
        [ "$issue" -ge "$maxissue" ] && maxissue="$issue"
done

MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF

}

# Function get_ccf_list to list on stdout the table of CCFs in a given CIF
#
# Requires that Heasoft is initialised. Otherwise exits with error.

get_ccf_list()
{
[ -z "$1" ] && return

cif_to_process="$1"

noversion=`which fversion | grep -c no`

if [ "${noversion}" != "0" ] ; then 
        echo "Error: Heasoft not initialised !"
        exit
fi

fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
                        columns="SCOPE TYPEID ISSUE"            \
                        rows=-  STDOUT  
}


# We need to get the list of all observed ODFs at the current date. 
# This is obtained from ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt.

VALID_CCF="$HOME/valid_ccf"

cd ${VALID_CCF}
wget -nc -q ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt


# Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host 

case "$host" in
        xvsoc01|xmm)
                export SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"
                export SAS_DIR=/data/xmm/ccflib/sas
                export SAS_PATH=$SAS_DIR
                source $SAS_DIR/sas-setup.sh
                ;;
        sasbld01|sasbld02)
                export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
                /sas/bin/confsas
                export HEADAS=/sasbuild/local/${host}/headas/architecture
                . $HEADAS/headas-init.sh
                source $HOME/setsas.sh 1> /dev/null
                export SAS_VERBOSITY=0
                ;;
        scigrid6|cn-*)
                export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
                /sas/bin/confsas
                export HEADAS=/sasbuild/local/sasbld02/headas/architecture
                . $HEADAS/headas-init.sh
                fversion
                source $HOME/setsas.sh 1> /dev/null
                export SAS_VERBOSITY=0
                ;;
        *) echo "Error: Do not know how to do it in host $host" ; exit ;;
esac

# Finds the latest MIF issue

find_latest_mif "${SAS_CCFPATH}"


# Now scans the list of OBDS ID registered in the previous list to get the start time
# and then runs cifbuild to get the respective CIF

# Output file
touch ${VALID_CCF}/all_ccfs_${now}.txt

n=0

while read line 
do
        obsid=`echo $line | awk -F"|" '{print $2}'` 
        stime=`echo $line | awk -F"|" '{print $9}'`
        stime=`echo $stime | tr " " "T"`
        n=$((n+1))
        echo "($n) Processing $obsid $stime" ; echo

        # Run cifbuild

        cif_file=${VALID_CCF}/${obsid}.cif      

        cifbuild --withmasterindexset='yes'  \
                 --masterindexset=${MIF}     \
                 --withobservationdate='yes' \
                 --observationdate=${stime}  \
                 --analysisdate='now'        \
                 --calindexset=${cif_file} 


        # Gets the list of CCFs included in the CIF

        get_ccf_list "${cif_file}" > ${VALID_CCF}/${obsid}_ccfs.tmp

        while read line 
        do
                [ "$line" = "" ] && continue
                class=`echo $line | awk -F" " '{print $1"_"$2}'`
                issue=`echo $line | awk -F" " '{print $3}'` 
                issue=`printf "%04d" ${issue}`
                echo ${class}_${issue}.CCF >> ${VALID_CCF}/${obsid}_ccfs.txt

        done < ${VALID_CCF}/${obsid}_ccfs.tmp

        rm ${VALID_CCF}/${obsid}_ccfs.tmp

        mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp

        cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt

        rm ${cif_file}
        rm ${VALID_CCF}/${obsid}_ccfs.txt
        rm ${VALID_CCF}/all_ccfs_${now}.tmp

done < ${VALID_CCF}/xsaobslog.txt

# Make a table of classes and issues type "Pipeline Release Notes"

# Lists all CCF classes

ccf_classes="${VALID_CCF}/ccf_classes.txt"

while read line
do 
        class=`echo $line | awk -F"_" '{print $1"_"$2}'`
        echo $class >> ${ccf_classes}
done < ${VALID_CCF}/all_ccfs_${now}.txt

cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt
mv ${VALID_CCF}/kk.txt ${ccf_classes}

# Output final file is named all_ccfs_${now}_table.txt

total_number_of_ccfs=0

all_ccfs_table="{VALID_CCF}/all_ccfs_${now}_table.txt

[ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}

echo "Table of valid CCFs at $now"                      >> ${all_ccfs_table}
echo                                                    >> ${all_ccfs_table}
echo "|================================|=============|" >> ${all_ccfs_table} 
echo "| Calibration File               | Issue range |" >> ${all_ccfs_table}
echo "|================================|=============|" >> ${all_ccfs_table}

while read ccf_class
do
        echo -n "Processing class $ccf_class ..." 
        issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
        [ -f "${issues_file}" ] && rm ${issues_file}
        while read line
        do
                class=`echo $line | awk -F"_" '{print $1"_"$2}'`
                [ "$class" != "$ccf_class" ] && continue
                issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`
                echo $issue >> ${issues_file}
        done < ${VALID_CCF}/all_ccfs_${now}.txt

        cat ${issues_file} | sort -u > kk.txt
        mv kk.txt ${issues_file}
        real_number_of_issues_for_this_class=`wc -l ${issues_file} | awk -F" " '{print $1}'`

        first_issue=`head -1 ${issues_file}`
        last_issue=`tail -1 ${issues_file}`

        li=$((10#$last_issue))
        fi=$((10#$first_issue))

        sequential_number_of_issues_for_this_class=$(($li - $fi + 1))
        if [ "${real_number_of_issues_for_this_class}"  -lt "${sequential_number_of_issues_for_this_class}" ] ; then 
                asterisk="(*)"
        else
                asterisk="   "
        fi
        total_number_of_ccfs=$(($real_number_of_issues_for_this_class + $total_number_of_ccfs))

        if [ "${first_issue}" != "${last_issue}" ] ; then 
                printf "| %-30s | %4s-%4s%3s|\n" ${ccf_class} ${first_issue} ${last_issue} ${asterisk} >> ${all_ccfs_table}
        else
                printf "| %-30s | %4s        |\n" ${ccf_class} ${first_issue}                          >> ${all_ccfs_table}
        fi
  
done < ${ccf_classes}  

echo "|================================|=============|"                                                >> ${all_ccfs_table}
echo                                                                                                   >> ${all_ccfs_table}
echo "(*): Some issues in the range are not required."                                                 >> ${all_ccfs_table} 
rm ${VALID_CCF}/ccf_classes.txt
rm ${VALID_CCF}/ccf_issues_for*.txt


echo ; echo "Number of valid CCFs required as of $now : ${total_number_of_ccfs}" 
1	ccflib	1.1	#! /bin/bash
2
3			# mlovccf (make list of valid ccfs)
4			#
5	ccflib	1.2	# Makes the list of valid CCFs that are all CCFs, among those published since the
6			# beginning of the project, that are required to process any ODF at a given date,
7			# typically the current date. Thus, this list will change with time as far as new
8			# CCFs are issued to deal with new calibration data and/or replace obsolete ones.
9	ccflib	1.1	#
10	ccflib	1.2	# To identify the set of CCFs required to process a given ODF is the task of cifbuild.
11			# To make such identification, cifbuild needs only the start time of the observation.
12	ccflib	1.1	#
13	ccflib	1.2	# If we get the list of required CCFs for all observed ODFs at a given date,
14			# we will obtain for such a moment, the list of valid CCFs required to process
15			# any observed ODF. Thus, we need as input the list of all observed ODFs at a given
16			# date. This is obtained from the report of all observations that the XSA makes to CDS,
17			# xsaobslog.txt (ftp://nxsa.esac.esa.int/pub/cfs_obslog/xsaobslog.txt).
18			#
19	ccflib	1.6	# $Id: mlovccf,v 1.5 2015/03/11 16:12:16 ccflib Exp $
20	ccflib	1.4
21			# Next two lines allow to submit this script to the grid
22			# request Bourne shell as shell for job
23	ccflib	1.6	#$ -S /bin/bash
24	ccflib	1.1
25	ccflib	1.5
26	ccflib	1.1	host=`hostname \| cut -d. -f1`
27
28	ccflib	1.5	now=`date +'%Y%m%d_%H%M'`
29	ccflib	1.3
30	ccflib	1.1	# Function find_latest_mif to get the latest XMM_CALINDEX from given directory
31
32			find_latest_mif()
33			{
34
35			[ -z "$1" ] && return
36
37			CCFPATH="$1"
38
39			maxissue="0"
40			for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF`
41			do
42			mifbase=`basename $mif`
43			issue=`echo $mifbase \| awk -F"." '{print $1}' \| awk -F"_" '{print $3}'`
44			[ "$issue" -ge "$maxissue" ] && maxissue="$issue"
45			done
46
47			MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF
48
49			}
50
51	ccflib	1.2	# Function get_ccf_list to list on stdout the table of CCFs in a given CIF
52	ccflib	1.1	#
53	ccflib	1.2	# Requires that Heasoft is initialised. Otherwise exits with error.
54	ccflib	1.1
55			get_ccf_list()
56			{
57			[ -z "$1" ] && return
58
59			cif_to_process="$1"
60
61			noversion=`which fversion \| grep -c no`
62
63			if [ "${noversion}" != "0" ] ; then
64			echo "Error: Heasoft not initialised !"
65			exit
66			fi
67
68			fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
69			columns="SCOPE TYPEID ISSUE" \
70			rows=- STDOUT
71			}
72
73
74	ccflib	1.2	# We need to get the list of all observed ODFs at the current date.
75			# This is obtained from ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt.
76	ccflib	1.1
77			VALID_CCF="$HOME/valid_ccf"
78
79			cd ${VALID_CCF}
80			wget -nc -q ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt
81
82
83			# Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host
84
85			case "$host" in
86			xvsoc01\|xmm)
87	ccflib	1.5	export SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"
88			export SAS_DIR=/data/xmm/ccflib/sas
89			export SAS_PATH=$SAS_DIR
90	ccflib	1.1	source $SAS_DIR/sas-setup.sh
91			;;
92			sasbld01\|sasbld02)
93	ccflib	1.5	export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
94	ccflib	1.1	/sas/bin/confsas
95			export HEADAS=/sasbuild/local/${host}/headas/architecture
96			. $HEADAS/headas-init.sh
97			source $HOME/setsas.sh 1> /dev/null
98			export SAS_VERBOSITY=0
99			;;
100	ccflib	1.5	scigrid6\|cn-*)
101			export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
102			/sas/bin/confsas
103			export HEADAS=/sasbuild/local/sasbld02/headas/architecture
104			. $HEADAS/headas-init.sh
105			fversion
106			source $HOME/setsas.sh 1> /dev/null
107			export SAS_VERBOSITY=0
108			;;
109			*) echo "Error: Do not know how to do it in host $host" ; exit ;;
110	ccflib	1.1	esac
111
112			# Finds the latest MIF issue
113
114			find_latest_mif "${SAS_CCFPATH}"
115
116
117			# Now scans the list of OBDS ID registered in the previous list to get the start time
118			# and then runs cifbuild to get the respective CIF
119
120	ccflib	1.2	# Output file
121	ccflib	1.3	touch ${VALID_CCF}/all_ccfs_${now}.txt
122	ccflib	1.2
123	ccflib	1.1	n=0
124	ccflib	1.2
125	ccflib	1.1	while read line
126			do
127			obsid=`echo $line \| awk -F"\|" '{print $2}'`
128			stime=`echo $line \| awk -F"\|" '{print $9}'`
129			stime=`echo $stime \| tr " " "T"`
130			n=$((n+1))
131			echo "($n) Processing $obsid $stime" ; echo
132
133			# Run cifbuild
134
135			cif_file=${VALID_CCF}/${obsid}.cif
136
137			cifbuild --withmasterindexset='yes' \
138			--masterindexset=${MIF} \
139			--withobservationdate='yes' \
140			--observationdate=${stime} \
141			--analysisdate='now' \
142			--calindexset=${cif_file}
143
144
145			# Gets the list of CCFs included in the CIF
146
147			get_ccf_list "${cif_file}" > ${VALID_CCF}/${obsid}_ccfs.tmp
148
149			while read line
150			do
151			[ "$line" = "" ] && continue
152			class=`echo $line \| awk -F" " '{print $1"_"$2}'`
153			issue=`echo $line \| awk -F" " '{print $3}'`
154			issue=`printf "%04d" ${issue}`
155			echo ${class}_${issue}.CCF >> ${VALID_CCF}/${obsid}_ccfs.txt
156	ccflib	1.2
157	ccflib	1.1	done < ${VALID_CCF}/${obsid}_ccfs.tmp
158
159			rm ${VALID_CCF}/${obsid}_ccfs.tmp
160
161	ccflib	1.3	mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp
162	ccflib	1.2
163	ccflib	1.3	cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp \| sort -u > ${VALID_CCF}/all_ccfs_${now}.txt
164	ccflib	1.1
165			rm ${cif_file}
166			rm ${VALID_CCF}/${obsid}_ccfs.txt
167	ccflib	1.3	rm ${VALID_CCF}/all_ccfs_${now}.tmp
168	ccflib	1.1
169			done < ${VALID_CCF}/xsaobslog.txt
170	ccflib	1.2
171			# Make a table of classes and issues type "Pipeline Release Notes"
172
173			# Lists all CCF classes
174
175			ccf_classes="${VALID_CCF}/ccf_classes.txt"
176
177			while read line
178			do
179			class=`echo $line \| awk -F"_" '{print $1"_"$2}'`
180			echo $class >> ${ccf_classes}
181	ccflib	1.3	done < ${VALID_CCF}/all_ccfs_${now}.txt
182	ccflib	1.2
183			cat ${ccf_classes} \| sort -u > ${VALID_CCF}/kk.txt
184			mv ${VALID_CCF}/kk.txt ${ccf_classes}
185
186	ccflib	1.3	# Output final file is named all_ccfs_${now}_table.txt
187
188			total_number_of_ccfs=0
189	ccflib	1.2
190	ccflib	1.3	all_ccfs_table="{VALID_CCF}/all_ccfs_${now}_table.txt
191	ccflib	1.2
192			[ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}
193
194	ccflib	1.3	echo "Table of valid CCFs at $now" >> ${all_ccfs_table}
195			echo >> ${all_ccfs_table}
196	ccflib	1.2	echo "\|================================\|=============\|" >> ${all_ccfs_table}
197			echo "\| Calibration File \| Issue range \|" >> ${all_ccfs_table}
198			echo "\|================================\|=============\|" >> ${all_ccfs_table}
199
200			while read ccf_class
201	ccflib	1.3	do
202			echo -n "Processing class $ccf_class ..."
203	ccflib	1.2	issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
204			[ -f "${issues_file}" ] && rm ${issues_file}
205			while read line
206			do
207			class=`echo $line \| awk -F"_" '{print $1"_"$2}'`
208			[ "$class" != "$ccf_class" ] && continue
209			issue=`echo $line \| awk -F"_" '{print $3}' \| cut -d. -f1`
210			echo $issue >> ${issues_file}
211	ccflib	1.3	done < ${VALID_CCF}/all_ccfs_${now}.txt
212	ccflib	1.2
213			cat ${issues_file} \| sort -u > kk.txt
214			mv kk.txt ${issues_file}
215	ccflib	1.3	real_number_of_issues_for_this_class=`wc -l ${issues_file} \| awk -F" " '{print $1}'`
216	ccflib	1.2
217			first_issue=`head -1 ${issues_file}`
218			last_issue=`tail -1 ${issues_file}`
219
220	ccflib	1.3	li=$((10#$last_issue))
221			fi=$((10#$first_issue))
222
223			sequential_number_of_issues_for_this_class=$(($li - $fi + 1))
224			if [ "${real_number_of_issues_for_this_class}" -lt "${sequential_number_of_issues_for_this_class}" ] ; then
225			asterisk="(*)"
226			else
227			asterisk=" "
228			fi
229			total_number_of_ccfs=$(($real_number_of_issues_for_this_class + $total_number_of_ccfs))
230
231	ccflib	1.2	if [ "${first_issue}" != "${last_issue}" ] ; then
232	ccflib	1.3	printf "\| %-30s \| %4s-%4s%3s\|\n" ${ccf_class} ${first_issue} ${last_issue} ${asterisk} >> ${all_ccfs_table}
233	ccflib	1.2	else
234	ccflib	1.3	printf "\| %-30s \| %4s \|\n" ${ccf_class} ${first_issue} >> ${all_ccfs_table}
235	ccflib	1.2	fi
236
237			done < ${ccf_classes}
238
239	ccflib	1.3	echo "\|================================\|=============\|" >> ${all_ccfs_table}
240			echo >> ${all_ccfs_table}
241			echo "(*): Some issues in the range are not required." >> ${all_ccfs_table}
242			rm ${VALID_CCF}/ccf_classes.txt
243			rm ${VALID_CCF}/ccf_issues_for*.txt
244
245
246
247			echo ; echo "Number of valid CCFs required as of $now : ${total_number_of_ccfs}"