Annotation of /bin/mlovccf

#! /bin/bash

# mlovccf (make list of valid ccfs)
#
# Makes the list of valid CCFs. These are all CCFs, among those published since the 
# beginning of the project, that are required to process any ODF at a given date,
# typically the current date. This list will change with time as far as new
# CCFs are issued to deal with new calibration data and/or replace obsolete ones. 
#
# To identify the set of CCFs required to process a given ODF is the ijob of task cifbuild. 
# To make such identification, cifbuild needs only the start time of the observation. 
# Therefore, if we provide a list of all observed ODFs at a given date together with their
# respective start time, we could compute all the CCFs required to process them.
# For such a moment, that list is the list of valid CCFs required to process 
# any observed ODF. 
#
# We need as input the list of all observed ODFs at a given date.
# This is obtained from the report of all observations that the XSA makes to CDS,
# xsaobslog.txt (ftp://nxsa.esac.esa.int/pub/cfs_obslog/xsaobslog.txt). 
#
# $Id: mlovccf,v 1.8 2015/09/07 13:39:15 ccflib Exp $

# Next two lines allow to submit this script to the grid
# request Bourne shell as shell for job
#$ -S /bin/bash


host=`hostname | cut -d. -f1`

now=`date +'%Y%m%d_%H%M'`
now_table=`date +'%Y-%m-%dT%H:%M:%S'`

# Function find_latest_mif to get the latest XMM_CALINDEX from given directory

find_latest_mif()
{

[ -z "$1" ] && return

CCFPATH="$1"

maxissue="0"
for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF`
do 
        mifbase=`basename $mif`
        issue=`echo $mifbase | awk -F"." '{print $1}' | awk -F"_" '{print $3}'`
        [ "$issue" -ge "$maxissue" ] && maxissue="$issue"
done

MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF

}

# Function get_ccf_list to list on stdout the table of CCFs in a given CIF
#
# Requires that Heasoft is initialised. Otherwise exits with error.

get_ccf_list()
{
[ -z "$1" ] && return

cif_to_process="$1"

noversion=`which fversion | grep -c no`

if [ "${noversion}" != "0" ] ; then 
        echo "Error: Heasoft not initialised !"
        exit
fi

fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
                        columns="SCOPE TYPEID ISSUE"            \
                        rows=-  STDOUT  
}


# We need to get the list of all observed ODFs at the current date. 
# This is obtained from ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt.

VALID_CCF="$HOME/valid_ccf"

cd ${VALID_CCF}

# Remove any xsaobslog.txt previously downloaded. 

[ -f "xsaobslog.txt" ] && rm -rf xsaobslog.txt

# Get the list latest list of ODFs available from nXSA server.
 
wget -q http://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt

# Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference 
# of which list of ODFs was used to compute the list of valid CCFs.

mv xsaobslog.txt xsaobslog_${now}.txt

# Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host 

case "$host" in
        xvsoc01|xmm)
                export SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"
                export SAS_DIR=/data/xmm/ccflib/sas
                export SAS_PATH=$SAS_DIR
                source $SAS_DIR/sas-setup.sh
                ;;
        sasbld01|sasbld02)
                export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
                /sas/bin/confsas
                export HEADAS=/sasbuild/local/${host}/headas/architecture
                . $HEADAS/headas-init.sh
                # To be able to mix several jobs on the same ccflib account
                [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
                export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
                echo -n "HEADAS version: " 
                fversion
                source $HOME/setsas.sh 1> /dev/null
                echo "SAS Version: "
                sasversion -V 1
                export SAS_VERBOSITY=0
                ;;
        scigrid6|cn-*)
                export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
                /sas/bin/confsas
                export HEADAS=/sasbuild/local/sasbld02/headas/architecture
                . $HEADAS/headas-init.sh
                # To be able to mix several jobs on the same ccflib account
                [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
                export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
                echo -n "HEADAS version: " 
                fversion
                source $HOME/setsas.sh 1> /dev/null
                echo "SAS Version: "
                sasversion -V 1
                export SAS_VERBOSITY=0
                ;;
        *) echo "Error: Do not know how to do it in host $host" ; exit ;;
esac

# Finds the latest MIF issue

find_latest_mif "${SAS_CCFPATH}"


# Now scans the list of OBDS ID registered in the previous list to get the start time
# and then runs cifbuild to get the respective CIF

# Output file
touch ${VALID_CCF}/all_ccfs_${now}.txt

n=0

while read line 
do
        obsid=`echo $line | awk -F"|" '{print $2}'` 
        stime=`echo $line | awk -F"|" '{print $9}'`
        stime=`echo $stime | tr " " "T"`
        n=$((n+1))
        echo "($n) Processing $obsid $stime" ; echo

        # Run cifbuild

        cif_file=${VALID_CCF}/${obsid}.cif      

        cifbuild --withmasterindexset='yes'  \
                 --masterindexset=${MIF}     \
                 --withobservationdate='yes' \
                 --observationdate=${stime}  \
                 --analysisdate='now'        \
                 --calindexset=${cif_file} 


        # Gets the list of CCFs included in the CIF

        get_ccf_list "${cif_file}" > ${VALID_CCF}/${obsid}_ccfs.tmp

        while read line 
        do
                [ "$line" = "" ] && continue
                class=`echo $line | awk -F" " '{print $1"_"$2}'`
                issue=`echo $line | awk -F" " '{print $3}'` 
                issue=`printf "%04d" ${issue}`
                echo ${class}_${issue}.CCF >> ${VALID_CCF}/${obsid}_ccfs.txt

        done < ${VALID_CCF}/${obsid}_ccfs.tmp

        rm ${VALID_CCF}/${obsid}_ccfs.tmp

        mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp

        cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt

        rm ${cif_file}
        rm ${VALID_CCF}/${obsid}_ccfs.txt
        rm ${VALID_CCF}/all_ccfs_${now}.tmp

done < ${VALID_CCF}/xsaobslog_${now}.txt

# Make a table of classes and issues type "Pipeline Release Notes"

# Lists all CCF classes

ccf_classes="${VALID_CCF}/ccf_classes.txt"

while read line
do 
        class=`echo $line | awk -F"_" '{print $1"_"$2}'`
        echo $class >> ${ccf_classes}
done < ${VALID_CCF}/all_ccfs_${now}.txt

cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt
mv ${VALID_CCF}/kk.txt ${ccf_classes}

# Output final file is named all_ccfs_${now}_table.txt

total_number_of_ccfs=0

all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt"

[ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}

echo "Table of valid CCFs at $now_table"                        >> ${all_ccfs_table}
echo                                                            >> ${all_ccfs_table}
echo "|================================|=============|=======|" >> ${all_ccfs_table} 
echo "| Calibration File               | Issue range | Count |" >> ${all_ccfs_table}
echo "|================================|=============|=======|" >> ${all_ccfs_table}

while read ccf_class
do
        echo -n "Processing class $ccf_class ..." 
        issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
        [ -f "${issues_file}" ] && rm ${issues_file}
        while read line
        do
                class=`echo $line | awk -F"_" '{print $1"_"$2}'`
                [ "$class" != "$ccf_class" ] && continue
                issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`
                echo $issue >> ${issues_file}
        done < ${VALID_CCF}/all_ccfs_${now}.txt

        cat ${issues_file} | sort -u > kk.txt
        mv kk.txt ${issues_file}
        real_number_of_issues_for_this_class=`wc -l ${issues_file} | awk -F" " '{print $1}'`

        first_issue=`head -1 ${issues_file}`
        last_issue=`tail -1 ${issues_file}`

        li=$((10#$last_issue))
        fi=$((10#$first_issue))

        sequential_number_of_issues_for_this_class=$((li - fi + 1))

        asterisk="   "
        [ "${real_number_of_issues_for_this_class}"  -lt "${sequential_number_of_issues_for_this_class}" ] &&  asterisk="(*)"

        total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs))

        rn=$real_number_of_issues_for_this_class

        if [ "${first_issue}" != "${last_issue}" ] ; then
           if [ "$asterisk" = "(*)" ] ; then
              printf "| %-30s | %4s-%4s(*)|  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
           else
              printf "| %-30s | %4s-%4s   |  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
           fi
        else
           printf "| %-30s | %4s        |  %4s |\n" ${ccf_class} ${first_issue} ${rn}                           >> ${all_ccfs_table}
        fi

        
        echo "done" 
done < ${ccf_classes}  

echo "|================================|=============|=======|"                                                 >> ${all_ccfs_table}
echo                                                                                                            >> ${all_ccfs_table}
echo "(*): Incomplete range."                                                                                   >> ${all_ccfs_table} 
echo ""                                                                                                         >> ${all_ccfs_table}
echo "Total number of valid CCFs required : ${total_number_of_ccfs}"                                            >> ${all_ccfs_table}
rm ${VALID_CCF}/ccf_classes.txt
rm ${VALID_CCF}/ccf_issues_for*.txt
1	ccflib	1.1	#! /bin/bash
2
3			# mlovccf (make list of valid ccfs)
4			#
5	ccflib	1.7	# Makes the list of valid CCFs. These are all CCFs, among those published since the
6	ccflib	1.2	# beginning of the project, that are required to process any ODF at a given date,
7	ccflib	1.7	# typically the current date. This list will change with time as far as new
8	ccflib	1.2	# CCFs are issued to deal with new calibration data and/or replace obsolete ones.
9	ccflib	1.1	#
10	ccflib	1.7	# To identify the set of CCFs required to process a given ODF is the ijob of task cifbuild.
11	ccflib	1.2	# To make such identification, cifbuild needs only the start time of the observation.
12	ccflib	1.7	# Therefore, if we provide a list of all observed ODFs at a given date together with their
13			# respective start time, we could compute all the CCFs required to process them.
14			# For such a moment, that list is the list of valid CCFs required to process
15			# any observed ODF.
16	ccflib	1.1	#
17	ccflib	1.7	# We need as input the list of all observed ODFs at a given date.
18			# This is obtained from the report of all observations that the XSA makes to CDS,
19	ccflib	1.2	# xsaobslog.txt (ftp://nxsa.esac.esa.int/pub/cfs_obslog/xsaobslog.txt).
20			#
21	ccflib	1.9	# $Id: mlovccf,v 1.8 2015/09/07 13:39:15 ccflib Exp $
22	ccflib	1.4
23			# Next two lines allow to submit this script to the grid
24			# request Bourne shell as shell for job
25	ccflib	1.6	#$ -S /bin/bash
26	ccflib	1.1
27	ccflib	1.5
28	ccflib	1.1	host=`hostname \| cut -d. -f1`
29
30	ccflib	1.5	now=`date +'%Y%m%d_%H%M'`
31	ccflib	1.7	now_table=`date +'%Y-%m-%dT%H:%M:%S'`
32	ccflib	1.3
33	ccflib	1.1	# Function find_latest_mif to get the latest XMM_CALINDEX from given directory
34
35			find_latest_mif()
36			{
37
38			[ -z "$1" ] && return
39
40			CCFPATH="$1"
41
42			maxissue="0"
43			for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF`
44			do
45			mifbase=`basename $mif`
46			issue=`echo $mifbase \| awk -F"." '{print $1}' \| awk -F"_" '{print $3}'`
47			[ "$issue" -ge "$maxissue" ] && maxissue="$issue"
48			done
49
50			MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF
51
52			}
53
54	ccflib	1.2	# Function get_ccf_list to list on stdout the table of CCFs in a given CIF
55	ccflib	1.1	#
56	ccflib	1.2	# Requires that Heasoft is initialised. Otherwise exits with error.
57	ccflib	1.1
58			get_ccf_list()
59			{
60			[ -z "$1" ] && return
61
62			cif_to_process="$1"
63
64			noversion=`which fversion \| grep -c no`
65
66			if [ "${noversion}" != "0" ] ; then
67			echo "Error: Heasoft not initialised !"
68			exit
69			fi
70
71			fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
72			columns="SCOPE TYPEID ISSUE" \
73			rows=- STDOUT
74			}
75
76
77	ccflib	1.2	# We need to get the list of all observed ODFs at the current date.
78			# This is obtained from ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt.
79	ccflib	1.1
80			VALID_CCF="$HOME/valid_ccf"
81
82			cd ${VALID_CCF}
83
84	ccflib	1.7	# Remove any xsaobslog.txt previously downloaded.
85
86			[ -f "xsaobslog.txt" ] && rm -rf xsaobslog.txt
87
88			# Get the list latest list of ODFs available from nXSA server.
89
90	ccflib	1.9	wget -q http://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt
91	ccflib	1.7
92			# Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference
93			# of which list of ODFs was used to compute the list of valid CCFs.
94
95			mv xsaobslog.txt xsaobslog_${now}.txt
96	ccflib	1.1
97			# Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host
98
99			case "$host" in
100			xvsoc01\|xmm)
101	ccflib	1.5	export SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"
102			export SAS_DIR=/data/xmm/ccflib/sas
103			export SAS_PATH=$SAS_DIR
104	ccflib	1.1	source $SAS_DIR/sas-setup.sh
105			;;
106			sasbld01\|sasbld02)
107	ccflib	1.5	export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
108	ccflib	1.1	/sas/bin/confsas
109			export HEADAS=/sasbuild/local/${host}/headas/architecture
110			. $HEADAS/headas-init.sh
111	ccflib	1.7	# To be able to mix several jobs on the same ccflib account
112			[ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
113			export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
114			echo -n "HEADAS version: "
115			fversion
116	ccflib	1.1	source $HOME/setsas.sh 1> /dev/null
117	ccflib	1.7	echo "SAS Version: "
118			sasversion -V 1
119	ccflib	1.1	export SAS_VERBOSITY=0
120			;;
121	ccflib	1.5	scigrid6\|cn-*)
122			export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
123			/sas/bin/confsas
124			export HEADAS=/sasbuild/local/sasbld02/headas/architecture
125			. $HEADAS/headas-init.sh
126	ccflib	1.7	# To be able to mix several jobs on the same ccflib account
127			[ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
128			export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
129			echo -n "HEADAS version: "
130			fversion
131	ccflib	1.5	source $HOME/setsas.sh 1> /dev/null
132	ccflib	1.7	echo "SAS Version: "
133			sasversion -V 1
134	ccflib	1.5	export SAS_VERBOSITY=0
135			;;
136			*) echo "Error: Do not know how to do it in host $host" ; exit ;;
137	ccflib	1.1	esac
138
139			# Finds the latest MIF issue
140
141			find_latest_mif "${SAS_CCFPATH}"
142
143
144			# Now scans the list of OBDS ID registered in the previous list to get the start time
145			# and then runs cifbuild to get the respective CIF
146
147	ccflib	1.2	# Output file
148	ccflib	1.3	touch ${VALID_CCF}/all_ccfs_${now}.txt
149	ccflib	1.2
150	ccflib	1.1	n=0
151	ccflib	1.2
152	ccflib	1.1	while read line
153			do
154			obsid=`echo $line \| awk -F"\|" '{print $2}'`
155			stime=`echo $line \| awk -F"\|" '{print $9}'`
156			stime=`echo $stime \| tr " " "T"`
157			n=$((n+1))
158			echo "($n) Processing $obsid $stime" ; echo
159
160			# Run cifbuild
161
162			cif_file=${VALID_CCF}/${obsid}.cif
163
164			cifbuild --withmasterindexset='yes' \
165			--masterindexset=${MIF} \
166			--withobservationdate='yes' \
167			--observationdate=${stime} \
168			--analysisdate='now' \
169			--calindexset=${cif_file}
170
171
172			# Gets the list of CCFs included in the CIF
173
174			get_ccf_list "${cif_file}" > ${VALID_CCF}/${obsid}_ccfs.tmp
175
176			while read line
177			do
178			[ "$line" = "" ] && continue
179			class=`echo $line \| awk -F" " '{print $1"_"$2}'`
180			issue=`echo $line \| awk -F" " '{print $3}'`
181			issue=`printf "%04d" ${issue}`
182			echo ${class}_${issue}.CCF >> ${VALID_CCF}/${obsid}_ccfs.txt
183	ccflib	1.2
184	ccflib	1.1	done < ${VALID_CCF}/${obsid}_ccfs.tmp
185
186			rm ${VALID_CCF}/${obsid}_ccfs.tmp
187
188	ccflib	1.3	mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp
189	ccflib	1.2
190	ccflib	1.3	cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp \| sort -u > ${VALID_CCF}/all_ccfs_${now}.txt
191	ccflib	1.1
192			rm ${cif_file}
193			rm ${VALID_CCF}/${obsid}_ccfs.txt
194	ccflib	1.3	rm ${VALID_CCF}/all_ccfs_${now}.tmp
195	ccflib	1.1
196	ccflib	1.7	done < ${VALID_CCF}/xsaobslog_${now}.txt
197	ccflib	1.2
198			# Make a table of classes and issues type "Pipeline Release Notes"
199
200			# Lists all CCF classes
201
202			ccf_classes="${VALID_CCF}/ccf_classes.txt"
203
204			while read line
205			do
206			class=`echo $line \| awk -F"_" '{print $1"_"$2}'`
207			echo $class >> ${ccf_classes}
208	ccflib	1.3	done < ${VALID_CCF}/all_ccfs_${now}.txt
209	ccflib	1.2
210			cat ${ccf_classes} \| sort -u > ${VALID_CCF}/kk.txt
211			mv ${VALID_CCF}/kk.txt ${ccf_classes}
212
213	ccflib	1.3	# Output final file is named all_ccfs_${now}_table.txt
214
215			total_number_of_ccfs=0
216	ccflib	1.2
217	ccflib	1.7	all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt"
218	ccflib	1.2
219			[ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}
220
221	ccflib	1.7	echo "Table of valid CCFs at $now_table" >> ${all_ccfs_table}
222			echo >> ${all_ccfs_table}
223			echo "\|================================\|=============\|=======\|" >> ${all_ccfs_table}
224			echo "\| Calibration File \| Issue range \| Count \|" >> ${all_ccfs_table}
225			echo "\|================================\|=============\|=======\|" >> ${all_ccfs_table}
226	ccflib	1.2
227			while read ccf_class
228	ccflib	1.3	do
229			echo -n "Processing class $ccf_class ..."
230	ccflib	1.2	issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
231			[ -f "${issues_file}" ] && rm ${issues_file}
232			while read line
233			do
234			class=`echo $line \| awk -F"_" '{print $1"_"$2}'`
235			[ "$class" != "$ccf_class" ] && continue
236			issue=`echo $line \| awk -F"_" '{print $3}' \| cut -d. -f1`
237			echo $issue >> ${issues_file}
238	ccflib	1.3	done < ${VALID_CCF}/all_ccfs_${now}.txt
239	ccflib	1.2
240			cat ${issues_file} \| sort -u > kk.txt
241			mv kk.txt ${issues_file}
242	ccflib	1.3	real_number_of_issues_for_this_class=`wc -l ${issues_file} \| awk -F" " '{print $1}'`
243	ccflib	1.2
244			first_issue=`head -1 ${issues_file}`
245			last_issue=`tail -1 ${issues_file}`
246
247	ccflib	1.3	li=$((10#$last_issue))
248			fi=$((10#$first_issue))
249
250	ccflib	1.7	sequential_number_of_issues_for_this_class=$((li - fi + 1))
251	ccflib	1.8
252			asterisk=" "
253			[ "${real_number_of_issues_for_this_class}" -lt "${sequential_number_of_issues_for_this_class}" ] && asterisk="(*)"
254
255	ccflib	1.7	total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs))
256
257			rn=$real_number_of_issues_for_this_class
258	ccflib	1.3
259	ccflib	1.8	if [ "${first_issue}" != "${last_issue}" ] ; then
260			if [ "$asterisk" = "(*)" ] ; then
261			printf "\| %-30s \| %4s-%4s(*)\| %4s \|\n" ${ccf_class} ${first_issue} ${last_issue} ${rn} >> ${all_ccfs_table}
262			else
263			printf "\| %-30s \| %4s-%4s \| %4s \|\n" ${ccf_class} ${first_issue} ${last_issue} ${rn} >> ${all_ccfs_table}
264			fi
265			else
266			printf "\| %-30s \| %4s \| %4s \|\n" ${ccf_class} ${first_issue} ${rn} >> ${all_ccfs_table}
267			fi
268
269	ccflib	1.7
270			echo "done"
271	ccflib	1.2	done < ${ccf_classes}
272
273	ccflib	1.7	echo "\|================================\|=============\|=======\|" >> ${all_ccfs_table}
274			echo >> ${all_ccfs_table}
275			echo "(*): Incomplete range." >> ${all_ccfs_table}
276			echo "" >> ${all_ccfs_table}
277			echo "Total number of valid CCFs required : ${total_number_of_ccfs}" >> ${all_ccfs_table}
278	ccflib	1.3	rm ${VALID_CCF}/ccf_classes.txt
279			rm ${VALID_CCF}/ccf_issues_for*.txt