Annotation of /bin/mlovccf

#! /bin/bash

# mlovccf (make list of valid ccfs)
#
# Makes the list of valid CCFs. These are all CCFs, among those published since the 
# beginning of the project, that are required to process any ODF at a given date,
# typically the current date. This list will change with time as far as new
# CCFs are issued to deal with new calibration data and/or replace obsolete ones. 
#
# To identify the set of CCFs required to process a given ODF is the job of task cifbuild. 
# To make such identification, cifbuild needs only the start time of the observation. 
# Therefore, if we provide a list of all observed ODFs at a given date together with their
# respective start time, we could compute all the CCFs required to process them.
# For such a moment, that list is the list of valid CCFs required to process 
# any observed ODF. 
#
# We need as input the list of all observed ODFs at a given date.
# This is obtained from the report of all observations that the XSA makes to CDS,
# xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt). 
#
# $Id: mlovccf,v 1.15 2016/04/13 10:26:13 ccflib Exp $

# Next two lines allow to submit this script to the grid
# request Bourne shell as shell for job
#$ -S /bin/bash

adminmail="eduardo.ojero@sciops.esa.int"

VALID_CCF="$HOME/valid_ccf"

mailfile="${VALID_CCF}/mailfile"

host=`hostname | cut -d. -f1`

now=`date +'%Y%m%d_%H%M'`
now_table=`date +'%Y-%m-%dT%H:%M:%S'`


# +++ Function mailnotify
#
# This function is used to notify adminmail on any issue while runnning this procedure
#
# Needs a subject as $1 and body as local file to be send.

mailnotify ()
{

[ "$#" -ne "2" ] && return

subject=$1 

bodyfile=$2

[ ! -f "${bodyfile}" ] && return 

echo ; echo "Sending message to ${adminmail} with subject ${subject}" ; echo

mailx -v -s "${subject}" "${adminmail}" < ${bodyfile}

}


# +++ Function getxsaobslog
#
# We need to get the list of all observed ODFs at the current date. 
# This is obtained from http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt.

getxsaobslog ()
{

XSAOBSURL="http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt"

# Remove any xsaobslog.txt previously downloaded. 

[ -f "${VALID_CCF}/xsaobslog.txt" ] && rm -rf ${VALID_CCF}/xsaobslog.txt

# Get the list latest list of ODFs available from nXSA server.
 
wget -q ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt 

# Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference 
# of which list of ODFs was used to compute the list of valid CCFs.

mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt

}


# +++ Function find_latest_mif to get the latest XMM_CALINDEX from given directory

find_latest_mif()
{

[ -z "$1" ] && return

CCFPATH="$1"

maxissue="0"
for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF`
do 
        mifbase=`basename $mif`
        issue=`echo $mifbase | awk -F"." '{print $1}' | awk -F"_" '{print $3}'`
        [ "$issue" -ge "$maxissue" ] && maxissue="$issue"
done

MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF

}


# +++Function get_ccf_list to list on stdout the table of CCFs in a given CIF.
#
# Requires that Heasoft is initialised. Otherwise exits with error.

get_ccf_list()
{
[ -z "$1" ] && return

cif_to_process="$1"

noversion=`which fversion | grep -c no`

if [ "${noversion}" != "0" ] ; then 
        echo "Error: Heasoft not initialised !"
        [ -f "${mailfile}" ] && rm -rf ${mailfile}
        echo "Running of fversion produced the following output:" >> $mailfile
        echo "$noversion" >> $mailfile
        mailnotify "mvloccf: Error: Heasoft not initialised!" $mailfile
        rm ${mailfile}
        exit
fi

fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
                        columns="SCOPE TYPEID ISSUE"            \
                        rows=-  STDOUT  
}


# +++ Fill in the valid_constituents directory with the Valid CCF set
#
# The directory to be filled in is ${VALIDCONSTITUENTSDISR}. 
# This directory is synchronized with the similar directory in the 
# external ftp server.
#
# In addition to that, we also use this function to fill in the 
# /ccf/valid directory available for users in the intranet.

fillvalidccfdir()
{

[ -z "$1" ] && return

validccflist=$1

if [ ! -f "${VALID_CCF}/${validccflist}"  ] ; then
        echo "Error: ${VALID_CCF}/${validccflist} not found - Abort!" 
        [ -f "${mailfile}" ] && rm -rf ${mailfile}
        echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found" >> $mailfile
        mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found - Abort!" $mailfile
        rm ${mailfile}
        exit
fi

VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents"
rm -rf ${VALIDCONSTITUENTSDIR}/*

VALIDCCFDIRINTRANET="/ccf/valid"
rm -rf ${VALIDCCFDIRINTRANET}/*

CONSTITUENTSDIR="/home/ccflib/ftp-area/constituents"
n=0
while read ccf
do
        n=$((n + 1))
        echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/"
        cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/
        echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/"
        cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/
done < ${VALID_CCF}/${validccflist}

}


#
#   M a i n    p r o g r a m 
#
# 1. Get the list of all ODFs


xsaobslogsize="0" 

getxsaobslog 

xsaobslogsize=`stat --format=%s ${VALID_CCF}/xsaobslog_${now}.txt` 

if [ "$xsaobslogsize"  = "0" ] ; then
        echo "Error: Failure to download the XSA Obs. Log file - Abort" 
        [ -f "${mailfile}" ] && rm -rf ${mailfile}
        echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile
        echo "." >> $mailfile
        mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile"
        rm ${mailfile}
        exit
fi 

# 2. Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host 

case "$host" in
        xvsoc01|xmm)
                export SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"
                export SAS_DIR=/data/xmm/ccflib/sas
                export SAS_PATH=$SAS_DIR
                source $SAS_DIR/sas-setup.sh
                ;;
        sasbld01|sasbld02)
                export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
                /sas/bin/confsas
                export HEADAS=/sasbuild/local/${host}/headas/architecture
                . $HEADAS/headas-init.sh
                # To be able to mix several jobs on the same ccflib account
                [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
                export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
                echo -n "HEADAS version: " 
                fversion
                source $HOME/setsas.sh 1> /dev/null
                echo "SAS Version: "
                sasversion -V 1
                export SAS_VERBOSITY=0
                ;;
        scigrid6|cn-*)
                export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
                /sas/bin/confsas
                export HEADAS=/sasbuild/local/sasbld02/headas/architecture
                . $HEADAS/headas-init.sh
                # To be able to mix several jobs on the same ccflib account
                [ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
                export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
                echo -n "HEADAS version: " 
                fversion
                source $HOME/setsas.sh 1> /dev/null
                echo "SAS Version: "
                sasversion -V 1
                export SAS_VERBOSITY=0
                ;;
        *) echo "Error: Do not know how to do it in host $host" ; exit ;;
esac

# 3. Finds the latest MIF issue

find_latest_mif "${SAS_CCFPATH}"


# 4. Scans the list of OBDS ID registered in the previous list to get the start time
#    Then runs cifbuild to get the respective CIF.
#    all_ccfs_${now}.txt is filled recursively on each iteration. Therefore we start
#    with an empty file.

#    Output file :          ${VALID_CCF}/all_ccfs_${now}.txt

touch ${VALID_CCF}/all_ccfs_${now}.txt

n=0

while read line 
do
        obsid=`echo $line | awk -F"|" '{print $2}'` 
        stime=`echo $line | awk -F"|" '{print $9}'`
        stime=`echo $stime | tr " " "T"`
        n=$((n+1))
        echo "($n) Processing $obsid $stime" ; echo

        # Run cifbuild

        cif_file=${VALID_CCF}/${obsid}.cif      

        cifbuild --withmasterindexset='yes'  \
                 --masterindexset=${MIF}     \
                 --withobservationdate='yes' \
                 --observationdate=${stime}  \
                 --analysisdate='now'        \
                 --calindexset=${cif_file} 


        # Gets the list of CCFs included in the CIF

        get_ccf_list "${cif_file}" > ${VALID_CCF}/${obsid}_ccfs.tmp

        while read line 
        do
                [ "$line" = "" ] && continue
                class=`echo $line | awk -F" " '{print $1"_"$2}'`
                issue=`echo $line | awk -F" " '{print $3}'` 
                issue=`printf "%04d" ${issue}`
                echo ${class}_${issue}.CCF >> ${VALID_CCF}/${obsid}_ccfs.txt

        done < ${VALID_CCF}/${obsid}_ccfs.tmp

        rm ${VALID_CCF}/${obsid}_ccfs.tmp

        mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp

        cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u  >  ${VALID_CCF}/all_ccfs_${now}.txt

        rm ${cif_file}
        rm ${VALID_CCF}/${obsid}_ccfs.txt
        rm ${VALID_CCF}/all_ccfs_${now}.tmp

done < ${VALID_CCF}/xsaobslog_${now}.txt

# 5. Make a table of classes and issues with the style "Pipeline Release Notes"
#    To get a classification, we need first to obtain all available classes.
#


# Lists all CCF classes

ccf_classes="${VALID_CCF}/ccf_classes.txt"

while read line
do 
        class=`echo $line | awk -F"_" '{print $1"_"$2}'`
        echo $class >> ${ccf_classes}
done < ${VALID_CCF}/all_ccfs_${now}.txt

cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt
mv ${VALID_CCF}/kk.txt ${ccf_classes}


# Output final file is named all_ccfs_${now}_table.txt

total_number_of_ccfs=0

all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt"

[ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}

echo "Table of valid CCFs at $now_table"                        >> ${all_ccfs_table}
echo                                                            >> ${all_ccfs_table}
echo "|================================|=============|=======|" >> ${all_ccfs_table} 
echo "| Calibration File               | Issue range | Count |" >> ${all_ccfs_table}
echo "|================================|=============|=======|" >> ${all_ccfs_table}

while read ccf_class
do
        echo -n "Processing class $ccf_class ..." 
        issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
        [ -f "${issues_file}" ] && rm ${issues_file}
        while read line
        do
                class=`echo $line | awk -F"_" '{print $1"_"$2}'`
                [ "$class" != "$ccf_class" ] && continue
                issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1`
                echo $issue >> ${issues_file}
        done < ${VALID_CCF}/all_ccfs_${now}.txt

        cat ${issues_file} | sort -u > kk.txt
        mv kk.txt ${issues_file}
        real_number_of_issues_for_this_class=`wc -l ${issues_file} | awk -F" " '{print $1}'`

        first_issue=`head -1 ${issues_file}`
        last_issue=`tail -1 ${issues_file}`

        li=$((10#$last_issue))
        fi=$((10#$first_issue))

        sequential_number_of_issues_for_this_class=$((li - fi + 1))

        asterisk="   "
        [ "${real_number_of_issues_for_this_class}"  -lt "${sequential_number_of_issues_for_this_class}" ] &&  asterisk="(*)"

        total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs))

        rn=$real_number_of_issues_for_this_class

        if [ "${first_issue}" != "${last_issue}" ] ; then
           if [ "$asterisk" = "(*)" ] ; then
              printf "| %-30s | %4s-%4s(*)|  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
           else
              printf "| %-30s | %4s-%4s   |  %4s |\n"  ${ccf_class} ${first_issue} ${last_issue} ${rn}          >> ${all_ccfs_table}
           fi
        else
           printf "| %-30s | %4s        |  %4s |\n" ${ccf_class} ${first_issue} ${rn}                           >> ${all_ccfs_table}
        fi

        
        echo "done" 
done < ${ccf_classes}  

echo "|================================|=============|=======|"                                                 >> ${all_ccfs_table}
echo                                                                                                            >> ${all_ccfs_table}
echo "(*): Incomplete range."                                                                                   >> ${all_ccfs_table} 
echo ""                                                                                                         >> ${all_ccfs_table}
echo "Total number of valid CCFs required : ${total_number_of_ccfs}"                                            >> ${all_ccfs_table}
rm ${VALID_CCF}/ccf_classes.txt
rm ${VALID_CCF}/ccf_issues_for*.txt


# 6. Fill in the Valid CCF Set dir with the proper CCFs

fillvalidccfdir "all_ccfs_${now}.txt"

[ -f "${mailfile}" ] && rm -rf ${mailfile}
echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile
echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}"  >> $mailfile 
mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile
rm ${mailfile}
1	ccflib	1.1	#! /bin/bash
2
3			# mlovccf (make list of valid ccfs)
4			#
5	ccflib	1.7	# Makes the list of valid CCFs. These are all CCFs, among those published since the
6	ccflib	1.2	# beginning of the project, that are required to process any ODF at a given date,
7	ccflib	1.7	# typically the current date. This list will change with time as far as new
8	ccflib	1.2	# CCFs are issued to deal with new calibration data and/or replace obsolete ones.
9	ccflib	1.1	#
10	ccflib	1.10	# To identify the set of CCFs required to process a given ODF is the job of task cifbuild.
11	ccflib	1.2	# To make such identification, cifbuild needs only the start time of the observation.
12	ccflib	1.7	# Therefore, if we provide a list of all observed ODFs at a given date together with their
13			# respective start time, we could compute all the CCFs required to process them.
14			# For such a moment, that list is the list of valid CCFs required to process
15			# any observed ODF.
16	ccflib	1.1	#
17	ccflib	1.7	# We need as input the list of all observed ODFs at a given date.
18			# This is obtained from the report of all observations that the XSA makes to CDS,
19	ccflib	1.10	# xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt).
20	ccflib	1.2	#
21	ccflib	1.16	# $Id: mlovccf,v 1.15 2016/04/13 10:26:13 ccflib Exp $
22	ccflib	1.4
23			# Next two lines allow to submit this script to the grid
24			# request Bourne shell as shell for job
25	ccflib	1.6	#$ -S /bin/bash
26	ccflib	1.1
27	ccflib	1.16	adminmail="eduardo.ojero@sciops.esa.int"
28	ccflib	1.5
29	ccflib	1.10	VALID_CCF="$HOME/valid_ccf"
30
31	ccflib	1.13	mailfile="${VALID_CCF}/mailfile"
32
33	ccflib	1.1	host=`hostname \| cut -d. -f1`
34
35	ccflib	1.5	now=`date +'%Y%m%d_%H%M'`
36	ccflib	1.7	now_table=`date +'%Y-%m-%dT%H:%M:%S'`
37	ccflib	1.3
38	ccflib	1.10
39	ccflib	1.13	# +++ Function mailnotify
40			#
41			# This function is used to notify adminmail on any issue while runnning this procedure
42			#
43			# Needs a subject as $1 and body as local file to be send.
44
45			mailnotify ()
46			{
47
48			[ "$#" -ne "2" ] && return
49
50			subject=$1
51
52			bodyfile=$2
53
54			[ ! -f "${bodyfile}" ] && return
55
56	ccflib	1.15	echo ; echo "Sending message to ${adminmail} with subject ${subject}" ; echo
57
58	ccflib	1.16	mailx -v -s "${subject}" "${adminmail}" < ${bodyfile}
59	ccflib	1.13
60			}
61
62
63
64			# +++ Function getxsaobslog
65			#
66	ccflib	1.10	# We need to get the list of all observed ODFs at the current date.
67			# This is obtained from http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt.
68
69			getxsaobslog ()
70			{
71
72			XSAOBSURL="http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt"
73
74			# Remove any xsaobslog.txt previously downloaded.
75
76			[ -f "${VALID_CCF}/xsaobslog.txt" ] && rm -rf ${VALID_CCF}/xsaobslog.txt
77
78			# Get the list latest list of ODFs available from nXSA server.
79
80	ccflib	1.11	wget -q ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt
81	ccflib	1.10
82			# Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference
83			# of which list of ODFs was used to compute the list of valid CCFs.
84
85			mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt
86
87			}
88
89
90	ccflib	1.13
91			# +++ Function find_latest_mif to get the latest XMM_CALINDEX from given directory
92	ccflib	1.1
93			find_latest_mif()
94			{
95
96			[ -z "$1" ] && return
97
98			CCFPATH="$1"
99
100			maxissue="0"
101			for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF`
102			do
103			mifbase=`basename $mif`
104			issue=`echo $mifbase \| awk -F"." '{print $1}' \| awk -F"_" '{print $3}'`
105			[ "$issue" -ge "$maxissue" ] && maxissue="$issue"
106			done
107
108			MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF
109
110			}
111
112	ccflib	1.13
113
114			# +++Function get_ccf_list to list on stdout the table of CCFs in a given CIF.
115			#
116	ccflib	1.2	# Requires that Heasoft is initialised. Otherwise exits with error.
117	ccflib	1.1
118			get_ccf_list()
119			{
120			[ -z "$1" ] && return
121
122			cif_to_process="$1"
123
124			noversion=`which fversion \| grep -c no`
125
126			if [ "${noversion}" != "0" ] ; then
127			echo "Error: Heasoft not initialised !"
128	ccflib	1.13	[ -f "${mailfile}" ] && rm -rf ${mailfile}
129			echo "Running of fversion produced the following output:" >> $mailfile
130			echo "$noversion" >> $mailfile
131			mailnotify "mvloccf: Error: Heasoft not initialised!" $mailfile
132			rm ${mailfile}
133	ccflib	1.1	exit
134			fi
135
136			fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \
137			columns="SCOPE TYPEID ISSUE" \
138			rows=- STDOUT
139			}
140
141	ccflib	1.13
142
143
144			# +++ Fill in the valid_constituents directory with the Valid CCF set
145			#
146			# The directory to be filled in is ${VALIDCONSTITUENTSDISR}.
147			# This directory is synchronized with the similar directory in the
148			# external ftp server.
149			#
150			# In addition to that, we also use this function to fill in the
151			# /ccf/valid directory available for users in the intranet.
152	ccflib	1.11
153			fillvalidccfdir()
154			{
155
156			[ -z "$1" ] && return
157
158			validccflist=$1
159
160			if [ ! -f "${VALID_CCF}/${validccflist}" ] ; then
161	ccflib	1.12	echo "Error: ${VALID_CCF}/${validccflist} not found - Abort!"
162	ccflib	1.13	[ -f "${mailfile}" ] && rm -rf ${mailfile}
163			echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found" >> $mailfile
164			mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found - Abort!" $mailfile
165			rm ${mailfile}
166	ccflib	1.11	exit
167			fi
168
169	ccflib	1.13	VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents"
170	ccflib	1.11	rm -rf ${VALIDCONSTITUENTSDIR}/*
171
172	ccflib	1.13	VALIDCCFDIRINTRANET="/ccf/valid"
173			rm -rf ${VALIDCCFDIRINTRANET}/*
174
175	ccflib	1.11	CONSTITUENTSDIR="/home/ccflib/ftp-area/constituents"
176	ccflib	1.12	n=0
177	ccflib	1.11	while read ccf
178			do
179	ccflib	1.12	n=$((n + 1))
180			echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/"
181	ccflib	1.11	cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/
182	ccflib	1.13	echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/"
183			cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/
184	ccflib	1.12	done < ${VALID_CCF}/${validccflist}
185	ccflib	1.11
186			}
187
188
189	ccflib	1.13	#
190			# M a i n p r o g r a m
191			#
192			# 1. Get the list of all ODFs
193
194	ccflib	1.1
195	ccflib	1.10	xsaobslogsize="0"
196	ccflib	1.1
197	ccflib	1.10	getxsaobslog
198	ccflib	1.1
199	ccflib	1.10	xsaobslogsize=`stat --format=%s ${VALID_CCF}/xsaobslog_${now}.txt`
200	ccflib	1.1
201	ccflib	1.10	if [ "$xsaobslogsize" = "0" ] ; then
202	ccflib	1.13	echo "Error: Failure to download the XSA Obs. Log file - Abort"
203			[ -f "${mailfile}" ] && rm -rf ${mailfile}
204			echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile
205			echo "." >> $mailfile
206			mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile"
207			rm ${mailfile}
208	ccflib	1.10	exit
209			fi
210	ccflib	1.1
211	ccflib	1.13	# 2. Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host
212	ccflib	1.1
213			case "$host" in
214			xvsoc01\|xmm)
215	ccflib	1.5	export SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents"
216			export SAS_DIR=/data/xmm/ccflib/sas
217			export SAS_PATH=$SAS_DIR
218	ccflib	1.1	source $SAS_DIR/sas-setup.sh
219			;;
220			sasbld01\|sasbld02)
221	ccflib	1.5	export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
222	ccflib	1.1	/sas/bin/confsas
223			export HEADAS=/sasbuild/local/${host}/headas/architecture
224			. $HEADAS/headas-init.sh
225	ccflib	1.7	# To be able to mix several jobs on the same ccflib account
226			[ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
227			export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
228			echo -n "HEADAS version: "
229			fversion
230	ccflib	1.1	source $HOME/setsas.sh 1> /dev/null
231	ccflib	1.7	echo "SAS Version: "
232			sasversion -V 1
233	ccflib	1.1	export SAS_VERBOSITY=0
234			;;
235	ccflib	1.5	scigrid6\|cn-*)
236			export SAS_CCFPATH="/home/ccflib/ftp-area/constituents"
237			/sas/bin/confsas
238			export HEADAS=/sasbuild/local/sasbld02/headas/architecture
239			. $HEADAS/headas-init.sh
240	ccflib	1.7	# To be able to mix several jobs on the same ccflib account
241			[ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host}
242			export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles"
243			echo -n "HEADAS version: "
244			fversion
245	ccflib	1.5	source $HOME/setsas.sh 1> /dev/null
246	ccflib	1.7	echo "SAS Version: "
247			sasversion -V 1
248	ccflib	1.5	export SAS_VERBOSITY=0
249			;;
250			*) echo "Error: Do not know how to do it in host $host" ; exit ;;
251	ccflib	1.1	esac
252
253	ccflib	1.13	# 3. Finds the latest MIF issue
254	ccflib	1.1
255			find_latest_mif "${SAS_CCFPATH}"
256
257
258	ccflib	1.13	# 4. Scans the list of OBDS ID registered in the previous list to get the start time
259			# Then runs cifbuild to get the respective CIF.
260			# all_ccfs_${now}.txt is filled recursively on each iteration. Therefore we start
261			# with an empty file.
262
263			# Output file : ${VALID_CCF}/all_ccfs_${now}.txt
264	ccflib	1.1
265	ccflib	1.3	touch ${VALID_CCF}/all_ccfs_${now}.txt
266	ccflib	1.2
267	ccflib	1.1	n=0
268	ccflib	1.2
269	ccflib	1.1	while read line
270			do
271			obsid=`echo $line \| awk -F"\|" '{print $2}'`
272			stime=`echo $line \| awk -F"\|" '{print $9}'`
273			stime=`echo $stime \| tr " " "T"`
274			n=$((n+1))
275			echo "($n) Processing $obsid $stime" ; echo
276
277			# Run cifbuild
278
279			cif_file=${VALID_CCF}/${obsid}.cif
280
281			cifbuild --withmasterindexset='yes' \
282			--masterindexset=${MIF} \
283			--withobservationdate='yes' \
284			--observationdate=${stime} \
285			--analysisdate='now' \
286			--calindexset=${cif_file}
287
288
289			# Gets the list of CCFs included in the CIF
290
291			get_ccf_list "${cif_file}" > ${VALID_CCF}/${obsid}_ccfs.tmp
292
293			while read line
294			do
295			[ "$line" = "" ] && continue
296			class=`echo $line \| awk -F" " '{print $1"_"$2}'`
297			issue=`echo $line \| awk -F" " '{print $3}'`
298			issue=`printf "%04d" ${issue}`
299			echo ${class}_${issue}.CCF >> ${VALID_CCF}/${obsid}_ccfs.txt
300	ccflib	1.2
301	ccflib	1.1	done < ${VALID_CCF}/${obsid}_ccfs.tmp
302
303			rm ${VALID_CCF}/${obsid}_ccfs.tmp
304
305	ccflib	1.3	mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp
306	ccflib	1.2
307	ccflib	1.3	cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp \| sort -u > ${VALID_CCF}/all_ccfs_${now}.txt
308	ccflib	1.1
309			rm ${cif_file}
310			rm ${VALID_CCF}/${obsid}_ccfs.txt
311	ccflib	1.3	rm ${VALID_CCF}/all_ccfs_${now}.tmp
312	ccflib	1.1
313	ccflib	1.7	done < ${VALID_CCF}/xsaobslog_${now}.txt
314	ccflib	1.2
315	ccflib	1.13	# 5. Make a table of classes and issues with the style "Pipeline Release Notes"
316			# To get a classification, we need first to obtain all available classes.
317			#
318
319	ccflib	1.2
320			# Lists all CCF classes
321
322			ccf_classes="${VALID_CCF}/ccf_classes.txt"
323
324			while read line
325			do
326			class=`echo $line \| awk -F"_" '{print $1"_"$2}'`
327			echo $class >> ${ccf_classes}
328	ccflib	1.3	done < ${VALID_CCF}/all_ccfs_${now}.txt
329	ccflib	1.2
330			cat ${ccf_classes} \| sort -u > ${VALID_CCF}/kk.txt
331			mv ${VALID_CCF}/kk.txt ${ccf_classes}
332
333	ccflib	1.13
334
335	ccflib	1.3	# Output final file is named all_ccfs_${now}_table.txt
336
337			total_number_of_ccfs=0
338	ccflib	1.2
339	ccflib	1.7	all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt"
340	ccflib	1.2
341			[ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table}
342
343	ccflib	1.7	echo "Table of valid CCFs at $now_table" >> ${all_ccfs_table}
344			echo >> ${all_ccfs_table}
345			echo "\|================================\|=============\|=======\|" >> ${all_ccfs_table}
346			echo "\| Calibration File \| Issue range \| Count \|" >> ${all_ccfs_table}
347			echo "\|================================\|=============\|=======\|" >> ${all_ccfs_table}
348	ccflib	1.2
349			while read ccf_class
350	ccflib	1.3	do
351			echo -n "Processing class $ccf_class ..."
352	ccflib	1.2	issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt"
353			[ -f "${issues_file}" ] && rm ${issues_file}
354			while read line
355			do
356			class=`echo $line \| awk -F"_" '{print $1"_"$2}'`
357			[ "$class" != "$ccf_class" ] && continue
358			issue=`echo $line \| awk -F"_" '{print $3}' \| cut -d. -f1`
359			echo $issue >> ${issues_file}
360	ccflib	1.3	done < ${VALID_CCF}/all_ccfs_${now}.txt
361	ccflib	1.2
362			cat ${issues_file} \| sort -u > kk.txt
363			mv kk.txt ${issues_file}
364	ccflib	1.3	real_number_of_issues_for_this_class=`wc -l ${issues_file} \| awk -F" " '{print $1}'`
365	ccflib	1.2
366			first_issue=`head -1 ${issues_file}`
367			last_issue=`tail -1 ${issues_file}`
368
369	ccflib	1.3	li=$((10#$last_issue))
370			fi=$((10#$first_issue))
371
372	ccflib	1.7	sequential_number_of_issues_for_this_class=$((li - fi + 1))
373	ccflib	1.8
374			asterisk=" "
375			[ "${real_number_of_issues_for_this_class}" -lt "${sequential_number_of_issues_for_this_class}" ] && asterisk="(*)"
376
377	ccflib	1.7	total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs))
378
379			rn=$real_number_of_issues_for_this_class
380	ccflib	1.3
381	ccflib	1.8	if [ "${first_issue}" != "${last_issue}" ] ; then
382			if [ "$asterisk" = "(*)" ] ; then
383			printf "\| %-30s \| %4s-%4s(*)\| %4s \|\n" ${ccf_class} ${first_issue} ${last_issue} ${rn} >> ${all_ccfs_table}
384			else
385			printf "\| %-30s \| %4s-%4s \| %4s \|\n" ${ccf_class} ${first_issue} ${last_issue} ${rn} >> ${all_ccfs_table}
386			fi
387			else
388			printf "\| %-30s \| %4s \| %4s \|\n" ${ccf_class} ${first_issue} ${rn} >> ${all_ccfs_table}
389			fi
390
391	ccflib	1.7
392			echo "done"
393	ccflib	1.2	done < ${ccf_classes}
394
395	ccflib	1.7	echo "\|================================\|=============\|=======\|" >> ${all_ccfs_table}
396			echo >> ${all_ccfs_table}
397			echo "(*): Incomplete range." >> ${all_ccfs_table}
398			echo "" >> ${all_ccfs_table}
399			echo "Total number of valid CCFs required : ${total_number_of_ccfs}" >> ${all_ccfs_table}
400	ccflib	1.3	rm ${VALID_CCF}/ccf_classes.txt
401			rm ${VALID_CCF}/ccf_issues_for*.txt
402	ccflib	1.11
403	ccflib	1.13
404
405			# 6. Fill in the Valid CCF Set dir with the proper CCFs
406	ccflib	1.11
407	ccflib	1.12	fillvalidccfdir "all_ccfs_${now}.txt"
408	ccflib	1.13
409			[ -f "${mailfile}" ] && rm -rf ${mailfile}
410			echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile
411			echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}" >> $mailfile
412			mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile
413			rm ${mailfile}