2 |
|
|
3 |
# mlovccf (make list of valid ccfs) |
# mlovccf (make list of valid ccfs) |
4 |
# |
# |
5 |
# Makes the list of valid CCF. This is the list of the CCFs required to process any ODF |
# Makes the list of valid CCFs. These are all CCFs, among those published since the |
6 |
# at the current date. |
# beginning of the project, that are required to process any ODF at a given date, |
7 |
|
# typically the current date. This list will change with time as far as new |
8 |
|
# CCFs are issued to deal with new calibration data and/or replace obsolete ones. |
9 |
# |
# |
10 |
# To get the set of CCFs required to process a given ODF is the task of cifbuild. |
# To identify the set of CCFs required to process a given ODF is the job of task cifbuild. |
11 |
# To get such list, cifbuild needs only the ODF observation time, typically the start |
# To make such identification, cifbuild needs only the start time of the observation. |
12 |
# of it. If we do this exercise for all observed ODFs at a given date, we will obtain |
# Therefore, if we provide a list of all observed ODFs at a given date together with their |
13 |
# for such a moment, the list of valid CCFs required to process any observed ODF. |
# respective start time, we could compute all the CCFs required to process them. |
14 |
|
# For such a moment, that list is the list of valid CCFs required to process |
15 |
|
# any observed ODF. |
16 |
# |
# |
17 |
# $Id$ |
# We need as input the list of all observed ODFs at a given date. |
18 |
|
# This is obtained from the report of all observations that the XSA makes to CDS, |
19 |
|
# xsaobslog.txt (http://nxsa.esac.esa.int/ftp_public/cfs_obslog/xsaobslog.txt). |
20 |
|
# |
21 |
|
# $Id: mlovccf,v 1.25 2021/11/08 13:23:09 ccflib Exp $ |
22 |
|
|
23 |
|
# Next two lines allow to submit this script to the grid |
24 |
|
# request Bourne shell as shell for job |
25 |
|
#$ -S /bin/bash |
26 |
|
|
27 |
|
export DEBUG="T" |
28 |
|
|
29 |
|
adminmail="eduardo.ojero@sciops.esa.int" |
30 |
|
|
31 |
|
VALID_CCF="$HOME/processing_valid_ccf" |
32 |
|
|
33 |
|
mailfile="${VALID_CCF}/mailfile" |
34 |
|
|
35 |
host=`hostname | cut -d. -f1` |
host=`hostname | cut -d. -f1` |
36 |
|
|
37 |
# Function find_latest_mif to get the latest XMM_CALINDEX from given directory |
now=`date +'%Y%m%d_%H%M'` |
38 |
|
now_table=`date +'%Y-%m-%dT%H:%M:%S'` |
39 |
|
|
40 |
|
|
41 |
|
# +++ Function mailnotify |
42 |
|
# |
43 |
|
# This function is used to notify adminmail on any issue while runnning this procedure |
44 |
|
# |
45 |
|
# Needs a subject as $1 and body as local file to be send. |
46 |
|
|
47 |
find_latest_mif() |
mailnotify () |
48 |
{ |
{ |
49 |
|
|
50 |
[ -z "$1" ] && return |
[ "$#" -ne "2" ] && return |
51 |
|
|
52 |
CCFPATH="$1" |
subject=$1 |
53 |
|
|
54 |
maxissue="0" |
bodyfile=$2 |
55 |
for mif in `ls -1 ${CCFPATH}/XMM_CALINDEX_*.CCF` |
|
56 |
do |
[ ! -f "${bodyfile}" ] && return |
57 |
mifbase=`basename $mif` |
|
58 |
issue=`echo $mifbase | awk -F"." '{print $1}' | awk -F"_" '{print $3}'` |
echo ; echo "Sending message to ${adminmail} with subject ${subject}" ; echo |
|
[ "$issue" -ge "$maxissue" ] && maxissue="$issue" |
|
|
done |
|
59 |
|
|
60 |
MIF=${CCFPATH}/XMM_CALINDEX_${maxissue}.CCF |
if [ "$DEBUG" ] ; then |
61 |
|
|
62 |
|
echo "DEBUG: mlovccf: mailnotify: Going to send mail as " |
63 |
|
echo "mailx -v -s ${subject} ${adminmail} < ${bodyfile}" |
64 |
|
fi |
65 |
|
|
66 |
|
mailx -v -s "${subject}" "${adminmail}" < ${bodyfile} |
67 |
|
|
68 |
|
if [ "$DEBUG" ] ; then |
69 |
|
echo "DEBUG: mlovccf: mailnotify: Mail sent!" |
70 |
|
fi |
71 |
} |
} |
72 |
|
|
73 |
# Function get_ccf_list to list on stdout the table of CCFS in a given CIF |
|
74 |
|
|
75 |
|
# +++ Function getxsaobslog |
76 |
# |
# |
77 |
# requires Heasoft be initialized. Otherwise exits with error. |
# We need to get the list of all observed ODFs at the current date. |
78 |
|
# This is obtained from http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt. |
79 |
|
|
80 |
|
getxsaobslog () |
81 |
|
{ |
82 |
|
|
83 |
|
XSAOBSURL="http://nxsa.esac.esa.int/ftp_public/cds_obslog/xsaobslog.txt" |
84 |
|
|
85 |
|
# Remove any xsaobslog.txt previously downloaded. |
86 |
|
|
87 |
|
[ -f "${VALID_CCF}/xsaobslog.txt" ] && rm -rf ${VALID_CCF}/xsaobslog.txt |
88 |
|
|
89 |
|
# Get the list latest list of ODFs available from nXSA server. |
90 |
|
|
91 |
|
wget --quiet ${XSAOBSURL} -O ${VALID_CCF}/xsaobslog.txt |
92 |
|
|
93 |
|
# Rename xsaobslog.txt to xsaobslog_${now}.txt just to have a reference |
94 |
|
# of which list of ODFs was used to compute the list of valid CCFs. |
95 |
|
|
96 |
|
mv ${VALID_CCF}/xsaobslog.txt ${VALID_CCF}/xsaobslog_${now}.txt |
97 |
|
|
98 |
|
if [ "$DEBUG" ] ; then |
99 |
|
echo "DEBUG: mlovccf: getxsaobslog: XSA Obs. log file obtained successfully and renamed to ${VALID_CCF}/xsaobslog_${now}.txt" |
100 |
|
fi |
101 |
|
|
102 |
|
} |
103 |
|
|
104 |
|
|
105 |
|
|
106 |
|
# +++Function get_ccf_list to list on stdout the table of CCFs in a given CIF. |
107 |
|
# |
108 |
|
# Requires that Heasoft is initialised. Otherwise exits with error. |
109 |
|
|
110 |
get_ccf_list() |
get_ccf_list() |
111 |
{ |
{ |
117 |
|
|
118 |
if [ "${noversion}" != "0" ] ; then |
if [ "${noversion}" != "0" ] ; then |
119 |
echo "Error: Heasoft not initialised !" |
echo "Error: Heasoft not initialised !" |
120 |
|
[ -f "${mailfile}" ] && rm -rf ${mailfile} |
121 |
|
echo "Running of fversion produced the following output:" >> $mailfile |
122 |
|
echo "$noversion" >> $mailfile |
123 |
|
mailnotify "mvloccf: Error: Heasoft not initialised!" $mailfile |
124 |
|
rm ${mailfile} |
125 |
exit |
exit |
126 |
fi |
fi |
127 |
|
|
128 |
fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \ |
fdump ${cif_to_process} prhead=no showcol=no showrow=no page=no \ |
129 |
columns="SCOPE TYPEID ISSUE" \ |
columns="SCOPE TYPEID ISSUE" \ |
130 |
rows=- STDOUT |
rows=- STDOUT |
131 |
|
|
132 |
|
} |
133 |
|
|
134 |
|
|
135 |
|
|
136 |
|
|
137 |
|
# +++ Fill in the valid_constituents directory with the Valid CCF set |
138 |
|
# |
139 |
|
# The directory to be filled in is ${VALIDCONSTITUENTSDISR}. |
140 |
|
# This directory is synchronized with the similar directory in the |
141 |
|
# external ftp server. |
142 |
|
# |
143 |
|
# In addition to that, we also use this function to fill in the |
144 |
|
# /ccf/valid directory available for users in the intranet. |
145 |
|
|
146 |
|
fillvalidccfdir() |
147 |
|
{ |
148 |
|
|
149 |
|
[ -z "$1" ] && return |
150 |
|
|
151 |
|
validccflist=$1 |
152 |
|
|
153 |
|
if [ ! -f "${VALID_CCF}/${validccflist}" -o ! -s "${VALID_CCF}/${validccflist}" ] ; then |
154 |
|
echo "Error: ${VALID_CCF}/${validccflist} not found or empty - Abort!" |
155 |
|
[ -f "${mailfile}" ] && rm -rf ${mailfile} |
156 |
|
echo "The list of valid CCF ${VALID_CCF}/${validccflist} is not found or it is empty" >> $mailfile |
157 |
|
mailnotify "mvloccf: Error: ${VALID_CCF}/${validccflist} not found or it is empty - Abort!" $mailfile |
158 |
|
rm ${mailfile} |
159 |
|
exit |
160 |
|
fi |
161 |
|
|
162 |
|
VALIDCONSTITUENTSDIR="/home/ccflib/ftp-area/valid_constituents" |
163 |
|
# Checks that the VALIDCONSTITUENTSDIR directory is writable by me |
164 |
|
touch ${VALIDCONSTITUENTSDIR}/test 2> /dev/null |
165 |
|
if [ "$?" != "0" ] ; then |
166 |
|
[ -f "${mailfile}" ] && rm -rf ${mailfile} |
167 |
|
echo "${VALIDCONSTITUENTSDIR} is not writable. Can not work" >> $mailfile |
168 |
|
mailnotify "mvloccf: Error: ${VALIDCONSTITUENTSDIR} is not writable - Abort!" $mailfile |
169 |
|
rm ${mailfile} |
170 |
|
exit |
171 |
|
fi |
172 |
|
rm -rf ${VALIDCONSTITUENTSDIR}/* |
173 |
|
|
174 |
|
# This operation can only succeed because the subdir valid has enable the write permission |
175 |
|
# for the group "sas" (g=rwx), provided that CCFs belong to sasbuild(5153):sas. |
176 |
|
VALIDCCFDIRINTRANET="/ccf/valid" |
177 |
|
# Checks that the VALIDCCFDIRINTRANET directory is writable by me |
178 |
|
touch ${VALIDCCFDIRINTRANET}/test 2> /dev/null |
179 |
|
if [ "$?" != "0" ] ; then |
180 |
|
[ -f "${mailfile}" ] && rm -rf ${mailfile} |
181 |
|
echo "${VALIDCCFDIRINTRANET} is not writable. Can not work" >> $mailfile |
182 |
|
mailnotify "mvloccf: Error: ${VALIDCCFDIRINTRANET} is not writable - Abort!" $mailfile |
183 |
|
rm ${mailfile} |
184 |
|
exit |
185 |
|
fi |
186 |
|
|
187 |
|
rm -rf ${VALIDCCFDIRINTRANET}/* |
188 |
|
|
189 |
|
CONSTITUENTSDIR="/home/ccflib/ftp-area/constituents" |
190 |
|
n=0 |
191 |
|
while read ccf |
192 |
|
do |
193 |
|
n=$((n + 1)) |
194 |
|
echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/" |
195 |
|
cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCONSTITUENTSDIR}/ |
196 |
|
echo "$n cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/" |
197 |
|
cp -a ${CONSTITUENTSDIR}/${ccf} ${VALIDCCFDIRINTRANET}/ |
198 |
|
done < ${VALID_CCF}/${validccflist} |
199 |
|
|
200 |
|
if [ "$DEBUG" ] ; then |
201 |
|
echo "DEBUG: mlovccf: fillvalidccfdir: Processed $validccflist" |
202 |
|
fi |
203 |
} |
} |
204 |
|
|
205 |
|
|
206 |
# We need to get the list of all observed ODFs at the current date. This is obtained from |
# |
207 |
# ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt. |
# M a i n p r o g r a m |
208 |
|
# |
209 |
|
# 1. Get the list of all ODFs |
210 |
|
|
211 |
|
|
212 |
|
xsaobslogsize="0" |
213 |
|
|
214 |
VALID_CCF="$HOME/valid_ccf" |
getxsaobslog |
215 |
|
|
216 |
cd ${VALID_CCF} |
xsaobslogsize=`stat --format=%s ${VALID_CCF}/xsaobslog_${now}.txt` |
217 |
wget -nc -q ftp://nxsa.esac.esa.int/pub/cds_obslog/xsaobslog.txt |
|
218 |
|
if [ "$xsaobslogsize" = "0" ] ; then |
219 |
|
echo "Error: Failure to download the XSA Obs. Log file - Abort" |
220 |
|
[ -f "${mailfile}" ] && rm -rf ${mailfile} |
221 |
|
echo "The XSA Obs. Log file has 0 size - Not downloaded?" >> $mailfile |
222 |
|
echo "." >> $mailfile |
223 |
|
if [ "$DEBUG" ] ; then |
224 |
|
echo "DEBUG: mlovccf: main: xsaobslogsize has 0 size. Mail notification being sent!" |
225 |
|
fi |
226 |
|
mailnotify "mlovccf: Error: Failure to download the XSA Obs. Log file - Abort" "$mailfile" |
227 |
|
rm ${mailfile} |
228 |
|
exit |
229 |
|
fi |
230 |
|
|
231 |
|
# |
232 |
|
# 2. Initialises HEADAS and SAS |
233 |
|
# |
234 |
|
|
235 |
# Sets SAS_CCFPATH and initialises HEADAS and SAS depending on host |
# Defines SASBUILD_LOCAL depending on host. |
236 |
|
# Then uses it to build up HEADAS and setup LD_LIBRARY_PATH accordingly. |
237 |
|
|
238 |
case "$host" in |
case "$host" in |
239 |
xvsoc01|xmm) |
sasbld01n|sasbld02n) |
240 |
SAS_CCFPATH="/data/xmm/ccflib/ftp-area/constituents" |
export SASBUILD_LOCAL="/sasbuild/local/${host}/GNU_CC_CXX_9.2.0" |
|
SAS_DIR=/data/xmm/ccflib/sas |
|
|
SAS_PATH=$SAS_DIR |
|
|
source $SAS_DIR/sas-setup.sh |
|
241 |
;; |
;; |
242 |
sasbld01|sasbld02) |
sciggw|scigrid-*) |
243 |
SAS_CCFPATH="/home/ccflib/ftp-area/constituents" |
export SASBUILD_LOCAL="/sasbuild/local/sasbld04n/GNU_CC_CXX_9.2.0" |
|
/sas/bin/confsas |
|
|
export HEADAS=/sasbuild/local/${host}/headas/architecture |
|
|
. $HEADAS/headas-init.sh |
|
|
source $HOME/setsas.sh 1> /dev/null |
|
|
export SAS_VERBOSITY=0 |
|
244 |
;; |
;; |
245 |
|
*) echo "Error: Unsupported host $host" ; exit 1 ;; |
246 |
esac |
esac |
247 |
|
|
248 |
# Finds the latest MIF issue |
export HEADAS=${SASBUILD_LOCAL}/headas/architecture |
249 |
|
|
250 |
|
headas_output="${VALID_CCF}/headas_output" |
251 |
|
source $HEADAS/headas-init.sh > ${headas_output} 2>&1 |
252 |
|
|
253 |
|
[ "$DEBUG" ] && echo "DEBUG: mlovccf: main: Executed Headas initialisation script for $HEADAS" |
254 |
|
|
255 |
|
# The next statement should not be necessary because the SAS initialisation |
256 |
|
# provides under $SAS_DIR/libextra all the necessary libraries from GNU C/C++ |
257 |
|
# which were required to build Heasoft. But, in order to make the HEADAS |
258 |
|
# initialisation independent of the SAS initialisation, we set the |
259 |
|
# LD_LIBRARY_PATH as it has to be to run Headas by itself without SAS been |
260 |
|
# ready. |
261 |
|
|
262 |
|
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${SASBUILD_LOCAL}/gcc/lib64 |
263 |
|
|
264 |
|
|
265 |
|
headas_ok=`cat ${headas_output}` |
266 |
|
|
267 |
|
if [ -n "${headas_ok}" ] ; then |
268 |
|
echo "mlovccf: HEADAS initialisation failed with error" >> ${mailfile} |
269 |
|
cat ${headas_output} >> ${mailfile} |
270 |
|
mailnotify "mlovccf: HEADAS initilisation failed" ${mailfile} |
271 |
|
if [ "$DEBUG" ] ; then |
272 |
|
echo "DEBUG: mlovccf: main: Headas initialization failed. Notification mail sent!" |
273 |
|
fi |
274 |
|
rm -v ${mailfile} |
275 |
|
rm -v ${headas_output} |
276 |
|
exit 1 |
277 |
|
fi |
278 |
|
|
279 |
|
# To be able to mix several jobs on the same ccflib account |
280 |
|
[ ! -d "${HOME}/pfiles/${host}" ] && mkdir ${HOME}/pfiles/${host} |
281 |
|
export PFILES="${HOME}/pfiles/${host};${HEADAS}/syspfiles" |
282 |
|
if [ "$DEBUG" ] ; then |
283 |
|
echo -n "DEBUG: mlovccf: main: HEADAS version: "`fversion` |
284 |
|
fi |
285 |
|
|
286 |
|
# |
287 |
|
# W a r n i n g o n S A S i n i t i a l i s a t i o n |
288 |
|
# |
289 |
|
# The SAS initialisation produced by setsas.sh sets SAS_CCFPATH to /ccf/pub. |
290 |
|
# Hence, if we set SAS_CCFPATH before running setsas.sh, we will always |
291 |
|
# use /ccf/pub as repository for CCF. This is right as long as /ccf/pub is in |
292 |
|
# strict sync with /home/ccflib/ccf_ftp/constituents. But if this is not true, |
293 |
|
# the resulting valid CCFs might be wrong. Therefore, we set it here explicitly. |
294 |
|
# |
295 |
|
|
296 |
find_latest_mif "${SAS_CCFPATH}" |
confsas_output="${VALID_CCF}/confsas_output" |
297 |
|
source /sas/bin/confsas > ${confsas_output} 2>&1 |
298 |
|
confsas_error=`cat ${confsas_output} | grep -c Error` |
299 |
|
|
300 |
|
if [ "$DEBUG" ] ; then |
301 |
|
echo "DEBUG: mlovccf: main: Executed /sas/bin/confsas with output: ${confsas_ok}" |
302 |
|
fi |
303 |
|
|
304 |
# Now scans the list of OBDS ID registered in the previous list to get the start time |
if [ "${confsas_error}" != "0" ] ; then |
305 |
# and then runs cifbuild to get the respective CIF |
echo "mlovccf: SAS initialization failed with error" >> ${mailfile} |
306 |
|
cat ${confsas_output} >> ${mailfile} |
307 |
|
mailnotify "mlovccf: SAS initialisation failed" ${mailfile} |
308 |
|
if [ "$DEBUG" ] ; then |
309 |
|
echo "DEBUG: mlovccf: main: SAS initialization failed. Notification mail sent!" |
310 |
|
fi |
311 |
|
exit 1 |
312 |
|
fi |
313 |
|
|
314 |
|
rm -v ${mailfile} |
315 |
|
rm -v ${confsas_output} |
316 |
|
|
317 |
|
source $HOME/setsas.sh 1> /dev/null |
318 |
|
|
319 |
|
if [ "$DEBUG" ] ; then |
320 |
|
echo "DEBUG: mlovccf: main: SAS setsas.sh script executed" |
321 |
|
echo |
322 |
|
echo "DEBUG: mlovccf: main: SAS Version: "`sasversion -V 1` |
323 |
|
fi |
324 |
|
|
325 |
|
|
326 |
|
# Set SAS vervbosity to 0 from now onwards |
327 |
|
|
328 |
|
export SAS_VERBOSITY=0 |
329 |
|
|
330 |
|
|
331 |
|
# Re-sets SAS_CCFPATH |
332 |
|
|
333 |
|
export SAS_CCFPATH="/home/ccflib/ftp-area/constituents" |
334 |
|
|
335 |
|
|
336 |
|
|
337 |
|
# 3. XMM_CALINDEX file from SAS_CCFPATH |
338 |
|
|
339 |
|
MIF=`ls -1 ${SAS_CCFPATH}/XMM_CALINDEX_*.CCF` |
340 |
|
|
341 |
|
# 4. For each OBDS ID registered in the XSA observations list, gets the start |
342 |
|
# time of the observation. Then runs cifbuild to get the respective CIF. |
343 |
|
# all_ccfs_${now}.txt is filled recursively on each iteration. |
344 |
|
# Therefore we start with an empty file. The final list will be this one. |
345 |
|
|
346 |
|
# Output file : ${VALID_CCF}/all_ccfs_${now}.txt |
347 |
|
|
348 |
|
touch ${VALID_CCF}/all_ccfs_${now}.txt |
349 |
|
|
|
touch ${VALID_CCF}/all_ccfs.txt |
|
350 |
n=0 |
n=0 |
351 |
|
|
352 |
while read line |
while read line |
353 |
do |
do |
354 |
obsid=`echo $line | awk -F"|" '{print $2}'` |
obsid=`echo $line | awk -F"|" '{print $2}'` |
380 |
issue=`echo $line | awk -F" " '{print $3}'` |
issue=`echo $line | awk -F" " '{print $3}'` |
381 |
issue=`printf "%04d" ${issue}` |
issue=`printf "%04d" ${issue}` |
382 |
echo ${class}_${issue}.CCF >> ${VALID_CCF}/${obsid}_ccfs.txt |
echo ${class}_${issue}.CCF >> ${VALID_CCF}/${obsid}_ccfs.txt |
383 |
|
|
384 |
done < ${VALID_CCF}/${obsid}_ccfs.tmp |
done < ${VALID_CCF}/${obsid}_ccfs.tmp |
385 |
|
|
386 |
rm ${VALID_CCF}/${obsid}_ccfs.tmp |
rm ${VALID_CCF}/${obsid}_ccfs.tmp |
387 |
|
|
388 |
mv ${VALID_CCF}/all_ccfs.txt ${VALID_CCF}/all_ccfs.tmp |
mv ${VALID_CCF}/all_ccfs_${now}.txt ${VALID_CCF}/all_ccfs_${now}.tmp |
389 |
cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs.tmp \ |
|
390 |
| sort -u > ${VALID_CCF}/all_ccfs.txt |
cat ${VALID_CCF}/${obsid}_ccfs.txt ${VALID_CCF}/all_ccfs_${now}.tmp | sort -u > ${VALID_CCF}/all_ccfs_${now}.txt |
391 |
|
|
392 |
|
comm -1 -3 --nocheck-order ${VALID_CCF}/all_ccfs_${now}.tmp ${VALID_CCF}/all_ccfs_${now}.txt > ${VALID_CCF}/diff_iteration_${n}.txt |
393 |
|
|
394 |
|
diffsize=`stat --format=%s ${VALID_CCF}/diff_iteration_${n}.txt` |
395 |
|
|
396 |
|
if [ "$diffsize" != "0" ] ; then |
397 |
|
echo "CCF added in this iteration: " |
398 |
|
cat ${VALID_CCF}/diff_iteration_${n}.txt |
399 |
|
fi |
400 |
|
|
401 |
|
echo |
402 |
|
echo "Number of valid CCFs found so far: "`cat ${VALID_CCF}/all_ccfs_${now}.txt | wc -l` ; echo |
403 |
|
|
404 |
|
rm ${VALID_CCF}/diff_iteration_${n}.txt |
405 |
rm ${cif_file} |
rm ${cif_file} |
406 |
rm ${VALID_CCF}/${obsid}_ccfs.txt |
rm ${VALID_CCF}/${obsid}_ccfs.txt |
407 |
|
rm ${VALID_CCF}/all_ccfs_${now}.tmp |
408 |
|
|
409 |
|
done < ${VALID_CCF}/xsaobslog_${now}.txt |
410 |
|
|
411 |
|
# 5. Make a table of classes and issues with the style "Pipeline Release Notes" |
412 |
|
# To get a classification, we need first to obtain all available classes. |
413 |
|
# |
414 |
|
|
415 |
|
|
416 |
|
# Lists all CCF classes |
417 |
|
|
418 |
|
ccf_classes="${VALID_CCF}/ccf_classes.txt" |
419 |
|
|
420 |
|
while read line |
421 |
|
do |
422 |
|
class=`echo $line | awk -F"_" '{print $1"_"$2}'` |
423 |
|
echo $class >> ${ccf_classes} |
424 |
|
done < ${VALID_CCF}/all_ccfs_${now}.txt |
425 |
|
|
426 |
|
cat ${ccf_classes} | sort -u > ${VALID_CCF}/kk.txt |
427 |
|
mv ${VALID_CCF}/kk.txt ${ccf_classes} |
428 |
|
|
429 |
|
|
430 |
|
|
431 |
|
# Output final file is named all_ccfs_${now}_table.txt |
432 |
|
|
433 |
|
total_number_of_ccfs=0 |
434 |
|
|
435 |
|
all_ccfs_table="${VALID_CCF}/all_ccfs_${now}_table.txt" |
436 |
|
|
437 |
|
[ -f "${all_ccfs_table}" ] && rm ${all_ccfs_table} |
438 |
|
|
439 |
|
echo "Table of valid CCFs at $now_table" >> ${all_ccfs_table} |
440 |
|
echo >> ${all_ccfs_table} |
441 |
|
echo "|================================|=============|=======|" >> ${all_ccfs_table} |
442 |
|
echo "| Calibration File | Issue range | Count |" >> ${all_ccfs_table} |
443 |
|
echo "|================================|=============|=======|" >> ${all_ccfs_table} |
444 |
|
|
445 |
|
while read ccf_class |
446 |
|
do |
447 |
|
echo -n "Processing class $ccf_class ..." |
448 |
|
issues_file="${VALID_CCF}/ccf_issues_for_${ccf_class}.txt" |
449 |
|
[ -f "${issues_file}" ] && rm ${issues_file} |
450 |
|
while read line |
451 |
|
do |
452 |
|
class=`echo $line | awk -F"_" '{print $1"_"$2}'` |
453 |
|
[ "$class" != "$ccf_class" ] && continue |
454 |
|
issue=`echo $line | awk -F"_" '{print $3}' | cut -d. -f1` |
455 |
|
echo $issue >> ${issues_file} |
456 |
|
done < ${VALID_CCF}/all_ccfs_${now}.txt |
457 |
|
|
458 |
|
cat ${issues_file} | sort -u > kk.txt |
459 |
|
mv kk.txt ${issues_file} |
460 |
|
real_number_of_issues_for_this_class=`wc -l ${issues_file} | awk -F" " '{print $1}'` |
461 |
|
|
462 |
|
first_issue=`head -1 ${issues_file}` |
463 |
|
last_issue=`tail -1 ${issues_file}` |
464 |
|
|
465 |
|
li=$((10#$last_issue)) |
466 |
|
fi=$((10#$first_issue)) |
467 |
|
|
468 |
|
sequential_number_of_issues_for_this_class=$((li - fi + 1)) |
469 |
|
|
470 |
|
asterisk=" " |
471 |
|
[ "${real_number_of_issues_for_this_class}" -lt "${sequential_number_of_issues_for_this_class}" ] && asterisk="(*)" |
472 |
|
|
473 |
|
total_number_of_ccfs=$((real_number_of_issues_for_this_class + total_number_of_ccfs)) |
474 |
|
|
475 |
|
rn=$real_number_of_issues_for_this_class |
476 |
|
|
477 |
|
if [ "${first_issue}" != "${last_issue}" ] ; then |
478 |
|
if [ "$asterisk" = "(*)" ] ; then |
479 |
|
printf "| %-30s | %4s-%4s(*)| %4s |\n" ${ccf_class} ${first_issue} ${last_issue} ${rn} >> ${all_ccfs_table} |
480 |
|
else |
481 |
|
printf "| %-30s | %4s-%4s | %4s |\n" ${ccf_class} ${first_issue} ${last_issue} ${rn} >> ${all_ccfs_table} |
482 |
|
fi |
483 |
|
else |
484 |
|
printf "| %-30s | %4s | %4s |\n" ${ccf_class} ${first_issue} ${rn} >> ${all_ccfs_table} |
485 |
|
fi |
486 |
|
|
487 |
|
|
488 |
|
echo "done" |
489 |
|
done < ${ccf_classes} |
490 |
|
|
491 |
|
echo "|================================|=============|=======|" >> ${all_ccfs_table} |
492 |
|
echo >> ${all_ccfs_table} |
493 |
|
echo "(*): Incomplete range." >> ${all_ccfs_table} |
494 |
|
echo "" >> ${all_ccfs_table} |
495 |
|
echo "Total number of valid CCFs required : ${total_number_of_ccfs}" >> ${all_ccfs_table} |
496 |
|
rm ${VALID_CCF}/ccf_classes.txt |
497 |
|
rm ${VALID_CCF}/ccf_issues_for*.txt |
498 |
|
|
499 |
|
|
500 |
|
|
501 |
|
# 6. Fill in the Valid CCF Set dir with the proper CCFs |
502 |
|
|
503 |
|
fillvalidccfdir "all_ccfs_${now}.txt" |
504 |
|
|
505 |
|
|
506 |
done < ${VALID_CCF}/xsaobslog.txt |
[ -f "${mailfile}" ] && rm -rf ${mailfile} |
507 |
|
echo "Valid CCF Set successfully copied to ${VALIDCONSTITUENTSDIR}" >> $mailfile |
508 |
|
echo "Valid CCF Set successfully copied to ${VALIDCCFDIRINTRANET}" >> $mailfile |
509 |
|
mailnotify "mvloccf: Valid CCF Set created successfully" $mailfile |
510 |
|
rm ${mailfile} |