Skip to content

Commit 2f04f48

Browse files
ymaan4ymaan4
authored
Merge pull request #7 from ymaan4/skipRFI_opts
New Skip-RFI-excision options, a little bit better documentation and other minor changes
2 parents a147da5 + 6abe252 commit 2f04f48

File tree

9 files changed

+264
-89
lines changed

9 files changed

+264
-89
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ Mkfile.old
5252
dkms.conf
5353

5454
##bin/rficlean.flags # include this in the package as an example
55-
bin/crp_rficlean_gm.sh
55+
#bin/crp_rficlean_gm.sh
5656
bin/rficlean
5757
bin/read_n_make_plot
5858
src/read_n_make_plot.c

‎COPYING‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ patent must be licensed for everyone's free use or not licensed at all.
5555

5656
The precise terms and conditions for copying, distribution and
5757
modification follow.
58-
58+
5959
GNU GENERAL PUBLIC LICENSE
6060
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
6161

@@ -110,7 +110,7 @@ above, provided that you also meet all of these conditions:
110110
License. (Exception: if the Program itself is interactive but
111111
does not normally print such an announcement, your work based on
112112
the Program is not required to print an announcement.)
113-
113+
114114
These requirements apply to the modified work as a whole. If
115115
identifiable sections of that work are not derived from the Program,
116116
and can be reasonably considered independent and separate works in
@@ -168,7 +168,7 @@ access to copy from a designated place, then offering equivalent
168168
access to copy the source code from the same place counts as
169169
distribution of the source code, even though third parties are not
170170
compelled to copy the source along with the object code.
171-
171+
172172
4. You may not copy, modify, sublicense, or distribute the Program
173173
except as expressly provided under this License. Any attempt
174174
otherwise to copy, modify, sublicense or distribute the Program is
@@ -225,7 +225,7 @@ impose that choice.
225225

226226
This section is intended to make thoroughly clear what is believed to
227227
be a consequence of the rest of this License.
228-
228+
229229
8. If the distribution and/or use of the Program is restricted in
230230
certain countries either by patents or by copyrighted interfaces, the
231231
original copyright holder who places the Program under this License
@@ -278,7 +278,7 @@ PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
278278
POSSIBILITY OF SUCH DAMAGES.
279279

280280
END OF TERMS AND CONDITIONS
281-
281+
282282
How to Apply These Terms to Your New Programs
283283

284284
If you develop a new program, and you want it to be of the greatest
@@ -338,3 +338,5 @@ proprietary programs. If your program is a subroutine library, you may
338338
consider it more useful to permit linking proprietary applications with the
339339
library. If this is what you want to do, use the GNU Library General
340340
Public License instead of this License.
341+
342+

‎COPYRIGHT‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,4 @@
1616
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1717
*
1818
*/
19+

‎bin/crp_rficlean_fil.sh‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
#=======================================================================
3-
# Crude parallelization of rfiClean for processing filterbank files.
3+
# Crude parallelization of RFIClean for processing filterbank files.
44
#
55
# Yogesh Maan, Dec. 2019
66
#=======================================================================
@@ -39,13 +39,13 @@ Nb=`echo "scale=0; $nsamples/($Np*$block) + 1" | bc`
3939

4040

4141

42-
echo "Starting parallel-rfiClean processing at: "
42+
echo "Starting parallel-RFIClean processing at: "
4343
date
4444
echo ""
4545

4646
##-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
4747
# prepare and execute the commands for all parts
48-
mkdir -p rfiClean_ps
48+
mkdir -p RFIClean_ps
4949
combine_cmd="cat "
5050
remove_cmd="rm "
5151
count=0
@@ -57,15 +57,15 @@ for (( c=1; c<=$Np; c++ )); do
5757
psfile=${rtag}_part${count}.ps
5858
#-------------------------------------
5959
if [ $count -eq 1 ]; then
60-
cmd="rficlean -t $block $flag -o $outfile -ps rfiClean_ps/$psfile $infile -bst $bst -nbl $Nb &"
60+
cmd="rficlean -t $block $flag -o $outfile -ps RFIClean_ps/$psfile $infile -bst $bst -nbl $Nb &"
6161
elif [ $count -eq $Np ]; then
6262
combine_cmd="${combine_cmd} ${tempout}"
6363
remove_cmd="${remove_cmd} ${tempout}"
64-
cmd="rficlean -t $block $flag -o $tempout -ps rfiClean_ps/$psfile $infile -bst $bst -headerless &"
64+
cmd="rficlean -t $block $flag -o $tempout -ps RFIClean_ps/$psfile $infile -bst $bst -headerless &"
6565
else
6666
combine_cmd="${combine_cmd} ${tempout}"
6767
remove_cmd="${remove_cmd} ${tempout}"
68-
cmd="rficlean -t $block $flag -o $tempout -ps rfiClean_ps/$psfile $infile -bst $bst -nbl $Nb -headerless &"
68+
cmd="rficlean -t $block $flag -o $tempout -ps RFIClean_ps/$psfile $infile -bst $bst -nbl $Nb -headerless &"
6969
fi
7070
echo $cmd
7171
eval $cmd
@@ -83,7 +83,7 @@ echo $cmd
8383
eval $cmd
8484

8585

86-
echo "Finished parallel-rfiClean processing at: "
86+
echo "Finished parallel-RFIClean processing at: "
8787
date
8888
echo ""
8989

‎bin/crp_rficlean_gm.sh‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/bin/bash
2+
#=======================================================================
3+
# Crude parallelization of RFIClean for processing GMRT data files.
4+
#
5+
# Yogesh Maan, Dec. 2019
6+
#=======================================================================
7+
#
8+
9+
10+
11+
#------------------------------------------------------------------------------
12+
if [ $# -lt 6 ] ; then
13+
echo "========================================================================================"
14+
echo " Usage: "
15+
echo " crp_rficlean_gm.sh <out_filename> <flag-file> <N_parallel> <in_filename> <gm_info> <extra-flag>"
16+
echo " "
17+
echo "========================================================================================"
18+
exit 0
19+
fi
20+
outfile=$1
21+
flagfile=$2
22+
Np=$3
23+
infile=$4
24+
gminfo_file=$5
25+
extra_flag=$6
26+
#------------------------------------------------------------------------------
27+
28+
rtag=${outfile%*.fil} # tag for part-files
29+
30+
##-------------------------------
31+
## read flags from the flag-file
32+
block=`sed -n 1p $flagfile`
33+
flag=`sed -n 2p $flagfile`
34+
#echo "block-size: $block"
35+
#echo "flag: '$flag'"
36+
## read nchan from the gm-info-file
37+
nchans=`sed -n 4p $gminfo_file`
38+
##-------------------------------
39+
40+
41+
# deduce number of blocks to be processed in each part
42+
fsize=$(stat -cL%s "$infile")
43+
nsamples=`echo "scale=0; $fsize/(2*$nchans)" | bc` ## gmrt data --> 2 bytes/sample
44+
Nb=`echo "scale=0; $nsamples/($Np*$block) + 1" | bc`
45+
#echo "nsamples, Nblocks: $nsamples, $Nb, $Np, $block ---"
46+
47+
48+
echo "Starting parallel-RFIClean processing at: "
49+
date
50+
echo ""
51+
52+
##-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
53+
# prepare and execute the commands for all parts
54+
mkdir -p RFIClean_ps
55+
combine_cmd="cat "
56+
remove_cmd="rm "
57+
count=0
58+
for (( c=1; c<=$Np; c++ )); do
59+
bst=$(($count*$Nb + 1))
60+
count=$((count+1))
61+
#echo $count
62+
tempout=${rtag}_part${count}.fil
63+
psfile=${rtag}_part${count}.ps
64+
pdffile=${rtag}_part${count}.pdf
65+
#-------------------------------------
66+
if [ $count -eq 1 ]; then
67+
cmd="/home/ymaan/bin/rficlean -t $block $flag -o $outfile -ps RFIClean_ps/$psfile $infile -bst $bst -nbl $Nb -gm $gminfo_file $extra_flag && ps2pdf RFIClean_ps/$psfile RFIClean_ps/$pdffile && rm RFIClean_ps/$psfile &"
68+
elif [ $count -eq $Np ]; then
69+
combine_cmd="${combine_cmd} ${tempout}"
70+
remove_cmd="${remove_cmd} ${tempout}"
71+
cmd="/home/ymaan/bin/rficlean -t $block $flag -o $tempout -ps RFIClean_ps/$psfile $infile -bst $bst -headerless -gm $gminfo_file $extra_flag && ps2pdf RFIClean_ps/$psfile RFIClean_ps/$pdffile && rm RFIClean_ps/$psfile &"
72+
else
73+
combine_cmd="${combine_cmd} ${tempout}"
74+
remove_cmd="${remove_cmd} ${tempout}"
75+
cmd="/home/ymaan/bin/rficlean -t $block $flag -o $tempout -ps RFIClean_ps/$psfile $infile -bst $bst -nbl $Nb -headerless -gm $gminfo_file $extra_flag && ps2pdf RFIClean_ps/$psfile RFIClean_ps/$pdffile && rm RFIClean_ps/$psfile &"
76+
fi
77+
echo $cmd
78+
eval $cmd
79+
done
80+
wait
81+
##-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
82+
83+
84+
# Now combine all parts and then remove the parts
85+
combine_cmd="${combine_cmd} >> ${outfile}"
86+
#echo $combine_cmd
87+
#echo $remove_cmd
88+
cmd="$combine_cmd && $remove_cmd"
89+
echo $cmd
90+
eval $cmd
91+
92+
93+
echo "Finished parallel-RFIClean processing at: "
94+
date
95+
echo ""
96+
97+
exit 0
98+

‎include/rficlean.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#include <stdlib.h>
22
#include <stdio.h>
3+
#include <stdbool.h>
34
#include <math.h>
45
#include "header.h"
56
#include <fftw3.h>
@@ -14,6 +15,7 @@ float *fftstat, *chanstat, *predist, *xpredist, *postdist, *xpostdist, *finaldis
1415
double *tfvar, *tfmean,meanvar,rmsvar ;
1516
double *chandata, *mspec, *rspec, *vspec, *wspec, *wt;
1617
long int *coff;
18+
bool RFIx,rfiFDx,rfiTx,rfiSx,rfiMSx,rfiVSx,rfiSclip;
1719

1820
fftw_complex *in;
1921
fftw_complex *out;

‎src/cleanit.c‎

Lines changed: 53 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -371,11 +371,19 @@ void cleanit(float *data, int nchans, long int nadd)
371371
isame = all_samef(chandata,nadd);
372372
}
373373
if( isame == 0){
374-
fftclean(in,out,nadd,inc); // clean some periodic RFIs
374+
if(rfiFDx){
375+
fftclean(in,out,nadd,inc); // clean some periodic RFIs
376+
}
375377
for (ii = 0; ii<nadd; ii++) chandata[ii] = in[ii][0];
376-
tsclip(chandata,nadd,sthresh); // clean some spiky RFIs
377-
mspec[channum]=chandata[nadd];
378-
rspec[channum]=chandata[nadd+1];
378+
if(rfiTx){
379+
tsclip(chandata,nadd,sthresh); // clean some spiky RFIs
380+
mspec[channum]=chandata[nadd];
381+
rspec[channum]=chandata[nadd+1];
382+
} else {
383+
robust_meanrms(chandata,nadd);
384+
mspec[channum]=chandata[nadd];
385+
rspec[channum]=chandata[nadd+1];
386+
}
379387
}
380388
else {
381389
mspec[channum]=chandata[0];
@@ -388,10 +396,14 @@ void cleanit(float *data, int nchans, long int nadd)
388396

389397

390398
for (i=0; i<nchans; i++) wspec[i]=+1.0;
391-
spfind(mspec,nchans,rthresh,wspec);
392-
spfind(vspec,nchans,rthresh,wspec);
393-
tsfind(mspec,nchans,rthresh,wspec);
394-
tsfind(vspec,nchans,rthresh,wspec);
399+
if(rfiMSx){
400+
spfind(mspec,nchans,rthresh,wspec);
401+
tsfind(mspec,nchans,rthresh,wspec);
402+
}
403+
if(rfiVSx){
404+
spfind(vspec,nchans,rthresh,wspec);
405+
tsfind(vspec,nchans,rthresh,wspec);
406+
}
395407
if(pcl>0){
396408
kk = (int)(0.9*nchans);
397409
for (i=kk;i<nchans;i++) wspec[i]=0.0;
@@ -437,28 +449,32 @@ void cleanit(float *data, int nchans, long int nadd)
437449

438450

439451
// Try clipping some channels in individual samples
440-
for (t=0; t<nadd; t++){
441-
nxc = t*nchans;
442-
for (c=0; c<nchans; c++) chandata[c] = data[nxc+c];
443-
spclip(chandata,nchans,clipthresh);
444-
for (c=0; c<nchans; c++) data[nxc+c] = chandata[c];
452+
if(rfiSclip){
453+
for (t=0; t<nadd; t++){
454+
nxc = t*nchans;
455+
for (c=0; c<nchans; c++) chandata[c] = data[nxc+c];
456+
spclip(chandata,nchans,clipthresh);
457+
for (c=0; c<nchans; c++) data[nxc+c] = chandata[c];
458+
}
445459
}
446460
// Now some timeseries cleaning
447-
an = (double)nadd;
448-
for (t=0; t<nadd; t++){
449-
nxc = t*nchans;
450-
chandata[t]=0.0;
451-
for (c=0; c<nchans; c++) chandata[t] = chandata[t]+data[nxc+c];
452-
chandata[t]=chandata[t]/an;
453-
}
454-
for (i=0; i<nadd; i++) wt[i]=+1.0;
455-
tsfind(chandata,nadd,sthresh,wt);
456-
spfind(chandata,nadd,sthresh,wt);
457-
for (t=0; t<nadd; t++){
458-
nxc = t*nchans;
459-
if(wt[t] < 0.0){
460-
for (c=0; c<nchans; c++) data[nxc+c] = mspec[c];
461-
}
461+
if(rfiTx){
462+
an = (double)nadd;
463+
for (t=0; t<nadd; t++){
464+
nxc = t*nchans;
465+
chandata[t]=0.0;
466+
for (c=0; c<nchans; c++) chandata[t] = chandata[t]+data[nxc+c];
467+
chandata[t]=chandata[t]/an;
468+
}
469+
for (i=0; i<nadd; i++) wt[i]=+1.0;
470+
tsfind(chandata,nadd,sthresh,wt);
471+
spfind(chandata,nadd,sthresh,wt);
472+
for (t=0; t<nadd; t++){
473+
nxc = t*nchans;
474+
if(wt[t] < 0.0){
475+
for (c=0; c<nchans; c++) data[nxc+c] = mspec[c];
476+
}
477+
}
462478
}
463479

464480
// sometimes gpt results might need additional checks
@@ -536,14 +552,15 @@ void cleanit(float *data, int nchans, long int nadd)
536552
}
537553

538554

539-
540-
if(iflip==1){ // flip the band
541-
for (t=0; t<nadd; t++){
542-
nxc = t*nchans;
543-
for (c=0; c<nchans; c++) mspec[c] = data[nxc+c];
544-
for (c=0; c<nchans; c++) data[nxc+c] = mspec[nchans-c-1];
545-
}
546-
}
555+
/* ***Now the following part is in rficlean_data.c ***
556+
// if(iflip==1){ // flip the band
557+
// for (t=0; t<nadd; t++){
558+
// nxc = t*nchans;
559+
// for (c=0; c<nchans; c++) mspec[c] = data[nxc+c];
560+
// for (c=0; c<nchans; c++) data[nxc+c] = mspec[nchans-c-1];
561+
// }
562+
// }
563+
***/
547564

548565
}
549566
//============================================================================

0 commit comments

Comments
 (0)