feat(rfimask): Add advanced RFI masking with frequency reversal support

- Introduced new `astrorfimask` command for batch RFI masking
- Implemented Savitzky-Golay and Spectral Kurtosis algorithms
- Added frequency reversal for descending order data
- Enhanced multiprocessing support for large datasets
- Improved error handling and progress tracking
- Updated documentation with detailed usage examples
- Added model download functionality for default weights
- Removed deprecated preprocess configuration

Author: lintian233

.github/workflows/ubuntu-22.04-build.yml

@@ -149,7 +149,7 @@ Below are the commonly used YAML configuration options:
 | `confidence` | Detection confidence threshold | `0.4` | `0.372` |
 | `snrhold` | SNR threshold | `5` | `5` |
 | `modelname` | Detector model type | current only `yolov11n`
-| `modelpath` | Path to model weights | `/path/to/yolo11n_0816_v1.pt` | - |
+| `modelpath` | Path to customodel weights | `model.pt` | - |
 | `timedownfactor` | Time downsampling factor | `8` | `1` |
 
 </details>
@@ -238,7 +238,7 @@ maskdir: /path/to/maskdir
 
 </details>
 
-<details>
+<!-- <details>
   <summary>Data Processing</summary>
 
 | Option | Description | Example |
@@ -252,7 +252,7 @@ preprocess:
   - guassion: 1 5            # Gaussian filter parameters
 ```
 
-</details>
+</details> -->
 
 <details>
   <summary>Plotting Configuration</summary>
@@ -300,7 +300,8 @@ cputhread: 32
 
 snrhold: 5
 modelname: yolov11n
-modelpath: yolo11n_0816_v1.pt
+# use default
+# modelpath: yolo11n_0816_v1.pt 
 
 rfi: ai
 maskfile: file.bad_chans
@@ -349,7 +350,8 @@ detgpu: 1
 cputhread: 64
 
 modelname: yolov11n
-modepath: yolo11n_0816_v1.pt
+# use default
+# modelpath: yolo11n_0816_v1.pt 
 plotworker: 16
 
 rfi: ai

README.md

@@ -141,7 +141,7 @@
 | `confidence` | 检测置信度阈值 | `0.4` | `0.372` |
 | `snrhold` | 信噪比阈值 | `5` | `5` |
 | `modelname` | 检测器模型类型 | 目前仅支持 `yolov11n` |
-| `modelpath` | 模型权重路径 | `/path/to/yolo11n_0816_v1.pt` | - |
+| `modelpath` | 自定义模型权重路径 | `/path/to/yolo11n_0816_v1.pt` | - |
 | `timedownfactor` | 时间降采样因子 | `8` | `1` |
 
 </details>
@@ -230,7 +230,7 @@ maskdir: /path/to/maskdir
 
 </details>
 
-<details>
+<!-- <details>
   <summary>数据处理</summary>
 
 | 选项 | 描述 | 示例 |
@@ -244,7 +244,7 @@ preprocess:
   - guassion: 1 5            # 高斯滤波参数
 ```
 
-</details>
+</details> -->
 
 <details>
   <summary>绘图配置</summary>
@@ -291,7 +291,8 @@ cputhread: 32
 
 snrhold: 5
 modelname: yolov11n
-modelpath: yolo11n_0816_v1.pt
+# use default
+# modelpath: yolo11n_0816_v1.pt 
 
 rfi: ai
 maskfile: file.bad_chans
@@ -340,7 +341,8 @@ detgpu: 1
 cputhread: 64
 
 modelname: yolov11n
-modepath: yolo11n_0816_v1.pt
+# use default
+# modelpath: yolo11n_0816_v1.pt 
 plotworker: 16
 
 rfi: ai

docs/README_zh-CN.md

@@ -12,7 +12,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "pulseflow"
-version = "0.1.0"
+version = "0.1.1"
 description = "High-performance radio astronomy single pulse Detect lib"
 requires-python = ">=3.10"
 readme = "README.md"
@@ -44,6 +44,7 @@ dev = ["pytest>=6.0", "pytest-cov", "black", "flake8"]
 
 [project.scripts]
 astroflow = "astroflow.cli:main"
+astrorfimask = "astroflow.rfimask:astrorfimask"
 
 [tool.setuptools]
 include-package-data = true

pyproject.toml

@@ -132,3 +132,4 @@ def main():
         print(f"Error: {e}")
         exit(1)
     exit(0)
+

python/astroflow/cli.py

@@ -1,12 +1,16 @@
 import os
 from sympy import preorder_traversal
 import yaml
+import urllib.request
+
+
 
 CENTERNET = 0
 YOLOV11N = 1
 DETECTNET = 2
 COMBINENET = 3
 
+
 class TaskConfig:
     _instance = None
 
@@ -81,15 +85,34 @@ def _checker_freqrange(self, freqrange):
         else:
             raise ValueError("Invalid format for freqrange in config file.")
 
-    def _checker_preprocess(self, preprocess):
-        if isinstance(preprocess, list):
-            for item in preprocess:
-                if not isinstance(item, dict) or len(item) != 1:
-                    raise ValueError(
-                        "Invalid format for preprocess in config file. Each item should be a single key-value pair."
-                    )
-        else:
-            raise ValueError("Invalid format for preprocess in config file.")
+    # def _checker_preprocess(self, preprocess):
+    #     if isinstance(preprocess, list):
+    #         for item in preprocess:
+    #             if not isinstance(item, dict) or len(item) != 1:
+    #                 raise ValueError(
+    #                     "Invalid format for preprocess in config file. Each item should be a single key-value pair."
+    #                 )
+    #     else:
+    #         raise ValueError("Invalid format for preprocess in config file.")
+
+
+
+    def get_model(self):
+        model_url_path = "https://github.com/lintian233/astroflow/releases/download/v0.1.1/yolo11n_0816_v1.pt" 
+        config_dir = "~/.config/astroflow"
+        config_dir = os.path.expanduser(config_dir)
+        os.makedirs(config_dir, exist_ok=True)
+        
+        model_filename = "yolov1n_0816_v1.pt"
+        local_model_path = os.path.join(config_dir, model_filename)
+        
+        if not os.path.exists(local_model_path):
+            print(f"Downloading model from {model_url_path}...")
+            urllib.request.urlretrieve(model_url_path, local_model_path)
+            print(f"Model downloaded to {local_model_path}")
+        
+        return local_model_path
+
 
     def _checker_dm_limt(self, dm_limt):
         if isinstance(dm_limt, list):
@@ -177,18 +200,16 @@ def candpath(self):
 
     @property
     def modelpath(self):
-        model_url_path = "" # yolo11n_pulsedetect.pt github release path
         # https get
         modelpath = self._config_data.get("modelpath")
         if modelpath is None:
-            raise ValueError("modelpath not found in config file.")
+            modelpath = self.get_model()
         if not isinstance(modelpath, str):
             raise ValueError("modelpath must be a string.")
         if not os.path.exists(modelpath):
             raise FileNotFoundError(f"Model path {modelpath} does not exist.")
         return modelpath
 
-
     @property
     def dedgpu(self):
         return self._config_data.get("dedgpu", 0)

python/astroflow/config/taskconfig.py

@@ -182,16 +182,16 @@ def _create_detector_and_plotter(task_config: TaskConfig) -> Tuple[Union[CenterN
 
     if task_config.modelname == CENTERNET:
         detector = CenterNetFrbDetector(
-            task_config.dm_limt, task_config.preprocess, task_config.confidence
+            task_config.dm_limt, None, task_config.confidence
         )
     elif task_config.modelname == YOLOV11N:
         detector = Yolo11nFrbDetector(
-            task_config.dm_limt, task_config.preprocess, task_config.confidence
+            task_config.dm_limt, None, task_config.confidence
         )
     else:
         logger.warning(f"Unknown model name {task_config.modelname}, using CenterNet")
         detector = CenterNetFrbDetector(
-            task_config.dm_limt, task_config.preprocess, task_config.confidence
+            task_config.dm_limt, None, task_config.confidence
         )
 
     return detector, plotter
@@ -229,14 +229,14 @@ def _process_single_file(
                 base_dir += f"-{config.freq_start}MHz-{config.freq_end}MHz"
                 base_dir += f"-{config.dm_step}DM-{config.t_sample}s"
 
-                cached_dir = os.path.join(output_dir, "cached").lower()
-                file_dir = os.path.join(cached_dir, base_dir, file_basename).lower()
+                cached_dir = os.path.join(output_dir, "cached")
+                file_dir = os.path.join(cached_dir, base_dir, file_basename)
 
 
                 if os.path.exists(file_dir):
                     logger.info(f"Skipping already processed file: {file_basename}")
                     # 检查 candidate_detect_dir 目录下是否有文件，如果没有则 detection_flag = 0
-                    candidate_detect_dir = os.path.join(output_dir, "candidate", file_basename).lower()
+                    candidate_detect_dir = os.path.join(output_dir, "candidate", file_basename)
                     if any(os.scandir(candidate_detect_dir)):
                         file_detected = True
                     continue

python/astroflow/dataset/generate.py

@@ -992,9 +992,25 @@ def plot_candidate(
                 spec_data, noise_range=None, threshold_sigma=5, toa_sample_idx=int((peak_time - spec_tstart) / header.tsamp)
             )
 
+            # Check SNR against threshold
+            snrhold = taskconfig.snrhold
+            if snr < snrhold:
+                print(f"Warning: SNR {snr:.2f} is below threshold {snrhold}. Skipping spectrum plot.")
+                plt.close('all')
+                if origin_data is not None:
+                    if hasattr(origin_data, "close"):
+                        try:
+                            origin_data.close()
+                        except Exception:
+                            pass
+                    del origin_data
+                
+                del spectrum, spec_data, initial_spectrum, initial_spec_data
+                gc.collect()
+                return  
+
             peak_time = spec_tstart + (peak_idx + 0.5) * header.tsamp
             pulse_width_ms = pulse_width * header.tsamp * 1e3 if pulse_width > 0 else -1  # Convert to milliseconds
-
 
             # Create time and frequency axes
             spec_time_axis = np.linspace(spec_tstart, spec_tend, spectrum.ntimes)