batch-run.py

import subprocess
import sys

def batch_run_trawler(start_batch, end_batch):
    for i in range(start_batch, end_batch+1) :
        print(f"Making directory for ct-10m-chunk-0{i} ")
        make_dir = f"mkdir output/ct-10m-chunk-0{i}"
        process = subprocess.run(make_dir, shell=True)
        if process.returncode != 0:
            print(f"Failed to make directory with return code {process.returncode}")
            break

        command = f"./dns-trawler --csv-file-name=input/ct-10m-chunk-0{i} \
        --threads=1000 \
        --verbosity=4 \
        --ip-rate-limit=10 \
        --domain-rate-limit=200 \
        --log-path=output/chunk_0{i}"

        print(f"Starting Trawler ct-10m-chunk-0{i} ")
        process = subprocess.run(command, shell=True)
        
        # Check if the process was successful
        if process.returncode != 0:
            print(f"Trawler failed with return code {process.returncode}")
            break

start_batch = int(sys.argv[1])
end_batch = int(sys.argv[2])
batch_run_trawler(start_batch, end_batch)

#         command = f"./dns-trawler/dns-trawler --domain-names=google.com\
#         --threads=1 \
#         --verbosity=4 \
#         --ip-rate-limit=10 \
#         --domain-rate-limit=120"