Aeternus-Argus/SetupData.py at master · neilhuang007/Aeternus-Argus · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
import os

import requests
import tqdm
import zipfile

vpndatasets = [
    "http://205.174.165.80/CICDataset/ISCX-VPN-NonVPN-2016/Dataset/CSVs/Scenario%20A1-ARFF.zip",
    "http://205.174.165.80/CICDataset/ISCX-VPN-NonVPN-2016/Dataset/CSVs/Scenario%20A2-ARFF.zip",
    "http://205.174.165.80/CICDataset/ISCX-VPN-NonVPN-2016/Dataset/CSVs/Scenario%20B-ARFF.zip"
]


def downloadvpndataset(dataset):
    # Create a folder to store the dataset
    os.makedirs("vpndata", exist_ok=True)

    #clear what we have
    # Progress bar for the total number of files
    total_progress = tqdm.tqdm(total=len(vpndatasets), desc="Total files")

    for dataset in vpndatasets:
        response = requests.get(dataset, stream=True)
        total_bytes = int(response.headers.get('content-length', 0))
        downloaded_bytes = 0
        # Progress bar for the current file
        file_progress = tqdm.tqdm(total=total_bytes, desc=dataset.split('/')[-1], unit='B', unit_scale=True, unit_divisor=1024,)

        with open(os.path.join("vpndata", dataset.split('/')[-1]), "wb") as f:
            for data in response.iter_content(chunk_size=1024):
                f.write(data)

        # unzip all of the files and delete the zip files
        with zipfile.ZipFile(os.path.join("vpndata", dataset.split('/')[-1]), 'r') as zip_ref:
            zip_ref.extractall("vpndata")

        os.remove(os.path.join("vpndata", dataset.split('/')[-1]))

        file_progress.close()
        total_progress.update(1)

    total_progress.close()


downloadvpndataset(vpndatasets)
# converttocsv("./vpndata")