Rename input dir to raw_block_data

LadyChristina · dimkarakostas · commit dab603ab5fc9 · 2023-08-24T16:39:17.000+03:00
diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,6 @@ data.json
 multi_pool_addresses*
 multi_pool_blocks
 output
-input/*_raw_data.json
+raw_block_data/*_raw_data.json
 .coverage
 site
diff --git a/README.md b/README.md
@@ -30,7 +30,7 @@ project:
 
 ## Run the tool
 
-Place all raw data (which could be collected from BigQuery for example) in the `input` directory, each file named as
+Place all raw data (which could be collected from BigQuery for example) in the `raw_block_data` directory, each file named as
 `<project_name>_raw_data.json` (e.g. `bitcoin_raw_data.json`). By default, there
 is a (very small) sample input file for some supported projects. To use the
 samples, remove the prefix `sample_`. For more extended raw data and instructions on how to retrieve it, see
diff --git a/docs/contribute.md b/docs/contribute.md
@@ -64,8 +64,8 @@ specifically:
 
 ### Data
 
-You can optionally commit small sample data for the new ledger in the input folder.
-Alternatively, make sure to add your raw data file in the input folder before
+You can optionally commit small sample data for the new ledger in the raw_block_data folder.
+Alternatively, make sure to add your raw data file in the raw_block_data folder before
 running the tool on the new ledger.
 
 ## Update existing mapping information
diff --git a/docs/parsers.md b/docs/parsers.md
@@ -3,7 +3,7 @@
 The parser obtains raw data from a full node (see [Data Collection](data.md) page on how to obtain the required data).
 It parses the data and outputs a `json` file with a list of entries, each entry corresponding to a block.
 
-The input file should be placed in the `input` directory and named as `<project_name>_raw_data.json`.
+The input file should be placed in the `raw_block_data` directory and named as `<project_name>_raw_data.json`.
 
 The output file is stored under `output/<project_name>/parsed_data.json` and is structured as follows:
 
diff --git a/docs/setup.md b/docs/setup.md
@@ -24,7 +24,7 @@ The `run.py` script in the root directory of the project invokes the required pa
 also possible to execute each module individually. The following process describes the most typical workflow.
 
 Place all raw data (which could be collected from BigQuery for example; see [Data Collection](data.md) for more details)
-in the `input` directory, each file named as `<project_name>_raw_data.json` (e.g., `bitcoin_raw_data.json`). By default,
+in the `raw_block_data` directory, each file named as `<project_name>_raw_data.json` (e.g., `bitcoin_raw_data.json`). By default,
 there is a (very small) sample input file for some supported projects; to use it, remove the prefix `sample_`.
 
 Run `python run.py --ledgers <ledger_1> <ledger_n> --timeframe <timeframe>` to
diff --git a/raw_block_data/sample_bitcoin_raw_data.json b/raw_block_data/sample_bitcoin_raw_data.json
diff --git a/raw_block_data/sample_cardano_raw_data.json b/raw_block_data/sample_cardano_raw_data.json
diff --git a/raw_block_data/sample_ethereum_raw_data.json b/raw_block_data/sample_ethereum_raw_data.json
diff --git a/raw_block_data/sample_tezos_raw_data.json b/raw_block_data/sample_tezos_raw_data.json
diff --git a/run.py b/run.py
@@ -4,7 +4,7 @@
 from src.analyze import analyze
 from src.parse import parse
 from src.plot import plot
-from src.helper import valid_date, INPUT_DIR, OUTPUT_DIR, get_default_ledgers, get_start_end_years
+from src.helper import valid_date, RAW_DATA_DIR, OUTPUT_DIR, get_default_ledgers, get_start_end_years
 
 logging.basicConfig(format='[%(asctime)s] %(message)s', datefmt='%Y/%m/%d %I:%M:%S %p', level=logging.INFO)
 
@@ -25,7 +25,7 @@ def main(projects, timeframes, force_parse, force_map, make_plots, make_animated
     """
     logging.info(f"The ledgers that will be analyzed are: {','.join(projects)}")
     for project in projects:
-        parse(project, INPUT_DIR, output_dir, force_parse)
+        parse(project, RAW_DATA_DIR, output_dir, force_parse)
         apply_mapping(project, timeframes, output_dir, force_map)
 
     used_metrics = analyze(projects, timeframes, output_dir)
diff --git a/src/collect_data.py b/src/collect_data.py
@@ -1,6 +1,6 @@
 """
-    This script can be used to run queries on BigQuery for any number of blockchains, and save the results in the input
-    directory of the project.
+    This script can be used to run queries on BigQuery for any number of blockchains, and save the results in the
+    raw_block_data directory of the project.
     The relevant queries must be stored in a file named 'queries.yaml' in the root directory of the project.
 
     Attention! Before running this script, you need to generate service account credentials from Google, as described
@@ -13,12 +13,12 @@
 import logging
 from yaml import safe_load
 
-from src.helper import ROOT_DIR, INPUT_DIR
+from src.helper import ROOT_DIR, RAW_DATA_DIR
 
 
 def collect_data(force_query):
-    if not INPUT_DIR.is_dir():
-        INPUT_DIR.mkdir()
+    if not RAW_DATA_DIR.is_dir():
+        RAW_DATA_DIR.mkdir()
 
     with open(ROOT_DIR / "queries.yaml") as f:
         queries = safe_load(f)
@@ -27,7 +27,7 @@ def collect_data(force_query):
 
     for ledger in queries.keys():
         filename = f'{ledger}_raw_data.json'
-        file = INPUT_DIR / filename
+        file = RAW_DATA_DIR / filename
         if not force_query and file.is_file():
             logging.info(f'{ledger} data already exists locally. '
                          f'For querying {ledger} anyway please run the script using the flag --force-query')
diff --git a/src/helper.py b/src/helper.py
@@ -12,7 +12,7 @@
 from yaml import safe_load
 
 ROOT_DIR = pathlib.Path(__file__).resolve().parent.parent
-INPUT_DIR = ROOT_DIR / 'input'
+RAW_DATA_DIR = ROOT_DIR / 'raw_block_data'
 OUTPUT_DIR = ROOT_DIR / 'output'
 MAPPING_INFO_DIR = ROOT_DIR / 'mapping_information'
 
diff --git a/src/parse.py b/src/parse.py
@@ -3,7 +3,7 @@
 from src.parsers.default_parser import DefaultParser
 from src.parsers.dummy_parser import DummyParser
 from src.parsers.ethereum_parser import EthereumParser
-from src.helper import INPUT_DIR, OUTPUT_DIR
+from src.helper import RAW_DATA_DIR, OUTPUT_DIR
 
 
 ledger_parser = {
@@ -51,4 +51,4 @@ def parse(project, input_dir, output_dir, force_parse=False):
     )
     args = parser.parse_args()
 
-    parse(args.ledger, INPUT_DIR, OUTPUT_DIR, args.force_parse)
+    parse(args.ledger, RAW_DATA_DIR, OUTPUT_DIR, args.force_parse)
diff --git a/tests/test_3_parsers.py b/tests/test_3_parsers.py
@@ -4,7 +4,7 @@
 from src.parse import parse, ledger_parser
 from src.parsers.default_parser import DefaultParser
 from src.parsers.dummy_parser import DummyParser
-from src.helper import INPUT_DIR, OUTPUT_DIR
+from src.helper import RAW_DATA_DIR, OUTPUT_DIR
 
 
 @pytest.fixture
@@ -15,9 +15,9 @@ def setup_and_cleanup():
     after (cleanup)
     """
     # Set up
-    test_input_dir = INPUT_DIR
+    test_raw_data_dir = RAW_DATA_DIR
     test_output_dir = OUTPUT_DIR / "test_output"
-    yield test_input_dir, test_output_dir
+    yield test_raw_data_dir, test_output_dir
     # Clean up
     shutil.rmtree(test_output_dir)
 
@@ -38,7 +38,7 @@ def compare_parsed_samples(correct_data, parsed_file):
 
 
 def test_default_parser(setup_and_cleanup):
-    test_input_dir, test_output_dir = setup_and_cleanup
+    test_raw_data_dir, test_output_dir = setup_and_cleanup
     sample_parsed_data = [
         {"number": "507516", "timestamp": "2018-02-04 02:36:23 UTC", "identifiers": 'b"\\x03|\\xbe\\x07A\\xd6\\x9d\\x9cj\\xcc\\xe4\\xd1A\\xd6\\x9d\\x9ci\\xf9\\xbe\\xf5/BTC.TOP/\\xfa\\xbemm\\x141 \\xf7\\xb3\\xda\\x91\\x8f\\x12\\xff\\xb3(\\xab\\x93_\\xbf\\xe2\\xd1\\xcd\\x9b\\xb4pre\\xd7\\xfe\\xe2?\\xd6\\xcf7\'\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00ZD\\xca\\xcf\\x00\\x00\\xf8\\xa4A \\x00\\x00"', "reward_addresses": "137YB5cpBLxLKvy8T6qXsycJ699iJjWCHH,1FVKW4rp5rN23dqFVk2tYGY4niAXMB8eZC"},
         {"number": "507715", "timestamp": "2018-02-05 04:54:34 UTC", "identifiers": "b'\\x03C\\xbf\\x07\\x13/mined by gbminers/,\\xfa\\xbemm\\x94\\x97n\\xce\\xbb\\xc7;=B\\x14\\xb3\\xd7\\xab3\\r\\xca!)\\xeb\\xfc\\xc8c\\xfaub<o\\x95\\x89\\x1esF\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x10\\xaf\\x8bf\\x00-\\xa9\\x10\\xef@\\x8cwhR\\x84\\x01\\x00'", "reward_addresses": "1J7FCFaafPRxqu4X9VsaiMZr1XMemx69GR,131RUhDyyjxXSbSPxGRCm3t6vcei1TB6MB"},
@@ -54,7 +54,7 @@ def test_default_parser(setup_and_cleanup):
 
     project_name = 'sample_bitcoin'
 
-    parser = DefaultParser(project_name, test_input_dir, test_output_dir)
+    parser = DefaultParser(project_name, test_raw_data_dir, test_output_dir)
     parser.parse()
 
     parsed_file = test_output_dir / f'{project_name}/parsed_data.json'
@@ -64,7 +64,7 @@ def test_default_parser(setup_and_cleanup):
 
 
 def test_dummy_parser(setup_and_cleanup):
-    test_input_dir, test_output_dir = setup_and_cleanup
+    test_raw_data_dir, test_output_dir = setup_and_cleanup
     sample_parsed_data = [
         {"number": "1649812", "timestamp": "2021-08-30 00:36:18 UTC", "identifiers": None, "reward_addresses": "tz1Kf25fX1VdmYGSEzwFy1wNmkbSEZ2V83sY"},
         {"number": "1650474", "timestamp": "2021-08-30 06:11:58 UTC", "identifiers": None, "reward_addresses": "tz1Vd1rXpV8hTHbFXCXN3c3qzCsgcU5BZw1e"},
@@ -76,7 +76,7 @@ def test_dummy_parser(setup_and_cleanup):
 
     project_name = 'sample_tezos'
 
-    parser = DummyParser(project_name, test_input_dir, test_output_dir)
+    parser = DummyParser(project_name, test_raw_data_dir, test_output_dir)
     parser.parse()
 
     parsed_file = test_output_dir / f'{project_name}/parsed_data.json'
@@ -86,16 +86,16 @@ def test_dummy_parser(setup_and_cleanup):
 
 
 def test_parse(setup_and_cleanup):
-    test_input_dir, test_output_dir = setup_and_cleanup
+    test_raw_data_dir, test_output_dir = setup_and_cleanup
     sample_block = {"number": "682736", "timestamp": "2021-05-09 11:12:32 UTC", "identifiers": "03f06a0a202f5669614254432f4d696e6564206279206a617669647361656964373037332f2cfabe6d6d6e43ef2e06f7137b897180388403ee5019b8ff0ca4a045ea3cd82e3e41620fe91000000000000000105462a20fc21591f70e691905660b0000", "reward_addresses": "18cBEMRxXHqzWWCxZNtU91F5sbUNKhL5PX"}
 
     project = 'sample_bitcoin'
     ledger_parser[project] = DefaultParser
 
     parsed_file = test_output_dir / f'{project}/parsed_data.json'
-    input_file = test_input_dir / f'{project}_raw_data.json'
+    input_file = test_raw_data_dir / f'{project}_raw_data.json'
 
-    parse(project, test_input_dir, test_output_dir)
+    parse(project, test_raw_data_dir, test_output_dir)
     with open(parsed_file) as f:
         test_data = json.load(f)
         for item in test_data:
@@ -108,14 +108,14 @@ def test_parse(setup_and_cleanup):
     with open(input_file, 'w') as f:
         f.write(sample_data)
 
-    parse(project, test_input_dir, test_output_dir)
+    parse(project, test_raw_data_dir, test_output_dir)
     with open(parsed_file) as f:
         test_data = json.load(f)
         for item in test_data:
             if item['number'] == '682736':
                 assert item['reward_addresses'] == sample_block['reward_addresses']
 
-    parse(project, test_input_dir, test_output_dir, True)
+    parse(project, test_raw_data_dir, test_output_dir, True)
     with open(parsed_file) as f:
         test_data = json.load(f)
         for item in test_data:
diff --git a/tests/test_4_mappings.py b/tests/test_4_mappings.py
@@ -12,7 +12,7 @@
 from src.mappings.ethereum_mapping import EthereumMapping
 from src.mappings.cardano_mapping import CardanoMapping
 from src.mappings.tezos_mapping import TezosMapping
-from src.helper import INPUT_DIR, OUTPUT_DIR
+from src.helper import RAW_DATA_DIR, OUTPUT_DIR
 
 
 @pytest.fixture
@@ -32,15 +32,15 @@ def setup_and_cleanup():
     ledger_mapping['sample_tezos'] = TezosMapping
     ledger_parser['sample_tezos'] = DummyParser
     mapping_info_dir = pathlib.Path(__file__).resolve().parent.parent / 'mapping_information'  # todo better to have separate helper files for the tests s.t. the tests don't break just because the info changes
-    test_input_dir = INPUT_DIR
+    test_raw_data_dir = RAW_DATA_DIR
     test_output_dir = OUTPUT_DIR / "test_output"
-    yield mapping_info_dir, test_input_dir, test_output_dir
+    yield mapping_info_dir, test_raw_data_dir, test_output_dir
     # Clean up
     shutil.rmtree(test_output_dir)
 
 
 def test_map(setup_and_cleanup):
-    pool_info_dir, test_input_dir, test_output_dir = setup_and_cleanup
+    pool_info_dir, test_raw_data_dir, test_output_dir = setup_and_cleanup
     project = 'sample_bitcoin'
 
     try:
@@ -64,7 +64,7 @@ def test_map(setup_and_cleanup):
     timeframes = ['2010', '2018-02', '2018-03']
     force_map = True
 
-    parse(project, test_input_dir, test_output_dir)
+    parse(project, test_raw_data_dir, test_output_dir)
     apply_mapping(project, timeframes, test_output_dir, force_map)
 
     output_file = test_output_dir / project / f'{timeframes[0]}.csv'
@@ -88,7 +88,7 @@ def test_map(setup_and_cleanup):
 
 
 def test_bitcoin_mapping(setup_and_cleanup):
-    pool_info_dir, test_input_dir, test_output_dir = setup_and_cleanup
+    pool_info_dir, test_raw_data_dir, test_output_dir = setup_and_cleanup
     project = 'sample_bitcoin'
 
     try:
@@ -121,7 +121,7 @@ def test_bitcoin_mapping(setup_and_cleanup):
     timeframes = ['2018-02']
     force_map = True
 
-    parse(project, test_input_dir, test_output_dir)
+    parse(project, test_raw_data_dir, test_output_dir)
     apply_mapping(project, timeframes, test_output_dir, force_map)
 
     expected_output = [
@@ -152,7 +152,7 @@ def test_bitcoin_mapping(setup_and_cleanup):
     timeframes = ['2020']
     force_map = True
 
-    parse(project, test_input_dir, test_output_dir)
+    parse(project, test_raw_data_dir, test_output_dir)
     apply_mapping(project, timeframes, test_output_dir, force_map)
 
     expected_output = [
@@ -181,7 +181,7 @@ def test_bitcoin_mapping(setup_and_cleanup):
 
 
 def test_ethereum_mapping(setup_and_cleanup):
-    pool_info_dir, test_input_dir, test_output_dir = setup_and_cleanup
+    pool_info_dir, test_raw_data_dir, test_output_dir = setup_and_cleanup
     project = 'sample_ethereum'
 
     try:
@@ -223,7 +223,7 @@ def test_ethereum_mapping(setup_and_cleanup):
     timeframes = ['2020-11']
     force_map = True
 
-    parse(project, test_input_dir, test_output_dir)
+    parse(project, test_raw_data_dir, test_output_dir)
     apply_mapping(project, timeframes, test_output_dir, force_map)
 
     expected_output = [
@@ -246,7 +246,7 @@ def test_ethereum_mapping(setup_and_cleanup):
     timeframes = ['2023']
     force_map = True
 
-    parse(project, test_input_dir, test_output_dir)
+    parse(project, test_raw_data_dir, test_output_dir)
     apply_mapping(project, timeframes, test_output_dir, force_map)
 
     expected_output = [
@@ -274,7 +274,7 @@ def test_ethereum_mapping(setup_and_cleanup):
 
 
 def test_cardano_mapping(setup_and_cleanup):
-    pool_info_dir, test_input_dir, test_output_dir = setup_and_cleanup
+    pool_info_dir, test_raw_data_dir, test_output_dir = setup_and_cleanup
     project = 'sample_cardano'
 
     try:
@@ -301,7 +301,7 @@ def test_cardano_mapping(setup_and_cleanup):
     timeframes = ['2020-12']
     force_map = True
 
-    parse(project, test_input_dir, test_output_dir)
+    parse(project, test_raw_data_dir, test_output_dir)
     apply_mapping(project, timeframes, test_output_dir, force_map)
 
     expected_output = [
@@ -338,7 +338,7 @@ def test_cardano_mapping(setup_and_cleanup):
 
 
 def test_tezos_mapping(setup_and_cleanup):
-    pool_info_dir, test_input_dir, test_output_dir = setup_and_cleanup
+    pool_info_dir, test_raw_data_dir, test_output_dir = setup_and_cleanup
     project = 'sample_tezos'
 
     try:
@@ -374,7 +374,7 @@ def test_tezos_mapping(setup_and_cleanup):
     timeframes = ['2021-08']
     force_map = True
 
-    parse(project, test_input_dir, test_output_dir)
+    parse(project, test_raw_data_dir, test_output_dir)
     apply_mapping(project, timeframes, test_output_dir, force_map)
 
     expected_output = [
@@ -398,7 +398,7 @@ def test_tezos_mapping(setup_and_cleanup):
     timeframes = ['2018']
     force_map = True
 
-    parse(project, test_input_dir, test_output_dir)
+    parse(project, test_raw_data_dir, test_output_dir)
     apply_mapping(project, timeframes, test_output_dir, force_map)
 
     expected_output = [
