petfinder/main.py at main · arun2596/petfinder · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
import os

import yaml
import pandas as pd
from finetuning import PawpularityModel

from shutil import copyfile

from datetime import date

from utils.train_handler import TrainHandler
from utils.utils import set_seed, create_folds


def parse_config(config):
    # used to check if config makes sense
    # make later versions of config backwards comaptible

    today = date.today()
    folder_prefix = str(today.strftime("%Y%m%d"))

    folder_list = os.listdir('model_output/finetuning')
    folder_suffixes = [x.split('-')[-1] for x in folder_list]

    if '-' in config['global']['folder_suffix']:
        raise Exception('suffix connot contain "-"')
    elif config['global']['folder_suffix'] in folder_suffixes:
        raise Exception('folder with the suffix already exists')
    else:
        os.makedirs(os.path.join('model_output', 'finetuning', folder_prefix + '-' + config['global']['folder_suffix']))
        config['global']['folder_name'] = folder_prefix + '-' + config['global']['folder_suffix']
        os.makedirs(os.path.join('model_output', 'finetuning', config['global']['folder_name'], 'head_only_model'))
        os.makedirs(os.path.join('model_output', 'finetuning', config['global']['folder_name'], 'full_model'))

    copyfile('config.yaml', os.path.join(config['global']['folder_name']))

    return config


def run_finetuning():
    with open("config.yaml", 'r') as stream:
        ft_config = yaml.safe_load(stream)

    ft_config = parse_config(ft_config)

    set_seed(ft_config['global']['seed'])

    train = pd.read_csv('data/raw/train.csv')
    train = create_folds(train, ft_config['global']['num_folds'], ft_config['global']['seed'])

    ft_handler = TrainHandler(model_class=PawpularityModel, train=train, config=ft_config)
    ft_handler.run()