spotify-data-pipeline/big_query.py at main · Battery0/spotify-data-pipeline · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
from google.cloud import bigquery


def upload_metadata_to_big_query(data_transformed_for_big_query, table_id, table_schema):
    client = bigquery.Client()
    table_id = table_id
    schema = table_schema

    job_config = bigquery.LoadJobConfig(schema=schema, write_disposition="WRITE_TRUNCATE")
    job = client.load_table_from_dataframe(
        dataframe=data_transformed_for_big_query,
        destination=table_id,
        job_config=job_config
    )
    job.result()

    table = client.get_table(table_id)
    print("Loaded {} rows and {} columns to table: {} in BigQuery".format(table.num_rows, len(table.schema), table_id))


def database_table_schema():
    schema = [
        bigquery.SchemaField(
            name="track_isrc_number",
            field_type="STRING",
            mode="REQUIRED",
            description="Recording unique ISRC number"
        ),
        bigquery.SchemaField(
            name="track_title",
            field_type="STRING",
            mode="REQUIRED",
            description="Track name"
        ),
        bigquery.SchemaField(
            name="track_artists",
            field_type="STRING",
            mode="REQUIRED",
            description="artists of track"
        ),
        bigquery.SchemaField(
            name="album_title",
            field_type="STRING",
            mode="REQUIRED",
            description="Album name"
        ),
        bigquery.SchemaField(
            name="album_release_date",
            field_type="DATE",
            mode="REQUIRED",
            description="Date of album release"
        ),
        bigquery.SchemaField(
            name="album_type",
            field_type="STRING",
            mode="REQUIRED",
            description="Type of album (album, single, compilation, appeared on)"
        ),
        bigquery.SchemaField(
            name="album_label",
            field_type="STRING",
            mode="REQUIRED",
            description="Record label"
        ),
        bigquery.SchemaField(
            name="album_upc",
            field_type="INTEGER",
            mode="REQUIRED",
            description="Album unique UPC number"
        )
    ]

    return schema