add script to load data in bigquery and databricks

Author: lio-p

blog-examples/Bench2Cost/bigquery/clickbench/bigquery_extended/README.md

@@ -99,3 +99,9 @@ This shows the dataset compresses roughly **10× smaller** on disk compared to t
 |-----------------|------------------------------|--------------------------------|
 | Active storage  | 94.4 GiB                     | 8.8 GiB                        |
 | Long-term (>90d)| 0 GiB                        | 0 GiB                          |
+
+## Inserting large number of data
+
+BigQuery’s on-demand model charges based on the amount of data scanned. If you try to scale a 1-billion-row table to 100 billion rows by running `insert into hits_100b select * from hits_1b` a hundred times, the same data is scanned repeatedly, which can become expensive.
+
+You can avoid this by duplicating the rows during a single scan. One option is to use `CROSS JOIN UNNEST(GENERATE_ARRAY(1, 20))`, which multiplies each row without rereading the source table. The load_data.sql script shows how to apply this approach.

blog-examples/Bench2Cost/bigquery/clickbench/bigquery_extended/load_data.sql

@@ -0,0 +1,5 @@
+-- Insert 20 billion rows into hits_100B table by cross joining hits_1B table
+INSERT INTO `test.hits_100B`
+SELECT t.*
+FROM `test.hits_1B` AS t
+CROSS JOIN UNNEST(GENERATE_ARRAY(1, 20));

blog-examples/Bench2Cost/databricks/clickbench/load_data.sql

@@ -0,0 +1,104 @@
+USE CATALOG bench2cost;
+USE SCHEMA hits;
+
+CREATE OR REPLACE TABLE bench2cost.hits.100m_clustered
+CLUSTER BY (CounterID, EventDate, UserID, EventTime)
+AS
+SELECT
+  CAST(WatchID AS BIGINT)             AS WatchID,
+  CAST(JavaEnable AS SMALLINT)        AS JavaEnable,
+  Title,
+  CAST(GoodEvent AS SMALLINT)         AS GoodEvent,
+  CAST(EventTime AS TIMESTAMP)        AS EventTime,
+  CAST(EventDate AS DATE)             AS EventDate,
+  CAST(CounterID AS INT)              AS CounterID,
+  CAST(ClientIP AS INT)               AS ClientIP,
+  CAST(RegionID AS INT)               AS RegionID,
+  CAST(UserID AS BIGINT)              AS UserID,
+  CAST(CounterClass AS SMALLINT)      AS CounterClass,
+  CAST(OS AS SMALLINT)                AS OS,
+  CAST(UserAgent AS SMALLINT)         AS UserAgent,
+  URL,
+  Referer,
+  CAST(IsRefresh AS SMALLINT)         AS IsRefresh,
+  CAST(RefererCategoryID AS SMALLINT) AS RefererCategoryID,
+  CAST(RefererRegionID AS INT)        AS RefererRegionID,
+  CAST(URLCategoryID AS SMALLINT)     AS URLCategoryID,
+  CAST(URLRegionID AS INT)            AS URLRegionID,
+  CAST(ResolutionWidth AS SMALLINT)   AS ResolutionWidth,
+  CAST(ResolutionHeight AS SMALLINT)  AS ResolutionHeight,
+  CAST(ResolutionDepth AS SMALLINT)   AS ResolutionDepth,
+  CAST(FlashMajor AS SMALLINT)        AS FlashMajor,
+  CAST(FlashMinor AS SMALLINT)        AS FlashMinor,
+  FlashMinor2,
+  CAST(NetMajor AS SMALLINT)          AS NetMajor,
+  CAST(NetMinor AS SMALLINT)          AS NetMinor,
+  CAST(UserAgentMajor AS SMALLINT)    AS UserAgentMajor,
+  UserAgentMinor,
+  CAST(CookieEnable AS SMALLINT)      AS CookieEnable,
+  CAST(JavascriptEnable AS SMALLINT)  AS JavascriptEnable,
+  CAST(IsMobile AS SMALLINT)          AS IsMobile,
+  CAST(MobilePhone AS SMALLINT)       AS MobilePhone,
+  MobilePhoneModel,
+  Params,
+  CAST(IPNetworkID AS INT)            AS IPNetworkID,
+  CAST(TraficSourceID AS SMALLINT)    AS TraficSourceID,
+  CAST(SearchEngineID AS SMALLINT)    AS SearchEngineID,
+  SearchPhrase,
+  CAST(AdvEngineID AS SMALLINT)       AS AdvEngineID,
+  CAST(IsArtifical AS SMALLINT)       AS IsArtifical,
+  CAST(WindowClientWidth AS SMALLINT)  AS WindowClientWidth,
+  CAST(WindowClientHeight AS SMALLINT) AS WindowClientHeight,
+  CAST(ClientTimeZone AS SMALLINT)     AS ClientTimeZone,
+  CAST(ClientEventTime AS TIMESTAMP)   AS ClientEventTime,
+  CAST(SilverlightVersion1 AS SMALLINT) AS SilverlightVersion1,
+  CAST(SilverlightVersion2 AS SMALLINT) AS SilverlightVersion2,
+  CAST(SilverlightVersion3 AS INT)      AS SilverlightVersion3,
+  CAST(SilverlightVersion4 AS SMALLINT) AS SilverlightVersion4,
+  PageCharset,
+  CAST(CodeVersion AS INT)            AS CodeVersion,
+  CAST(IsLink AS SMALLINT)            AS IsLink,
+  CAST(IsDownload AS SMALLINT)        AS IsDownload,
+  CAST(IsNotBounce AS SMALLINT)       AS IsNotBounce,
+  CAST(FUniqID AS BIGINT)             AS FUniqID,
+  OriginalURL,
+  CAST(HID AS INT)                    AS HID,
+  CAST(IsOldCounter AS SMALLINT)      AS IsOldCounter,
+  CAST(IsEvent AS SMALLINT)           AS IsEvent,
+  CAST(IsParameter AS SMALLINT)       AS IsParameter,
+  CAST(DontCountHits AS SMALLINT)     AS DontCountHits,
+  CAST(WithHash AS SMALLINT)          AS WithHash,
+  HitColor,
+  CAST(LocalEventTime AS TIMESTAMP)   AS LocalEventTime,
+  CAST(Age AS SMALLINT)               AS Age,
+  CAST(Sex AS SMALLINT)               AS Sex,
+  CAST(Income AS SMALLINT)            AS Income,
+  CAST(Interests AS SMALLINT)         AS Interests,
+  CAST(Robotness AS SMALLINT)         AS Robotness,
+  CAST(RemoteIP AS INT)               AS RemoteIP,
+  CAST(WindowName AS INT)             AS WindowName,
+  CAST(OpenerName AS INT)             AS OpenerName,
+  CAST(HistoryLength AS SMALLINT)     AS HistoryLength,
+  BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction,
+  CAST(HTTPError AS SMALLINT)         AS HTTPError,
+  CAST(SendTiming AS INT)             AS SendTiming,
+  CAST(DNSTiming AS INT)              AS DNSTiming,
+  CAST(ConnectTiming AS INT)          AS ConnectTiming,
+  CAST(ResponseStartTiming AS INT)    AS ResponseStartTiming,
+  CAST(ResponseEndTiming AS INT)      AS ResponseEndTiming,
+  CAST(FetchTiming AS INT)            AS FetchTiming,
+  CAST(SocialSourceNetworkID AS SMALLINT) AS SocialSourceNetworkID,
+  SocialSourcePage,
+  CAST(ParamPrice AS BIGINT)          AS ParamPrice,
+  ParamOrderID, ParamCurrency,
+  CAST(ParamCurrencyID AS SMALLINT)   AS ParamCurrencyID,
+  OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID,
+  UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag,
+  CAST(HasGCLID AS SMALLINT)          AS HasGCLID,
+  CAST(RefererHash AS BIGINT)         AS RefererHash,
+  CAST(URLHash AS BIGINT)             AS URLHash,
+  CAST(CLID AS INT)                   AS CLID
+FROM read_files(
+  's3://hits-parquet-100m-sorted-zstd/*.parquet',
+  format => 'parquet'
+);