Merge pull request #726 from data-integrations/ch-hidden-datasets

add option to show hidden datasets

Author: yaojiefeng

docs/BigQuery-connector.md

@@ -26,4 +26,6 @@ it can automatically be read from the environment. In other environments, the se
   auto-detect' when running on a Dataproc cluster. When running on other clusters, the file must be present on every
   node in the cluster.
 
-* **JSON**: Contents of the service account JSON file.
+* **JSON**: Contents of the service account JSON file.
+
+**Show Hidden Datasets:** Whether to show hidden datasets.

src/main/java/io/cdap/plugin/gcp/bigquery/connector/BigQueryConnector.java

@@ -195,7 +195,8 @@ private BrowseDetail listTables(BigQuery bigQuery, String datasetProject, String
   }
 
   private BrowseDetail listDatasets(BigQuery bigQuery, Integer limit) {
-    Page<Dataset> datasetPage = bigQuery.listDatasets(BigQuery.DatasetListOption.all());
+    Page<Dataset> datasetPage = config.showHiddenDatasets() ?
+                                  bigQuery.listDatasets(BigQuery.DatasetListOption.all()) : bigQuery.listDatasets();
     int countLimit = limit == null || limit <= 0 ? Integer.MAX_VALUE : limit;
     int count = 0;
     BrowseDetail.Builder browseDetailBuilder = BrowseDetail.builder();

src/main/java/io/cdap/plugin/gcp/bigquery/connector/BigQueryConnectorSpecificConfig.java

@@ -29,11 +29,20 @@ public class BigQueryConnectorSpecificConfig extends BigQueryConnectorConfig {
   @Description("The root dataset to browse when a root path is given")
   protected String rootDataset;
 
+  @Nullable
+  @Description("Whether to show hidden datasets")
+  protected Boolean showHiddenDatasets;
+
   public BigQueryConnectorSpecificConfig(@Nullable String project, @Nullable String datasetProject,
                                          @Nullable String serviceAccountType, @Nullable String serviceFilePath,
                                          @Nullable String serviceAccountJson,
                                          @Nullable String rootDataset) {
     super(project, datasetProject, serviceAccountType, serviceFilePath, serviceAccountJson);
     this.rootDataset = rootDataset;
+    this.showHiddenDatasets = false;
+  }
+
+  public boolean showHiddenDatasets() {
+    return showHiddenDatasets == null ? false : showHiddenDatasets;
   }
 }

widgets/BigQuery-connector.json

@@ -26,6 +26,22 @@
         {
           "widget-type": "hidden",
           "name": "rootDataset"
+        },
+        {
+          "widget-type": "toggle",
+          "label": "Show Hidden Datasets",
+          "name": "showHiddenDatasets",
+          "widget-attributes": {
+            "on": {
+              "value": "true",
+              "label": "YES"
+            },
+            "off": {
+              "value": "false",
+              "label": "NO"
+            },
+            "default": "false"
+          }
         }
       ]
     },