fix: preserve gsheet column order during sync (#574)

Author: stevenle

.changeset/twenty-bulldogs-begin.md

@@ -0,0 +1,5 @@
+---
+'@blinkk/root-cms': patch
+---
+
+fix: preserve column headers in data sources (#574)

packages/root-cms/core/client.ts

@@ -73,6 +73,8 @@ export interface DataSource {
 export interface DataSourceData<T = any> {
   dataSource: DataSource;
   data: T;
+  /** Optional list of column headers (for gsheet sources). */
+  headers?: string[];
 }
 
 export type DataSourceMode = 'draft' | 'published';
@@ -822,6 +824,7 @@ export class RootCMSClient {
     batch.set(dataDocRefPublished, {
       dataSource: updatedDataSource,
       data: dataRes?.data || null,
+      ...(dataRes?.headers ? {headers: dataRes.headers} : {}),
     });
     batch.update(dataDocRefDraft, {
       dataSource: updatedDataSource,

packages/root-cms/ui/pages/DataSourcePage/DataSourcePage.tsx

@@ -137,7 +137,7 @@ DataSourcePage.DataSection = (props: {
   dataSource: DataSource;
   data: DataSourceData;
 }) => {
-  const {data} = props.data || {};
+  const {data, headers: storedHeaders} = props.data || {};
   const dataSource = props.dataSource;
 
   if (!data) {
@@ -146,22 +146,25 @@ DataSourcePage.DataSection = (props: {
 
   const dataFormat = dataSource.dataFormat || 'map';
   if (dataSource.type === 'gsheet') {
-    let headers: string[] | undefined = undefined;
+    let headers: string[] | undefined = storedHeaders;
     let rows: any[] = [];
     if (dataFormat === 'array') {
       rows = data as string[][];
     } else if (dataFormat === 'map') {
-      // Reformat Array<Record<string, string>> to string[][].
-      const headersSet = new Set<string>();
       const items = data as any[];
-      items.forEach((item) => {
-        for (const key in item) {
-          if (key) {
-            headersSet.add(key);
+      if (!headers) {
+        // Reformat Array<Record<string, string>> to string[][]. Preserve key order
+        // by iterating through object keys as encountered.
+        const headersSet = new Set<string>();
+        items.forEach((item) => {
+          for (const key in item) {
+            if (key) {
+              headersSet.add(key);
+            }
           }
-        }
-      });
-      headers = Array.from(headersSet);
+        });
+        headers = Array.from(headersSet);
+      }
       items.forEach((item) => {
         rows.push(headers!.map((header) => item[header] || ''));
       });

packages/root-cms/ui/utils/data-source.ts

@@ -49,6 +49,8 @@ export interface DataSource {
 export interface DataSourceData<T = any> {
   dataSource: DataSource;
   data: T;
+  /** Optional list of column headers (for gsheet sources). */
+  headers?: string[];
 }
 
 export async function addDataSource(
@@ -157,7 +159,7 @@ export async function syncDataSource(id: string) {
       throw new Error(`sync failed: ${err}`);
     }
   } else {
-    const data = await fetchData(dataSource);
+    const {data, headers} = await fetchData(dataSource);
 
     const projectId = window.__ROOT_CTX.rootConfig.projectId;
     const db = window.firebase.db;
@@ -183,6 +185,7 @@ export async function syncDataSource(id: string) {
     batch.set(dataDocRef, {
       dataSource: updatedDataSource,
       data: data,
+      ...(headers ? {headers} : {}),
     });
     batch.update(dataSourceDocRef, {
       syncedAt: Timestamp.now(),
@@ -235,6 +238,7 @@ export async function publishDataSource(id: string) {
   batch.set(dataDocRefPublished, {
     dataSource: updatedDataSource,
     data: dataRes?.data || null,
+    ...(dataRes?.headers ? {headers: dataRes.headers} : {}),
   });
   batch.update(dataDocRefDraft, {
     dataSource: updatedDataSource,
@@ -280,17 +284,23 @@ export async function deleteDataSource(id: string) {
   logAction('datasource.delete', {metadata: {datasourceId: id}});
 }
 
-async function fetchData(dataSource: DataSource) {
+interface FetchedData {
+  data: any;
+  headers?: string[];
+}
+
+async function fetchData(dataSource: DataSource): Promise<FetchedData> {
   if (dataSource.type === 'http') {
-    return await fetchHttpData(dataSource);
+    const data = await fetchHttpData(dataSource);
+    return {data};
   }
   if (dataSource.type === 'gsheet') {
     return await fetchGsheetData(dataSource);
   }
   throw new Error(`unsupported data source: ${dataSource.type}`);
 }
 
-async function fetchGsheetData(dataSource: DataSource) {
+async function fetchGsheetData(dataSource: DataSource): Promise<FetchedData> {
   const gsheetId = parseSpreadsheetUrl(dataSource.url);
   if (!gsheetId?.spreadsheetId) {
     throw new Error(`failed to parse google sheet url: ${dataSource.url}`);
@@ -303,10 +313,21 @@ async function fetchGsheetData(dataSource: DataSource) {
   }
 
   const dataFormat = dataSource.dataFormat || 'map';
+  const [headers, rows] = await gsheet.getValues();
   if (dataFormat === 'array') {
-    return await gsheet.getValues();
+    return {data: [headers, rows], headers};
   }
-  return await gsheet.getValuesMap();
+  const mapData = rows.map((row) => {
+    const item: Record<string, string> = {};
+    row.forEach((val, i) => {
+      const key = headers[i];
+      if (key) {
+        item[key] = String(val || '');
+      }
+    });
+    return item;
+  });
+  return {data: mapData, headers};
 }
 
 async function fetchHttpData(dataSource: DataSource) {