[ui-storagebrowser] adds and reads configs from .ini files (#3922)

* [ui-storagebrowser] adds and reads configs from .ini files

* removes show download button from global constants

* moves the storage browser config to separate group

* adds storage_browser key in config

Author: ramprasadagarwal

desktop/core/src/desktop/api2.py

@@ -78,8 +78,9 @@
   uuid_default,
 )
 from desktop.views import get_banner_message, serve_403_error
-from filebrowser.conf import RESTRICT_FILE_EXTENSIONS
+from filebrowser.conf import CONCURRENT_MAX_CONNECTIONS, FILE_UPLOAD_CHUNK_SIZE, RESTRICT_FILE_EXTENSIONS, SHOW_DOWNLOAD_BUTTON
 from filebrowser.tasks import check_disk_usage_and_clean_task, document_cleanup_task
+from filebrowser.views import MAX_FILEEDITOR_SIZE
 from hadoop.cluster import is_yarn
 from metadata.catalog_api import (
   _highlight,
@@ -124,10 +125,14 @@ def get_config(request):
   config = get_cluster_config(request.user)
   config['hue_config']['is_admin'] = is_admin(request.user)
   config['hue_config']['is_yarn_enabled'] = is_yarn()
-  config['hue_config']['enable_new_storage_browser'] = ENABLE_NEW_STORAGE_BROWSER.get()
-  config['hue_config']['enable_chunked_file_uploader'] = ENABLE_CHUNKED_FILE_UPLOADER.get()
   config['hue_config']['enable_task_server'] = TASK_SERVER_V2.ENABLED.get()
-  config['hue_config']['restrict_file_extensions'] = RESTRICT_FILE_EXTENSIONS.get()
+  config['storage_browser']['enable_chunked_file_upload'] = ENABLE_CHUNKED_FILE_UPLOADER.get()
+  config['storage_browser']['enable_new_storage_browser'] = ENABLE_NEW_STORAGE_BROWSER.get()
+  config['storage_browser']['restrict_file_extensions'] = RESTRICT_FILE_EXTENSIONS.get()
+  config['storage_browser']['concurrent_max_connection'] = CONCURRENT_MAX_CONNECTIONS.get()
+  config['storage_browser']['file_upload_chunk_size'] = FILE_UPLOAD_CHUNK_SIZE.get()
+  config['storage_browser']['enable_file_download_button'] = SHOW_DOWNLOAD_BUTTON.get()
+  config['storage_browser']['max_file_editor_size'] = MAX_FILEEDITOR_SIZE
   config['clusters'] = list(get_clusters(request.user).values())
   config['documents'] = {
     'types': list(Document2.objects.documents(user=request.user).order_by().values_list('type', flat=True).distinct())

desktop/core/src/desktop/js/apps/storageBrowser/StorageFilePage/StorageFilePage.test.tsx

@@ -22,7 +22,6 @@ import '@testing-library/jest-dom';
 import userEvent from '@testing-library/user-event';
 import { DOWNLOAD_API_URL } from '../../../reactComponents/FileChooser/api';
 import huePubSub from '../../../utils/huePubSub';
-import { hueWindow } from '../../../types/types';
 
 jest.mock('../../../utils/dateTimeUtils', () => ({
   ...jest.requireActual('../../../utils/dateTimeUtils'),
@@ -50,6 +49,16 @@ jest.mock('../../../utils/hooks/useLoadData', () => {
   }));
 });
 
+const mockConfig = {
+  storage_browser: {
+    enable_file_download_button: true,
+    max_file_editor_size: 1000000000
+  }
+};
+jest.mock('../../../config/hueConfig', () => ({
+  getLastKnownConfig: () => mockConfig
+}));
+
 const mockFileStats: FileStats = {
   path: '/path/to/file.txt',
   size: 123456,
@@ -67,19 +76,6 @@ const mockFileName = 'file.txt';
 const mockReload = jest.fn();
 
 describe('StorageFilePage', () => {
-  let oldShowDownloadButton: boolean;
-  let oldMaxFileEditorSize: number;
-  beforeEach(() => {
-    oldShowDownloadButton = (window as hueWindow).SHOW_DOWNLOAD_BUTTON as boolean;
-    oldMaxFileEditorSize = (window as hueWindow).MAX_FILEEDITOR_SIZE as number;
-    (window as hueWindow).SHOW_DOWNLOAD_BUTTON = true;
-    (window as hueWindow).MAX_FILEEDITOR_SIZE = 1000000000;
-  });
-  afterAll(() => {
-    (window as hueWindow).SHOW_DOWNLOAD_BUTTON = oldShowDownloadButton;
-    (window as hueWindow).MAX_FILEEDITOR_SIZE = oldMaxFileEditorSize;
-  });
-
   it('should render file metadata and content', () => {
     render(
       <StorageFilePage fileName={mockFileName} fileStats={mockFileStats} onReload={mockReload} />
@@ -224,7 +220,7 @@ describe('StorageFilePage', () => {
   });
 
   it('should not render the download button when show_download_button is false', () => {
-    (window as hueWindow).SHOW_DOWNLOAD_BUTTON = false;
+    mockConfig.storage_browser.enable_file_download_button = false;
 
     render(
       <StorageFilePage fileName={mockFileName} fileStats={mockFileStats} onReload={mockReload} />

desktop/core/src/desktop/js/apps/storageBrowser/StorageFilePage/StorageFilePage.tsx

@@ -15,7 +15,6 @@
 // limitations under the License.
 
 import React, { useMemo, useState } from 'react';
-import { hueWindow } from 'types/types';
 import {
   BrowserViewType,
   FilePreview,
@@ -39,6 +38,7 @@ import {
 } from '../../../utils/constants/storageBrowser';
 import { Spin } from 'antd';
 import useLoadData from '../../../utils/hooks/useLoadData';
+import { getLastKnownConfig } from '../../../config/hueConfig';
 
 interface StorageFilePageProps {
   onReload: () => void;
@@ -47,7 +47,9 @@ interface StorageFilePageProps {
 }
 
 const StorageFilePage = ({ fileName, fileStats, onReload }: StorageFilePageProps): JSX.Element => {
+  const config = getLastKnownConfig();
   const { t } = i18nReact.useTranslation();
+
   const [isEditing, setIsEditing] = useState<boolean>(false);
   const [fileContent, setFileContent] = useState<FilePreview['contents']>();
 
@@ -114,7 +116,8 @@ const StorageFilePage = ({ fileName, fileStats, onReload }: StorageFilePageProps
 
   const isEditingEnabled =
     !isEditing &&
-    (window as hueWindow).MAX_FILEEDITOR_SIZE > fileStats.size &&
+    config?.storage_browser.max_file_editor_size &&
+    config?.storage_browser.max_file_editor_size > fileStats.size &&
     EDITABLE_FILE_FORMATS[fileType] &&
     fileData?.compression?.toLocaleLowerCase() === 'none';
 
@@ -167,7 +170,7 @@ const StorageFilePage = ({ fileName, fileStats, onReload }: StorageFilePageProps
                   </Button>
                 </>
               )}
-              {(window as hueWindow).SHOW_DOWNLOAD_BUTTON && (
+              {config?.storage_browser.enable_file_download_button && (
                 <a href={`${DOWNLOAD_API_URL}${fileStats.path}`}>
                   <PrimaryButton
                     data-testid="preview--download--button"

desktop/core/src/desktop/js/config/types.ts

@@ -60,6 +60,15 @@ export enum AppType {
   sdkapps = 'sdkapps'
 }
 
+interface StorageBrowserConfig {
+  concurrent_max_connection: number;
+  enable_chunked_file_uploader: boolean;
+  enable_file_download_button: boolean;
+  enable_new_storage_browser: boolean;
+  file_upload_chunk_size: number;
+  max_file_editor_size: number;
+}
+
 export interface HueConfig extends GenericApiResponse {
   app_config: {
     [AppType.browser]?: AppConfig<BrowserInterpreter>;
@@ -83,7 +92,11 @@ export interface HueConfig extends GenericApiResponse {
   hue_config: {
     enable_sharing: boolean;
     collect_usage: boolean;
+    enable_task_server: boolean;
+    is_admin: boolean;
+    is_yarn_enabled: boolean;
   };
+  storage_browser: StorageBrowserConfig;
   hue_version?: string;
   img_version?: string;
   vw_name?: string;

desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.tsx

@@ -27,7 +27,11 @@ import StatusStoppedIcon from '@cloudera/cuix-core/icons/react/StatusStoppedIcon
 import StatusErrorIcon from '@cloudera/cuix-core/icons/react/StatusErrorIcon';
 import { UploadItem } from '../../utils/hooks/useFileUpload/util';
 import useFileUpload from '../../utils/hooks/useFileUpload/useFileUpload';
-import { FileUploadStatus } from '../../utils/constants/storageBrowser';
+import {
+  DEFAULT_ENABLE_CHUNK_UPLOAD,
+  FileUploadStatus
+} from '../../utils/constants/storageBrowser';
+import { getLastKnownConfig } from '../../config/hueConfig';
 
 interface FileUploadQueueProps {
   filesQueue: UploadItem[];
@@ -47,11 +51,14 @@ const sortOrder = [
 }, {});
 
 const FileUploadQueue: React.FC<FileUploadQueueProps> = ({ filesQueue, onClose, onComplete }) => {
+  const config = getLastKnownConfig();
+  const isChunkUpload =
+    config?.storage_browser.enable_chunked_file_uploader ?? DEFAULT_ENABLE_CHUNK_UPLOAD;
   const { t } = i18nReact.useTranslation();
   const [isExpanded, setIsExpanded] = useState(true);
 
   const { uploadQueue, onCancel } = useFileUpload(filesQueue, {
-    isChunkUpload: true,
+    isChunkUpload,
     onComplete
   });
 

desktop/core/src/desktop/js/types/types.ts

@@ -59,8 +59,6 @@ export interface hueWindow {
   HUE_LANG?: string;
   HUE_VERSION?: string;
   LOGGED_USERNAME?: string;
-  MAX_FILEEDITOR_SIZE?: number;
-  SHOW_DOWNLOAD_BUTTON?: boolean;
   SQL_ANALYZER_MODE?: string;
   USER_IS_ADMIN?: boolean;
   USER_IS_HUE_ADMIN?: boolean;

desktop/core/src/desktop/js/utils/constants/storageBrowser.ts

@@ -16,8 +16,8 @@
 
 export const DEFAULT_PAGE_SIZE = 50;
 export const DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024; // 5 MiB
-export const DEFAULT_CONCURRENT_UPLOAD = 3;
-export const DEFAULT_CONCURRENT_CHUNK_UPLOAD = 3;
+export const DEFAULT_CONCURRENT_MAX_CONNECTIONS = 3;
+export const DEFAULT_ENABLE_CHUNK_UPLOAD = false;
 
 export enum SupportedFileTypes {
   IMAGE = 'image',

desktop/core/src/desktop/js/utils/hooks/useFileUpload/useChunkUpload.ts

@@ -15,11 +15,12 @@
 // limitations under the License.
 
 import { useEffect, useState } from 'react';
+import { getLastKnownConfig } from '../../../config/hueConfig';
 import useSaveData from '../useSaveData';
 import useQueueProcessor from '../useQueueProcessor';
 import {
   DEFAULT_CHUNK_SIZE,
-  DEFAULT_CONCURRENT_CHUNK_UPLOAD,
+  DEFAULT_CONCURRENT_MAX_CONNECTIONS,
   FileUploadStatus
 } from '../../constants/storageBrowser';
 import useLoadData from '../useLoadData';
@@ -42,14 +43,18 @@ interface UseUploadQueueResponse {
 }
 
 interface ChunkUploadOptions {
+  concurrentProcess?: number;
   onStatusUpdate: (item: UploadItem, newStatus: FileUploadStatus) => void;
   onComplete: () => void;
 }
 
 const useChunkUpload = ({
+  concurrentProcess = DEFAULT_CONCURRENT_MAX_CONNECTIONS,
   onStatusUpdate,
   onComplete
 }: ChunkUploadOptions): UseUploadQueueResponse => {
+  const config = getLastKnownConfig();
+  const chunkSize = config?.storage_browser?.file_upload_chunk_size ?? DEFAULT_CHUNK_SIZE;
   const [pendingItems, setPendingItems] = useState<UploadItem[]>([]);
 
   const { save } = useSaveData(undefined, {
@@ -99,30 +104,30 @@ const useChunkUpload = ({
   };
 
   const onChunksUploadComplete = async (item: UploadItem) => {
-    const { url, payload } = getChunksCompletePayload(item);
+    const { url, payload } = getChunksCompletePayload(item, chunkSize);
     return save(payload, {
       url,
       onSuccess: () => addItemToPending(item)
     });
   };
 
   const uploadChunk = async (chunkItem: UploadChunkItem) => {
-    const { url, payload } = getChunkItemPayload(chunkItem);
+    const { url, payload } = getChunkItemPayload(chunkItem, chunkSize);
     return save(payload, { url });
   };
 
   const { enqueueAsync } = useQueueProcessor<UploadChunkItem>(uploadChunk, {
-    concurrentProcess: DEFAULT_CONCURRENT_CHUNK_UPLOAD
+    concurrentProcess
   });
 
   const uploadItemInChunks = async (item: UploadItem) => {
-    const chunks = createChunks(item, DEFAULT_CHUNK_SIZE);
+    const chunks = createChunks(item, chunkSize);
     await enqueueAsync(chunks);
     return onChunksUploadComplete(item);
   };
 
   const uploadItemInSingleChunk = (item: UploadItem) => {
-    const { url, payload } = getChunkSinglePayload(item);
+    const { url, payload } = getChunkSinglePayload(item, chunkSize);
     return save(payload, {
       url,
       onSuccess: () => addItemToPending(item)
@@ -131,7 +136,7 @@ const useChunkUpload = ({
 
   const uploadItem = async (item: UploadItem) => {
     onStatusUpdate(item, FileUploadStatus.Uploading);
-    const chunks = getTotalChunk(item.file.size, DEFAULT_CHUNK_SIZE);
+    const chunks = getTotalChunk(item.file.size, chunkSize);
     if (chunks === 1) {
       return uploadItemInSingleChunk(item);
     }

desktop/core/src/desktop/js/utils/hooks/useFileUpload/useFileUpload.ts

@@ -18,7 +18,11 @@ import { useCallback, useEffect, useState } from 'react';
 import useRegularUpload from './useRegularUpload';
 import useChunkUpload from './useChunkUpload';
 import { getNewFileItems, UploadItem } from './util';
-import { FileUploadStatus } from '../../constants/storageBrowser';
+import {
+  DEFAULT_CONCURRENT_MAX_CONNECTIONS,
+  FileUploadStatus
+} from '../../constants/storageBrowser';
+import { getLastKnownConfig } from '../../../config/hueConfig';
 
 interface UseUploadQueueResponse {
   uploadQueue: UploadItem[];
@@ -35,6 +39,10 @@ const useFileUpload = (
   filesQueue: UploadItem[],
   { isChunkUpload = false, onComplete }: UploadQueueOptions
 ): UseUploadQueueResponse => {
+  const config = getLastKnownConfig();
+  const concurrentProcess =
+    config?.storage_browser.concurrent_max_connection ?? DEFAULT_CONCURRENT_MAX_CONNECTIONS;
+
   const [uploadQueue, setUploadQueue] = useState<UploadItem[]>([]);
 
   const onStatusUpdate = (item: UploadItem, newStatus: FileUploadStatus) =>
@@ -52,6 +60,7 @@ const useFileUpload = (
     removeFile: removeFromChunkUpload,
     isLoading: isChunkLoading
   } = useChunkUpload({
+    concurrentProcess,
     onStatusUpdate,
     onComplete
   });
@@ -61,6 +70,7 @@ const useFileUpload = (
     removeFile: removeFromRegularUpload,
     isLoading: isNonChunkLoading
   } = useRegularUpload({
+    concurrentProcess,
     onStatusUpdate,
     onComplete
   });

desktop/core/src/desktop/js/utils/hooks/useFileUpload/useRegularUpload.ts

@@ -16,7 +16,10 @@
 
 import useQueueProcessor from '../useQueueProcessor';
 import { UPLOAD_FILE_URL } from '../../../reactComponents/FileChooser/api';
-import { DEFAULT_CONCURRENT_UPLOAD, FileUploadStatus } from '../../constants/storageBrowser';
+import {
+  DEFAULT_CONCURRENT_MAX_CONNECTIONS,
+  FileUploadStatus
+} from '../../constants/storageBrowser';
 import useSaveData from '../useSaveData';
 import { UploadItem } from './util';
 
@@ -27,11 +30,13 @@ interface UseUploadQueueResponse {
 }
 
 interface UploadQueueOptions {
+  concurrentProcess?: number;
   onStatusUpdate: (item: UploadItem, newStatus: FileUploadStatus) => void;
   onComplete: () => void;
 }
 
 const useRegularUpload = ({
+  concurrentProcess = DEFAULT_CONCURRENT_MAX_CONNECTIONS,
   onStatusUpdate,
   onComplete
 }: UploadQueueOptions): UseUploadQueueResponse => {
@@ -68,7 +73,7 @@ const useRegularUpload = ({
     dequeue: removeFile,
     isLoading
   } = useQueueProcessor<UploadItem>(processUploadItem, {
-    concurrentProcess: DEFAULT_CONCURRENT_UPLOAD,
+    concurrentProcess,
     onSuccess: onComplete
   });