Make study export clickhouse only (#11893)

* Make study export clickhouse only

Drop MySQL support

* Fix virtual study download

where sample id in (IDS) filtering

Author: forus

src/main/java/org/cbioportal/application/file/export/ExportConfig.java

@@ -84,6 +84,7 @@
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.Resource;
 import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
 
@@ -218,31 +219,25 @@ public VirtualStudyExportDecoratorService virtualStudyAwareExportService(
     return new VirtualStudyExportDecoratorService(virtualStudyService, exportService);
   }
 
+  @Bean("exportMapperLocations")
+  public Resource[] clickhouseExportMapperLocations(ApplicationContext applicationContext)
+      throws IOException {
+    return applicationContext.getResources("classpath:mappers/export/*.xml");
+  }
+
   @Bean("exportSqlSessionFactory")
   public SqlSessionFactoryBean exportSqlSessionFactory(
-      @Qualifier("exportDataSource") DataSource dataSource, ApplicationContext applicationContext)
-      throws IOException {
+      @Qualifier("exportDataSource") DataSource dataSource, Resource[] exportMapperLocations) {
     SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean();
     sessionFactory.setDataSource(dataSource);
-    sessionFactory.setMapperLocations(
-        applicationContext.getResources("classpath:mappers/export/*.xml"));
+    sessionFactory.setMapperLocations(exportMapperLocations);
     return sessionFactory;
   }
 
-  @Bean
-  public DataSource exportDataSource(DataSourceProperties dataSourceProperties) {
-    HikariConfig hikariConfig = new HikariConfig();
-    hikariConfig.setJdbcUrl(dataSourceProperties.getUrl());
-    hikariConfig.setUsername(dataSourceProperties.getUsername());
-    hikariConfig.setPassword(dataSourceProperties.getPassword());
-
-    // Set streaming properties for data export
-    Properties dsProperties = new Properties();
-    dsProperties.setProperty("useCursorFetch", "true");
-    dsProperties.setProperty("defaultFetchSize", "1000");
-    hikariConfig.setDataSourceProperties(dsProperties);
-
-    return new HikariDataSource(hikariConfig);
+  @Bean("exportDataSource")
+  public DataSource clickhouseExportDataSource(DataSourceProperties dataSourceProperties) {
+    // TODO How to use cursor fetch with ClickHouse to minimize memory usage during export?
+    return createDataSource(dataSourceProperties, null);
   }
 
   @Value("${feature.study.export.timeout_ms:600000}") // 10 minutes timeout by default
@@ -253,6 +248,23 @@ public void configureAsyncSupport(AsyncSupportConfigurer configurer) {
     configurer.setDefaultTimeout(timeoutMs);
   }
 
+  private DataSource createDataSource(
+      DataSourceProperties dataSourceProperties, Properties dataSourcePropertiesOverrides) {
+    HikariConfig hikariConfig = new HikariConfig();
+    hikariConfig.setJdbcUrl(dataSourceProperties.getUrl());
+    hikariConfig.setUsername(dataSourceProperties.getUsername());
+    hikariConfig.setPassword(dataSourceProperties.getPassword());
+
+    if (dataSourceProperties.getDriverClassName() != null) {
+      hikariConfig.setDriverClassName(dataSourceProperties.getDriverClassName());
+    }
+    if (dataSourcePropertiesOverrides != null) {
+      hikariConfig.setDataSourceProperties(dataSourcePropertiesOverrides);
+    }
+
+    return new HikariDataSource(hikariConfig);
+  }
+
   @Bean
   public List<Exporter> exporters(
       CancerStudyMetadataExporter cancerStudyMetadataExporter,

src/main/resources/mappers/export/CancerStudyMetadataMapper.xml

@@ -4,44 +4,44 @@
 <mapper namespace="org.cbioportal.application.file.export.repositories.mybatis.CancerStudyMetadataMapper">
     <select id="getCancerStudyMetadata" resultType="org.cbioportal.application.file.model.CancerStudyMetadata">
         SELECT
-        cs.TYPE_OF_CANCER_ID AS typeOfCancer,
-        cs.CANCER_STUDY_IDENTIFIER AS cancerStudyIdentifier,
-        cs.NAME AS name,
-        cs.DESCRIPTION AS description,
-        cs.CITATION AS citation,
-        cs.PMID AS pmid,
-        cs.`GROUPS` AS `groups`,
-        rg.NAME AS referenceGenome
+        cs.type_of_cancer_id AS typeOfCancer,
+        cs.cancer_study_identifier AS cancerStudyIdentifier,
+        cs.name AS name,
+        cs.description AS description,
+        cs.citation AS citation,
+        cs.pmid AS pmid,
+        cs.`groups` AS `groups`,
+        rg.name AS referenceGenome
         FROM cancer_study cs
-        JOIN reference_genome rg ON rg.REFERENCE_GENOME_ID = cs.REFERENCE_GENOME_ID
-        WHERE cs.CANCER_STUDY_IDENTIFIER = #{studyId}
+        JOIN reference_genome rg ON rg.reference_genome_id = cs.reference_genome_id
+        WHERE cs.cancer_study_identifier = #{studyId}
     </select>
     <select id="getCancerTypeHierarchy" resultType="org.cbioportal.application.file.model.CancerType">
         WITH RECURSIVE cancer_type_hierarchy AS (
         -- Anchor member: get the initial cancer type from the study
         SELECT
-            ct.TYPE_OF_CANCER_ID AS typeOfCancerId,
-            ct.NAME AS name,
-            ct.DEDICATED_COLOR AS dedicatedColor,
-            ct.SHORT_NAME AS shortName,
-            ct.PARENT AS parent
+            ct.type_of_cancer_id AS typeOfCancerId,
+            ct.name AS name,
+            ct.dedicated_color AS dedicatedColor,
+            ct.short_name AS shortName,
+            ct.parent AS parent
         FROM cancer_study cs
-        JOIN type_of_cancer ct ON ct.TYPE_OF_CANCER_ID = cs.TYPE_OF_CANCER_ID
-        WHERE cs.CANCER_STUDY_IDENTIFIER = #{studyId}
+        JOIN type_of_cancer ct ON ct.type_of_cancer_id = cs.type_of_cancer_id
+        WHERE cs.cancer_study_identifier = #{studyId}
 
         UNION ALL
 
         -- Recursive member: follow the parent chain
         SELECT
-            parent_ct.TYPE_OF_CANCER_ID AS typeOfCancerId,
-            parent_ct.NAME AS name,
-            parent_ct.DEDICATED_COLOR AS dedicatedColor,
-            parent_ct.SHORT_NAME AS shortName,
-            parent_ct.PARENT AS parent
+            parent_ct.type_of_cancer_id AS typeOfCancerId,
+            parent_ct.name AS name,
+            parent_ct.dedicated_color AS dedicatedColor,
+            parent_ct.short_name AS shortName,
+            parent_ct.parent AS parent
         FROM type_of_cancer parent_ct
-        JOIN cancer_type_hierarchy cth ON cth.parent = parent_ct.TYPE_OF_CANCER_ID
+        JOIN cancer_type_hierarchy cth ON cth.parent = parent_ct.type_of_cancer_id
         )
 
-        SELECT * FROM cancer_type_hierarchy;
+        SELECT * FROM cancer_type_hierarchy
     </select>
-</mapper>
+</mapper>

src/main/resources/mappers/export/CaseListMetadataMapper.xml

@@ -3,39 +3,33 @@
 
 <mapper namespace="org.cbioportal.application.file.export.repositories.mybatis.CaseListMetadataMapper">
     <resultMap id="CaseListMetadataResultMap" type="org.cbioportal.application.file.model.CaseListMetadata">
-        <id column="LIST_ID"/>
-        <result property="cancerStudyIdentifier" column="cancerStudyIdentifier"/>
-        <result property="stableId" column="stableId"/>
+        <id column="list_id"/>
+        <result property="cancerStudyIdentifier" column="cancer_study_identifier"/>
+        <result property="stableId" column="stable_id"/>
         <result property="name" column="name"/>
         <result property="description" column="description"/>
         <collection property="sampleIds" ofType="String" javaType="java.util.TreeSet">
-            <result column="sampleId"/>
+            <result column="sample_id"/>
         </collection>
     </resultMap>
     <select
             id="getCaseListsMetadata"
             resultMap="CaseListMetadataResultMap">
+        <bind name="sampleIdsArray" value="sampleIds != null ? sampleIds.toArray(new String[sampleIds.size()]) : null" />
         SELECT
-            sl.LIST_ID,
-            cs.CANCER_STUDY_IDENTIFIER as cancerStudyIdentifier,
-            sl.STABLE_ID as stableId,
-            sl.NAME as name,
-            sl.DESCRIPTION as description,
-            s.STABLE_ID as sampleId
+            sl.list_id AS list_id,
+            cs.cancer_study_identifier AS cancer_study_identifier,
+            sl.stable_id AS stable_id,
+            sl.name AS name,
+            sl.description AS description,
+            s.stable_id AS sample_id
         FROM sample_list sl
-        JOIN cancer_study cs ON cs.CANCER_STUDY_ID = sl.CANCER_STUDY_ID
-        JOIN sample_list_list sll ON sll.LIST_ID = sl.LIST_ID
-        JOIN sample s ON s.INTERNAL_ID = sll.SAMPLE_ID
+        JOIN cancer_study cs ON cs.cancer_study_id = sl.cancer_study_id
+        JOIN sample_list_list sll ON sll.list_id = sl.list_id
+        JOIN sample s ON s.internal_id = sll.sample_id
+        WHERE cs.cancer_study_identifier = #{studyId}
         <if test="sampleIds != null">
-            JOIN (
-                SELECT *
-                FROM (VALUES
-                <foreach item="sid" collection="sampleIds" separator=",">
-                    ROW(#{sid})
-                </foreach>
-                ) AS temp(sample_id)
-            ) AS sample_ids_subquery ON sample_ids_subquery.sample_id = s.STABLE_ID
+            AND s.stable_id IN (#{sampleIdsArray, typeHandler=org.apache.ibatis.type.ArrayTypeHandler})
         </if>
-        WHERE cs.CANCER_STUDY_IDENTIFIER = #{studyId}
     </select>
-</mapper>
+</mapper>