RFC: FACETS data type for cBioPortal

scripts/importer/cbioportal_common.py

@@ -63,6 +63,8 @@ class MetaFileTypes(object):
     CNA_LOG2 = 'meta_log2CNA'
     CNA_CONTINUOUS = 'meta_contCNA'
     SEG = 'meta_segment'
+    FACETS_CNCF = 'meta_facets_cncf'
+    FACETS_GENE_LEVEL = 'meta_facets_gene_level'
     EXPRESSION = 'meta_expression'
     MUTATION = 'meta_mutations_extended'
     MUTATION_UNCALLED = 'meta_mutations_uncalled'
@@ -175,6 +177,22 @@ def __init__(self, process_return_status, message):
         'data_filename': True,
         'description': True
     },
+    MetaFileTypes.FACETS_CNCF: {
+        'cancer_study_identifier': True,
+        'genetic_alteration_type': True,
+        'datatype': True,
+        'reference_genome_id': True,
+        'data_filename': True,
+        'description': True
+    },
+    MetaFileTypes.FACETS_GENE_LEVEL: {
+        'cancer_study_identifier': True,
+        'genetic_alteration_type': True,
+        'datatype': True,
+        'reference_genome_id': True,
+        'data_filename': True,
+        'description': True
+    },
     MetaFileTypes.MUTATION: {
         'cancer_study_identifier': True,
         'genetic_alteration_type': True,
@@ -390,6 +408,8 @@ def __init__(self, process_return_status, message):
     MetaFileTypes.GENE_PANEL_MATRIX,
     MetaFileTypes.STRUCTURAL_VARIANT,
     MetaFileTypes.SEG,
+    MetaFileTypes.FACETS_CNCF,
+    MetaFileTypes.FACETS_GENE_LEVEL,
 ]
 
 IMPORTER_CLASSNAME_BY_META_TYPE = {
@@ -402,6 +422,8 @@ def __init__(self, process_return_status, message):
     MetaFileTypes.CNA_LOG2: "org.mskcc.cbio.portal.scripts.ImportProfileData",
     MetaFileTypes.CNA_CONTINUOUS: "org.mskcc.cbio.portal.scripts.ImportProfileData",
     MetaFileTypes.SEG: "org.mskcc.cbio.portal.scripts.ImportCopyNumberSegmentData",
+    MetaFileTypes.FACETS_CNCF: "org.mskcc.cbio.portal.scripts.ImportFacetsCncfData",
+    MetaFileTypes.FACETS_GENE_LEVEL: "org.mskcc.cbio.portal.scripts.ImportFacetsGeneLevelData",
     MetaFileTypes.EXPRESSION: "org.mskcc.cbio.portal.scripts.ImportProfileData",
     MetaFileTypes.MUTATION: "org.mskcc.cbio.portal.scripts.ImportProfileData",
     MetaFileTypes.MUTATION_UNCALLED: "org.mskcc.cbio.portal.scripts.ImportProfileData",
@@ -427,6 +449,8 @@ def __init__(self, process_return_status, message):
 IMPORTER_REQUIRES_METADATA = {
     "org.mskcc.cbio.portal.scripts.ImportClinicalData" : True,
     "org.mskcc.cbio.portal.scripts.ImportCopyNumberSegmentData" : True,
+    "org.mskcc.cbio.portal.scripts.ImportFacetsCncfData" : True,
+    "org.mskcc.cbio.portal.scripts.ImportFacetsGeneLevelData" : True,
     "org.mskcc.cbio.portal.scripts.ImportGisticData" : False,
     "org.mskcc.cbio.portal.scripts.ImportMutSigData" : False,
     "org.mskcc.cbio.portal.scripts.ImportProfileData" : True,
@@ -666,6 +690,8 @@ def get_meta_file_type(meta_dictionary, logger, filename):
         ("COPY_NUMBER_ALTERATION", "CONTINUOUS"): MetaFileTypes.CNA_CONTINUOUS,
         ("COPY_NUMBER_ALTERATION", "LOG2-VALUE"): MetaFileTypes.CNA_LOG2,
         ("COPY_NUMBER_ALTERATION", "SEG"): MetaFileTypes.SEG,
+        ("COPY_NUMBER_ALTERATION", "FACETS_CNCF"): MetaFileTypes.FACETS_CNCF,
+        ("COPY_NUMBER_ALTERATION", "FACETS_GENE_LEVEL"): MetaFileTypes.FACETS_GENE_LEVEL,
         # expression
         ("MRNA_EXPRESSION", "CONTINUOUS"): MetaFileTypes.EXPRESSION,
         ("MRNA_EXPRESSION", "Z-SCORE"): MetaFileTypes.EXPRESSION,

src/main/java/org/mskcc/cbio/portal/dao/ClickHouseAutoIncrement.java

@@ -90,6 +90,8 @@ private SequenceConfig(String tableName, String columnName) {
         register("seq_copy_number_seg", "copy_number_seg", "seg_id");
         register("seq_copy_number_seg_file", "copy_number_seg_file", "seg_file_id");
         register("seq_clinical_event", "clinical_event", "clinical_event_id");
+        register("seq_facets_cncf", "facets_cncf", "seg_id");
+        register("seq_facets_genes", "facets_genes", "gene_id");
         registerShutdownHook();
     }
 

src/main/java/org/mskcc/cbio/portal/dao/ClickHouseConstraintChecker.java

@@ -296,6 +296,12 @@ private static List<ForeignKey> schemaForeignKeys() {
         fks.add(new ForeignKey("copy_number_seg", List.of("sample_id"), "sample", List.of("internal_id")));
         fks.add(new ForeignKey("copy_number_seg_file", List.of("cancer_study_id"), "cancer_study", List.of("cancer_study_id")));
 
+        // facets_cncf / facets_genes
+        fks.add(new ForeignKey("facets_cncf", List.of("cancer_study_id"), "cancer_study", List.of("cancer_study_id")));
+        fks.add(new ForeignKey("facets_cncf", List.of("sample_id"), "sample", List.of("internal_id")));
+        fks.add(new ForeignKey("facets_genes", List.of("cancer_study_id"), "cancer_study", List.of("cancer_study_id")));
+        fks.add(new ForeignKey("facets_genes", List.of("sample_id"), "sample", List.of("internal_id")));
+
         // clinical_event / clinical_event_data
         fks.add(new ForeignKey("clinical_event", List.of("patient_id"), "patient", List.of("internal_id")));
         fks.add(new ForeignKey("clinical_event_data", List.of("clinical_event_id"), "clinical_event", List.of("clinical_event_id")));
@@ -450,6 +456,10 @@ private static List<UniqueKey> schemaUniqueKeys() {
         uniqueKeys.add(new UniqueKey("copy_number_seg", List.of("seg_id")));
         uniqueKeys.add(new UniqueKey("copy_number_seg_file", List.of("seg_file_id")));
 
+        // facets_cncf / facets_genes
+        uniqueKeys.add(new UniqueKey("facets_cncf", List.of("seg_id")));
+        uniqueKeys.add(new UniqueKey("facets_genes", List.of("gene_id")));
+
         // clinical_event
         uniqueKeys.add(new UniqueKey("clinical_event", List.of("clinical_event_id")));
 

src/main/java/org/mskcc/cbio/portal/dao/DaoFacetsCncf.java

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2026 Memorial Sloan-Kettering Cancer Center.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS
+ * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder
+ * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no
+ * obligations to provide maintenance, support, updates, enhancements or
+ * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be
+ * liable to any party for direct, indirect, special, incidental or
+ * consequential damages, including lost profits, arising out of the use of this
+ * software and its documentation, even if Memorial Sloan-Kettering Cancer
+ * Center has been advised of the possibility of such damage.
+ */
+
+/*
+ * This file is part of cBioPortal.
+ *
+ * cBioPortal is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package org.mskcc.cbio.portal.dao;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import org.mskcc.cbio.portal.model.FacetsCncfSegment;
+
+/**
+ * DAO for FACETS CNCF (allele-specific copy number segment) data.
+ *
+ * Mirrors {@link DaoCopyNumberSegment}: rows are exclusively appended through
+ * {@link ClickHouseBulkLoader}, and the `facets_cncf` table is ordered by
+ * (cancer_study_id, sample_id, chr, start) to make genome-ordered, per-sample
+ * segment reads (e.g. segment plots) efficient.
+ */
+public final class DaoFacetsCncf {
+
+    private static final String TABLE = "facets_cncf";
+
+    private DaoFacetsCncf() {}
+
+    public static void addFacetsCncfSegment(FacetsCncfSegment seg) throws DaoException {
+        FacetsDaoUtil.requireBulkLoad("insert FACETS CNCF data");
+        ClickHouseBulkLoader.getClickHouseBulkLoader(TABLE).insertRecord(
+                Long.toString(seg.getId()),
+                Integer.toString(seg.getCancerStudyId()),
+                Integer.toString(seg.getSampleId()),
+                seg.getChr(),
+                Long.toString(seg.getStart()),
+                Long.toString(seg.getEnd()),
+                seg.getTcn() == null ? null : Double.toString(seg.getTcn()),
+                seg.getLcn() == null ? null : Double.toString(seg.getLcn()),
+                seg.getCellularFraction() == null ? null : Double.toString(seg.getCellularFraction()),
+                seg.getPurity() == null ? null : Double.toString(seg.getPurity())
+        );
+    }
+
+    public static void addFacetsCncfSegments(List<FacetsCncfSegment> segs) throws DaoException {
+        for (FacetsCncfSegment seg : segs) {
+            addFacetsCncfSegment(seg);
+        }
+    }
+
+    /**
+     * Reserves and returns the next unique id for a new `facets_cncf` row.
+     * Callers should invoke this once per row before {@link #addFacetsCncfSegment}.
+     */
+    public static long getNextId() throws DaoException {
+        return ClickHouseAutoIncrement.nextId("seq_facets_cncf");
+    }
+
+    public static List<FacetsCncfSegment> getSegmentsForSample(int sampleId, int cancerStudyId) throws DaoException {
+        return getSegmentsForSamples(Collections.singleton(sampleId), cancerStudyId);
+    }
+
+    public static List<FacetsCncfSegment> getSegmentsForSamples(Collection<Integer> sampleIds, int cancerStudyId) throws DaoException {
+        return FacetsDaoUtil.queryForSamples(TABLE, sampleIds, cancerStudyId, DaoFacetsCncf::mapRow);
+    }
+
+    public static boolean facetsCncfDataExistForCancerStudy(int cancerStudyId) throws DaoException {
+        return FacetsDaoUtil.dataExistsForCancerStudy(TABLE, cancerStudyId);
+    }
+
+    public static void deleteFacetsCncfDataForSamples(int cancerStudyId, Set<Integer> sampleIds) throws DaoException {
+        FacetsDaoUtil.deleteDataForSamples(TABLE, cancerStudyId, sampleIds);
+    }
+
+    private static FacetsCncfSegment mapRow(ResultSet rs) throws SQLException {
+        FacetsCncfSegment seg = new FacetsCncfSegment(
+                rs.getInt("cancer_study_id"),
+                rs.getInt("sample_id"),
+                rs.getString("chr"),
+                rs.getLong("start"),
+                rs.getLong("end"),
+                getNullableDouble(rs, "tcn"),
+                getNullableDouble(rs, "lcn"),
+                getNullableDouble(rs, "cellular_fraction"),
+                getNullableDouble(rs, "purity"));
+        seg.setId(rs.getLong("seg_id"));
+        return seg;
+    }
+
+    private static Double getNullableDouble(ResultSet rs, String column) throws SQLException {
+        double value = rs.getDouble(column);
+        return rs.wasNull() ? null : value;
+    }
+}

src/main/java/org/mskcc/cbio/portal/dao/DaoFacetsGenes.java

@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2026 Memorial Sloan-Kettering Cancer Center.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS
+ * FOR A PARTICULAR PURPOSE. The software and documentation provided hereunder
+ * is on an "as is" basis, and Memorial Sloan-Kettering Cancer Center has no
+ * obligations to provide maintenance, support, updates, enhancements or
+ * modifications. In no event shall Memorial Sloan-Kettering Cancer Center be
+ * liable to any party for direct, indirect, special, incidental or
+ * consequential damages, including lost profits, arising out of the use of this
+ * software and its documentation, even if Memorial Sloan-Kettering Cancer
+ * Center has been advised of the possibility of such damage.
+ */
+
+/*
+ * This file is part of cBioPortal.
+ *
+ * cBioPortal is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package org.mskcc.cbio.portal.dao;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import org.mskcc.cbio.portal.model.FacetsGeneLevelRecord;
+
+/**
+ * DAO for FACETS gene-level data, derived from (but not directly mapped to)
+ * FACETS CNCF segment calls (see {@link DaoFacetsCncf}).
+ *
+ * Rows are exclusively appended through {@link ClickHouseBulkLoader}. The
+ * `facets_genes` table is ordered by (cancer_study_id, sample_id,
+ * hugo_gene_symbol) for efficient per-sample gene table reads, with a bloom
+ * filter index on hugo_gene_symbol to keep cross-sample/cohort gene lookups
+ * (e.g. "show gene X across all samples") cheap as well.
+ */
+public final class DaoFacetsGenes {
+
+    private static final String TABLE = "facets_genes";
+
+    private DaoFacetsGenes() {}
+
+    public static void addFacetsGeneLevelRecord(FacetsGeneLevelRecord rec) throws DaoException {
+        FacetsDaoUtil.requireBulkLoad("insert FACETS gene-level data");
+        ClickHouseBulkLoader.getClickHouseBulkLoader(TABLE).insertRecord(
+                Long.toString(rec.getId()),
+                Integer.toString(rec.getCancerStudyId()),
+                Integer.toString(rec.getSampleId()),
+                rec.getHugoGeneSymbol(),
+                Long.toString(rec.getEntrezGeneId()),
+                rec.getChr(),
+                Long.toString(rec.getStart()),
+                Long.toString(rec.getEnd()),
+                rec.getTcn() == null ? null : Double.toString(rec.getTcn()),
+                rec.getLcn() == null ? null : Double.toString(rec.getLcn()),
+                rec.getCellularFraction() == null ? null : Double.toString(rec.getCellularFraction()),
+                rec.getPurity() == null ? null : Double.toString(rec.getPurity())
+        );
+    }
+
+    public static void addFacetsGeneLevelRecords(List<FacetsGeneLevelRecord> recs) throws DaoException {
+        for (FacetsGeneLevelRecord rec : recs) {
+            addFacetsGeneLevelRecord(rec);
+        }
+    }
+
+    /**
+     * Reserves and returns the next unique id for a new `facets_genes` row.
+     * Callers should invoke this once per row before {@link #addFacetsGeneLevelRecord}.
+     */
+    public static long getNextId() throws DaoException {
+        return ClickHouseAutoIncrement.nextId("seq_facets_genes");
+    }
+
+    public static List<FacetsGeneLevelRecord> getGeneLevelDataForSample(int sampleId, int cancerStudyId) throws DaoException {
+        return getGeneLevelDataForSamples(Collections.singleton(sampleId), cancerStudyId);
+    }
+
+    public static List<FacetsGeneLevelRecord> getGeneLevelDataForSamples(Collection<Integer> sampleIds, int cancerStudyId) throws DaoException {
+        return FacetsDaoUtil.queryForSamples(TABLE, sampleIds, cancerStudyId, DaoFacetsGenes::mapRow);
+    }
+
+    /**
+     * Fetches gene-level FACETS records for a single gene across a set of samples
+     * (e.g. for an oncoprint-style, cohort-wide view). Relies on the bloom filter
+     * index on `hugo_gene_symbol` for efficient filtering.
+     */
+    public static List<FacetsGeneLevelRecord> getGeneLevelDataForGene(String hugoGeneSymbol, Collection<Integer> sampleIds, int cancerStudyId) throws DaoException {
+        if (sampleIds == null || sampleIds.isEmpty()) {
+            return Collections.emptyList();
+        }
+        return ClickHouseBulkUploader.upload(sampleIds, stagingTable -> {
+            List<FacetsGeneLevelRecord> results = new ArrayList<>();
+            Connection con = null;
+            try {
+                con = JdbcUtil.getDbConnection(DaoFacetsGenes.class);
+                try (PreparedStatement pstmt = con.prepareStatement(
+                        "SELECT * FROM `" + TABLE + "`" +
+                        " WHERE `hugo_gene_symbol`=?" +
+                        " AND `cancer_study_id`=?" +
+                        " AND `sample_id` IN (SELECT id FROM " + stagingTable + ")")) {
+                    pstmt.setString(1, hugoGeneSymbol);
+                    pstmt.setInt(2, cancerStudyId);
+                    try (ResultSet rs = pstmt.executeQuery()) {
+                        while (rs.next()) {
+                            results.add(mapRow(rs));
+                        }
+                    }
+                }
+                return results;
+            } finally {
+                JdbcUtil.closeAll(DaoFacetsGenes.class, con, null, null);
+            }
+        });
+    }
+
+    public static boolean facetsGenesDataExistForCancerStudy(int cancerStudyId) throws DaoException {
+        return FacetsDaoUtil.dataExistsForCancerStudy(TABLE, cancerStudyId);
+    }
+
+    public static void deleteFacetsGenesDataForSamples(int cancerStudyId, Set<Integer> sampleIds) throws DaoException {
+        FacetsDaoUtil.deleteDataForSamples(TABLE, cancerStudyId, sampleIds);
+    }
+
+    private static FacetsGeneLevelRecord mapRow(ResultSet rs) throws SQLException {
+        FacetsGeneLevelRecord rec = new FacetsGeneLevelRecord(
+                rs.getInt("cancer_study_id"),
+                rs.getInt("sample_id"),
+                rs.getString("hugo_gene_symbol"),
+                rs.getLong("entrez_gene_id"),
+                rs.getString("chr"),
+                rs.getLong("start"),
+                rs.getLong("end"),
+                getNullableDouble(rs, "tcn"),
+                getNullableDouble(rs, "lcn"),
+                getNullableDouble(rs, "cellular_fraction"),
+                getNullableDouble(rs, "purity"));
+        rec.setId(rs.getLong("gene_id"));
+        return rec;
+    }
+
+    private static Double getNullableDouble(ResultSet rs, String column) throws SQLException {
+        double value = rs.getDouble(column);
+        return rs.wasNull() ? null : value;
+    }
+}