Merge commit '19ffa7a2f137f2781ffca64bf39b31f5f99f199d'

Author: milot-mirdita

lib/foldseek/lib/foldcomp/CMakeLists.txt

@@ -1,5 +1,5 @@
 # Author: Milot Mirdita (milot@mirdita.de), Hyunbin Kim (khb7840@gmail.com)
-cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -14,6 +14,11 @@ option(GCS_SUPPORT "Enable Google Cloud Storage support" OFF)
 include_directories(src)
 add_subdirectory(src)
 
+# For windows, include lib/windows
+if(WIN32)
+    include_directories(lib/windows)
+endif(WIN32)
+
 if(HAVE_SANITIZER)
     include(FindUBSan)
     include(FindASan)
@@ -26,6 +31,11 @@ if(BUILD_LIBRARY)
         ${foldcomp_header_files}
         ${foldcomp_source_files})
 elseif(BUILD_PYTHON)
+        if(MSVC)
+            install(FILES lib/windows/dirent.h DESTINATION include)
+        endif(MSVC)
+        find_package(PythonInterp)
+    	find_package(PythonLibs) # Trying to fix cibuildwheel ubuntu error
         find_package(PythonExtensions REQUIRED)
         add_library(foldcomp MODULE
             ${foldcomp_header_files}
@@ -45,26 +55,34 @@ else()
     target_compile_definitions(foldcomp PUBLIC FOLDCOMP_EXECUTABLE)
     # For debugging
     # target_compile_definitions(foldcomp PUBLIC _GLIBCXX_DEBUG=1 _LIBCPP_DEBUG=1)
-
-    find_package(OpenMP REQUIRED)
-    if(OPENMP_CXX_FOUND)
-        if((CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang"))
-            target_link_libraries(foldcomp PUBLIC OpenMP::OpenMP_CXX)
-        else()
-            target_link_libraries(foldcomp PRIVATE "${OpenMP_CXX_FLAGS}")
-            target_compile_options(foldcomp PRIVATE "${OpenMP_CXX_FLAGS}")
+    
+    if(NOT EMSCRIPTEN)
+        # For local compilation, openmp and zlib are required
+        find_package(OpenMP REQUIRED)
+        if(OPENMP_CXX_FOUND)
+            if((CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang"))
+                target_link_libraries(foldcomp PUBLIC OpenMP::OpenMP_CXX)
+            else()
+                target_link_libraries(foldcomp PRIVATE "${OpenMP_CXX_FLAGS}")
+                target_compile_options(foldcomp PRIVATE "${OpenMP_CXX_FLAGS}")
+            endif()
             target_compile_definitions(foldcomp PUBLIC OPENMP)
         endif()
+        find_package(ZLIB REQUIRED)
+        target_link_libraries(foldcomp PUBLIC ZLIB::ZLIB)
+    else()
+        # For webassembly, not using openmp. zlib is added to compile flags
+        set_target_properties(
+            foldcomp
+            PROPERTIES
+            COMPILE_FLAGS -sUSE_ZLIB=1
+            LINK_FLAGS "-sUSE_ZLIB=1 -sEXPORTED_RUNTIME_METHODS=callMain,FS -sINVOKE_RUN=0 -sFILESYSTEM=1 -sALLOW_MEMORY_GROWTH=1 -sTOTAL_MEMORY=256MB -sENVIRONMENT=web -sMODULARIZE=1 -s EXPORT_ES6=1 -sEXPORT_NAME=createFoldcomp -sSINGLE_FILE=0 -sASSERTIONS=0")
     endif()
 
-    find_package(ZLIB REQUIRED)
-    target_link_libraries(foldcomp PUBLIC ZLIB::ZLIB)
-
     include_directories(lib/gemmi)
 
     include_directories(lib/microtar)
     add_subdirectory(lib/microtar)
-
     target_link_libraries(foldcomp PUBLIC microtar)
 
     if(GCS_SUPPORT)

lib/foldseek/lib/foldcomp/README.md

@@ -1,8 +1,9 @@
 # Foldcomp
+
 <p align="center">
 <img src="https://raw.githubusercontent.com/steineggerlab/foldcomp/master/.github/img/foldcomp_strong_marv.png" max-height="300px" height="300" display="block" margin-left="auto" margin-right="auto" display="block"/>
 </p>
-Foldcomp compresses protein structures with torsion angles effectively. It compresses the backbone atoms to 8 bytes and the side chain to additionally 4-5 byes per residue, thus an averaged-sized protein of 350 residues requires ~6kb. 
+Foldcomp compresses protein structures with torsion angles effectively. It compresses the backbone atoms to 8 bytes and the side chain to additionally 4-5 byes per residue, thus an averaged-sized protein of 350 residues requires ~6kb.
 
 Foldcomp efficient compressed format stores protein structures requiring only 13 bytes per residue, which reduces the required storage space by an order of magnitude compared to saving 3D coordinates directly. We achieve this reduction by encoding the torsion angles of the backbone as well as the side-chain angles in a compact binary file format (FCZ).
 
@@ -16,6 +17,18 @@ Foldcomp efficient compressed format stores protein structures requiring only 13
 </picture>
 </p>
 
+## Publications
+
+[Hyunbin Kim, Milot Mirdita, Martin Steinegger, Foldcomp: a library and format for compressing and indexing large protein structure sets, Bioinformatics, 2023;, btad153,](https://doi.org/10.1093/bioinformatics/btad153)
+
+## Presentation Video
+
+We presented Foldcomp at ISMB/ECCB2023. Check it out:
+
+<a href="https://www.youtube.com/watch?v=aFtqH0VqE7w" target="_blank">
+  <img src="https://raw.githubusercontent.com/steineggerlab/foldcomp/master/.github/img/ismb_thumbnail.png" alt="Foldcomp presented at ISMB/ECCB2023" max-width="720px" max-height="400px" width="auto" height="auto">
+</a>
+
 ## Usage
 
 ### Installing Foldcomp
@@ -32,40 +45,89 @@ wget https://mmseqs.com/foldcomp/foldcomp-linux-arm64.tar.gz
 
 # Download binary for macOS
 wget https://mmseqs.com/foldcomp/foldcomp-macos-universal.tar.gz
+
+# Download binary for Windows (x64)
+wget https://mmseqs.com/foldcomp/foldcomp-windows-x64.zip
 ```
 
 ### Executable
 ```
 # Compression
-foldcomp compress <pdb_file|cif_file> [<fcz_file>]
-foldcomp compress [-t number] <pdb_dir|cif_dir> [<fcz_dir>]
+foldcomp compress <pdb|cif> [<fcz>]
+foldcomp compress [-t number] <dir|tar(.gz)> [<dir|tar|db>]
 
 # Decompression
-foldcomp decompress <fcz_file> [<pdb_file>]
-foldcomp decompress [-t number] <fcz_dir> [<pdb_dir>]
+foldcomp decompress <fcz|tar> [<pdb>]
+foldcomp decompress [-t number] <dir|tar(.gz)|db> [<dir|tar>]
+
+# Decompressing a subset of Foldcomp database
+foldcomp decompress [-t number] --id-list <idlist.txt> <db> [<dir|tar>]
 
 # Extraction of sequence or pLDDT
-foldcomp extract [--plddt|--fasta] <fcz_file> [<txt_file|fasta_file>]
-foldcomp extract [--plddt|--fasta] [-t number] <fcz_dir|tar> [<output_dir>]
+foldcomp extract [--plddt|--amino-acid] <fcz> [<fasta>]
+foldcomp extract [--plddt|--amino-acid] [-t number] <dir|tar(.gz)|db> [<fasta_out>]
 
 # Check
-foldcomp check <fcz_file>
-foldcomp check [-t number] <fcz_dir|tar>
+foldcomp check <fcz>
+foldcomp check [-t number] <dir|tar(.gz)|db>
 
 # RMSD
-foldcomp rmsd <pdb1|cif1> <pdb2|cif2>
+foldcomp rmsd <pdb|cif> <pdb|cif>
 
 # Options
- -h, --help           print this help message
- -t, --threads        threads for (de)compression of folders/tar files [default=1]
- -a, --alt            use alternative atom order [default=false]
- -b, --break          interval size to save absolute atom coordinates [default=25]
- -z, --tar            save as tar file [default=false]
- --plddt              extract pLDDT score (only for extraction mode)
- --fasta              extract amino acid sequence (only for extraction mode)
- --no-merge           do not merge output files (only for extraction mode)
+ -h, --help               print this help message
+ -v, --version            print version
+ -t, --threads            threads for (de)compression of folders/tar files [default=1]
+ -r, --recursive          recursively look for files in directory [default=0]
+ -f, --file               input is a list of files [default=0]
+ -a, --alt                use alternative atom order [default=false]
+ -b, --break              interval size to save absolute atom coordinates [default=25]
+ -z, --tar                save as tar file [default=false]
+ -d, --db                 save as database [default=false]
+ -y, --overwrite          overwrite existing files [default=false]
+ -l, --id-list            a file of id list to be processed (only for database input)
+ --skip-discontinuous     skip PDB with with discontinuous residues (only batch compression)
+ --check                  check FCZ before and skip entries with error (only for batch decompression)
+ --plddt                  extract pLDDT score (only for extraction mode)
+ -p, --plddt-digits       extract pLDDT score with specified number of digits (only for extraction mode)
+                          - 1: single digit (fasta-like format), 2: 2-digit(00-99; tsv), 3: 3-digit, 4: 4-digit (max)
+ --fasta, --amino-acid    extract amino acid sequence (only for extraction mode)
+ --no-merge               do not merge output files (only for extraction mode)
+ --use-title              use TITLE as the output file name (only for extraction mode)
+ --time                   measure time for compression/decompression
+```
+
+### Downloading Databases
+We offer prebuilt databases for multiple large sets of predicted protein structures and a Python helper to download the database files.
+
+You can download the AlphaFoldDB Swiss-Prot with the following command:
+```
+python -c "import foldcomp; foldcomp.setup('afdb_swissprot_v4');
 ```
 
+Currently we offer the following databases:
+* [ESMAtlas](https://esmatlas.com/) full (v0 + v2023_02): `foldcomp.setup('esmatlas')`
+* ESMAtlas v2023_02: `foldcomp.setup('esmatlas_v2023_02')`
+* ESMAtlas high-quality: `foldcomp.setup('highquality_clust30')`
+
+  **Note:** We skipped all structures with discontinous residues or other issues.
+   Here is a list with the affected predictions;
+   [full](https://foldcomp.steineggerlab.workers.dev/esmatlas.err.log) (~21M),
+   [high-quality](https://foldcomp.steineggerlab.workers.dev/highquality_clust30_issues.txt) (~100k),
+   [v2023_02](https://foldcomp.steineggerlab.workers.dev/esmatlas_v2023_02.err.log) (~10k)
+
+* [AlphaFoldDB Uniprot](https://alphafold.ebi.ac.uk/): `foldcomp.setup('afdb_uniprot_v4')`
+* AlphaFoldDB Swiss-Prot: `foldcomp.setup('afdb_swissprot_v4')`
+* AlphaFoldDB Model Organisms: `foldcomp.setup('h_sapiens')`
+  * `a_thaliana`, `c_albicans`, `c_elegans`, `d_discoideum`, `d_melanogaster`, `d_rerio`, `e_coli`, `g_max`,
+    `h_sapiens`, `m_jannaschii`, `m_musculus`, `o_sativa`, `r_norvegicus`, `s_cerevisiae`, `s_pombe`, `z_mays`
+* [AlphaFoldDB Cluster Representatives](https://afdb-cluster.steineggerlab.workers.dev/): `foldcomp.setup('afdb_rep_v4')`
+* AlphaFoldDB Cluster Representatives (Dark Clusters): `foldcomp.setup('afdb_rep_dark_v4')`
+
+If you want other prebuilt datasets, please get in touch with us through our [GitHub issues](https://github.com/steineggerlab/foldcomp/issues).
+
+If you have issues downloading the databases you can navigate directly to our [download server](https://foldcomp.steineggerlab.workers.dev/) and download the required files. E.g. `afdb_uniprot_v4`, `afdb_uniprot_v4.index`, `afdb_uniprot_v4.dbtype`, `afdb_uniprot_v4.lookup`, and optionally `afdb_uniprot_v4.source`.
+
 ### Python API
 
 You can find more in-depth examples of using Foldcomp's Python interface in the example notebook:
@@ -85,6 +147,17 @@ with open("test/compressed.fcz", "rb") as fcz:
   with open(name, "w") as pdb_file:
     pdb_file.write(pdb)
 
+  # Get data as dictionary
+  data_dict = foldcomp.get_data(fcz_binary) # foldcomp.get_data(pdb) also works
+  # Keys: phi, psi, omega, torsion_angles, residues, bond_angles, coordinates
+  data_dict["phi"] # phi angles (C-N-CA-C)
+  data_dict["psi"] # psi angles (N-CA-C-N)
+  data_dict["omega"] # omega angles (CA-C-N-CA)
+  data_dict["torsion_angles"] # torsion angles of the backbone as list (phi + psi + omega)
+  data_dict["bond_angles"] # bond angles of the backbone as list
+  data_dict["residues"] # amino acid residues as string
+  data_dict["coordinates"] # coordinates of the backbone as list
+
 # 02. Iterate over a database of FCZ files
 # Open a foldcomp database
 ids = ["d1asha_", "d1it2a_"]
@@ -96,6 +169,19 @@ with foldcomp.open("test/example_db", ids=ids) as db:
         pdb_file.write(pdb)
 ```
 
+## Subsetting Databases
+If you are dealing with millions of entries, we recommend using `createsubdb` command
+of [mmseqs2](https://mmseqs.com) to subset databases.
+The following commands can be used to subset the AlphaFold Uniprot DB with given IDs.
+```sh
+# mmseqs createsubdb --subdb-mode 0 --id-mode 1 id_list.txt input_foldcomp_db output_foldcomp_db
+mmseqs createsubdb --subdb-mode 0 --id-mode 1 id_list.txt afdb_uniprot_v4 afdb_subset
+```
+Please note that the IDs in afdb_uniprot_v4 are in the format `AF-A0A5S3Y9Q7-F1-model_v4` .
+
+## Community Contributions
+* [PyMOL Plugin for reading Foldcomp files](https://github.com/yakomaxa/load_fcz_PyMOL) by @yakomaxa
+
 ## Contributor
 <a href="https://github.com/steineggerlab/foldcomp/graphs/contributors">
   <img src="https://contributors-img.firebaseapp.com/image?repo=steineggerlab/foldcomp" />

lib/foldseek/lib/foldcomp/src/CMakeLists.txt

@@ -4,11 +4,13 @@ set(foldcomp_header_files
     src/bond_info.h
     src/execution_timer.h
     src/database_reader.h
+    src/database_writer.h
     src/discretizer.h
     src/float3d.h
     src/foldcomp.h
     src/nerf.h
     src/sidechain.h
+    src/tcbspan.h
     src/torsion_angle.h
     src/utility.h
     PARENT_SCOPE
@@ -18,6 +20,7 @@ set(foldcomp_source_files
     src/amino_acid.cpp
     src/atom_coordinate.cpp
     src/database_reader.cpp
+    src/database_writer.cpp
     src/discretizer.cpp
     src/foldcomp.cpp
     src/nerf.cpp