refactor(dataset): dataset add/import refactoring (#3001)

Author: mohammad-alisafaee

docs/cheatsheet/conf.py

@@ -169,7 +169,7 @@
 napoleon_numpy_docstring = False
 
 # sphinx type references only work for types that documentation is generated for
-# Suppress warnungs for these types that are referenced but not documented themselves.
+# Suppress warnings for these types that are referenced but not documented themselves.
 nitpick_ignore = [
     ("py:class", "Path"),
     ("py:class", "OID_TYPE"),

docs/reference/core.rst

@@ -135,6 +135,14 @@ Providers for dataset import and export
    :members:
    :show-inheritance:
 
+.. automodule:: renku.core.dataset.providers.git
+   :members:
+   :show-inheritance:
+
+.. automodule:: renku.core.dataset.providers.local
+   :members:
+   :show-inheritance:
+
 .. automodule:: renku.core.dataset.providers.models
    :members:
    :show-inheritance:
@@ -147,6 +155,14 @@ Providers for dataset import and export
    :members:
    :show-inheritance:
 
+.. automodule:: renku.core.dataset.providers.repository
+   :members:
+   :show-inheritance:
+
+.. automodule:: renku.core.dataset.providers.web
+   :members:
+   :show-inheritance:
+
 .. automodule:: renku.core.dataset.providers.zenodo
    :members:
    :show-inheritance:

docs/spelling_wordlist.txt

@@ -55,6 +55,7 @@ deployer
 deserialization
 deserialize
 Deserialize
+deserialized
 Deserialized
 deserializing
 discoverable

renku/command/dataset.py

@@ -32,7 +32,7 @@
     show_dataset,
     update_datasets,
 )
-from renku.core.dataset.dataset_add import add_data_to_dataset
+from renku.core.dataset.dataset_add import add_to_dataset
 from renku.core.dataset.tag import add_dataset_tag, list_dataset_tags, remove_dataset_tags
 
 
@@ -65,7 +65,7 @@ def show_dataset_command():
 
 def add_to_dataset_command():
     """Create a command for adding data to datasets."""
-    command = Command().command(add_data_to_dataset).lock_dataset().with_database(write=True)
+    command = Command().command(add_to_dataset).lock_dataset().with_database(write=True)
     return command.require_migration().with_commit(raise_if_empty=True, commit_only=DATASET_METADATA_PATHS)
 
 

renku/core/dataset/context.py

@@ -16,6 +16,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Dataset context managers."""
+
+import contextlib
+import time
 from typing import Optional
 
 from renku.command.command_builder.command import inject
@@ -71,3 +74,15 @@ def __exit__(self, exc_type, exc_value, traceback):
             self.datasets_provenance = DatasetsProvenance()
             self.datasets_provenance.add_or_update(self.dataset, creator=self.creator)
             self.database_dispatcher.current_database.commit()
+
+
+@contextlib.contextmanager
+def wait_for(delay: float):
+    """Make sure that at least ``delay`` seconds are passed during the execution of the wrapped code block."""
+    start = time.time()
+
+    yield
+
+    exec_time = time.time() - start
+    if exec_time < delay:
+        time.sleep(delay - exec_time)