diff --git a/docs/howtos/artifacts.rst b/docs/howtos/artifacts.rst index b3075507f4..ea22ffcb6f 100644 --- a/docs/howtos/artifacts.rst +++ b/docs/howtos/artifacts.rst @@ -106,13 +106,8 @@ Qiskit Experiments. Saving and loading artifacts ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note:: - This feature is only for those who have access to the cloud service. You can - check whether you do by logging into the IBM Quantum interface - and seeing if you can see the `database `__. - -Artifacts are saved and loaded to and from the cloud service along with the rest of the -:class:`ExperimentData` object. Artifacts are stored as ``.zip`` files in the cloud service grouped by +Artifacts are saved and loaded to and from an experiment service along with the rest of the +:class:`ExperimentData` object. Artifacts are stored as ``.zip`` files in the service grouped by the artifact name. For example, the composite experiment above will generate two artifact files, ``fit_summary.zip`` and ``curve_data.zip``. Each of these zipfiles will contain serialized artifact data in JSON format named by their unique artifact ID: @@ -130,14 +125,8 @@ by their unique artifact ID: print(f"|- {data.artifacts('experiment_notes').artifact_id}.json") Note that for performance reasons, the auto save feature does not apply to artifacts. You must still -call :meth:`.ExperimentData.save` once the experiment analysis has completed to upload artifacts to the -cloud service. - -Note also though individual artifacts can be deleted, currently artifact files cannot be removed from the -cloud service. Instead, you can delete all artifacts of that name -using :meth:`~.delete_artifact` and then call :meth:`.ExperimentData.save`. -This will save an empty file to the service, and the loaded experiment data will not contain -these artifacts. +call :meth:`.ExperimentData.save` once the experiment analysis has completed to save artifacts in the +service. See Also -------- diff --git a/docs/howtos/cloud_service.rst b/docs/howtos/cloud_service.rst deleted file mode 100644 index a26709cb5f..0000000000 --- a/docs/howtos/cloud_service.rst +++ /dev/null @@ -1,150 +0,0 @@ -Save and load experiment data with the cloud service -==================================================== - -.. note:: - The cloud service at `database ` has been - sunset in the move to the new IBM Quantum cloud platform. Saving and loading - to the cloud will not work. We are working on a local saving solution. - -Problem -------- - -You want to save and retrieve experiment data from the cloud service. - -Solution --------- - -Saving -~~~~~~ - -.. note:: - This guide requires :external+qiskit_ibm_runtime:doc:`qiskit-ibm-runtime ` version 0.15 and up, which can be installed with ``python -m pip install qiskit-ibm-runtime``. - For how to migrate from the older ``qiskit-ibm-provider`` to :external+qiskit_ibm_runtime:doc:`qiskit-ibm-runtime `, - consult the `migration guide `_.\ - -You must run the experiment on a real IBM -backend and not a simulator to be able to save the experiment data. This is done by calling -:meth:`~.ExperimentData.save`: - -.. jupyter-input:: - - from qiskit_ibm_runtime import QiskitRuntimeService - from qiskit_experiments.library.characterization import T1 - import numpy as np - - service = QiskitRuntimeService(channel="ibm_quantum") - backend = service.backend("ibm_osaka") - - t1_delays = np.arange(1e-6, 600e-6, 50e-6) - - exp = T1(physical_qubits=(0,), delays=t1_delays) - - t1_expdata = exp.run(backend=backend).block_for_results() - t1_expdata.save() - -.. jupyter-output:: - - You can view the experiment online at - https://quantum.ibm.com/experiments/10a43cb0-7cb9-41db-ad74-18ea6cf63704 - -Loading -~~~~~~~ - -Let's load a `previous T1 -experiment `__ -(requires login to view), which we've made public by editing the ``Share level`` field: - -.. jupyter-input:: - - from qiskit_experiments.framework import ExperimentData - load_expdata = ExperimentData.load("9640736e-d797-4321-b063-d503f8e98571", provider=service) - -Now we can display the figure from the loaded experiment data: - -.. jupyter-input:: - - load_expdata.figure(0) - -.. image:: ./experiment_cloud_service/t1_loaded.png - -The analysis results have been retrieved as well and can be accessed normally. - -.. jupyter-input:: - - results = load_expdata.analysis_results(dataframe=True)) - -Discussion ----------- - -Note that calling :meth:`~.ExperimentData.save` before the experiment is complete will -instantiate an experiment entry in the database, but it will not have -complete data. To fix this, you can call :meth:`~.ExperimentData.save` again once the -experiment is done running. - -Sometimes the metadata of an experiment can be very large and cannot be stored directly in the database. -In this case, a separate ``metadata.json`` file will be stored along with the experiment. Saving and loading -this file is done automatically in :meth:`~.ExperimentData.save` and :meth:`~.ExperimentData.load`. - -Auto-saving an experiment -~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :meth:`~.ExperimentData.auto_save` feature automatically saves changes to the -:class:`.ExperimentData` object to the cloud service whenever it's updated. - -.. jupyter-input:: - - exp = T1(physical_qubits=(0,), delays=t1_delays) - - t1_expdata = exp.run(backend=backend, shots=1000) - t1_expdata.auto_save = True - t1_expdata.block_for_results() - -.. jupyter-output:: - - You can view the experiment online at https://quantum.ibm.com/experiments/cdaff3fa-f621-4915-a4d8-812d05d9a9ca - - -Setting ``auto_save = True`` works by triggering :meth:`.ExperimentData.save`. - -When working with composite experiments, setting ``auto_save`` will propagate this -setting to the child experiments. - -Deleting an experiment -~~~~~~~~~~~~~~~~~~~~~~ - -Both figures and analysis results can be deleted. Note that unless you -have auto save on, the update has to be manually saved to the remote -database by calling :meth:`~.ExperimentData.save`. Because there are two analysis -results, one for the T1 parameter and one for the curve fitting results, we must -delete twice to fully remove the analysis results. - -.. jupyter-input:: - - t1_expdata.delete_figure(0) - t1_expdata.delete_analysis_result(0) - t1_expdata.delete_analysis_result(0) - -.. jupyter-output:: - - Are you sure you want to delete the experiment plot? [y/N]: y - Are you sure you want to delete the analysis result? [y/N]: y - Are you sure you want to delete the analysis result? [y/N]: y - -Tagging and sharing experiments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Tags and notes can be added to experiments to help identify specific experiments in the interface. -For example, an experiment can be tagged and made public with the following code. - -.. jupyter-input:: - - t1_expdata.tags = ['tag1', 'tag2'] - t1_expdata.share_level = "public" - t1_expdata.notes = "Example note." - -Web interface -~~~~~~~~~~~~~ - -You can also view experiment results as well as change the tags and share level at the `IBM Quantum Experiments -pane `__ -on the cloud. diff --git a/docs/howtos/experiment_cloud_service/t1_loaded.png b/docs/howtos/experiment_cloud_service/t1_loaded.png deleted file mode 100644 index a20f738ab3..0000000000 Binary files a/docs/howtos/experiment_cloud_service/t1_loaded.png and /dev/null differ diff --git a/docs/howtos/experiment_service.rst b/docs/howtos/experiment_service.rst new file mode 100644 index 0000000000..a9da2a48bd --- /dev/null +++ b/docs/howtos/experiment_service.rst @@ -0,0 +1,160 @@ +Save and load experiment data with an experiment service +======================================================== + +.. note:: + The cloud service at https://quantum.ibm.com/experiments has been + sunset in the move to the new IBM Quantum cloud platform. Saving and loading + to the cloud will not work. + +Problem +------- + +You want to save and retrieve experiment data from an experiment service. + +Solution +-------- + +The :class:`~.ExperimentData` supports saving and loading data with an +experiment service class satisfying the :class:`~.ExperimentService` protocol. +Here we demonstrate with the :class:`~.LocalExperimentService` class in Qiskit +Experiments. + +Saving +~~~~~~ + +.. note:: + + In the examples below, the service is instantiated with + ``LocalExperimentService()`` which only saves results to + memory. You might want to use ``LocalExperimentService(db_dir=db_dir)`` + instead specifying some local file path ``db_dir`` to save results to. Keep + in mind that :class:`~.LocalExperimentService` was not designed to scale to + saving a large amount of data. + +Saving results is done by calling :meth:`.ExperimentData.save`: + +.. jupyter-execute:: + :hide-code: + + # backend + from qiskit_ibm_runtime.fake_provider import FakeManilaV2 + from qiskit_aer import AerSimulator + backend = AerSimulator.from_backend(FakeManilaV2()) + +.. jupyter-execute:: + + import numpy as np + from qiskit_experiments.library import T1 + from qiskit_experiments.database_service import LocalExperimentService + + delays = np.arange(1.e-6, 300.e-6, 30.e-6) + exp = T1(physical_qubits=(0, ), delays=delays, backend=backend) + + exp_data = exp.run().block_for_results() + service = LocalExperimentService() + exp_data.service = service + exp_data.save() + +Loading +~~~~~~~ + +Let's load the previous experiment again from the service. First, we create a +:class:`~qiskit_experiments.framework.Provider` object that has a +``job(job_id)`` method that can return a +:class:`~qiskit_experiments.framework.Job` instance. Since this is a local +test, a fake provider class that just returns jobs it has been given is used. +Another provider like :class:`~qiskit_ibm_runtime.QiskitRuntimeService` could +be used instead. Also, the provider is only needed for reloading the raw job +data for rerunning analysis. If only the experiment results and figures are +needed, the ``provider`` argument to :meth:`.ExperimentData.load` can omitted. +A warning about not being able to access the job data will be emitted in this +case. + +.. jupyter-execute:: + + from qiskit_experiments.test.utils import FakeProvider + + provider = FakeProvider() + for job in exp_data.jobs(): + provider.add_job(job) + +Now the experiment data can be reloaded: + +.. jupyter-execute:: + + from qiskit_experiments.framework import ExperimentData + load_expdata = ExperimentData.load(exp_data.experiment_id, service=service, provider=provider) + +Now we can display the figure from the loaded experiment data: + +.. jupyter-execute:: + + load_expdata.figure(0) + +The analysis results have been retrieved as well and can be accessed normally. + +.. jupyter-execute:: + + load_expdata.analysis_results(dataframe=True) + +Discussion +---------- + +Note that calling :meth:`~.ExperimentData.save` before the experiment is complete will +instantiate an experiment entry in the database, but it will not have +complete data. To fix this, you can call :meth:`~.ExperimentData.save` again once the +experiment is done running. + +Sometimes the metadata of an experiment can be very large and cannot be stored directly in the database. +In this case, a separate ``metadata.json`` file will be stored along with the experiment. Saving and loading +this file is done automatically in :meth:`~.ExperimentData.save` and :meth:`~.ExperimentData.load`. + +Auto-saving an experiment +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The `auto_save` feature automatically saves changes to the +:class:`.ExperimentData` object to the experiment service whenever it's updated. + +.. jupyter-execute:: + + delays = np.arange(1.e-6, 300.e-6, 30.e-6) + exp = T1(physical_qubits=(0, ), delays=delays, backend=backend) + + exp_data = exp.run() + service = LocalExperimentService() + exp_data.service = service + exp_data.auto_save = True + exp_data.block_for_results() + + +Setting ``auto_save = True`` works by triggering :meth:`.ExperimentData.save` +once the experiment's analysis completes. + +When working with composite experiments, setting ``auto_save`` will propagate this +setting to the child experiments. + +Deleting an experiment +~~~~~~~~~~~~~~~~~~~~~~ + +Both figures and analysis results can be deleted. Note that unless you +have auto save on, the update has to be manually saved to the +database by calling :meth:`~.ExperimentData.save`. Because there are two analysis +results, one for the T1 parameter and one for the curve fitting results, we must +delete twice to fully remove the analysis results. + +.. jupyter-input:: + + t1_expdata.delete_figure(0) + t1_expdata.delete_analysis_result(0) + t1_expdata.delete_analysis_result(0) + +Tagging experiments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Tags and notes can be added to experiments to help identify specific experiments in the interface. +For example, an experiment can be tagged with the following code. + +.. jupyter-input:: + + t1_expdata.tags = ['tag1', 'tag2'] + t1_expdata.notes = "Example note." diff --git a/docs/howtos/experiment_times.rst b/docs/howtos/experiment_times.rst index 8be8b1dc8b..1733c83a61 100644 --- a/docs/howtos/experiment_times.rst +++ b/docs/howtos/experiment_times.rst @@ -21,17 +21,6 @@ are all of type ``datetime.datetime`` and in your local timezone: could not be added to the :class:`.ExperimentData` object for some other reason, :attr:`.ExperimentData.end_datetime` will not update. -.. note:: - The below attributes are only relevant for those who have access to the cloud service. You can - check whether you do by logging into the IBM Quantum interface - and seeing if you can see the `database `__. - -- :attr:`.ExperimentData.creation_datetime` is the time when the experiment data was saved via the - service. This defaults to ``None`` if experiment data has not yet been saved. - -- :attr:`.ExperimentData.updated_datetime` is the time the experiment data entry in the service was - last updated. This defaults to ``None`` if experiment data has not yet been saved. - Discussion ---------- diff --git a/docs/howtos/rerun_analysis.rst b/docs/howtos/rerun_analysis.rst index 0b6b9c4496..d1bcd3be5c 100644 --- a/docs/howtos/rerun_analysis.rst +++ b/docs/howtos/rerun_analysis.rst @@ -11,11 +11,6 @@ execution successfully. Solution -------- -.. note:: - This guide requires :external+qiskit_ibm_runtime:doc:`qiskit-ibm-runtime ` version 0.15 and up, which can be installed with ``python -m pip install qiskit-ibm-runtime``. - For how to migrate from the older ``qiskit-ibm-provider`` to :external+qiskit_ibm_runtime:doc:`qiskit-ibm-runtime `, - consult the `migration guide `_.\ - Once you recreate the exact experiment you ran and all of its parameters and options, you can call the :meth:`.ExperimentData.add_jobs` method with a list of :class:`Job ` objects to generate the new :class:`.ExperimentData` object. @@ -122,4 +117,4 @@ first component experiment. See Also -------- -* `Saving and loading experiment data with the cloud service `_ +* `Saving and loading experiment data with an experiment service `_ diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 1c91354c00..3b920e6b68 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -708,7 +708,7 @@ API Changes for Experiment Authors :class:`~qiskit_experiments.framework.ExtendedJob`, :class:`~qiskit_experiments.framework.Job`, :class:`~qiskit_experiments.framework.BaseProvider`, - :class:`~qiskit_experiments.framework.IBMProvider`, and + ``qiskit_experiments.framework.IBMProvider``, and :class:`~qiskit_experiments.framework.Provider` to document the interfaces needed by :class:`~.ExperimentData` to work with jobs and results. @@ -1522,11 +1522,11 @@ Deprecation Notes .. releasenotes/notes/0.6/experiment-artifacts-c481f4e07226ce9e.yaml @ b'e8531c4f6af9432827bc28c772c5a179737f0c3c' - Direct access to the curve fit summary in :class:`.ExperimentData` has moved from - :meth:`.analysis_results` to :meth:`.artifacts`, where values are stored in the + :meth:`.ExperimentDat.analysis_results` to :meth:`.artifacts`, where values are stored in the :attr:`~.ArtifactData.data` attribute of :class:`.ArtifactData` objects. For example, to access the chi-squared of the fit, ``expdata.analysis_results(0).chisq`` is deprecated in favor of ``expdata.artifacts("fit_summary").data.chisq``. In a future release, the curve fit summary - will be removed from :meth:`.analysis_results` and the option ``return_fit_parameters`` will be + will be removed from :meth:`.ExperimentData.analysis_results` and the option ``return_fit_parameters`` will be removed. For more information on artifacts, see the :doc:`artifacts how-to `. .. releasenotes/notes/0.6/experiment-artifacts-c481f4e07226ce9e.yaml @ b'e8531c4f6af9432827bc28c772c5a179737f0c3c' diff --git a/pyproject.toml b/pyproject.toml index b6041dbbad..ae0a4e0fab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,8 +10,7 @@ dependencies = [ "numpy>=1.17", "scipy>=1.4", "qiskit>=1.3", - "qiskit-ibm-experiment>=0.4.6", - "qiskit_ibm_runtime>=0.34.0", + "qiskit-ibm-runtime>=0.34.0", "matplotlib>=3.4", "uncertainties", "lmfit", @@ -51,8 +50,9 @@ Documentation = "https://qiskit-community.github.io/qiskit-experiments" [project.optional-dependencies] extras = [ "cvxpy>=1.3.2", # for tomography + "pyyaml>=6.0.0", # for storing yaml files in local experiment service + "qiskit-aer>=0.13.2", # for simulating QV circuits or using test backends "scikit-learn", # for discriminators - "qiskit-aer>=0.13.2", ] [project.entry-points."qiskit.synthesis"] diff --git a/qiskit_experiments/curve_analysis/scatter_table.py b/qiskit_experiments/curve_analysis/scatter_table.py index 78df3b496f..47daca7807 100644 --- a/qiskit_experiments/curve_analysis/scatter_table.py +++ b/qiskit_experiments/curve_analysis/scatter_table.py @@ -81,7 +81,7 @@ def __init__(self): def from_dataframe( cls, data: pd.DataFrame, - ) -> "ScatterTable": + ) -> ScatterTable: """Create new dataset with existing dataframe. Args: @@ -99,7 +99,7 @@ def from_dataframe( def _create_new_instance( cls, data: pd.DataFrame, - ) -> "ScatterTable": + ) -> ScatterTable: # A shortcut for creating instance. # This bypasses data formatting and column compatibility check. # User who calls this method must guarantee the quality of the input data. @@ -312,7 +312,7 @@ def filter( filt_data = filt_data.loc[index, :] return ScatterTable._create_new_instance(filt_data) - def iter_by_series_id(self) -> Iterator[tuple[int, "ScatterTable"]]: + def iter_by_series_id(self) -> Iterator[tuple[int, ScatterTable]]: """Iterate over subset of data sorted by the data series index. Yields: @@ -325,7 +325,7 @@ def iter_by_series_id(self) -> Iterator[tuple[int, "ScatterTable"]]: def iter_groups( self, *group_by: str, - ) -> Iterator[tuple[tuple[Any, ...], "ScatterTable"]]: + ) -> Iterator[tuple[tuple[Any, ...], ScatterTable]]: """Iterate over the subset sorted by multiple column values. Args: @@ -420,7 +420,7 @@ def __json_encode__(self) -> dict[str, Any]: } @classmethod - def __json_decode__(cls, value: dict[str, Any]) -> "ScatterTable": + def __json_decode__(cls, value: dict[str, Any]) -> ScatterTable: if not value.get("class", None) == "ScatterTable": raise ValueError("JSON decoded value for ScatterTable is not valid class type.") tmp_df = pd.DataFrame.from_dict(value.get("data", {}), orient="index") diff --git a/qiskit_experiments/database_service/__init__.py b/qiskit_experiments/database_service/__init__.py index b3cd7eaa1d..b6d73f4948 100644 --- a/qiskit_experiments/database_service/__init__.py +++ b/qiskit_experiments/database_service/__init__.py @@ -32,6 +32,31 @@ UnknownComponent to_component +Dataclasses +=========== + +.. autosummary:: + :toctree: ../stubs/ + + DbExperimentData + DbAnalysisResultData + +Constants +========= + +.. autosummary:: + :toctree: ../stubs/ + + ResultQuality + +Services +======== + +.. autosummary:: + :toctree: ../stubs/ + + LocalExperimentService + Exceptions ========== @@ -39,9 +64,19 @@ :toctree: ../stubs/ ExperimentDataError + ExperimentDataSaveFailed ExperimentEntryExists ExperimentEntryNotFound """ -from .exceptions import ExperimentDataError, ExperimentEntryExists, ExperimentEntryNotFound +from .exceptions import ( + ExperimentDataError, + ExperimentDataSaveFailed, + ExperimentEntryExists, + ExperimentEntryNotFound, +) from .device_component import DeviceComponent, Qubit, Resonator, UnknownComponent, to_component +from .constants import ResultQuality +from .db_experiment_data import DbExperimentData +from .db_analysis_result_data import DbAnalysisResultData +from .local_experiment_service import LocalExperimentService diff --git a/qiskit_experiments/database_service/constants.py b/qiskit_experiments/database_service/constants.py new file mode 100644 index 0000000000..e3c7868d0f --- /dev/null +++ b/qiskit_experiments/database_service/constants.py @@ -0,0 +1,41 @@ +# This code is part of Qiskit. +# +# (C) Copyright IBM 2021, 2022. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE.txt file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +"""Experiment constants.""" + +from __future__ import annotations + +import enum + + +class ResultQuality(enum.Enum): + """Possible values for analysis result quality.""" + + BAD = "bad" + GOOD = "good" + UNKNOWN = "unknown" + + @staticmethod + def from_str(quality: str) -> ResultQuality: + """Convert quality to a ResultQuality, defaulting to UNKNOWN""" + try: + result = ResultQuality(str(quality).lower()) + except ValueError: + result = ResultQuality.UNKNOWN + return result + + @staticmethod + def to_str(quality: ResultQuality) -> str: + """Convert quality to string, defaulting to "unknown" """ + if isinstance(quality, ResultQuality): + return quality.value + return "unknown" diff --git a/qiskit_experiments/database_service/db_analysis_result_data.py b/qiskit_experiments/database_service/db_analysis_result_data.py new file mode 100644 index 0000000000..f8fddcc25b --- /dev/null +++ b/qiskit_experiments/database_service/db_analysis_result_data.py @@ -0,0 +1,89 @@ +# This code is part of Qiskit. +# +# (C) Copyright IBM 2022. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE.txt file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +"""Dataclass for analysis result data in the database""" +import copy +import uuid + +from dataclasses import dataclass, field +from typing import Any +from datetime import datetime + +from .constants import ResultQuality +from .device_component import DeviceComponent + + +@dataclass +class DbAnalysisResultData: + """Dataclass for experiment analysis results in the database. + + .. note:: + + The documentation does not currently render all the fields of this + dataclass. + + .. note:: + + This class is named DbAnalysisResultData to avoid confusion with the + :class:`~qiskit_experiments.framework.AnalysisResult` class. + """ + + result_id: str | None = field(default_factory=lambda: str(uuid.uuid4())) + experiment_id: str | None = None + result_type: str | None = None + result_data: dict[str, Any] | None = field(default_factory=dict) + device_components: list[str | DeviceComponent] | str | DeviceComponent | None = field( + default_factory=list + ) + quality: ResultQuality | None = ResultQuality.UNKNOWN + verified: bool | None = False + tags: list[str] | None = field(default_factory=list) + backend_name: str | None = None + creation_datetime: datetime | None = None + updated_datetime: datetime | None = None + chisq: float | None = None + + def __str__(self): + ret = f"Result {self.result_type}" + ret += f"\nResult ID: {self.result_id}" + ret += f"\nExperiment ID: {self.experiment_id}" + ret += f"\nBackend: {self.backend_name}" + ret += f"\nQuality: {self.quality}" + ret += f"\nVerified: {self.verified}" + ret += f"\nDevice components: {self.device_components}" + ret += f"\nData: {self.result_data}" + if self.chisq: + ret += f"\nChi Square: {self.chisq}" + if self.tags: + ret += f"\nTags: {self.tags}" + if self.creation_datetime: + ret += f"\nCreated at: {self.creation_datetime}" + if self.updated_datetime: + ret += f"\nUpdated at: {self.updated_datetime}" + return ret + + def copy(self): + """Creates a deep copy of the data""" + return DbAnalysisResultData( + result_id=self.result_id, + experiment_id=self.experiment_id, + result_type=self.result_type, + result_data=copy.deepcopy(self.result_data), + device_components=copy.copy(self.device_components), + quality=self.quality, + verified=self.verified, + tags=copy.copy(self.tags), + backend_name=self.backend_name, + creation_datetime=self.creation_datetime, + updated_datetime=self.updated_datetime, + chisq=self.chisq, + ) diff --git a/qiskit_experiments/database_service/db_experiment_data.py b/qiskit_experiments/database_service/db_experiment_data.py new file mode 100644 index 0000000000..99d00c4da3 --- /dev/null +++ b/qiskit_experiments/database_service/db_experiment_data.py @@ -0,0 +1,98 @@ +# This code is part of Qiskit. +# +# (C) Copyright IBM 2022. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE.txt file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +"""Dataclass for experiment data in the database""" +import copy +import uuid +from dataclasses import dataclass, field +from datetime import datetime + + +@dataclass +class DbExperimentData: + """Dataclass for experiments in the database. + + .. note:: + + The documentation does not currently render all the fields of this + dataclass. + + .. note:: + + This is named DbExperimentData to avoid confusion with the main + :class:`~qiskit_experiments.framework.ExperimentData` class. + """ + + experiment_id: str = field(default_factory=lambda: str(uuid.uuid4())) + parent_id: str | None = None + experiment_type: str | None = None + backend: str | None = None + tags: list[str] | None = field(default_factory=list) + job_ids: list[str] | None = field(default_factory=list) + share_level: str | None = None + metadata: dict[str, str] | None = field(default_factory=dict) + figure_names: list[str] | None = field(default_factory=list) + notes: str | None = None + hub: str | None = None + group: str | None = None + project: str | None = None + owner: str | None = None + creation_datetime: datetime | None = None + start_datetime: datetime | None = None + end_datetime: datetime | None = None + updated_datetime: datetime | None = None + + def __str__(self): + ret = "" + ret += f"Experiment: {self.experiment_type}" + ret += f"\nExperiment ID: {self.experiment_id}" + if self.backend: + ret += f"\nBackend: {self.backend}" + if self.tags: + ret += f"\nTags: {self.tags}" + ret += f"\nHub\\Group\\Project: {self.hub}\\{self.group}\\{self.project}" + if self.creation_datetime: + ret += f"\nCreated at: {self.creation_datetime}" + if self.start_datetime: + ret += f"\nStarted at: {self.start_datetime}" + if self.end_datetime: + ret += f"\nEnded at: {self.end_datetime}" + if self.updated_datetime: + ret += f"\nUpdated at: {self.updated_datetime}" + if self.metadata: + ret += f"\nMetadata: {self.metadata}" + if self.figure_names: + ret += f"\nFigures: {self.figure_names}" + return ret + + def copy(self): + """Creates a deep copy of the data""" + return DbExperimentData( + experiment_id=self.experiment_id, + parent_id=self.parent_id, + experiment_type=self.experiment_type, + backend=self.backend, + tags=copy.copy(self.tags), + job_ids=copy.copy(self.job_ids), + share_level=self.share_level, + metadata=copy.deepcopy(self.metadata), + figure_names=copy.copy(self.figure_names), + notes=self.notes, + hub=self.hub, + group=self.group, + project=self.project, + owner=self.owner, + creation_datetime=self.creation_datetime, + start_datetime=self.start_datetime, + end_datetime=self.end_datetime, + updated_datetime=self.updated_datetime, + ) diff --git a/qiskit_experiments/database_service/local_experiment_service.py b/qiskit_experiments/database_service/local_experiment_service.py new file mode 100644 index 0000000000..7b6e9757ef --- /dev/null +++ b/qiskit_experiments/database_service/local_experiment_service.py @@ -0,0 +1,1112 @@ +# This code is part of Qiskit. +# +# (C) Copyright IBM 2021, 2022. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE.txt file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +"""Local experiment service for storing experiment data locally.""" + +import json +import logging +import os +from dataclasses import fields +from datetime import datetime, timezone + +import pandas as pd +import numpy as np + +from qiskit_experiments.database_service import ( + DbAnalysisResultData, + DbExperimentData, + ExperimentEntryNotFound, + ExperimentEntryExists, + ResultQuality, +) + +logger = logging.getLogger(__name__) + + +class LocalExperimentService: + """Provides local experiment database services. + + This class provides a service for storing experiment data locally + without connecting to a remote service. Data can be persisted to + disk or kept only in memory. + + .. note:: + + This class is designed for demonstration and testing purposes and will + not scale well to storing many results. It stores all results in memory + and writes all data out to disk at every save. It could serve as a + reference for writing a more scalable system for saving experiments. + """ + + experiment_db_columns = [f.name for f in fields(DbExperimentData)] + results_db_columns = [f.name for f in fields(DbAnalysisResultData)] + + def __init__( + self, + db_dir: str | None = None, + ) -> None: + """LocalExperimentService constructor. + + Args: + db_dir: The directory in which to place the database files. + If None, results are saved in memory only and lost when the + Python process ends. + """ + self._experiments = pd.DataFrame() + self._results = pd.DataFrame() + self._figures = None + self._files = None + self._files_list = {} + self._options = {} + + self.db_dir = db_dir + self.figures_dir = os.path.join(self.db_dir, "figures") if db_dir else None + self.files_dir = os.path.join(self.db_dir, "files") if db_dir else None + self.experiments_file = os.path.join(self.db_dir, "experiments.json") if db_dir else None + self.results_file = os.path.join(self.db_dir, "results.json") if db_dir else None + if db_dir: + self._create_directories() + + self._init_db() + + def _create_directories(self): + """Creates the directories needed for the DB if they do not exist (internal method)""" + dirs_to_create = [self.db_dir, self.figures_dir, self.files_dir] + for dir_to_create in dirs_to_create: + if not os.path.exists(dir_to_create): + os.makedirs(dir_to_create, exist_ok=True) + + def save(self): + """Saves the db to disk""" + if self.db_dir: + self._experiments.to_json(self.experiments_file) + self._results.to_json(self.results_file) + self._save_figures() + self._save_files() + + def _save_figures(self): + """Saves the figures to disk""" + for exp_id in self._figures: + for figure_name, figure_data in self._figures[exp_id].items(): + filename = f"{exp_id}_{figure_name}" + with open(os.path.join(self.figures_dir, filename), "wb") as file: + file.write(figure_data) + + def _save_files(self): + """Saves the files to disk""" + db_files = set() + for exp_id in self._files: + for file_name, file_data in self._files[exp_id].items(): + full_filename = f"{exp_id}_{file_name}" + db_files.add(full_filename) + file_ext = os.path.splitext(full_filename)[1] + mode = "wb" if file_ext == ".zip" else "w" + encoding = None if mode == "wb" else "utf-8" + with open( + os.path.join(self.files_dir, full_filename), mode, encoding=encoding + ) as file: + file.write(file_data) + current_files = set(os.listdir(self.files_dir)) + stray_files = current_files.difference(db_files) + for file in stray_files: + try: + os.unlink(os.path.join(self.files_dir, file)) + except (OSError, FileNotFoundError): + pass + + def _init_db(self): + """Initializes the db (internal method)""" + if self.db_dir: + if os.path.exists(self.experiments_file): + self._experiments = pd.read_json(self.experiments_file) + else: + self._experiments = pd.DataFrame(columns=self.experiment_db_columns) + + if os.path.exists(self.results_file): + self._results = pd.read_json(self.results_file) + else: + self._results = pd.DataFrame(columns=self.results_db_columns) + + if os.path.exists(self.figures_dir): + self._figures = self._get_figure_list() + else: + self._figures = {} + if os.path.exists(self.files_dir): + self._files, self._files_list = self._get_files() + else: + self._files = {} + else: + self._experiments = pd.DataFrame(columns=self.experiment_db_columns) + self._results = pd.DataFrame(columns=self.results_db_columns) + self._figures = {} + self._files = {} + + self.save() + + @property + def options(self) -> dict: + """Return service options dictionary.""" + return self._options + + def backends(self) -> dict: + """Return the backend list from the experiment DB.""" + return self._experiments.backend.unique().tolist() + + def experiments(self) -> list[str]: + """Retrieve experiment ids + + Returns: + A list of experiment ids. + """ + return self._experiments.experiment_id.unique().tolist() + + def experiment( + self, + experiment_id: str, + json_decoder: type[json.JSONDecoder] = None, # pylint: disable=unused-argument + ) -> DbExperimentData: + """Retrieve a single experiment from the database. + + Args: + experiment_id: Experiment ID + json_decoder: Custom JSON decoder (unused in local service) + + Returns: + Retrieved experiment data + + Raises: + ExperimentEntryNotFound: If the experiment is not found + """ + exp = self._experiments.loc[self._experiments.experiment_id == experiment_id] + if exp.empty: + raise ExperimentEntryNotFound(f"Experiment {experiment_id} not found") + + # Convert the first (and only) row to DbExperimentData + exp_dict = self._prepare_experiment_data(exp.iloc[0].to_dict()) + return DbExperimentData(**exp_dict) + + def create_or_update_experiment( + self, + data: "DbExperimentData", + json_encoder: type[json.JSONEncoder] = json.JSONEncoder, # pylint: disable=unused-argument + create: bool = True, + max_attempts: int = 3, + **kwargs, # pylint: disable=unused-argument + ) -> str: + """Creates a new experiment, or updates an existing one. + + Args: + data: The experiment data to save + json_encoder: Custom JSON encoder (unused) + create: Whether to attempt create first + max_attempts: Maximum number of attempts + **kwargs: Additional parameters (ignored for local service) + + Returns: + Experiment ID + """ + + # Convert DbExperimentData to API format + api_data = {f.name: val for f in fields(data) if (val := getattr(data, f.name)) is not None} + for field in ("creation_datetime", "start_datetime", "end_datetime", "updated_datetime"): + if api_data.get(field): + api_data[field] = api_data[field].isoformat() + + def create_exp(): + return self._experiment_create(api_data) + + def update_exp(): + # Remove fields that shouldn't be updated + update_data = api_data.copy() + for field in [ + "experiment_id", + "device_name", + "group_id", + "hub_id", + "project_id", + "type", + "start_time", + "parent_id", + ]: + update_data.pop(field, None) + return self._experiment_update(data.experiment_id, update_data) + + params = {} + result = self._create_or_update(create_exp, update_exp, params, create, max_attempts) + return DbExperimentData(**result) + + def delete_experiment(self, experiment_id: str) -> None: + """Delete an experiment from the database. + + Args: + experiment_id: Experiment ID to delete + + Raises: + ExperimentEntryNotFound: If the experiment is not found + """ + exp = self._experiments.loc[self._experiments.experiment_id == experiment_id] + if exp.empty: + raise ExperimentEntryNotFound(f"Experiment {experiment_id} not found") + + self._experiments.drop( + self._experiments.loc[self._experiments.experiment_id == experiment_id].index, + inplace=True, + ) + self.save() + + def _prepare_experiment_data(self, row: dict) -> dict: + """Prepare database entry fields for dataclass + + Args: + row: Dataframe row containing experiment data + + Returns: + Dictionary suitable for DbExperimentData initialization + """ + data = row.copy() + + # Convert timestamps + for field in ("creation_datetime", "start_datetime", "end_datetime", "updated_datetime"): + if pd.notna(data.get(field)): + data[field] = datetime.fromisoformat(data[field]) + + list_fields = {"tags", "job_ids"} + str_fields = {"notes", "hub", "group", "project", "owner"} + dict_fields = {"metadata"} + + for key, val in data.items(): + if isinstance(val, float) and pd.isna(val): + if key in list_fields: + data[key] = [] + elif key in str_fields: + data[key] = "" + elif key in dict_fields: + data[key] = {} + else: + data[key] = None + + return data + + def _experiment_create(self, data: dict) -> dict: + """Create an experiment (internal method). + + Args: + data: Experiment data. + + Returns: + Experiment data. + + Raises: + ExperimentEntryExists: If the experiment already exists + + """ + data_dict = data.copy() + now = datetime.now(timezone.utc).isoformat() + for time_field in ("start_datetime", "creation_datetime", "updated_datetime"): + if time_field not in data_dict: + data_dict[time_field] = now + if "tags" not in data_dict: + data_dict["tags"] = [] + if "figure_names" not in data_dict: + data_dict["figure_names"] = [] + + exp = self._experiments.loc[self._experiments.experiment_id == data_dict["experiment_id"]] + if not exp.empty: + raise ExperimentEntryExists + + new_df = pd.DataFrame([data_dict], columns=self._experiments.columns) + self._experiments = pd.concat([self._experiments, new_df], ignore_index=True) + self.save() + exp = self._experiments.loc[self._experiments.experiment_id == data_dict["experiment_id"]] + return self._prepare_experiment_data(exp.to_dict("records")[0]) + + def _experiment_update(self, experiment_id: str, new_data: dict) -> dict: + """Update an experiment (internal method). + + Args: + experiment_id: Experiment UUID. + new_data: New experiment data. + + Returns: + Experiment data. + + Raises: + ExperimentEntryNotFound: If the experiment is not found + """ + new_data = new_data.copy() + exp = self._experiments.loc[self._experiments.experiment_id == experiment_id] + if exp.empty: + raise ExperimentEntryNotFound + exp_index = exp.index[0] + new_data["updated_datetime"] = datetime.now(timezone.utc).isoformat() + for key, value in new_data.items(): + self._experiments.at[exp_index, key] = value + self.save() + exp = self._experiments.loc[self._experiments.experiment_id == experiment_id] + return self._prepare_experiment_data(exp.to_dict("records")[0]) + + def analysis_results( + self, + limit: int | None = None, + backend_name: str | None = None, + device_components: list[str] | None = None, + experiment_id: str | None = None, + result_type: str | None = None, + quality: str | list[str] | None = None, + verified: bool | None = None, + tags: list[str] | None = None, + created_at: list | None = None, + json_decoder: type[json.JSONDecoder] = None, # pylint: disable=unused-argument + ) -> list[DbAnalysisResultData]: + """Return a list of analysis results. + + Args: + limit: Number of analysis results to retrieve. + backend_name: Name of the backend. + device_components: A list of device components used for filtering. + experiment_id: Experiment UUID used for filtering. + result_type: Analysis result type used for filtering. + quality: Quality value used for filtering. + verified: Indicates whether this result has been verified. + tags: Filter by tags assigned to analysis results. + created_at: A list of timestamps used to filter by creation time. + json_decoder: Custom JSON decoder (unused in local service) + + Returns: + A list of analysis results. + Raises: + ValueError: If the parameters are unsuitable for filtering + """ + # pylint: disable=unused-argument + df = self._results + + # TODO: skipping device components for now until we conslidate more with the provider service + # (in the qiskit-experiments service there is no operator for device components, + # so the specification for filtering is not clearly defined) + + if experiment_id is not None: + df = df.loc[df.experiment_id == experiment_id] + if result_type is not None: + df = df.loc[df.result_type == result_type] + if backend_name is not None: + df = df.loc[df.backend_name == backend_name] + if quality is not None: + df = df.loc[df.quality == quality] + if verified is not None: + df = df.loc[df.verified == verified] + + if tags is not None: + tags = tags.split(",") + df = df.loc[df.tags.apply(lambda dftags: any(x in dftags for x in tags))] + + df = df.sort_values(["creation_datetime", "experiment_id"], ascending=[False, True]) + + if limit is not None: + df = df.iloc[:limit] + + # Convert dataframe rows to DbAnalysisResultData objects + results = [] + for _, row in df.iterrows(): + result_dict = self._prepare_analysis_result_data(row.to_dict()) + results.append(DbAnalysisResultData(**result_dict)) + + return results + + def analysis_result( + self, + result_id: str, + json_decoder: type[json.JSONDecoder] = None, # pylint: disable=unused-argument + ) -> DbAnalysisResultData: + """Retrieve a single analysis result from the database. + + Args: + result_id: Analysis result ID + json_decoder: Custom JSON decoder (unused in local service) + + Returns: + Retrieved analysis result data + + Raises: + ExperimentEntryNotFound: If the analysis result is not found + """ + result = self._results.loc[self._results.result_id == result_id] + if result.empty: + raise ExperimentEntryNotFound(f"Analysis result {result_id} not found") + + # Convert the first (and only) row to DbAnalysisResultData + result_dict = self._prepare_analysis_result_data(result.iloc[0].to_dict()) + return DbAnalysisResultData(**result_dict) + + def create_or_update_analysis_result( + self, + data: "DbAnalysisResultData", + json_encoder: type[json.JSONEncoder] = json.JSONEncoder, # pylint: disable=unused-argument + create: bool = True, + max_attempts: int = 3, + ) -> str: + """Creates or updates an analysis result. + + Args: + data: The analysis result data to save + json_encoder: Custom JSON encoder (unused) + create: Whether to attempt create first + max_attempts: Maximum number of attempts + + Returns: + Analysis result ID + """ + + # Convert DbAnalysisResultData to API format + api_data = {f.name: val for f in fields(data) if (val := getattr(data, f.name)) is not None} + if api_data.get("quality"): + api_data["quality"] = ResultQuality.to_str(api_data["quality"]) + for field in ("creation_datetime", "updated_datetime"): + if api_data.get(field): + api_data[field] = api_data[field].isoformat() + + def create_result(): + return self._analysis_result_create(api_data) + + def update_result(): + # Remove fields that shouldn't be updated + update_data = api_data.copy() + for field in ["result_id", "experiment_id", "device_components", "type"]: + update_data.pop(field, None) + return self._analysis_result_update(data.result_id, update_data) + + params = {} + result = self._create_or_update(create_result, update_result, params, create, max_attempts) + # result is a dict from analysis_result_create or _analysis_result_update + return result["result_id"] + + def create_analysis_results( + self, + data: list["DbAnalysisResultData"], + blocking: bool = True, # pylint: disable=unused-argument + max_workers: int = 100, # pylint: disable=unused-argument + json_encoder: type[json.JSONEncoder] = json.JSONEncoder, + ): + """Create multiple analysis results (simplified without threading for local). + + Args: + data: List of analysis result data to save + blocking: Ignored for local service (always blocking) + max_workers: Ignored for local service + json_encoder: Custom JSON encoder + + Returns: + Status dictionary with results + """ + successful = [] + failed = [] + + for result_data in data: + try: + self.create_or_update_analysis_result( + result_data, json_encoder=json_encoder, create=True, max_attempts=3 + ) + successful.append(result_data) + except Exception as ex: # pylint: disable=broad-exception-caught + failed.append({"data": result_data, "exception": ex}) + + return { + "running": [], + "done": successful, + "fail": failed, + } + + def delete_analysis_result(self, result_id: str) -> dict: + """Delete an analysis result. + + Args: + result_id: Analysis result ID. + + Raises: + ExperimentEntryNotFound: If the analysis result is not found + """ + result = self._results.loc[self._results.result_id == result_id] + if result.empty: + raise ExperimentEntryNotFound + self._results.drop( + self._results.loc[self._results.result_id == result_id].index, inplace=True + ) + self.save() + + def _analysis_result_create(self, result: dict) -> dict: + """Upload an analysis result. + + Args: + result: The analysis result to upload + + Returns: + Analysis result data. + + Raises: + ValueError: If experiment id is missing + ExperimentEntryNotFound: If experiment is not found + """ + data_dict = result.copy() + + exp_id = data_dict.get("experiment_id") + if exp_id is None: + raise ValueError("Cannot create analysis result without experiment id") + exp = self._experiments.loc[self._experiments.experiment_id == exp_id] + if exp.empty: + raise ExperimentEntryNotFound(f"Experiment {exp_id} not found") + exp_index = exp.index[0] + data_dict["backend_name"] = self._experiments.at[exp_index, "backend"] + now = datetime.now(timezone.utc).isoformat() + if data_dict.get("creation_datetime") is None: + data_dict["creation_datetime"] = now + if data_dict.get("updated_datetime") is None: + data_dict["updated_datetime"] = now + + new_df = pd.DataFrame([data_dict], columns=self._results.columns) + self._results = pd.concat([self._results, new_df], ignore_index=True) + self.save() + return data_dict + + def _analysis_result_update(self, result_id: str, new_data: dict) -> dict: + """Update an analysis result (internal method). + + Args: + result_id: Analysis result ID. + new_data: New analysis result data. + + Returns: + Analysis result data. + + Raises: + ExperimentEntryNotFound: If the analysis result is not found + """ + new_data = new_data.copy() + result = self._results.loc[self._results.result_id == result_id] + if result.empty: + raise ExperimentEntryNotFound + result_index = result.index[0] + new_data["updated_datetime"] = datetime.now(timezone.utc).isoformat() + for key, value in new_data.items(): + self._results.at[result_index, key] = value + self.save() + result = self._results.loc[self._results.result_id == result_id] + return self._prepare_analysis_result_data(result.to_dict("records")[0]) + + def _prepare_analysis_result_data(self, row: dict) -> dict: + """Prepare row dict from database for analysis result dataclass. + + Args: + row: Dataframe row containing analysis result data + + Returns: + Dictionary suitable for DbAnalysisResultData initialization + """ + data = row.copy() + + # Convert timestamps + for field in ("creation_datetime", "updated_datetime"): + if pd.notna(data.get(field)): + data[field] = datetime.fromisoformat(data[field]) + + list_fields = {"device_components", "tags"} + str_fields = {"notes", "hub", "group", "project", "owner"} + dict_fields = {"result_data"} + bool_fields = {"verified"} + + for key, val in data.items(): + if isinstance(val, float) and pd.isna(val): + if key in list_fields: + data[key] = [] + elif key in str_fields: + data[key] = "" + elif key in dict_fields: + data[key] = {} + elif key in bool_fields: + data[key] = False + else: + data[key] = None + + if "quality" in data and isinstance(data["quality"], str): + data["quality"] = ResultQuality.from_str(data["quality"]) + + return data + + def figure( + self, experiment_id: str, figure_name: str, file_name: str | None = None + ) -> int | bytes: + """Retrieve an existing figure. + + Args: + experiment_id: Experiment ID + figure_name: Name of the figure + file_name: Local file to save to (if None, returns bytes) + + Returns: + Size if file_name given, otherwise figure bytes + """ + data = self._figure_get(experiment_id, figure_name) + + if file_name: + with open(file_name, "wb") as file: + num_bytes = file.write(data) + return num_bytes + + return data + + def create_or_update_figure( + self, + experiment_id: str, + figure: str | bytes, + figure_name: str | None = None, + create: bool = True, + max_attempts: int = 3, + ) -> tuple: + """Creates a figure if it doesn't exist, otherwise updates it. + + Args: + experiment_id: Experiment ID + figure: Figure file name or figure data + figure_name: Name of the figure + create: Whether to attempt create first + max_attempts: Maximum number of attempts + + Returns: + Tuple of (figure_name, size) + """ + params = { + "experiment_id": experiment_id, + "figure": figure, + "figure_name": figure_name, + } + return self._create_or_update( + self._figure_create, self._figure_update, params, create, max_attempts + ) + + def create_figures( + self, + experiment_id: str, + figure_list: list[tuple], + blocking: bool = True, # pylint: disable=unused-argument + max_workers: int = 100, # pylint: disable=unused-argument + ): + """Create multiple figures (simplified without threading for local). + + Args: + experiment_id: ID of the experiment + figure_list: List of (figure, name) tuples + blocking: Ignored for local service + max_workers: Ignored for local service + + Returns: + Status dictionary with results + """ + successful = [] + failed = [] + + for figure, figure_name in figure_list: + try: + self.create_or_update_figure( + experiment_id=experiment_id, + figure=figure, + figure_name=figure_name, + create=True, + max_attempts=3, + ) + successful.append((figure, figure_name)) + except Exception as ex: # pylint: disable=broad-exception-caught + failed.append({"data": (figure, figure_name), "exception": ex}) + + return { + "running": [], + "done": successful, + "fail": failed, + } + + def delete_figure(self, experiment_id: str, figure_name: str) -> None: + """Delete an experiment plot. + + Args: + experiment_id: Experiment ID + figure_name: Name of the figure + """ + try: + self._figure_delete(experiment_id, figure_name) + except ExperimentEntryNotFound: + logger.warning("Figure %s not found.", figure_name) + + def _get_figure_list(self): + """Generates the figure dictionary based on stored data on disk""" + figures = {} + for exp_id in self._experiments.experiment_id: + # exp_id should be str to begin with, so just in case + exp_id_string = str(exp_id) + figures_for_exp = {} + for filename in os.listdir(self.figures_dir): + if filename.startswith(exp_id_string): + with open(os.path.join(self.figures_dir, filename), "rb") as file: + figure_data = file.read() + figure_name = filename[len(exp_id_string) + 1 :] + figures_for_exp[figure_name] = figure_data + figures[exp_id] = figures_for_exp + return figures + + def _figure_create( + self, + experiment_id: str, + figure: str | bytes, + figure_name: str | None = None, + ) -> tuple: + """Store a new figure in the database (internal method). + + Args: + experiment_id: ID of the experiment + figure: Figure file name or figure data + figure_name: Name of the figure + + Returns: + Tuple of (figure_name, size) + """ + if figure_name is None: + if isinstance(figure, str): + figure_name = figure + else: + figure_name = f"figure_{datetime.now(timezone.utc).isoformat()}.svg" + + if not figure_name.endswith(".svg"): + figure_name += ".svg" + + if isinstance(figure, str): + with open(figure, "rb") as file: + figure = file.read() + + if experiment_id not in self._figures: + self._figures[experiment_id] = {} + exp_figures = self._figures[experiment_id] + if figure_name in exp_figures: + raise ExperimentEntryExists(f"Figure {figure_name} already exists") + exp_figures[figure_name] = figure + self.save() + + return figure_name, len(figure) + + def _figure_get(self, experiment_id: str, plot_name: str) -> bytes: + """Retrieve an experiment plot (internal method). + + Args: + experiment_id: Experiment UUID. + plot_name: Name of the plot. + + Returns: + Retrieved experiment plot. + + Raises: + ExperimentEntryNotFound: If the figure is not found + """ + + exp_figures = self._figures[experiment_id] + if plot_name not in exp_figures: + raise ExperimentEntryNotFound(f"Figure {plot_name} not found") + return exp_figures[plot_name] + + def _figure_update( + self, + experiment_id: str, + figure: str | bytes, + figure_name: str, + ) -> tuple: + """Update an existing figure (internal method). + + Args: + experiment_id: Experiment ID + figure: Figure file name or figure data + figure_name: Name of the figure + + Returns: + Tuple of (figure_name, size) + """ + if not figure_name.endswith(".svg"): + figure_name += ".svg" + + if isinstance(figure, str): + with open(figure, "rb") as file: + figure = file.read() + + exp_figures = self._figures[experiment_id] + if figure_name not in exp_figures: + raise ExperimentEntryNotFound(f"Figure {figure_name} not found") + exp_figures[figure_name] = figure + self.save() + + return figure_name, len(figure) + + def _figure_delete(self, experiment_id: str, plot_name: str) -> None: + """Delete an experiment plot (internal method). + + Args: + experiment_id: Experiment UUID. + plot_name: Plot file name. + + Raises: + ExperimentEntryNotFound: If the figure is not found + """ + exp_figures = self._figures[experiment_id] + if plot_name not in exp_figures: + raise ExperimentEntryNotFound(f"Figure {plot_name} not found") + del exp_figures[plot_name] + + def files(self, experiment_id: str) -> dict: + """Retrieve the file list for an experiment. + + Args: + experiment_id: Experiment ID + + Returns: + Dictionary with file list metadata (format: {"files": [...]}) + """ + return {"files": self._files_list.get(experiment_id, [])} + + def experiment_has_file(self, experiment_id: str, filename: str) -> bool: + """Check if an experiment has a specific file. + + Args: + experiment_id: Experiment ID + filename: Name of the file to check + + Returns: + True if the file exists, False otherwise + """ + if experiment_id not in self._files: + return False + return filename in self._files[experiment_id] + + def file_upload( + self, + experiment_id: str, + file_name: str, + file_data: dict | str | bytes, + json_encoder: type[json.JSONEncoder] = json.JSONEncoder, + ): + """Uploads a data file to the DB. + + Args: + experiment_id: The experiment the file belongs to + file_name: The expected filename + file_data: Dictionary or JSON string or bytes to save + json_encoder: Custom JSON encoder + + Raises: + RuntimeError: pyyaml not available and a yaml file requested + """ + # Ensure proper file extension + if not ( + file_name.endswith(".json") or file_name.endswith(".yaml") or file_name.endswith(".zip") + ): + file_name += ".json" + + if isinstance(file_data, dict): + if file_name.endswith(".yaml"): + try: + import yaml + except ImportError as err: + raise RuntimeError("pyyaml required to store yaml file!") from err + file_data = yaml.dump(file_data) + elif file_name.endswith(".json"): + file_data = json.dumps(file_data, cls=json_encoder) + + if experiment_id not in self._files_list: + self._files_list[experiment_id] = [] + if experiment_id not in self._files: + self._files[experiment_id] = {} + size = len(file_data) + new_file_element = { + "Key": file_name, + "Size": size, + "LastModified": datetime.now(timezone.utc).isoformat(), + } + self._files_list[experiment_id].append(new_file_element) + self._files[experiment_id][file_name] = file_data + self.save() + + def file_delete( + self, + experiment_id: str, + file_name: str, + ): + """Delete a file from the database""" + if not ( + file_name.endswith(".json") or file_name.endswith(".yaml") or file_name.endswith(".zip") + ): + file_name += ".json" + + if experiment_id not in self._files: + raise ExperimentEntryNotFound + if file_name not in self._files[experiment_id]: + raise ExperimentEntryNotFound + + del self._files[experiment_id][file_name] + self._files_list[experiment_id] = [ + e for e in self._files_list[experiment_id] if e["Key"] != file_name + ] + self.save() + + def file_download( + self, + experiment_id: str, + file_name: str, + json_decoder: type[json.JSONDecoder] = json.JSONDecoder, + ) -> dict: + """Downloads a data file from the DB. + + Args: + experiment_id: The experiment the file belongs to + file_name: The filename + json_decoder: Custom JSON decoder + + Returns: + Deserialized file data + + Raises: + ExperimentEntryNotFound: File not found + RuntimeError: pyyaml not available and a yaml file requested + """ + if not ( + file_name.endswith(".json") or file_name.endswith(".yaml") or file_name.endswith(".zip") + ): + file_name += ".json" + + if experiment_id not in self._files: + raise ExperimentEntryNotFound + if file_name not in self._files[experiment_id]: + raise ExperimentEntryNotFound + if file_name.endswith(".yaml"): + try: + import yaml + except ImportError as err: + raise RuntimeError("pyyaml required to load yaml file!") from err + return yaml.safe_load(self._files[experiment_id][file_name]) + elif file_name.endswith(".json"): + return json.loads(self._files[experiment_id][file_name], cls=json_decoder) + return self._files[experiment_id][file_name] + + def _get_files(self): + """Generates the figure dictionary based on stored data on disk""" + files = {} + files_list = {} + for exp_id in self._experiments.experiment_id: + # exp_id should be str to begin with, so just in case + exp_id_string = str(exp_id) + file_list_for_exp = [] + files_for_exp = {} + for filename in os.listdir(self.files_dir): + if filename.startswith(exp_id_string): + file_full_path = os.path.join(self.files_dir, filename) + file_ext = os.path.splitext(filename)[1] + mode = "rb" if file_ext == ".zip" else "r" + encoding = None if mode == "rb" else "utf-8" + with open(file_full_path, mode, encoding=encoding) as file: + file_data = file.read() + file_size = len(file_data) + file_name = filename[len(exp_id_string) + 1 :] + files_for_exp[file_name] = file_data + new_file_element = { + "Key": file_name, + "Size": file_size, + "LastModified": os.path.getmtime(file_full_path), + } + file_list_for_exp.append(new_file_element) + files_list[exp_id] = file_list_for_exp + files[exp_id] = files_for_exp + return files, files_list + + def _experiment_files_get(self, experiment_id: str) -> dict[str, list[str]]: + """Retrieve experiment related files (internal method). + + Args: + experiment_id: Experiment ID. + + Returns: + Experiment files. + """ + return {"files": self._files_list.get(experiment_id, [])} + + def _experiment_file_download_impl( + self, experiment_id: str, file_name: str, json_decoder: type[json.JSONDecoder] + ) -> dict: + """Downloads a data file from the DB (internal implementation) + + Args: + experiment_id: Experiment ID. + file_name: The name of the data file + json_decoder: Custom decoder to use to decode the retrieved experiment. + + Returns: + The Dictionary of contents of the file + + Raises: + ExperimentEntryNotFound: if experiment or file not found + """ + if experiment_id not in self._files: + raise ExperimentEntryNotFound + if file_name not in self._files[experiment_id]: + raise ExperimentEntryNotFound + if file_name.endswith(".yaml"): + try: + import yaml + except ImportError as err: + raise RuntimeError("pyyaml required to load yaml file!") from err + return yaml.safe_load(self._files[experiment_id][file_name]) + elif file_name.endswith(".json"): + return json.loads(self._files[experiment_id][file_name], cls=json_decoder) + return self._files[experiment_id][file_name] + + def _create_or_update( + self, + create_func, + update_func, + params, + create: bool = True, + max_attempts: int = 3, + ): + """Creates or updates a database entry using the given functions. + + Args: + create_func: Function to create new entry + update_func: Function to update existing entry + params: Parameters to pass to the functions + create: Whether to attempt create first + max_attempts: Maximum number of attempts + + Returns: + Result from the successful function call + """ + attempts = 0 + success = False + result = None + while attempts < max_attempts and not success: + attempts += 1 + if create: + try: + result = create_func(**params) + success = True + except ExperimentEntryExists: + create = False + else: + try: + result = update_func(**params) + success = True + except ExperimentEntryNotFound: + create = True + return result + + def _convert_db_to_dict(self, dataframe: pd.DataFrame): + """Prepares db values for dataclasses""" + result = dataframe.replace({np.nan: None}).to_dict("records")[0] + return result diff --git a/qiskit_experiments/database_service/utils.py b/qiskit_experiments/database_service/utils.py index 073d0cf3e4..2b585881f8 100644 --- a/qiskit_experiments/database_service/utils.py +++ b/qiskit_experiments/database_service/utils.py @@ -27,11 +27,6 @@ import dateutil.parser from dateutil import tz -from qiskit_ibm_experiment import ( - IBMExperimentEntryExists, - IBMExperimentEntryNotFound, -) - from .exceptions import ExperimentEntryNotFound, ExperimentEntryExists, ExperimentDataError LOG = logging.getLogger(__name__) @@ -91,7 +86,7 @@ def objs_to_zip( zip_file.writestr(f"{filename}.json", json.dumps(data, cls=json_encoder)) zip_buffer.seek(0) - return zip_buffer + return zip_buffer.read() def zip_to_objs(zip_bytes: bytes, json_decoder: json.JSONDecoder | None = None) -> Iterator[any]: @@ -163,8 +158,6 @@ def save_data( ExperimentDataError: If unable to determine whether the entry exists. """ attempts = 0 - no_entry_exception = (ExperimentEntryNotFound, IBMExperimentEntryNotFound) - dup_entry_exception = (ExperimentEntryExists, IBMExperimentEntryExists) try: kwargs = {} @@ -180,13 +173,13 @@ def save_data( kwargs.update(new_data) kwargs.update(update_data) return True, new_func(**kwargs) - except dup_entry_exception: + except ExperimentEntryExists: is_new = False else: try: kwargs.update(update_data) return True, update_func(**kwargs) - except no_entry_exception: + except ExperimentEntryNotFound: is_new = True raise ExperimentDataError("Unable to determine the existence of the entry.") except Exception: # pylint: disable=broad-except diff --git a/qiskit_experiments/framework/__init__.py b/qiskit_experiments/framework/__init__.py index b4a35eaad1..51aad8777c 100644 --- a/qiskit_experiments/framework/__init__.py +++ b/qiskit_experiments/framework/__init__.py @@ -94,7 +94,7 @@ FigureData Provider BaseProvider - IBMProvider + ExperimentService Job BaseJob ExtendedJob @@ -149,8 +149,8 @@ from .provider_interfaces import ( BaseJob, BaseProvider, + ExperimentService, ExtendedJob, - IBMProvider, Job, MeasLevel, MeasReturnType, diff --git a/qiskit_experiments/framework/analysis_result.py b/qiskit_experiments/framework/analysis_result.py index 8bb59d9bd4..2cbf55b3a0 100644 --- a/qiskit_experiments/framework/analysis_result.py +++ b/qiskit_experiments/framework/analysis_result.py @@ -21,8 +21,6 @@ import uncertainties -from qiskit_ibm_experiment import IBMExperimentService, AnalysisResultData -from qiskit_ibm_experiment import ResultQuality from qiskit.exceptions import QiskitError from qiskit_experiments.framework.json import ( @@ -30,10 +28,15 @@ ExperimentDecoder, _serialize_safe_float, ) - -from qiskit_experiments.database_service.device_component import DeviceComponent, to_component -from qiskit_experiments.database_service.exceptions import ExperimentDataError +from qiskit_experiments.database_service import ( + DbAnalysisResultData, + DeviceComponent, + ExperimentDataError, + ResultQuality, + to_component, +) from qiskit_experiments.framework.package_deps import qiskit_version +from qiskit_experiments.framework.provider_interfaces import ExperimentService LOG = logging.getLogger(__name__) @@ -78,12 +81,6 @@ class AnalysisResult: _extra_data = {} - RESULT_QUALITY_TO_TEXT = { - ResultQuality.GOOD: "good", - ResultQuality.BAD: "bad", - ResultQuality.UNKNOWN: "unknown", - } - def __init__( self, name: str = None, @@ -92,11 +89,11 @@ def __init__( experiment_id: str = None, result_id: str | None = None, chisq: float | None = None, - quality: str | None = RESULT_QUALITY_TO_TEXT[ResultQuality.UNKNOWN], + quality: str | None = ResultQuality.UNKNOWN.value, extra: dict[str, Any] | None = None, verified: bool = False, tags: list[str] | None = None, - service: IBMExperimentService | None = None, + service: ExperimentService | None = None, source: dict[str, str] | None = None, ) -> "AnalysisResult": """AnalysisResult constructor. @@ -120,12 +117,12 @@ def __init__( The AnalysisResult object. """ # Data to be stored in DB. - self._db_data = AnalysisResultData( + self._db_data = DbAnalysisResultData( experiment_id=experiment_id, result_id=result_id or str(uuid.uuid4()), result_type=name, chisq=chisq, - quality=quality, + quality=ResultQuality.from_str(quality), verified=verified, tags=tags or [], ) @@ -145,12 +142,11 @@ def __init__( except AttributeError: pass - def set_data(self, data: AnalysisResultData): + def set_data(self, data: DbAnalysisResultData): """Sets the analysis data stored in the class""" self._db_data = data new_device_components = [to_component(comp) for comp in self._db_data.device_components] self._db_data.device_components = new_device_components - self._db_data.quality = self.RESULT_QUALITY_TO_TEXT.get(self._db_data.quality, "unknown") @classmethod def default_source(cls) -> dict[str, str]: @@ -189,7 +185,7 @@ def format_result_data(value, extra, chisq, source): return result_data @classmethod - def load(cls, result_id: str, service: IBMExperimentService) -> "AnalysisResult": + def load(cls, result_id: str, service: ExperimentService) -> "AnalysisResult": """Load a saved analysis result from a database service. Args: @@ -399,7 +395,7 @@ def tags(self, new_tags: list[str]) -> None: self.save() @property - def service(self) -> IBMExperimentService | None: + def service(self) -> ExperimentService | None: """Return the database service. Returns: @@ -409,7 +405,7 @@ def service(self) -> IBMExperimentService | None: return self._service @service.setter - def service(self, service: IBMExperimentService) -> None: + def service(self, service: ExperimentService) -> None: """Set the service to be used for storing result data in a database. Args: diff --git a/qiskit_experiments/framework/analysis_result_data.py b/qiskit_experiments/framework/analysis_result_data.py index 956c23453e..0b16fd062b 100644 --- a/qiskit_experiments/framework/analysis_result_data.py +++ b/qiskit_experiments/framework/analysis_result_data.py @@ -16,7 +16,7 @@ import logging from typing import Any -from qiskit_experiments.database_service.device_component import DeviceComponent +from qiskit_experiments.database_service import DeviceComponent, ResultQuality LOG = logging.getLogger(__name__) @@ -30,7 +30,7 @@ class AnalysisResultData: value: Any experiment: str = None chisq: float | None = None - quality: str | None = None + quality: str = ResultQuality.UNKNOWN.value experiment_id: str | None = None result_id: str | None = None tags: list = dataclasses.field(default_factory=list) diff --git a/qiskit_experiments/framework/containers/artifact_data.py b/qiskit_experiments/framework/containers/artifact_data.py index 0a89622d65..06d47d482c 100644 --- a/qiskit_experiments/framework/containers/artifact_data.py +++ b/qiskit_experiments/framework/containers/artifact_data.py @@ -32,7 +32,7 @@ class ArtifactData: fit status, and any other JSON-based data needed to serialize experiments and experiment data. Attributes: - name: The name of the artifact. When saved to the cloud service, this will be the name + name: The name of the artifact. When saved in an experiment service, this will be the name of the zipfile this artifact object is stored in. data: The artifact payload. artifact_id: Artifact ID. Must be unique inside an :class:`ExperimentData` object. diff --git a/qiskit_experiments/framework/experiment_data.py b/qiskit_experiments/framework/experiment_data.py index fab589eed3..6d94b179e2 100644 --- a/qiskit_experiments/framework/experiment_data.py +++ b/qiskit_experiments/framework/experiment_data.py @@ -40,13 +40,6 @@ from qiskit.providers import Backend from qiskit.primitives import BitArray, SamplerPubResult -from qiskit_ibm_experiment import ( - IBMExperimentService, - ExperimentData as ExperimentDataclass, - AnalysisResultData as AnalysisResultDataclass, - ResultQuality, -) - from qiskit_experiments.framework.json import ExperimentEncoder, ExperimentDecoder from qiskit_experiments.database_service.utils import ( plot_to_svg_bytes, @@ -62,15 +55,18 @@ from qiskit_experiments.framework import ExperimentStatus, AnalysisStatus, AnalysisCallback from qiskit_experiments.framework.deprecation import warn_from_qe from qiskit_experiments.framework.package_deps import qiskit_version -from qiskit_experiments.database_service.exceptions import ( +from qiskit_experiments.database_service import ( + DbAnalysisResultData, + DbExperimentData, ExperimentDataError, ExperimentEntryNotFound, ExperimentDataSaveFailed, + ResultQuality, ) from qiskit_experiments.database_service.utils import objs_to_zip, zip_to_objs from .containers.figure_data import FigureData, FigureType -from .provider_interfaces import Job, Provider +from .provider_interfaces import ExperimentService, Job, Provider if TYPE_CHECKING: @@ -125,11 +121,6 @@ def get_job_status(job: Job) -> JobStatus: class ExperimentData: """Experiment data container class. - .. note:: - Saving experiment data to the cloud database is currently a limited access feature. You can - check whether you have access by logging into the IBM Quantum interface - and seeing if you can see the `database `__. - This class handles the following: 1. Storing the data related to an experiment: raw data, metadata, analysis results, @@ -137,7 +128,7 @@ class ExperimentData: 2. Managing jobs and adding data from jobs automatically 3. Saving and loading data from the database service - The field ``db_data`` is a dataclass (``ExperimentDataclass``) containing + The field ``db_data`` is a :class:`.DbExperimentData` dataclass containing all the data that can be stored in the database and loaded from it, and as such is subject to strict conventions. @@ -159,13 +150,13 @@ def __init__( self, experiment: BaseExperiment | None = None, backend: Backend | None = None, - service: IBMExperimentService | None = None, + service: ExperimentService | None = None, provider: Provider | None = None, parent_id: str | None = None, job_ids: list[str] | None = None, child_data: list[ExperimentData] | None = None, verbose: bool | None = True, - db_data: ExperimentDataclass | None = None, + db_data: DbExperimentData | None = None, start_datetime: datetime | None = None, **kwargs, ): @@ -176,8 +167,7 @@ def __init__( backend: Backend the experiment runs on. This overrides the backend in the experiment object. service: The service that stores the experiment results to the database - provider: The provider used for the experiments - (can be used to automatically obtain the service) + provider: The provider used for the experiment parent_id: ID of the parent experiment data in the setting of a composite experiment job_ids: IDs of jobs submitted for the experiment. @@ -187,15 +177,6 @@ def __init__( This overrides other db parameters. start_datetime: The time when the experiment started running. If none, defaults to the current time. - - Additional info: - In order to save the experiment data to the cloud service, the class - needs access to the experiment service provider. It can be obtained - via three different methods, given here by priority: - - 1. Passing it directly via the ``service`` parameter. - 2. Implicitly obtaining it from the ``provider`` parameter. - 3. Implicitly obtaining it from the ``backend`` parameter, using that backend's provider. """ if experiment is not None: backend = backend or experiment.backend @@ -223,7 +204,7 @@ def __init__( metadata["_source"] = source experiment_id = kwargs.get("experiment_id", str(uuid.uuid4())) if db_data is None: - self._db_data = ExperimentDataclass( + self._db_data = DbExperimentData( experiment_id=experiment_id, experiment_type=experiment_type, parent_id=parent_id, @@ -254,8 +235,8 @@ def __init__( # qiskit_ibm_runtime.IBMBackend stores its Provider-like object in # the "service" attribute self._provider = backend.service - # Experiment service like qiskit_ibm_experiment.IBMExperimentService, - # not to be confused with qiskit_ibm_runtime.QiskitRuntimeService + # ExperimentService service not to be confused with + # qiskit_ibm_runtime.QiskitRuntimeService self._service = service self._auto_save = False self._created_in_db = False @@ -362,10 +343,14 @@ def metadata(self) -> dict: def creation_datetime(self) -> datetime: """Return the creation datetime of this experiment data. - Returns: - The timestamp when this experiment data was saved to the cloud service - in the local timezone. + .. note:: + Typically this value is set within the experiment service. It might + not be available on a new :class:`.ExperimentData` instance prior + to reloading it from the service. + + Returns: + The timestamp when this experiment data was saved to the experiment service. """ return self._db_data.creation_datetime @@ -374,8 +359,7 @@ def start_datetime(self) -> datetime: """Return the start datetime of this experiment data. Returns: - The timestamp when this experiment began running in the local timezone. - + The timestamp when this experiment began running. """ return self._db_data.start_datetime @@ -387,9 +371,14 @@ def start_datetime(self, new_start_datetime: datetime) -> None: def updated_datetime(self) -> datetime: """Return the update datetime of this experiment data. + .. note:: + + Typically this value is set within the experiment service. It might + not be available on a new :class:`.ExperimentData` instance prior + to reloading it from the service. + Returns: - The timestamp when this experiment data was last updated in the service - in the local timezone. + The timestamp when this experiment data was last updated in the service. """ return self._db_data.updated_datetime @@ -417,8 +406,6 @@ def end_datetime(self) -> datetime: Returns: The timestamp when the last job of this experiment finished - in the local timezone. - """ return self._db_data.end_datetime @@ -432,7 +419,6 @@ def hub(self) -> str: Returns: The hub of this experiment data. - """ return self._db_data.hub @@ -452,7 +438,6 @@ def project(self) -> str: Returns: The project of this experiment data. - """ return self._db_data.project @@ -527,9 +512,7 @@ def share_level(self, new_level: str) -> None: to this experiment itself and its descendants. Args: - new_level: New experiment share level. Valid share levels are provider- - specified. For example, IBM Quantum experiment service allows - "public", "hub", "group", "project", and "private". + new_level: New experiment share level. """ self._db_data.share_level = new_level for data in self._child_data.values(): @@ -633,7 +616,7 @@ def _clear_results(self): self._db_data.figure_names.clear() @property - def service(self) -> IBMExperimentService | None: + def service(self) -> ExperimentService | None: """Return the database service. Returns: @@ -642,7 +625,7 @@ def service(self) -> IBMExperimentService | None: return self._service @service.setter - def service(self, service: IBMExperimentService) -> None: + def service(self, service: ExperimentService) -> None: """Set the service to be used for storing experiment data Args: @@ -653,30 +636,7 @@ def service(self, service: IBMExperimentService) -> None: """ self._set_service(service) - def _infer_service(self, warn: bool): - """Try to configure service if it has not been configured - - This method should be called before any method that needs to work with - the experiment service. - - Args: - warn: Warn if the service could not be set up from the backend or - provider attributes. - - Returns: - True if a service instance has been set up - """ - if self.service is None: - self.service = self.get_service_from_backend(self.backend) - if self.service is None: - self.service = self.get_service_from_provider(self.provider) - - if warn and self.service is None: - LOG.warning("Experiment service has not been configured. Can not save!") - - return self.service is not None - - def _set_service(self, service: IBMExperimentService) -> None: + def _set_service(self, service: ExperimentService) -> None: """Set the service to be used for storing experiment data, to this experiment itself and its descendants. @@ -694,32 +654,6 @@ def _set_service(self, service: IBMExperimentService) -> None: for data in self.child_data(): data._set_service(service) - @staticmethod - def get_service_from_backend(backend) -> IBMExperimentService | None: - """Initializes the service from the backend data""" - # backend.provider is not checked since currently the only viable way - # to set up the experiment service is using the credentials from - # QiskitRuntimeService on a qiskit_ibm_runtime.IBMBackend. - provider = getattr(backend, "service", None) - return ExperimentData.get_service_from_provider(provider) - - @staticmethod - def get_service_from_provider(provider) -> IBMExperimentService | None: - """Initializes the service from the provider data""" - if not hasattr(provider, "active_account"): - return None - - account = provider.active_account() - url = account.get("url") - token = account.get("token") - try: - if url is not None and token is not None: - return IBMExperimentService(token=token, url=url) - except Exception: # pylint: disable=broad-except - LOG.warning("Failed to connect to experiment service", exc_info=True) - - return None - @property def provider(self) -> Provider | None: """Return the backend provider. @@ -1174,8 +1108,7 @@ def _retrieve_data(self): if not self._result_data.copy(): # Adding warning so the user will have indication why the analysis may fail. LOG.warning( - "Provider for ExperimentData object doesn't exist, resulting in a failed attempt to" - " retrieve data from the server; no stored result data exists" + "Provider for ExperimentData object is unset. Job data is not available." ) return retrieved_jobs = {} @@ -1351,7 +1284,7 @@ def add_figures( self._db_data.figure_names.append(fig_name) save = save_figure if save_figure is not None else self.auto_save - if save and self._infer_service(warn=True): + if save: if isinstance(figure, pyplot.Figure): figure = plot_to_svg_bytes(figure) self.service.create_or_update_figure( @@ -1369,7 +1302,7 @@ def delete_figure( self, figure_key: str | int, ) -> str: - """Add the experiment figure. + """Delete the experiment figure. Args: figure_key: Name or index of the figure. @@ -1384,8 +1317,9 @@ def delete_figure( del self._figures[figure_key] self._deleted_figures.append(figure_key) + self._db_data.figure_names.remove(figure_key) - if self.auto_save and self._infer_service(warn=True): + if self.auto_save: with service_exception_to_warning(): self.service.delete_figure(experiment_id=self.experiment_id, figure_name=figure_key) self._deleted_figures.remove(figure_key) @@ -1434,7 +1368,7 @@ def figure( figure_key = self._find_figure_key(figure_key) figure_data = self._figures.get(figure_key, None) - if figure_data is None and self._infer_service(warn=False): + if figure_data is None and self.service: figure = self.service.figure(experiment_id=self.experiment_id, figure_name=figure_key) figure_data = FigureData(figure=figure, name=figure_key) self._figures[figure_key] = figure_data @@ -1606,7 +1540,7 @@ def add_analysis_results( created_time=created_time, **extra_values, ) - if self.auto_save and self._infer_service(warn=True): + if self.auto_save: service_result = _series_to_service_result( series=self._analysis_results.get_data(uid, columns="all").iloc[0], service=self.service, @@ -1630,14 +1564,16 @@ def delete_analysis_result( Raises: ExperimentEntryNotFound: If analysis result not found or multiple entries are found. """ + deleted_data = self._analysis_results.get_data(result_key, columns="all") + result_ids = deleted_data.result_id.unique() uids = self._analysis_results.del_data(result_key) - if self.auto_save and self._infer_service(warn=True): + if self.auto_save: with service_exception_to_warning(): - for uid in uids: + for uid in result_ids: self.service.delete_analysis_result(result_id=uid) else: - self._deleted_analysis_results.extend(uids) + self._deleted_analysis_results.extend(result_ids) return uids @@ -1654,9 +1590,8 @@ def _retrieve_analysis_results(self, refresh: bool = False): experiment_id=self.experiment_id, limit=None, json_decoder=self._json_decoder ) for result in retrieved_results: - # Canonicalize IBM specific data structure. # TODO define proper data schema on frontend and delegate this to service. - cano_quality = AnalysisResult.RESULT_QUALITY_TO_TEXT.get(result.quality, "unknown") + cano_quality = ResultQuality.to_str(result.quality) cano_components = [to_component(c) for c in result.device_components] extra = result.result_data["_extra"] if result.chisq is not None: @@ -1781,7 +1716,7 @@ def analysis_results( ), DeprecationWarning, ) - # Convert back into List[AnalysisResult] which is payload for IBM experiment service. + # Convert back into List[AnalysisResult]. # This will be removed in future version. tmp_df = self._analysis_results.get_data(index, columns="all") service_results = [] @@ -1813,34 +1748,32 @@ def save_metadata(self) -> None: This method does not save analysis results, figures, or artifacts. Use :meth:`save` for general saving of all experiment data. - See :meth:`qiskit.providers.experiment.IBMExperimentService.create_experiment` - for fields that are saved. + This method uses + :meth:`qiskit_experiments.database_service.ExperimentService.create_or_update_experiment` """ - self._infer_service(warn=False) self._save_experiment_metadata() for data in self.child_data(): data.save_metadata() def _save_experiment_metadata(self, suppress_errors: bool = True) -> None: """Save this experiments metadata to a database service. + Args: suppress_errors: should the method catch exceptions (true) or pass them on, potentially aborting the experiment (false) + Raises: QiskitError: If the save to the database failed + .. note:: This method does not save analysis results nor figures. Use :meth:`save` for general saving of all experiment data. - See :meth:`qiskit.providers.experiment.IBMExperimentService.create_experiment` - for fields that are saved. + This method uses + :meth:`qiskit_experiments.database_service.ExperimentService.create_or_update_experiment` """ if not self.service: - LOG.warning( - "Experiment cannot be saved because no experiment service is available. " - "An experiment service is available, for example, " - "when using an IBM Quantum backend." - ) + LOG.warning("Experiment cannot be saved because no experiment service is available.") return try: handle_metadata_separately = self._metadata_too_large() @@ -1851,11 +1784,10 @@ def _save_experiment_metadata(self, suppress_errors: bool = True) -> None: result = self.service.create_or_update_experiment( self._db_data, json_encoder=self._json_encoder, create=not self._created_in_db ) - if isinstance(result, dict): - created_datetime = result.get("created_at", None) - updated_datetime = result.get("updated_at", None) - self._db_data.creation_datetime = parse_utc_datetime(created_datetime) - self._db_data.updated_datetime = parse_utc_datetime(updated_datetime) + if hasattr(result, "start_datetime"): + self._db_data.creation_datetime = result.start_datetime + if hasattr(result, "updated_datetime"): + self._db_data.updated_datetime = result.updated_datetime self._created_in_db = True @@ -1913,13 +1845,8 @@ def save( additional tags or notes) use :meth:`save_metadata`. """ # TODO - track changes - self._infer_service(warn=False) if not self.service: - LOG.warning( - "Experiment cannot be saved because no experiment service is available. " - "An experiment service is available, for example, " - "when using an IBM Quantum backend." - ) + LOG.warning("Experiment cannot be saved because no experiment service is available.") if suppress_errors: return else: @@ -2021,27 +1948,14 @@ def save( except Exception: # pylint: disable=broad-except: LOG.error("Unable to save artifacts: %s", traceback.format_exc()) - # Upload a blank file if the whole file should be deleted - # TODO: replace with direct artifact deletion when available for artifact_name in self._deleted_artifacts.copy(): - try: # Don't overwrite with a blank file if there's still artifacts with this name - self.artifacts(artifact_name) - except Exception: # pylint: disable=broad-except: - with service_exception_to_warning(): - self.service.file_upload( - experiment_id=self.experiment_id, - file_name=f"{artifact_name}.zip", - file_data=None, - ) - # Even if we didn't overwrite an artifact file, we don't need to keep this because - # an existing artifact(s) needs to be deleted to delete the artifact file in the future + with service_exception_to_warning(): + self.service.file_delete( + experiment_id=self.experiment_id, + file_name=f"{artifact_name}.zip", + ) self._deleted_artifacts.remove(artifact_name) - if not self.service.local and self.verbose: - print( - "You can view the experiment online at " - f"https://quantum.ibm.com/experiments/{self.experiment_id}" - ) # handle children, but without additional prints if save_children: for data in self._child_data.values(): @@ -2490,7 +2404,7 @@ def child_data( def load( cls, experiment_id: str, - service: IBMExperimentService | None = None, + service: ExperimentService | None = None, provider: Provider | None = None, ) -> ExperimentData: """Load a saved experiment data from a database service. @@ -2498,12 +2412,7 @@ def load( Args: experiment_id: Experiment ID. service: the database service. - provider: an IBMProvider required for loading job data and - can be used to initialize the service. When using - :external+qiskit_ibm_runtime:doc:`qiskit-ibm-runtime `, - this is the :class:`~qiskit_ibm_runtime.QiskitRuntimeService` and should - not be confused with the experiment database service - :meth:`qiskit_ibm_experiment.IBMExperimentService`. + provider: provider to load job results from Returns: The loaded experiment data. @@ -2511,11 +2420,7 @@ def load( ExperimentDataError: If not service nor provider were given. """ if service is None: - if provider is None: - raise ExperimentDataError( - "Loading an experiment requires a valid Qiskit provider or experiment service." - ) - service = cls.get_service_from_provider(provider) + raise ExperimentDataError("Loading an experiment requires a valid experiment service.") data = service.experiment(experiment_id, json_decoder=cls._json_decoder) if service.experiment_has_file(experiment_id, cls._metadata_filename): metadata = service.file_download( @@ -2546,9 +2451,7 @@ def load( expdata._created_in_db = True child_data_ids = expdata.metadata.pop("child_data_ids", []) - child_data = [ - ExperimentData.load(child_id, service, provider) for child_id in child_data_ids - ] + child_data = [ExperimentData.load(child_id, service) for child_id in child_data_ids] expdata._set_child_data(child_data) return expdata @@ -2929,7 +2832,7 @@ def service_exception_to_warning(): def _series_to_service_result( series: pd.Series, - service: IBMExperimentService, + service: ExperimentService, auto_save: bool, source: dict[str, Any] | None = None, ) -> AnalysisResult: @@ -2940,7 +2843,7 @@ def _series_to_service_result( Now :class:`.AnalysisResult` is only used to save data in the experiment service. All local operations must be done with :class:`.AnalysisResultTable` dataframe. ExperimentData._analysis_results are totally decoupled from - the model of IBM experiment service until this function is implicitly called. + the model of the experiment service until this function is implicitly called. Args: series: Pandas dataframe Series (a row of dataframe). @@ -2969,23 +2872,18 @@ def _series_to_service_result( result_data["_value"] = qe_result.value result_data["_extra"] = qe_result.extra - # IBM Experiment Service doesn't have data field for experiment and run time. + # DbAnalysisResultData doesn't have data field for experiment and run time. # These are added to extra field so that these data can be saved. result_data["_extra"]["experiment"] = qe_result.experiment result_data["_extra"]["run_time"] = qe_result.run_time - try: - quality = ResultQuality(str(qe_result.quality).upper()) - except ValueError: - quality = "unknown" - - experiment_service_payload = AnalysisResultDataclass( + experiment_service_payload = DbAnalysisResultData( result_id=qe_result.result_id, experiment_id=qe_result.experiment_id, result_type=qe_result.name, result_data=result_data, device_components=list(map(to_component, qe_result.device_components)), - quality=quality, + quality=ResultQuality.from_str(qe_result.quality), tags=qe_result.tags, backend_name=qe_result.backend, creation_datetime=qe_result.created_time, diff --git a/qiskit_experiments/framework/provider_interfaces.py b/qiskit_experiments/framework/provider_interfaces.py index 93e1b12b78..27bc085eb4 100644 --- a/qiskit_experiments/framework/provider_interfaces.py +++ b/qiskit_experiments/framework/provider_interfaces.py @@ -20,13 +20,19 @@ experiment results. """ +from __future__ import annotations + from enum import Enum, IntEnum -from typing import Protocol +from json import JSONEncoder, JSONDecoder +from typing import Protocol, TYPE_CHECKING from qiskit.result import Result from qiskit.primitives import PrimitiveResult from qiskit.providers import Backend, JobStatus +if TYPE_CHECKING: + from qiskit_experiments.database_service import DbExperimentData, DbAnalysisResultData + class BaseJob(Protocol): """Required interface definition of a job class as needed for experiment data""" @@ -79,45 +85,301 @@ def job(self, job_id: str) -> Job: raise NotImplementedError -class IBMProvider(BaseProvider, Protocol): - """Provider interface needed for supporting features like IBM Quantum +Provider = BaseProvider +"""Type alias of provider interface supported by Qiskit Experiments""" + + +class MeasReturnType(str, Enum): + """Backend return types for Qobj and backend.run jobs""" + + AVERAGE = "avg" + SINGLE = "single" + + +class MeasLevel(IntEnum): + """Measurement level types for legacy Qobj and Sampler jobs""" - This interface is the subset of - :class:`~qiskit_ibm_runtime.QiskitRuntimeService` needed for all features - of Qiskit Experiments. Another provider could implement this interface to - support these features as well. + RAW = 0 + KERNELED = 1 + CLASSIFIED = 2 + + +class ExperimentService(Protocol): + """Interface definition for experiment database service. + + This interface defines the methods needed by ExperimentData to interact + with an experiment database service, whether local or remote. + + .. note:: + + Some of the type signatures of the methods of this protocol could + change in future versions of Qiskit Experiments without a transition + period. """ - def active_account(self) -> dict[str, str] | None: - """Return the IBM Quantum account information currently in use + @property + def options(self) -> dict: + """Return service options dictionary. + + Returns: + Dictionary of service options + """ + raise NotImplementedError + + def create_or_update_experiment( + self, + data: DbExperimentData, + json_encoder: type[JSONEncoder] | None = None, + create: bool = True, + max_attempts: int = 3, + **kwargs, + ) -> DbExperimentData: + """Create or update an experiment in the database. + + Args: + data: Experiment data to save + json_encoder: Custom JSON encoder + create: Whether to attempt create first + max_attempts: Maximum number of attempts + **kwargs: Additional parameters + + Returns: + Experiment data of the experiment + """ + raise NotImplementedError + + def create_or_update_analysis_result( + self, + data: DbAnalysisResultData, + json_encoder: type[JSONEncoder] | None = None, + create: bool = True, + max_attempts: int = 3, + ) -> str: + """Create or update an analysis result in the database. - This method returns the current account information in a dictionary - format. It is used to copy the credentials for use with - ``qiskit-ibm-experiment`` without requiring specifying the credentials - for the provider and ``qiskit-ibm-experiment`` separately - It should include ``"url"`` and ``"token"`` as keys for the - authentication to work. + Args: + data: Analysis result data to save + json_encoder: Custom JSON encoder + create: Whether to attempt create first + max_attempts: Maximum number of attempts Returns: - A dictionary with information about the account currently in the session. + Analysis result ID """ raise NotImplementedError + def create_analysis_results( + self, + data: list, + blocking: bool = True, + max_workers: int = 100, + json_encoder: type[JSONEncoder] | None = None, + ): + """Create multiple analysis results in the database. -Provider = BaseProvider | IBMProvider -"""Union type of provider interfaces supported by Qiskit Experiments""" + Args: + data: List of analysis result data to save + blocking: Whether to wait for completion + max_workers: Maximum number of worker threads + json_encoder: Custom JSON encoder + Returns: + Status dictionary or handler object + """ + raise NotImplementedError -class MeasReturnType(str, Enum): - """Backend return types for Qobj and backend.run jobs""" + def experiment( + self, + experiment_id: str, + json_decoder: type[JSONDecoder] | None = None, + ) -> DbExperimentData: + """Retrieve a single experiment from the database. - AVERAGE = "avg" - SINGLE = "single" + Args: + experiment_id: Experiment ID + json_decoder: Custom JSON decoder + Returns: + Retrieved experiment data + """ + raise NotImplementedError -class MeasLevel(IntEnum): - """Measurement level types for legacy Qobj and Sampler jobs""" + def delete_experiment(self, experiment_id: str) -> None: + """Delete an experiment from the database. - RAW = 0 - KERNELED = 1 - CLASSIFIED = 2 + Args: + experiment_id: Experiment ID to delete + """ + raise NotImplementedError + + def analysis_result( + self, + result_id: str, + json_decoder: type[JSONDecoder] | None = None, + ) -> DbAnalysisResultData: + """Retrieve a single analysis result from the database. + + Args: + result_id: Analysis result ID + json_decoder: Custom JSON decoder + + Returns: + Retrieved analysis result data + """ + raise NotImplementedError + + def analysis_results( + self, + experiment_id: str | None = None, + limit: int | None = None, + json_decoder: type[JSONDecoder] | None = None, + **filters, + ) -> list: + """Query analysis results from the database. + + Args: + experiment_id: Filter by experiment ID + limit: Maximum number of results + json_decoder: Custom JSON decoder + **filters: Additional filter parameters + + Returns: + List of analysis results + """ + raise NotImplementedError + + def delete_analysis_result(self, result_id: str) -> None: + """Delete an analysis result from the database. + + Args: + result_id: Analysis result ID to delete + """ + raise NotImplementedError + + def create_or_update_figure( + self, + experiment_id: str, + figure: bytes | str, + figure_name: str | None = None, + create: bool = True, + max_attempts: int = 3, + ) -> tuple: + """Create or update a figure in the database. + + Args: + experiment_id: Experiment ID + figure: Figure data or file path + figure_name: Name for the figure + create: Whether to attempt create first + max_attempts: Maximum number of attempts + + Returns: + Tuple of (figure_name, size) + """ + raise NotImplementedError + + def create_figures( + self, + experiment_id: str, + figure_list: list, + blocking: bool = True, + max_workers: int = 100, + ): + """Create multiple figures in the database. + + Args: + experiment_id: Experiment ID + figure_list: List of (figure, name) tuples + blocking: Whether to wait for completion + max_workers: Maximum number of worker threads + + Returns: + Status dictionary or handler object + """ + raise NotImplementedError + + def figure( + self, + experiment_id: str, + figure_name: str, + file_name: str | None = None, + ) -> bytes | int: + """Retrieve a figure from the database. + + Args: + experiment_id: Experiment ID + figure_name: Name of the figure + file_name: Optional local file to save to + + Returns: + Figure bytes if file_name is None, otherwise size written + """ + raise NotImplementedError + + def delete_figure(self, experiment_id: str, figure_name: str) -> None: + """Delete a figure from the database. + + Args: + experiment_id: Experiment ID + figure_name: Name of the figure to delete + """ + raise NotImplementedError + + def files(self, experiment_id: str) -> dict: + """Retrieve the file list for an experiment. + + Args: + experiment_id: Experiment ID + + Returns: + Dictionary with file list metadata + """ + raise NotImplementedError + + def file_upload( + self, + experiment_id: str, + file_name: str, + file_data: dict | str | bytes, + json_encoder: type[JSONEncoder] | None = None, + ) -> None: + """Upload a file to the database. + + Args: + experiment_id: Experiment ID + file_name: Name for the file + file_data: File data (dict or JSON string or file bytes) + json_encoder: Custom JSON encoder + """ + raise NotImplementedError + + def file_delete( + self, + experiment_id: str, + file_name: str, + ): + """Delete a file from the database + + Args: + experiment_id: Experiment ID + file_name: Name for the file + """ + raise NotImplementedError + + def file_download( + self, + experiment_id: str, + file_name: str, + json_decoder: type[JSONDecoder] | None = None, + ) -> dict: + """Download a file from the database. + + Args: + experiment_id: Experiment ID + file_name: Name of the file + json_decoder: Custom JSON decoder + + Returns: + Deserialized file data + """ + raise NotImplementedError diff --git a/qiskit_experiments/test/__init__.py b/qiskit_experiments/test/__init__.py index 0571c2ca21..64a2c0ff4f 100644 --- a/qiskit_experiments/test/__init__.py +++ b/qiskit_experiments/test/__init__.py @@ -54,4 +54,3 @@ from .mock_iq_helpers import MockIQExperimentHelper, MockIQParallelExperimentHelper from .noisy_delay_aer_simulator import NoisyDelayAerBackend from .t2hahn_backend import T2HahnBackend -from .fake_service import FakeService diff --git a/qiskit_experiments/test/fake_service.py b/qiskit_experiments/test/fake_service.py deleted file mode 100644 index 3ec338826e..0000000000 --- a/qiskit_experiments/test/fake_service.py +++ /dev/null @@ -1,494 +0,0 @@ -# This code is part of Qiskit. -# -# (C) Copyright IBM 2021. -# -# This code is licensed under the Apache License, Version 2.0. You may -# obtain a copy of this license in the LICENSE.txt file in the root directory -# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. -# -# Any modifications or derivative works of this code must retain this -# copyright notice, and modified files need to carry a notice indicating -# that they have been altered from the originals. - -"""Fake service class for tests.""" - -from typing import Any -import json -from datetime import datetime, timedelta -import uuid - -import pandas as pd -from qiskit_ibm_experiment import AnalysisResultData - -from qiskit_experiments.test.fake_backend import FakeBackend -from qiskit_experiments.database_service.device_component import DeviceComponent -from qiskit_experiments.database_service.exceptions import ( - ExperimentEntryExists, - ExperimentEntryNotFound, -) - - -class FakeService: - """ - This extremely simple database is designated for testing and as a playground for developers. - It does not support multi-threading. - It is not guaranteed to perform well for a large amount of data. - It implements most of the methods of `DatabaseService`. - """ - - def __init__(self): - self.exps = pd.DataFrame( - columns=[ - "experiment_type", - "backend_name", - "metadata", - "experiment_id", - "parent_id", - "job_ids", - "tags", - "notes", - "share_level", - "start_datetime", - "device_components", - "figure_names", - "backend", - ] - ) - self.results = pd.DataFrame( - columns=[ - "experiment_id", - "result_data", - "result_type", - "device_components", - "tags", - "quality", - "verified", - "result_id", - "chisq", - "creation_datetime", - "backend_name", - ] - ) - - def create_experiment( - self, - experiment_type: str, - backend_name: str, - metadata: dict | None = None, - experiment_id: str | None = None, - parent_id: str | None = None, - job_ids: list[str] | None = None, - tags: list[str] | None = None, - notes: str | None = None, - json_encoder: type[json.JSONEncoder] = json.JSONEncoder, - **kwargs: Any, - ) -> str: - """Creates a new experiment""" - # currently not using json_encoder; this should be changed - # pylint: disable = unused-argument - if experiment_id is None: - experiment_id = uuid.uuid4() - - if experiment_id in self.exps.experiment_id.values: - raise ExperimentEntryExists("Cannot add experiment with existing id") - - # Clarifications about some of the columns: - # share_level - not a parameter of `DatabaseService.create_experiment` but a parameter of - # `IBMExperimentService.create_experiment`. It must be supported because it is used - # in `DbExperimentData`. - # device_components - the user specifies the device components when adding a result - # (this is not a local decision of the fake service but the interface of DatabaseService - # and IBMExperimentService). The components of the different results of the same - # experiment are aggregated here in the device_components column. - # start_datetime - not a parameter of `DatabaseService.create_experiment` but a parameter of - # `IBMExperimentService.create_experiment`. Since `DbExperimentData` does not set it - # via kwargs (as it does with share_level), the user cannot control the time and the - # service alone decides about it. Here we've chosen to set a unique time for each - # experiment, with the first experiment dated to midnight of January 1st, 2022, the - # second experiment an hour later, etc. - # figure_names - the fake service currently does not support figures. The column - # (degenerated to []) is required to prevent a flaw in the work with DbExperimentData. - # backend - the query methods `experiment` and `experiments` are supposed to return an - # an instantiated backend object, and not only the backend name. We assume that the fake - # service works with the fake backend (class FakeBackend). - row = pd.DataFrame( - [ - { - "experiment_type": experiment_type, - "experiment_id": experiment_id, - "parent_id": parent_id, - "backend_name": backend_name, - "metadata": metadata, - "job_ids": job_ids, - "tags": tags, - "notes": notes, - "share_level": kwargs.get("share_level", None), - "device_components": [], - "start_datetime": datetime(2022, 1, 1) + timedelta(hours=len(self.exps)), - "figure_names": [], - "backend": FakeBackend(backend_name=backend_name), - } - ], - columns=self.exps.columns, - ) - if len(self.exps) > 0: - self.exps = pd.concat( - [ - self.exps, - row, - ], - ignore_index=True, - ) - else: - # Avoid the FutureWarning on concatenating empty DataFrames - # introduced in https://github.com/pandas-dev/pandas/pull/52532 - self.exps = row - - return experiment_id - - def update_experiment( - self, - experiment_id: str, - metadata: dict | None = None, - job_ids: list[str] | None = None, - notes: str | None = None, - tags: list[str] | None = None, - **kwargs: Any, - ) -> None: - """Updates an existing experiment""" - if experiment_id not in self.exps.experiment_id.values: - raise ExperimentEntryNotFound("Attempt to update a non-existing experiment") - - row = self.exps.experiment_id == experiment_id - if metadata is not None: - self.exps.loc[row, "metadata"] = metadata - if job_ids is not None: - self.exps.loc[row, "job_ids"] = job_ids - if tags is not None: - self.exps.loc[row, "tags"] = tags - if notes is not None: - self.exps.loc[row, "notes"] = notes - - for field_name in ["share_level", "parent_id"]: - if field_name in kwargs: - self.exps.loc[row, field_name] = kwargs[field_name] - - def experiment( - self, experiment_id: str, json_decoder: type[json.JSONDecoder] = json.JSONDecoder - ) -> dict: - """Returns an experiment by experiment_id""" - # pylint: disable = unused-argument - if experiment_id not in self.exps.experiment_id.values: - raise ExperimentEntryNotFound("Experiment does not exist") - - return self.exps.loc[self.exps.experiment_id == experiment_id].to_dict("records")[0] - - def experiments( - self, - limit: int | None = 10, - json_decoder: type[json.JSONDecoder] = json.JSONDecoder, - device_components: str | DeviceComponent | None = None, - experiment_type: str | None = None, - backend_name: str | None = None, - tags: list[str] | None = None, - parent_id: str | None = None, - tags_operator: str | None = "OR", - **filters: Any, - ) -> list[dict]: - """Returns a list of experiments filtered by given criteria""" - # pylint: disable = unused-argument - df = self.exps - - if experiment_type is not None: - df = df.loc[df.experiment_type == experiment_type] - - if backend_name is not None: - df = df.loc[df.backend_name == backend_name] - - # Note a bug in the interface for all services: - # It is impossible to filter by experiments whose parent id is None - # (i.e., root experiments) - if parent_id is not None: - df = df.loc[df.parent_id == parent_id] - - # Waiting for consistency between provider service and qiskit-experiments service, - # currently they have different types for `device_components` - if device_components is not None: - raise ValueError( - "The fake service currently does not support filtering on device components" - ) - - if tags is not None: - if tags_operator == "OR": - df = df.loc[df.tags.apply(lambda dftags: any(x in dftags for x in tags))] - elif tags_operator == "AND": - df = df.loc[df.tags.apply(lambda dftags: all(x in dftags for x in tags))] - else: - raise ValueError("Unrecognized tags operator") - - # These are parameters of IBMExperimentService.experiments - if "start_datetime_before" in filters: - df = df.loc[df.start_datetime <= filters["start_datetime_before"]] - if "start_datetime_after" in filters: - df = df.loc[df.start_datetime >= filters["start_datetime_after"]] - - # This is a parameter of IBMExperimentService.experiments - sort_by = filters.get("sort_by", "start_datetime:desc") - - if not isinstance(sort_by, list): - sort_by = [sort_by] - - # TODO: support also experiment_type - if len(sort_by) != 1: - raise ValueError("The fake service currently supports only sorting by start_datetime") - - sortby_split = sort_by[0].split(":") - # TODO: support also experiment_type - if ( - len(sortby_split) != 2 - or sortby_split[0] != "start_datetime" - or (sortby_split[1] != "asc" and sortby_split[1] != "desc") - ): - raise ValueError( - "The fake service currently supports only sorting by start_datetime, which can be " - "either asc or desc" - ) - - df = df.sort_values( - ["start_datetime", "experiment_id"], ascending=[(sortby_split[1] == "asc"), True] - ) - - df = df.iloc[:limit] - - return df.to_dict("records") - - def delete_experiment(self, experiment_id: str) -> None: - """Deletes an experiment""" - if experiment_id not in self.exps.experiment_id.values: - return - - index = self.exps[self.exps.experiment_id == experiment_id].index - self.exps.drop(index, inplace=True) - - def create_analysis_result( - self, - experiment_id: str, - result_data: dict, - result_type: str, - device_components: str | DeviceComponent | None = None, - tags: list[str] | None = None, - quality: str | None = None, - verified: bool = False, - result_id: str | None = None, - json_encoder: type[json.JSONEncoder] = json.JSONEncoder, - **kwargs: Any, - ) -> str: - """Creates an analysis result""" - # pylint: disable = unused-argument - if result_id is None: - result_id = uuid.uuid4() - - if result_id in self.results.result_id.values: - raise ExperimentEntryExists("Cannot add analysis result with existing id") - - # Clarifications about some of the columns: - # backend_name - taken from the experiment. - # creation_datetime - start_datetime - not a parameter of - # `DatabaseService.create_analysis_result` but a parameter of - # `IBMExperimentService.create_analysis_result`. Since `DbExperimentData` does not set it - # via kwargs (as it does with chisq), the user cannot control the time and the service - # alone decides about it. Here we've chosen to set the start date of the experiment. - row = pd.DataFrame( - [ - { - "result_data": result_data, - "result_id": result_id, - "result_type": result_type, - "device_components": device_components, - "experiment_id": experiment_id, - "quality": quality, - "verified": verified, - "tags": tags, - "backend_name": self.exps.loc[self.exps.experiment_id == experiment_id] - .iloc[0] - .backend_name, - "chisq": kwargs.get("chisq", None), - "creation_datetime": self.exps.loc[self.exps.experiment_id == experiment_id] - .iloc[0] - .start_datetime, - } - ] - ) - if len(self.results) > 0: - self.results = pd.concat( - [ - self.results, - row, - ], - ignore_index=True, - ) - else: - # Avoid the FutureWarning on concatenating empty DataFrames - # introduced in https://github.com/pandas-dev/pandas/pull/52532 - self.results = row - - # a helper method for updating the experiment's device components, see usage below - def add_new_components(expcomps): - for dc in device_components: - if dc not in expcomps: - expcomps.append(dc) - - # update the experiment's device components - self.exps.loc[self.exps.experiment_id == experiment_id, "device_components"].apply( - add_new_components - ) - - return result_id - - def update_analysis_result( - self, - result_id: str, - result_data: dict | None = None, - tags: list[str] | None = None, - quality: str | None = None, - verified: bool = None, - **kwargs: Any, - ) -> None: - """Updates an analysis result""" - if result_id not in self.results.result_id.values: - raise ExperimentEntryNotFound("Attempt to update a non-existing analysis result") - - row = self.results.result_id == result_id - if result_data is not None: - self.results.loc[row, "result_data"] = result_data - if tags is not None: - self.results.loc[row, "tags"] = tags - if quality is not None: - self.results.loc[row, "quality"] = quality - if verified is not None: - self.results.loc[row, "verified"] = verified - if "chisq" in kwargs: - self.results.loc[row, "chisq"] = kwargs["chisq"] - - def analysis_result( - self, result_id: str, json_decoder: type[json.JSONDecoder] = json.JSONDecoder - ) -> dict: - """Gets an analysis result by result_id""" - # pylint: disable = unused-argument - if result_id not in self.results.result_id.values: - raise ExperimentEntryNotFound("Analysis result does not exist") - - # The `experiment` method implements special handling of the backend, we skip it here. - # It's a bit strange, so, if not required by `DbExperimentData` then we'd better skip. - return self.results.loc[self.results.result_id == result_id].to_dict("records")[0] - - def analysis_results( - self, - limit: int | None = 10, - json_decoder: type[json.JSONDecoder] = json.JSONDecoder, - device_components: str | DeviceComponent | None = None, - experiment_id: str | None = None, - result_type: str | None = None, - backend_name: str | None = None, - quality: str | None = None, - verified: bool | None = None, - tags: list[str] | None = None, - tags_operator: str | None = "OR", - **filters: Any, - ) -> list[AnalysisResultData]: - """Returns a list of analysis results filtered by the given criteria""" - # pylint: disable = unused-argument - df = self.results - - # TODO: skipping device components for now until we consolidate more with the provider service - # (in the qiskit-experiments service there is no operator for device components, - # so the specification for filtering is not clearly defined) - - if experiment_id is not None: - df = df.loc[df.experiment_id == experiment_id] - if result_type is not None: - df = df.loc[df.result_type == result_type] - if backend_name is not None: - df = df.loc[df.backend_name == backend_name] - if quality is not None: - df = df.loc[df.quality == quality] - if verified is not None: - df = df.loc[df.verified == verified] - - if tags is not None: - if tags_operator == "OR": - df = df.loc[df.tags.apply(lambda dftags: any(x in dftags for x in tags))] - elif tags_operator == "AND": - df = df.loc[df.tags.apply(lambda dftags: all(x in dftags for x in tags))] - else: - raise ValueError("Unrecognized tags operator") - - # This is a parameter of IBMExperimentService.experiments - sort_by = filters.get("sort_by", "creation_datetime:desc") - - if not isinstance(sort_by, list): - sort_by = [sort_by] - - # TODO: support also device components and result type - if len(sort_by) != 1: - raise ValueError( - "The fake service currently supports only sorting by creation_datetime" - ) - - sortby_split = sort_by[0].split(":") - # TODO: support also device components and result type - if ( - len(sortby_split) != 2 - or sortby_split[0] != "creation_datetime" - or (sortby_split[1] != "asc" and sortby_split[1] != "desc") - ): - raise ValueError( - "The fake service currently supports only sorting by creation_datetime, " - "which can be either asc or desc" - ) - - df = df.sort_values( - ["creation_datetime", "result_id"], ascending=[(sortby_split[1] == "asc"), True] - ) - - df = df.iloc[:limit] - return [AnalysisResultData(**res) for res in df.to_dict("records")] - - def delete_analysis_result(self, result_id: str) -> None: - """Deletes an analysis result""" - if result_id not in self.results.result_id.values: - return - - index = self.results[self.results.result_id == result_id].index - self.results.drop(index, inplace=True) - - def create_figure( - self, experiment_id: str, figure: str | bytes, figure_name: str | None - ) -> tuple[str, int]: - """Creates a figure""" - pass - - def update_figure( - self, experiment_id: str, figure: str | bytes, figure_name: str - ) -> tuple[str, int]: - """Updates a figure""" - pass - - def figure( - self, experiment_id: str, figure_name: str, file_name: str | None = None - ) -> int | bytes: - """Returns a figure by experiment id and figure name""" - pass - - def delete_figure( - self, - experiment_id: str, - figure_name: str, - ) -> None: - """Deletes a figure""" - pass - - @property - def preferences(self) -> dict: - """Returns the db service preferences""" - return {"auto_save": False} diff --git a/qiskit_experiments/test/utils.py b/qiskit_experiments/test/utils.py index 61773c3ae4..49b5c6762d 100644 --- a/qiskit_experiments/test/utils.py +++ b/qiskit_experiments/test/utils.py @@ -12,8 +12,12 @@ """Test utility functions.""" +from __future__ import annotations + import uuid +from collections.abc import Callable from datetime import datetime, timezone +from typing import TYPE_CHECKING from qiskit.providers.job import JobV1 as Job from qiskit.providers.jobstatus import JobStatus @@ -21,10 +25,37 @@ from qiskit.result import Result +if TYPE_CHECKING: + from qiskit_experiments.framework import Job as JobLike + + +class FakeProvider: + """Dummy Provider class for test purposes only""" + + def __init__(self): + self._jobs: dict[str, JobLike] = {} + + def add_job(self, job: JobLike): + """Add job to provider""" + self._jobs[job.job_id()] = job + + def job(self, job_id: str) -> JobLike: + """Retrieve job by job ID""" + return self._jobs[job_id] + + class FakeJob(Job): """Fake job.""" - def __init__(self, backend: Backend, result: Result | None = None): + def __init__( + self, + backend: Backend, + result: Result | None = None, + status: JobStatus | None = None, + cancel_callback: Callable | None = None, + result_callback: Callable | None = None, + status_callback: Callable | None = None, + ): """Initialize FakeJob.""" if result: job_id = result.job_id @@ -32,11 +63,24 @@ def __init__(self, backend: Backend, result: Result | None = None): job_id = uuid.uuid4().hex super().__init__(backend, job_id) self._result = result + self._status = status + + self._cancel_callback = cancel_callback + self._result_callback = result_callback + self._status_callback = status_callback def result(self): """Return job result.""" + if self._result_callback: + return self._result_callback() return self._result + def cancel(self): + """Cancel the job""" + self._status = JobStatus.CANCELLED + if self._cancel_callback: + self._cancel_callback() + def submit(self): """Submit the job to the backend for execution.""" pass @@ -48,6 +92,11 @@ def time_per_step() -> dict[str, datetime]: def status(self) -> JobStatus: """Return the status of the job, among the values of ``JobStatus``.""" + if self._status_callback: + return self._status_callback() + + if self._status is not None: + return self._status if self._result: return JobStatus.DONE return JobStatus.RUNNING diff --git a/releasenotes/notes/adopt-qiskit-ibm-experiment-9bc997dd31535ffc.yaml b/releasenotes/notes/adopt-qiskit-ibm-experiment-9bc997dd31535ffc.yaml new file mode 100644 index 0000000000..e4e5fc84ad --- /dev/null +++ b/releasenotes/notes/adopt-qiskit-ibm-experiment-9bc997dd31535ffc.yaml @@ -0,0 +1,34 @@ +--- +upgrade: + - | + The dependency on ``qiskit-ibm-experiment`` has been removed following that + project's discontinuation. Its classes related to storing experiment + results in a database (referred to in the documentation as an experiment + service) have been moved into Qiskit Experiments as + :class:`.DbExperimentData`, :class:`.DbAnalysisResultData`, and + :class:`.ResultQuality`. These classes are used internally by + :class:`.ExperimentData` and :class:`.AnalysisResult`, so the move should + be transparent to the user in most cases. Additionally, ``pyyaml`` was + added an optional dependency for the new :class:`.LocalExperimentService`. + ``pyyaml`` was previously pulled in as a dependency of + ``qiskit-ibm-experiment``. + - | + The support within :class:`.ExperimentData` for looking up an experiment + service from an experiment's backend or provider has been removed. This + support was only relevant for the IBM experiment service which was + discontinued. + - | + The ``IBMProvider`` interface class was removed. This class was only used + for type annotations to indicate where a provider could be passed that + would allow looking up an experiment service. +developer: + - | + :class:`.LocalExperimentService` has been added as a reference + implementation of a service for saving and loading experiment results. As + noted in its documentation, it was written for testing purposes and is not + currently suitable for storing a large amount of results. +other: + - | + The `howto `_ on working with the cloud experiment + service has been reworked to demonstrate using + :class:`.LocalExperimentService` instead. diff --git a/releasenotes/notes/lazy-imports-730268f4cd8763dc.yaml b/releasenotes/notes/lazy-imports-730268f4cd8763dc.yaml index f6d354dbeb..0022ee55a9 100644 --- a/releasenotes/notes/lazy-imports-730268f4cd8763dc.yaml +++ b/releasenotes/notes/lazy-imports-730268f4cd8763dc.yaml @@ -3,7 +3,7 @@ features: - | Importing of optional dependencies has been made delayed until first use. In particular, ``cvxpy`` is now not imported until a tomography fitter that - uses it is used. Additionally, ``qiskit_ibm_runtime`` is not imported an + uses it is used. Additionally, ``qiskit_ibm_runtime`` is not imported until an experiment is run without being passed a sampler argument. Previously, both packages were imported when ``qiskit_experiments`` was imported (``cvxpy`` is optional, but it was always imported if it was available). diff --git a/test/database_service/test_db_analysis_result.py b/test/database_service/test_db_analysis_result.py index 8be2b2368b..79d1beeb1e 100644 --- a/test/database_service/test_db_analysis_result.py +++ b/test/database_service/test_db_analysis_result.py @@ -14,7 +14,6 @@ """Test AnalysisResult.""" from test.base import QiskitExperimentsTestCase -from unittest import mock import json import math @@ -22,10 +21,16 @@ import numpy as np import uncertainties -from qiskit_ibm_experiment import IBMExperimentService, ExperimentData from qiskit_experiments.framework import AnalysisResult -from qiskit_experiments.database_service.device_component import Qubit, Resonator, to_component -from qiskit_experiments.database_service.exceptions import ExperimentDataError +from qiskit_experiments.database_service import ( + DbExperimentData, + ExperimentDataError, + LocalExperimentService, + Qubit, + Resonator, + ResultQuality, + to_component, +) @ddt @@ -39,28 +44,31 @@ def test_analysis_result_attributes(self): "device_components": [Qubit(1), Qubit(2)], "experiment_id": "1234", "result_id": "5678", - "quality": "Good", + "quality": "good", "verified": False, } result = AnalysisResult(value={"foo": "bar"}, tags=["tag1", "tag2"], **attrs) self.assertEqual({"foo": "bar"}, result.value) self.assertEqual(["tag1", "tag2"], result.tags) for key, val in attrs.items(): + if key == "quality": + val = ResultQuality.from_str(val) self.assertEqual(val, getattr(result, key)) def test_save(self): """Test saving analysis result.""" - mock_service = mock.create_autospec(IBMExperimentService) + service = LocalExperimentService() result = self._new_analysis_result() - result.service = mock_service + result.service = service + service.create_or_update_experiment(DbExperimentData(experiment_id=result.experiment_id)) result.save() - mock_service.create_or_update_analysis_result.assert_called_once() + service.analysis_result(result.result_id) def test_load(self): """Test loading analysis result.""" - service = IBMExperimentService(local=True, local_save=False) + service = LocalExperimentService() result = self._new_analysis_result() - service.create_experiment(ExperimentData(experiment_id=result.experiment_id)) + service.create_or_update_experiment(DbExperimentData(experiment_id=result.experiment_id)) result.service = service result.save() loaded_result = AnalysisResult.load(result_id=result.result_id, service=service) @@ -69,40 +77,37 @@ def test_load(self): def test_auto_save(self): """Test auto saving.""" - mock_service = mock.create_autospec(IBMExperimentService) - result = self._new_analysis_result(service=mock_service) + service = LocalExperimentService() + result = self._new_analysis_result(service=service) + service.create_or_update_experiment(DbExperimentData(experiment_id=result.experiment_id)) result.auto_save = True - mock_service.reset_mock() # since setting auto_save = True initiated save - - subtests = [ - # update function, update parameters, service called - (setattr, (result, "tags", ["foo"])), - (setattr, (result, "value", {"foo": "bar"})), - (setattr, (result, "quality", "GOOD")), - (setattr, (result, "verified", True)), - ] - - for func, params in subtests: - with self.subTest(func=func): - func(*params) - mock_service.create_or_update_analysis_result.assert_called_once() - mock_service.reset_mock() + + result.tags = ["foo"] + result.value = {"foo": "bar"} + result.quality = ResultQuality.GOOD + result.verified = True + + loaded_result = AnalysisResult.load(result.result_id, service) + self.assertEqual(result.tags, loaded_result.tags) + self.assertEqual(result.value, loaded_result.value) + self.assertEqual(result.quality, loaded_result.quality) + self.assertEqual(result.verified, loaded_result.verified) def test_set_service_init(self): """Test setting service in init.""" - mock_service = mock.create_autospec(IBMExperimentService) - result = self._new_analysis_result(service=mock_service) - self.assertEqual(mock_service, result.service) + service = LocalExperimentService() + result = self._new_analysis_result(service=service) + self.assertEqual(service, result.service) def test_set_service_direct(self): """Test setting service directly.""" - mock_service = mock.create_autospec(IBMExperimentService) + service = LocalExperimentService() result = self._new_analysis_result() - result.service = mock_service - self.assertEqual(mock_service, result.service) + result.service = service + self.assertEqual(service, result.service) with self.assertRaises(ExperimentDataError): - result.service = mock_service + result.service = service def test_set_data(self): """Test setting data.""" diff --git a/test/database_service/test_db_experiment_data.py b/test/database_service/test_db_experiment_data.py index 4632d9f338..8034ddda97 100644 --- a/test/database_service/test_db_experiment_data.py +++ b/test/database_service/test_db_experiment_data.py @@ -15,39 +15,40 @@ """Test ExperimentData.""" from test.base import QiskitExperimentsTestCase from test.fake_experiment import FakeExperiment -import os -from unittest import mock import copy -from random import randrange -import time -import threading import json +import os import re +import threading +import time import uuid from datetime import datetime, timedelta +from random import randrange import matplotlib.pyplot as plt import numpy as np from qiskit_ibm_runtime.fake_provider import FakeMelbourneV2 from qiskit.result import Result -from qiskit.providers import JobV1 as Job from qiskit.providers import JobStatus -from qiskit.providers.backend import Backend -from qiskit_ibm_experiment import IBMExperimentService -from qiskit_experiments.framework import ExperimentData, BackendData, ArtifactData - -from qiskit_experiments.database_service.exceptions import ( +from qiskit_experiments.framework import ( + ArtifactData, + ExperimentData, + BackendData, +) +from qiskit_experiments.database_service import ( ExperimentDataError, ExperimentEntryNotFound, + LocalExperimentService, + Qubit, ) -from qiskit_experiments.database_service.device_component import Qubit from qiskit_experiments.framework.experiment_data import ( AnalysisStatus, ExperimentStatus, ) from qiskit_experiments.framework.matplotlib import get_non_gui_ax from qiskit_experiments.test.fake_backend import FakeBackend +from qiskit_experiments.test.utils import FakeJob class TestDbExperimentData(QiskitExperimentsTestCase): @@ -57,15 +58,6 @@ def setUp(self): super().setUp() self.backend = FakeMelbourneV2() - def generate_mock_job(self): - """Helper method to generate a mock job.""" - job = mock.create_autospec(Job, instance=True) - # mock a backend without provider - backend = mock.create_autospec(Backend, instance=True) - backend.provider = None - job.backend.return_value = backend - return job - def test_db_experiment_data_attributes(self): """Test DB experiment data attributes.""" attrs = { @@ -128,14 +120,13 @@ def test_add_data_result_metadata(self): def test_add_data_job(self): """Test add job data.""" - a_job = self.generate_mock_job() - a_job.result.return_value = self._get_job_result(3) + a_job = FakeJob(self.backend, self._get_job_result(3, job_id="0")) num_circs = 3 jobs = [] for _ in range(2): - job = self.generate_mock_job() - job.result.return_value = self._get_job_result(2, label_from=num_circs) - job.status.return_value = JobStatus.DONE + job = FakeJob( + self.backend, self._get_job_result(2, label_from=num_circs, job_id=str(num_circs)) + ) jobs.append(job) num_circs = num_circs + 2 @@ -170,9 +161,7 @@ def _callback(_exp_data): nonlocal called_back called_back = True - a_job = self.generate_mock_job() - a_job.result.return_value = self._get_job_result(2) - a_job.status.return_value = JobStatus.DONE + a_job = FakeJob(self.backend, self._get_job_result(2)) called_back = False exp_data = ExperimentData(backend=self.backend, experiment_type="qiskit_test") @@ -224,10 +213,7 @@ def _callback(_exp_data, **kwargs): nonlocal called_back called_back = True - a_job = self.generate_mock_job() - a_job.backend.return_value = mock.create_autospec(Backend, instance=True) - a_job.result.return_value = self._get_job_result(2) - a_job.status.return_value = JobStatus.DONE + a_job = FakeJob(self.backend, self._get_job_result(2)) called_back = False callback_kwargs = "foo" @@ -243,9 +229,7 @@ def test_add_data_pending_post_processing(self): def _callback(_exp_data, **kwargs): kwargs["event"].wait(timeout=3) - a_job = self.generate_mock_job() - a_job.result.return_value = self._get_job_result(2) - a_job.status.return_value = JobStatus.DONE + a_job = FakeJob(self.backend, self._get_job_result(2)) event = threading.Event() self.addCleanup(event.set) @@ -296,16 +280,15 @@ def test_add_figure_plot(self): """Test adding a matplotlib figure.""" figure, ax = plt.subplots() ax.plot([1, 2, 3]) + fig_name = "figure.svg" - service = self._set_mock_service() + service = LocalExperimentService() exp_data = ExperimentData( backend=self.backend, experiment_type="qiskit_test", service=service ) - exp_data.add_figures(figure, save_figure=True) + exp_data.add_figures(figure, fig_name, save_figure=True) self.assertEqual(figure, exp_data.figure(0).figure) - service.create_or_update_figure.assert_called_once() - _, kwargs = service.create_or_update_figure.call_args - self.assertIsInstance(kwargs["figure"], bytes) + self.assertIsInstance(service.figure(exp_data.experiment_id, fig_name), bytes) def test_add_figures(self): """Test adding multiple new figures.""" @@ -359,15 +342,14 @@ def test_add_figure_overwrite(self): def test_add_figure_save(self): """Test saving a figure in the database.""" hello_bytes = str.encode("hello world") - service = self._set_mock_service() + fig_name = "hello.svg" + service = LocalExperimentService() exp_data = ExperimentData( backend=self.backend, experiment_type="qiskit_test", service=service ) - exp_data.add_figures(hello_bytes, save_figure=True) - service.create_or_update_figure.assert_called_once() - _, kwargs = service.create_or_update_figure.call_args - self.assertEqual(kwargs["figure"], hello_bytes) - self.assertEqual(kwargs["experiment_id"], exp_data.experiment_id) + exp_data.add_figures(hello_bytes, fig_name, save_figure=True) + loaded = service.figure(exp_data.experiment_id, fig_name) + self.assertEqual(hello_bytes, loaded) def test_add_figure_metadata(self): hello_bytes = str.encode("hello world") @@ -460,14 +442,14 @@ def test_delayed_backend(self): exp_data = ExperimentData(experiment_type="qiskit_test") self.assertIsNone(exp_data.backend) exp_data.save_metadata() - a_job = self.generate_mock_job() + a_job = FakeJob(self.backend, self._get_job_result(2)) exp_data.add_jobs(a_job) self.assertIsNotNone(exp_data.backend) def test_different_backend(self): """Test setting a different backend.""" exp_data = ExperimentData(backend=self.backend, experiment_type="qiskit_test") - a_job = self.generate_mock_job() + a_job = FakeJob(FakeBackend(), self._get_job_result(2)) self.assertNotEqual(exp_data.backend, a_job.backend()) with self.assertLogs("qiskit_experiments", "WARNING"): exp_data.add_jobs(a_job) @@ -524,99 +506,89 @@ def test_delete_analysis_result(self): def test_save_metadata(self): """Test saving experiment metadata.""" exp_data = ExperimentData(backend=self.backend, experiment_type="qiskit_test") - service = mock.create_autospec(IBMExperimentService, instance=True) + service = LocalExperimentService() exp_data.service = service exp_data.save_metadata() - service.create_or_update_experiment.assert_called_once() - data = service.create_or_update_experiment.call_args[0][0] - self.assertEqual(exp_data.experiment_id, data.experiment_id) + loaded = service.experiment(exp_data.experiment_id) + self.assertEqual(exp_data.experiment_id, loaded.experiment_id) def test_save(self): """Test saving all experiment related data.""" exp_data = ExperimentData(backend=self.backend, experiment_type="qiskit_test") - service = mock.create_autospec(IBMExperimentService, instance=True) + service = LocalExperimentService() exp_data.add_figures(str.encode("hello world")) exp_data.add_analysis_results(result_id=str(uuid.uuid4())) exp_data.service = service exp_data.save() - service.create_or_update_experiment.assert_called_once() - service.create_figures.assert_called_once() - service.create_analysis_results.assert_called_once() + loaded = service.experiment(exp_data.experiment_id) + self.assertEqual(exp_data.experiment_id, loaded.experiment_id) + self.assertEqual(len(loaded.figure_names), 1) + results = service.analysis_results(experiment_id=exp_data.experiment_id) + self.assertEqual(len(results), 1) def test_save_delete(self): """Test saving all deletion.""" exp_data = ExperimentData(backend=self.backend, experiment_type="qiskit_test") - service = mock.create_autospec(IBMExperimentService, instance=True) + service = LocalExperimentService() + exp_data.service = service exp_data.add_figures(str.encode("hello world")) exp_data.add_analysis_results(result_id=str(uuid.uuid4())) + exp_data.save() + exp_data.delete_analysis_result(0) exp_data.delete_figure(0) - exp_data.service = service - exp_data.save() - service.create_or_update_experiment.assert_called_once() - service.delete_figure.assert_called_once() - service.delete_analysis_result.assert_called_once() + + loaded = service.experiment(exp_data.experiment_id) + self.assertEqual(len(loaded.figure_names), 0) + results = service.analysis_results(experiment_id=exp_data.experiment_id) + self.assertEqual(len(results), 0) def test_set_service_direct(self): """Test setting service directly.""" exp_data = ExperimentData(experiment_type="qiskit_test") self.assertIsNone(exp_data.service) - mock_service = mock.MagicMock() - exp_data.service = mock_service - self.assertEqual(mock_service, exp_data.service) + service = LocalExperimentService() + exp_data.service = service + self.assertEqual(service, exp_data.service) with self.assertRaises(ExperimentDataError): - exp_data.service = mock_service + exp_data.service = service def test_auto_save(self): """Test auto save.""" - service = self._set_mock_service() + service = LocalExperimentService() exp_data = ExperimentData( backend=self.backend, experiment_type="qiskit_test", service=service ) exp_data.auto_save = True - mock_result = mock.MagicMock() - mock_result.result_id = str(uuid.uuid4()) - subtests = [ - # update function, update parameters, service called - ( - exp_data.add_analysis_results, - (), - {"result_id": str(uuid.uuid4())}, - service.create_or_update_analysis_result, - ), - ( - exp_data.add_figures, - (str.encode("hello world"),), - {}, - service.create_or_update_figure, - ), - (exp_data.delete_figure, (0,), {}, service.delete_figure), - (exp_data.delete_analysis_result, (0,), {}, service.delete_analysis_result), - (setattr, (exp_data, "tags", ["foo"]), {}, service.create_or_update_experiment), - (setattr, (exp_data, "notes", "foo"), {}, service.create_or_update_experiment), - (setattr, (exp_data, "share_level", "hub"), {}, service.create_or_update_experiment), - ] + exp_data.add_figures(str.encode("hello world")) + exp_data.add_analysis_results(result_id=str(uuid.uuid4())) + + exp_data.delete_analysis_result(0) + exp_data.delete_figure(0) + + exp_data.tags = ["foo"] + exp_data.notes = "foo" - for func, params, kwargs, called in subtests: - with self.subTest(func=func): - func(*params, **kwargs) - if called: - called.assert_called_once() - service.reset_mock() + loaded = service.experiment(exp_data.experiment_id) + self.assertEqual(len(loaded.figure_names), 0) + results = service.analysis_results(experiment_id=exp_data.experiment_id) + self.assertEqual(len(results), 0) + self.assertEqual(loaded.tags, ["foo"]) + self.assertEqual(loaded.notes, "foo") def test_status_job_pending(self): """Test experiment status when job is pending.""" - job1 = self.generate_mock_job() - job1.result.return_value = self._get_job_result(3) - job1.status.return_value = JobStatus.DONE + job1 = FakeJob(self.backend, self._get_job_result(3)) event = threading.Event() - job2 = self.generate_mock_job() - job2.result = lambda *args, **kwargs: event.wait(timeout=15) - job2.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING + job2 = FakeJob( + self.backend, + result_callback=lambda: event.wait(timeout=15), + status_callback=lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING, + ) self.addCleanup(event.set) exp_data = ExperimentData(experiment_type="qiskit_test") @@ -634,24 +606,17 @@ def test_status_job_pending(self): def test_status_job_error(self): """Test experiment status when job failed.""" - job1 = self.generate_mock_job() - job1.result.return_value = self._get_job_result(3) - job1.status.return_value = JobStatus.DONE + job1 = FakeJob(self.backend, self._get_job_result(3)) - job2 = self.generate_mock_job() - job2.status.return_value = JobStatus.ERROR + job2 = FakeJob(self.backend, status=JobStatus.ERROR) exp_data = ExperimentData(experiment_type="qiskit_test") - with self.assertLogs(logger="qiskit_experiments.framework", level="WARN") as cm: - exp_data.add_jobs([job1, job2]) - self.assertIn("Adding a job from a backend", ",".join(cm.output)) + exp_data.add_jobs([job1, job2]) self.assertEqual(ExperimentStatus.ERROR, exp_data.status()) def test_status_post_processing(self): """Test experiment status during post processing.""" - job = self.generate_mock_job() - job.result.return_value = self._get_job_result(3) - job.status.return_value = JobStatus.DONE + job = FakeJob(self.backend, self._get_job_result(3)) event = threading.Event() self.addCleanup(event.set) @@ -664,9 +629,7 @@ def test_status_post_processing(self): def test_status_cancelled_analysis(self): """Test experiment status during post processing.""" - job = self.generate_mock_job() - job.result.return_value = self._get_job_result(3) - job.status.return_value = JobStatus.DONE + job = FakeJob(self.backend, self._get_job_result(3)) event = threading.Event() self.addCleanup(event.set) @@ -686,9 +649,7 @@ def test_status_post_processing_error(self): def _post_processing(*args, **kwargs): raise ValueError("Kaboom!") - job = self.generate_mock_job() - job.result.return_value = self._get_job_result(3) - job.status.return_value = JobStatus.DONE + job = FakeJob(self.backend, self._get_job_result(3)) exp_data = ExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) @@ -701,9 +662,7 @@ def _post_processing(*args, **kwargs): def test_status_done(self): """Test experiment status when all jobs are done.""" - job = self.generate_mock_job() - job.result.return_value = self._get_job_result(3) - job.status.return_value = JobStatus.DONE + job = FakeJob(self.backend, self._get_job_result(3)) exp_data = ExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_jobs(job) @@ -734,11 +693,12 @@ def _job_cancel(): exp_data = ExperimentData(experiment_type="qiskit_test") event = threading.Event() self.addCleanup(event.set) - job = self.generate_mock_job() - job.job_id.return_value = "1234" - job.cancel = _job_cancel - job.result = _job_result - job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING + job = FakeJob( + self.backend, + result_callback=_job_result, + cancel_callback=_job_cancel, + status_callback=lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING, + ) exp_data.add_jobs(job) with self.assertLogs("qiskit_experiments", "WARNING"): @@ -760,10 +720,11 @@ def _job_result(): def _analysis(*args): # pylint: disable = unused-argument event.wait(timeout=15) - job = self.generate_mock_job() - job.job_id.return_value = "1234" - job.result = _job_result - job.status = lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING + job = FakeJob( + self.backend, + result_callback=_job_result, + status_callback=lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING, + ) exp_data = ExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) @@ -796,10 +757,11 @@ def _analysis(expdata, name=None, timeout=0): # pylint: disable = unused-argume event.wait(timeout=timeout) run_analysis.append(name) - job = self.generate_mock_job() - job.job_id.return_value = "1234" - job.result = _job_result - job.status = lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING + job = FakeJob( + self.backend, + result_callback=_job_result, + status_callback=lambda: JobStatus.DONE if event.is_set() else JobStatus.RUNNING, + ) exp_data = ExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) @@ -848,11 +810,12 @@ def _status(): return JobStatus.CANCELLED return JobStatus.RUNNING - job = self.generate_mock_job() - job.job_id.return_value = "1234" - job.result = _job_result - job.cancel = event.set - job.status = _status + job = FakeJob( + self.backend, + result_callback=_job_result, + cancel_callback=event.set, + status_callback=lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING, + ) exp_data = ExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) @@ -874,11 +837,12 @@ def _job_result(): event.wait(timeout=15) raise ValueError("Job was cancelled.") - job = self.generate_mock_job() - job.job_id.return_value = "1234" - job.result = _job_result - job.cancel = event.set - job.status = lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING + job = FakeJob( + self.backend, + result_callback=_job_result, + cancel_callback=event.set, + status_callback=lambda: JobStatus.CANCELLED if event.is_set() else JobStatus.RUNNING, + ) exp_data = ExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job, timeout=0.5) @@ -906,13 +870,8 @@ def test_errors(self): def _post_processing(*args, **kwargs): # pylint: disable=unused-argument raise ValueError("Kaboom!") - job1 = self.generate_mock_job() - job1.job_id.return_value = "1234" - job1.status.return_value = JobStatus.DONE - - job2 = self.generate_mock_job() - job2.status.return_value = JobStatus.ERROR - job2.job_id.return_value = "5678" + job1 = FakeJob(self.backend, self._get_job_result(3)) + job2 = FakeJob(self.backend, status=JobStatus.ERROR) exp_data = ExperimentData(experiment_type="qiskit_test") with self.assertLogs(logger="qiskit_experiments.framework", level="WARN") as cm: @@ -922,7 +881,9 @@ def _post_processing(*args, **kwargs): # pylint: disable=unused-argument exp_data.block_for_results() self.assertEqual(ExperimentStatus.ERROR, exp_data.status()) self.assertIn("Kaboom", ",".join(cm.output)) - self.assertTrue(re.match(r".*5678.*Kaboom!", exp_data.errors(), re.DOTALL)) + self.assertTrue( + re.match(rf".*{exp_data.experiment_id}.*Kaboom!", exp_data.errors(), re.DOTALL) + ) def test_simple_methods_from_callback(self): """Test that simple methods used in call back function don't hang @@ -1034,8 +995,11 @@ def _sleeper(*args, **kwargs): # pylint: disable=unused-argument return self._get_job_result(1) sleep_count = 0 - job = self.generate_mock_job() - job.result = _sleeper + job = FakeJob( + self.backend, + result_callback=_sleeper, + status=JobStatus.DONE, + ) exp_data = ExperimentData(experiment_type="qiskit_test") exp_data.add_jobs(job) exp_data.add_analysis_callback(_sleeper) @@ -1085,13 +1049,17 @@ def _job2_result(): return job_results2 exp_data = ExperimentData(experiment_type="qiskit_test") - job = self.generate_mock_job() - job.result = _job1_result + job = FakeJob( + self.backend, + result_callback=_job1_result, + ) exp_data.add_jobs(job) copied = exp_data.copy(copy_results=False) - job2 = self.generate_mock_job() - job2.result = _job2_result + job2 = FakeJob( + self.backend, + result_callback=_job2_result, + ) copied.add_jobs(job2) event.set() @@ -1104,13 +1072,13 @@ def _job2_result(): exp_data.data(0)["counts"], [copied.data(0)["counts"], copied.data(1)["counts"]] ) - def _get_job_result(self, circ_count, label_from=0, no_metadata=False): + def _get_job_result(self, circ_count, label_from=0, no_metadata=False, job_id="some_job_id"): """Return a job result with random counts.""" job_result = { "backend_name": BackendData(self.backend).name, "backend_version": "1.1.1", "qobj_id": "1234", - "job_id": "some_job_id", + "job_id": job_id, "success": True, "results": [], } @@ -1126,14 +1094,6 @@ def _get_job_result(self, circ_count, label_from=0, no_metadata=False): return Result.from_dict(job_result) - def _set_mock_service(self): - """Add a mock service to the backend.""" - mock_provider = mock.MagicMock() - self.backend._provider = mock_provider - mock_service = mock.create_autospec(IBMExperimentService, instance=True) - mock_provider.service.return_value = mock_service - return mock_service - def test_getters(self): """Test the getters return the expected result""" data = ExperimentData() @@ -1201,7 +1161,7 @@ def test_add_delete_artifact(self): exp_data.add_artifacts(new_artifact) self.assertEqual(exp_data.artifacts("test"), new_artifact) - service = mock.create_autospec(IBMExperimentService, instance=True) + service = LocalExperimentService() exp_data.service = service exp_data.save() diff --git a/test/database_service/test_fake_service.py b/test/database_service/test_fake_service.py deleted file mode 100644 index 679a6efd00..0000000000 --- a/test/database_service/test_fake_service.py +++ /dev/null @@ -1,448 +0,0 @@ -# This code is part of Qiskit. -# -# (C) Copyright IBM 2022. -# -# This code is licensed under the Apache License, Version 2.0. You may -# obtain a copy of this license in the LICENSE.txt file in the root directory -# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. -# -# Any modifications or derivative works of this code must retain this -# copyright notice, and modified files need to carry a notice indicating -# that they have been altered from the originals. -""" -Test the fake service -""" - -from datetime import datetime -from test.base import QiskitExperimentsTestCase -from qiskit_experiments.test import FakeService - - -class TestFakeService(QiskitExperimentsTestCase): - """ - Test the fake service - """ - - def setUp(self): - super().setUp() - - self.service = FakeService() - - # A copy of the database, in the form of a dictionary - # To serve as a reference - self.expdict = {} - - expid = 0 - for experiment_type in range(2): - for backend_name in range(2): - for tags in range(2): - expentry = { - "experiment_id": str(expid), - "experiment_type": str(experiment_type), - "backend_name": str(backend_name), - "tags": ["a" + str(tags), "b" + str(tags)], - } - - if expid > 2: - expentry["parent_id"] = str(expid % 3) - else: - expentry["parent_id"] = None - - # Create the experiment in the service - self.service.create_experiment(**expentry) - - # We have sent the experiment for creation in the service. - # We will now update the reference dictionary self.expdict - # with columns that are set internally by the service. - - # Below we add analysis results to the experiments 0, 1, 6, and 7. - # For each of these experiments, some of the results have device - # components [0], an d some have device components [1]. - # This means that each of these experiments should eventually have - # device components [0, 1]. - if expid in [0, 1, 6, 7]: - expentry["device_components"] = [0, 1] - else: - expentry["device_components"] = [] - - # The service determines the time (see documentation in - # FakeService.create_experiment). - expentry["start_datetime"] = datetime(2022, 1, 1, expid) - - # Update the reference dictionary - self.expdict[str(expid)] = expentry - - expid += 1 - - # A reference dictionary for the analysis results - self.resdict = {} - - resid = 0 - for experiment_id in [0, 1, 6, 7]: - for result_type in range(2): - for samebool4all in range(2): - # We don't branch of each column because it makes the data too large and slows - # down the test - tags = samebool4all - quality = samebool4all - verified = samebool4all - result_data = samebool4all - device_components = samebool4all - - resentry = { - "experiment_id": str(experiment_id), - "result_type": str(result_type), - "result_id": str(resid), - "tags": ["a" + str(tags), "b" + str(tags)], - "quality": quality, - "verified": verified, - "result_data": {"value": result_data}, - "device_components": [device_components], - } - - # Create the result in the service - self.service.create_analysis_result(**resentry) - - # We have sent the experiment for creation in the service. - # We will now update the reference dictionary self.expdict - # with columns that are set internally by the service. - - # The service sets the backend to be the experiment's backend - resentry["backend_name"] = self.expdict[str(experiment_id)]["backend_name"] - - # The service determines the time (see documentation in - # FakeService.create_analysis_result). - resentry["creation_datetime"] = self.expdict[str(experiment_id)][ - "start_datetime" - ] - - # Update the reference dictionary - self.resdict[str(resid)] = resentry - - resid += 1 - - def test_creation(self): - """Test FakeService methods create_experiment and create_analysis_result""" - for df, reference_dict, id_field in zip( - [self.service.exps, self.service.results], - [self.expdict, self.resdict], - ["experiment_id", "result_id"], - ): - self.assertEqual(len(df), len(reference_dict)) - is_in_frame = [] - for i in range(len(df)): - full_entry = df.loc[i, :].to_dict() - id_value = full_entry[id_field] - self.assertTrue(id_value not in is_in_frame) - is_in_frame.append(id_value) - self.assertTrue(id_value in reference_dict) - entry = reference_dict[id_value] - self.assertTrue(entry.items() <= full_entry.items()) - - def test_query_for_single(self): - """Test FakeService methods experiment and analysis_result""" - for ( - query_method, - reference_dict, - ) in zip( - [self.service.experiment, self.service.analysis_result], [self.expdict, self.resdict] - ): - for id_value in range(len(reference_dict)): - full_entry = query_method(str(id_value)) - entry = reference_dict[str(id_value)] - self.assertTrue(entry.items() <= full_entry.items()) - - def test_experiments_query(self): - """Test FakeService.experiments""" - for experiment_type in range(2): - expids = sorted( - [ - exp["experiment_id"] - for exp in self.service.experiments( - experiment_type=str(experiment_type), limit=None - ) - ] - ) - ref_expids = sorted( - [ - exp["experiment_id"] - for exp in self.expdict.values() - if exp["experiment_type"] == str(experiment_type) - ] - ) - self.assertTrue(len(expids) > 0) - self.assertEqual(expids, ref_expids) - - for backend_name in range(2): - expids = sorted( - [ - exp["experiment_id"] - for exp in self.service.experiments(backend_name=str(backend_name), limit=None) - ] - ) - ref_expids = sorted( - [ - exp["experiment_id"] - for exp in self.expdict.values() - if exp["backend_name"] == str(backend_name) - ] - ) - self.assertTrue(len(expids) > 0) - self.assertEqual(expids, ref_expids) - - for parent_id in range(3): - expids = sorted( - [ - exp["experiment_id"] - for exp in self.service.experiments(parent_id=str(parent_id), limit=None) - ] - ) - ref_expids = sorted( - [ - exp["experiment_id"] - for exp in self.expdict.values() - if exp["parent_id"] == str(parent_id) - ] - ) - self.assertTrue(len(expids) > 0) - self.assertEqual(expids, ref_expids) - - expids = sorted( - [ - exp["experiment_id"] - for exp in self.service.experiments( - tags=["a1", "b1"], tags_operator="AND", limit=None - ) - ] - ) - ref_expids = sorted( - [ - exp["experiment_id"] - for exp in self.expdict.values() - if "a1" in exp["tags"] and "b1" in exp["tags"] - ] - ) - self.assertTrue(len(expids) > 0) - self.assertEqual(expids, ref_expids) - - expids = sorted( - [ - exp["experiment_id"] - for exp in self.service.experiments( - tags=["a1", "c1"], tags_operator="AND", limit=None - ) - ] - ) - self.assertEqual(len(expids), 0) - - expids = sorted( - [ - exp["experiment_id"] - for exp in self.service.experiments(tags=["a0", "c0"], limit=None) - ] - ) - ref_expids = sorted( - [exp["experiment_id"] for exp in self.expdict.values() if "a0" in exp["tags"]] - ) - self.assertTrue(len(expids) > 0) - self.assertEqual(expids, ref_expids) - - expids = sorted( - [ - exp["experiment_id"] - for exp in self.service.experiments( - start_datetime_before=datetime(2022, 1, 1, 6), - start_datetime_after=datetime(2022, 1, 1, 3), - limit=None, - ) - ] - ) - self.assertEqual(expids, ["3", "4", "5", "6"]) - - datetimes = [exp["start_datetime"] for exp in self.service.experiments(limit=None)] - self.assertTrue(len(datetimes) > 0) - for i in range(len(datetimes) - 1): - self.assertTrue(datetimes[i] >= datetimes[i + 1]) - - datetimes = [ - exp["start_datetime"] - for exp in self.service.experiments(sort_by="start_datetime:asc", limit=None) - ] - self.assertTrue(len(datetimes) > 0) - for i in range(len(datetimes) - 1): - self.assertTrue(datetimes[i] <= datetimes[i + 1]) - - self.assertEqual(len(self.service.experiments(limit=4)), 4) - - def test_update_experiment(self): - """Test FakeService.update_experiment""" - self.service.update_experiment(experiment_id="1", metadata="hey", notes="hi") - exp = self.service.experiment(experiment_id="1") - self.assertEqual(exp["metadata"], "hey") - self.assertEqual(exp["notes"], "hi") - - def test_delete_experiment(self): - """Test FakeService.delete_experiment""" - exps = self.service.experiments( - start_datetime_before=datetime(2022, 1, 1, 2), - start_datetime_after=datetime(2022, 1, 1, 2), - ) - self.assertEqual(len(exps), 1) - self.service.delete_experiment(experiment_id="2") - exps = self.service.experiments( - start_datetime_before=datetime(2022, 1, 1, 2), - start_datetime_after=datetime(2022, 1, 1, 2), - ) - self.assertEqual(len(exps), 0) - - def test_update_result(self): - """Test FakeService.update_analysis_result""" - self.service.update_analysis_result(result_id="1", tags=["hey"]) - res = self.service.analysis_result(result_id="1") - self.assertEqual(res["tags"], "hey") - - def test_results_query(self): - """Test FakeService.analysis_results""" - for result_type in range(2): - resids = sorted( - [ - res.result_id - for res in self.service.analysis_results( - result_type=str(result_type), limit=None - ) - ] - ) - ref_resids = sorted( - [ - res["result_id"] - for res in self.resdict.values() - if res["result_type"] == str(result_type) - ] - ) - self.assertTrue(len(resids) > 0) - self.assertEqual(resids, ref_resids) - - for experiment_id in range(2): - resids = sorted( - [ - res.result_id - for res in self.service.analysis_results( - experiment_id=str(experiment_id), limit=None - ) - ] - ) - ref_resids = sorted( - [ - res["result_id"] - for res in self.resdict.values() - if res["experiment_id"] == str(experiment_id) - ] - ) - self.assertTrue(len(resids) > 0) - self.assertEqual(resids, ref_resids) - - for quality in range(2): - resids = sorted( - [ - res.result_id - for res in self.service.analysis_results(quality=quality, limit=None) - ] - ) - ref_resids = sorted( - [res["result_id"] for res in self.resdict.values() if res["quality"] == quality] - ) - self.assertTrue(len(resids) > 0) - self.assertEqual(resids, ref_resids) - - for verified in range(2): - resids = sorted( - [ - res.result_id - for res in self.service.analysis_results(verified=verified, limit=None) - ] - ) - ref_resids = sorted( - [res["result_id"] for res in self.resdict.values() if res["verified"] == verified] - ) - self.assertTrue(len(resids) > 0) - self.assertEqual(resids, ref_resids) - - for backend_name in range(2): - resids = sorted( - [ - res.result_id - for res in self.service.analysis_results( - backend_name=str(backend_name), limit=None - ) - ] - ) - ref_resids = sorted( - [ - res["result_id"] - for res in self.resdict.values() - if res["backend_name"] == str(backend_name) - ] - ) - self.assertTrue(len(resids) > 0) - self.assertEqual(resids, ref_resids) - - resids = sorted( - [ - res.result_id - for res in self.service.analysis_results( - tags=["a1", "b1"], tags_operator="AND", limit=None - ) - ] - ) - ref_resids = sorted( - [ - res["result_id"] - for res in self.resdict.values() - if "a1" in res["tags"] and "b1" in res["tags"] - ] - ) - self.assertTrue(len(resids) > 0) - self.assertEqual(resids, ref_resids) - - resids = sorted( - [ - res.result_id - for res in self.service.analysis_results( - tags=["a1", "c1"], tags_operator="AND", limit=None - ) - ] - ) - self.assertEqual(len(resids), 0) - - resids = sorted( - [res.result_id for res in self.service.analysis_results(tags=["a0", "c0"], limit=None)] - ) - ref_resids = sorted( - [res["result_id"] for res in self.resdict.values() if "a0" in res["tags"]] - ) - self.assertTrue(len(resids) > 0) - self.assertEqual(resids, ref_resids) - - datetimes = [res.creation_datetime for res in self.service.analysis_results(limit=None)] - self.assertTrue(len(datetimes) > 0) - for i in range(len(datetimes) - 1): - self.assertTrue(datetimes[i] >= datetimes[i + 1]) - - datetimes = [ - res.creation_datetime - for res in self.service.analysis_results(sort_by="creation_datetime:asc", limit=None) - ] - self.assertTrue(len(datetimes) > 0) - for i in range(len(datetimes) - 1): - self.assertTrue(datetimes[i] <= datetimes[i + 1]) - - self.assertEqual(len(self.service.analysis_results(limit=4)), 4) - - def test_delete_result(self): - """Test FakeService.delete_analysis_result""" - results = self.service.analysis_results(experiment_id="6") - old_number = len(results) - to_delete = results[0].result_id - self.service.delete_analysis_result(result_id=to_delete) - results = self.service.analysis_results(experiment_id="6") - self.assertEqual(len(results), old_number - 1) diff --git a/test/database_service/test_local_experiment_service.py b/test/database_service/test_local_experiment_service.py new file mode 100644 index 0000000000..db61c0aff6 --- /dev/null +++ b/test/database_service/test_local_experiment_service.py @@ -0,0 +1,357 @@ +# This code is part of Qiskit. +# +# (C) Copyright IBM 2021-2022. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE.txt file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +"""Local experiment client tests""" +import unittest +import json +from dataclasses import asdict +from datetime import datetime, timezone +from tempfile import TemporaryDirectory +from typing import Any + +from test.base import QiskitExperimentsTestCase + +import yaml + +from qiskit_experiments.database_service import ( + DbAnalysisResultData, + DbExperimentData, + ExperimentEntryNotFound, + LocalExperimentService, + ResultQuality, +) + + +class TestExperimentLocalClient(QiskitExperimentsTestCase): + """Test experiment modules.""" + + def setUp(self): + """Initial class level setup.""" + super().setUp() + self.service = LocalExperimentService() + + def test_create_or_update_experiment(self): + """Tests creating an experiment""" + data = DbExperimentData( + experiment_type="test_experiment", + backend="ibmq_qasm_simulator", + metadata={"float_data": 3.14, "string_data": "foo"}, + ) + exp_id = self.service.create_or_update_experiment(data).experiment_id + self.assertIsNotNone(exp_id) + + exp = self.service.experiment(experiment_id=exp_id) + self.assertEqual(exp.experiment_type, "test_experiment") + self.assertEqual(exp.backend, "ibmq_qasm_simulator") + self.assertEqual(exp.metadata["float_data"], 3.14) + self.assertEqual(exp.metadata["string_data"], "foo") + + def test_update_experiment(self): + """Tests updating an experiment""" + data = DbExperimentData( + experiment_type="test_experiment", + backend="ibmq_qasm_simulator", + metadata={"float_data": 3.14, "string_data": "foo"}, + ) + exp_id = self.service.create_or_update_experiment(data).experiment_id + data = self.service.experiment(exp_id) + data.metadata["float_data"] = 2.71 + data.experiment_type = "foo_type" + data.notes = ["foo_note"] + self.service.create_or_update_experiment(data) + result = self.service.experiment(exp_id) + self.assertEqual(result.metadata["float_data"], 2.71) + self.assertEqual(result.experiment_type, "foo_type") + self.assertEqual(result.notes[0], "foo_note") + + def test_delete_experiment(self): + """Tests deleting an experiment""" + data = DbExperimentData( + experiment_type="test_experiment", + backend="ibmq_qasm_simulator", + ) + exp_id = self.service.create_or_update_experiment(data).experiment_id + # Check the experiment exists + self.service.experiment(experiment_id=exp_id) + self.service.delete_experiment(exp_id) + with self.assertRaises(ExperimentEntryNotFound): + self.service.experiment(experiment_id=exp_id) + + def test_create_or_update_analysis_result(self): + """Tests creating an analysis result""" + exp_id = self.service.create_or_update_experiment( + DbExperimentData(experiment_type="test_experiment", backend="ibmq_qasm_simulator") + ).experiment_id + analysis_result_value = {"str": "foo", "float": 3.14} + analysis_data = DbAnalysisResultData( + experiment_id=exp_id, + result_data=analysis_result_value, + result_type="qiskit_test", + ) + analysis_id = self.service.create_or_update_analysis_result(analysis_data) + result = self.service.analysis_result(result_id=analysis_id) + self.assertEqual(result.result_type, "qiskit_test") + self.assertEqual(result.result_data["str"], analysis_result_value["str"]) + self.assertEqual(result.result_data["float"], analysis_result_value["float"]) + + def test_get_analysis_results(self): + """Tests getting an analysis result""" + exp_id = self.service.create_or_update_experiment( + DbExperimentData(experiment_type="test_experiment", backend="ibmq_qasm_simulator") + ).experiment_id + result_ids = ["00", "01", "10", "11"] + for result_id in result_ids: + analysis_result_value = { + "str": f"foo_{result_id}", + "float": 3.14 + int(result_id), + } + analysis_data = DbAnalysisResultData( + experiment_id=exp_id, + result_id=result_id, + result_data=analysis_result_value, + result_type=f"test_get_analysis_results_{result_id[0]}", + ) + self.service.create_or_update_analysis_result(analysis_data) + results = self.service.analysis_results( + result_type="test_get_analysis_results_0", + ) + self.assertEqual(len(results), 2) + results = self.service.analysis_results(result_type="test_get_analysis_results_1") + self.assertEqual(len(results), 2) + self.assertSetEqual({r.result_data["float"] for r in results}, {3.14 + 10, 3.14 + 11}) + + def test_delete_analysis_result(self): + """Tests deleting an analysis result""" + exp_id = self.service.create_or_update_experiment( + DbExperimentData(experiment_type="test_experiment", backend="ibmq_qasm_simulator") + ).experiment_id + analysis_data = DbAnalysisResultData( + experiment_id=exp_id, + result_data={"foo": "delete_bar"}, + result_type="test_result", + ) + result_id = self.service.create_or_update_analysis_result(analysis_data) + result = self.service.analysis_result(result_id) + self.assertEqual(result.result_data["foo"], "delete_bar") + self.service.delete_analysis_result(result_id) + with self.assertRaises(ExperimentEntryNotFound): + result = self.service.analysis_result(result_id) + + def test_update_analysis_result(self): + """Test updating an analysis result.""" + result_id = self._create_analysis_result() + fit = {"value": 41.456, "variance": 4.051} + chisq = 1.3253 + + self.service.create_or_update_analysis_result( + DbAnalysisResultData( + result_id=result_id, + result_data=fit, + tags=["qiskit_test"], + quality=ResultQuality.GOOD, + verified=True, + chisq=chisq, + ), + create=False, + ) + + rresult = self.service.analysis_result(result_id) + self.assertEqual(result_id, rresult.result_id) + self.assertEqual(fit, rresult.result_data) + self.assertEqual(["qiskit_test"], rresult.tags) + self.assertEqual(ResultQuality.GOOD, rresult.quality) + self.assertTrue(rresult.verified) + self.assertEqual(chisq, rresult.chisq) + + def test_figure(self): + """Test getting a figure.""" + exp_id = self.service.create_or_update_experiment( + DbExperimentData(experiment_type="test_experiment", backend="ibmq_qasm_simulator") + ).experiment_id + hello_bytes = str.encode("hello world") + figure_name = "hello.svg" + self.service.create_or_update_figure( + experiment_id=exp_id, figure=hello_bytes, figure_name=figure_name + ) + fig = self.service.figure(exp_id, figure_name) + self.assertEqual(fig, hello_bytes) + hello_bytes = str.encode("hello world version 2") + self.service.create_or_update_figure( + experiment_id=exp_id, + figure=hello_bytes, + figure_name=figure_name, + create=False, + ) + fig = self.service.figure(exp_id, figure_name) + self.assertEqual(fig, hello_bytes) + + def test_files(self): + """Test upload and download of files""" + exp_id = self.service.create_or_update_experiment( + DbExperimentData(experiment_type="test_experiment", backend="ibmq_qasm_simulator") + ).experiment_id + hello_data = {"hello": "world", "foo": "bar"} + filename = "test_file.json" + self.service.file_upload(exp_id, filename, hello_data) + rfile_data = self.service.file_download(exp_id, filename) + self.assertEqual(hello_data, rfile_data) + self.assertTrue(self.service.experiment_has_file(exp_id, filename)) + file_list = self.service.files(exp_id)["files"] + self.assertEqual(len(file_list), 1) + self.assertEqual(file_list[0]["Key"], filename) + + exp_id2 = self.service.create_or_update_experiment( + DbExperimentData(experiment_type="test_experiment", backend="ibmq_qasm_simulator") + ).experiment_id + file_list = self.service.files(exp_id2)["files"] + self.assertEqual(len(file_list), 0) + + def test_server_setting_start_time(self): + """Tests that start time is initialized by the server unless already present""" + ref_start_dt = datetime.now(timezone.utc) + exp_id = self.service.create_or_update_experiment( + DbExperimentData( + experiment_type="qiskit_time_test", + backend="ibmq_qasm_simulator", + ) + ).experiment_id + experiments = self.service.experiments() + found = False + for exp_id in experiments: + exp = self.service.experiment(exp_id) + if exp.experiment_id == exp_id: + found = True + self.assertTrue(found) + self.assertGreaterEqual(exp.start_datetime, ref_start_dt) + + def test_file_upload_formats(self): + """Test file upload/download for JSON and YAML formats""" + exp_id = self._create_experiment() + data = {"string": "b-string", "int": 10, "float": 0.333} + yaml_data = yaml.dump(data) + json_data = json.dumps(data) + yaml_filename = "data.yaml" + json_filename = "data.json" + + self.service.file_upload(exp_id, json_filename, json_data) + rjson_data = self.service.file_download(exp_id, json_filename) + self.assertEqual(data, rjson_data) + + self.service.file_upload(exp_id, yaml_filename, yaml_data) + ryaml_data = self.service.file_download(exp_id, yaml_filename) + self.assertEqual(data, ryaml_data) + file_list = self.service.files(exp_id)["files"] + self.assertEqual(len(file_list), 2) + + def test_save_to_disk(self): + """Test round trip of data to disk""" + # Make an experiemnt, add result, add figure, add json, add artifact (zip) + # Read it all back + data = DbExperimentData( + experiment_type="test_experiment", + backend="ibmq_qasm_simulator", + metadata={"float_data": 3.14, "string_data": "foo"}, + ) + + result_value = {"str": "foo", "float": 3.14} + result_data = DbAnalysisResultData( + experiment_id=data.experiment_id, + result_data=result_value, + result_type="qiskit_test", + ) + + figure_data = b"figure_data" + figure_name = "figure.svg" + + file_data = {"string": "b-string", "int": 10, "float": 0.333} + yaml_filename = "data.yaml" + json_filename = "data.json" + + zip_data = b"zip_data" + zip_name = "data.zip" + + with TemporaryDirectory() as tmpdirname: + save_service = LocalExperimentService(db_dir=tmpdirname) + save_service.create_or_update_experiment(data) + save_service.create_or_update_analysis_result(result_data) + save_service.create_or_update_figure(data.experiment_id, figure_data, figure_name) + save_service.file_upload(data.experiment_id, json_filename, file_data) + save_service.file_upload(data.experiment_id, yaml_filename, file_data) + save_service.file_upload(data.experiment_id, zip_name, zip_data) + + load_service = LocalExperimentService(db_dir=tmpdirname) + load_data = load_service.experiment(data.experiment_id) + load_result = load_service.analysis_result(result_data.result_id) + load_figure = load_service.figure(data.experiment_id, figure_name) + load_json = load_service.file_download(data.experiment_id, json_filename) + load_yaml = load_service.file_download(data.experiment_id, yaml_filename) + load_zip = load_service.file_download(data.experiment_id, zip_name) + + # Filter values because service can insert dates and change empty values' types + data_dict = {k: v for k, v in asdict(data).items() if v} + load_data_dict = {k: v for k, v in asdict(load_data).items() if k in data_dict} + self.assertDictEqual(data_dict, load_data_dict) + + result_data_dict = {k: v for k, v in asdict(result_data).items() if v} + load_result_data_dict = { + k: v for k, v in asdict(load_result).items() if k in result_data_dict + } + self.assertEqual(result_data_dict, load_result_data_dict) + + self.assertEqual(figure_data, load_figure) + self.assertEqual(file_data, load_json) + self.assertEqual(file_data, load_yaml) + self.assertEqual(zip_data, load_zip) + + def _create_experiment( + self, + experiment_type: str | None = None, + json_encoder: json.JSONEncoder | None = None, + **kwargs, + ) -> str: + """Create a new experiment.""" + experiment_type = experiment_type or "qiskit_test" + exp_id = self.service.create_or_update_experiment( + DbExperimentData( + experiment_type=experiment_type, + **kwargs, + ), + json_encoder=json_encoder, + ).experiment_id + return exp_id + + def _create_analysis_result( + self, + exp_id: str | None = None, + result_type: str | None = None, + result_data: dict | None = None, + json_encoder: json.JSONEncoder | None = None, + **kwargs: Any, + ): + """Create a simple analysis result.""" + experiment_id = exp_id or self._create_experiment() + result_type = result_type or "qiskit_test" + result_data = result_data or {} + aresult_id = self.service.create_or_update_analysis_result( + DbAnalysisResultData( + experiment_id=experiment_id, + result_data=result_data, + result_type=result_type, + **kwargs, + ), + json_encoder=json_encoder, + ) + return aresult_id + + +if __name__ == "__main__": + unittest.main() diff --git a/test/extended_equality.py b/test/extended_equality.py index 4039e3e0fa..e3bb7d0447 100644 --- a/test/extended_equality.py +++ b/test/extended_equality.py @@ -387,9 +387,17 @@ def _check_experiment_data( data2.child_data(), **kwargs, ) + artifacts1 = data1.artifacts() + if not isinstance(artifacts1, list): + artifacts1 = [artifacts1] + artifacts2 = data2.artifacts() + if not isinstance(artifacts2, list): + artifacts2 = [artifacts2] + artifacts1.sort(key=lambda a: a.name) + artifacts2.sort(key=lambda a: a.name) artifact_equiv = is_equivalent( - data1.artifacts(), - data2.artifacts(), + artifacts1, + artifacts2, **kwargs, ) diff --git a/test/fake_experiment.py b/test/fake_experiment.py index 220a1ddd94..0d833631c0 100644 --- a/test/fake_experiment.py +++ b/test/fake_experiment.py @@ -13,7 +13,6 @@ """A FakeExperiment for testing.""" import numpy as np -import pandas as pd from matplotlib.figure import Figure as MatplotlibFigure from qiskit import QuantumCircuit from qiskit_experiments.framework import ( @@ -41,7 +40,18 @@ def _run_analysis(self, experiment_data): analysis_results = [ AnalysisResultData(f"result_{i}", value) for i, value in enumerate(rng.random(3)) ] - scatter_table = ScatterTable.from_dataframe(pd.DataFrame(columns=ScatterTable.COLUMNS)) + scatter_table = ScatterTable() + for val in range(3): + scatter_table.add_row( + xval=float(val), + yval=0.1 * val, + yerr=0.1, + series_name="model1", + series_id=0, + category="raw", + shots=1000, + analysis="FakeAnalysis", + ) fit_data = CurveFitResult( method="some_method", model_repr={"s1": "par0 * x + par1"}, diff --git a/test/framework/test_composite.py b/test/framework/test_composite.py index 5034a82f9f..c98c55bc9e 100644 --- a/test/framework/test_composite.py +++ b/test/framework/test_composite.py @@ -18,7 +18,6 @@ from test.fake_experiment import FakeExperiment, FakeAnalysis from test.base import QiskitExperimentsTestCase -from unittest import mock from ddt import ddt, data from qiskit import QuantumCircuit @@ -26,8 +25,7 @@ from qiskit_aer import AerSimulator, noise -from qiskit_ibm_experiment import IBMExperimentService - +from qiskit_experiments.database_service import LocalExperimentService from qiskit_experiments.exceptions import QiskitError from qiskit_experiments.test.utils import FakeJob from qiskit_experiments.test.fake_backend import FakeBackend @@ -284,7 +282,7 @@ def test_composite_save_load(self): Verify that saving and loading restores the original composite experiment data object """ - self.rootdata.service = IBMExperimentService(local=True, local_save=False) + self.rootdata.service = LocalExperimentService() self.rootdata.save() loaded_data = ExperimentData.load(self.rootdata.experiment_id, self.rootdata.service) self.check_if_equal(loaded_data, self.rootdata, is_a_copy=False, check_artifact=True) @@ -293,7 +291,7 @@ def test_composite_save_metadata(self): """ Verify that saving metadata and loading restores the original composite experiment data object """ - self.rootdata.service = IBMExperimentService(local=True, local_save=False) + self.rootdata.service = LocalExperimentService() self.rootdata.save_metadata() loaded_data = ExperimentData.load(self.rootdata.experiment_id, self.rootdata.service) self.check_if_equal(loaded_data, self.rootdata, is_a_copy=False) @@ -436,7 +434,7 @@ def test_composite_figures(self): par_exp = BatchExperiment([exp1, exp2], flatten_results=False) expdata = par_exp.run(FakeBackend(num_qubits=4)) self.assertExperimentDone(expdata) - expdata.service = IBMExperimentService(local=True, local_save=False) + expdata.service = LocalExperimentService() expdata.auto_save = True par_exp.analysis.run(expdata) self.assertExperimentDone(expdata) @@ -445,7 +443,7 @@ def test_composite_auto_save(self): """ Test setting autosave when using composite experiments """ - service = mock.create_autospec(IBMExperimentService, instance=True) + service = LocalExperimentService() exp1 = FakeExperiment([0, 2]) exp2 = FakeExperiment([1, 3]) par_exp = BatchExperiment([exp1, exp2], flatten_results=False) @@ -453,7 +451,9 @@ def test_composite_auto_save(self): expdata.service = service self.assertExperimentDone(expdata) expdata.auto_save = True - self.assertEqual(service.create_or_update_experiment.call_count, 3) + for child_data in expdata.child_data(): + results = service.analysis_results(experiment_id=child_data.experiment_id) + self.assertEqual(len(results), 3) def test_composite_subexp_data(self): """ diff --git a/test/framework/test_framework.py b/test/framework/test_framework.py index ce51c57930..f14dae5ec9 100644 --- a/test/framework/test_framework.py +++ b/test/framework/test_framework.py @@ -15,7 +15,9 @@ import datetime import json import pickle +import uuid from itertools import product +from tempfile import TemporaryDirectory from test.fake_experiment import FakeExperiment, FakeAnalysis from test.base import QiskitExperimentsTestCase @@ -24,10 +26,11 @@ from qiskit import QuantumCircuit from qiskit.providers.jobstatus import JobStatus +from qiskit.result import Result from qiskit.exceptions import QiskitError from qiskit_ibm_runtime.fake_provider import FakeVigoV2 -from qiskit_experiments.database_service import Qubit +from qiskit_experiments.database_service import LocalExperimentService, Qubit from qiskit_experiments.exceptions import AnalysisError from qiskit_experiments.framework import ( ExperimentData, @@ -40,7 +43,7 @@ AnalysisStatus, ) from qiskit_experiments.test.fake_backend import FakeBackend -from qiskit_experiments.test.utils import FakeJob +from qiskit_experiments.test.utils import FakeJob, FakeProvider @ddt.ddt @@ -175,6 +178,41 @@ def test_experiment_data_analysis_results_json_roundtrip(self): result2 = next(expdata2.analysis_results(dataframe=True).itertuples()) self.assertEqual(result1, result2) + def test_run_analysis_experiment_data_experiment_service_roundtrip(self): + """Test ExperimentData after experiment service roundtrip""" + provider = FakeProvider() + backend = FakeBackend() + job = FakeJob( + backend, + Result.from_dict( + { + "backend_name": backend.name, + "job_id": uuid.uuid4().hex, + "success": True, + "results": [{"shots": 100, "success": True, "data": {"counts": {"0": 100}}}], + } + ), + ) + provider.add_job(job) + expdata1 = ExperimentData() + analysis = FakeAnalysis() + expdata1.add_jobs([job]) + # Set physical qubit for more complete comparison + expdata1.metadata["physical_qubits"] = (1,) + expdata1 = analysis.run(expdata1, seed=54321) + self.assertExperimentDone(expdata1) + + with TemporaryDirectory() as tmpdir: + service = LocalExperimentService(db_dir=tmpdir) + expdata1.service = service + expdata1.save() + + expdata2 = ExperimentData.load( + expdata1.experiment_id, provider=provider, service=service + ) + + self.assertEqualExtended(expdata1, expdata2) + def test_analysis_replace_results_true(self): """Test running analysis with replace_results=True""" analysis = FakeAnalysis() diff --git a/test/library/characterization/test_readout_error.py b/test/library/characterization/test_readout_error.py index ba46d96341..c60d1cd26f 100644 --- a/test/library/characterization/test_readout_error.py +++ b/test/library/characterization/test_readout_error.py @@ -20,9 +20,9 @@ import numpy as np from qiskit.quantum_info.operators.predicates import matrix_equal from qiskit_aer import AerSimulator -from qiskit_ibm_experiment import IBMExperimentService from qiskit_ibm_runtime.fake_provider import FakeParisV2 from qiskit_experiments.library.characterization import LocalReadoutError, CorrelatedReadoutError +from qiskit_experiments.database_service import LocalExperimentService from qiskit_experiments.framework import ExperimentData from qiskit_experiments.framework import ParallelExperiment from qiskit_experiments.framework.json import ExperimentEncoder, ExperimentDecoder @@ -212,7 +212,7 @@ def test_database_save_and_load(self): exp = LocalReadoutError(qubits) exp_data = exp.run(backend) self.assertExperimentDone(exp_data) - exp_data.service = IBMExperimentService(local=True, local_save=False) + exp_data.service = LocalExperimentService() exp_data.save() loaded_data = ExperimentData.load(exp_data.experiment_id, exp_data.service) exp_res = exp_data.analysis_results(dataframe=True)