From 1eb9decddf70348678eb1e9b2904656748e0bc4a Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Fri, 15 May 2026 11:51:06 +0200 Subject: [PATCH 01/28] add hybridization class --- petab/v2/core.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/petab/v2/core.py b/petab/v2/core.py index 2bf6a49e..be459bdd 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -1108,6 +1108,77 @@ def n_estimated(self) -> int: return sum(p.estimate for p in self.parameters) +class Hybridization(BaseModel): + """Assigns NN inputs and outputs.""" + + #: The target ID. + target_id: str = Field(alias=C.TARGET_ID) + #: The target value. + target_val: sp.Basic = Field(alias=C.TARGET_VALUE) + + #: :meta private: + model_config = ConfigDict( + arbitrary_types_allowed=True, + populate_by_name=True, + extra="allow", + validate_assignment=True, + ) + + @field_validator("target_val", mode="before") + @classmethod + def _sympify(cls, v): + if v is None or isinstance(v, sp.Basic): + return v + if isinstance(v, float) and np.isnan(v): + return None + + return sympify_petab(v) + + +class HybridizationTable(BaseTable[Hybridization]): + """PEtab hybridization table.""" + + @property + def hybridizations(self) -> list[Hybridization]: + """List of hybridizations.""" + return self.elements + + @classmethod + def from_df(cls, df: pd.DataFrame, **kwargs) -> HybridizationTable: + """Create a HybridizationTable from a DataFrame.""" + if df is None: + return cls(**kwargs) + + hybridizations = [ + Hybridization( + **row.to_dict(), + ) + for _, row in df.reset_index().iterrows() + ] + + return cls(hybridizations, **kwargs) + + def to_df(self) -> pd.DataFrame: + """Convert the HybridizationTable to a DataFrame.""" + records = self.model_dump(by_alias=True)["elements"] + + return pd.DataFrame(records) + + def __getitem__(self, target_id: str) -> Hybridization: + """Get a hybridization by target ID.""" + for hybridization in self.hybridizations: + if hybridization.target_id == target_id: + return hybridization + raise KeyError(f"Target ID {target_id} not found") + + def get(self, target_id, default=None): + """Get a hybridization by target ID or return a default value.""" + try: + return self[target_id] + except KeyError: + return default + + class Problem: """ PEtab parameter estimation problem From 11652181ed4ca266d2ad4d2343dd240e35ae2d03 Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Fri, 15 May 2026 12:15:12 +0200 Subject: [PATCH 02/28] add sciml problem config --- petab/v2/C.py | 2 ++ petab/v2/core.py | 77 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/petab/v2/C.py b/petab/v2/C.py index e640ae5c..7b141b77 100644 --- a/petab/v2/C.py +++ b/petab/v2/C.py @@ -258,6 +258,8 @@ MAPPING_FILES = "mapping_files" #: Extensions key in the YAML file EXTENSIONS = "extensions" +#: PEtab SciML extension +SCIML = "sciml" # MAPPING diff --git a/petab/v2/core.py b/petab/v2/core.py index be459bdd..d3be5638 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -2012,14 +2012,21 @@ def validate( validation_results = ValidationResultList() - if self.config and self.config.extensions: - extensions = ",".join(self.config.extensions.keys()) + supported_extensions = {C.SCIML} + if ( + self.config + and self.config.extensions + and (self.config.extensions.keys() - supported_extensions) + ): + extensions_without_support = ",".join( + self.config.extensions.keys() - supported_extensions + ) validation_results.append( ValidationIssue( ValidationIssueSeverity.WARNING, - "Validation of PEtab extensions is not yet implemented, " - "but the given problem uses the following extensions: " - f"{extensions}", + "The given problem uses the following extensions for " + "which validation is not yet implemented: " + f"{extensions_without_support}", ) ) @@ -2521,6 +2528,18 @@ class ModelFile(BaseModel): ) +class NeuralNetConfig(BaseModel): + """A neural net in the PEtab SciML problem configuration.""" + + location: AnyUrl | Path + pre_initialization: bool + format: str + + model_config = ConfigDict( + validate_assignment=True, + ) + + class ExtensionConfig(BaseModel): """The configuration of a PEtab extension.""" @@ -2528,6 +2547,25 @@ class ExtensionConfig(BaseModel): config: dict +class SciMLConfig(BaseModel): + """The extended configuration of a PEtab SciML problem.""" + + #: The PEtab SciML format version. + version: str = "0.1.0" + #: The paths to the array data files. + # Absolute or relative to `base_path`. + array_files: list[AnyUrl | Path] = [] + #: The paths to the hybridization tables. + # Absolute or relative to `base_path`. + hybridization_files: list[AnyUrl | Path] = [] + #: The neural network IDs and info. + neural_nets: dict[str, NeuralNetConfig] | None = {} + + model_config = ConfigDict( + validate_assignment=True, + ) + + class ProblemConfig(BaseModel): """The PEtab problem configuration.""" @@ -2577,6 +2615,23 @@ class ProblemConfig(BaseModel): validate_assignment=True, ) + @field_validator("extensions", mode="before") + @classmethod + def _parse_extensions(cls, v): + """Parse extensions dict and convert known extensions to their specific + config classes.""" + if isinstance(v, dict): + parsed_extensions = {} + for ext_name, ext_config in v.items(): + if ext_name == C.SCIML: + # Convert sciml extension to SciMLConfig + parsed_extensions[ext_name] = SciMLConfig(**ext_config) + else: + # Keep other extensions as ExtensionConfig + parsed_extensions[ext_name] = ExtensionConfig(**ext_config) + return parsed_extensions + return v + # convert parameter_file to list @field_validator( "parameter_files", @@ -2614,12 +2669,22 @@ def to_yaml(self, filename: str | Path): for model_id in data.get("model_files", {}): data["model_files"][model_id][C.MODEL_LOCATION] = str( - data["model_files"][model_id]["location"] + data["model_files"][model_id][C.MODEL_LOCATION] ) if data["id"] is None: # The schema requires a valid id or no id field at all. del data["id"] + for ext_id, d_ext in data[C.EXTENSIONS].items(): + if ext_id == C.SCIML: + # convert Paths to strings + for key in ("array_files", "hybridization_files"): + d_ext[key] = list(map(str, data[key])) + for nn in d_ext["neural_nets"]: + d_ext["neural_nets"][nn][C.MODEL_LOCATION] = str( + d_ext["neural_nets"][nn][C.MODEL_LOCATION] + ) + write_yaml(data, filename) @property From f2c41931332ab2d9c7391e20ca4fe2be7764b257 Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Mon, 25 May 2026 17:22:39 +0200 Subject: [PATCH 03/28] create Problem for sciml extension --- petab/v2/core.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/petab/v2/core.py b/petab/v2/core.py index d3be5638..1067440e 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -1109,7 +1109,7 @@ def n_estimated(self) -> int: class Hybridization(BaseModel): - """Assigns NN inputs and outputs.""" + """Assigns PEtab SciML NN inputs and outputs.""" #: The target ID. target_id: str = Field(alias=C.TARGET_ID) @@ -1136,7 +1136,7 @@ def _sympify(cls, v): class HybridizationTable(BaseTable[Hybridization]): - """PEtab hybridization table.""" + """PEtab SciML hybridization table.""" @property def hybridizations(self) -> list[Hybridization]: @@ -1153,7 +1153,7 @@ def from_df(cls, df: pd.DataFrame, **kwargs) -> HybridizationTable: Hybridization( **row.to_dict(), ) - for _, row in df.reset_index().iterrows() + for _, row in df.iterrows() ] return cls(hybridizations, **kwargs) @@ -1205,6 +1205,9 @@ def __init__( measurement_tables: list[MeasurementTable] = None, parameter_tables: list[ParameterTable] = None, mapping_tables: list[MappingTable] = None, + neural_networks: list[NNModel] | None = None, + hybridization_tables: list[HybridizationTable] | None = None, + array_data_files: list[ArrayData] | None = None, config: ProblemConfig = None, ): from ..v2.lint import default_validation_tasks @@ -1221,6 +1224,11 @@ def __init__( self.measurement_tables = measurement_tables or [MeasurementTable()] self.mapping_tables = mapping_tables or [MappingTable()] self.parameter_tables = parameter_tables or [ParameterTable()] + self.neural_networks = neural_networks or [] + self.hybridization_tables = hybridization_tables or [ + HybridizationTable() + ] + self.array_data_files = array_data_files or [] def __repr__(self): return f"<{self.__class__.__name__} id={self.id!r}>" @@ -1393,6 +1401,59 @@ def from_yaml( else None ) + # sciml extension + if config.extensions and config.extensions[C.SCIML]: + try: + from petab_sciml import ( + ArrayDataStandard, + NNModel, + NNModelStandard, + ) + except ImportError as e: + raise ImportError( + "To generate a PEtab SciML problem, (petab_sciml) must be" + "installed." + ) from e + + # Neural network classes are constructed via pytorch for now to get the + # proper inputs + neural_networks = ( + [ + NNModel.from_pytorch_module( + NNModelStandard.load_data( + _generate_path( + file_path=nn_config.location, + base_path=base_path, + ) + ).to_pytorch_module(), + nn_model_id=nn_id, + ) + for nn_id, nn_config in ( + config.extensions[C.SCIML].neural_nets or {} + ).items() + ] + if config.extensions and config.extensions[C.SCIML] + else None + ) + + hybridization_tables = ( + [ + HybridizationTable.from_tsv(f, base_path) + for f in config.extensions[C.SCIML].hybridization_files + ] + if config.extensions and config.extensions[C.SCIML] + else None + ) + + array_data_files = ( + [ + ArrayDataStandard.load_data(_generate_path(f, base_path)) + for f in config.extensions[C.SCIML].array_files + ] + if config.extensions and config.extensions[C.SCIML] + else None + ) + return Problem( config=config, models=models, @@ -1402,6 +1463,9 @@ def from_yaml( measurement_tables=measurement_tables, parameter_tables=parameter_tables, mapping_tables=mapping_tables, + neural_networks=neural_networks, + hybridization_tables=hybridization_tables, + array_data_files=array_data_files, ) @staticmethod @@ -1708,6 +1772,15 @@ def id(self, value: str): self.config = ProblemConfig(format_version="2.0.0") self.config.id = value + @property + def hybridizations(self) -> list[Hybridization]: + """List of hybridizations in the hybridization table(s).""" + return list( + chain.from_iterable( + ht.hybridizations for ht in self.hybridization_tables + ) + ) + def get_optimization_parameters(self) -> list[str]: """ Get the list of optimization parameter IDs from parameter table. From 96858f230af8b47995c9074234464316354ce435 Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Tue, 26 May 2026 13:09:11 +0200 Subject: [PATCH 04/28] Assign SciML-specific lint checks --- petab/v2/core.py | 13 +++++++++---- petab/v2/lint.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/petab/v2/core.py b/petab/v2/core.py index 1067440e..fac583ac 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -1210,13 +1210,18 @@ def __init__( array_data_files: list[ArrayData] | None = None, config: ProblemConfig = None, ): - from ..v2.lint import default_validation_tasks + from ..v2.lint import default_validation_tasks, sciml_validation_tasks self.config = config self.models: list[Model] = models or [] - self.validation_tasks: list[ValidationTask] = ( - default_validation_tasks.copy() - ) + if config.extensions and config.extensions[C.SCIML]: + self.validation_tasks: list[ValidationTask] = ( + sciml_validation_tasks.copy() + ) + else: + self.validation_tasks: list[ValidationTask] = ( + default_validation_tasks.copy() + ) self.observable_tables = observable_tables or [ObservableTable()] self.condition_tables = condition_tables or [ConditionTable()] diff --git a/petab/v2/lint.py b/petab/v2/lint.py index a287c915..105715e3 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -970,6 +970,40 @@ def run(self, problem: Problem) -> ValidationIssue | None: return None +class CheckHybridizationTable(ValidationTask): + """Validate the SciML hybridization table.""" + + def run(self, problem: Problem) -> ValidationIssue | None: + messages = [] + + condition_targets = { + c.target_id for ct in problem.conditions for c in ct.changes + } + nn_input_ids = { + inp.input_id for nn in problem.neural_networks for inp in nn.inputs + } + hyb_target_ids = {hyb.target_id for hyb in problem.hybridizations} + hyb_target_vals = {hyb.target_value for hyb in problem.hybridizations} + + # Hybridization targets are not also targets in the condition table + if culprits := (hyb_target_ids & condition_targets): + messages.append( + f"Hybridization target ids `{culprits}` are also " + "target ids in the condition table." + ) + # NN inputs are not used as target values + if culprits := (hyb_target_vals & nn_input_ids): + messages.append( + "The following neural net inputs were used as target values " + f"in the Hybridization tbale: `{culprits}`. Please simplify." + ) + + if messages: + return ValidationError("\n".join(messages)) + + return None + + def get_valid_parameters_for_parameter_table( problem: Problem, ) -> set[str]: @@ -1177,3 +1211,13 @@ def get_placeholders( CheckInitialChangeSymbols(), CheckMappingTable(), ] + +#: Validation tasks that should be run PEtab SciML problems +sciml_validation_tasks = [ + CheckHybridizationTable(), +] + list( + set(default_validation_tasks) + - { + CheckAllParametersPresentInParameterTable(), + } +) From dd3f87ba2da1db31619c952ddc307ff9759901e2 Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Tue, 26 May 2026 14:14:40 +0200 Subject: [PATCH 05/28] Proble.add_hybridization method --- petab/v2/core.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/petab/v2/core.py b/petab/v2/core.py index fac583ac..1de26993 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -2402,6 +2402,23 @@ def add_experiment(self, id_: str, *args): Experiment(id=id_, periods=periods) ) + def add_hybridization(self, target_id: str, target_value: str): + """Add a SciML hybridization table entry to the problem. + + If there are more than one hybridization tables, the hybridization + is added to the last one. + + Arguments: + target_id: The ID of the target entity in the PEtab problem + or neural network model + target_value: The value that is assigned to the target id. + """ + if not self.hybridization_tables: + self.hybridization_tables.append(HybridizationTable()) + self.hybridization_tables[-1].hybridizations.append( + Hybridization(target_id=target_id, target_value=target_value) + ) + def __iadd__(self, other): """Add Observable, Parameter, Measurement, Condition, or Experiment""" from .core import ( From 8c1a8a9a670d09cfd6dd5ff25769c0e6179a1353 Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Tue, 26 May 2026 15:09:25 +0200 Subject: [PATCH 06/28] add hybridization df property, setter --- petab/v2/core.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/petab/v2/core.py b/petab/v2/core.py index 1de26993..be55d1eb 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -1786,6 +1786,19 @@ def hybridizations(self) -> list[Hybridization]: ) ) + @property + def hybridization_df(self) -> pd.DataFrame | None: + """Combined SciML hybridization tables as DataFrame.""" + return ( + HybridizationTable(hybridizations).to_df() + if (hybridizations := self.hybridizations) + else None + ) + + @hybridization_df.setter + def hybridization_df(self, value: pd.DataFrame): + self.hybridization_tables = [HybridizationTable.from_df(value)] + def get_optimization_parameters(self) -> list[str]: """ Get the list of optimization parameter IDs from parameter table. From 8c5cd2b9de7aa2caaa562aac281475f64ba47fbf Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Tue, 26 May 2026 23:59:40 +0200 Subject: [PATCH 07/28] fix --- petab/v2/core.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/petab/v2/core.py b/petab/v2/core.py index be55d1eb..c055d023 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -61,6 +61,7 @@ __all__ = [ "Problem", "ProblemConfig", + "SciMLConfig", "Observable", "ObservableTable", "NoiseDistribution", @@ -1114,7 +1115,7 @@ class Hybridization(BaseModel): #: The target ID. target_id: str = Field(alias=C.TARGET_ID) #: The target value. - target_val: sp.Basic = Field(alias=C.TARGET_VALUE) + target_value: sp.Basic = Field(alias=C.TARGET_VALUE) #: :meta private: model_config = ConfigDict( @@ -1124,7 +1125,7 @@ class Hybridization(BaseModel): validate_assignment=True, ) - @field_validator("target_val", mode="before") + @field_validator("target_value", mode="before") @classmethod def _sympify(cls, v): if v is None or isinstance(v, sp.Basic): @@ -1214,7 +1215,7 @@ def __init__( self.config = config self.models: list[Model] = models or [] - if config.extensions and config.extensions[C.SCIML]: + if config and config.extensions and config.extensions[C.SCIML]: self.validation_tasks: list[ValidationTask] = ( sciml_validation_tasks.copy() ) @@ -2787,7 +2788,7 @@ def to_yaml(self, filename: str | Path): if ext_id == C.SCIML: # convert Paths to strings for key in ("array_files", "hybridization_files"): - d_ext[key] = list(map(str, data[key])) + d_ext[key] = list(map(str, d_ext[key])) for nn in d_ext["neural_nets"]: d_ext["neural_nets"][nn][C.MODEL_LOCATION] = str( d_ext["neural_nets"][nn][C.MODEL_LOCATION] From 238829bd0d75008a5aa2a8a47172aaa9efebe365 Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Wed, 27 May 2026 00:25:14 +0200 Subject: [PATCH 08/28] add a basic test for sciml --- petab/v2/lint.py | 27 ++++++-- tests/v2/test_sciml.py | 138 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 tests/v2/test_sciml.py diff --git a/petab/v2/lint.py b/petab/v2/lint.py index 105715e3..3676604d 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -45,6 +45,7 @@ "CheckUndefinedExperiments", "CheckInitialChangeSymbols", "CheckMappingTable", + "CheckHybridizationTable", "lint_problem", "default_validation_tasks", ] @@ -1214,10 +1215,24 @@ def get_placeholders( #: Validation tasks that should be run PEtab SciML problems sciml_validation_tasks = [ + CheckProblemConfig(), + CheckModel(), + CheckUniquePrimaryKeys(), + CheckMeasurementModelId(), + CheckMeasuredObservablesDefined(), + CheckPosLogMeasurements(), + CheckOverridesMatchPlaceholders(), + CheckValidConditionTargets(), + CheckExperimentTable(), + CheckExperimentConditionsExist(), + CheckUndefinedExperiments(), + CheckObservablesDoNotShadowModelEntities(), + # CheckAllParametersPresentInParameterTable(), + CheckValidParameterInConditionOrParameterTable(), + CheckUnusedExperiments(), + CheckUnusedConditions(), + CheckPriorDistribution(), + CheckInitialChangeSymbols(), + CheckMappingTable(), CheckHybridizationTable(), -] + list( - set(default_validation_tasks) - - { - CheckAllParametersPresentInParameterTable(), - } -) +] diff --git a/tests/v2/test_sciml.py b/tests/v2/test_sciml.py new file mode 100644 index 00000000..9e39fc80 --- /dev/null +++ b/tests/v2/test_sciml.py @@ -0,0 +1,138 @@ +import numpy as np +from pydantic import ConfigDict + +from petab.v2.core import * +from petab.v2.core import ModelFile, NeuralNetConfig +from petab.v2.lint import sciml_validation_tasks +from petab.v2.models.sbml_model import SbmlModel + + +def _get_test_problem(): + problem = Problem() + problem.validation_tasks = sciml_validation_tasks + problem.config = ProblemConfig( + format_version="2.0.0", + model_files=ConfigDict( + {"lv": ModelFile(location="lv.xml", language="sbml")} + ), + parameter_files=["parameters.tsv"], + measurement_files=["measurements.tsv"], + observable_files=["observables.tsv"], + experiment_files=["experiments.tsv"], + mapping_files=["mappings.tsv"], + extensions={ + "sciml": { + "version": "0.1.0", + "array_files": ["net1_ps.hdf5"], + "hybridization_files": ["hybridizations.tsv"], + "neural_nets": { + "net1": NeuralNetConfig( + location="net1.yaml", + pre_initialization=False, + format="YAML", + ) + }, + } + }, + ) + problem.model = SbmlModel.from_antimony(""" + model lv + species A, B; + A = 0.442; + B = 4.63; + alpha = 1.3; + gamma_ = 0.8; + -> A; alpha * A; + B -> ; 1.8 * B; + A -> ; 0.9 * A * B; + -> B; gamma_; + end + """) + problem.add_experiment("e1", 0, "") + problem.add_mapping("net1_input1", "net1.inputs[0][0]") + problem.add_mapping("net1_input2", "net1.inputs[0][1]") + problem.add_mapping("net1_output1", "net1.outputs[0][0]") + problem.add_mapping("net1_ps", "net1.parameters") + problem.add_measurement("B_obs", time=1, measurement=1, experiment_id="e1") + problem.add_observable("B_obs", "B", noise_formula="0.05") + problem.add_parameter( + "alpha", estimate=True, lb=0, ub=15, nominal_value=1.3 + ) + problem.add_parameter( + "net1_ps", estimate=True, lb=-np.inf, ub=np.inf, nominal_value="array" + ) + problem.add_hybridization("net1_input1", "A") + problem.add_hybridization("net1_input2", "B") + problem.add_hybridization("gamma_", "net1_output_1") + problem.add_neural_network_from_dict( + "net1", + nn_dict={ + "nn_model_id": "net1", + "inputs": [{"input_id": "input0"}], + "layers": [ + { + "layer_id": "layer1", + "layer_type": "Linear", + "args": { + "in_features": 2, + "out_features": 1, + "bias": True, + }, + } + ], + "forward": [ + { + "name": "net_input", + "op": "placeholder", + "target": "net_input", + }, + { + "name": "layer1", + "op": "call_module", + "target": "layer1", + "args": ["net_input"], + }, + { + "name": "tanh", + "op": "call_method", + "target": "tanh", + "args": ["layer1"], + }, + ], + }, + ) + + # array data + problem.add_array_data_from_dict( + { + "metadata": {"pytorch_format": True}, + "inputs": {}, + "parameters": { + "net1": { + "layer1": { + "bias": np.random.randn(2), + "weight": np.random.randn(2), + } + } + }, + } + ) + + # set the filenames + problem.config.filepath = "problem.yaml" + problem.model.rel_path = "lv.xml" + problem.experiment_tables[0].rel_path = "experiments.tsv" + problem.mapping_tables[0].rel_path = "mappings.tsv" + problem.measurement_tables[0].rel_path = "measurements.tsv" + problem.observable_tables[0].rel_path = "observables.tsv" + problem.parameter_tables[0].rel_path = "parameters.tsv" + problem.hybridization_tables[0].rel_path = "hybridizations.tsv" + # problem.neural_networks[0].rel_path = "net1.yaml" + # problem.array_data_files[0].rel_path = "net1_ps.hdf5" + + return problem + + +def test_lint(): + problem = _get_test_problem() + assert problem.validate() == [] From d4d8a94c6e121b2f3bc40b2f88c943c4f7c72bb0 Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Wed, 27 May 2026 00:32:11 +0200 Subject: [PATCH 09/28] methods for adding nn, array data to problem --- petab/v2/core.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/petab/v2/core.py b/petab/v2/core.py index c055d023..77fa873b 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -2433,6 +2433,51 @@ def add_hybridization(self, target_id: str, target_value: str): Hybridization(target_id=target_id, target_value=target_value) ) + def add_neural_network_from_dict(self, model_id: str, nn_dict: dict): + """Add a SciML neural net from a dictionary.""" + from petab_sciml import NNModel + + nn_model = NNModel.model_validate(nn_dict) + nn_model.nn_model_id = model_id + self.neural_networks.append(nn_model) + + def add_neural_network_from_yaml( + self, + model_id: str, + file_path: str | Path, + base_path: str | Path | None = None, + ): + """Add a SciML neural net from a yaml file.""" + from petab_sciml import NNModelStandard + + self.neural_networks.append( + NNModelStandard.load_data( + _generate_path( + file_path=file_path, + base_path=base_path, + ), + nn_model_id=model_id, + ) + ) + + def add_array_data_from_dict(self, array_data: dict): + """Add SciML array data from a dictionary.""" + from petab_sciml import ArrayData + + self.array_data_files.append(ArrayData.model_validate(array_data)) + + def add_array_data_from_hdf5( + self, + file_path: str | Path, + base_path: str | Path | None = None, + ): + """Add SciML array data from an hdf5 file.""" + from petab_sciml import ArrayDataStandard + + self.array_data_files.append( + ArrayDataStandard.load_data(_generate_path(file_path, base_path)) + ) + def __iadd__(self, other): """Add Observable, Parameter, Measurement, Condition, or Experiment""" from .core import ( From 653ccc7a678116c7573ffea5d4f4c74c8e4e1cf7 Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Wed, 27 May 2026 08:27:15 +0200 Subject: [PATCH 10/28] Non-local petab_sciml import --- petab/v2/core.py | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/petab/v2/core.py b/petab/v2/core.py index 77fa873b..64c362d1 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -54,6 +54,16 @@ from ..versions import parse_version from . import C, get_observable_df +try: + from petab_sciml import ( + ArrayData, + ArrayDataStandard, + NNModel, + NNModelStandard, + ) +except ModuleNotFoundError: + pass + if TYPE_CHECKING: from ..v2.lint import ValidationResultList, ValidationTask @@ -1408,18 +1418,18 @@ def from_yaml( ) # sciml extension - if config.extensions and config.extensions[C.SCIML]: - try: - from petab_sciml import ( - ArrayDataStandard, - NNModel, - NNModelStandard, - ) - except ImportError as e: - raise ImportError( - "To generate a PEtab SciML problem, (petab_sciml) must be" - "installed." - ) from e + # if config.extensions and config.extensions[C.SCIML]: + # try: + # from petab_sciml import ( + # ArrayDataStandard, + # NNModel, + # NNModelStandard, + # ) + # except ImportError as e: + # raise ImportError( + # "To generate a PEtab SciML problem, (petab_sciml) must" + # "be installed." + # ) from e # Neural network classes are constructed via pytorch for now to get the # proper inputs @@ -2435,8 +2445,7 @@ def add_hybridization(self, target_id: str, target_value: str): def add_neural_network_from_dict(self, model_id: str, nn_dict: dict): """Add a SciML neural net from a dictionary.""" - from petab_sciml import NNModel - + # from petab_sciml import NNModel nn_model = NNModel.model_validate(nn_dict) nn_model.nn_model_id = model_id self.neural_networks.append(nn_model) @@ -2448,8 +2457,7 @@ def add_neural_network_from_yaml( base_path: str | Path | None = None, ): """Add a SciML neural net from a yaml file.""" - from petab_sciml import NNModelStandard - + # from petab_sciml import NNModelStandard self.neural_networks.append( NNModelStandard.load_data( _generate_path( @@ -2462,8 +2470,7 @@ def add_neural_network_from_yaml( def add_array_data_from_dict(self, array_data: dict): """Add SciML array data from a dictionary.""" - from petab_sciml import ArrayData - + # from petab_sciml import ArrayData self.array_data_files.append(ArrayData.model_validate(array_data)) def add_array_data_from_hdf5( @@ -2472,8 +2479,7 @@ def add_array_data_from_hdf5( base_path: str | Path | None = None, ): """Add SciML array data from an hdf5 file.""" - from petab_sciml import ArrayDataStandard - + # from petab_sciml import ArrayDataStandard self.array_data_files.append( ArrayDataStandard.load_data(_generate_path(file_path, base_path)) ) From 15c4486d9e802a8705803341911dde24c6574742 Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Wed, 27 May 2026 12:09:24 +0200 Subject: [PATCH 11/28] add dependency --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 0295cfa6..d6dfccc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ maintainers = [ tests = [ "antimony>=3.1.0", "copasi-basico>=0.85", + "petab_sciml", "pysb", "pytest", "pytest-cov", @@ -71,6 +72,9 @@ vis = [ "seaborn", "scipy" ] +sciml = [ + "petab_sciml", +] [project.scripts] petablint = "petab.petablint:main" From 3d8901bae0df33d184ec2aeaf992e09e681bcb82 Mon Sep 17 00:00:00 2001 From: Dilan Pathirana <59329744+dilpath@users.noreply.github.com> Date: Wed, 27 May 2026 12:16:42 +0200 Subject: [PATCH 12/28] Update petab_sciml dependency to use Git URL --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d6dfccc9..ed228c04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ maintainers = [ tests = [ "antimony>=3.1.0", "copasi-basico>=0.85", - "petab_sciml", + "petab_sciml @ git+https://github.com/PEtab-dev/petab_sciml.git", "pysb", "pytest", "pytest-cov", @@ -73,7 +73,7 @@ vis = [ "scipy" ] sciml = [ - "petab_sciml", + "petab_sciml @ git+https://github.com/PEtab-dev/petab_sciml.git", ] [project.scripts] From 4ef0a10b435e8e2c2d77afc22810c8c46ad3c29b Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Thu, 11 Jun 2026 15:29:05 +0100 Subject: [PATCH 13/28] neural_nets to neural_networks --- petab/v2/core.py | 10 +++++----- tests/v2/test_sciml.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/petab/v2/core.py b/petab/v2/core.py index 64c362d1..dd860436 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -1445,7 +1445,7 @@ def from_yaml( nn_model_id=nn_id, ) for nn_id, nn_config in ( - config.extensions[C.SCIML].neural_nets or {} + config.extensions[C.SCIML].neural_networks or {} ).items() ] if config.extensions and config.extensions[C.SCIML] @@ -2719,7 +2719,7 @@ class SciMLConfig(BaseModel): # Absolute or relative to `base_path`. hybridization_files: list[AnyUrl | Path] = [] #: The neural network IDs and info. - neural_nets: dict[str, NeuralNetConfig] | None = {} + neural_networks: dict[str, NeuralNetConfig] | None = {} model_config = ConfigDict( validate_assignment=True, @@ -2840,9 +2840,9 @@ def to_yaml(self, filename: str | Path): # convert Paths to strings for key in ("array_files", "hybridization_files"): d_ext[key] = list(map(str, d_ext[key])) - for nn in d_ext["neural_nets"]: - d_ext["neural_nets"][nn][C.MODEL_LOCATION] = str( - d_ext["neural_nets"][nn][C.MODEL_LOCATION] + for nn in d_ext["neural_networks"]: + d_ext["neural_networks"][nn][C.MODEL_LOCATION] = str( + d_ext["neural_networks"][nn][C.MODEL_LOCATION] ) write_yaml(data, filename) diff --git a/tests/v2/test_sciml.py b/tests/v2/test_sciml.py index 9e39fc80..49dc0d13 100644 --- a/tests/v2/test_sciml.py +++ b/tests/v2/test_sciml.py @@ -25,7 +25,7 @@ def _get_test_problem(): "version": "0.1.0", "array_files": ["net1_ps.hdf5"], "hybridization_files": ["hybridizations.tsv"], - "neural_nets": { + "neural_networks": { "net1": NeuralNetConfig( location="net1.yaml", pre_initialization=False, From 58c549a2876448ccff47f0e402db7b438e42ac6e Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Mon, 15 Jun 2026 10:38:11 +0100 Subject: [PATCH 14/28] add required params check to sciml validations --- petab/v2/core.py | 14 -------------- petab/v2/lint.py | 7 ++++++- tests/v2/test_sciml.py | 2 +- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/petab/v2/core.py b/petab/v2/core.py index dd860436..c4b073e7 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -1417,20 +1417,6 @@ def from_yaml( else None ) - # sciml extension - # if config.extensions and config.extensions[C.SCIML]: - # try: - # from petab_sciml import ( - # ArrayDataStandard, - # NNModel, - # NNModelStandard, - # ) - # except ImportError as e: - # raise ImportError( - # "To generate a PEtab SciML problem, (petab_sciml) must" - # "be installed." - # ) from e - # Neural network classes are constructed via pytorch for now to get the # proper inputs neural_networks = ( diff --git a/petab/v2/lint.py b/petab/v2/lint.py index 3676604d..5a61ecff 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -1156,6 +1156,11 @@ def append_overrides(overrides): } parameter_ids -= condition_targets + hybridization_targets = {hyb.target_id for hyb in problem.hybridizations} + parameter_ids -= hybridization_targets + hybridization_target_values = {str(hyb.target_value) for hyb in problem.hybridizations} + parameter_ids -= hybridization_target_values + return parameter_ids @@ -1227,7 +1232,7 @@ def get_placeholders( CheckExperimentConditionsExist(), CheckUndefinedExperiments(), CheckObservablesDoNotShadowModelEntities(), - # CheckAllParametersPresentInParameterTable(), + CheckAllParametersPresentInParameterTable(), CheckValidParameterInConditionOrParameterTable(), CheckUnusedExperiments(), CheckUnusedConditions(), diff --git a/tests/v2/test_sciml.py b/tests/v2/test_sciml.py index 49dc0d13..1f0c35e4 100644 --- a/tests/v2/test_sciml.py +++ b/tests/v2/test_sciml.py @@ -63,7 +63,7 @@ def _get_test_problem(): ) problem.add_hybridization("net1_input1", "A") problem.add_hybridization("net1_input2", "B") - problem.add_hybridization("gamma_", "net1_output_1") + problem.add_hybridization("gamma_", "net1_output1") problem.add_neural_network_from_dict( "net1", nn_dict={ From f1d45400fa09de03ed16a449edf3309adbdfa0c9 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Mon, 15 Jun 2026 10:43:29 +0100 Subject: [PATCH 15/28] ruff format --- petab/v2/lint.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/petab/v2/lint.py b/petab/v2/lint.py index 5a61ecff..ea7ebc23 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -1158,7 +1158,9 @@ def append_overrides(overrides): hybridization_targets = {hyb.target_id for hyb in problem.hybridizations} parameter_ids -= hybridization_targets - hybridization_target_values = {str(hyb.target_value) for hyb in problem.hybridizations} + hybridization_target_values = { + str(hyb.target_value) for hyb in problem.hybridizations + } parameter_ids -= hybridization_target_values return parameter_ids From 2f76d3ee2b57be640c322e8928eb0b7f63e900a5 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Wed, 17 Jun 2026 11:43:18 +0100 Subject: [PATCH 16/28] implement feedback from code review --- petab/v2/core.py | 5 +++-- petab/v2/lint.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/petab/v2/core.py b/petab/v2/core.py index c4b073e7..5852c02b 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -938,6 +938,7 @@ class Parameter(BaseModel): ) #: Nominal value. nominal_value: Annotated[ + # PEtab SciML supports arrays via "array" nominal values float | Literal["array"] | None, BeforeValidator(_convert_nan_to_none) ] = Field(alias=C.NOMINAL_VALUE, default=None) #: Is the parameter to be estimated? @@ -2415,8 +2416,8 @@ def add_experiment(self, id_: str, *args): def add_hybridization(self, target_id: str, target_value: str): """Add a SciML hybridization table entry to the problem. - If there are more than one hybridization tables, the hybridization - is added to the last one. + If there is more than one hybridization table, the hybridization + is added to the last table. Arguments: target_id: The ID of the target entity in the PEtab problem diff --git a/petab/v2/lint.py b/petab/v2/lint.py index ea7ebc23..5f35c462 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -996,7 +996,7 @@ def run(self, problem: Problem) -> ValidationIssue | None: if culprits := (hyb_target_vals & nn_input_ids): messages.append( "The following neural net inputs were used as target values " - f"in the Hybridization tbale: `{culprits}`. Please simplify." + f"in the Hybridization table: `{culprits}`." ) if messages: From 81292a70f4d5ab8c7e41da93b0a05f6e418ff0b7 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Wed, 17 Jun 2026 14:36:23 +0100 Subject: [PATCH 17/28] move sciml code to separate files and resolve circular imports --- lint.py | 0 petab/v2/base.py | 225 ++++++++++++++++++++++++++++++++++++ petab/v2/core.py | 227 +++---------------------------------- petab/v2/lint.py | 206 ++------------------------------- petab/v2/sciml/__init__.py | 0 petab/v2/sciml/core.py | 127 +++++++++++++++++++++ petab/v2/sciml/lint.py | 36 ++++++ tests/v2/test_sciml.py | 3 +- 8 files changed, 417 insertions(+), 407 deletions(-) create mode 100644 lint.py create mode 100644 petab/v2/base.py create mode 100644 petab/v2/sciml/__init__.py create mode 100644 petab/v2/sciml/core.py create mode 100644 petab/v2/sciml/lint.py diff --git a/lint.py b/lint.py new file mode 100644 index 00000000..e69de29b diff --git a/petab/v2/base.py b/petab/v2/base.py new file mode 100644 index 00000000..3decff94 --- /dev/null +++ b/petab/v2/base.py @@ -0,0 +1,225 @@ +"""Base classes shared across petab.v2 to avoid circular imports.""" + +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import IntEnum +from pathlib import Path +from typing import TYPE_CHECKING, Generic, TypeVar, get_args + +import pandas as pd +from pydantic import AnyUrl, BaseModel, Field + +from .._utils import _generate_path + +if TYPE_CHECKING: + from .core import Problem + +logger = logging.getLogger(__name__) + + +class ValidationIssueSeverity(IntEnum): + """The severity of a validation issue.""" + + INFO = 10 + WARNING = 20 + ERROR = 30 + CRITICAL = 40 + + +@dataclass +class ValidationIssue: + """The result of a validation task.""" + + level: ValidationIssueSeverity + message: str + task: str | None = None + + def __post_init__(self): + if not isinstance(self.level, ValidationIssueSeverity): + raise TypeError( + "`level` must be an instance of ValidationIssueSeverity." + ) + + def __str__(self): + return f"{self.level.name}: {self.message}" + + @staticmethod + def _get_task_name() -> str | None: + """Get the name of the ValidationTask that raised this error.""" + import inspect + + for frame_info in inspect.stack(): + frame = frame_info.frame + if "self" in frame.f_locals: + task = frame.f_locals["self"] + if isinstance(task, ValidationTask): + return task.__class__.__name__ + return None + + +@dataclass +class ValidationError(ValidationIssue): + """A validation result with level ERROR.""" + + level: ValidationIssueSeverity = field( + default=ValidationIssueSeverity.ERROR, init=False + ) + + def __post_init__(self): + if self.task is None: + self.task = self._get_task_name() + + +@dataclass +class ValidationWarning(ValidationIssue): + """A validation result with level WARNING.""" + + level: ValidationIssueSeverity = field( + default=ValidationIssueSeverity.WARNING, init=False + ) + + def __post_init__(self): + if self.task is None: + self.task = self._get_task_name() + + +class ValidationResultList(list): + """A list of validation results.""" + + def log( + self, + *, + logger: logging.Logger = logger, + min_level: ValidationIssueSeverity = ValidationIssueSeverity.INFO, + max_level: ValidationIssueSeverity = ValidationIssueSeverity.CRITICAL, + ): + """Log the validation results.""" + for result in self: + if result.level < min_level or result.level > max_level: + continue + msg = f"{result.level.name}: {result.message} [{result.task}]" + if result.level == ValidationIssueSeverity.INFO: + logger.info(msg) + elif result.level == ValidationIssueSeverity.WARNING: + logger.warning(msg) + elif result.level >= ValidationIssueSeverity.ERROR: + logger.error(msg) + + if not self: + logger.info("PEtab format check completed successfully.") + + def has_errors(self) -> bool: + """Check if there are any errors in the validation results.""" + return any( + result.level >= ValidationIssueSeverity.ERROR for result in self + ) + + +class ValidationTask(ABC): + """A task to validate a PEtab problem.""" + + @abstractmethod + def run(self, problem: Problem) -> ValidationIssue | None: + """Run the validation task.""" + ... + + def __call__(self, *args, **kwargs): + return self.run(*args, **kwargs) + + +T = TypeVar("T", bound=BaseModel) + + +class BaseTable(BaseModel, Generic[T]): + """Base class for PEtab tables.""" + + #: The table elements + elements: list[T] + #: The path to the table file, if applicable. + #: Relative to the base path, if the base path is set and rel_path is not + #: an absolute path. + rel_path: AnyUrl | Path | None = Field(exclude=True, default=None) + #: The base path for the table file, if applicable. + #: This is usually the directory of the PEtab YAML file. + base_path: AnyUrl | Path | None = Field(exclude=True, default=None) + + def __init__(self, elements: list[T] = None, **kwargs) -> None: + """Initialize the BaseTable with a list of elements.""" + if elements is None: + elements = [] + super().__init__(elements=elements, **kwargs) + + def __getitem__(self, id_: str) -> T: + """Get an element by ID. + + :param id_: The ID of the element to retrieve. + :return: The element with the given ID. + :raises KeyError: If no element with the given ID exists. + :raises NotImplementedError: + If the element type does not have an ID attribute. + """ + if "id" not in self._element_class().model_fields: + raise NotImplementedError( + f"__getitem__ is not implemented for {self.__class__.__name__}" + ) + + for element in self.elements: + if element.id == id_: + return element + + raise KeyError(f"{T.__name__} ID {id_} not found") + + @classmethod + @abstractmethod + def from_df(cls, df: pd.DataFrame, **kwargs) -> BaseTable[T]: + """Create a table from a DataFrame.""" + pass + + @abstractmethod + def to_df(self) -> pd.DataFrame: + """Convert the table to a DataFrame.""" + pass + + @classmethod + def from_tsv( + cls, file_path: str | Path, base_path: str | Path | None = None + ) -> BaseTable[T]: + """Create table from a TSV file.""" + df = pd.read_csv(_generate_path(file_path, base_path), sep="\t") + return cls.from_df(df, rel_path=file_path, base_path=base_path) + + def to_tsv(self, file_path: str | Path = None) -> None: + """Write the table to a TSV file.""" + df = self.to_df() + df.to_csv( + file_path or _generate_path(self.rel_path, self.base_path), + sep="\t", + index=not isinstance(df.index, pd.RangeIndex), + ) + + @classmethod + def _element_class(cls) -> type[T]: + """Get the class of the elements in the table.""" + return get_args(cls.model_fields["elements"].annotation)[0] + + def __add__(self, other: T) -> BaseTable[T]: + """Add an item to the table.""" + if not isinstance(other, self._element_class()): + raise TypeError( + f"Can only add {self._element_class().__name__} " + f"to {self.__class__.__name__}" + ) + return self.__class__(elements=self.elements + [other]) + + def __iadd__(self, other: T) -> BaseTable[T]: + """Add an item to the table in place.""" + if not isinstance(other, self._element_class()): + raise TypeError( + f"Can only add {self._element_class().__name__} " + f"to {self.__class__.__name__}" + ) + self.elements.append(other) + return self diff --git a/petab/v2/core.py b/petab/v2/core.py index 5852c02b..6f348f22 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -7,7 +7,6 @@ import os import tempfile import traceback -from abc import abstractmethod from collections.abc import Sequence from enum import Enum from itertools import chain @@ -18,11 +17,8 @@ TYPE_CHECKING, Annotated, Any, - Generic, Literal, Self, - TypeVar, - get_args, ) import numpy as np @@ -41,7 +37,18 @@ model_validator, ) +try: + from petab_sciml import ( + ArrayData, + ArrayDataStandard, + NNModel, + NNModelStandard, + ) +except ModuleNotFoundError: + pass + from .._utils import _generate_path +from .base import BaseTable from ..v1 import ( validate_yaml_syntax, yaml, @@ -53,16 +60,11 @@ from ..v1.yaml import get_path_prefix from ..versions import parse_version from . import C, get_observable_df - -try: - from petab_sciml import ( - ArrayData, - ArrayDataStandard, - NNModel, - NNModelStandard, - ) -except ModuleNotFoundError: - pass +from .sciml.core import ( + Hybridization, + HybridizationTable, + SciMLConfig, +) if TYPE_CHECKING: from ..v2.lint import ValidationResultList, ValidationTask @@ -225,101 +227,6 @@ class PriorDistribution(str, Enum): ) -T = TypeVar("T", bound=BaseModel) - - -class BaseTable(BaseModel, Generic[T]): - """Base class for PEtab tables.""" - - #: The table elements - elements: list[T] - #: The path to the table file, if applicable. - #: Relative to the base path, if the base path is set and rel_path is not - #: an absolute path. - rel_path: AnyUrl | Path | None = Field(exclude=True, default=None) - #: The base path for the table file, if applicable. - #: This is usually the directory of the PEtab YAML file. - base_path: AnyUrl | Path | None = Field(exclude=True, default=None) - - def __init__(self, elements: list[T] = None, **kwargs) -> None: - """Initialize the BaseTable with a list of elements.""" - if elements is None: - elements = [] - super().__init__(elements=elements, **kwargs) - - def __getitem__(self, id_: str) -> T: - """Get an element by ID. - - :param id_: The ID of the element to retrieve. - :return: The element with the given ID. - :raises KeyError: If no element with the given ID exists. - :raises NotImplementedError: - If the element type does not have an ID attribute. - """ - if "id" not in self._element_class().model_fields: - raise NotImplementedError( - f"__getitem__ is not implemented for {self.__class__.__name__}" - ) - - for element in self.elements: - if element.id == id_: - return element - - raise KeyError(f"{T.__name__} ID {id_} not found") - - @classmethod - @abstractmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> BaseTable[T]: - """Create a table from a DataFrame.""" - pass - - @abstractmethod - def to_df(self) -> pd.DataFrame: - """Convert the table to a DataFrame.""" - pass - - @classmethod - def from_tsv( - cls, file_path: str | Path, base_path: str | Path | None = None - ) -> BaseTable[T]: - """Create table from a TSV file.""" - df = pd.read_csv(_generate_path(file_path, base_path), sep="\t") - return cls.from_df(df, rel_path=file_path, base_path=base_path) - - def to_tsv(self, file_path: str | Path = None) -> None: - """Write the table to a TSV file.""" - df = self.to_df() - df.to_csv( - file_path or _generate_path(self.rel_path, self.base_path), - sep="\t", - index=not isinstance(df.index, pd.RangeIndex), - ) - - @classmethod - def _element_class(cls) -> type[T]: - """Get the class of the elements in the table.""" - return get_args(cls.model_fields["elements"].annotation)[0] - - def __add__(self, other: T) -> BaseTable[T]: - """Add an item to the table.""" - if not isinstance(other, self._element_class()): - raise TypeError( - f"Can only add {self._element_class().__name__} " - f"to {self.__class__.__name__}" - ) - return self.__class__(elements=self.elements + [other]) - - def __iadd__(self, other: T) -> BaseTable[T]: - """Add an item to the table in place.""" - if not isinstance(other, self._element_class()): - raise TypeError( - f"Can only add {self._element_class().__name__} " - f"to {self.__class__.__name__}" - ) - self.elements.append(other) - return self - - class Observable(BaseModel): """Observable definition.""" @@ -1120,77 +1027,6 @@ def n_estimated(self) -> int: return sum(p.estimate for p in self.parameters) -class Hybridization(BaseModel): - """Assigns PEtab SciML NN inputs and outputs.""" - - #: The target ID. - target_id: str = Field(alias=C.TARGET_ID) - #: The target value. - target_value: sp.Basic = Field(alias=C.TARGET_VALUE) - - #: :meta private: - model_config = ConfigDict( - arbitrary_types_allowed=True, - populate_by_name=True, - extra="allow", - validate_assignment=True, - ) - - @field_validator("target_value", mode="before") - @classmethod - def _sympify(cls, v): - if v is None or isinstance(v, sp.Basic): - return v - if isinstance(v, float) and np.isnan(v): - return None - - return sympify_petab(v) - - -class HybridizationTable(BaseTable[Hybridization]): - """PEtab SciML hybridization table.""" - - @property - def hybridizations(self) -> list[Hybridization]: - """List of hybridizations.""" - return self.elements - - @classmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> HybridizationTable: - """Create a HybridizationTable from a DataFrame.""" - if df is None: - return cls(**kwargs) - - hybridizations = [ - Hybridization( - **row.to_dict(), - ) - for _, row in df.iterrows() - ] - - return cls(hybridizations, **kwargs) - - def to_df(self) -> pd.DataFrame: - """Convert the HybridizationTable to a DataFrame.""" - records = self.model_dump(by_alias=True)["elements"] - - return pd.DataFrame(records) - - def __getitem__(self, target_id: str) -> Hybridization: - """Get a hybridization by target ID.""" - for hybridization in self.hybridizations: - if hybridization.target_id == target_id: - return hybridization - raise KeyError(f"Target ID {target_id} not found") - - def get(self, target_id, default=None): - """Get a hybridization by target ID or return a default value.""" - try: - return self[target_id] - except KeyError: - return default - - class Problem: """ PEtab parameter estimation problem @@ -2675,18 +2511,6 @@ class ModelFile(BaseModel): ) -class NeuralNetConfig(BaseModel): - """A neural net in the PEtab SciML problem configuration.""" - - location: AnyUrl | Path - pre_initialization: bool - format: str - - model_config = ConfigDict( - validate_assignment=True, - ) - - class ExtensionConfig(BaseModel): """The configuration of a PEtab extension.""" @@ -2694,25 +2518,6 @@ class ExtensionConfig(BaseModel): config: dict -class SciMLConfig(BaseModel): - """The extended configuration of a PEtab SciML problem.""" - - #: The PEtab SciML format version. - version: str = "0.1.0" - #: The paths to the array data files. - # Absolute or relative to `base_path`. - array_files: list[AnyUrl | Path] = [] - #: The paths to the hybridization tables. - # Absolute or relative to `base_path`. - hybridization_files: list[AnyUrl | Path] = [] - #: The neural network IDs and info. - neural_networks: dict[str, NeuralNetConfig] | None = {} - - model_config = ConfigDict( - validate_assignment=True, - ) - - class ProblemConfig(BaseModel): """The PEtab problem configuration.""" diff --git a/petab/v2/lint.py b/petab/v2/lint.py index 5f35c462..f8434fee 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -3,17 +3,24 @@ from __future__ import annotations import logging -from abc import ABC, abstractmethod from collections import Counter, OrderedDict from collections.abc import Set -from dataclasses import dataclass, field -from enum import IntEnum from itertools import chain from pathlib import Path import pandas as pd import sympy as sp +from .base import ( + ValidationError, + ValidationIssue, + ValidationIssueSeverity, + ValidationResultList, + ValidationTask, + ValidationWarning, +) +from petab.v2.sciml.lint import CheckHybridizationTable + from ..v2.C import * from .core import PriorDistribution, Problem @@ -51,126 +58,6 @@ ] -class ValidationIssueSeverity(IntEnum): - """The severity of a validation issue.""" - - # INFO: Informational message, no action required - INFO = 10 - # WARNING: Warning message, potential issues - WARNING = 20 - # ERROR: Error message, action required - ERROR = 30 - # CRITICAL: Critical error message, stops further validation - CRITICAL = 40 - - -@dataclass -class ValidationIssue: - """The result of a validation task. - - Attributes: - level: The level of the validation event. - message: The message of the validation event. - """ - - level: ValidationIssueSeverity - message: str - task: str | None = None - - def __post_init__(self): - if not isinstance(self.level, ValidationIssueSeverity): - raise TypeError( - "`level` must be an instance of ValidationIssueSeverity." - ) - - def __str__(self): - return f"{self.level.name}: {self.message}" - - @staticmethod - def _get_task_name() -> str | None: - """Get the name of the ValidationTask that raised this error. - - Expected to be called from below a `ValidationTask.run`. - """ - import inspect - - # walk up the stack until we find the ValidationTask.run method - for frame_info in inspect.stack(): - frame = frame_info.frame - if "self" in frame.f_locals: - task = frame.f_locals["self"] - if isinstance(task, ValidationTask): - return task.__class__.__name__ - return None - - -@dataclass -class ValidationError(ValidationIssue): - """A validation result with level ERROR.""" - - level: ValidationIssueSeverity = field( - default=ValidationIssueSeverity.ERROR, init=False - ) - - def __post_init__(self): - if self.task is None: - self.task = self._get_task_name() - - -@dataclass -class ValidationWarning(ValidationIssue): - """A validation result with level WARNING.""" - - level: ValidationIssueSeverity = field( - default=ValidationIssueSeverity.WARNING, init=False - ) - - def __post_init__(self): - if self.task is None: - self.task = self._get_task_name() - - -class ValidationResultList(list[ValidationIssue]): - """A list of validation results. - - Contains all issues found during the validation of a PEtab problem. - """ - - def log( - self, - *, - logger: logging.Logger = logger, - min_level: ValidationIssueSeverity = ValidationIssueSeverity.INFO, - max_level: ValidationIssueSeverity = ValidationIssueSeverity.CRITICAL, - ): - """Log the validation results. - - :param logger: The logger to use for logging. - Defaults to the module logger. - :param min_level: The minimum severity level to log. - :param max_level: The maximum severity level to log. - """ - for result in self: - if result.level < min_level or result.level > max_level: - continue - msg = f"{result.level.name}: {result.message} [{result.task}]" - if result.level == ValidationIssueSeverity.INFO: - logger.info(msg) - elif result.level == ValidationIssueSeverity.WARNING: - logger.warning(msg) - elif result.level >= ValidationIssueSeverity.ERROR: - logger.error(msg) - - if not self: - logger.info("PEtab format check completed successfully.") - - def has_errors(self) -> bool: - """Check if there are any errors in the validation results.""" - return any( - result.level >= ValidationIssueSeverity.ERROR for result in self - ) - - def lint_problem(problem: Problem | str | Path) -> ValidationResultList: """Validate a PEtab problem. @@ -187,24 +74,6 @@ def lint_problem(problem: Problem | str | Path) -> ValidationResultList: return problem.validate() -class ValidationTask(ABC): - """A task to validate a PEtab problem.""" - - @abstractmethod - def run(self, problem: Problem) -> ValidationIssue | None: - """Run the validation task. - - Arguments: - problem: PEtab problem to check. - Returns: - Validation results or ``None`` - """ - ... - - def __call__(self, *args, **kwargs): - return self.run(*args, **kwargs) - - class CheckProblemConfig(ValidationTask): """A task to validate the configuration of a PEtab problem. @@ -971,40 +840,6 @@ def run(self, problem: Problem) -> ValidationIssue | None: return None -class CheckHybridizationTable(ValidationTask): - """Validate the SciML hybridization table.""" - - def run(self, problem: Problem) -> ValidationIssue | None: - messages = [] - - condition_targets = { - c.target_id for ct in problem.conditions for c in ct.changes - } - nn_input_ids = { - inp.input_id for nn in problem.neural_networks for inp in nn.inputs - } - hyb_target_ids = {hyb.target_id for hyb in problem.hybridizations} - hyb_target_vals = {hyb.target_value for hyb in problem.hybridizations} - - # Hybridization targets are not also targets in the condition table - if culprits := (hyb_target_ids & condition_targets): - messages.append( - f"Hybridization target ids `{culprits}` are also " - "target ids in the condition table." - ) - # NN inputs are not used as target values - if culprits := (hyb_target_vals & nn_input_ids): - messages.append( - "The following neural net inputs were used as target values " - f"in the Hybridization table: `{culprits}`." - ) - - if messages: - return ValidationError("\n".join(messages)) - - return None - - def get_valid_parameters_for_parameter_table( problem: Problem, ) -> set[str]: @@ -1221,25 +1056,6 @@ def get_placeholders( ] #: Validation tasks that should be run PEtab SciML problems -sciml_validation_tasks = [ - CheckProblemConfig(), - CheckModel(), - CheckUniquePrimaryKeys(), - CheckMeasurementModelId(), - CheckMeasuredObservablesDefined(), - CheckPosLogMeasurements(), - CheckOverridesMatchPlaceholders(), - CheckValidConditionTargets(), - CheckExperimentTable(), - CheckExperimentConditionsExist(), - CheckUndefinedExperiments(), - CheckObservablesDoNotShadowModelEntities(), - CheckAllParametersPresentInParameterTable(), - CheckValidParameterInConditionOrParameterTable(), - CheckUnusedExperiments(), - CheckUnusedConditions(), - CheckPriorDistribution(), - CheckInitialChangeSymbols(), - CheckMappingTable(), +sciml_validation_tasks = default_validation_tasks + [ CheckHybridizationTable(), ] diff --git a/petab/v2/sciml/__init__.py b/petab/v2/sciml/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/petab/v2/sciml/core.py b/petab/v2/sciml/core.py new file mode 100644 index 00000000..e4c29a46 --- /dev/null +++ b/petab/v2/sciml/core.py @@ -0,0 +1,127 @@ +from pathlib import Path +from typing import Self + +import numpy as np +import pandas as pd +import sympy as sp +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + Field, + field_validator, +) + +import petab.v2.C as C + +from ...v1.math.sympify import sympify_petab +from ..base import BaseTable + +__all__ = [ + "Hybridization", + "HybridizationTable", + "NeuralNetConfig", + "SciMLConfig", +] + + +class Hybridization(BaseModel): + """Assigns PEtab SciML NN inputs and outputs.""" + + #: The target ID. + target_id: str = Field(alias=C.TARGET_ID) + #: The target value. + target_value: sp.Basic = Field(alias=C.TARGET_VALUE) + + #: :meta private: + model_config = ConfigDict( + arbitrary_types_allowed=True, + populate_by_name=True, + extra="allow", + validate_assignment=True, + ) + + @field_validator("target_value", mode="before") + @classmethod + def _sympify(cls, v): + if v is None or isinstance(v, sp.Basic): + return v + if isinstance(v, float) and np.isnan(v): + return None + + return sympify_petab(v) + + +class HybridizationTable(BaseTable[Hybridization]): + """PEtab SciML hybridization table.""" + + @property + def hybridizations(self) -> list[Hybridization]: + """List of hybridizations.""" + return self.elements + + @classmethod + def from_df(cls, df: pd.DataFrame, **kwargs) -> Self: + """Create a HybridizationTable from a DataFrame.""" + if df is None: + return cls(**kwargs) + + hybridizations = [ + Hybridization( + **row.to_dict(), + ) + for _, row in df.iterrows() + ] + + return cls(hybridizations, **kwargs) + + def to_df(self) -> pd.DataFrame: + """Convert the HybridizationTable to a DataFrame.""" + records = self.model_dump(by_alias=True)["elements"] + + return pd.DataFrame(records) + + def __getitem__(self, target_id: str) -> Hybridization: + """Get a hybridization by target ID.""" + for hybridization in self.hybridizations: + if hybridization.target_id == target_id: + return hybridization + raise KeyError(f"Target ID {target_id} not found") + + def get(self, target_id, default=None): + """Get a hybridization by target ID or return a default value.""" + try: + return self[target_id] + except KeyError: + return default + + +class NeuralNetConfig(BaseModel): + """A neural net in the PEtab SciML problem configuration.""" + + location: AnyUrl | Path + pre_initialization: bool + format: str + + model_config = ConfigDict( + validate_assignment=True, + ) + + +class SciMLConfig(BaseModel): + """The extended configuration of a PEtab SciML problem.""" + + #: The PEtab SciML format version. + version: str = "0.1.0" + #: The paths to the array data files. + # Absolute or relative to `base_path`. + array_files: list[AnyUrl | Path] = [] + #: The paths to the hybridization tables. + # Absolute or relative to `base_path`. + hybridization_files: list[AnyUrl | Path] = [] + #: The neural network IDs and info. + neural_networks: dict[str, NeuralNetConfig] | None = {} + + model_config = ConfigDict( + validate_assignment=True, + ) diff --git a/petab/v2/sciml/lint.py b/petab/v2/sciml/lint.py new file mode 100644 index 00000000..dc087a36 --- /dev/null +++ b/petab/v2/sciml/lint.py @@ -0,0 +1,36 @@ +from petab.v1.problem import Problem +from petab.v2.base import ValidationError, ValidationIssue, ValidationTask + + +class CheckHybridizationTable(ValidationTask): + """Validate the SciML hybridization table.""" + + def run(self, problem: Problem) -> ValidationIssue | None: + messages = [] + + condition_targets = { + c.target_id for ct in problem.conditions for c in ct.changes + } + nn_input_ids = { + inp.input_id for nn in problem.neural_networks for inp in nn.inputs + } + hyb_target_ids = {hyb.target_id for hyb in problem.hybridizations} + hyb_target_vals = {hyb.target_value for hyb in problem.hybridizations} + + # Hybridization targets are not also targets in the condition table + if culprits := (hyb_target_ids & condition_targets): + messages.append( + f"Hybridization target ids `{culprits}` are also " + "target ids in the condition table." + ) + # NN inputs are not used as target values + if culprits := (hyb_target_vals & nn_input_ids): + messages.append( + "The following neural net inputs were used as target values " + f"in the Hybridization table: `{culprits}`." + ) + + if messages: + return ValidationError("\n".join(messages)) + + return None diff --git a/tests/v2/test_sciml.py b/tests/v2/test_sciml.py index 1f0c35e4..23737a0e 100644 --- a/tests/v2/test_sciml.py +++ b/tests/v2/test_sciml.py @@ -2,9 +2,10 @@ from pydantic import ConfigDict from petab.v2.core import * -from petab.v2.core import ModelFile, NeuralNetConfig +from petab.v2.core import ModelFile from petab.v2.lint import sciml_validation_tasks from petab.v2.models.sbml_model import SbmlModel +from petab.v2.sciml.core import NeuralNetConfig def _get_test_problem(): From 840ae7ec7379d1167a1c490df5b8ad3ba6d29907 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Wed, 17 Jun 2026 14:48:39 +0100 Subject: [PATCH 18/28] fixup ruff --- petab/v2/core.py | 2 +- petab/v2/lint.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/petab/v2/core.py b/petab/v2/core.py index 6f348f22..c0477f29 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -48,7 +48,6 @@ pass from .._utils import _generate_path -from .base import BaseTable from ..v1 import ( validate_yaml_syntax, yaml, @@ -60,6 +59,7 @@ from ..v1.yaml import get_path_prefix from ..versions import parse_version from . import C, get_observable_df +from .base import BaseTable from .sciml.core import ( Hybridization, HybridizationTable, diff --git a/petab/v2/lint.py b/petab/v2/lint.py index f8434fee..7d33f33a 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -11,6 +11,9 @@ import pandas as pd import sympy as sp +from petab.v2.sciml.lint import CheckHybridizationTable + +from ..v2.C import * from .base import ( ValidationError, ValidationIssue, @@ -19,9 +22,6 @@ ValidationTask, ValidationWarning, ) -from petab.v2.sciml.lint import CheckHybridizationTable - -from ..v2.C import * from .core import PriorDistribution, Problem logger = logging.getLogger(__name__) From 9af9a54be17c8cf97622999dda1fe9c69ff7e7f5 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Wed, 17 Jun 2026 15:23:31 +0100 Subject: [PATCH 19/28] fix docs build issue --- doc/conf.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/conf.py b/doc/conf.py index 975cad03..41c41496 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -9,6 +9,8 @@ import sys import warnings +from pydantic import BaseModel + # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, @@ -126,6 +128,9 @@ def skip_some_objects(app, what, name, obj, skip, options): """Exclude some objects from the documentation""" if getattr(obj, "__module__", None) == "collections": return True + # Napoleon + Pydantic v2 bug: BaseModel itself triggers __getattr__ error + if obj is BaseModel: + return True def setup(app): From e342f26daf260364d03533f4642c1d12d2ff98ae Mon Sep 17 00:00:00 2001 From: BSnelling Date: Wed, 17 Jun 2026 16:26:12 +0100 Subject: [PATCH 20/28] Update petab/v2/core.py Co-authored-by: Dilan Pathirana <59329744+dilpath@users.noreply.github.com> --- petab/v2/core.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/petab/v2/core.py b/petab/v2/core.py index c0477f29..37fa350c 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -1254,10 +1254,13 @@ def from_yaml( else None ) - # Neural network classes are constructed via pytorch for now to get the - # proper inputs - neural_networks = ( - [ + neural_networks = None + hybridization_tables = None + array_data_files = None + if config.extensions and config.extensions[C.SCIML]: + # Neural network classes are constructed via pytorch for now to get the + # proper inputs + neural_networks = [ NNModel.from_pytorch_module( NNModelStandard.load_data( _generate_path( @@ -1271,27 +1274,16 @@ def from_yaml( config.extensions[C.SCIML].neural_networks or {} ).items() ] - if config.extensions and config.extensions[C.SCIML] - else None - ) - - hybridization_tables = ( - [ + + hybridization_tables = [ HybridizationTable.from_tsv(f, base_path) for f in config.extensions[C.SCIML].hybridization_files ] - if config.extensions and config.extensions[C.SCIML] - else None - ) - - array_data_files = ( - [ + + array_data_files = [ ArrayDataStandard.load_data(_generate_path(f, base_path)) for f in config.extensions[C.SCIML].array_files ] - if config.extensions and config.extensions[C.SCIML] - else None - ) return Problem( config=config, From 254a5964b24222befa60e96474cd09a48a1a6ef6 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Wed, 17 Jun 2026 16:36:58 +0100 Subject: [PATCH 21/28] add to docstrings --- petab/v2/core.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/petab/v2/core.py b/petab/v2/core.py index 37fa350c..0d3211e4 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -1258,8 +1258,8 @@ def from_yaml( hybridization_tables = None array_data_files = None if config.extensions and config.extensions[C.SCIML]: - # Neural network classes are constructed via pytorch for now to get the - # proper inputs + # Neural network classes are constructed via pytorch for now to get + # the proper inputs neural_networks = [ NNModel.from_pytorch_module( NNModelStandard.load_data( @@ -1274,12 +1274,12 @@ def from_yaml( config.extensions[C.SCIML].neural_networks or {} ).items() ] - + hybridization_tables = [ HybridizationTable.from_tsv(f, base_path) for f in config.extensions[C.SCIML].hybridization_files ] - + array_data_files = [ ArrayDataStandard.load_data(_generate_path(f, base_path)) for f in config.extensions[C.SCIML].array_files @@ -1605,7 +1605,10 @@ def id(self, value: str): @property def hybridizations(self) -> list[Hybridization]: - """List of hybridizations in the hybridization table(s).""" + """ + List of hybridizations in the hybridization table(s). + Note that hybridizations are specific to PEtab SciML problems. + """ return list( chain.from_iterable( ht.hybridizations for ht in self.hybridization_tables @@ -1614,7 +1617,10 @@ def hybridizations(self) -> list[Hybridization]: @property def hybridization_df(self) -> pd.DataFrame | None: - """Combined SciML hybridization tables as DataFrame.""" + """ + Combined SciML hybridization tables as DataFrame. + Note that hybridizations are specific to PEtab SciML problems. + """ return ( HybridizationTable(hybridizations).to_df() if (hybridizations := self.hybridizations) @@ -2245,7 +2251,8 @@ def add_hybridization(self, target_id: str, target_value: str): """Add a SciML hybridization table entry to the problem. If there is more than one hybridization table, the hybridization - is added to the last table. + is added to the last table. Note that hybridizations are specific + to PEtab SciML problems. Arguments: target_id: The ID of the target entity in the PEtab problem @@ -2259,7 +2266,9 @@ def add_hybridization(self, target_id: str, target_value: str): ) def add_neural_network_from_dict(self, model_id: str, nn_dict: dict): - """Add a SciML neural net from a dictionary.""" + """ + Add a SciML neural net from a dictionary (or PEtab SciML problems). + """ # from petab_sciml import NNModel nn_model = NNModel.model_validate(nn_dict) nn_model.nn_model_id = model_id @@ -2271,7 +2280,9 @@ def add_neural_network_from_yaml( file_path: str | Path, base_path: str | Path | None = None, ): - """Add a SciML neural net from a yaml file.""" + """ + Add a SciML neural net from a yaml file (for PEtab SciML problems). + """ # from petab_sciml import NNModelStandard self.neural_networks.append( NNModelStandard.load_data( @@ -2284,7 +2295,9 @@ def add_neural_network_from_yaml( ) def add_array_data_from_dict(self, array_data: dict): - """Add SciML array data from a dictionary.""" + """ + Add SciML array data from a dictionary (for PEtab SciML problems). + """ # from petab_sciml import ArrayData self.array_data_files.append(ArrayData.model_validate(array_data)) @@ -2293,7 +2306,9 @@ def add_array_data_from_hdf5( file_path: str | Path, base_path: str | Path | None = None, ): - """Add SciML array data from an hdf5 file.""" + """ + Add SciML array data from an hdf5 file (for PEtab SciML problems). + """ # from petab_sciml import ArrayDataStandard self.array_data_files.append( ArrayDataStandard.load_data(_generate_path(file_path, base_path)) From d95f09a3592876646482a34aa023d16e5ea89a3b Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Thu, 18 Jun 2026 14:06:48 +0100 Subject: [PATCH 22/28] Revert "move sciml code to separate files and resolve circular imports" This reverts commit 153289ffd789f9e0a7baebaac86214a40a9d7b90. --- lint.py | 0 petab/v2/base.py | 225 ------------------------------------ petab/v2/core.py | 227 ++++++++++++++++++++++++++++++++++--- petab/v2/lint.py | 206 +++++++++++++++++++++++++++++++-- petab/v2/sciml/__init__.py | 0 petab/v2/sciml/core.py | 127 --------------------- petab/v2/sciml/lint.py | 36 ------ tests/v2/test_sciml.py | 3 +- 8 files changed, 407 insertions(+), 417 deletions(-) delete mode 100644 lint.py delete mode 100644 petab/v2/base.py delete mode 100644 petab/v2/sciml/__init__.py delete mode 100644 petab/v2/sciml/core.py delete mode 100644 petab/v2/sciml/lint.py diff --git a/lint.py b/lint.py deleted file mode 100644 index e69de29b..00000000 diff --git a/petab/v2/base.py b/petab/v2/base.py deleted file mode 100644 index 3decff94..00000000 --- a/petab/v2/base.py +++ /dev/null @@ -1,225 +0,0 @@ -"""Base classes shared across petab.v2 to avoid circular imports.""" - -from __future__ import annotations - -import logging -from abc import ABC, abstractmethod -from dataclasses import dataclass, field -from enum import IntEnum -from pathlib import Path -from typing import TYPE_CHECKING, Generic, TypeVar, get_args - -import pandas as pd -from pydantic import AnyUrl, BaseModel, Field - -from .._utils import _generate_path - -if TYPE_CHECKING: - from .core import Problem - -logger = logging.getLogger(__name__) - - -class ValidationIssueSeverity(IntEnum): - """The severity of a validation issue.""" - - INFO = 10 - WARNING = 20 - ERROR = 30 - CRITICAL = 40 - - -@dataclass -class ValidationIssue: - """The result of a validation task.""" - - level: ValidationIssueSeverity - message: str - task: str | None = None - - def __post_init__(self): - if not isinstance(self.level, ValidationIssueSeverity): - raise TypeError( - "`level` must be an instance of ValidationIssueSeverity." - ) - - def __str__(self): - return f"{self.level.name}: {self.message}" - - @staticmethod - def _get_task_name() -> str | None: - """Get the name of the ValidationTask that raised this error.""" - import inspect - - for frame_info in inspect.stack(): - frame = frame_info.frame - if "self" in frame.f_locals: - task = frame.f_locals["self"] - if isinstance(task, ValidationTask): - return task.__class__.__name__ - return None - - -@dataclass -class ValidationError(ValidationIssue): - """A validation result with level ERROR.""" - - level: ValidationIssueSeverity = field( - default=ValidationIssueSeverity.ERROR, init=False - ) - - def __post_init__(self): - if self.task is None: - self.task = self._get_task_name() - - -@dataclass -class ValidationWarning(ValidationIssue): - """A validation result with level WARNING.""" - - level: ValidationIssueSeverity = field( - default=ValidationIssueSeverity.WARNING, init=False - ) - - def __post_init__(self): - if self.task is None: - self.task = self._get_task_name() - - -class ValidationResultList(list): - """A list of validation results.""" - - def log( - self, - *, - logger: logging.Logger = logger, - min_level: ValidationIssueSeverity = ValidationIssueSeverity.INFO, - max_level: ValidationIssueSeverity = ValidationIssueSeverity.CRITICAL, - ): - """Log the validation results.""" - for result in self: - if result.level < min_level or result.level > max_level: - continue - msg = f"{result.level.name}: {result.message} [{result.task}]" - if result.level == ValidationIssueSeverity.INFO: - logger.info(msg) - elif result.level == ValidationIssueSeverity.WARNING: - logger.warning(msg) - elif result.level >= ValidationIssueSeverity.ERROR: - logger.error(msg) - - if not self: - logger.info("PEtab format check completed successfully.") - - def has_errors(self) -> bool: - """Check if there are any errors in the validation results.""" - return any( - result.level >= ValidationIssueSeverity.ERROR for result in self - ) - - -class ValidationTask(ABC): - """A task to validate a PEtab problem.""" - - @abstractmethod - def run(self, problem: Problem) -> ValidationIssue | None: - """Run the validation task.""" - ... - - def __call__(self, *args, **kwargs): - return self.run(*args, **kwargs) - - -T = TypeVar("T", bound=BaseModel) - - -class BaseTable(BaseModel, Generic[T]): - """Base class for PEtab tables.""" - - #: The table elements - elements: list[T] - #: The path to the table file, if applicable. - #: Relative to the base path, if the base path is set and rel_path is not - #: an absolute path. - rel_path: AnyUrl | Path | None = Field(exclude=True, default=None) - #: The base path for the table file, if applicable. - #: This is usually the directory of the PEtab YAML file. - base_path: AnyUrl | Path | None = Field(exclude=True, default=None) - - def __init__(self, elements: list[T] = None, **kwargs) -> None: - """Initialize the BaseTable with a list of elements.""" - if elements is None: - elements = [] - super().__init__(elements=elements, **kwargs) - - def __getitem__(self, id_: str) -> T: - """Get an element by ID. - - :param id_: The ID of the element to retrieve. - :return: The element with the given ID. - :raises KeyError: If no element with the given ID exists. - :raises NotImplementedError: - If the element type does not have an ID attribute. - """ - if "id" not in self._element_class().model_fields: - raise NotImplementedError( - f"__getitem__ is not implemented for {self.__class__.__name__}" - ) - - for element in self.elements: - if element.id == id_: - return element - - raise KeyError(f"{T.__name__} ID {id_} not found") - - @classmethod - @abstractmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> BaseTable[T]: - """Create a table from a DataFrame.""" - pass - - @abstractmethod - def to_df(self) -> pd.DataFrame: - """Convert the table to a DataFrame.""" - pass - - @classmethod - def from_tsv( - cls, file_path: str | Path, base_path: str | Path | None = None - ) -> BaseTable[T]: - """Create table from a TSV file.""" - df = pd.read_csv(_generate_path(file_path, base_path), sep="\t") - return cls.from_df(df, rel_path=file_path, base_path=base_path) - - def to_tsv(self, file_path: str | Path = None) -> None: - """Write the table to a TSV file.""" - df = self.to_df() - df.to_csv( - file_path or _generate_path(self.rel_path, self.base_path), - sep="\t", - index=not isinstance(df.index, pd.RangeIndex), - ) - - @classmethod - def _element_class(cls) -> type[T]: - """Get the class of the elements in the table.""" - return get_args(cls.model_fields["elements"].annotation)[0] - - def __add__(self, other: T) -> BaseTable[T]: - """Add an item to the table.""" - if not isinstance(other, self._element_class()): - raise TypeError( - f"Can only add {self._element_class().__name__} " - f"to {self.__class__.__name__}" - ) - return self.__class__(elements=self.elements + [other]) - - def __iadd__(self, other: T) -> BaseTable[T]: - """Add an item to the table in place.""" - if not isinstance(other, self._element_class()): - raise TypeError( - f"Can only add {self._element_class().__name__} " - f"to {self.__class__.__name__}" - ) - self.elements.append(other) - return self diff --git a/petab/v2/core.py b/petab/v2/core.py index 0d3211e4..7595d629 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -7,6 +7,7 @@ import os import tempfile import traceback +from abc import abstractmethod from collections.abc import Sequence from enum import Enum from itertools import chain @@ -17,8 +18,11 @@ TYPE_CHECKING, Annotated, Any, + Generic, Literal, Self, + TypeVar, + get_args, ) import numpy as np @@ -37,16 +41,6 @@ model_validator, ) -try: - from petab_sciml import ( - ArrayData, - ArrayDataStandard, - NNModel, - NNModelStandard, - ) -except ModuleNotFoundError: - pass - from .._utils import _generate_path from ..v1 import ( validate_yaml_syntax, @@ -59,12 +53,16 @@ from ..v1.yaml import get_path_prefix from ..versions import parse_version from . import C, get_observable_df -from .base import BaseTable -from .sciml.core import ( - Hybridization, - HybridizationTable, - SciMLConfig, -) + +try: + from petab_sciml import ( + ArrayData, + ArrayDataStandard, + NNModel, + NNModelStandard, + ) +except ModuleNotFoundError: + pass if TYPE_CHECKING: from ..v2.lint import ValidationResultList, ValidationTask @@ -227,6 +225,101 @@ class PriorDistribution(str, Enum): ) +T = TypeVar("T", bound=BaseModel) + + +class BaseTable(BaseModel, Generic[T]): + """Base class for PEtab tables.""" + + #: The table elements + elements: list[T] + #: The path to the table file, if applicable. + #: Relative to the base path, if the base path is set and rel_path is not + #: an absolute path. + rel_path: AnyUrl | Path | None = Field(exclude=True, default=None) + #: The base path for the table file, if applicable. + #: This is usually the directory of the PEtab YAML file. + base_path: AnyUrl | Path | None = Field(exclude=True, default=None) + + def __init__(self, elements: list[T] = None, **kwargs) -> None: + """Initialize the BaseTable with a list of elements.""" + if elements is None: + elements = [] + super().__init__(elements=elements, **kwargs) + + def __getitem__(self, id_: str) -> T: + """Get an element by ID. + + :param id_: The ID of the element to retrieve. + :return: The element with the given ID. + :raises KeyError: If no element with the given ID exists. + :raises NotImplementedError: + If the element type does not have an ID attribute. + """ + if "id" not in self._element_class().model_fields: + raise NotImplementedError( + f"__getitem__ is not implemented for {self.__class__.__name__}" + ) + + for element in self.elements: + if element.id == id_: + return element + + raise KeyError(f"{T.__name__} ID {id_} not found") + + @classmethod + @abstractmethod + def from_df(cls, df: pd.DataFrame, **kwargs) -> BaseTable[T]: + """Create a table from a DataFrame.""" + pass + + @abstractmethod + def to_df(self) -> pd.DataFrame: + """Convert the table to a DataFrame.""" + pass + + @classmethod + def from_tsv( + cls, file_path: str | Path, base_path: str | Path | None = None + ) -> BaseTable[T]: + """Create table from a TSV file.""" + df = pd.read_csv(_generate_path(file_path, base_path), sep="\t") + return cls.from_df(df, rel_path=file_path, base_path=base_path) + + def to_tsv(self, file_path: str | Path = None) -> None: + """Write the table to a TSV file.""" + df = self.to_df() + df.to_csv( + file_path or _generate_path(self.rel_path, self.base_path), + sep="\t", + index=not isinstance(df.index, pd.RangeIndex), + ) + + @classmethod + def _element_class(cls) -> type[T]: + """Get the class of the elements in the table.""" + return get_args(cls.model_fields["elements"].annotation)[0] + + def __add__(self, other: T) -> BaseTable[T]: + """Add an item to the table.""" + if not isinstance(other, self._element_class()): + raise TypeError( + f"Can only add {self._element_class().__name__} " + f"to {self.__class__.__name__}" + ) + return self.__class__(elements=self.elements + [other]) + + def __iadd__(self, other: T) -> BaseTable[T]: + """Add an item to the table in place.""" + if not isinstance(other, self._element_class()): + raise TypeError( + f"Can only add {self._element_class().__name__} " + f"to {self.__class__.__name__}" + ) + self.elements.append(other) + return self + + class Observable(BaseModel): """Observable definition.""" @@ -1027,6 +1120,77 @@ def n_estimated(self) -> int: return sum(p.estimate for p in self.parameters) +class Hybridization(BaseModel): + """Assigns PEtab SciML NN inputs and outputs.""" + + #: The target ID. + target_id: str = Field(alias=C.TARGET_ID) + #: The target value. + target_value: sp.Basic = Field(alias=C.TARGET_VALUE) + + #: :meta private: + model_config = ConfigDict( + arbitrary_types_allowed=True, + populate_by_name=True, + extra="allow", + validate_assignment=True, + ) + + @field_validator("target_value", mode="before") + @classmethod + def _sympify(cls, v): + if v is None or isinstance(v, sp.Basic): + return v + if isinstance(v, float) and np.isnan(v): + return None + + return sympify_petab(v) + + +class HybridizationTable(BaseTable[Hybridization]): + """PEtab SciML hybridization table.""" + + @property + def hybridizations(self) -> list[Hybridization]: + """List of hybridizations.""" + return self.elements + + @classmethod + def from_df(cls, df: pd.DataFrame, **kwargs) -> HybridizationTable: + """Create a HybridizationTable from a DataFrame.""" + if df is None: + return cls(**kwargs) + + hybridizations = [ + Hybridization( + **row.to_dict(), + ) + for _, row in df.iterrows() + ] + + return cls(hybridizations, **kwargs) + + def to_df(self) -> pd.DataFrame: + """Convert the HybridizationTable to a DataFrame.""" + records = self.model_dump(by_alias=True)["elements"] + + return pd.DataFrame(records) + + def __getitem__(self, target_id: str) -> Hybridization: + """Get a hybridization by target ID.""" + for hybridization in self.hybridizations: + if hybridization.target_id == target_id: + return hybridization + raise KeyError(f"Target ID {target_id} not found") + + def get(self, target_id, default=None): + """Get a hybridization by target ID or return a default value.""" + try: + return self[target_id] + except KeyError: + return default + + class Problem: """ PEtab parameter estimation problem @@ -2518,6 +2682,18 @@ class ModelFile(BaseModel): ) +class NeuralNetConfig(BaseModel): + """A neural net in the PEtab SciML problem configuration.""" + + location: AnyUrl | Path + pre_initialization: bool + format: str + + model_config = ConfigDict( + validate_assignment=True, + ) + + class ExtensionConfig(BaseModel): """The configuration of a PEtab extension.""" @@ -2525,6 +2701,25 @@ class ExtensionConfig(BaseModel): config: dict +class SciMLConfig(BaseModel): + """The extended configuration of a PEtab SciML problem.""" + + #: The PEtab SciML format version. + version: str = "0.1.0" + #: The paths to the array data files. + # Absolute or relative to `base_path`. + array_files: list[AnyUrl | Path] = [] + #: The paths to the hybridization tables. + # Absolute or relative to `base_path`. + hybridization_files: list[AnyUrl | Path] = [] + #: The neural network IDs and info. + neural_networks: dict[str, NeuralNetConfig] | None = {} + + model_config = ConfigDict( + validate_assignment=True, + ) + + class ProblemConfig(BaseModel): """The PEtab problem configuration.""" diff --git a/petab/v2/lint.py b/petab/v2/lint.py index 7d33f33a..5f35c462 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -3,25 +3,18 @@ from __future__ import annotations import logging +from abc import ABC, abstractmethod from collections import Counter, OrderedDict from collections.abc import Set +from dataclasses import dataclass, field +from enum import IntEnum from itertools import chain from pathlib import Path import pandas as pd import sympy as sp -from petab.v2.sciml.lint import CheckHybridizationTable - from ..v2.C import * -from .base import ( - ValidationError, - ValidationIssue, - ValidationIssueSeverity, - ValidationResultList, - ValidationTask, - ValidationWarning, -) from .core import PriorDistribution, Problem logger = logging.getLogger(__name__) @@ -58,6 +51,126 @@ ] +class ValidationIssueSeverity(IntEnum): + """The severity of a validation issue.""" + + # INFO: Informational message, no action required + INFO = 10 + # WARNING: Warning message, potential issues + WARNING = 20 + # ERROR: Error message, action required + ERROR = 30 + # CRITICAL: Critical error message, stops further validation + CRITICAL = 40 + + +@dataclass +class ValidationIssue: + """The result of a validation task. + + Attributes: + level: The level of the validation event. + message: The message of the validation event. + """ + + level: ValidationIssueSeverity + message: str + task: str | None = None + + def __post_init__(self): + if not isinstance(self.level, ValidationIssueSeverity): + raise TypeError( + "`level` must be an instance of ValidationIssueSeverity." + ) + + def __str__(self): + return f"{self.level.name}: {self.message}" + + @staticmethod + def _get_task_name() -> str | None: + """Get the name of the ValidationTask that raised this error. + + Expected to be called from below a `ValidationTask.run`. + """ + import inspect + + # walk up the stack until we find the ValidationTask.run method + for frame_info in inspect.stack(): + frame = frame_info.frame + if "self" in frame.f_locals: + task = frame.f_locals["self"] + if isinstance(task, ValidationTask): + return task.__class__.__name__ + return None + + +@dataclass +class ValidationError(ValidationIssue): + """A validation result with level ERROR.""" + + level: ValidationIssueSeverity = field( + default=ValidationIssueSeverity.ERROR, init=False + ) + + def __post_init__(self): + if self.task is None: + self.task = self._get_task_name() + + +@dataclass +class ValidationWarning(ValidationIssue): + """A validation result with level WARNING.""" + + level: ValidationIssueSeverity = field( + default=ValidationIssueSeverity.WARNING, init=False + ) + + def __post_init__(self): + if self.task is None: + self.task = self._get_task_name() + + +class ValidationResultList(list[ValidationIssue]): + """A list of validation results. + + Contains all issues found during the validation of a PEtab problem. + """ + + def log( + self, + *, + logger: logging.Logger = logger, + min_level: ValidationIssueSeverity = ValidationIssueSeverity.INFO, + max_level: ValidationIssueSeverity = ValidationIssueSeverity.CRITICAL, + ): + """Log the validation results. + + :param logger: The logger to use for logging. + Defaults to the module logger. + :param min_level: The minimum severity level to log. + :param max_level: The maximum severity level to log. + """ + for result in self: + if result.level < min_level or result.level > max_level: + continue + msg = f"{result.level.name}: {result.message} [{result.task}]" + if result.level == ValidationIssueSeverity.INFO: + logger.info(msg) + elif result.level == ValidationIssueSeverity.WARNING: + logger.warning(msg) + elif result.level >= ValidationIssueSeverity.ERROR: + logger.error(msg) + + if not self: + logger.info("PEtab format check completed successfully.") + + def has_errors(self) -> bool: + """Check if there are any errors in the validation results.""" + return any( + result.level >= ValidationIssueSeverity.ERROR for result in self + ) + + def lint_problem(problem: Problem | str | Path) -> ValidationResultList: """Validate a PEtab problem. @@ -74,6 +187,24 @@ def lint_problem(problem: Problem | str | Path) -> ValidationResultList: return problem.validate() +class ValidationTask(ABC): + """A task to validate a PEtab problem.""" + + @abstractmethod + def run(self, problem: Problem) -> ValidationIssue | None: + """Run the validation task. + + Arguments: + problem: PEtab problem to check. + Returns: + Validation results or ``None`` + """ + ... + + def __call__(self, *args, **kwargs): + return self.run(*args, **kwargs) + + class CheckProblemConfig(ValidationTask): """A task to validate the configuration of a PEtab problem. @@ -840,6 +971,40 @@ def run(self, problem: Problem) -> ValidationIssue | None: return None +class CheckHybridizationTable(ValidationTask): + """Validate the SciML hybridization table.""" + + def run(self, problem: Problem) -> ValidationIssue | None: + messages = [] + + condition_targets = { + c.target_id for ct in problem.conditions for c in ct.changes + } + nn_input_ids = { + inp.input_id for nn in problem.neural_networks for inp in nn.inputs + } + hyb_target_ids = {hyb.target_id for hyb in problem.hybridizations} + hyb_target_vals = {hyb.target_value for hyb in problem.hybridizations} + + # Hybridization targets are not also targets in the condition table + if culprits := (hyb_target_ids & condition_targets): + messages.append( + f"Hybridization target ids `{culprits}` are also " + "target ids in the condition table." + ) + # NN inputs are not used as target values + if culprits := (hyb_target_vals & nn_input_ids): + messages.append( + "The following neural net inputs were used as target values " + f"in the Hybridization table: `{culprits}`." + ) + + if messages: + return ValidationError("\n".join(messages)) + + return None + + def get_valid_parameters_for_parameter_table( problem: Problem, ) -> set[str]: @@ -1056,6 +1221,25 @@ def get_placeholders( ] #: Validation tasks that should be run PEtab SciML problems -sciml_validation_tasks = default_validation_tasks + [ +sciml_validation_tasks = [ + CheckProblemConfig(), + CheckModel(), + CheckUniquePrimaryKeys(), + CheckMeasurementModelId(), + CheckMeasuredObservablesDefined(), + CheckPosLogMeasurements(), + CheckOverridesMatchPlaceholders(), + CheckValidConditionTargets(), + CheckExperimentTable(), + CheckExperimentConditionsExist(), + CheckUndefinedExperiments(), + CheckObservablesDoNotShadowModelEntities(), + CheckAllParametersPresentInParameterTable(), + CheckValidParameterInConditionOrParameterTable(), + CheckUnusedExperiments(), + CheckUnusedConditions(), + CheckPriorDistribution(), + CheckInitialChangeSymbols(), + CheckMappingTable(), CheckHybridizationTable(), ] diff --git a/petab/v2/sciml/__init__.py b/petab/v2/sciml/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/petab/v2/sciml/core.py b/petab/v2/sciml/core.py deleted file mode 100644 index e4c29a46..00000000 --- a/petab/v2/sciml/core.py +++ /dev/null @@ -1,127 +0,0 @@ -from pathlib import Path -from typing import Self - -import numpy as np -import pandas as pd -import sympy as sp -from pydantic import ( - AnyUrl, - BaseModel, - ConfigDict, - Field, - field_validator, -) - -import petab.v2.C as C - -from ...v1.math.sympify import sympify_petab -from ..base import BaseTable - -__all__ = [ - "Hybridization", - "HybridizationTable", - "NeuralNetConfig", - "SciMLConfig", -] - - -class Hybridization(BaseModel): - """Assigns PEtab SciML NN inputs and outputs.""" - - #: The target ID. - target_id: str = Field(alias=C.TARGET_ID) - #: The target value. - target_value: sp.Basic = Field(alias=C.TARGET_VALUE) - - #: :meta private: - model_config = ConfigDict( - arbitrary_types_allowed=True, - populate_by_name=True, - extra="allow", - validate_assignment=True, - ) - - @field_validator("target_value", mode="before") - @classmethod - def _sympify(cls, v): - if v is None or isinstance(v, sp.Basic): - return v - if isinstance(v, float) and np.isnan(v): - return None - - return sympify_petab(v) - - -class HybridizationTable(BaseTable[Hybridization]): - """PEtab SciML hybridization table.""" - - @property - def hybridizations(self) -> list[Hybridization]: - """List of hybridizations.""" - return self.elements - - @classmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> Self: - """Create a HybridizationTable from a DataFrame.""" - if df is None: - return cls(**kwargs) - - hybridizations = [ - Hybridization( - **row.to_dict(), - ) - for _, row in df.iterrows() - ] - - return cls(hybridizations, **kwargs) - - def to_df(self) -> pd.DataFrame: - """Convert the HybridizationTable to a DataFrame.""" - records = self.model_dump(by_alias=True)["elements"] - - return pd.DataFrame(records) - - def __getitem__(self, target_id: str) -> Hybridization: - """Get a hybridization by target ID.""" - for hybridization in self.hybridizations: - if hybridization.target_id == target_id: - return hybridization - raise KeyError(f"Target ID {target_id} not found") - - def get(self, target_id, default=None): - """Get a hybridization by target ID or return a default value.""" - try: - return self[target_id] - except KeyError: - return default - - -class NeuralNetConfig(BaseModel): - """A neural net in the PEtab SciML problem configuration.""" - - location: AnyUrl | Path - pre_initialization: bool - format: str - - model_config = ConfigDict( - validate_assignment=True, - ) - - -class SciMLConfig(BaseModel): - """The extended configuration of a PEtab SciML problem.""" - - #: The PEtab SciML format version. - version: str = "0.1.0" - #: The paths to the array data files. - # Absolute or relative to `base_path`. - array_files: list[AnyUrl | Path] = [] - #: The paths to the hybridization tables. - # Absolute or relative to `base_path`. - hybridization_files: list[AnyUrl | Path] = [] - #: The neural network IDs and info. - neural_networks: dict[str, NeuralNetConfig] | None = {} - - model_config = ConfigDict( - validate_assignment=True, - ) diff --git a/petab/v2/sciml/lint.py b/petab/v2/sciml/lint.py deleted file mode 100644 index dc087a36..00000000 --- a/petab/v2/sciml/lint.py +++ /dev/null @@ -1,36 +0,0 @@ -from petab.v1.problem import Problem -from petab.v2.base import ValidationError, ValidationIssue, ValidationTask - - -class CheckHybridizationTable(ValidationTask): - """Validate the SciML hybridization table.""" - - def run(self, problem: Problem) -> ValidationIssue | None: - messages = [] - - condition_targets = { - c.target_id for ct in problem.conditions for c in ct.changes - } - nn_input_ids = { - inp.input_id for nn in problem.neural_networks for inp in nn.inputs - } - hyb_target_ids = {hyb.target_id for hyb in problem.hybridizations} - hyb_target_vals = {hyb.target_value for hyb in problem.hybridizations} - - # Hybridization targets are not also targets in the condition table - if culprits := (hyb_target_ids & condition_targets): - messages.append( - f"Hybridization target ids `{culprits}` are also " - "target ids in the condition table." - ) - # NN inputs are not used as target values - if culprits := (hyb_target_vals & nn_input_ids): - messages.append( - "The following neural net inputs were used as target values " - f"in the Hybridization table: `{culprits}`." - ) - - if messages: - return ValidationError("\n".join(messages)) - - return None diff --git a/tests/v2/test_sciml.py b/tests/v2/test_sciml.py index 23737a0e..1f0c35e4 100644 --- a/tests/v2/test_sciml.py +++ b/tests/v2/test_sciml.py @@ -2,10 +2,9 @@ from pydantic import ConfigDict from petab.v2.core import * -from petab.v2.core import ModelFile +from petab.v2.core import ModelFile, NeuralNetConfig from petab.v2.lint import sciml_validation_tasks from petab.v2.models.sbml_model import SbmlModel -from petab.v2.sciml.core import NeuralNetConfig def _get_test_problem(): From 239e1fffaf3f7a295e97bbe362d864ad64c36fc6 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Thu, 18 Jun 2026 16:27:46 +0100 Subject: [PATCH 23/28] use problem.extensions.sciml for separation --- petab/v2/C.py | 2 +- petab/v2/core.py | 272 +++++--------------------------- petab/v2/extensions/__init__.py | 0 petab/v2/extensions/sciml.py | 233 +++++++++++++++++++++++++++ petab/v2/lint.py | 40 ++--- tests/v2/test_sciml.py | 21 +-- 6 files changed, 303 insertions(+), 265 deletions(-) create mode 100644 petab/v2/extensions/__init__.py create mode 100644 petab/v2/extensions/sciml.py diff --git a/petab/v2/C.py b/petab/v2/C.py index 7b141b77..d738804f 100644 --- a/petab/v2/C.py +++ b/petab/v2/C.py @@ -259,7 +259,7 @@ #: Extensions key in the YAML file EXTENSIONS = "extensions" #: PEtab SciML extension -SCIML = "sciml" +EXT_ID_SCIML = "sciml" # MAPPING diff --git a/petab/v2/core.py b/petab/v2/core.py index 7595d629..fb7c9863 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -54,16 +54,6 @@ from ..versions import parse_version from . import C, get_observable_df -try: - from petab_sciml import ( - ArrayData, - ArrayDataStandard, - NNModel, - NNModelStandard, - ) -except ModuleNotFoundError: - pass - if TYPE_CHECKING: from ..v2.lint import ValidationResultList, ValidationTask @@ -71,7 +61,6 @@ __all__ = [ "Problem", "ProblemConfig", - "SciMLConfig", "Observable", "ObservableTable", "NoiseDistribution", @@ -320,6 +309,22 @@ def __iadd__(self, other: T) -> BaseTable[T]: return self +# SciML extension classes — imported after BaseTable is defined to avoid +# circular imports (sciml.py does not import from core.py). +from .extensions.sciml import ( # noqa: E402 + HybridizationTable, + SciMLConfig, + SciMLExt, +) + + +class ProblemExtensions: + """Runtime extension state attached to a :class:`Problem`.""" + + def __init__(self, sciml: SciMLExt = None): + self.sciml: SciMLExt = sciml or SciMLExt() + + class Observable(BaseModel): """Observable definition.""" @@ -1120,77 +1125,6 @@ def n_estimated(self) -> int: return sum(p.estimate for p in self.parameters) -class Hybridization(BaseModel): - """Assigns PEtab SciML NN inputs and outputs.""" - - #: The target ID. - target_id: str = Field(alias=C.TARGET_ID) - #: The target value. - target_value: sp.Basic = Field(alias=C.TARGET_VALUE) - - #: :meta private: - model_config = ConfigDict( - arbitrary_types_allowed=True, - populate_by_name=True, - extra="allow", - validate_assignment=True, - ) - - @field_validator("target_value", mode="before") - @classmethod - def _sympify(cls, v): - if v is None or isinstance(v, sp.Basic): - return v - if isinstance(v, float) and np.isnan(v): - return None - - return sympify_petab(v) - - -class HybridizationTable(BaseTable[Hybridization]): - """PEtab SciML hybridization table.""" - - @property - def hybridizations(self) -> list[Hybridization]: - """List of hybridizations.""" - return self.elements - - @classmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> HybridizationTable: - """Create a HybridizationTable from a DataFrame.""" - if df is None: - return cls(**kwargs) - - hybridizations = [ - Hybridization( - **row.to_dict(), - ) - for _, row in df.iterrows() - ] - - return cls(hybridizations, **kwargs) - - def to_df(self) -> pd.DataFrame: - """Convert the HybridizationTable to a DataFrame.""" - records = self.model_dump(by_alias=True)["elements"] - - return pd.DataFrame(records) - - def __getitem__(self, target_id: str) -> Hybridization: - """Get a hybridization by target ID.""" - for hybridization in self.hybridizations: - if hybridization.target_id == target_id: - return hybridization - raise KeyError(f"Target ID {target_id} not found") - - def get(self, target_id, default=None): - """Get a hybridization by target ID or return a default value.""" - try: - return self[target_id] - except KeyError: - return default - - class Problem: """ PEtab parameter estimation problem @@ -1217,16 +1151,18 @@ def __init__( measurement_tables: list[MeasurementTable] = None, parameter_tables: list[ParameterTable] = None, mapping_tables: list[MappingTable] = None, - neural_networks: list[NNModel] | None = None, - hybridization_tables: list[HybridizationTable] | None = None, - array_data_files: list[ArrayData] | None = None, + extensions: ProblemExtensions = None, config: ProblemConfig = None, ): from ..v2.lint import default_validation_tasks, sciml_validation_tasks self.config = config self.models: list[Model] = models or [] - if config and config.extensions and config.extensions[C.SCIML]: + if ( + config + and config.extensions + and config.extensions.get(C.EXT_ID_SCIML) + ): self.validation_tasks: list[ValidationTask] = ( sciml_validation_tasks.copy() ) @@ -1241,11 +1177,7 @@ def __init__( self.measurement_tables = measurement_tables or [MeasurementTable()] self.mapping_tables = mapping_tables or [MappingTable()] self.parameter_tables = parameter_tables or [ParameterTable()] - self.neural_networks = neural_networks or [] - self.hybridization_tables = hybridization_tables or [ - HybridizationTable() - ] - self.array_data_files = array_data_files or [] + self.extensions = extensions or ProblemExtensions() def __repr__(self): return f"<{self.__class__.__name__} id={self.id!r}>" @@ -1418,10 +1350,10 @@ def from_yaml( else None ) - neural_networks = None - hybridization_tables = None - array_data_files = None - if config.extensions and config.extensions[C.SCIML]: + extensions = ProblemExtensions() + if config.extensions and config.extensions.get(C.EXT_ID_SCIML): + from petab_sciml import ArrayDataStandard, NNModel, NNModelStandard + # Neural network classes are constructed via pytorch for now to get # the proper inputs neural_networks = [ @@ -1435,20 +1367,28 @@ def from_yaml( nn_model_id=nn_id, ) for nn_id, nn_config in ( - config.extensions[C.SCIML].neural_networks or {} + config.extensions[C.EXT_ID_SCIML].neural_networks or {} ).items() ] hybridization_tables = [ HybridizationTable.from_tsv(f, base_path) - for f in config.extensions[C.SCIML].hybridization_files + for f in config.extensions[C.EXT_ID_SCIML].hybridization_files ] array_data_files = [ ArrayDataStandard.load_data(_generate_path(f, base_path)) - for f in config.extensions[C.SCIML].array_files + for f in config.extensions[C.EXT_ID_SCIML].array_files ] + extensions = ProblemExtensions( + sciml=SciMLExt( + neural_networks=neural_networks, + hybridization_tables=hybridization_tables, + array_data_files=array_data_files, + ) + ) + return Problem( config=config, models=models, @@ -1458,9 +1398,7 @@ def from_yaml( measurement_tables=measurement_tables, parameter_tables=parameter_tables, mapping_tables=mapping_tables, - neural_networks=neural_networks, - hybridization_tables=hybridization_tables, - array_data_files=array_data_files, + extensions=extensions, ) @staticmethod @@ -1767,34 +1705,6 @@ def id(self, value: str): self.config = ProblemConfig(format_version="2.0.0") self.config.id = value - @property - def hybridizations(self) -> list[Hybridization]: - """ - List of hybridizations in the hybridization table(s). - Note that hybridizations are specific to PEtab SciML problems. - """ - return list( - chain.from_iterable( - ht.hybridizations for ht in self.hybridization_tables - ) - ) - - @property - def hybridization_df(self) -> pd.DataFrame | None: - """ - Combined SciML hybridization tables as DataFrame. - Note that hybridizations are specific to PEtab SciML problems. - """ - return ( - HybridizationTable(hybridizations).to_df() - if (hybridizations := self.hybridizations) - else None - ) - - @hybridization_df.setter - def hybridization_df(self, value: pd.DataFrame): - self.hybridization_tables = [HybridizationTable.from_df(value)] - def get_optimization_parameters(self) -> list[str]: """ Get the list of optimization parameter IDs from parameter table. @@ -2099,7 +2009,7 @@ def validate( validation_results = ValidationResultList() - supported_extensions = {C.SCIML} + supported_extensions = {C.EXT_ID_SCIML} if ( self.config and self.config.extensions @@ -2411,73 +2321,6 @@ def add_experiment(self, id_: str, *args): Experiment(id=id_, periods=periods) ) - def add_hybridization(self, target_id: str, target_value: str): - """Add a SciML hybridization table entry to the problem. - - If there is more than one hybridization table, the hybridization - is added to the last table. Note that hybridizations are specific - to PEtab SciML problems. - - Arguments: - target_id: The ID of the target entity in the PEtab problem - or neural network model - target_value: The value that is assigned to the target id. - """ - if not self.hybridization_tables: - self.hybridization_tables.append(HybridizationTable()) - self.hybridization_tables[-1].hybridizations.append( - Hybridization(target_id=target_id, target_value=target_value) - ) - - def add_neural_network_from_dict(self, model_id: str, nn_dict: dict): - """ - Add a SciML neural net from a dictionary (or PEtab SciML problems). - """ - # from petab_sciml import NNModel - nn_model = NNModel.model_validate(nn_dict) - nn_model.nn_model_id = model_id - self.neural_networks.append(nn_model) - - def add_neural_network_from_yaml( - self, - model_id: str, - file_path: str | Path, - base_path: str | Path | None = None, - ): - """ - Add a SciML neural net from a yaml file (for PEtab SciML problems). - """ - # from petab_sciml import NNModelStandard - self.neural_networks.append( - NNModelStandard.load_data( - _generate_path( - file_path=file_path, - base_path=base_path, - ), - nn_model_id=model_id, - ) - ) - - def add_array_data_from_dict(self, array_data: dict): - """ - Add SciML array data from a dictionary (for PEtab SciML problems). - """ - # from petab_sciml import ArrayData - self.array_data_files.append(ArrayData.model_validate(array_data)) - - def add_array_data_from_hdf5( - self, - file_path: str | Path, - base_path: str | Path | None = None, - ): - """ - Add SciML array data from an hdf5 file (for PEtab SciML problems). - """ - # from petab_sciml import ArrayDataStandard - self.array_data_files.append( - ArrayDataStandard.load_data(_generate_path(file_path, base_path)) - ) - def __iadd__(self, other): """Add Observable, Parameter, Measurement, Condition, or Experiment""" from .core import ( @@ -2682,18 +2525,6 @@ class ModelFile(BaseModel): ) -class NeuralNetConfig(BaseModel): - """A neural net in the PEtab SciML problem configuration.""" - - location: AnyUrl | Path - pre_initialization: bool - format: str - - model_config = ConfigDict( - validate_assignment=True, - ) - - class ExtensionConfig(BaseModel): """The configuration of a PEtab extension.""" @@ -2701,25 +2532,6 @@ class ExtensionConfig(BaseModel): config: dict -class SciMLConfig(BaseModel): - """The extended configuration of a PEtab SciML problem.""" - - #: The PEtab SciML format version. - version: str = "0.1.0" - #: The paths to the array data files. - # Absolute or relative to `base_path`. - array_files: list[AnyUrl | Path] = [] - #: The paths to the hybridization tables. - # Absolute or relative to `base_path`. - hybridization_files: list[AnyUrl | Path] = [] - #: The neural network IDs and info. - neural_networks: dict[str, NeuralNetConfig] | None = {} - - model_config = ConfigDict( - validate_assignment=True, - ) - - class ProblemConfig(BaseModel): """The PEtab problem configuration.""" @@ -2777,7 +2589,7 @@ def _parse_extensions(cls, v): if isinstance(v, dict): parsed_extensions = {} for ext_name, ext_config in v.items(): - if ext_name == C.SCIML: + if ext_name == C.EXT_ID_SCIML: # Convert sciml extension to SciMLConfig parsed_extensions[ext_name] = SciMLConfig(**ext_config) else: @@ -2830,7 +2642,7 @@ def to_yaml(self, filename: str | Path): del data["id"] for ext_id, d_ext in data[C.EXTENSIONS].items(): - if ext_id == C.SCIML: + if ext_id == C.EXT_ID_SCIML: # convert Paths to strings for key in ("array_files", "hybridization_files"): d_ext[key] = list(map(str, d_ext[key])) diff --git a/petab/v2/extensions/__init__.py b/petab/v2/extensions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/petab/v2/extensions/sciml.py b/petab/v2/extensions/sciml.py new file mode 100644 index 00000000..0b483783 --- /dev/null +++ b/petab/v2/extensions/sciml.py @@ -0,0 +1,233 @@ +"""PEtab SciML extension — classes and runtime state for hybrid ODE/ML +problems.""" + +from __future__ import annotations + +from itertools import chain +from pathlib import Path + +import numpy as np +import pandas as pd +import sympy as sp +from pydantic import AnyUrl, BaseModel, ConfigDict, Field, field_validator + +from petab._utils import _generate_path +from petab.v1.math import sympify_petab + +try: + from petab_sciml import ( + ArrayData, + ArrayDataStandard, + NNModel, + NNModelStandard, + ) +except ModuleNotFoundError: + pass + +from .. import C + + +class Hybridization(BaseModel): + """Assigns PEtab SciML NN inputs and outputs.""" + + #: The target ID. + target_id: str = Field(alias=C.TARGET_ID) + #: The target value. + target_value: sp.Basic = Field(alias=C.TARGET_VALUE) + + #: :meta private: + model_config = ConfigDict( + arbitrary_types_allowed=True, + populate_by_name=True, + extra="allow", + validate_assignment=True, + ) + + @field_validator("target_value", mode="before") + @classmethod + def _sympify(cls, v): + if v is None or isinstance(v, sp.Basic): + return v + if isinstance(v, float) and np.isnan(v): + return None + return sympify_petab(v) + + +class HybridizationTable: + """PEtab SciML hybridization table.""" + + def __init__(self, hybridizations: list[Hybridization] = None, **kwargs): + self.hybridizations: list[Hybridization] = hybridizations or [] + self.rel_path: AnyUrl | Path | None = kwargs.get("rel_path") + self.base_path: AnyUrl | Path | None = kwargs.get("base_path") + + @property + def elements(self) -> list[Hybridization]: + return self.hybridizations + + @classmethod + def from_df(cls, df: pd.DataFrame, **kwargs) -> HybridizationTable: + """Create a HybridizationTable from a DataFrame.""" + if df is None: + return cls(**kwargs) + + hybridizations = [ + Hybridization(**row.to_dict()) for _, row in df.iterrows() + ] + return cls(hybridizations, **kwargs) + + @classmethod + def from_tsv( + cls, + file_path: str | Path, + base_path: str | Path | None = None, + ) -> HybridizationTable: + """Create a HybridizationTable from a TSV file.""" + df = pd.read_csv(_generate_path(file_path, base_path), sep="\t") + return cls.from_df(df, rel_path=file_path, base_path=base_path) + + def to_df(self) -> pd.DataFrame: + """Convert the HybridizationTable to a DataFrame.""" + records = [h.model_dump(by_alias=True) for h in self.hybridizations] + return pd.DataFrame(records) + + def to_tsv(self, file_path: str | Path = None) -> None: + """Write the table to a TSV file.""" + df = self.to_df() + df.to_csv( + file_path or _generate_path(self.rel_path, self.base_path), + sep="\t", + index=False, + ) + + def __getitem__(self, target_id: str) -> Hybridization: + """Get a hybridization by target ID.""" + for hybridization in self.hybridizations: + if hybridization.target_id == target_id: + return hybridization + raise KeyError(f"Target ID {target_id} not found") + + def get(self, target_id, default=None): + """Get a hybridization by target ID or return a default value.""" + try: + return self[target_id] + except KeyError: + return default + + +class NeuralNetConfig(BaseModel): + """A neural net in the PEtab SciML problem configuration.""" + + location: AnyUrl | Path + pre_initialization: bool + format: str + + model_config = ConfigDict( + validate_assignment=True, + ) + + +class SciMLConfig(BaseModel): + """The extended configuration of a PEtab SciML problem.""" + + #: The PEtab SciML format version. + version: str = "0.1.0" + #: The paths to the array data files. + array_files: list[AnyUrl | Path] = [] + #: The paths to the hybridization tables. + hybridization_files: list[AnyUrl | Path] = [] + #: The neural network IDs and info. + neural_networks: dict[str, NeuralNetConfig] | None = {} + + model_config = ConfigDict( + validate_assignment=True, + ) + + +class SciMLExt: + """SciML extension runtime state. + + Accessible as ``Problem.extensions.sciml``. + """ + + def __init__( + self, + neural_networks: list = None, + hybridization_tables: list[HybridizationTable] = None, + array_data_files: list = None, + ): + self.neural_networks: list = neural_networks or [] + self.hybridization_tables: list[HybridizationTable] = ( + hybridization_tables or [HybridizationTable()] + ) + self.array_data_files: list = array_data_files or [] + + @property + def hybridizations(self) -> list[Hybridization]: + """Flat list of all hybridizations across all hybridization tables.""" + return list( + chain.from_iterable( + ht.hybridizations for ht in self.hybridization_tables + ) + ) + + @property + def hybridization_df(self) -> pd.DataFrame | None: + """Combined hybridization tables as a single DataFrame.""" + hybs = self.hybridizations + return HybridizationTable(hybs).to_df() if hybs else None + + @hybridization_df.setter + def hybridization_df(self, value: pd.DataFrame): + self.hybridization_tables = [HybridizationTable.from_df(value)] + + def add_hybridization(self, target_id: str, target_value: str): + """Add a hybridization entry. + + If there is more than one hybridization table the entry is added to + the last one. + + Arguments: + target_id: The ID of the target entity in the PEtab problem + or neural network model + target_value: The value that is assigned to the target id. + """ + if not self.hybridization_tables: + self.hybridization_tables.append(HybridizationTable()) + self.hybridization_tables[-1].hybridizations.append( + Hybridization(target_id=target_id, target_value=target_value) + ) + + def add_neural_network_from_dict(self, model_id: str, nn_dict: dict): + """Add a neural network from a dictionary.""" + nn_model = NNModel.model_validate(nn_dict) + nn_model.nn_model_id = model_id + self.neural_networks.append(nn_model) + + def add_neural_network_from_yaml( + self, + model_id: str, + file_path: str | Path, + base_path: str | Path | None = None, + ): + """Add a neural network from a YAML file.""" + self.neural_networks.append( + NNModelStandard.load_data( + _generate_path(file_path=file_path, base_path=base_path), + nn_model_id=model_id, + ) + ) + + def add_array_data_from_dict(self, array_data: dict): + """Add array data from a dictionary.""" + self.array_data_files.append(ArrayData.model_validate(array_data)) + + def add_array_data_from_hdf5( + self, + file_path: str | Path, + base_path: str | Path | None = None, + ): + """Add array data from an HDF5 file.""" + self.array_data_files.append( + ArrayDataStandard.load_data(_generate_path(file_path, base_path)) + ) diff --git a/petab/v2/lint.py b/petab/v2/lint.py index 5f35c462..8fde413f 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -981,10 +981,16 @@ def run(self, problem: Problem) -> ValidationIssue | None: c.target_id for ct in problem.conditions for c in ct.changes } nn_input_ids = { - inp.input_id for nn in problem.neural_networks for inp in nn.inputs + inp.input_id + for nn in problem.extensions.sciml.neural_networks + for inp in nn.inputs + } + hyb_target_ids = { + hyb.target_id for hyb in problem.extensions.sciml.hybridizations + } + hyb_target_vals = { + hyb.target_value for hyb in problem.extensions.sciml.hybridizations } - hyb_target_ids = {hyb.target_id for hyb in problem.hybridizations} - hyb_target_vals = {hyb.target_value for hyb in problem.hybridizations} # Hybridization targets are not also targets in the condition table if culprits := (hyb_target_ids & condition_targets): @@ -1156,10 +1162,13 @@ def append_overrides(overrides): } parameter_ids -= condition_targets - hybridization_targets = {hyb.target_id for hyb in problem.hybridizations} + hybridization_targets = { + hyb.target_id for hyb in problem.extensions.sciml.hybridizations + } parameter_ids -= hybridization_targets hybridization_target_values = { - str(hyb.target_value) for hyb in problem.hybridizations + str(hyb.target_value) + for hyb in problem.extensions.sciml.hybridizations } parameter_ids -= hybridization_target_values @@ -1221,25 +1230,6 @@ def get_placeholders( ] #: Validation tasks that should be run PEtab SciML problems -sciml_validation_tasks = [ - CheckProblemConfig(), - CheckModel(), - CheckUniquePrimaryKeys(), - CheckMeasurementModelId(), - CheckMeasuredObservablesDefined(), - CheckPosLogMeasurements(), - CheckOverridesMatchPlaceholders(), - CheckValidConditionTargets(), - CheckExperimentTable(), - CheckExperimentConditionsExist(), - CheckUndefinedExperiments(), - CheckObservablesDoNotShadowModelEntities(), - CheckAllParametersPresentInParameterTable(), - CheckValidParameterInConditionOrParameterTable(), - CheckUnusedExperiments(), - CheckUnusedConditions(), - CheckPriorDistribution(), - CheckInitialChangeSymbols(), - CheckMappingTable(), +sciml_validation_tasks = default_validation_tasks + [ CheckHybridizationTable(), ] diff --git a/tests/v2/test_sciml.py b/tests/v2/test_sciml.py index 1f0c35e4..f25b85b8 100644 --- a/tests/v2/test_sciml.py +++ b/tests/v2/test_sciml.py @@ -2,7 +2,8 @@ from pydantic import ConfigDict from petab.v2.core import * -from petab.v2.core import ModelFile, NeuralNetConfig +from petab.v2.core import ModelFile +from petab.v2.extensions.sciml import NeuralNetConfig from petab.v2.lint import sciml_validation_tasks from petab.v2.models.sbml_model import SbmlModel @@ -61,10 +62,10 @@ def _get_test_problem(): problem.add_parameter( "net1_ps", estimate=True, lb=-np.inf, ub=np.inf, nominal_value="array" ) - problem.add_hybridization("net1_input1", "A") - problem.add_hybridization("net1_input2", "B") - problem.add_hybridization("gamma_", "net1_output1") - problem.add_neural_network_from_dict( + problem.extensions.sciml.add_hybridization("net1_input1", "A") + problem.extensions.sciml.add_hybridization("net1_input2", "B") + problem.extensions.sciml.add_hybridization("gamma_", "net1_output1") + problem.extensions.sciml.add_neural_network_from_dict( "net1", nn_dict={ "nn_model_id": "net1", @@ -103,7 +104,7 @@ def _get_test_problem(): ) # array data - problem.add_array_data_from_dict( + problem.extensions.sciml.add_array_data_from_dict( { "metadata": {"pytorch_format": True}, "inputs": {}, @@ -126,9 +127,11 @@ def _get_test_problem(): problem.measurement_tables[0].rel_path = "measurements.tsv" problem.observable_tables[0].rel_path = "observables.tsv" problem.parameter_tables[0].rel_path = "parameters.tsv" - problem.hybridization_tables[0].rel_path = "hybridizations.tsv" - # problem.neural_networks[0].rel_path = "net1.yaml" - # problem.array_data_files[0].rel_path = "net1_ps.hdf5" + problem.extensions.sciml.hybridization_tables[ + 0 + ].rel_path = "hybridizations.tsv" + # problem.extensions.sciml.neural_networks[0].rel_path = "net1.yaml" + # problem.extensions.sciml.array_data_files[0].rel_path = "net1_ps.hdf5" return problem From 3c1a09f6232828bbf21f017f57d7d4d2ef435acd Mon Sep 17 00:00:00 2001 From: user <59329744+dilpath@users.noreply.github.com> Date: Fri, 19 Jun 2026 16:45:17 +0200 Subject: [PATCH 24/28] revert mapping table fixes that should be implemented in the other pr --- petab/v2/lint.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/petab/v2/lint.py b/petab/v2/lint.py index 8fde413f..5048d09a 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -1055,15 +1055,11 @@ def get_valid_parameters_for_parameter_table( # aliases an invalid PEtab ID? See # https://github.com/PEtab-dev/libpetab-python/pull/482#discussion_r3420762034 for mapping in problem.mappings: - if mapping.petab_id not in invalid: + if mapping.model_id and mapping.model_id in parameter_ids.keys(): parameter_ids[mapping.petab_id] = None - # An aliased model id is not a valid parameter id - if ( - mapping.model_id - and mapping.model_id != mapping.petab_id - and mapping.model_id in parameter_ids - ): - del parameter_ids[mapping.model_id] + # An aliased model id is not a valid parameter id + if mapping.model_id in parameter_ids: + del parameter_ids[mapping.model_id] # add output parameters from observable table output_parameters = problem.get_output_parameters() From 118aed7b6294f2db1ecba8ce686770e9db86ed0f Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Mon, 22 Jun 2026 14:38:08 +0100 Subject: [PATCH 25/28] fix aliased model logic - again --- petab/v2/lint.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/petab/v2/lint.py b/petab/v2/lint.py index 5048d09a..8fde413f 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -1055,11 +1055,15 @@ def get_valid_parameters_for_parameter_table( # aliases an invalid PEtab ID? See # https://github.com/PEtab-dev/libpetab-python/pull/482#discussion_r3420762034 for mapping in problem.mappings: - if mapping.model_id and mapping.model_id in parameter_ids.keys(): + if mapping.petab_id not in invalid: parameter_ids[mapping.petab_id] = None - # An aliased model id is not a valid parameter id - if mapping.model_id in parameter_ids: - del parameter_ids[mapping.model_id] + # An aliased model id is not a valid parameter id + if ( + mapping.model_id + and mapping.model_id != mapping.petab_id + and mapping.model_id in parameter_ids + ): + del parameter_ids[mapping.model_id] # add output parameters from observable table output_parameters = problem.get_output_parameters() From e133f217b61ed47b79c9613f8adc81c31a4d3e9e Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 23 Jun 2026 10:32:10 +0100 Subject: [PATCH 26/28] move sciml validation to own file --- petab/v2/extensions/sciml_lint.py | 43 +++++++++++++++++++++++++++++++ petab/v2/lint.py | 43 +++---------------------------- 2 files changed, 46 insertions(+), 40 deletions(-) create mode 100644 petab/v2/extensions/sciml_lint.py diff --git a/petab/v2/extensions/sciml_lint.py b/petab/v2/extensions/sciml_lint.py new file mode 100644 index 00000000..c4f65179 --- /dev/null +++ b/petab/v2/extensions/sciml_lint.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from .. import core, lint + + +class CheckHybridizationTable(lint.ValidationTask): + """Validate the SciML hybridization table.""" + + def run(self, problem: core.Problem) -> lint.ValidationIssue | None: + messages = [] + + condition_targets = { + c.target_id for ct in problem.conditions for c in ct.changes + } + nn_input_ids = { + inp.input_id + for nn in problem.extensions.sciml.neural_networks + for inp in nn.inputs + } + hyb_target_ids = { + hyb.target_id for hyb in problem.extensions.sciml.hybridizations + } + hyb_target_vals = { + hyb.target_value for hyb in problem.extensions.sciml.hybridizations + } + + # Hybridization targets are not also targets in the condition table + if culprits := (hyb_target_ids & condition_targets): + messages.append( + f"Hybridization target ids `{culprits}` are also " + "target ids in the condition table." + ) + # NN inputs are not used as target values + if culprits := (hyb_target_vals & nn_input_ids): + messages.append( + "The following neural net inputs were used as target values " + f"in the Hybridization table: `{culprits}`." + ) + + if messages: + return lint.ValidationError("\n".join(messages)) + + return None diff --git a/petab/v2/lint.py b/petab/v2/lint.py index 8fde413f..4260e40a 100644 --- a/petab/v2/lint.py +++ b/petab/v2/lint.py @@ -971,46 +971,6 @@ def run(self, problem: Problem) -> ValidationIssue | None: return None -class CheckHybridizationTable(ValidationTask): - """Validate the SciML hybridization table.""" - - def run(self, problem: Problem) -> ValidationIssue | None: - messages = [] - - condition_targets = { - c.target_id for ct in problem.conditions for c in ct.changes - } - nn_input_ids = { - inp.input_id - for nn in problem.extensions.sciml.neural_networks - for inp in nn.inputs - } - hyb_target_ids = { - hyb.target_id for hyb in problem.extensions.sciml.hybridizations - } - hyb_target_vals = { - hyb.target_value for hyb in problem.extensions.sciml.hybridizations - } - - # Hybridization targets are not also targets in the condition table - if culprits := (hyb_target_ids & condition_targets): - messages.append( - f"Hybridization target ids `{culprits}` are also " - "target ids in the condition table." - ) - # NN inputs are not used as target values - if culprits := (hyb_target_vals & nn_input_ids): - messages.append( - "The following neural net inputs were used as target values " - f"in the Hybridization table: `{culprits}`." - ) - - if messages: - return ValidationError("\n".join(messages)) - - return None - - def get_valid_parameters_for_parameter_table( problem: Problem, ) -> set[str]: @@ -1229,6 +1189,9 @@ def get_placeholders( CheckMappingTable(), ] +# Import SciML validation from sciml_lint at the end to avoid circular imports +from ..v2.extensions.sciml_lint import CheckHybridizationTable # noqa: E402 + #: Validation tasks that should be run PEtab SciML problems sciml_validation_tasks = default_validation_tasks + [ CheckHybridizationTable(), From 81148e6b439448e41f5800107014325fde64acff Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 23 Jun 2026 10:37:57 +0100 Subject: [PATCH 27/28] use pypi petab_sciml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ed228c04..d6dfccc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ maintainers = [ tests = [ "antimony>=3.1.0", "copasi-basico>=0.85", - "petab_sciml @ git+https://github.com/PEtab-dev/petab_sciml.git", + "petab_sciml", "pysb", "pytest", "pytest-cov", @@ -73,7 +73,7 @@ vis = [ "scipy" ] sciml = [ - "petab_sciml @ git+https://github.com/PEtab-dev/petab_sciml.git", + "petab_sciml", ] [project.scripts] From 7759b50bb45b906ae25b124b446d0293dce92446 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 23 Jun 2026 14:06:21 +0100 Subject: [PATCH 28/28] cherry-pick ci fix --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3f3bbe46..ffeb503c 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ description = extras = tests,reports,combine,vis deps= git+https://github.com/PEtab-dev/petab_test_suite@main - git+https://github.com/Benchmarking-Initiative/Benchmark-Models-PEtab.git@master\#subdirectory=src/python + -e git+https://github.com/Benchmarking-Initiative/Benchmark-Models-PEtab.git@master\#subdirectory=src/python&egg=benchmark_models_petab commands = python -m pip install sympy>=1.12.1