Save pulse shape parameters directly in json (and not in a table file)

docs/changes/2088.feature.md

@@ -0,0 +1,2 @@
+Added a unified parameter export flow for both file-backed and dict-backed table parameters, with stricter validation and normalization of table content including required `column_units` for `fadc_pulse_shape`.
+Plot table generation now selects the schema document by `model_parameter_schema_version` (with fallback), and sim_telarray/model/database handling was aligned with expanded unit test coverage.

docs/source/api-reference/data_model.md

@@ -67,3 +67,12 @@ Data products ingested or produced by simtools generally follows the CTAO data m
 .. automodule:: data_model.validate_data
    :members:
 ```
+
+## row_table_utils
+
+(row-table-utils-1)=
+
+```{eval-rst}
+.. automodule:: data_model.row_table_utils
+   :members:
+```

docs/source/api-reference/db_handler.md

@@ -30,3 +30,12 @@ Modules for database access. See the databases sections for details.
 .. automodule:: db.db_model_upload
    :members:
 ```
+
+## parameter_exporter
+
+(db-parameter-exporter)=
+
+```{eval-rst}
+.. automodule:: db.parameter_exporter
+   :members:
+```

docs/source/api-reference/sim_telarray.md

@@ -40,6 +40,15 @@ Support modules for running sim_telarray.
    :members:
 ```
 
+## simtel_table_writer
+
+(simtel-table-writer-1)=
+
+```{eval-rst}
+.. automodule:: simtel.simtel_table_writer
+   :members:
+```
+
 ## simtel_io_metadata
 
 (simtel-io-metadata-1)=

src/simtools/applications/db_get_file_from_db.py

@@ -61,15 +61,16 @@ def main():
     app_context = startup_application(_parse)
 
     db = db_handler.DatabaseHandler()
-    file_id = db.export_model_files(
-        dest=app_context.io_handler.get_output_directory(),
-        file_names=app_context.args["file_name"],
-    )
-    if file_id is None:
-        app_context.logger.error(
-            f"The file {app_context.args['file_name']} was not found in {db.db_name}."
+    try:
+        db.export_model_files(
+            dest=app_context.io_handler.get_output_directory(),
+            file_names=app_context.args["file_name"],
         )
-        raise FileNotFoundError
+    except FileNotFoundError as exc:
+        raise FileNotFoundError(
+            f"The file {app_context.args['file_name']} was not found in {db.db_name}."
+        ) from exc
+
     app_context.logger.info(
         f"Got file {app_context.args['file_name']} from DB {db.db_name} "
         f"and saved into {app_context.io_handler.get_output_directory()}"

src/simtools/applications/db_get_parameter_from_db.py

@@ -3,10 +3,19 @@
 r"""
     Get a parameter entry from DB for a specific telescope or a site.
 
-    The application receives a parameter name, a site, a telescope (if applicable) and
-    a version. Allow to print the parameter entry to screen or save it to a file.
-    Parameter describing a table file can be written to disk or exported as an astropy table
-    (if available).
+    The application supports three output modes:
+
+    1. Print the database entry to stdout.
+    2. Write the database entry to a JSON or YAML file using output_file.
+    3. Export table-type model parameters using export_model_file.
+
+    The export_model_file mode is type-dependent:
+
+    - File-backed parameters are exported with their original file name from the database.
+    - Dict-backed table parameters are exported as ECSV, using output_file as the base name.
+
+    For file-backed parameters, export_model_file_as_table can be added to also write an
+    ECSV representation next to the exported file.
 
     Command line arguments
     ----------------------
@@ -26,30 +35,41 @@
         Telescope model name (e.g. LST-1, SST-D, ...)
 
     output_file (str, optional)
-        Output file name. If not given, print to stdout.
+        Output file name for writing the database entry, or base file name for
+        exporting dict-backed tables as ECSV.
 
     export_model_file (bool, optional)
-        Export model file (if parameter describes a file).
+        Export parameter data. File-backed parameters are written as model files.
+        Embedded dict-typed table parameters are written as ECSV using output_file.
 
     export_model_file_as_table (bool, optional)
-        Export model file as astropy table (if parameter describes a file).
+        Export file-backed parameters as astropy tables in addition to the
+        original file export. Use together with export_model_file.
 
     Raises
     ------
     KeyError in case the parameter requested does not exist in the model parameters.
 
     Example
     -------
-    Get the mirror_list parameter used for a given model_version from the DB.
+    Print the mirror_list parameter entry used for a given model_version.
 
     .. code-block:: console
 
         simtools-db-get-parameter-from-db --parameter mirror_list \\
                 --site North --telescope LSTN-01 \\
                 --model_version 5.0.0
 
-    Get the mirror_list parameter using the parameter_version from the DB.
-    Write the mirror list to disk.
+    Write the database entry for a parameter to a JSON file.
+
+    .. code-block:: console
+
+        simtools-db-get-parameter-from-db --parameter array_element_position_ground \\
+                --site North --telescope LSTN-01 \\
+                --parameter_version 6.0.0 \\
+                --output_file array_element_position_ground.json
+
+    Export a file-backed parameter using the original file name stored in the database.
 
     .. code-block:: console
 
@@ -58,6 +78,24 @@
                 --parameter_version 1.0.0 \\
                 --export_model_file
 
+    Export a file-backed parameter and also write an ECSV table representation.
+
+    .. code-block:: console
+
+        simtools-db-get-parameter-from-db --parameter mirror_reflectivity \\
+                --site North --telescope LSTN-01 \\
+                --model_version 6.0.2 \\
+                --export_model_file --export_model_file_as_table
+
+    Export a dict-backed table parameter as ECSV. The .ecsv suffix is added automatically.
+
+    .. code-block:: console
+
+        simtools-db-get-parameter-from-db --parameter fadc_pulse_shape \\
+                --site North --telescope LSTN-01 \\
+                --parameter_version 2.0.0 \\
+                --export_model_file --output_file fadc_pulse_shape
+
 """
 
 from pprint import pprint
@@ -72,33 +110,39 @@ def _parse():
     """Parse command line configuration."""
     config = configurator.Configurator(
         label=get_application_label(__file__),
-        description=(
-            "Get a parameter entry from DB for a specific telescope or a site. "
-            "The application receives a parameter name, a site, a telescope (if applicable), "
-            "and a version. It then prints out the parameter entry. "
-        ),
+        description=("Export a parameter entry from model parameter database."),
     )
 
     config.parser.add_argument("--parameter", help="Parameter name", type=str, required=True)
     config.parser.add_argument(
         "--output_file",
-        help="output file name (if not given: print to stdout)",
+        help=(
+            "Output file name for writing the DB entry, or base name for ECSV export of "
+            "dict-backed tables."
+        ),
         type=str,
         required=False,
     )
     config.parser.add_argument(
         "--export_model_file",
-        help="Export model file (if parameter describes a file)",
+        help=(
+            "Export parameter data. File-backed parameters are written as files; "
+            "embedded dict-typed table parameters are written as ECSV using --output_file."
+        ),
         action="store_true",
         required=False,
     )
     config.parser.add_argument(
         "--export_model_file_as_table",
-        help="Export model file as astropy table (if parameter describes a file)",
+        help=(
+            "Also export file-backed parameters as ECSV. Use together with "
+            "--export_model_file. "
+            "(legacy option; as file-backed parameters will be replaced by table-backed ones, "
+            "this option will be removed in the future)"
+        ),
         action="store_true",
         required=False,
     )
-
     return config.initialize(
         db_config=True, simulation_model=["telescope", "parameter_version", "model_version"]
     )
@@ -110,30 +154,28 @@ def main():
 
     db = db_handler.DatabaseHandler()
 
+    if app_context.args["export_model_file"] or app_context.args["export_model_file_as_table"]:
+        output_files = db.export_parameter_data(
+            parameter=app_context.args["parameter"],
+            site=app_context.args["site"],
+            array_element_name=app_context.args.get("telescope"),
+            parameter_version=app_context.args.get("parameter_version"),
+            model_version=app_context.args.get("model_version"),
+            output_file=app_context.args.get("output_file"),
+            export_model_file=app_context.args["export_model_file"],
+            export_model_file_as_table=app_context.args["export_model_file_as_table"],
+        )
+        for output_file in output_files:
+            app_context.logger.info(f"Exported parameter output to {output_file}")
+        return
+
     pars = db.get_model_parameter(
         parameter=app_context.args["parameter"],
         site=app_context.args["site"],
         array_element_name=app_context.args.get("telescope"),
         parameter_version=app_context.args.get("parameter_version"),
         model_version=app_context.args.get("model_version"),
     )
-    if app_context.args["export_model_file"] or app_context.args["export_model_file_as_table"]:
-        table = db.export_model_file(
-            parameter=app_context.args["parameter"],
-            site=app_context.args["site"],
-            array_element_name=app_context.args["telescope"],
-            parameter_version=app_context.args.get("parameter_version"),
-            model_version=app_context.args.get("model_version"),
-            export_file_as_table=app_context.args["export_model_file_as_table"],
-        )
-        param_value = pars[app_context.args["parameter"]]["value"]
-        table_file = app_context.io_handler.get_output_file(param_value)
-        app_context.logger.info(f"Exported model file {param_value} to {table_file}")
-        if table and table_file.suffix != ".ecsv":
-            table.write(table_file.with_suffix(".ecsv"), format="ascii.ecsv", overwrite=True)
-            app_context.logger.info(
-                f"Exported model file {param_value} to {table_file.with_suffix('.ecsv')}"
-            )
 
     if app_context.args["output_file"] is not None:
         pars[app_context.args["parameter"]].pop("_id")

src/simtools/applications/submit_model_parameter_from_external.py

@@ -18,6 +18,8 @@
         site location.
     parameter_version (str)
         Parameter version.
+    model_parameter_schema_version (str, optional)
+        Version of the model-parameter schema to use for validation and value interpretation.
     input_meta (str, optional)
         input meta data file (yml format)
 
@@ -43,6 +45,7 @@
 import simtools.data_model.model_data_writer as writer
 from simtools.application_control import get_application_label, startup_application
 from simtools.configuration import configurator
+from simtools.simtel import simtel_table_reader
 
 
 def _parse():
@@ -60,6 +63,12 @@ def _parse():
     config.parser.add_argument(
         "--parameter_version", type=str, required=True, help="Parameter version"
     )
+    config.parser.add_argument(
+        "--model_parameter_schema_version",
+        type=str,
+        required=False,
+        help="Model-parameter schema version to use for validation and value interpretation",
+    )
     config.parser.add_argument(
         "--value",
         type=str,
@@ -88,6 +97,23 @@ def _parse():
 def main():
     """Submit and validate a model parameter value and metadata."""
     app_context = startup_application(_parse)
+    model_parameter_schema_version = app_context.args.get("model_parameter_schema_version")
+    value = app_context.args["value"]
+    data_writer = writer.ModelDataWriter()
+    parameter_type = data_writer.get_parameter_type_for_schema(
+        app_context.args["parameter"],
+        model_parameter_schema_version,
+    )
+
+    if parameter_type == "dict" and data_writer.parameter_uses_row_table_schema(
+        app_context.args["parameter"],
+        model_parameter_schema_version,
+    ):
+        value = simtel_table_reader.resolve_dict_parameter_value(
+            value,
+            app_context.args["parameter"],
+            app_context.args.get("data_path"),
+        )
 
     if app_context.args.get("output_path"):
         output_path = app_context.io_handler.get_output_directory(
@@ -98,7 +124,7 @@ def main():
 
     writer.ModelDataWriter.dump_model_parameter(
         parameter_name=app_context.args["parameter"],
-        value=app_context.args["value"],
+        value=value,
         instrument=app_context.args["instrument"],
         parameter_version=app_context.args["parameter_version"],
         output_file=Path(
@@ -107,6 +133,7 @@ def main():
         output_path=output_path,
         metadata_input_dict=app_context.args,
         check_db_for_existing_parameter=app_context.args.get("check_parameter_version", False),
+        model_parameter_schema_version=model_parameter_schema_version,
     )