Skip to content

Logging a metric with a trailing space causes get_metrics to throw "Exception: Malformed metric value" #1392

Open
@marcindulak

Description

@marcindulak

This issue is provided for reference purposes.

Tested with azureml-sdk version 1.22.0 on the jupyter notebook instance and the current default SKLearn backend.

Example:

from azureml.core import Workspace, Experiment, Run
from azureml.train.sklearn import SKLearn
from azureml.train.hyperdrive.sampling import RandomParameterSampling
from azureml.train.hyperdrive.parameter_expressions import choice
from azureml.train.hyperdrive.runconfig import HyperDriveConfig
from azureml.train.hyperdrive.run import PrimaryMetricGoal
from azureml.core.compute import ComputeTarget, AmlCompute

print(azureml.core.VERSION)

ws = Workspace.from_config()
experiment_name = 'test'
experiment = Experiment(ws, experiment_name)

compute_config = AmlCompute.provisioning_configuration(vm_size='Standard_D2_V2',
                                                       max_nodes=1)
compute_target = ComputeTarget.create(ws, "test", compute_config)
compute_target.wait_for_completion(show_output=True)

estimator = SKLearn(source_directory=".",
                    entry_script="train.py",
                    compute_target=compute_target)

param_sampling = RandomParameterSampling({"--dummy": choice(1)})

run_config = HyperDriveConfig(estimator=estimator,
                              hyperparameter_sampling=param_sampling,
                              policy=None,
                              primary_metric_name="Accuracy",
                              primary_metric_goal=PrimaryMetricGoal.MAXIMIZE,
                              max_total_runs=1,
                              max_concurrent_runs=1)

run = experiment.submit(config=run_config, show_output=True)

run.wait_for_completion(show_output=True)

best_run = run.get_best_run_by_primary_metric()
print('best_run.id: ', best_run.id)

best_run_metrics = best_run.get_metrics()
print("best_run.get_metrics():", best_run_metrics)

with the following train.py

import argparse

from azureml.core import Run

run = Run.get_context()

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('--dummy', type=int, default=1, help="Nothing to do")

    args = parser.parse_args()

    run.log("Metric ending with a space: ", 0.1)
    run.log("Accuracy", 1.0)

if __name__ == "__main__":
    main()

The exception is thrown

---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-12-70e7c0ca2c98> in <module>
----> 1 best_run_metrics = best_run.get_metrics()
      2 print("best_run.get_metrics():", best_run_metrics)

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/core/run.py in get_metrics(self, name, recursive, run_type, populate)
   1272         """
   1273         return self._client.get_metrics(name=name, recursive=recursive, run_type=run_type,
-> 1274                                         populate=populate, root_run_id=self._root_run_id)
   1275 
   1276     def _get_outputs_datapath(self):

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_run_impl/run_history_facade.py in get_metrics(self, name, recursive, run_type, populate, root_run_id, run_ids, use_batch)
    363             return self.metrics.get_all_metrics_v2(name=name, run_ids=run_ids, populate=populate,
    364                                                    artifact_client=self.artifacts,
--> 365                                                    data_container=self._data_container_id)
    366 
    367         return self.metrics.get_all_metrics(run_ids=run_ids, populate=populate, artifact_client=self.artifacts,

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/metrics_client.py in get_all_metrics_v2(self, name, run_ids, populate, artifact_client, data_container, after_timestamp, custom_headers)
    311         return self.get_metrics_for_run_ids_v2(name=name, run_ids=run_ids, start_time=after_timestamp,
    312                                                populate=populate, artifact_client=artifact_client,
--> 313                                                data_container=data_container, custom_headers=custom_headers)
    314 
    315     def get_all_metrics(self, run_ids=None, populate=False, artifact_client=None,

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/metrics_client.py in get_metrics_for_run_ids_v2(self, name, run_ids, start_time, end_time, populate, artifact_client, data_container, custom_headers)
    298                                                   end_time=end_time, populate=populate,
    299                                                   artifact_client=artifact_client, data_container=data_container,
--> 300                                                   custom_headers=custom_headers)
    301             if len(metrics) != 0:
    302                 returned_metrics_for_runs[run_id] = metrics

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/metrics_client.py in get_metrics_for_run_v2(self, run_id, name, start_time, end_time, populate, artifact_client, data_container, custom_headers)
    284                 metric = self.get_metric_for_run_v2(metric_name, run_id=run_id, start_time=start_time,
    285                                                     end_time=end_time, artifact_client=artifact_client,
--> 286                                                     populate=populate, custom_headers=custom_headers)
    287                 if metric is not None:
    288                     returned_metrics[metric_name] = metric

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/metrics_client.py in get_metric_for_run_v2(self, name, run_id, start_time, end_time, artifact_client, populate, custom_headers)
    263         if metric_dto is None or len(metric_dto.value) == 0:
    264             return None
--> 265         return MetricsClient.dto_v2_to_metric_cells(metric_dto, artifact_client, populate)
    266 
    267     def get_metrics_for_run_v2(self, run_id=None, name=None, start_time=None, end_time=None, populate=False,

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/_restclient/metrics_client.py in dto_v2_to_metric_cells(cls, metric_dto, artifact_client, populate)
    115         metric_type = metric_dto.properties.ux_metric_type
    116         if metric_type in INLINE_METRICS:
--> 117             return InlineMetric.get_cells_from_metric_v2_dto(metric_dto)
    118 
    119         if populate and artifact_client is None:

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/core/_metrics.py in get_cells_from_metric_v2_dto(metric_dto)
    415             values = []
    416             for metric_value in metric_dto.value:
--> 417                 cell = InlineMetric.get_cell_v2(columns, metric_value, metric_dto.name)
    418                 values.append(cell)
    419             if len(values) == 1:

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/azureml/core/_metrics.py in get_cell_v2(columns, metric_v2_value, column_name)
    424     def get_cell_v2(columns, metric_v2_value, column_name):
    425         if column_name not in columns.keys() or column_name not in metric_v2_value.data.keys():
--> 426             raise Exception("Malformed metric value")
    427         cell_type = str.lower(columns[column_name])
    428         value = metric_v2_value.data[column_name]

Exception: Malformed metric value

Document Details

Do not edit this section. It is required for docs.microsoft.com ➟ GitHub issue linking.

Metadata

Metadata

Assignees

No one assigned

    Labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions