.. DO NOT EDIT.
.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
.. "auto_revalidation/re_orientation/_01_reorientation_analysis.py"
.. LINE NUMBERS ARE GIVEN BELOW.

.. only:: html

    .. note::
        :class: sphx-glr-download-link-note

        :ref:`Go to the end <sphx_glr_download_auto_revalidation_re_orientation__01_reorientation_analysis.py>`
        to download the full example code.

.. rst-class:: sphx-glr-example-title

.. _sphx_glr_auto_revalidation_re_orientation__01_reorientation_analysis.py:


.. _reorientation_val_results:

Performance of the reorientation algorithm on simulated TVS misorientations
===========================================================================

The TVS dataset does not provide recordings with known lower-back sensor
misorientations. For this revalidation, we therefore use the INDIP reference
walking bouts, simulate all supported rough mounting orientations, and treat the
algorithm response as a multiclass classification problem.

We compare the full and trust-gravity variants of the
:class:`~mobgap.re_orientation.ReorientationMethodDM` algorithm.

.. note:: If you are interested in how these results are calculated, head over to
    the :ref:`processing page <reorientation_val_gen>`.

.. GENERATED FROM PYTHON SOURCE LINES 21-25

Algorithms
----------
The result generation script stores one result folder per algorithm variant.
Here, we map these folder names to display labels used in plots and tables.

.. GENERATED FROM PYTHON SOURCE LINES 25-30

.. code-block:: Python

    algorithms = {
        "MethodDM__full": ("ReorientationMethodDM", "Full"),
        "MethodDM__trust_gravity": ("ReorientationMethodDM", "Trust gravity"),
    }








.. GENERATED FROM PYTHON SOURCE LINES 31-36

Loading Results
---------------
By default, the data will be downloaded from the validation result repository.
During development, set `MOBGAP_VALIDATION_USE_LOCAL_DATA=1` and point
`MOBGAP_VALIDATION_DATA_PATH` to the local validation-data folder.

.. GENERATED FROM PYTHON SOURCE LINES 36-161

.. code-block:: Python

    from pathlib import Path

    import matplotlib.pyplot as plt
    import numpy as np
    import pandas as pd
    import seaborn as sns
    from mobgap.data.validation_results import ValidationResultLoader
    from mobgap.re_orientation.pipeline import REORIENTATION_LABELS
    from mobgap.utils.misc import get_env_var

    IDENTITY_LABEL = "identity"
    UNCORRECTABLE_TRUST_GRAVITY_LABEL = "pa_flipped__rot_pa_0"
    FULL_VERSION = "Full"
    TRUST_GRAVITY_VERSION = "Trust gravity"


    def format_loaded_results(
        values: dict[tuple[str, str], pd.DataFrame],
        index_cols: list[str],
    ) -> pd.DataFrame:
        formatted = (
            pd.concat(values, names=["algo", "version", *index_cols])
            .reset_index()
            .assign(
                algo_with_version=lambda df: (
                    df["algo"] + " (" + df["version"] + ")"
                ),
                _combined="combined",
            )
        )
        return formatted


    def load_raw_predictions(
        loader: ValidationResultLoader,
        algo_name: str,
        condition: str,
    ) -> pd.DataFrame:
        return loader.load_single_csv_file(
            algo_name, condition, "raw_predictions.csv"
        ).reset_index()


    def format_loaded_predictions(
        values: dict[tuple[str, str], pd.DataFrame],
    ) -> pd.DataFrame:
        formatted_predictions = []
        for (algo, version), df in values.items():
            formatted_predictions.append(
                df.assign(
                    algo=algo,
                    version=version,
                    algo_with_version=f"{algo} ({version})",
                    is_correct=lambda data: data["label"] == data["prediction"],
                )
            )

        return pd.concat(
            formatted_predictions,
            ignore_index=True,
        )


    local_data_path = (
        Path(get_env_var("MOBGAP_VALIDATION_DATA_PATH")) / "results"
        if int(get_env_var("MOBGAP_VALIDATION_USE_LOCAL_DATA", 0))
        else None
    )
    __RESULT_VERSION = "main"
    loader = ValidationResultLoader(
        "re_orientation", result_path=local_data_path, version=__RESULT_VERSION
    )

    free_living_index_cols = [
        "cohort",
        "participant_id",
        "time_measure",
        "recording",
        "recording_name",
        "recording_name_pretty",
    ]

    free_living_results = format_loaded_results(
        {
            v: loader.load_single_results(k, "free_living")
            for k, v in algorithms.items()
        },
        free_living_index_cols,
    )
    free_living_predictions = format_loaded_predictions(
        {
            v: load_raw_predictions(loader, k, "free_living")
            for k, v in algorithms.items()
        }
    )

    lab_index_cols = [
        "cohort",
        "participant_id",
        "time_measure",
        "test",
        "trial",
        "test_name",
        "test_name_pretty",
    ]

    lab_results = format_loaded_results(
        {
            v: loader.load_single_results(k, "laboratory")
            for k, v in algorithms.items()
        },
        lab_index_cols,
    )
    lab_predictions = format_loaded_predictions(
        {
            v: load_raw_predictions(loader, k, "laboratory")
            for k, v in algorithms.items()
        }
    )
    combined_predictions = pd.concat(
        [free_living_predictions, lab_predictions], ignore_index=True
    )

    cohort_order = ["HA", "CHF", "COPD", "MS", "PD", "PFF"]





.. rst-class:: sphx-glr-script-out

 .. code-block:: none


      0%|                                              | 0.00/1.02k [00:00<?, ?B/s]
      0%|                                              | 0.00/1.02k [00:00<?, ?B/s]
    100%|█████████████████████████████████████| 1.02k/1.02k [00:00<00:00, 4.67MB/s]

      0%|                                              | 0.00/1.02k [00:00<?, ?B/s]
      0%|                                              | 0.00/1.02k [00:00<?, ?B/s]
    100%|█████████████████████████████████████| 1.02k/1.02k [00:00<00:00, 6.05MB/s]

      0%|                                               | 0.00/136k [00:00<?, ?B/s]
      0%|                                               | 0.00/136k [00:00<?, ?B/s]
    100%|████████████████████████████████████████| 136k/136k [00:00<00:00, 650MB/s]

      0%|                                               | 0.00/135k [00:00<?, ?B/s]
      0%|                                               | 0.00/135k [00:00<?, ?B/s]
    100%|████████████████████████████████████████| 135k/135k [00:00<00:00, 590MB/s]

      0%|                                              | 0.00/4.73k [00:00<?, ?B/s]
      0%|                                              | 0.00/4.73k [00:00<?, ?B/s]
    100%|█████████████████████████████████████| 4.73k/4.73k [00:00<00:00, 28.1MB/s]

      0%|                                              | 0.00/4.70k [00:00<?, ?B/s]
      0%|                                              | 0.00/4.70k [00:00<?, ?B/s]
    100%|█████████████████████████████████████| 4.70k/4.70k [00:00<00:00, 27.1MB/s]

      0%|                                              | 0.00/41.6k [00:00<?, ?B/s]
      0%|                                              | 0.00/41.6k [00:00<?, ?B/s]
    100%|██████████████████████████████████████| 41.6k/41.6k [00:00<00:00, 206MB/s]

      0%|                                              | 0.00/41.1k [00:00<?, ?B/s]
      0%|                                              | 0.00/41.1k [00:00<?, ?B/s]
    100%|██████████████████████████████████████| 41.1k/41.1k [00:00<00:00, 230MB/s]




.. GENERATED FROM PYTHON SOURCE LINES 162-170

Performance Metrics
-------------------
We report two accuracy views:

- The average per-recording accuracy. Every datapoint contributes one value.
- The combined accuracy over every simulated walking-bout orientation row.

The confusion matrices use the combined predictions.

.. GENERATED FROM PYTHON SOURCE LINES 170-394

.. code-block:: Python

    from functools import partial

    from mobgap.pipeline.evaluation import CustomErrorAggregations as A
    from mobgap.utils.df_operations import (
        CustomOperation,
        apply_aggregations,
        apply_transformations,
    )
    from mobgap.utils.tables import FormatTransformer as F
    from mobgap.utils.tables import RevalidationInfo, revalidation_table_styles

    custom_aggs = [
        CustomOperation(
            identifier=None,
            function=A.n_datapoints,
            column_name=[("n_datapoints", "all")],
        ),
        ("accuracy", ["mean", A.conf_intervals]),
    ]

    format_transforms = [
        CustomOperation(
            identifier=None,
            function=lambda df_: df_[("n_datapoints", "all")].astype(int),
            column_name="n_datapoints",
        ),
        CustomOperation(
            identifier=None,
            function=partial(
                F.value_with_metadata,
                value_col=("mean", "accuracy"),
                other_columns={"range": ("conf_intervals", "accuracy")},
            ),
            column_name="accuracy",
        ),
    ]


    def validation_thresholds(datapoint_label: str) -> dict[str, RevalidationInfo]:
        return {
            f"Accuracy per {datapoint_label}": RevalidationInfo(
                threshold=0.8, higher_is_better=True
            ),
            "Combined accuracy": RevalidationInfo(
                threshold=0.8, higher_is_better=True
            ),
        }


    def calculate_combined_accuracy(
        predictions: pd.DataFrame,
        groupby: list[str],
    ) -> pd.Series:
        return (
            predictions.groupby(groupby)["is_correct"]
            .mean()
            .rename("combined_accuracy")
        )


    def format_tables(
        single_results: pd.DataFrame,
        raw_predictions: pd.DataFrame,
        groupby: list[str],
        datapoint_label: str,
    ) -> pd.DataFrame:
        final_names = {
            "n_datapoints": f"# {datapoint_label}s",
            "accuracy": f"Accuracy per {datapoint_label}",
            "combined_accuracy": "Combined accuracy",
        }
        formatted_single_results = (
            single_results.groupby(groupby)
            .apply(apply_aggregations, custom_aggs, include_groups=False)
            .pipe(apply_transformations, format_transforms)
            .rename(columns=final_names)
            .loc[:, [final_names["n_datapoints"], final_names["accuracy"]]]
        )
        combined_accuracy = calculate_combined_accuracy(raw_predictions, groupby)
        formatted_single_results["Combined accuracy"] = combined_accuracy
        return formatted_single_results.loc[:, list(final_names.values())]


    def calculate_confusion_matrix(predictions: pd.DataFrame) -> pd.DataFrame:
        known_labels = list(REORIENTATION_LABELS)
        extra_labels = sorted(
            set(predictions["label"]).union(predictions["prediction"])
            - set(known_labels)
        )
        labels = [*known_labels, *extra_labels]
        matrix = pd.crosstab(predictions["label"], predictions["prediction"])
        return matrix.reindex(index=labels, columns=labels, fill_value=0)


    def calculate_label_accuracies(
        predictions: pd.DataFrame,
        groupby: list[str],
    ) -> pd.DataFrame:
        return predictions.pivot_table(
            index=groupby,
            columns="label",
            values="is_correct",
            aggfunc="mean",
        ).reindex(columns=REORIENTATION_LABELS)


    def orientation_prevalence_weights(
        total_misorientation_prevalence: float,
    ) -> pd.Series:
        weights = pd.Series(
            total_misorientation_prevalence / (len(REORIENTATION_LABELS) - 1),
            index=REORIENTATION_LABELS,
        )
        weights[IDENTITY_LABEL] = 1 - total_misorientation_prevalence
        return weights


    def calculate_weighted_accuracy_by_prevalence(
        predictions: pd.DataFrame,
        prevalence_scenarios: pd.Series,
    ) -> pd.DataFrame:
        label_accuracies = calculate_label_accuracies(
            predictions, ["algo", "version"]
        )
        return pd.DataFrame(
            {
                scenario: label_accuracies.mul(
                    orientation_prevalence_weights(prevalence),
                    axis=1,
                ).sum(axis=1)
                for scenario, prevalence in prevalence_scenarios.items()
            }
        )


    def calculate_weighted_accuracy_curve(
        predictions: pd.DataFrame,
        prevalence_grid: np.ndarray,
    ) -> pd.DataFrame:
        label_accuracies = calculate_label_accuracies(predictions, ["version"])
        rows = []
        for version, accuracies in label_accuracies.iterrows():
            for prevalence in prevalence_grid:
                rows.append(
                    {
                        "version": version,
                        "total_misorientation_prevalence": prevalence,
                        "weighted_accuracy": accuracies.mul(
                            orientation_prevalence_weights(prevalence)
                        ).sum(),
                    }
                )
        return pd.DataFrame(rows)


    def _break_even_prevalence(identity_diff: float, error_diff: float) -> float:
        denominator = error_diff - identity_diff
        if np.isclose(denominator, 0):
            return np.nan

        break_even = -identity_diff / denominator
        if 0 <= break_even <= 1:
            return break_even
        return np.nan


    def calculate_mode_break_even_points(predictions: pd.DataFrame) -> pd.Series:
        label_accuracies = calculate_label_accuracies(predictions, ["version"])
        diff = (
            label_accuracies.loc[TRUST_GRAVITY_VERSION]
            - label_accuracies.loc[FULL_VERSION]
        )

        return pd.Series(
            {
                "Total error prevalence": _break_even_prevalence(
                    diff[IDENTITY_LABEL],
                    diff.drop(index=IDENTITY_LABEL).mean(),
                ),
                "Specific PA-flip prevalence": _break_even_prevalence(
                    diff[IDENTITY_LABEL],
                    diff[UNCORRECTABLE_TRUST_GRAVITY_LABEL],
                ),
            }
        )


    def calculate_mode_break_even_inputs(predictions: pd.DataFrame) -> pd.DataFrame:
        label_accuracies = calculate_label_accuracies(predictions, ["version"])
        non_identity_labels = [
            label for label in REORIENTATION_LABELS if label != IDENTITY_LABEL
        ]
        input_labels = {
            "Identity orientation": IDENTITY_LABEL,
            "Mean non-identity orientation": non_identity_labels,
            "Uncorrectable PA-flip orientation": (
                UNCORRECTABLE_TRUST_GRAVITY_LABEL
            ),
        }

        rows = []
        for name, label_or_labels in input_labels.items():
            labels = (
                [label_or_labels]
                if isinstance(label_or_labels, str)
                else label_or_labels
            )
            full_accuracy = label_accuracies.loc[FULL_VERSION, labels].mean()
            trust_gravity_accuracy = label_accuracies.loc[
                TRUST_GRAVITY_VERSION, labels
            ].mean()
            rows.append(
                {
                    "Input": name,
                    "Full accuracy": full_accuracy,
                    "Trust gravity accuracy": trust_gravity_accuracy,
                    "Trust gravity - full": (
                        trust_gravity_accuracy - full_accuracy
                    ),
                }
            )
        return pd.DataFrame(rows).set_index("Input")









.. GENERATED FROM PYTHON SOURCE LINES 395-399

Free-Living Comparison
----------------------
The free-living condition is the expected use case for unknown sensor mounting
orientations.

.. GENERATED FROM PYTHON SOURCE LINES 399-420

.. code-block:: Python

    fig, ax = plt.subplots()
    sns.boxplot(
        data=free_living_results,
        x="algo_with_version",
        y="accuracy",
        ax=ax,
    )
    plt.xticks(rotation=45, ha="right")
    fig.tight_layout()
    fig.show()

    free_living_perf_metrics_all = format_tables(
        free_living_results,
        free_living_predictions,
        ["algo", "version"],
        "recording",
    )
    free_living_perf_metrics_all.style.pipe(
        revalidation_table_styles, validation_thresholds("recording"), ["algo"]
    )




.. image-sg:: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_001.png
   :alt:  01 reorientation analysis
   :srcset: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_001.png
   :class: sphx-glr-single-img



.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <style type="text/css">
    #T_67351_row0_col1, #T_67351_row0_col2 {
      font-weight: bold;
      background-color: lightgreen;
    }
    #T_67351_row1_col0 {
      border-bottom: 2px solid black;
    }
    #T_67351_row1_col1, #T_67351_row1_col2 {
      background-color: lightgreen;
      border-bottom: 2px solid black;
    }
    </style>
    <table id="T_67351" class="dataframe">
      <thead>
        <tr>
          <th class="blank" >&nbsp;</th>
          <th class="blank level0" >&nbsp;</th>
          <th id="T_67351_level0_col0" class="col_heading level0 col0" ># recordings</th>
          <th id="T_67351_level0_col1" class="col_heading level0 col1" >Accuracy per recording</th>
          <th id="T_67351_level0_col2" class="col_heading level0 col2" >Combined accuracy</th>
        </tr>
        <tr>
          <th class="index_name level0" >algo</th>
          <th class="index_name level1" >version</th>
          <th class="blank col0" >&nbsp;</th>
          <th class="blank col1" >&nbsp;</th>
          <th class="blank col2" >&nbsp;</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th id="T_67351_level0_row0" class="row_heading level0 row0" rowspan="2">ReorientationMethodDM</th>
          <th id="T_67351_level1_row0" class="row_heading level1 row0" >Full</th>
          <td id="T_67351_row0_col0" class="data row0 col0" >101</td>
          <td id="T_67351_row0_col1" class="data row0 col1" ><span>0.92 [0.89, 0.94]</span></td>
          <td id="T_67351_row0_col2" class="data row0 col2" >0.9262472885032538</td>
        </tr>
        <tr>
          <th id="T_67351_level1_row1" class="row_heading level1 row1" >Trust gravity</th>
          <td id="T_67351_row1_col0" class="data row1 col0" >101</td>
          <td id="T_67351_row1_col1" class="data row1 col1" ><span>0.81 [0.79, 0.83]</span></td>
          <td id="T_67351_row1_col2" class="data row1 col2" >0.8195375665549202</td>
        </tr>
      </tbody>
    </table>

    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 421-423

Per Cohort
~~~~~~~~~~

.. GENERATED FROM PYTHON SOURCE LINES 423-447

.. code-block:: Python

    fig, ax = plt.subplots()
    sns.boxplot(
        data=free_living_results,
        x="cohort",
        y="accuracy",
        hue="algo_with_version",
        order=cohort_order,
        ax=ax,
    )
    ax.set_title("Free-living accuracy per recording")
    fig.show()

    free_living_perf_metrics_cohort = format_tables(
        free_living_results,
        free_living_predictions,
        ["cohort", "algo", "version"],
        "recording",
    ).loc[cohort_order]
    free_living_perf_metrics_cohort.style.pipe(
        revalidation_table_styles,
        validation_thresholds("recording"),
        ["cohort", "algo"],
    )




.. image-sg:: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_002.png
   :alt: Free-living accuracy per recording
   :srcset: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_002.png
   :class: sphx-glr-single-img



.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <style type="text/css">
    #T_e1659_row0_col1, #T_e1659_row0_col2, #T_e1659_row2_col1, #T_e1659_row2_col2, #T_e1659_row4_col1, #T_e1659_row4_col2, #T_e1659_row6_col1, #T_e1659_row6_col2, #T_e1659_row8_col1, #T_e1659_row8_col2, #T_e1659_row10_col1, #T_e1659_row10_col2 {
      font-weight: bold;
      background-color: lightgreen;
    }
    #T_e1659_row1_col0, #T_e1659_row3_col0, #T_e1659_row5_col0, #T_e1659_row7_col0, #T_e1659_row9_col0, #T_e1659_row11_col0 {
      border-bottom: 2px solid black;
    }
    #T_e1659_row1_col1, #T_e1659_row1_col2, #T_e1659_row5_col1, #T_e1659_row5_col2, #T_e1659_row7_col1, #T_e1659_row7_col2, #T_e1659_row9_col2, #T_e1659_row11_col1, #T_e1659_row11_col2 {
      background-color: lightgreen;
      border-bottom: 2px solid black;
    }
    #T_e1659_row3_col1, #T_e1659_row3_col2, #T_e1659_row9_col1 {
      background-color: lightcoral;
      border-bottom: 2px solid black;
    }
    </style>
    <table id="T_e1659" class="dataframe">
      <thead>
        <tr>
          <th class="blank" >&nbsp;</th>
          <th class="blank" >&nbsp;</th>
          <th class="blank level0" >&nbsp;</th>
          <th id="T_e1659_level0_col0" class="col_heading level0 col0" ># recordings</th>
          <th id="T_e1659_level0_col1" class="col_heading level0 col1" >Accuracy per recording</th>
          <th id="T_e1659_level0_col2" class="col_heading level0 col2" >Combined accuracy</th>
        </tr>
        <tr>
          <th class="index_name level0" >cohort</th>
          <th class="index_name level1" >algo</th>
          <th class="index_name level2" >version</th>
          <th class="blank col0" >&nbsp;</th>
          <th class="blank col1" >&nbsp;</th>
          <th class="blank col2" >&nbsp;</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th id="T_e1659_level0_row0" class="row_heading level0 row0" rowspan="2">HA</th>
          <th id="T_e1659_level1_row0" class="row_heading level1 row0" rowspan="2">ReorientationMethodDM</th>
          <th id="T_e1659_level2_row0" class="row_heading level2 row0" >Full</th>
          <td id="T_e1659_row0_col0" class="data row0 col0" >20</td>
          <td id="T_e1659_row0_col1" class="data row0 col1" ><span>0.94 [0.92, 0.97]</span></td>
          <td id="T_e1659_row0_col2" class="data row0 col2" >0.9458762886597938</td>
        </tr>
        <tr>
          <th id="T_e1659_level2_row1" class="row_heading level2 row1" >Trust gravity</th>
          <td id="T_e1659_row1_col0" class="data row1 col0" >20</td>
          <td id="T_e1659_row1_col1" class="data row1 col1" ><span>0.83 [0.81, 0.85]</span></td>
          <td id="T_e1659_row1_col2" class="data row1 col2" >0.8344072164948454</td>
        </tr>
        <tr>
          <th id="T_e1659_level0_row2" class="row_heading level0 row2" rowspan="2">CHF</th>
          <th id="T_e1659_level1_row2" class="row_heading level1 row2" rowspan="2">ReorientationMethodDM</th>
          <th id="T_e1659_level2_row2" class="row_heading level2 row2" >Full</th>
          <td id="T_e1659_row2_col0" class="data row2 col0" >10</td>
          <td id="T_e1659_row2_col1" class="data row2 col1" ><span>0.89 [0.78, 1.00]</span></td>
          <td id="T_e1659_row2_col2" class="data row2 col2" >0.8286516853932584</td>
        </tr>
        <tr>
          <th id="T_e1659_level2_row3" class="row_heading level2 row3" >Trust gravity</th>
          <td id="T_e1659_row3_col0" class="data row3 col0" >10</td>
          <td id="T_e1659_row3_col1" class="data row3 col1" ><span>0.79 [0.71, 0.87]</span></td>
          <td id="T_e1659_row3_col2" class="data row3 col2" >0.7457865168539326</td>
        </tr>
        <tr>
          <th id="T_e1659_level0_row4" class="row_heading level0 row4" rowspan="2">COPD</th>
          <th id="T_e1659_level1_row4" class="row_heading level1 row4" rowspan="2">ReorientationMethodDM</th>
          <th id="T_e1659_level2_row4" class="row_heading level2 row4" >Full</th>
          <td id="T_e1659_row4_col0" class="data row4 col0" >17</td>
          <td id="T_e1659_row4_col1" class="data row4 col1" ><span>0.93 [0.87, 0.98]</span></td>
          <td id="T_e1659_row4_col2" class="data row4 col2" >0.9209065679925994</td>
        </tr>
        <tr>
          <th id="T_e1659_level2_row5" class="row_heading level2 row5" >Trust gravity</th>
          <td id="T_e1659_row5_col0" class="data row5 col0" >17</td>
          <td id="T_e1659_row5_col1" class="data row5 col1" ><span>0.82 [0.78, 0.86]</span></td>
          <td id="T_e1659_row5_col2" class="data row5 col2" >0.8156799259944496</td>
        </tr>
        <tr>
          <th id="T_e1659_level0_row6" class="row_heading level0 row6" rowspan="2">MS</th>
          <th id="T_e1659_level1_row6" class="row_heading level1 row6" rowspan="2">ReorientationMethodDM</th>
          <th id="T_e1659_level2_row6" class="row_heading level2 row6" >Full</th>
          <td id="T_e1659_row6_col0" class="data row6 col0" >18</td>
          <td id="T_e1659_row6_col1" class="data row6 col1" ><span>0.95 [0.92, 0.97]</span></td>
          <td id="T_e1659_row6_col2" class="data row6 col2" >0.9544270833333334</td>
        </tr>
        <tr>
          <th id="T_e1659_level2_row7" class="row_heading level2 row7" >Trust gravity</th>
          <td id="T_e1659_row7_col0" class="data row7 col0" >18</td>
          <td id="T_e1659_row7_col1" class="data row7 col1" ><span>0.83 [0.81, 0.85]</span></td>
          <td id="T_e1659_row7_col2" class="data row7 col2" >0.8406575520833334</td>
        </tr>
        <tr>
          <th id="T_e1659_level0_row8" class="row_heading level0 row8" rowspan="2">PD</th>
          <th id="T_e1659_level1_row8" class="row_heading level1 row8" rowspan="2">ReorientationMethodDM</th>
          <th id="T_e1659_level2_row8" class="row_heading level2 row8" >Full</th>
          <td id="T_e1659_row8_col0" class="data row8 col0" >19</td>
          <td id="T_e1659_row8_col1" class="data row8 col1" ><span>0.87 [0.76, 0.97]</span></td>
          <td id="T_e1659_row8_col2" class="data row8 col2" >0.9247851002865329</td>
        </tr>
        <tr>
          <th id="T_e1659_level2_row9" class="row_heading level2 row9" >Trust gravity</th>
          <td id="T_e1659_row9_col0" class="data row9 col0" >19</td>
          <td id="T_e1659_row9_col1" class="data row9 col1" ><span>0.77 [0.70, 0.85]</span></td>
          <td id="T_e1659_row9_col2" class="data row9 col2" >0.8185888252148997</td>
        </tr>
        <tr>
          <th id="T_e1659_level0_row10" class="row_heading level0 row10" rowspan="2">PFF</th>
          <th id="T_e1659_level1_row10" class="row_heading level1 row10" rowspan="2">ReorientationMethodDM</th>
          <th id="T_e1659_level2_row10" class="row_heading level2 row10" >Full</th>
          <td id="T_e1659_row10_col0" class="data row10 col0" >17</td>
          <td id="T_e1659_row10_col1" class="data row10 col1" ><span>0.92 [0.87, 0.97]</span></td>
          <td id="T_e1659_row10_col2" class="data row10 col2" >0.9430379746835443</td>
        </tr>
        <tr>
          <th id="T_e1659_level2_row11" class="row_heading level2 row11" >Trust gravity</th>
          <td id="T_e1659_row11_col0" class="data row11 col0" >17</td>
          <td id="T_e1659_row11_col1" class="data row11 col1" ><span>0.82 [0.78, 0.85]</span></td>
          <td id="T_e1659_row11_col2" class="data row11 col2" >0.8318829113924051</td>
        </tr>
      </tbody>
    </table>

    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 448-450

Confusion Matrices
~~~~~~~~~~~~~~~~~~

.. GENERATED FROM PYTHON SOURCE LINES 450-475

.. code-block:: Python

    fig, axes = plt.subplots(
        1,
        len(algorithms),
        figsize=(6 * len(algorithms), 5),
        constrained_layout=True,
    )
    if len(algorithms) == 1:
        axes = [axes]

    for ax, ((algo, version), data) in zip(
        axes, free_living_predictions.groupby(["algo", "version"], sort=False)
    ):
        sns.heatmap(
            calculate_confusion_matrix(data),
            annot=True,
            fmt="d",
            cmap="Blues",
            cbar=False,
            ax=ax,
        )
        ax.set_title(f"{algo} ({version})")

    fig.suptitle("Free-living confusion matrices")
    fig.show()




.. image-sg:: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_003.png
   :alt: Free-living confusion matrices, ReorientationMethodDM (Full), ReorientationMethodDM (Trust gravity)
   :srcset: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_003.png
   :class: sphx-glr-single-img





.. GENERATED FROM PYTHON SOURCE LINES 476-480

Laboratory Comparison
---------------------
Every datapoint below is one lab trial. The simulated orientation rows are
created within every reference walking bout of that trial.

.. GENERATED FROM PYTHON SOURCE LINES 480-493

.. code-block:: Python

    fig, ax = plt.subplots()
    sns.boxplot(data=lab_results, x="algo_with_version", y="accuracy", ax=ax)
    plt.xticks(rotation=45, ha="right")
    fig.tight_layout()
    fig.show()

    lab_perf_metrics_all = format_tables(
        lab_results, lab_predictions, ["algo", "version"], "trial"
    )
    lab_perf_metrics_all.style.pipe(
        revalidation_table_styles, validation_thresholds("trial"), ["algo"]
    )




.. image-sg:: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_004.png
   :alt:  01 reorientation analysis
   :srcset: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_004.png
   :class: sphx-glr-single-img



.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <style type="text/css">
    #T_02974_row0_col1, #T_02974_row0_col2 {
      font-weight: bold;
      background-color: lightgreen;
    }
    #T_02974_row1_col0 {
      border-bottom: 2px solid black;
    }
    #T_02974_row1_col1, #T_02974_row1_col2 {
      background-color: lightgreen;
      border-bottom: 2px solid black;
    }
    </style>
    <table id="T_02974" class="dataframe">
      <thead>
        <tr>
          <th class="blank" >&nbsp;</th>
          <th class="blank level0" >&nbsp;</th>
          <th id="T_02974_level0_col0" class="col_heading level0 col0" ># trials</th>
          <th id="T_02974_level0_col1" class="col_heading level0 col1" >Accuracy per trial</th>
          <th id="T_02974_level0_col2" class="col_heading level0 col2" >Combined accuracy</th>
        </tr>
        <tr>
          <th class="index_name level0" >algo</th>
          <th class="index_name level1" >version</th>
          <th class="blank col0" >&nbsp;</th>
          <th class="blank col1" >&nbsp;</th>
          <th class="blank col2" >&nbsp;</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th id="T_02974_level0_row0" class="row_heading level0 row0" rowspan="2">ReorientationMethodDM</th>
          <th id="T_02974_level1_row0" class="row_heading level1 row0" >Full</th>
          <td id="T_02974_row0_col0" class="data row0 col0" >1168</td>
          <td id="T_02974_row0_col1" class="data row0 col1" ><span>0.90 [0.89, 0.92]</span></td>
          <td id="T_02974_row0_col2" class="data row0 col2" >0.9003533568904594</td>
        </tr>
        <tr>
          <th id="T_02974_level1_row1" class="row_heading level1 row1" >Trust gravity</th>
          <td id="T_02974_row1_col0" class="data row1 col0" >1168</td>
          <td id="T_02974_row1_col1" class="data row1 col1" ><span>0.80 [0.79, 0.82]</span></td>
          <td id="T_02974_row1_col2" class="data row1 col2" >0.8001766784452297</td>
        </tr>
      </tbody>
    </table>

    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 494-496

Per Cohort
~~~~~~~~~~

.. GENERATED FROM PYTHON SOURCE LINES 496-520

.. code-block:: Python

    fig, ax = plt.subplots()
    sns.boxplot(
        data=lab_results,
        x="cohort",
        y="accuracy",
        hue="algo_with_version",
        order=cohort_order,
        ax=ax,
    )
    ax.set_title("Laboratory accuracy per trial")
    fig.show()

    lab_perf_metrics_cohort = format_tables(
        lab_results,
        lab_predictions,
        ["cohort", "algo", "version"],
        "trial",
    ).loc[cohort_order]
    lab_perf_metrics_cohort.style.pipe(
        revalidation_table_styles,
        validation_thresholds("trial"),
        ["cohort", "algo"],
    )




.. image-sg:: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_005.png
   :alt: Laboratory accuracy per trial
   :srcset: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_005.png
   :class: sphx-glr-single-img



.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <style type="text/css">
    #T_592f0_row0_col1, #T_592f0_row0_col2, #T_592f0_row2_col1, #T_592f0_row2_col2, #T_592f0_row4_col1, #T_592f0_row4_col2, #T_592f0_row6_col1, #T_592f0_row6_col2, #T_592f0_row8_col1, #T_592f0_row8_col2, #T_592f0_row10_col1, #T_592f0_row10_col2 {
      font-weight: bold;
      background-color: lightgreen;
    }
    #T_592f0_row1_col0, #T_592f0_row3_col0, #T_592f0_row5_col0, #T_592f0_row7_col0, #T_592f0_row9_col0, #T_592f0_row11_col0 {
      border-bottom: 2px solid black;
    }
    #T_592f0_row1_col1, #T_592f0_row1_col2, #T_592f0_row3_col1, #T_592f0_row3_col2, #T_592f0_row7_col1, #T_592f0_row7_col2 {
      background-color: lightcoral;
      border-bottom: 2px solid black;
    }
    #T_592f0_row5_col1, #T_592f0_row5_col2, #T_592f0_row9_col1, #T_592f0_row9_col2, #T_592f0_row11_col1, #T_592f0_row11_col2 {
      background-color: lightgreen;
      border-bottom: 2px solid black;
    }
    </style>
    <table id="T_592f0" class="dataframe">
      <thead>
        <tr>
          <th class="blank" >&nbsp;</th>
          <th class="blank" >&nbsp;</th>
          <th class="blank level0" >&nbsp;</th>
          <th id="T_592f0_level0_col0" class="col_heading level0 col0" ># trials</th>
          <th id="T_592f0_level0_col1" class="col_heading level0 col1" >Accuracy per trial</th>
          <th id="T_592f0_level0_col2" class="col_heading level0 col2" >Combined accuracy</th>
        </tr>
        <tr>
          <th class="index_name level0" >cohort</th>
          <th class="index_name level1" >algo</th>
          <th class="index_name level2" >version</th>
          <th class="blank col0" >&nbsp;</th>
          <th class="blank col1" >&nbsp;</th>
          <th class="blank col2" >&nbsp;</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th id="T_592f0_level0_row0" class="row_heading level0 row0" rowspan="2">HA</th>
          <th id="T_592f0_level1_row0" class="row_heading level1 row0" rowspan="2">ReorientationMethodDM</th>
          <th id="T_592f0_level2_row0" class="row_heading level2 row0" >Full</th>
          <td id="T_592f0_row0_col0" class="data row0 col0" >227</td>
          <td id="T_592f0_row0_col1" class="data row0 col1" ><span>0.88 [0.84, 0.92]</span></td>
          <td id="T_592f0_row0_col2" class="data row0 col2" >0.8824626865671642</td>
        </tr>
        <tr>
          <th id="T_592f0_level2_row1" class="row_heading level2 row1" >Trust gravity</th>
          <td id="T_592f0_row1_col0" class="data row1 col0" >227</td>
          <td id="T_592f0_row1_col1" class="data row1 col1" ><span>0.78 [0.75, 0.81]</span></td>
          <td id="T_592f0_row1_col2" class="data row1 col2" >0.7868470149253731</td>
        </tr>
        <tr>
          <th id="T_592f0_level0_row2" class="row_heading level0 row2" rowspan="2">CHF</th>
          <th id="T_592f0_level1_row2" class="row_heading level1 row2" rowspan="2">ReorientationMethodDM</th>
          <th id="T_592f0_level2_row2" class="row_heading level2 row2" >Full</th>
          <td id="T_592f0_row2_col0" class="data row2 col0" >106</td>
          <td id="T_592f0_row2_col1" class="data row2 col1" ><span>0.87 [0.80, 0.93]</span></td>
          <td id="T_592f0_row2_col2" class="data row2 col2" >0.8821138211382114</td>
        </tr>
        <tr>
          <th id="T_592f0_level2_row3" class="row_heading level2 row3" >Trust gravity</th>
          <td id="T_592f0_row3_col0" class="data row3 col0" >106</td>
          <td id="T_592f0_row3_col1" class="data row3 col1" ><span>0.77 [0.73, 0.82]</span></td>
          <td id="T_592f0_row3_col2" class="data row3 col2" >0.7865853658536586</td>
        </tr>
        <tr>
          <th id="T_592f0_level0_row4" class="row_heading level0 row4" rowspan="2">COPD</th>
          <th id="T_592f0_level1_row4" class="row_heading level1 row4" rowspan="2">ReorientationMethodDM</th>
          <th id="T_592f0_level2_row4" class="row_heading level2 row4" >Full</th>
          <td id="T_592f0_row4_col0" class="data row4 col0" >214</td>
          <td id="T_592f0_row4_col1" class="data row4 col1" ><span>0.92 [0.88, 0.95]</span></td>
          <td id="T_592f0_row4_col2" class="data row4 col2" >0.9027777777777778</td>
        </tr>
        <tr>
          <th id="T_592f0_level2_row5" class="row_heading level2 row5" >Trust gravity</th>
          <td id="T_592f0_row5_col0" class="data row5 col0" >214</td>
          <td id="T_592f0_row5_col1" class="data row5 col1" ><span>0.81 [0.79, 0.84]</span></td>
          <td id="T_592f0_row5_col2" class="data row5 col2" >0.8015873015873016</td>
        </tr>
        <tr>
          <th id="T_592f0_level0_row6" class="row_heading level0 row6" rowspan="2">MS</th>
          <th id="T_592f0_level1_row6" class="row_heading level1 row6" rowspan="2">ReorientationMethodDM</th>
          <th id="T_592f0_level2_row6" class="row_heading level2 row6" >Full</th>
          <td id="T_592f0_row6_col0" class="data row6 col0" >228</td>
          <td id="T_592f0_row6_col1" class="data row6 col1" ><span>0.87 [0.82, 0.91]</span></td>
          <td id="T_592f0_row6_col2" class="data row6 col2" >0.8620071684587813</td>
        </tr>
        <tr>
          <th id="T_592f0_level2_row7" class="row_heading level2 row7" >Trust gravity</th>
          <td id="T_592f0_row7_col0" class="data row7 col0" >228</td>
          <td id="T_592f0_row7_col1" class="data row7 col1" ><span>0.77 [0.74, 0.81]</span></td>
          <td id="T_592f0_row7_col2" class="data row7 col2" >0.771505376344086</td>
        </tr>
        <tr>
          <th id="T_592f0_level0_row8" class="row_heading level0 row8" rowspan="2">PD</th>
          <th id="T_592f0_level1_row8" class="row_heading level1 row8" rowspan="2">ReorientationMethodDM</th>
          <th id="T_592f0_level2_row8" class="row_heading level2 row8" >Full</th>
          <td id="T_592f0_row8_col0" class="data row8 col0" >224</td>
          <td id="T_592f0_row8_col1" class="data row8 col1" ><span>0.94 [0.91, 0.97]</span></td>
          <td id="T_592f0_row8_col2" class="data row8 col2" >0.9226415094339623</td>
        </tr>
        <tr>
          <th id="T_592f0_level2_row9" class="row_heading level2 row9" >Trust gravity</th>
          <td id="T_592f0_row9_col0" class="data row9 col0" >224</td>
          <td id="T_592f0_row9_col1" class="data row9 col1" ><span>0.83 [0.81, 0.85]</span></td>
          <td id="T_592f0_row9_col2" class="data row9 col2" >0.8169811320754717</td>
        </tr>
        <tr>
          <th id="T_592f0_level0_row10" class="row_heading level0 row10" rowspan="2">PFF</th>
          <th id="T_592f0_level1_row10" class="row_heading level1 row10" rowspan="2">ReorientationMethodDM</th>
          <th id="T_592f0_level2_row10" class="row_heading level2 row10" >Full</th>
          <td id="T_592f0_row10_col0" class="data row10 col0" >169</td>
          <td id="T_592f0_row10_col1" class="data row10 col1" ><span>0.96 [0.93, 0.99]</span></td>
          <td id="T_592f0_row10_col2" class="data row10 col2" >0.9495614035087719</td>
        </tr>
        <tr>
          <th id="T_592f0_level2_row11" class="row_heading level2 row11" >Trust gravity</th>
          <td id="T_592f0_row11_col0" class="data row11 col0" >169</td>
          <td id="T_592f0_row11_col1" class="data row11 col1" ><span>0.84 [0.82, 0.86]</span></td>
          <td id="T_592f0_row11_col2" class="data row11 col2" >0.837171052631579</td>
        </tr>
      </tbody>
    </table>

    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 521-523

Confusion Matrices
~~~~~~~~~~~~~~~~~~

.. GENERATED FROM PYTHON SOURCE LINES 523-549

.. code-block:: Python

    fig, axes = plt.subplots(
        1,
        len(algorithms),
        figsize=(6 * len(algorithms), 5),
        constrained_layout=True,
    )
    if len(algorithms) == 1:
        axes = [axes]

    for ax, ((algo, version), data) in zip(
        axes, lab_predictions.groupby(["algo", "version"], sort=False)
    ):
        sns.heatmap(
            calculate_confusion_matrix(data),
            annot=True,
            fmt="d",
            cmap="Blues",
            cbar=False,
            ax=ax,
        )
        ax.set_title(f"{algo} ({version})")

    fig.suptitle("Laboratory confusion matrices")
    fig.show()





.. image-sg:: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_006.png
   :alt: Laboratory confusion matrices, ReorientationMethodDM (Full), ReorientationMethodDM (Trust gravity)
   :srcset: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_006.png
   :class: sphx-glr-single-img





.. GENERATED FROM PYTHON SOURCE LINES 550-598

Prevalence-weighted mode choice
-------------------------------
The simulated validation data contains all supported orientation labels with
equal weight. This is useful to stress-test every class, but it is not the
expected real-world prevalence. In practice, most walking bouts should already
be correctly oriented. We therefore also calculate weighted accuracies for
explicit prevalence scenarios.

- ``5% total errors``: 95% identity orientation and 5% errors split equally
  across the seven non-identity orientation classes.
- ``Equal simulated classes``: the class-balanced simulation view. This
  corresponds to 12.5% identity and 87.5% total orientation errors.

The break-even calculation uses the accuracy difference
``trust_gravity - full``. ``trust_gravity`` usually wins on the identity
class, because it does not try to correct a front-back flip when gravity is
already plausible. ``full`` wins on the one front-back flip class that
``trust_gravity`` intentionally leaves unresolved. The relevant question is
therefore how common these true orientation errors are compared to correctly
mounted walking bouts.

We calculate the break-even point by solving:

.. math::

   (1 - p) \Delta_\mathrm{identity} + p \Delta_\mathrm{error} = 0

for ``p``:

.. math::

   p = -\Delta_\mathrm{identity}
       / (\Delta_\mathrm{error} - \Delta_\mathrm{identity})

The reported thresholds use two choices for :math:`\Delta_\mathrm{error}`:

- ``Total error prevalence``: identity has prevalence ``1 - p`` and all seven
  non-identity classes have prevalence ``p / 7``. Here,
  :math:`\Delta_\mathrm{error}` is the mean difference across all non-identity
  labels.
- ``Uncorrectable PA-flip prevalence``: only the correctly mounted identity
  orientation and ``pa_flipped__rot_pa_0`` vary. This isolates the
  same-gravity, front-back flip case that ``trust_gravity`` intentionally
  cannot distinguish from identity.

The ``Combined TVS`` rows pool the free-living and laboratory prediction rows
to provide a single rough TVS-level threshold. Use the condition-specific rows
if your target setting maps clearly to one of the two validation conditions.

.. GENERATED FROM PYTHON SOURCE LINES 598-622

.. code-block:: Python

    prevalence_scenarios = pd.Series(
        {
            "5% total errors": 0.05,
            "33% total errors": 0.33,
            "Equal simulated classes": 1 - 1 / len(REORIENTATION_LABELS),
        }
    )

    weighted_accuracy_scenarios = pd.concat(
        {
            "Combined TVS": calculate_weighted_accuracy_by_prevalence(
                combined_predictions, prevalence_scenarios
            ),
            "Free-living": calculate_weighted_accuracy_by_prevalence(
                free_living_predictions, prevalence_scenarios
            ),
            "Laboratory": calculate_weighted_accuracy_by_prevalence(
                lab_predictions, prevalence_scenarios
            ),
        },
        names=["condition"],
    )
    weighted_accuracy_scenarios.style.format("{:.1%}")






.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <style type="text/css">
    </style>
    <table id="T_d2254">
      <thead>
        <tr>
          <th class="blank" >&nbsp;</th>
          <th class="blank" >&nbsp;</th>
          <th class="blank level0" >&nbsp;</th>
          <th id="T_d2254_level0_col0" class="col_heading level0 col0" >5% total errors</th>
          <th id="T_d2254_level0_col1" class="col_heading level0 col1" >33% total errors</th>
          <th id="T_d2254_level0_col2" class="col_heading level0 col2" >Equal simulated classes</th>
        </tr>
        <tr>
          <th class="index_name level0" >condition</th>
          <th class="index_name level1" >algo</th>
          <th class="index_name level2" >version</th>
          <th class="blank col0" >&nbsp;</th>
          <th class="blank col1" >&nbsp;</th>
          <th class="blank col2" >&nbsp;</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th id="T_d2254_level0_row0" class="row_heading level0 row0" rowspan="2">Combined TVS</th>
          <th id="T_d2254_level1_row0" class="row_heading level1 row0" rowspan="2">ReorientationMethodDM</th>
          <th id="T_d2254_level2_row0" class="row_heading level2 row0" >Full</th>
          <td id="T_d2254_row0_col0" class="data row0 col0" >91.9%</td>
          <td id="T_d2254_row0_col1" class="data row0 col1" >92.0%</td>
          <td id="T_d2254_row0_col2" class="data row0 col2" >92.1%</td>
        </tr>
        <tr>
          <th id="T_d2254_level2_row1" class="row_heading level2 row1" >Trust gravity</th>
          <td id="T_d2254_row1_col0" class="data row1 col0" >98.8%</td>
          <td id="T_d2254_row1_col1" class="data row1 col1" >93.0%</td>
          <td id="T_d2254_row1_col2" class="data row1 col2" >81.5%</td>
        </tr>
        <tr>
          <th id="T_d2254_level0_row2" class="row_heading level0 row2" rowspan="2">Free-living</th>
          <th id="T_d2254_level1_row2" class="row_heading level1 row2" rowspan="2">ReorientationMethodDM</th>
          <th id="T_d2254_level2_row2" class="row_heading level2 row2" >Full</th>
          <td id="T_d2254_row2_col0" class="data row2 col0" >92.4%</td>
          <td id="T_d2254_row2_col1" class="data row2 col1" >92.5%</td>
          <td id="T_d2254_row2_col2" class="data row2 col2" >92.6%</td>
        </tr>
        <tr>
          <th id="T_d2254_level2_row3" class="row_heading level2 row3" >Trust gravity</th>
          <td id="T_d2254_row3_col0" class="data row3 col0" >98.9%</td>
          <td id="T_d2254_row3_col1" class="data row3 col1" >93.1%</td>
          <td id="T_d2254_row3_col2" class="data row3 col2" >82.0%</td>
        </tr>
        <tr>
          <th id="T_d2254_level0_row4" class="row_heading level0 row4" rowspan="2">Laboratory</th>
          <th id="T_d2254_level1_row4" class="row_heading level1 row4" rowspan="2">ReorientationMethodDM</th>
          <th id="T_d2254_level2_row4" class="row_heading level2 row4" >Full</th>
          <td id="T_d2254_row4_col0" class="data row4 col0" >90.3%</td>
          <td id="T_d2254_row4_col1" class="data row4 col1" >90.2%</td>
          <td id="T_d2254_row4_col2" class="data row4 col2" >90.0%</td>
        </tr>
        <tr>
          <th id="T_d2254_level2_row5" class="row_heading level2 row5" >Trust gravity</th>
          <td id="T_d2254_row5_col0" class="data row5 col0" >98.8%</td>
          <td id="T_d2254_row5_col1" class="data row5 col1" >92.4%</td>
          <td id="T_d2254_row5_col2" class="data row5 col2" >80.0%</td>
        </tr>
      </tbody>
    </table>

    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 623-635

.. code-block:: Python

    break_even_inputs = pd.concat(
        {
            "Combined TVS": calculate_mode_break_even_inputs(combined_predictions),
            "Free-living": calculate_mode_break_even_inputs(
                free_living_predictions
            ),
            "Laboratory": calculate_mode_break_even_inputs(lab_predictions),
        },
        names=["condition", "input"],
    )
    break_even_inputs.style.format("{:.1%}")






.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <style type="text/css">
    </style>
    <table id="T_06def">
      <thead>
        <tr>
          <th class="blank" >&nbsp;</th>
          <th class="blank level0" >&nbsp;</th>
          <th id="T_06def_level0_col0" class="col_heading level0 col0" >Full accuracy</th>
          <th id="T_06def_level0_col1" class="col_heading level0 col1" >Trust gravity accuracy</th>
          <th id="T_06def_level0_col2" class="col_heading level0 col2" >Trust gravity - full</th>
        </tr>
        <tr>
          <th class="index_name level0" >condition</th>
          <th class="index_name level1" >input</th>
          <th class="blank col0" >&nbsp;</th>
          <th class="blank col1" >&nbsp;</th>
          <th class="blank col2" >&nbsp;</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th id="T_06def_level0_row0" class="row_heading level0 row0" rowspan="3">Combined TVS</th>
          <th id="T_06def_level1_row0" class="row_heading level1 row0" >Identity orientation</th>
          <td id="T_06def_row0_col0" class="data row0 col0" >91.9%</td>
          <td id="T_06def_row0_col1" class="data row0 col1" >99.9%</td>
          <td id="T_06def_row0_col2" class="data row0 col2" >8.0%</td>
        </tr>
        <tr>
          <th id="T_06def_level1_row1" class="row_heading level1 row1" >Mean non-identity orientation</th>
          <td id="T_06def_row1_col0" class="data row1 col0" >92.1%</td>
          <td id="T_06def_row1_col1" class="data row1 col1" >78.9%</td>
          <td id="T_06def_row1_col2" class="data row1 col2" >-13.2%</td>
        </tr>
        <tr>
          <th id="T_06def_level1_row2" class="row_heading level1 row2" >Uncorrectable PA-flip orientation</th>
          <td id="T_06def_row2_col0" class="data row2 col0" >92.2%</td>
          <td id="T_06def_row2_col1" class="data row2 col1" >0.0%</td>
          <td id="T_06def_row2_col2" class="data row2 col2" >-92.2%</td>
        </tr>
        <tr>
          <th id="T_06def_level0_row3" class="row_heading level0 row3" rowspan="3">Free-living</th>
          <th id="T_06def_level1_row3" class="row_heading level1 row3" >Identity orientation</th>
          <td id="T_06def_row3_col0" class="data row3 col0" >92.4%</td>
          <td id="T_06def_row3_col1" class="data row3 col1" >99.9%</td>
          <td id="T_06def_row3_col2" class="data row3 col2" >7.5%</td>
        </tr>
        <tr>
          <th id="T_06def_level1_row4" class="row_heading level1 row4" >Mean non-identity orientation</th>
          <td id="T_06def_row4_col0" class="data row4 col0" >92.7%</td>
          <td id="T_06def_row4_col1" class="data row4 col1" >79.4%</td>
          <td id="T_06def_row4_col2" class="data row4 col2" >-13.3%</td>
        </tr>
        <tr>
          <th id="T_06def_level1_row5" class="row_heading level1 row5" >Uncorrectable PA-flip orientation</th>
          <td id="T_06def_row5_col0" class="data row5 col0" >92.9%</td>
          <td id="T_06def_row5_col1" class="data row5 col1" >0.0%</td>
          <td id="T_06def_row5_col2" class="data row5 col2" >-92.9%</td>
        </tr>
        <tr>
          <th id="T_06def_level0_row6" class="row_heading level0 row6" rowspan="3">Laboratory</th>
          <th id="T_06def_level1_row6" class="row_heading level1 row6" >Identity orientation</th>
          <td id="T_06def_row6_col0" class="data row6 col0" >90.3%</td>
          <td id="T_06def_row6_col1" class="data row6 col1" >99.9%</td>
          <td id="T_06def_row6_col2" class="data row6 col2" >9.6%</td>
        </tr>
        <tr>
          <th id="T_06def_level1_row7" class="row_heading level1 row7" >Mean non-identity orientation</th>
          <td id="T_06def_row7_col0" class="data row7 col0" >90.0%</td>
          <td id="T_06def_row7_col1" class="data row7 col1" >77.2%</td>
          <td id="T_06def_row7_col2" class="data row7 col2" >-12.8%</td>
        </tr>
        <tr>
          <th id="T_06def_level1_row8" class="row_heading level1 row8" >Uncorrectable PA-flip orientation</th>
          <td id="T_06def_row8_col0" class="data row8 col0" >89.8%</td>
          <td id="T_06def_row8_col1" class="data row8 col1" >0.0%</td>
          <td id="T_06def_row8_col2" class="data row8 col2" >-89.8%</td>
        </tr>
      </tbody>
    </table>

    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 636-653

.. code-block:: Python

    break_even_points = pd.DataFrame(
        {
            "Combined TVS": calculate_mode_break_even_points(combined_predictions),
            "Free-living": calculate_mode_break_even_points(
                free_living_predictions
            ),
            "Laboratory": calculate_mode_break_even_points(lab_predictions),
        }
    ).T
    break_even_points = break_even_points.rename(
        columns={
            "Total error prevalence": "Total error prevalence p",
            "Specific PA-flip prevalence": "Uncorrectable PA-flip prevalence q",
        }
    )
    break_even_points.style.format("{:.1%}", na_rep="outside [0, 100%]")






.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <style type="text/css">
    </style>
    <table id="T_70be8">
      <thead>
        <tr>
          <th class="blank level0" >&nbsp;</th>
          <th id="T_70be8_level0_col0" class="col_heading level0 col0" >Total error prevalence p</th>
          <th id="T_70be8_level0_col1" class="col_heading level0 col1" >Uncorrectable PA-flip prevalence q</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th id="T_70be8_level0_row0" class="row_heading level0 row0" >Combined TVS</th>
          <td id="T_70be8_row0_col0" class="data row0 col0" >37.7%</td>
          <td id="T_70be8_row0_col1" class="data row0 col1" >7.9%</td>
        </tr>
        <tr>
          <th id="T_70be8_level0_row1" class="row_heading level0 row1" >Free-living</th>
          <td id="T_70be8_row1_col0" class="data row1 col0" >36.1%</td>
          <td id="T_70be8_row1_col1" class="data row1 col1" >7.5%</td>
        </tr>
        <tr>
          <th id="T_70be8_level0_row2" class="row_heading level0 row2" >Laboratory</th>
          <td id="T_70be8_row2_col0" class="data row2 col0" >42.8%</td>
          <td id="T_70be8_row2_col1" class="data row2 col1" >9.7%</td>
        </tr>
      </tbody>
    </table>

    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 654-656

The free-living curve shows why the preferred mode depends on the expected
prevalence of orientation errors.

.. GENERATED FROM PYTHON SOURCE LINES 656-689

.. code-block:: Python

    prevalence_grid = np.linspace(0, 1, 101)
    free_living_weighted_accuracy_curve = calculate_weighted_accuracy_curve(
        free_living_predictions,
        prevalence_grid,
    )

    fig, ax = plt.subplots()
    sns.lineplot(
        data=free_living_weighted_accuracy_curve,
        x="total_misorientation_prevalence",
        y="weighted_accuracy",
        hue="version",
        ax=ax,
    )
    free_living_break_even = break_even_points.loc[
        "Free-living", "Total error prevalence p"
    ]
    if not np.isnan(free_living_break_even):
        ax.axvline(
            free_living_break_even,
            color="black",
            linestyle="--",
            label="Break-even",
        )
    ax.set(
        xlabel="Total misorientation prevalence",
        ylabel="Weighted accuracy",
        title="Free-living prevalence-weighted accuracy",
    )
    ax.legend()
    fig.show()





.. image-sg:: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_007.png
   :alt: Free-living prevalence-weighted accuracy
   :srcset: /auto_revalidation/re_orientation/images/sphx_glr__01_reorientation_analysis_007.png
   :class: sphx-glr-single-img






.. rst-class:: sphx-glr-timing

   **Total running time of the script:** (0 minutes 8.916 seconds)

**Estimated memory usage:**  82 MB


.. _sphx_glr_download_auto_revalidation_re_orientation__01_reorientation_analysis.py:

.. only:: html

  .. container:: sphx-glr-footer sphx-glr-footer-example

    .. container:: sphx-glr-download sphx-glr-download-jupyter

      :download:`Download Jupyter notebook: _01_reorientation_analysis.ipynb <_01_reorientation_analysis.ipynb>`

    .. container:: sphx-glr-download sphx-glr-download-python

      :download:`Download Python source code: _01_reorientation_analysis.py <_01_reorientation_analysis.py>`

    .. container:: sphx-glr-download sphx-glr-download-zip

      :download:`Download zipped: _01_reorientation_analysis.zip <_01_reorientation_analysis.zip>`


.. only:: html

 .. rst-class:: sphx-glr-signature

    `Gallery generated by Sphinx-Gallery <https://sphinx-gallery.github.io>`_