Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions rsmtool/analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from sklearn.decomposition import PCA
from sklearn.metrics import confusion_matrix, mean_squared_error, r2_score
from skll.metrics import kappa
from wandb.sdk.lib import RunDisabled
from wandb.wandb_run import Run

from .configuration_parser import Configuration
Expand Down Expand Up @@ -1701,7 +1700,7 @@ def run_prediction_analyses(
self,
data_container: DataContainer,
configuration: Configuration,
wandb_run: Union[Run, RunDisabled, None] = None,
wandb_run: Union[Run, None] = None,
) -> Tuple[Configuration, DataContainer]:
"""
Run all analyses on the system scores (predictions).
Expand All @@ -1718,7 +1717,7 @@ def run_prediction_analyses(
parameters (keys): {"subgroups", "second_human_score_column",
"use_scaled_predictions"}.

wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
The wandb run object if wandb is enabled, ``None`` otherwise.
If enabled, all the output data frames will be logged to this run
as tables.
Expand Down
5 changes: 2 additions & 3 deletions rsmtool/reporter.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
from nbconvert.exporters.templateexporter import default_filters
from nbformat.warnings import DuplicateCellId, MissingIDFieldWarning
from traitlets.config import Config
from wandb.sdk.lib import RunDisabled
from wandb.wandb_run import Run

from rsmtool.configuration_parser import Configuration
Expand Down Expand Up @@ -126,7 +125,7 @@ class Reporter:
def __init__(
self,
logger: Optional[logging.Logger] = None,
wandb_run: Union[Run, RunDisabled, None] = None,
wandb_run: Union[Run, None] = None,
):
"""
Initialize the Reporter object.
Expand All @@ -136,7 +135,7 @@ def __init__(
logger: Optional[logging.Logger]
A Logger object. If ``None`` is passed, get logger from ``__name__``.
Defaults to ``None``.
wandb_run: Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run: Union[wandb.wandb_run.Run, None]
A wandb run object that will be used to log artifacts and tables.
If ``None`` is passed, a new wandb run will be initialized if
wandb is enabled in the configuration.
Expand Down
5 changes: 2 additions & 3 deletions rsmtool/rsmeval.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
from pathlib import Path
from typing import Any, Dict, List, Optional, Union

from wandb.sdk.lib import RunDisabled
from wandb.wandb_run import Run

from .analyzer import Analyzer
Expand All @@ -37,7 +36,7 @@ def run_evaluation(
output_dir: str,
overwrite_output: bool = False,
logger: Optional[logging.Logger] = None,
wandb_run: Union[Run, RunDisabled, None] = None,
wandb_run: Union[Run, None] = None,
) -> None:
"""
Run an rsmeval experiment using the given configuration.
Expand Down Expand Up @@ -65,7 +64,7 @@ def run_evaluation(
logger : Optional[logging.Logger]
A Logger object. If ``None`` is passed, get logger from ``__name__``.
Defaults to ``None``.
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
A wandb run object that will be used to log artifacts and tables.
If ``None`` is passed, a new wandb run will be initialized if
wandb is enabled in the configuration.
Expand Down
9 changes: 4 additions & 5 deletions rsmtool/rsmexplain.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import shap
from skll.data import FeatureSet
from skll.learner import Learner
from wandb.sdk.lib import RunDisabled
from wandb.wandb_run import Run

from .configuration_parser import Configuration, configure
Expand Down Expand Up @@ -147,7 +146,7 @@ def generate_explanation(
output_dir: str,
overwrite_output: bool = False,
logger: Optional[logging.Logger] = None,
wandb_run: Union[Run, RunDisabled, None] = None,
wandb_run: Union[Run, None] = None,
):
"""
Generate a shap.Explanation object.
Expand Down Expand Up @@ -176,7 +175,7 @@ def generate_explanation(
logger : Optional[logging.Logger]
A Logger object. If ``None`` is passed, get logger from ``__name__``.
Defaults to ``None``.
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
A wandb run object that will be used to log artifacts and tables.
If ``None`` is passed, a new wandb run will be initialized if
wandb is enabled in the configuration.
Expand Down Expand Up @@ -440,7 +439,7 @@ def generate_report(
ids: Dict[int, str],
configuration: Configuration,
logger: Optional[logging.Logger] = None,
wandb_run: Union[Run, RunDisabled, None] = None,
wandb_run: Union[Run, None] = None,
) -> None:
"""
Generate an rsmexplain report.
Expand All @@ -463,7 +462,7 @@ def generate_report(
logger : Optional[logging.Logger]
A Logger object. If ``None`` is passed, get logger from ``__name__``.
Defaults to ``None``.
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
A wandb run object that will be used to log artifacts and tables.
If ``None`` is passed, a new wandb run will be initialized if
wandb is enabled in the configuration. Defaults to ``None``.
Expand Down
5 changes: 2 additions & 3 deletions rsmtool/rsmpredict.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import numpy as np
import pandas as pd
from wandb.sdk.lib import RunDisabled
from wandb.wandb_run import Run

from .configuration_parser import Configuration, configure
Expand Down Expand Up @@ -316,7 +315,7 @@ def compute_and_save_predictions(
output_file: str,
feats_file: Optional[str] = None,
logger: Optional[logging.Logger] = None,
wandb_run: Union[Run, RunDisabled, None] = None,
wandb_run: Union[Run, None] = None,
) -> None:
"""
Run rsmpredict using the given configuration.
Expand Down Expand Up @@ -345,7 +344,7 @@ def compute_and_save_predictions(
logger : Optional[logging.Logger]
A Logger object. If ``None`` is passed, get logger from ``__name__``.
Defaults to ``None``.
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
A wandb run object that will be used to log artifacts and tables.
If ``None`` is passed, a new wandb run will be initialized if
wandb is enabled in the configuration.
Expand Down
5 changes: 2 additions & 3 deletions rsmtool/rsmsummarize.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Union

from wandb.sdk.lib import RunDisabled
from wandb.wandb_run import Run

from .configuration_parser import Configuration, configure
Expand Down Expand Up @@ -103,7 +102,7 @@ def run_summary(
output_dir: str,
overwrite_output: bool = False,
logger: Optional[logging.Logger] = None,
wandb_run: Union[Run, RunDisabled, None] = None,
wandb_run: Union[Run, None] = None,
) -> None:
"""
Run rsmsummarize experiment using the given configuration.
Expand Down Expand Up @@ -133,7 +132,7 @@ def run_summary(
logger : Optional[logging.Logger]
A Logger object. If ``None`` is passed, get logger from ``__name__``.
Defaults to ``None``.
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
A wandb run object that will be used to log artifacts and tables.
If ``None`` is passed, a new wandb run will be initialized if
wandb is enabled in the configuration.
Expand Down
5 changes: 2 additions & 3 deletions rsmtool/rsmtool.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
from pathlib import Path
from typing import Any, Dict, List, Optional, Union

from wandb.sdk.lib import RunDisabled
from wandb.wandb_run import Run

from rsmtool.container import DatasetDict
Expand All @@ -40,7 +39,7 @@ def run_experiment(
output_dir: str,
overwrite_output: bool = False,
logger: Optional[logging.Logger] = None,
wandb_run: Union[Run, RunDisabled, None] = None,
wandb_run: Union[Run, None] = None,
) -> None:
"""
Run an rsmtool experiment using the given configuration.
Expand Down Expand Up @@ -70,7 +69,7 @@ def run_experiment(
logger : Optional[logging.Logger]
A Logger object. If ``None`` is passed, get logger from ``__name__``.
Defaults to ``None``.
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
A wandb run object that will be used to log artifacts and tables.
If ``None`` is passed, a new wandb run will be initialized if
wandb is enabled in the configuration.
Expand Down
21 changes: 10 additions & 11 deletions rsmtool/utils/wandb.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

import pandas as pd
import wandb
from wandb.sdk.lib import RunDisabled
from wandb.wandb_run import Run

from ..configuration_parser import Configuration
Expand All @@ -23,7 +22,7 @@
METRICS_LOGGED = ["consistency", "eval_short", "true_score_eval"]


def init_wandb_run(config_obj: Configuration) -> Union[Run, RunDisabled, None]:
def init_wandb_run(config_obj: Configuration) -> Union[Run, None]:
"""
Initialize a wandb run if logging to wandb is enabled in the configuration.

Expand All @@ -37,7 +36,7 @@ def init_wandb_run(config_obj: Configuration) -> Union[Run, RunDisabled, None]:

Returns
-------
Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
Union[wandb.wandb_run.Run, None]
A wandb Run object, or ``None`` if logging to wandb is disabled.
"""
use_wandb = config_obj["use_wandb"]
Expand All @@ -50,14 +49,14 @@ def init_wandb_run(config_obj: Configuration) -> Union[Run, RunDisabled, None]:


def log_configuration_to_wandb(
wandb_run: Union[Run, RunDisabled, None], configuration: Configuration
wandb_run: Union[Run, None], configuration: Configuration
) -> None:
"""
Log a configuration object to wandb if logging to wandb is enabled.

Parameters
----------
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
The wandb Run object, or ``None``, if logging to wandb is disabled.
configuration : rsmtool.configuration_parser.Configuration
The Configuration object to log to the run.
Expand All @@ -67,7 +66,7 @@ def log_configuration_to_wandb(


def log_dataframe_to_wandb(
wandb_run: Union[Run, RunDisabled, None],
wandb_run: Union[Run, None],
df: pd.DataFrame,
frame_name: str,
section: Optional[str] = None,
Expand All @@ -81,7 +80,7 @@ def log_dataframe_to_wandb(

Parameters
----------
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
The wandb Run object, or ``None``, if logging to wandb is disabled.
df : pandas.DataFrame
The dataframe object to log.
Expand Down Expand Up @@ -148,7 +147,7 @@ def get_metric_name(


def log_confusion_matrix(
wandb_run: Union[Run, RunDisabled, None],
wandb_run: Union[Run, None],
human_scores: pd.Series,
system_scores: pd.Series,
name: str,
Expand All @@ -161,7 +160,7 @@ def log_confusion_matrix(

Parameters
----------
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
The wandb Run object, or ``None``, if logging to wandb is disabled.
human_scores : pandas.Series
The human scores for the responses in the data
Expand All @@ -186,7 +185,7 @@ def log_confusion_matrix(


def log_report_to_wandb(
wandb_run: Union[Run, RunDisabled, None], report_name: str, report_path: str
wandb_run: Union[Run, None], report_name: str, report_path: str
):
"""
Log a report to wandb if logging to wandb is enabled.
Expand All @@ -196,7 +195,7 @@ def log_report_to_wandb(

Parameters
----------
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
The wandb Run object, or ``None``, if logging to wandb is disabled.
report_name: str
The report's name to use in the log.
Expand Down
5 changes: 2 additions & 3 deletions rsmtool/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from typing import Dict, List, Optional, Union

import pandas as pd
from wandb.sdk.lib import RunDisabled
from wandb.wandb_run import Run

from rsmtool.container import DataContainer
Expand All @@ -28,7 +27,7 @@ def __init__(
self,
experiment_id: Optional[str] = None,
context: Optional[str] = None,
wandb_run: Union[Run, RunDisabled, None] = None,
wandb_run: Union[Run, None] = None,
):
"""
Initialize the DataWriter object.
Expand All @@ -41,7 +40,7 @@ def __init__(
context : Optional[str]
The context in which this writer is used.
Defaults to ``None``.
wandb_run : Union[wandb.wandb_run.Run, wandb.sdk.lib.RunDisabled, None]
wandb_run : Union[wandb.wandb_run.Run, None]
The wandb run object if wandb is enabled, None otherwise.
If enabled, all the output data frames will be logged to
this run as tables.
Expand Down
Loading