Visitar URL original
Robustness fix: Annotation file not dropping likelihood column if present from machine labels by C-Achard · Pull Request #3323 · DeepLabCut/DeepLabCut · GitHub
Skip to content
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Drop likelihood columns from annotations
Add helper _drop_likelihood_columns to remove any 'likelihood' coord columns from MultiIndex annotation DataFrames (using 'coords' level if present or the last level otherwise). Use this sanitizer in merge_annotateddatasets and as a final check in format_training_data to avoid concatenated likelihood columns (fixes issues such as napari-deeplabcut #204 and DeepLabCut #3319). Also log when likelihood columns are detected and dropped, tighten x/y coordinate validation, and apply minor formatting cleanups.
  • Loading branch information
C-Achard committed May 13, 2026
commit 84e9f589690e51c3d4a840325629d9dddb9a3dfe
46 changes: 34 additions & 12 deletions deeplabcut/generate_training_dataset/trainingsetmanipulation.py
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,26 @@ def parse_video_filenames(videos: list[str]) -> list[str]:
return filenames


def _drop_likelihood_columns(df: pd.DataFrame) -> pd.DataFrame:
"""Drop any columns whose coord level is named 'likelihood'.

This sanitizes annotation DataFrames coming from h5/csv files before they are
used for training dataset generation.
"""
if not isinstance(df.columns, pd.MultiIndex):
return df

coord_level = "coords" if "coords" in df.columns.names else df.columns.names[-1]
coord_values = df.columns.get_level_values(coord_level)

likelihood_mask = coord_values == "likelihood"
if likelihood_mask.any():
logging.info("Detected likelihood columns in annotation data; dropping them.")
df = df.drop(columns=df.columns[likelihood_mask])

return df


def merge_annotateddatasets(cfg, trainingsetfolder_full):
"""Merges all the h5 files for all labeled-datasets (from individual videos).

Expand Down Expand Up @@ -547,11 +567,16 @@ def merge_annotateddatasets(cfg, trainingsetfolder_full):
else:
bodyparts = cfg["bodyparts"]
AnnotationData = AnnotationData.reindex(bodyparts, axis=1, level=AnnotationData.columns.names.index("bodyparts"))
# Filter out any stray likelihood columns that may have been concatenated in
# see napari-deeplabcut #204 and DeepLabCut #3319
AnnotationData = _drop_likelihood_columns(AnnotationData)

if AnnotationData.empty:
logging.warning(
"The annotated dataframe is empty after reindexing using config. "
"Hint: are bodyparts correctly listed in the configuration?"
)

filename = os.path.join(trainingsetfolder_full, f"CollectedData_{cfg['scorer']}")
AnnotationData.to_hdf(filename + ".h5", key="df_with_missing", mode="w")
AnnotationData.to_csv(filename + ".csv") # human readable.
Expand Down Expand Up @@ -728,22 +753,17 @@ def to_matlab_cell(array):
outer[0, 0] = array.astype("int64")
return outer

# Keep only x/y columns if likelihood is present
# Again, remove likelihood if present
df = _drop_likelihood_columns(df)

if isinstance(df.columns, pd.MultiIndex):
coord_level = "coords" if "coords" in df.columns.names else df.columns.names[-1]
coord_values = df.columns.get_level_values(coord_level)

has_x = "x" in coord_values
has_y = "y" in coord_values
has_likelihood = "likelihood" in coord_values

if has_x and has_y:
if has_likelihood:
logging.info(
"Detected likelihood columns in training data; dropping them before formatting training dataset."
)
df = df.loc[:, coord_values.isin(["x", "y"])]
else:
if not (has_x and has_y):
raise ValueError(
f"Training data must contain x/y coordinates. Found coordinate labels: {list(pd.unique(coord_values))}"
)
Expand All @@ -768,15 +788,14 @@ def to_matlab_cell(array):
joints = np.c_[range(nbodyparts), temp]
joints = joints[~np.isnan(joints).any(axis=1)].astype(int)

# Check that points lie within the image
inside = np.logical_and(
np.logical_and(joints[:, 1] < img_shape[2], joints[:, 1] > 0),
np.logical_and(joints[:, 2] < img_shape[1], joints[:, 2] > 0),
)
if not all(inside):
joints = joints[inside]

if joints.size: # Exclude images without labels
if joints.size:
data["joints"] = joints
train_data.append(data)
matlab_data.append(
Expand All @@ -787,7 +806,10 @@ def to_matlab_cell(array):
)
)

matlab_data = np.asarray(matlab_data, dtype=[("image", "O"), ("size", "O"), ("joints", "O")])
matlab_data = np.asarray(
matlab_data,
dtype=[("image", "O"), ("size", "O"), ("joints", "O")],
)
return train_data, matlab_data


Expand Down