Interactive validation

This notebooks was used to record the ‘interactive validation’ demonstration video. The widget state is saved, but interactive elements will not function properly without a live Python kernel (e.g., in the online documentation).

import os
from pathlib import Path

from glob import glob
import pandas as pd

from vassi.config import cfg
from vassi.io import load_trajectories

# observation_library is a small package to separate the user iterface from the vassi code
# it will be installed alongside vassi as a dependency

from observation_library import ObservationLibrary
from observation_library.render_settings import RenderSettings

Social cichlids (annotations)

Configure trajectory keys

These keys need to be set so that the automated-scoring package can load trajectories from a H5 storage file

cfg.key_keypoints = "pose"
cfg.key_timestamp = "time_stamp"

cfg.trajectory_keys = ("pose", "time_stamp")

Organize videos, trajectories and observations

Prepare observations (can be either annotations or predictions). Here, load all available annotations.

Note that some columns should be categorical for better interactive filtering. The code below also rearranges columns for a better table structure.

observations = pd.read_csv("../../datasets/social_cichlids/cichlids_annotations.csv")
categorical_columns = ["category", "actor", "recipient", "group"]
sorted_columns = ["group", "actor", "recipient", "category", "start", "stop", "duration"]

observations = observations.dropna(axis=0)
observations[categorical_columns] = observations[categorical_columns].astype("category")
observations = observations[sorted_columns]

# some prefiltering for the example usecase
observations = observations[(observations["duration"] > 5) & (observations["category"] != "none")]

Create two dictionaries that link group IDs to both video files and trajectories.

video_directory = Path("~/Documents/social_cichlids/videos/").expanduser()
videos = sorted(video_directory.glob("*.MP4"))

video_lookup = {}
trajectory_lookup = {}

for idx, group in enumerate(sorted(observations["group"].unique())):
    video_lookup[group] = [videos[idx]]
    trajectory_lookup[group] = load_trajectories(
        "../../datasets/social_cichlids/cichlids_trajectories.h5",
        data_path=group,
    )

With these preparations done, display the observation library, an interactive widget to inspect behavioral datasets and to render video snippets.

observation_library = ObservationLibrary(
    observations,
    video_lookup=video_lookup,
    trajectory_lookup=trajectory_lookup,
    num_keypoints=3,
    highlight_observations_mode="category",
    filter_dependencies={
        "recipient": ("actor", "group", "category"),
        "actor": ("group", "category"),
    },
)

# preset a few rendering options
observation_library.render_settings.max_render_width = 2704
observation_library.render_settings.max_render_height = 1520
observation_library.render_settings.interval_padding = 2
observation_library.render_settings.highlight = True

observation_library

CALMS21 (predictions)

cfg.key_keypoints = "keypoints"
cfg.key_timestamp = "timestamps"

cfg.trajectory_keys = ("keypoints", "timestamps")

observations = pd.read_csv("../../datasets/CALMS21/pred/mice_pred_predictions.csv")
categorical_columns = ["category", "actor", "recipient", "true_category", "group"]
sorted_columns = ["group", "actor", "recipient", "category", "start", "stop", "start_min", "stop_min", "duration_s", "true_category", "mean_probability"]

observations = observations.dropna(axis=0)
observations[categorical_columns] = observations[categorical_columns].astype("category")

observations[["start_min", "stop_min"]] = observations[["start", "stop"]] / (30 * 60)
observations["duration_s"] = observations["duration"] / 30
observations = observations[sorted_columns]

observations["comment"] = ""
observations["valid"] = False

video_directory = Path("~/Downloads/task1_videos_mp4/test").expanduser()
videos = sorted(video_directory.glob("*.mp4"))

video_lookup = {}
trajectory_lookup = {}

for sequence in range(19):
    video_lookup[sequence] = [videos[sequence]]
    trajectory_lookup[sequence] = load_trajectories(
        "../../datasets/CALMS21/test/mice_test_trajectories.h5",
        data_path=str(sequence),
    )

Figure 1 from paper:

observation_library_mice = ObservationLibrary(
    observations,
    video_lookup=video_lookup,
    trajectory_lookup=trajectory_lookup,
    num_keypoints=7,
    selected_observations_mode="selected",
    visible_columns=[
        column for column in observations.columns if column not in ["start", "stop"]
    ]
)

# preset a few rendering options
observation_library_mice.render_settings.max_render_width = 2704
observation_library_mice.render_settings.max_render_height = 1520
observation_library_mice.render_settings.interval_padding = 2
observation_library_mice.render_settings.segments = [
    "0 - 1", "0 - 2", "1 - 3", "2 - 3", "3 - 4", "3 - 5", "4 - 6", "5 - 6"
]

observation_library_mice