"""
Doubling time per generation across all seeds of one experiment and variant, to
show whether doubling time drifts over generations.
Top panel: one line per seed, with a red x where a lineage ended in a cell that
failed and a purple square where a cell reached ``max_duration``. Run this after
the workflow finishes, because a cell that is still running also has no success
marker.
Middle panel: mean and standard deviation across seeds for each generation.
Bottom panel: histogram of doubling times after skipping initial generations.
Failed cells are excluded, except cells that reached ``max_duration``. Those are
counted at their doubling time even if ``fail_at_max_duration`` marked them as
failed, because they were still alive at the time limit.
All panels mark the target doubling time of the condition the cells ran in
(the config ``condition`` if set, otherwise the variant's sim_data condition).
Optional params:
skip_n_gens: leave this many initial generations out of the histogram and of
the printed and titled average. All generations are still shown in the
first two panels. Default: 2.
max_doubling_time: largest doubling time shown in the first two panels, in
minutes. Cells above this are drawn as triangles at the top of the first
panel. Default: fits the data, up to 3 x target.
"""
import os
import pickle
from typing import Any, cast
# noinspection PyUnresolvedReferences
from duckdb import DuckDBPyConnection
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator
import numpy as np
import polars as pl
from ecoli.library.parquet_emitter import read_stacked_columns
from wholecell.utils import units
DEATH_COLOR = "#d03030"
MAX_DURATION_COLOR = "#7b2fbe"
MEAN_COLOR = "#2a78d6"
TARGET_COLOR = "#52514e"
GRID_COLOR = "#e6e5e1"
[docs]
def get_config_condition(conn: DuckDBPyConnection, config_sql: str) -> str | None:
"""Return the ``condition`` set in the sim config, if any."""
try:
row = conn.sql(
f"SELECT DISTINCT condition FROM ({config_sql}) WHERE condition IS NOT NULL"
).fetchone()
except Exception:
# Config has no condition column
return None
return None if row is None else str(row[0])
[docs]
def get_target_doubling_time(
sim_data_dict: dict[str, dict[int, str]], condition: str | None
) -> float | None:
"""
Return the target doubling time in minutes of the condition the cells ran
in, if available. ``sim_data.doubling_time`` is always the basal doubling
time, so the target is looked up by condition instead: the ``condition``
from the sim config if set, otherwise the condition of the variant's
sim_data.
"""
for variants in sim_data_dict.values():
for sim_data_path in variants.values():
with open(sim_data_path, "rb") as f:
sim_data = pickle.load(f)
condition = condition or sim_data.condition
return float(
sim_data.condition_to_doubling_time[condition].asNumber(units.min)
)
return None
[docs]
def get_max_duration_minutes(conn: DuckDBPyConnection, config_sql: str) -> float | None:
"""
Return the shortest doubling time in minutes that counts as reaching
``max_duration``, allowing for the final time step not being emitted.
"""
row = conn.sql(
f"SELECT max(max_duration), max(time_step) FROM ({config_sql})"
).fetchone()
if row is None or row[0] is None:
return None
return (row[0] - 2 * (row[1] or 0)) / 60
[docs]
def load_doubling_times(
conn: DuckDBPyConnection, history_sql: str, success_sql: str
) -> pl.DataFrame:
"""
Return one row per cell with its doubling time in minutes and whether it
has a success marker.
"""
times_sql = cast(
str, read_stacked_columns(history_sql, ["time"], order_results=False)
)
return conn.sql(f"""
WITH cells AS (
SELECT experiment_id, variant, lineage_seed, generation, agent_id,
(max(time) - min(time)) / 60 AS doubling_time
FROM ({times_sql})
GROUP BY experiment_id, variant, lineage_seed, generation, agent_id
),
succeeded AS (
SELECT DISTINCT experiment_id, variant, lineage_seed, generation, agent_id
FROM ({success_sql})
)
SELECT cells.lineage_seed, cells.generation, cells.agent_id,
cells.doubling_time, succeeded.generation IS NOT NULL AS success
FROM cells
LEFT JOIN succeeded
ON cells.experiment_id = succeeded.experiment_id
AND cells.variant = succeeded.variant
AND cells.lineage_seed = succeeded.lineage_seed
AND cells.generation = succeeded.generation
AND cells.agent_id = succeeded.agent_id
ORDER BY cells.lineage_seed, cells.generation, cells.agent_id
""").pl()
[docs]
def plot(
params: dict[str, Any],
conn: DuckDBPyConnection,
history_sql: str,
config_sql: str,
success_sql: str,
sim_data_dict: dict[str, dict[int, str]],
validation_data_paths: list[str],
outdir: str,
variant_metadata: dict[str, dict[int, Any]],
variant_names: dict[str, str],
):
"""Plot doubling time trajectories for each seed and their mean and SD."""
cells = load_doubling_times(conn, history_sql, success_sql)
if cells.is_empty():
raise ValueError("No cells found in the data.")
completed = cells.filter(pl.col("success"))
if completed.is_empty():
raise ValueError("No successfully divided cells found in the data.")
# Cells of one seed and generation are averaged if single_daughters is off
per_seed = (
completed.group_by(["lineage_seed", "generation"])
.agg(pl.col("doubling_time").mean())
.sort(["lineage_seed", "generation"])
)
per_generation = (
per_seed.group_by("generation")
.agg(
pl.col("doubling_time").mean().alias("mean"),
pl.col("doubling_time").std().fill_null(0).alias("std"),
pl.len().alias("n"),
)
.sort("generation")
)
skip_n_gens = params.get("skip_n_gens", 2)
first_generation = cast(int, per_seed["generation"].min())
averaged = per_seed.filter(pl.col("generation") >= first_generation + skip_n_gens)
averaged_mean = cast(float, averaged["doubling_time"].mean())
averaged_std = cast(float, averaged["doubling_time"].std())
summary = (
f"{averaged_mean:.1f} ± {averaged_std:.1f} min after skipping "
f"{skip_n_gens} generations (n={averaged.height})"
if averaged.height > 1
else "too few cells for an average"
)
target = get_target_doubling_time(
sim_data_dict, get_config_condition(conn, config_sql)
)
observed_min = cast(float, per_seed["doubling_time"].min())
observed_max = cast(float, per_seed["doubling_time"].max())
y_max = params.get("max_doubling_time")
if y_max is None:
y_max = 1.05 * max(observed_max, target or 0)
if target:
y_max = min(y_max, 3 * target)
y_min = 0.9 * min(observed_min, target or observed_min)
seeds = per_seed["lineage_seed"].unique().sort().to_list()
cmap = plt.get_cmap("viridis")
fig = plt.figure(figsize=(10, 11.5))
grid = fig.add_gridspec(3, 1)
ax_seeds = fig.add_subplot(grid[0])
ax_mean = fig.add_subplot(grid[1], sharex=ax_seeds)
ax_hist = fig.add_subplot(grid[2])
for i, seed in enumerate(seeds):
seed_data = per_seed.filter(pl.col("lineage_seed") == seed)
generations = seed_data["generation"].to_numpy()
doubling_times = seed_data["doubling_time"].to_numpy()
clipped = np.minimum(doubling_times, y_max)
color = cmap(i / max(1, len(seeds) - 1))
ax_seeds.plot(
generations,
clipped,
color=color,
linewidth=0.9,
alpha=0.65,
marker=".",
markersize=3,
)
above = doubling_times > y_max
ax_seeds.plot(
generations[above], clipped[above], "^", color=color, markersize=7
)
max_duration = get_max_duration_minutes(conn, config_sql)
reached_limit = (
pl.col("doubling_time") >= max_duration
if max_duration is not None
else pl.lit(False)
)
at_limit = cells.filter(~pl.col("success") & reached_limit)
failed = cells.filter(~pl.col("success") & ~reached_limit)
# Mark each cell at the time limit at its own doubling time, connected to
# the last completed generation of its seed
for row in at_limit.iter_rows(named=True):
earlier = per_seed.filter(
(pl.col("lineage_seed") == row["lineage_seed"])
& (pl.col("generation") < row["generation"])
)
if earlier.height and row["lineage_seed"] in seeds:
seed_index = seeds.index(row["lineage_seed"])
ax_seeds.plot(
[earlier["generation"][-1], row["generation"]],
[
min(earlier["doubling_time"][-1], y_max),
min(row["doubling_time"], y_max),
],
color=cmap(seed_index / max(1, len(seeds) - 1)),
linewidth=0.9,
alpha=0.65,
)
ax_seeds.plot(
row["generation"],
min(row["doubling_time"], y_max),
"s",
color=MAX_DURATION_COLOR,
markersize=7,
zorder=5,
)
# Mark each failed cell at its lineage's last completed doubling time
for row in failed.iter_rows(named=True):
earlier = per_seed.filter(
(pl.col("lineage_seed") == row["lineage_seed"])
& (pl.col("generation") < row["generation"])
)
y_value = earlier["doubling_time"][-1] if earlier.height else y_min
ax_seeds.plot(
row["generation"],
min(y_value, y_max),
"x",
color=DEATH_COLOR,
markersize=9,
markeredgewidth=2.2,
zorder=5,
)
n_failed_seeds = failed["lineage_seed"].n_unique()
ax_seeds.plot(
[],
[],
"x",
color=DEATH_COLOR,
markersize=9,
markeredgewidth=2.2,
label=f"lineage failed ({n_failed_seeds} of {len(seeds)} seeds)",
)
if at_limit.height:
ax_seeds.plot(
[],
[],
"s",
color=MAX_DURATION_COLOR,
markersize=7,
label=(
f"reached max duration ({at_limit.height} "
f"{'cell' if at_limit.height == 1 else 'cells'})"
),
)
generations = per_generation["generation"].to_numpy()
means = per_generation["mean"].to_numpy()
stds = per_generation["std"].to_numpy()
ax_mean.plot(generations, means, color=MEAN_COLOR, linewidth=2, label="mean")
ax_mean.fill_between(
generations,
means - stds,
means + stds,
color=MEAN_COLOR,
alpha=0.18,
linewidth=0,
label="± SD across seeds",
)
for ax in (ax_seeds, ax_mean):
if target:
ax.axhline(
target,
color=TARGET_COLOR,
linestyle="--",
linewidth=1.2,
label=f"target ({target:.0f} min)",
)
ax.set_ylim(y_min, y_max * 1.03)
ax.set_ylabel("doubling time (min)")
ax.spines[["top", "right"]].set_visible(False)
ax.grid(axis="y", color=GRID_COLOR, linewidth=0.8)
ax.set_axisbelow(True)
ax.legend(fontsize=8, frameon=False, loc="upper right")
ax_seeds.set_title(f"Doubling time per seed: {summary}", loc="left")
ax_mean.set_title("Mean and SD across seeds", loc="left")
ax_mean.set_xlabel("generation")
ax_mean.xaxis.set_major_locator(MaxNLocator(integer=True))
# Histogram of completed cells and cells alive at the time limit
after_skip = cells.filter(pl.col("generation") >= first_generation + skip_n_gens)
included = after_skip.filter(pl.col("success") | reached_limit)
n_at_limit = after_skip.filter(~pl.col("success") & reached_limit).height
n_failed = after_skip.height - included.height
if included.is_empty():
ax_hist.text(0.5, 0.5, "no cells to show", transform=ax_hist.transAxes)
else:
values = included["doubling_time"].to_numpy()
failed_word = "generation" if n_failed == 1 else "generations"
label = (
f"{included.height} cells included "
f"({n_failed} failed {failed_word} excluded"
)
label += f", {n_at_limit} at max duration kept)" if n_at_limit else ")"
ax_hist.hist(values, bins=40, color=MEAN_COLOR, alpha=0.75, label=label)
ax_hist.axvline(
values.mean(),
color=DEATH_COLOR,
linewidth=1.5,
label=f"mean {values.mean():.1f} ± {values.std(ddof=1) if values.size > 1 else 0:.1f} min",
)
if target:
ax_hist.axvline(
target,
color=TARGET_COLOR,
linestyle="--",
linewidth=1.2,
label=f"target ({target:.0f} min)",
)
ax_hist.set_xlabel("doubling time (min)")
ax_hist.set_ylabel("cells")
ax_hist.yaxis.set_major_locator(MaxNLocator(integer=True))
ax_hist.set_title(
f"Doubling times, skipping the first {skip_n_gens} generations", loc="left"
)
ax_hist.spines[["top", "right"]].set_visible(False)
ax_hist.grid(axis="y", color=GRID_COLOR, linewidth=0.8)
ax_hist.set_axisbelow(True)
ax_hist.legend(fontsize=8, frameon=False, loc="upper right")
fig.tight_layout()
fig.savefig(os.path.join(outdir, "doubling_time_trajectories.pdf"))
fig.savefig(os.path.join(outdir, "doubling_time_trajectories.png"), dpi=150)
plt.close(fig)
print(f"Doubling time: {summary}")