Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Bug fix slow plot with datetimeindex#61414

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Closed
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletionsdoc/source/whatsnew/v3.0.0.rst
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -653,6 +653,7 @@ Performance improvements
- Performance improvement in :meth:`CategoricalDtype.update_dtype` when ``dtype`` is a :class:`CategoricalDtype` with non ``None`` categories and ordered (:issue:`59647`)
- Performance improvement in :meth:`DataFrame.__getitem__` when ``key`` is a :class:`DataFrame` with many columns (:issue:`61010`)
- Performance improvement in :meth:`DataFrame.astype` when converting to extension floating dtypes, e.g. "Float64" (:issue:`60066`)
- Performance improvement in :meth:`DataFrame.plot` when plotting DataFrames with DatetimeIndex and multiple columns (:issue:`61398`)
- Performance improvement in :meth:`DataFrame.stack` when using ``future_stack=True`` and the DataFrame does not have a :class:`MultiIndex` (:issue:`58391`)
- Performance improvement in :meth:`DataFrame.where` when ``cond`` is a :class:`DataFrame` with many columns (:issue:`61010`)
- Performance improvement in :meth:`to_hdf` avoid unnecessary reopenings of the HDF5 file to speedup data addition to files with a very large number of groups . (:issue:`58248`)
Expand Down
52 changes: 50 additions & 2 deletionspandas/plotting/_matplotlib/converter.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,7 @@
is_integer_dtype,
is_nested_list_like,
)
from pandas.core.dtypes.generic import ABCDatetimeIndex

from pandas import (
Index,
Expand DownExpand Up@@ -301,17 +302,37 @@ def try_parse(values):
except Exception:
return values

# Fast path for single values
if isinstance(values, (datetime, pydt.date, np.datetime64, pydt.time)):
return mdates.date2num(values)
elif is_integer(values) or is_float(values):
return values
elif isinstance(values, str):
return try_parse(values)
elif isinstance(values, (list, tuple, np.ndarray, Index, Series)):
# Check for cache to avoid redundant conversions
# This is especially important for DataFrames with the same DatetimeIndex
# for all columns
if isinstance(values, Index) and hasattr(axis, "_converter_cache"):
cache_key = id(values)
if cache_key in axis._converter_cache:
return axis._converter_cache[cache_key]

if isinstance(values, Series):
# https://github.com/matplotlib/matplotlib/issues/11391
# Series was skipped. Convert to DatetimeIndex to get asi8
values = Index(values)

# For DatetimeIndex objects, directly use _mpl_repr() for better efficiency
if isinstance(values, ABCDatetimeIndex):
result = values._mpl_repr()
# Cache result for reuse with subsequent columns
if hasattr(axis, "_converter_cache"):
axis._converter_cache[id(values)] = result
elif axis is not None:
axis._converter_cache = {id(values): result}
return result

if isinstance(values, Index):
values = values.values
if not isinstance(values, np.ndarray):
Expand All@@ -325,7 +346,15 @@ def try_parse(values):
except Exception:
pass

values = mdates.date2num(values)
result = mdates.date2num(values)

# Cache result if possible
if hasattr(axis, "_converter_cache"):
axis._converter_cache[id(values)] = result
elif axis is not None:
axis._converter_cache = {id(values): result}

return result

return values

Expand DownExpand Up@@ -426,10 +455,29 @@ def __call__(self):
)

interval = self._get_interval()
freq = f"{interval}ms"

# Use seconds instead of milliseconds for large intervals to improve performance
if interval >= 1000:
# Use seconds instead of ms for better performance
sec_interval = interval / 1000
freq = f"{sec_interval}s"
else:
freq = f"{interval}ms"

tz = self.tz.tzname(None)
st = dmin.replace(tzinfo=None)
ed = dmax.replace(tzinfo=None)

# Limit ticks for large date ranges to improve performance
date_diff = (ed - st).total_seconds()
if (
date_diff > 86400 * 365 and interval < 1000
): # Year+ of data with small interval
# Generate limited ticks for large datasets instead of a full date range
num_ticks = max_millis_ticks
tick_locs = np.linspace(mdates.date2num(st), mdates.date2num(ed), num_ticks)
return tick_locs

all_dates = date_range(start=st, end=ed, freq=freq, tz=tz).astype(object)

try:
Expand Down
184 changes: 176 additions & 8 deletionspandas/plotting/_matplotlib/core.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1549,9 +1549,157 @@ def __init__(self, data, **kwargs) -> None:
self.data = self.data.fillna(value=0)

def _make_plot(self, fig: Figure) -> None:
"""Create the plot.

This method contains a fast path optimization for DataFrames with DatetimeIndex
and multiple columns. For large DataFrames with DatetimeIndex, plotting can be
very slow due to the overhead of date conversions for each column.

The optimization follows this strategy:
1. For the first column only: Use standard DatetimeIndex plotting to get ticks
2. For remaining columns: Plot with a simpler numeric index (much faster)
3. Apply the datetime tick labels from the first plot to all other plots

This avoids redundant DatetimeIndex -> PeriodIndex conversions and tick
calculations when plotting many columns with the same index.

The optimization can yield a 10x+ speedup on large DataFrames with many columns.
"""
# Fast path for DatetimeIndex with many columns
# Implement the same strategy as the user's workaround that showed 11x speedup
if (
self._is_ts_plot()
and isinstance(self.data.index, ABCDatetimeIndex)
and len(self.data.columns) >= 2
): # Need at least 2 columns for this optimization
# Get the first axis for the plot
ax = self._get_ax(0)

# STEP 1: Plot only the first column to get datetime ticks
first_column = self.data.iloc[:, 0]
first_series = first_column.copy()
first_style = None

# Apply colors and style just for first column
colors = self._get_colors()
first_col_label = self.data.columns[0]
kwds = self.kwds.copy()
if self.color is not None:
kwds["color"] = self.color

# Set up style for first column
first_style, kwds = self._apply_style_colors(
colors,
kwds,
0,
first_col_label, # type: ignore[arg-type]
)

# Add label to kwds for the first column
first_label = pprint_thing(first_col_label)
first_label = self._mark_right_label(first_label, index=0)
kwds["label"] = first_label

# Plot the first column with DatetimeIndex to set up ticks
first_ax = self._get_ax(0)
# We need to specifically add column_num for stacking
kwds["column_num"] = 0
lines = self._ts_plot(
first_ax, None, first_series, style=first_style, **kwds
)

# Get the x-ticks and labels from the first plot
xticks = first_ax.get_xticks()
xticklabels = [label.get_text() for label in first_ax.get_xticklabels()]

# Keep reference to the first line for the legend
first_line = lines[0]
self._append_legend_handles_labels(first_line, first_label)

# STEP 2: Plot all columns with a numeric index (much faster)
# Reset axes for faster plotting
data_without_index = self.data.reset_index(drop=True)

# Plot remaining columns
stacking_id = self._get_stacking_id()
is_errorbar = com.any_not_none(*self.errors.values())

# Skip the first column and process the remaining ones
for i, (col_idx, (label, y)) in enumerate(
zip(
range(1, len(data_without_index.columns)),
list(data_without_index.items())[1:],
)
):
# Get the actual axis for this column - use the right column index
# Note: i is 0-based for the remaining columns after skipping the first
ax = self._get_ax(col_idx) # Use col_idx which starts from 1

# Reset kwds for each column
kwds = self.kwds.copy()
if self.color is not None:
kwds["color"] = self.color

# Apply style and colors
style, kwds = self._apply_style_colors(
colors,
kwds,
col_idx, # Use 1-based index to match column
label, # type: ignore[arg-type]
)

# Handle any error bars
errors = self._get_errorbars(label=label, index=col_idx)
kwds = dict(kwds, **errors)

# Format the label
label_str = pprint_thing(label)
label_str = self._mark_right_label(label_str, index=col_idx)
kwds["label"] = label_str

# Add column number for stacking
kwds["column_num"] = col_idx

try:
# Use regular plot (not ts_plot) for better performance
newlines = self._plot(
ax,
data_without_index.index, # Use numeric index for speed
np.asarray(y.values),
style=style,
stacking_id=stacking_id,
is_errorbar=is_errorbar,
**kwds,
)
self._append_legend_handles_labels(newlines[0], label_str)

# STEP 3: Apply the datetime x-axis formatting to each plot
# Use ticks from first plot for all subsequent plots
num_ticks = len(xticks)
new_xticks = np.linspace(0, len(self.data.index) - 1, num_ticks)
ax.set_xlim(0, len(self.data.index) - 1)
ax.set_xticks(new_xticks)
ax.set_xticklabels(xticklabels)
except Exception as e:
# If anything goes wrong with the plotting, log it but don't crash
# This ensures the fix doesn't introduce new issues
import warnings

warnings.warn(
f"Fast path plotting failed for column {col_idx}: {e!s}. "
"Falling back to regular plotting method for remaining columns",
stacklevel=2,
)
# Return without 'return' to fall back to the normal plotting path
break
else:
# If we've successfully plotted all columns, return from the method
# We've already plotted everything with the fast path
return

# Regular path for other cases
if self._is_ts_plot():
data = maybe_convert_index(self._get_ax(0), self.data)

x = data.index # dummy, not used
plotf = self._ts_plot
it = data.items()
Expand All@@ -1570,6 +1718,7 @@ def _make_plot(self, fig: Figure) -> None:
is_errorbar = com.any_not_none(*self.errors.values())

colors = self._get_colors()

for i, (label, y) in enumerate(it):
ax = self._get_ax(i)
kwds = self.kwds.copy()
Expand DownExpand Up@@ -1636,15 +1785,34 @@ def _ts_plot(self, ax: Axes, x, data: Series, style=None, **kwds):
# accept x to be consistent with normal plot func,
# x is not passed to tsplot as it uses data.index as x coordinate
# column_num must be in kwds for stacking purpose
freq, data = prepare_ts_data(data, ax, kwds)

# TODO #54485
ax._plot_data.append((data, self._kind, kwds)) # type: ignore[attr-defined]
# Optimization for multi-column DatetimeIndex plots
if hasattr(ax, "_datetime_ticks_setup_done") and kwds.get("column_num", 0) > 0:
# Skip the expensive date axis setup for columns after the first one
# We'll just copy the ticks from the first plot
freq = getattr(ax, "freq", None)
lines = self._plot(
ax, data.index, np.asarray(data.values), style=style, **kwds
)

if hasattr(ax, "_xticks") and hasattr(ax, "_xticklabels"):
# Use the stored ticks and labels from the first column plot
ax.set_xticks(ax._xticks)
ax.set_xticklabels(ax._xticklabels)
else:
# Regular path for first column or non-optimized plots
freq, data = prepare_ts_data(data, ax, kwds)

# TODO #54485
ax._plot_data.append((data, self._kind, kwds)) # type: ignore[attr-defined]

lines = self._plot(
ax, data.index, np.asarray(data.values), style=style, **kwds
)
# set date formatter, locators and rescale limits
# TODO #54485
format_dateaxis(ax, ax.freq, data.index) # type: ignore[arg-type, attr-defined]

lines = self._plot(ax, data.index, np.asarray(data.values), style=style, **kwds)
# set date formatter, locators and rescale limits
# TODO #54485
format_dateaxis(ax, ax.freq, data.index) # type: ignore[arg-type, attr-defined]
return lines

@final
Expand Down
Loading
Loading
[8]ページ先頭
©2009-2025 Movatter.jp