diff --git a/pybaseball/datahelpers/postprocessing.py b/pybaseball/datahelpers/postprocessing.py index 5fb4bae9..6723c2ba 100644 --- a/pybaseball/datahelpers/postprocessing.py +++ b/pybaseball/datahelpers/postprocessing.py @@ -30,14 +30,15 @@ def try_parse_dataframe( if parse_numerics: data_copy = coalesce_nulls(data_copy, null_replacement) - data_copy = data_copy.apply( - pd.to_numeric, - errors='ignore', - downcast='signed' - ).convert_dtypes(convert_string=False) + for col in data_copy.columns: + try: + data_copy[col] = pd.to_numeric(data_copy[col], downcast='signed') + except (ValueError, TypeError): + pass + data_copy = data_copy.convert_dtypes(convert_string=False) string_columns = [ - dtype_tuple[0] for dtype_tuple in data_copy.dtypes.items() if str(dtype_tuple[1]) in ["object", "string"] + dtype_tuple[0] for dtype_tuple in data_copy.dtypes.items() if str(dtype_tuple[1]) in ["object", "string", "str"] ] for column in string_columns: # Only check the first value of the column and test that; diff --git a/pybaseball/retrosheet.py b/pybaseball/retrosheet.py index 391f55bb..7063900c 100644 --- a/pybaseball/retrosheet.py +++ b/pybaseball/retrosheet.py @@ -24,7 +24,7 @@ from pybaseball.utils import get_text_file from datetime import datetime from io import StringIO -from github import Github +from github import Auth, Github import os from getpass import getuser, getpass from github.GithubException import RateLimitExceededException @@ -131,7 +131,7 @@ def events(season, type='regular', export_dir='.'): "the valid types are: 'regular', 'post', and 'asg'.") try: - g = Github(GH_TOKEN) + g = Github(auth=Auth.Token(GH_TOKEN)) if GH_TOKEN else Github() repo = g.get_repo('chadwickbureau/retrosheet') season_folder = [f.path[f.path.rfind('/')+1:] for f in repo.get_contents(f'seasons/{season}')] season_events = [t for t in season_folder if t.endswith(file_extension)] @@ -156,7 +156,7 @@ def rosters(season): GH_TOKEN=os.getenv('GH_TOKEN', '') try: - g = Github(GH_TOKEN) + g = Github(auth=Auth.Token(GH_TOKEN)) if GH_TOKEN else Github() repo = g.get_repo('chadwickbureau/retrosheet') season_folder = [f.path[f.path.rfind('/')+1:] for f in repo.get_contents(f'seasons/{season}')] rosters = [t for t in season_folder if t.endswith('.ROS')] @@ -179,7 +179,7 @@ def _roster(team, season, checked = False): GH_TOKEN=os.getenv('GH_TOKEN', '') if not checked: - g = Github(GH_TOKEN) + g = Github(auth=Auth.Token(GH_TOKEN)) if GH_TOKEN else Github() try: repo = g.get_repo('chadwickbureau/retrosheet') season_folder = [f.path[f.path.rfind('/')+1:] for f in repo.get_contents(f'seasons/{season}')] @@ -213,7 +213,7 @@ def schedules(season): """ GH_TOKEN=os.getenv('GH_TOKEN', '') # validate input - g = Github(GH_TOKEN) + g = Github(auth=Auth.Token(GH_TOKEN)) if GH_TOKEN else Github() repo = g.get_repo('chadwickbureau/retrosheet') season_folder = [f.path[f.path.rfind('/')+1:] for f in repo.get_contents(f'seasons/{season}')] file_name = f'{season}schedule.csv' @@ -231,7 +231,7 @@ def season_game_logs(season): """ GH_TOKEN=os.getenv('GH_TOKEN', '') # validate input - g = Github(GH_TOKEN) + g = Github(auth=Auth.Token(GH_TOKEN)) if GH_TOKEN else Github() repo = g.get_repo('chadwickbureau/retrosheet') season_folder = [f.path[f.path.rfind('/')+1:] for f in repo.get_contents(f'seasons/{season}')] gamelog_file_name = f'GL{season}.TXT' diff --git a/pybaseball/team_batting.py b/pybaseball/team_batting.py index 8d8504dd..bcf7d1b0 100644 --- a/pybaseball/team_batting.py +++ b/pybaseball/team_batting.py @@ -40,10 +40,15 @@ def team_batting_bref(team: str, start_season: int, end_season: Optional[int]=No response = session.get(stats_url) soup = BeautifulSoup(response.content, 'html.parser') - table = soup.find_all('table', {'class': 'sortable stats_table'})[0] + table = soup.find('table', {'id': 'players_standard_batting'}) + thead = table.find('thead') if table is not None else None + if table is None or thead is None: + raise ValueError( + "Could not find batting data for {} {}. The page structure may have changed.".format(team, season) + ) if headings is None: - headings = [row.text.strip() for row in table.find_all('th')[1:28]] + headings = [th.text.strip() for th in thead.find_all('th')[1:]] rows = table.find_all('tr') for row in rows: diff --git a/pybaseball/team_fielding.py b/pybaseball/team_fielding.py index 7546e5e2..45fa7b4a 100644 --- a/pybaseball/team_fielding.py +++ b/pybaseball/team_fielding.py @@ -32,7 +32,12 @@ def team_fielding_bref(team: str, start_season: int, end_season: Optional[int]=N ) if end_season is None: end_season = start_season + if end_season < start_season: + raise ValueError( + "end_season must be greater than or equal to start_season." + ) + team = team.upper() url = "https://www.baseball-reference.com/teams/{}".format(team) raw_data = [] diff --git a/pybaseball/team_game_logs.py b/pybaseball/team_game_logs.py index 12e741a2..7470a165 100644 --- a/pybaseball/team_game_logs.py +++ b/pybaseball/team_game_logs.py @@ -22,6 +22,14 @@ def get_table(season: int, team: str, log_type: str) -> pd.DataFrame: return data +def _to_numeric_or_keep(column: pd.Series) -> pd.Series: + # pandas 3 removed errors="ignore" from pd.to_numeric + try: + return pd.to_numeric(column) + except (ValueError, TypeError): + return column + + def postprocess(data: pd.DataFrame) -> pd.DataFrame: #print(data.columns) data.drop([('Unnamed: 0_level_0', 'Rk')], axis=1, inplace=True) # drop index column @@ -36,7 +44,7 @@ def postprocess(data: pd.DataFrame) -> pd.DataFrame: data = data.rename(columns= repl_dict).copy() data[('Unnamed: 3_level_0','Home')] = data[('Unnamed: 3_level_0','Home')].isnull() # '@' if away, empty if home data = data[data[('Unnamed: 1_level_0','Game')] != 'Gtm'].copy() # drop empty month rows - data = data.apply(pd.to_numeric, errors="ignore") + data = data.apply(_to_numeric_or_keep) data[('Unnamed: 1_level_0','Game')] = data[('Unnamed: 1_level_0','Game')].astype(int) return data.reset_index(drop=True) diff --git a/pybaseball/team_pitching.py b/pybaseball/team_pitching.py index 0facc4f3..942d5449 100644 --- a/pybaseball/team_pitching.py +++ b/pybaseball/team_pitching.py @@ -42,10 +42,15 @@ def team_pitching_bref(team: str, start_season: int, end_season: Optional[int]=N response = session.get(stats_url) soup = BeautifulSoup(response.content, 'html.parser') - table = soup.find_all('table', {'id': 'team_pitching'})[0] + table = soup.find('table', {'id': 'players_standard_pitching'}) + thead = table.find('thead') if table is not None else None + if table is None or thead is None: + raise ValueError( + "Could not find pitching data for {} {}. The page structure may have changed.".format(team, season) + ) if headings is None: - headings = [row.text.strip() for row in table.find_all('th')[1:34]] + headings = [th.text.strip() for th in thead.find_all('th')[1:]] rows = table.find_all('tr') for row in rows: diff --git a/pybaseball/team_results.py b/pybaseball/team_results.py index 6e7c44ce..796b9615 100644 --- a/pybaseball/team_results.py +++ b/pybaseball/team_results.py @@ -72,7 +72,7 @@ def get_table(soup: BeautifulSoup, team: str) -> pd.DataFrame: df = df.rename(columns=df.iloc[0]) df = df.reindex(df.index.drop(0)) df = df.drop('', axis=1) #not a useful column - df['Attendance'].replace(r'^Unknown$', np.nan, regex=True, inplace = True) # make this a NaN so the column can benumeric + df['Attendance'] = df['Attendance'].replace(r'^Unknown$', np.nan, regex=True) # make this a NaN so the column can be numeric return df def process_win_streak(data: pd.DataFrame) -> pd.DataFrame: diff --git a/pybaseball/teamid_lookup.py b/pybaseball/teamid_lookup.py index 07bcba97..38af1503 100644 --- a/pybaseball/teamid_lookup.py +++ b/pybaseball/teamid_lookup.py @@ -24,6 +24,20 @@ def team_ids(season: Optional[int] = None, league: str = 'ALL') -> pd.DataFrame: fg_team_data = pd.read_csv(_DATA_FILENAME, index_col=0) + max_year = int(fg_team_data['yearID'].max()) + + # If the requested season is beyond the data, extrapolate from the last known year. + # The 30 franchises are unchanged since 2021 (when the bundled data ends), but the + # Athletics moved to Sacramento in 2025 and Baseball Reference and Retrosheet + # list them as ATH from that season on. + if season is not None and season > max_year: + last_year_data = fg_team_data[fg_team_data['yearID'] == max_year].copy() + last_year_data['yearID'] = season + if season >= 2025: + athletics = last_year_data['franchID'] == 'OAK' + last_year_data.loc[athletics, ['teamIDBR', 'teamIDretro']] = 'ATH' + fg_team_data = pd.concat([fg_team_data, last_year_data], ignore_index=True) + if season is not None: fg_team_data = fg_team_data.query(f"yearID == {season}") diff --git a/setup.py b/setup.py index 963e7f10..75aa4b15 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ 'requests>=2.18.1', 'lxml>=4.2.1', 'pyarrow>=1.0.1', - 'pygithub>=1.51', + 'pygithub>=1.59', 'scipy>=1.4.0', 'matplotlib>=2.0.0', 'tqdm>=4.50.0', diff --git a/tests/pybaseball/data/team_batting_bref.html b/tests/pybaseball/data/team_batting_bref.html new file mode 100644 index 00000000..c0e44b04 --- /dev/null +++ b/tests/pybaseball/data/team_batting_bref.html @@ -0,0 +1,11 @@ + + + + + + + + + +
RkTmBatAgeGPARHHRRBI
1New York Yankees28.516263009001400250860
2Boston Red Sox27.916262508501450220820
+ diff --git a/tests/pybaseball/data/team_pitching_bref.html b/tests/pybaseball/data/team_pitching_bref.html new file mode 100644 index 00000000..f1388976 --- /dev/null +++ b/tests/pybaseball/data/team_pitching_bref.html @@ -0,0 +1,11 @@ + + + + + + + + + +
RkTmPAgeGIPHERERASOBB
1New York Yankees29.11621450.013006003.721500500
2Boston Red Sox28.41621440.013506504.061400520
+ diff --git a/tests/pybaseball/data/team_results.html b/tests/pybaseball/data/team_results.html new file mode 100644 index 00000000..3c933b0a --- /dev/null +++ b/tests/pybaseball/data/team_results.html @@ -0,0 +1,12 @@ + + + + + + + + + + +
Gm#DateTmOppHAWLRRAInnRankGBWinLossSaveTimeDNStreakAttendance
2019-04-01NYYBALHomeW5391--SmithAJonesBNone3:01D+1Unknownx
2019-04-02NYYBALHomeL24921.0DoeCRoeDNone2:55N-140,000x
Description row to be skipped
+ diff --git a/tests/pybaseball/test_retrosheet.py b/tests/pybaseball/test_retrosheet.py new file mode 100644 index 00000000..3604ee78 --- /dev/null +++ b/tests/pybaseball/test_retrosheet.py @@ -0,0 +1,38 @@ +from unittest.mock import MagicMock + +import pytest + +import pybaseball.retrosheet as retrosheet + + +def test_events_uses_token_auth(monkeypatch: pytest.MonkeyPatch) -> None: + # Regression test for #455: when GH_TOKEN is set, the GitHub client must be + # built with the modern keyword auth API (Auth.Token) rather than the + # deprecated positional-token form. + monkeypatch.setenv('GH_TOKEN', 'dummy_token') + fake_github = MagicMock() + fake_github.return_value.get_repo.side_effect = RuntimeError("stop") + monkeypatch.setattr(retrosheet, 'Github', fake_github) + + with pytest.raises(RuntimeError): + retrosheet.events(2019) + + args, kwargs = fake_github.call_args + assert args == () + assert 'auth' in kwargs + + +def test_events_anonymous_without_token(monkeypatch: pytest.MonkeyPatch) -> None: + # Regression test for #455: with no GH_TOKEN, the client must be created + # anonymously (Github()) instead of passing an empty token string. + monkeypatch.delenv('GH_TOKEN', raising=False) + fake_github = MagicMock() + fake_github.return_value.get_repo.side_effect = RuntimeError("stop") + monkeypatch.setattr(retrosheet, 'Github', fake_github) + + with pytest.raises(RuntimeError): + retrosheet.events(2019) + + args, kwargs = fake_github.call_args + assert args == () + assert 'auth' not in kwargs diff --git a/tests/pybaseball/test_statcast.py b/tests/pybaseball/test_statcast.py index 91413fb5..c4bd913b 100644 --- a/tests/pybaseball/test_statcast.py +++ b/tests/pybaseball/test_statcast.py @@ -18,7 +18,10 @@ def _single_game_raw(get_data_file_contents: Callable[[str], str]) -> str: @pytest.fixture(name="single_game") def _single_game(get_data_file_dataframe: GetDataFrameCallable) -> pd.DataFrame: data = get_data_file_dataframe('single_game_request.csv', parse_dates=[2]) - data[data.columns[2]].apply(pd.to_datetime, errors='ignore', format=DATE_FORMAT) + try: + data[data.columns[2]] = pd.to_datetime(data[data.columns[2]], format=DATE_FORMAT) + except (ValueError, TypeError): + pass return data diff --git a/tests/pybaseball/test_team_batting.py b/tests/pybaseball/test_team_batting.py index d5e142ba..ceb4429e 100644 --- a/tests/pybaseball/test_team_batting.py +++ b/tests/pybaseball/test_team_batting.py @@ -25,3 +25,36 @@ def test_team_batting(response_get_monkeypatch: Callable, sample_html: str, samp team_batting_result = team_batting(season).reset_index(drop=True) pd.testing.assert_frame_equal(team_batting_result, sample_processed_result, check_dtype=False) + + +@pytest.fixture(name="sample_bref_html") +def _sample_bref_html(get_data_file_contents: Callable[[str], str]) -> str: + return get_data_file_contents('team_batting_bref.html') + + +def test_team_batting_bref(bref_get_monkeypatch: Callable, sample_bref_html: str) -> None: + # Regression test for #461: Baseball Reference changed the batting table to + # id='players_standard_batting' with a . team_batting_bref must parse + # the new structure instead of the removed 'sortable stats_table' class. + from pybaseball.team_batting import team_batting_bref + + bref_get_monkeypatch(sample_bref_html) + + result = team_batting_bref('NYY', 2019) + + assert result is not None + assert not result.empty + assert 'Tm' in result.columns + assert 'Year' in result.columns + assert (result['Year'] == 2019).all() + + +def test_team_batting_bref_missing_table_raises(bref_get_monkeypatch: Callable) -> None: + # Regression test for #461: a page without the expected table should raise a + # clear ValueError instead of an opaque IndexError. + from pybaseball.team_batting import team_batting_bref + + bref_get_monkeypatch("

no table here

") + + with pytest.raises(ValueError): + team_batting_bref('NYY', 2019) diff --git a/tests/pybaseball/test_team_fielding.py b/tests/pybaseball/test_team_fielding.py index 2ab17fa7..047813e5 100644 --- a/tests/pybaseball/test_team_fielding.py +++ b/tests/pybaseball/test_team_fielding.py @@ -25,3 +25,12 @@ def test_team_fielding(response_get_monkeypatch: Callable, sample_html: str, sam team_fielding_result = team_fielding(season).reset_index(drop=True) pd.testing.assert_frame_equal(team_fielding_result, sample_processed_result, check_dtype=False) + + +def test_team_fielding_bref_invalid_season_range() -> None: + # Regression test for #462: an end_season earlier than start_season should + # raise a clear ValueError before any network request is made. + from pybaseball.team_fielding import team_fielding_bref + + with pytest.raises(ValueError): + team_fielding_bref('NYY', 2019, 2018) diff --git a/tests/pybaseball/test_team_game_logs.py b/tests/pybaseball/test_team_game_logs.py new file mode 100644 index 00000000..3e2a2c28 --- /dev/null +++ b/tests/pybaseball/test_team_game_logs.py @@ -0,0 +1,45 @@ +import pandas as pd + +from pybaseball.team_game_logs import _to_numeric_or_keep, postprocess + + +def test_postprocess_converts_numeric_columns_and_keeps_text() -> None: + # Regression test: postprocess used DataFrame.apply(pd.to_numeric, errors="ignore"), + # and pandas 3 rejects errors="ignore", so team_game_logs raised ValueError. + columns = pd.MultiIndex.from_tuples([ + ('Unnamed: 0_level_0', 'Rk'), + ('Unnamed: 1_level_0', 'Gtm'), + ('Unnamed: 2_level_0', 'Date'), + ('Unnamed: 3_level_0', 'Unnamed: 3_level_1'), + ('Unnamed: 4_level_0', 'Opp'), + ('Batting', 'R'), + ]) + data = pd.DataFrame([ + ['1', '1', 'Mar 28', '@', 'NYY', '3'], + ['2', '2', 'Mar 29', None, 'NYY', '5'], + ['Rk', 'Gtm', 'Date', None, 'Opp', 'R'], + ['3', '3', 'Mar 30', None, 'NYY', '0'], + ['', '', '', None, '', '8'], + ], columns=columns) + + result = postprocess(data) + + assert len(result) == 3 + assert result[('Unnamed: 1_level_0', 'Game')].tolist() == [1, 2, 3] + assert result[('Unnamed: 3_level_0', 'Home')].tolist() == [False, True, True] + assert pd.api.types.is_numeric_dtype(result[('Batting', 'R')]) + assert result[('Batting', 'R')].tolist() == [3, 5, 0] + assert result[('Unnamed: 4_level_0', 'Opp')].tolist() == ['NYY', 'NYY', 'NYY'] + + +def test_numeric_conversion_handles_duplicate_column_labels() -> None: + # Selecting a duplicated label returns a DataFrame, so a per-column + # pd.to_numeric loop would silently leave every column as text. + columns = pd.MultiIndex.from_tuples([('Batting', 'R'), ('Batting', 'R'), ('Opp', 'Opp')]) + data = pd.DataFrame([['3', '4', 'NYY'], ['5', '6', 'BOS']], columns=columns) + + result = data.apply(_to_numeric_or_keep) + + assert result[('Batting', 'R')].values.tolist() == [[3, 4], [5, 6]] + assert all(pd.api.types.is_numeric_dtype(dtype) for dtype in result[('Batting', 'R')].dtypes) + assert result.iloc[:, 2].tolist() == ['NYY', 'BOS'] diff --git a/tests/pybaseball/test_team_pitching.py b/tests/pybaseball/test_team_pitching.py index 5a554fc9..61888971 100644 --- a/tests/pybaseball/test_team_pitching.py +++ b/tests/pybaseball/test_team_pitching.py @@ -45,3 +45,36 @@ def test_team_pitching_relievers(response_get_monkeypatch: Callable, sample_html team_pitching_relievers_result = team_pitching_relievers(season).reset_index(drop=True) pd.testing.assert_frame_equal(team_pitching_relievers_result, sample_processed_result, check_dtype=False) + + +@pytest.fixture(name="sample_bref_html") +def _sample_bref_html(get_data_file_contents: Callable[[str], str]) -> str: + return get_data_file_contents('team_pitching_bref.html') + + +def test_team_pitching_bref(bref_get_monkeypatch: Callable, sample_bref_html: str) -> None: + # Regression test for #461: Baseball Reference changed the pitching table to + # id='players_standard_pitching' with a . team_pitching_bref must parse + # the new structure instead of the removed 'team_pitching' id. + from pybaseball.team_pitching import team_pitching_bref + + bref_get_monkeypatch(sample_bref_html) + + result = team_pitching_bref('NYY', 2019) + + assert result is not None + assert not result.empty + assert 'Tm' in result.columns + assert 'Year' in result.columns + assert (result['Year'] == 2019).all() + + +def test_team_pitching_bref_missing_table_raises(bref_get_monkeypatch: Callable) -> None: + # Regression test for #461: a page without the expected table should raise a + # clear ValueError instead of an opaque IndexError. + from pybaseball.team_pitching import team_pitching_bref + + bref_get_monkeypatch("

no table here

") + + with pytest.raises(ValueError): + team_pitching_bref('NYY', 2019) diff --git a/tests/pybaseball/test_team_results.py b/tests/pybaseball/test_team_results.py new file mode 100644 index 00000000..42f0232b --- /dev/null +++ b/tests/pybaseball/test_team_results.py @@ -0,0 +1,19 @@ +from typing import Callable + +from bs4 import BeautifulSoup + +from pybaseball.team_results import get_table + + +def test_get_table_unknown_attendance_becomes_nan(get_data_file_contents: Callable[[str], str]) -> None: + # Regression test for #459: 'Unknown' attendance values must be converted to + # NaN so the column can be made numeric. The previous chained-assignment + # `inplace=True` form silently failed under pandas copy-on-write (and emitted + # a FutureWarning), leaving the value as the string 'Unknown'. + soup = BeautifulSoup(get_data_file_contents('team_results.html'), 'lxml') + + result = get_table(soup, 'NYY') + + assert 'Attendance' in result.columns + assert result['Attendance'].isna().any() + assert 'Unknown' not in result['Attendance'].tolist() diff --git a/tests/pybaseball/test_teamid_lookup.py b/tests/pybaseball/test_teamid_lookup.py index 87d05e00..5ea6d8ef 100644 --- a/tests/pybaseball/test_teamid_lookup.py +++ b/tests/pybaseball/test_teamid_lookup.py @@ -141,3 +141,40 @@ def test_get_close_team_matches() -> None: for _, row in lahman_teams.iterrows(): assert _get_close_team_matches(row, fg_teams) == row.expected + + +def test_team_id_lookup_recent_season() -> None: + # Regression test for #486: seasons after the bundled data's final year + # previously returned an empty DataFrame. team_ids should now extrapolate + # from the most recent known year (the 30 franchises are unchanged). + from pybaseball.teamid_lookup import _DATA_FILENAME + + max_year = int(pd.read_csv(_DATA_FILENAME, index_col=0)['yearID'].max()) + recent_season = max_year + 2 + + result = team_ids(recent_season) + + assert result is not None + assert not result.empty + assert len(result.columns) == 7 + assert len(result) == 30 + assert (result['yearID'] == recent_season).all() + + +def test_team_id_lookup_athletics_abbreviation_after_move() -> None: + # The Athletics are OAK through 2024 and ATH on Baseball Reference and + # Retrosheet from 2025, when they moved to Sacramento. + from pybaseball.teamid_lookup import _DATA_FILENAME + + max_year = int(pd.read_csv(_DATA_FILENAME, index_col=0)['yearID'].max()) + if max_year >= 2024: + pytest.skip('bundled data now covers 2024; extrapolation is not used') + + before = team_ids(2024).set_index('franchID').loc['OAK'] + assert before['teamIDBR'] == 'OAK' + assert before['teamIDretro'] == 'OAK' + + after = team_ids(2025).set_index('franchID').loc['OAK'] + assert after['teamIDBR'] == 'ATH' + assert after['teamIDretro'] == 'ATH' + assert after['teamIDfg'] == before['teamIDfg']