diff --git a/pybaseball/datahelpers/postprocessing.py b/pybaseball/datahelpers/postprocessing.py index 5fb4bae9..6723c2ba 100644 --- a/pybaseball/datahelpers/postprocessing.py +++ b/pybaseball/datahelpers/postprocessing.py @@ -30,14 +30,15 @@ def try_parse_dataframe( if parse_numerics: data_copy = coalesce_nulls(data_copy, null_replacement) - data_copy = data_copy.apply( - pd.to_numeric, - errors='ignore', - downcast='signed' - ).convert_dtypes(convert_string=False) + for col in data_copy.columns: + try: + data_copy[col] = pd.to_numeric(data_copy[col], downcast='signed') + except (ValueError, TypeError): + pass + data_copy = data_copy.convert_dtypes(convert_string=False) string_columns = [ - dtype_tuple[0] for dtype_tuple in data_copy.dtypes.items() if str(dtype_tuple[1]) in ["object", "string"] + dtype_tuple[0] for dtype_tuple in data_copy.dtypes.items() if str(dtype_tuple[1]) in ["object", "string", "str"] ] for column in string_columns: # Only check the first value of the column and test that; diff --git a/pybaseball/retrosheet.py b/pybaseball/retrosheet.py index 391f55bb..7063900c 100644 --- a/pybaseball/retrosheet.py +++ b/pybaseball/retrosheet.py @@ -24,7 +24,7 @@ from pybaseball.utils import get_text_file from datetime import datetime from io import StringIO -from github import Github +from github import Auth, Github import os from getpass import getuser, getpass from github.GithubException import RateLimitExceededException @@ -131,7 +131,7 @@ def events(season, type='regular', export_dir='.'): "the valid types are: 'regular', 'post', and 'asg'.") try: - g = Github(GH_TOKEN) + g = Github(auth=Auth.Token(GH_TOKEN)) if GH_TOKEN else Github() repo = g.get_repo('chadwickbureau/retrosheet') season_folder = [f.path[f.path.rfind('/')+1:] for f in repo.get_contents(f'seasons/{season}')] season_events = [t for t in season_folder if t.endswith(file_extension)] @@ -156,7 +156,7 @@ def rosters(season): GH_TOKEN=os.getenv('GH_TOKEN', '') try: - g = Github(GH_TOKEN) + g = Github(auth=Auth.Token(GH_TOKEN)) if GH_TOKEN else Github() repo = g.get_repo('chadwickbureau/retrosheet') season_folder = [f.path[f.path.rfind('/')+1:] for f in repo.get_contents(f'seasons/{season}')] rosters = [t for t in season_folder if t.endswith('.ROS')] @@ -179,7 +179,7 @@ def _roster(team, season, checked = False): GH_TOKEN=os.getenv('GH_TOKEN', '') if not checked: - g = Github(GH_TOKEN) + g = Github(auth=Auth.Token(GH_TOKEN)) if GH_TOKEN else Github() try: repo = g.get_repo('chadwickbureau/retrosheet') season_folder = [f.path[f.path.rfind('/')+1:] for f in repo.get_contents(f'seasons/{season}')] @@ -213,7 +213,7 @@ def schedules(season): """ GH_TOKEN=os.getenv('GH_TOKEN', '') # validate input - g = Github(GH_TOKEN) + g = Github(auth=Auth.Token(GH_TOKEN)) if GH_TOKEN else Github() repo = g.get_repo('chadwickbureau/retrosheet') season_folder = [f.path[f.path.rfind('/')+1:] for f in repo.get_contents(f'seasons/{season}')] file_name = f'{season}schedule.csv' @@ -231,7 +231,7 @@ def season_game_logs(season): """ GH_TOKEN=os.getenv('GH_TOKEN', '') # validate input - g = Github(GH_TOKEN) + g = Github(auth=Auth.Token(GH_TOKEN)) if GH_TOKEN else Github() repo = g.get_repo('chadwickbureau/retrosheet') season_folder = [f.path[f.path.rfind('/')+1:] for f in repo.get_contents(f'seasons/{season}')] gamelog_file_name = f'GL{season}.TXT' diff --git a/pybaseball/team_batting.py b/pybaseball/team_batting.py index 8d8504dd..bcf7d1b0 100644 --- a/pybaseball/team_batting.py +++ b/pybaseball/team_batting.py @@ -40,10 +40,15 @@ def team_batting_bref(team: str, start_season: int, end_season: Optional[int]=No response = session.get(stats_url) soup = BeautifulSoup(response.content, 'html.parser') - table = soup.find_all('table', {'class': 'sortable stats_table'})[0] + table = soup.find('table', {'id': 'players_standard_batting'}) + thead = table.find('thead') if table is not None else None + if table is None or thead is None: + raise ValueError( + "Could not find batting data for {} {}. The page structure may have changed.".format(team, season) + ) if headings is None: - headings = [row.text.strip() for row in table.find_all('th')[1:28]] + headings = [th.text.strip() for th in thead.find_all('th')[1:]] rows = table.find_all('tr') for row in rows: diff --git a/pybaseball/team_fielding.py b/pybaseball/team_fielding.py index 7546e5e2..45fa7b4a 100644 --- a/pybaseball/team_fielding.py +++ b/pybaseball/team_fielding.py @@ -32,7 +32,12 @@ def team_fielding_bref(team: str, start_season: int, end_season: Optional[int]=N ) if end_season is None: end_season = start_season + if end_season < start_season: + raise ValueError( + "end_season must be greater than or equal to start_season." + ) + team = team.upper() url = "https://www.baseball-reference.com/teams/{}".format(team) raw_data = [] diff --git a/pybaseball/team_game_logs.py b/pybaseball/team_game_logs.py index 12e741a2..7470a165 100644 --- a/pybaseball/team_game_logs.py +++ b/pybaseball/team_game_logs.py @@ -22,6 +22,14 @@ def get_table(season: int, team: str, log_type: str) -> pd.DataFrame: return data +def _to_numeric_or_keep(column: pd.Series) -> pd.Series: + # pandas 3 removed errors="ignore" from pd.to_numeric + try: + return pd.to_numeric(column) + except (ValueError, TypeError): + return column + + def postprocess(data: pd.DataFrame) -> pd.DataFrame: #print(data.columns) data.drop([('Unnamed: 0_level_0', 'Rk')], axis=1, inplace=True) # drop index column @@ -36,7 +44,7 @@ def postprocess(data: pd.DataFrame) -> pd.DataFrame: data = data.rename(columns= repl_dict).copy() data[('Unnamed: 3_level_0','Home')] = data[('Unnamed: 3_level_0','Home')].isnull() # '@' if away, empty if home data = data[data[('Unnamed: 1_level_0','Game')] != 'Gtm'].copy() # drop empty month rows - data = data.apply(pd.to_numeric, errors="ignore") + data = data.apply(_to_numeric_or_keep) data[('Unnamed: 1_level_0','Game')] = data[('Unnamed: 1_level_0','Game')].astype(int) return data.reset_index(drop=True) diff --git a/pybaseball/team_pitching.py b/pybaseball/team_pitching.py index 0facc4f3..942d5449 100644 --- a/pybaseball/team_pitching.py +++ b/pybaseball/team_pitching.py @@ -42,10 +42,15 @@ def team_pitching_bref(team: str, start_season: int, end_season: Optional[int]=N response = session.get(stats_url) soup = BeautifulSoup(response.content, 'html.parser') - table = soup.find_all('table', {'id': 'team_pitching'})[0] + table = soup.find('table', {'id': 'players_standard_pitching'}) + thead = table.find('thead') if table is not None else None + if table is None or thead is None: + raise ValueError( + "Could not find pitching data for {} {}. The page structure may have changed.".format(team, season) + ) if headings is None: - headings = [row.text.strip() for row in table.find_all('th')[1:34]] + headings = [th.text.strip() for th in thead.find_all('th')[1:]] rows = table.find_all('tr') for row in rows: diff --git a/pybaseball/team_results.py b/pybaseball/team_results.py index 6e7c44ce..796b9615 100644 --- a/pybaseball/team_results.py +++ b/pybaseball/team_results.py @@ -72,7 +72,7 @@ def get_table(soup: BeautifulSoup, team: str) -> pd.DataFrame: df = df.rename(columns=df.iloc[0]) df = df.reindex(df.index.drop(0)) df = df.drop('', axis=1) #not a useful column - df['Attendance'].replace(r'^Unknown$', np.nan, regex=True, inplace = True) # make this a NaN so the column can benumeric + df['Attendance'] = df['Attendance'].replace(r'^Unknown$', np.nan, regex=True) # make this a NaN so the column can be numeric return df def process_win_streak(data: pd.DataFrame) -> pd.DataFrame: diff --git a/pybaseball/teamid_lookup.py b/pybaseball/teamid_lookup.py index 07bcba97..38af1503 100644 --- a/pybaseball/teamid_lookup.py +++ b/pybaseball/teamid_lookup.py @@ -24,6 +24,20 @@ def team_ids(season: Optional[int] = None, league: str = 'ALL') -> pd.DataFrame: fg_team_data = pd.read_csv(_DATA_FILENAME, index_col=0) + max_year = int(fg_team_data['yearID'].max()) + + # If the requested season is beyond the data, extrapolate from the last known year. + # The 30 franchises are unchanged since 2021 (when the bundled data ends), but the + # Athletics moved to Sacramento in 2025 and Baseball Reference and Retrosheet + # list them as ATH from that season on. + if season is not None and season > max_year: + last_year_data = fg_team_data[fg_team_data['yearID'] == max_year].copy() + last_year_data['yearID'] = season + if season >= 2025: + athletics = last_year_data['franchID'] == 'OAK' + last_year_data.loc[athletics, ['teamIDBR', 'teamIDretro']] = 'ATH' + fg_team_data = pd.concat([fg_team_data, last_year_data], ignore_index=True) + if season is not None: fg_team_data = fg_team_data.query(f"yearID == {season}") diff --git a/setup.py b/setup.py index 963e7f10..75aa4b15 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ 'requests>=2.18.1', 'lxml>=4.2.1', 'pyarrow>=1.0.1', - 'pygithub>=1.51', + 'pygithub>=1.59', 'scipy>=1.4.0', 'matplotlib>=2.0.0', 'tqdm>=4.50.0', diff --git a/tests/pybaseball/data/team_batting_bref.html b/tests/pybaseball/data/team_batting_bref.html new file mode 100644 index 00000000..c0e44b04 --- /dev/null +++ b/tests/pybaseball/data/team_batting_bref.html @@ -0,0 +1,11 @@ +
+| Rk | Tm | BatAge | G | PA | R | H | HR | RBI |
|---|---|---|---|---|---|---|---|---|
| 1 | New York Yankees | 28.5 | 162 | 6300 | 900 | 1400 | 250 | 860 |
| 2 | Boston Red Sox | 27.9 | 162 | 6250 | 850 | 1450 | 220 | 820 |
| Rk | Tm | PAge | G | IP | H | ER | ERA | SO | BB |
|---|---|---|---|---|---|---|---|---|---|
| 1 | New York Yankees | 29.1 | 162 | 1450.0 | 1300 | 600 | 3.72 | 1500 | 500 |
| 2 | Boston Red Sox | 28.4 | 162 | 1440.0 | 1350 | 650 | 4.06 | 1400 | 520 |
| Gm# | Date | Tm | Opp | HA | WL | R | RA | Inn | Rank | GB | Win | Loss | Save | Time | DN | Streak | Attendance | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2019-04-01 | NYY | BAL | Home | W | 5 | 3 | 9 | 1 | -- | SmithA | JonesB | None | 3:01 | D | +1 | Unknown | x | |
| 2019-04-02 | NYY | BAL | Home | L | 2 | 4 | 9 | 2 | 1.0 | DoeC | RoeD | None | 2:55 | N | -1 | 40,000 | x | |
| Description row to be skipped |
no table here
") + + with pytest.raises(ValueError): + team_batting_bref('NYY', 2019) diff --git a/tests/pybaseball/test_team_fielding.py b/tests/pybaseball/test_team_fielding.py index 2ab17fa7..047813e5 100644 --- a/tests/pybaseball/test_team_fielding.py +++ b/tests/pybaseball/test_team_fielding.py @@ -25,3 +25,12 @@ def test_team_fielding(response_get_monkeypatch: Callable, sample_html: str, sam team_fielding_result = team_fielding(season).reset_index(drop=True) pd.testing.assert_frame_equal(team_fielding_result, sample_processed_result, check_dtype=False) + + +def test_team_fielding_bref_invalid_season_range() -> None: + # Regression test for #462: an end_season earlier than start_season should + # raise a clear ValueError before any network request is made. + from pybaseball.team_fielding import team_fielding_bref + + with pytest.raises(ValueError): + team_fielding_bref('NYY', 2019, 2018) diff --git a/tests/pybaseball/test_team_game_logs.py b/tests/pybaseball/test_team_game_logs.py new file mode 100644 index 00000000..3e2a2c28 --- /dev/null +++ b/tests/pybaseball/test_team_game_logs.py @@ -0,0 +1,45 @@ +import pandas as pd + +from pybaseball.team_game_logs import _to_numeric_or_keep, postprocess + + +def test_postprocess_converts_numeric_columns_and_keeps_text() -> None: + # Regression test: postprocess used DataFrame.apply(pd.to_numeric, errors="ignore"), + # and pandas 3 rejects errors="ignore", so team_game_logs raised ValueError. + columns = pd.MultiIndex.from_tuples([ + ('Unnamed: 0_level_0', 'Rk'), + ('Unnamed: 1_level_0', 'Gtm'), + ('Unnamed: 2_level_0', 'Date'), + ('Unnamed: 3_level_0', 'Unnamed: 3_level_1'), + ('Unnamed: 4_level_0', 'Opp'), + ('Batting', 'R'), + ]) + data = pd.DataFrame([ + ['1', '1', 'Mar 28', '@', 'NYY', '3'], + ['2', '2', 'Mar 29', None, 'NYY', '5'], + ['Rk', 'Gtm', 'Date', None, 'Opp', 'R'], + ['3', '3', 'Mar 30', None, 'NYY', '0'], + ['', '', '', None, '', '8'], + ], columns=columns) + + result = postprocess(data) + + assert len(result) == 3 + assert result[('Unnamed: 1_level_0', 'Game')].tolist() == [1, 2, 3] + assert result[('Unnamed: 3_level_0', 'Home')].tolist() == [False, True, True] + assert pd.api.types.is_numeric_dtype(result[('Batting', 'R')]) + assert result[('Batting', 'R')].tolist() == [3, 5, 0] + assert result[('Unnamed: 4_level_0', 'Opp')].tolist() == ['NYY', 'NYY', 'NYY'] + + +def test_numeric_conversion_handles_duplicate_column_labels() -> None: + # Selecting a duplicated label returns a DataFrame, so a per-column + # pd.to_numeric loop would silently leave every column as text. + columns = pd.MultiIndex.from_tuples([('Batting', 'R'), ('Batting', 'R'), ('Opp', 'Opp')]) + data = pd.DataFrame([['3', '4', 'NYY'], ['5', '6', 'BOS']], columns=columns) + + result = data.apply(_to_numeric_or_keep) + + assert result[('Batting', 'R')].values.tolist() == [[3, 4], [5, 6]] + assert all(pd.api.types.is_numeric_dtype(dtype) for dtype in result[('Batting', 'R')].dtypes) + assert result.iloc[:, 2].tolist() == ['NYY', 'BOS'] diff --git a/tests/pybaseball/test_team_pitching.py b/tests/pybaseball/test_team_pitching.py index 5a554fc9..61888971 100644 --- a/tests/pybaseball/test_team_pitching.py +++ b/tests/pybaseball/test_team_pitching.py @@ -45,3 +45,36 @@ def test_team_pitching_relievers(response_get_monkeypatch: Callable, sample_html team_pitching_relievers_result = team_pitching_relievers(season).reset_index(drop=True) pd.testing.assert_frame_equal(team_pitching_relievers_result, sample_processed_result, check_dtype=False) + + +@pytest.fixture(name="sample_bref_html") +def _sample_bref_html(get_data_file_contents: Callable[[str], str]) -> str: + return get_data_file_contents('team_pitching_bref.html') + + +def test_team_pitching_bref(bref_get_monkeypatch: Callable, sample_bref_html: str) -> None: + # Regression test for #461: Baseball Reference changed the pitching table to + # id='players_standard_pitching' with a . team_pitching_bref must parse + # the new structure instead of the removed 'team_pitching' id. + from pybaseball.team_pitching import team_pitching_bref + + bref_get_monkeypatch(sample_bref_html) + + result = team_pitching_bref('NYY', 2019) + + assert result is not None + assert not result.empty + assert 'Tm' in result.columns + assert 'Year' in result.columns + assert (result['Year'] == 2019).all() + + +def test_team_pitching_bref_missing_table_raises(bref_get_monkeypatch: Callable) -> None: + # Regression test for #461: a page without the expected table should raise a + # clear ValueError instead of an opaque IndexError. + from pybaseball.team_pitching import team_pitching_bref + + bref_get_monkeypatch("no table here
") + + with pytest.raises(ValueError): + team_pitching_bref('NYY', 2019) diff --git a/tests/pybaseball/test_team_results.py b/tests/pybaseball/test_team_results.py new file mode 100644 index 00000000..42f0232b --- /dev/null +++ b/tests/pybaseball/test_team_results.py @@ -0,0 +1,19 @@ +from typing import Callable + +from bs4 import BeautifulSoup + +from pybaseball.team_results import get_table + + +def test_get_table_unknown_attendance_becomes_nan(get_data_file_contents: Callable[[str], str]) -> None: + # Regression test for #459: 'Unknown' attendance values must be converted to + # NaN so the column can be made numeric. The previous chained-assignment + # `inplace=True` form silently failed under pandas copy-on-write (and emitted + # a FutureWarning), leaving the value as the string 'Unknown'. + soup = BeautifulSoup(get_data_file_contents('team_results.html'), 'lxml') + + result = get_table(soup, 'NYY') + + assert 'Attendance' in result.columns + assert result['Attendance'].isna().any() + assert 'Unknown' not in result['Attendance'].tolist() diff --git a/tests/pybaseball/test_teamid_lookup.py b/tests/pybaseball/test_teamid_lookup.py index 87d05e00..5ea6d8ef 100644 --- a/tests/pybaseball/test_teamid_lookup.py +++ b/tests/pybaseball/test_teamid_lookup.py @@ -141,3 +141,40 @@ def test_get_close_team_matches() -> None: for _, row in lahman_teams.iterrows(): assert _get_close_team_matches(row, fg_teams) == row.expected + + +def test_team_id_lookup_recent_season() -> None: + # Regression test for #486: seasons after the bundled data's final year + # previously returned an empty DataFrame. team_ids should now extrapolate + # from the most recent known year (the 30 franchises are unchanged). + from pybaseball.teamid_lookup import _DATA_FILENAME + + max_year = int(pd.read_csv(_DATA_FILENAME, index_col=0)['yearID'].max()) + recent_season = max_year + 2 + + result = team_ids(recent_season) + + assert result is not None + assert not result.empty + assert len(result.columns) == 7 + assert len(result) == 30 + assert (result['yearID'] == recent_season).all() + + +def test_team_id_lookup_athletics_abbreviation_after_move() -> None: + # The Athletics are OAK through 2024 and ATH on Baseball Reference and + # Retrosheet from 2025, when they moved to Sacramento. + from pybaseball.teamid_lookup import _DATA_FILENAME + + max_year = int(pd.read_csv(_DATA_FILENAME, index_col=0)['yearID'].max()) + if max_year >= 2024: + pytest.skip('bundled data now covers 2024; extrapolation is not used') + + before = team_ids(2024).set_index('franchID').loc['OAK'] + assert before['teamIDBR'] == 'OAK' + assert before['teamIDretro'] == 'OAK' + + after = team_ids(2025).set_index('franchID').loc['OAK'] + assert after['teamIDBR'] == 'ATH' + assert after['teamIDretro'] == 'ATH' + assert after['teamIDfg'] == before['teamIDfg']