TESTecdata.py

import pandas as pd
from datetime import datetime
import html
import re
from playwright.sync_api import sync_playwright
import time

CURRENCY_CHARS = "¥€$£₽₩₺₹"
currency_pattern = re.compile(f"[{re.escape(CURRENCY_CHARS)}]")

def clean_value(text: str) -> str:
    if not text:
        return ""
    s = html.unescape(str(text))
    s = "".join(ch for ch in s if ch.isprintable()).replace("\xa0", " ")
    s = currency_pattern.sub("", s)
    return re.sub(r"\s+", " ", s).strip()

def parse_month_year(text: str) -> datetime:
    return datetime.strptime(text.strip(), "%B %Y")

def scrape_mql5_calendar(first_year=2020, last_year=2026, first_month=1, last_month=1):
    all_data = []

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=False)
        context = browser.new_context()
        page = context.new_page()

        # Блокируем только тяжёлые ресурсы, JS не трогаем
        page.route("**/*", lambda route: (
            route.abort()
            if route.request.resource_type in ["image", "font"]
            else route.continue_()
        ))

        print("⏳ Заходим на сайт...")
        page.goto("https://www.mql5.com/en/economic-calendar", wait_until="domcontentloaded")

        # Открываем календарь
        date_input = page.locator("#filterDateInput")
        date_input.wait_for(state="visible", timeout=15000)
        date_input.click()
        page.wait_for_selector(".date-picker__part-left", state="visible", timeout=15000)

        first_target = datetime(first_year, first_month, 1)
        last_target  = datetime(last_year, last_month, 1)

        # ОТМОТКА НАЗАД до первой нужной даты
        while True:
            left_part  = page.query_selector(".date-picker__part-left")
            right_part = page.query_selector(".date-picker__part-right")

            left_text  = left_part.query_selector(".date-picker__month-title").inner_text().strip()
            right_text = right_part.query_selector(".date-picker__month-title").inner_text().strip()

            left_dt  = parse_month_year(left_text)
            right_dt = parse_month_year(right_text)

            if right_dt <= first_target:
                break

            left_part.query_selector(".date-picker__prev").click()
            right_part.query_selector(".date-picker__prev").click()  # чуть подождать перерисовку

        current = first_target
        print(f"Период: {first_year}-{last_year}")

        # ОСНОВНОЙ ЦИКЛ ВПЕРЁД
        while current < last_target:
            # убеждаемся, что календарь открыт
            if not page.is_visible(".date-picker__part-left"):
                page.locator("#filterDateInput").click()
                page.wait_for_selector(".date-picker__part-left", state="visible", timeout=15000)

            left_part  = page.query_selector(".date-picker__part-left")
            right_part = page.query_selector(".date-picker__part-right")

            left_text  = left_part.query_selector(".date-picker__month-title").inner_text().strip()
            right_text = right_part.query_selector(".date-picker__month-title").inner_text().strip()

            left_dt  = parse_month_year(left_text)
            right_dt = parse_month_year(right_text)

            if right_dt >= last_target:
                break

            # Выставляем 1 число в обоих календарях
            for part in (left_part, right_part):
                days_list = part.query_selector(".date-picker__days-list")
                if not days_list:
                    continue
                for li in days_list.query_selector_all("li"):
                    if li.get_attribute("data-day") == "1":
                        li.click()
                        break

            # Жмём Save
            save_btn = page.query_selector("button.buttonActive")
            if not save_btn:
                print("❌ Не нашли кнопку Save, выходим")
                break
            save_btn.click()
            
            # Ждём таблицу: сначала появление, потом обновление
            try:
                body = page.wait_for_selector(".ec-table__body", timeout=15000)
                # Доп. пауза, чтобы JS успел подгрузить строки
                page.wait_for_timeout(4500)
            except:
                print(f" Data scraping from-to {left_text} - {right_text} ... table is not visible")
                # Пробуем перейти к следующему месяцу
                page.locator("#filterDateInput").click()
                page.wait_for_selector(".date-picker__next", timeout=15000)
                left_part.query_selector(".date-picker__next").click()
                right_part.query_selector(".date-picker__next").click()
                if current.month == 12:
                    current = datetime(current.year + 1, 1, 1)
                else:
                    current = datetime(current.year, current.month + 1, 1)
                continue

            print(f"📥 Сбор данных за {left_text} - {right_text} ... ", end="")

            def grab_rows():
                return page.evaluate(
                    """
                    () => {
                        const results = [];
                        const rows = document.querySelectorAll('.ec-table__item, .ec-table__title');
                        let currentDateStr = "";
                        rows.forEach(row => {
                            if (row.classList.contains('ec-table__title')) {
                                currentDateStr = row.innerText.trim();
                            } else if (row.classList.contains('ec-table__item')) {
                                const txt = (sel) => {
                                    const el = row.querySelector(sel);
                                    return el ? el.innerText.trim() : "";
                                };
                                const attr = (sel, at) => {
                                    const el = row.querySelector(sel);
                                    return el ? el.getAttribute(at) : "";
                                };
                                results.push({
                                    d_str: currentDateStr,
                                    time: txt('.ec-table__col_time'),
                                    cur: txt('.ec-table__curency-name'),
                                    evt: txt('.ec-table__col_event a'),
                                    act: txt('.ec-table__col_actual'),
                                    fc: txt('.ec-table__col_forecast'),
                                    prev: txt('.ec-table__col_previous'),
                                    imp: attr('.ec-table__importance', 'title')
                                });
                            }
                        });
                        return results;
                    }
                    """
                )

            # первый заход
            js_data = grab_rows()

            # если строк подозрительно мало — подождать и попробовать ещё раз
            if len(js_data) < 1000:
                page.wait_for_timeout(5000)   # доп. ожидание 3 сек
                js_data = grab_rows()

            year_for_block = right_dt.year

            if js_data:
                for item in js_data:
                    raw_date = item["d_str"].split(",")[0].strip()
                    full_date_str = f"{raw_date} {year_for_block}"
                    try:
                        dt_obj = datetime.strptime(full_date_str, "%d %B %Y").strftime("%Y-%m-%d")
                    except Exception:
                        dt_obj = full_date_str

                    all_data.append({
                        "Date": dt_obj,
                        "Time": clean_value(item["time"]),
                        "Currency": clean_value(item["cur"]),
                        "Event": clean_value(item["evt"]),
                        "Actual": clean_value(item["act"]),
                        "Forecast": clean_value(item["fc"]),
                        "Previous": clean_value(item["prev"]),
                        "Importance": clean_value(item["imp"]),
                    })
                print(f"✅ +{len(js_data)} строк")
            else:
                print("❌ пусто")


            # Переход к следующему месяцу
            page.locator("#filterDateInput").click()
            page.wait_for_selector(".date-picker__next", state="visible", timeout=15000)
            left_part  = page.query_selector(".date-picker__part-left")
            right_part = page.query_selector(".date-picker__part-right")
            left_part.query_selector(".date-picker__next").click()
            right_part.query_selector(".date-picker__next").click()


            if current.month == 12:
                current = datetime(current.year + 1, 1, 1)
            else:
                current = datetime(current.year, current.month + 1, 1)

        browser.close()

    if all_data:
        df = pd.DataFrame(all_data)
        df = df.drop_duplicates()
        df = df.sort_values(by=["Date", "Time"])
        filename = f"mql5_calendar_{first_year}_{last_year}.csv"
        df.to_csv(filename, index=False, encoding="utf-8-sig")
        print(f"Ready! Saved in: {filename} ({len(df)} rows)")
        return df
    else:
        print("No data in")
        return None

if __name__ == "__main__":
    try:
        first_year = int(input("Please input the first year you wanna scrape data: "))
        last_year = int(input("Please input the last year you wanna scrape data: "))
        first_month = int(input("Please input the first month you wanna scrape data: "))
        last_month = int(input("Please input the last month you wanna scrape data: "))
        scrape_mql5_calendar(first_year, last_year, first_month, last_month)

    except ValueError:
        print("\n[Error]: Please enter only numbers (integers) for years and months.")

    except KeyboardInterrupt:
        print("\n\n[Stopped]: Process interrupted by the user. Goodbye!")

    except Exception as e:
        print(f"\n[Unexpected Error]: {e}")