montana/Русский/Логистика/mt_detail_probe.py

#!/usr/bin/env python3
"""
MT Detail Probe — Find the XHR endpoint that returns ownership data with website/address.
Intercepts all network calls on vessel detail page and saves them.
"""
import asyncio, json, sys, os, time, struct, hmac, hashlib, base64

os.chdir(os.path.dirname(os.path.abspath(__file__)))
if hasattr(sys.stdout, 'reconfigure'):
    sys.stdout.reconfigure(encoding='utf-8', errors='replace', line_buffering=True)

EMAIL       = "operation@mrlogisticcorp.com"
PASSWORD    = "NKh9i8Z!7fU9jfi"
TOTP_SECRET = "MNWTEPTFJZBUC32GJFEWY6LVKQ2GGYKH"

SHIP_IDS = ['437855', '756199', '733924']  # CALUMET, ANTONIA S, GOLDEN JOY


def totp(secret):
    s = secret.upper().replace(' ', '')
    pad = (-len(s)) % 8
    key = base64.b32decode(s + '=' * pad)
    counter = int(time.time()) // 30
    msg = struct.pack('>Q', counter)
    h = hmac.new(key, msg, hashlib.sha1).digest()
    offset = h[-1] & 0x0f
    code = struct.unpack('>I', h[offset:offset+4])[0] & 0x7fffffff
    return str(code % 1000000).zfill(6)


async def do_login(page):
    print("LOGIN...")
    await page.goto('https://www.marinetraffic.com/en/users/login',
                    wait_until='domcontentloaded', timeout=30000)
    await asyncio.sleep(3)
    await page.fill('input[name="username"]', EMAIL)
    await page.click('button[type="submit"]')
    await asyncio.sleep(3)
    await page.fill('input[type="password"]', PASSWORD)
    await page.click('button[type="submit"]')
    await asyncio.sleep(4)
    if 'mfa' in page.url.lower() or 'auth.kpler' in page.url:
        try:
            await page.click('button:has-text("Google Authenticator")', timeout=3000)
            await asyncio.sleep(2)
        except Exception:
            pass
        otp = totp(TOTP_SECRET)
        print(f"  TOTP: {otp}")
        try:
            await page.fill('input[name="code"]', otp)
            await page.click('button[type="submit"]')
        except Exception:
            try:
                await page.fill('input[type="text"]', otp)
                await page.click('button[type="submit"]')
            except Exception as e:
                print(f"  2FA error: {e}")
        await asyncio.sleep(8)
    ok = 'marinetraffic.com' in page.url and 'auth.kpler' not in page.url
    print(f"  Login: {'OK' if ok else 'FAILED'} | {page.url[:80]}")
    return ok


async def probe_vessel(page, ship_id):
    """Navigate to vessel detail page, intercept ALL XHR responses."""
    print(f"\n--- Probing ship_id={ship_id} ---")
    captured = []

    async def on_response(response):
        url = response.url
        # Only care about API/JSON calls
        if any(x in url for x in ['/api/', '/vessels/', '/ships/', '/ownership', '/details',
                                    '/summary', '/company', '/management', 'json']):
            try:
                ct = response.headers.get('content-type', '')
                if 'json' in ct or 'javascript' in ct:
                    body = await response.body()
                    text = body.decode('utf-8', errors='replace')
                    if len(text) > 50 and ('owner' in text.lower() or 'address' in text.lower()
                                            or 'contact' in text.lower() or 'website' in text.lower()):
                        captured.append({'url': url, 'body': text[:2000]})
                        print(f"  [XHR] {url[:100]}")
                        print(f"         {text[:200]}")
            except Exception:
                pass

    page.on('response', on_response)

    url = f'https://www.marinetraffic.com/en/ais/details/ships/shipid:{ship_id}'
    await page.goto(url, wait_until='domcontentloaded', timeout=30000)
    await asyncio.sleep(8)

    page.remove_listener('response', on_response)

    # Also try direct JS fetch for known endpoints
    print(f"\n  Trying direct JS fetches...")
    endpoints = [
        f'/en/vessels/summary-details/shipid:{ship_id}',
        f'/en/vessels/vesselDetails/shipid:{ship_id}',
        f'/en/vessels/ownership/shipid:{ship_id}',
        f'/en/ais/details/ships/shipid:{ship_id}/tab:ownership',
        f'/en/vessels/{ship_id}/ownership',
        f'/en/vessels/{ship_id}/details',
        f'/en/api/vessels/{ship_id}',
    ]
    for ep in endpoints:
        url_full = f'https://www.marinetraffic.com{ep}'
        js = f"""
        async () => {{
            try {{
                const r = await fetch({json.dumps(url_full)}, {{
                    credentials: 'include',
                    headers: {{
                        'X-Requested-With': 'XMLHttpRequest',
                        'Accept': 'application/json',
                    }}
                }});
                const text = await r.text();
                return {{status: r.status, body: text.slice(0,500)}};
            }} catch(e) {{ return {{error: e.message}}; }}
        }}
        """
        try:
            result = await page.evaluate(js)
            status = result.get('status', '?')
            body = result.get('body', result.get('error', ''))[:150]
            if status == 200 and len(body) > 20:
                print(f"  [200] {ep}")
                print(f"        {body}")
                captured.append({'url': url_full, 'status': status, 'body': body})
            else:
                print(f"  [{status}] {ep}")
        except Exception as e:
            print(f"  [ERR] {ep}: {e}")
        await asyncio.sleep(0.5)

    return captured


async def main():
    from playwright.async_api import async_playwright

    all_captured = []

    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=False,
            args=['--no-sandbox', '--disable-blink-features=AutomationControlled']
        )
        context = await browser.new_context(
            viewport={'width': 1440, 'height': 900},
            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
                       '(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
        )
        page = await context.new_page()

        if not await do_login(page):
            await browser.close()
            return

        await asyncio.sleep(3)

        for ship_id in SHIP_IDS:
            captured = await probe_vessel(page, ship_id)
            all_captured.extend(captured)
            await asyncio.sleep(3)

        # Save all captured data
        with open('mt_detail_probe_results.json', 'w', encoding='utf-8') as f:
            json.dump(all_captured, f, ensure_ascii=False, indent=2)
        print(f"\n\nSaved {len(all_captured)} captured responses to mt_detail_probe_results.json")

        await browser.close()


asyncio.run(main())