#!/usr/bin/env python3 """ Probe MT Reports API for bulk filtering by vessel type. Goal: Get ALL cargo vessels via reports endpoint with pagination, without needing map tile scanning. """ import asyncio, json, sys, os, time, struct, hmac, hashlib, base64 import psycopg2 os.chdir(os.path.dirname(os.path.abspath(__file__))) if hasattr(sys.stdout, 'reconfigure'): sys.stdout.reconfigure(encoding='utf-8', errors='replace') EMAIL = "operation@mrlogisticcorp.com" PASSWORD = "NKh9i8Z!7fU9jfi" TOTP_SECRET = "MNWTEPTFJZBUC32GJFEWY6LVKQ2GGYKH" DB_URL = 'postgresql://seafare:SF_m0ntana_2026@127.0.0.1:15432/seafare_db' def totp(secret): s = secret.upper().replace(' ', '') pad = (-len(s)) % 8 key = base64.b32decode(s + '=' * pad) counter = int(time.time()) // 30 msg = struct.pack('>Q', counter) h = hmac.new(key, msg, hashlib.sha1).digest() offset = h[-1] & 0x0f code = struct.unpack('>I', h[offset:offset+4])[0] & 0x7fffffff return str(code % 1000000).zfill(6) async def do_login(page): print("Logging in...") await page.goto('https://www.marinetraffic.com/en/users/login', wait_until='domcontentloaded', timeout=30000) await asyncio.sleep(3) await page.fill('input[name="username"]', EMAIL) await page.click('button[type="submit"]') await asyncio.sleep(3) await page.fill('input[type="password"]', PASSWORD) await page.click('button[type="submit"]') await asyncio.sleep(4) if 'mfa' in page.url.lower() or 'auth.kpler' in page.url: try: await page.click('button:has-text("Google Authenticator")', timeout=3000) await asyncio.sleep(2) except Exception: pass await asyncio.sleep(1) otp = totp(TOTP_SECRET) print(f" TOTP: {otp}") await page.fill('input[name="code"]', otp) await page.click('button[type="submit"]') await asyncio.sleep(8) ok = 'marinetraffic.com' in page.url and 'auth.kpler' not in page.url print(f" Login: {'OK' if ok else 'FAILED'} | {page.url[:80]}") return ok async def probe_reports(page, url, label): """Fetch reports API URL via page.evaluate(fetch()) and show results.""" js = f""" async () => {{ try {{ const r = await fetch({json.dumps(url)}, {{ credentials: 'include', cache: 'no-store', headers: {{ 'X-Requested-With': 'XMLHttpRequest', 'Accept': 'application/json', 'Referer': 'https://www.marinetraffic.com/en/data/?asset_type=vessels', }} }}); const text = await r.text(); return {{status: r.status, body: text.substring(0, 5000)}}; }} catch(e) {{ return {{error: e.message}}; }} }} """ result = await page.evaluate(js) print(f"\n=== {label} ===") print(f" URL: {url[:120]}") if result.get('error'): print(f" ERROR: {result['error']}") return None status = result.get('status') body = result.get('body', '') print(f" Status: {status}") try: data = json.loads(body) if isinstance(data, dict): total = data.get('totalCount', '?') rows = data.get('data', []) print(f" totalCount: {total}") print(f" rows returned: {len(rows)}") if rows and isinstance(rows, list) and isinstance(rows[0], dict): print(f" First row keys: {list(rows[0].keys())[:15]}") for i, row in enumerate(rows[:3]): name = row.get('SHIPNAME', '?') mmsi = row.get('MMSI', '?') imo = row.get('IMO', '?') gt = row.get('GT_SHIPTYPE', '?') dwt = row.get('DWT', '?') owner = row.get('BENEFICIAL_OWNER', '?') print(f" [{i+1}] {name} | MMSI={mmsi} | IMO={imo} | GT={gt} | DWT={dwt} | Owner={owner}") return data else: print(f" Response: {str(data)[:200]}") return None except (json.JSONDecodeError, ValueError): print(f" Not JSON: {body[:200]}") return None async def main(): from playwright.async_api import async_playwright async with async_playwright() as p: browser = await p.chromium.launch( headless=False, args=['--no-sandbox', '--disable-blink-features=AutomationControlled'] ) context = await browser.new_context( viewport={'width': 1440, 'height': 900}, user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' '(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', ) page = await context.new_page() if not await do_login(page): await browser.close() return # Warm up data page await page.goto('https://www.marinetraffic.com/en/data/?asset_type=vessels', wait_until='load', timeout=40000) await asyncio.sleep(5) base = 'https://www.marinetraffic.com/en/reports/?asset_type=vessels' cols = 'shipname,imo,mmsi,flag,ship_type,dwt,beneficial_owner,operator,registered_owner' # Test 1: gt_shiptype_in with pipe separator await probe_reports(page, f'{base}&columns={cols}>_shiptype_in|6|', 'gt_shiptype_in|6| (pipe)') await asyncio.sleep(1) # Test 2: ship_type_in with pipe separator await probe_reports(page, f'{base}&columns={cols}&ship_type_in|7|', 'ship_type_in|7| (pipe)') await asyncio.sleep(1) # Test 3: Using column filter format (like MT data page) await probe_reports(page, f'{base}&columns={cols}&column:gt_shiptype_in|6|', 'column:gt_shiptype_in|6|') await asyncio.sleep(1) # Test 4: Using quicksearch_ship_type_in (guessing MT param name) await probe_reports(page, f'{base}&columns={cols}&quicksearch_ship_type_in|7|', 'quicksearch_ship_type_in|7|') await asyncio.sleep(1) # Test 5: Simple ship_type=7 (cargo) await probe_reports(page, f'{base}&columns={cols}&ship_type=7', 'ship_type=7') await asyncio.sleep(1) # Test 6: No filter but with columns including gt_shiptype cols2 = 'shipname,imo,mmsi,flag,ship_type,gt_shiptype,dwt,beneficial_owner,operator,registered_owner' await probe_reports(page, f'{base}&columns={cols2}', 'No filter, just columns with gt_shiptype') await asyncio.sleep(1) # Test 7: With per_page await probe_reports(page, f'{base}&columns={cols2}&per_page=50', 'No filter, per_page=50') await asyncio.sleep(1) # Test 8: With page=2 await probe_reports(page, f'{base}&columns={cols2}&per_page=50&page=2', 'No filter, per_page=50, page=2') await asyncio.sleep(1) # Test 9: With time_of_latest_position (active vessels only) await probe_reports(page, f'{base}&columns={cols2}&per_page=100&time_of_latest_position_between|60|', 'Active in last 60 min, per_page=100') print("\n=== PROBE DONE ===") await browser.close() asyncio.run(main())