202 lines
7.3 KiB
Python
202 lines
7.3 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""
|
||
|
|
Probe MT Reports API for bulk filtering by vessel type.
|
||
|
|
Goal: Get ALL cargo vessels via reports endpoint with pagination,
|
||
|
|
without needing map tile scanning.
|
||
|
|
"""
|
||
|
|
import asyncio, json, sys, os, time, struct, hmac, hashlib, base64
|
||
|
|
import psycopg2
|
||
|
|
|
||
|
|
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
||
|
|
if hasattr(sys.stdout, 'reconfigure'):
|
||
|
|
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
||
|
|
|
||
|
|
EMAIL = "operation@mrlogisticcorp.com"
|
||
|
|
PASSWORD = "NKh9i8Z!7fU9jfi"
|
||
|
|
TOTP_SECRET = "MNWTEPTFJZBUC32GJFEWY6LVKQ2GGYKH"
|
||
|
|
DB_URL = 'postgresql://seafare:SF_m0ntana_2026@127.0.0.1:15432/seafare_db'
|
||
|
|
|
||
|
|
|
||
|
|
def totp(secret):
|
||
|
|
s = secret.upper().replace(' ', '')
|
||
|
|
pad = (-len(s)) % 8
|
||
|
|
key = base64.b32decode(s + '=' * pad)
|
||
|
|
counter = int(time.time()) // 30
|
||
|
|
msg = struct.pack('>Q', counter)
|
||
|
|
h = hmac.new(key, msg, hashlib.sha1).digest()
|
||
|
|
offset = h[-1] & 0x0f
|
||
|
|
code = struct.unpack('>I', h[offset:offset+4])[0] & 0x7fffffff
|
||
|
|
return str(code % 1000000).zfill(6)
|
||
|
|
|
||
|
|
|
||
|
|
async def do_login(page):
|
||
|
|
print("Logging in...")
|
||
|
|
await page.goto('https://www.marinetraffic.com/en/users/login',
|
||
|
|
wait_until='domcontentloaded', timeout=30000)
|
||
|
|
await asyncio.sleep(3)
|
||
|
|
await page.fill('input[name="username"]', EMAIL)
|
||
|
|
await page.click('button[type="submit"]')
|
||
|
|
await asyncio.sleep(3)
|
||
|
|
await page.fill('input[type="password"]', PASSWORD)
|
||
|
|
await page.click('button[type="submit"]')
|
||
|
|
await asyncio.sleep(4)
|
||
|
|
if 'mfa' in page.url.lower() or 'auth.kpler' in page.url:
|
||
|
|
try:
|
||
|
|
await page.click('button:has-text("Google Authenticator")', timeout=3000)
|
||
|
|
await asyncio.sleep(2)
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
otp = totp(TOTP_SECRET)
|
||
|
|
print(f" TOTP: {otp}")
|
||
|
|
await page.fill('input[name="code"]', otp)
|
||
|
|
await page.click('button[type="submit"]')
|
||
|
|
await asyncio.sleep(8)
|
||
|
|
ok = 'marinetraffic.com' in page.url and 'auth.kpler' not in page.url
|
||
|
|
print(f" Login: {'OK' if ok else 'FAILED'} | {page.url[:80]}")
|
||
|
|
return ok
|
||
|
|
|
||
|
|
|
||
|
|
async def probe_reports(page, url, label):
|
||
|
|
"""Fetch reports API URL via page.evaluate(fetch()) and show results."""
|
||
|
|
js = f"""
|
||
|
|
async () => {{
|
||
|
|
try {{
|
||
|
|
const r = await fetch({json.dumps(url)}, {{
|
||
|
|
credentials: 'include', cache: 'no-store',
|
||
|
|
headers: {{
|
||
|
|
'X-Requested-With': 'XMLHttpRequest',
|
||
|
|
'Accept': 'application/json',
|
||
|
|
'Referer': 'https://www.marinetraffic.com/en/data/?asset_type=vessels',
|
||
|
|
}}
|
||
|
|
}});
|
||
|
|
const text = await r.text();
|
||
|
|
return {{status: r.status, body: text.substring(0, 5000)}};
|
||
|
|
}} catch(e) {{ return {{error: e.message}}; }}
|
||
|
|
}}
|
||
|
|
"""
|
||
|
|
result = await page.evaluate(js)
|
||
|
|
print(f"\n=== {label} ===")
|
||
|
|
print(f" URL: {url[:120]}")
|
||
|
|
|
||
|
|
if result.get('error'):
|
||
|
|
print(f" ERROR: {result['error']}")
|
||
|
|
return None
|
||
|
|
|
||
|
|
status = result.get('status')
|
||
|
|
body = result.get('body', '')
|
||
|
|
print(f" Status: {status}")
|
||
|
|
|
||
|
|
try:
|
||
|
|
data = json.loads(body)
|
||
|
|
if isinstance(data, dict):
|
||
|
|
total = data.get('totalCount', '?')
|
||
|
|
rows = data.get('data', [])
|
||
|
|
print(f" totalCount: {total}")
|
||
|
|
print(f" rows returned: {len(rows)}")
|
||
|
|
if rows and isinstance(rows, list) and isinstance(rows[0], dict):
|
||
|
|
print(f" First row keys: {list(rows[0].keys())[:15]}")
|
||
|
|
for i, row in enumerate(rows[:3]):
|
||
|
|
name = row.get('SHIPNAME', '?')
|
||
|
|
mmsi = row.get('MMSI', '?')
|
||
|
|
imo = row.get('IMO', '?')
|
||
|
|
gt = row.get('GT_SHIPTYPE', '?')
|
||
|
|
dwt = row.get('DWT', '?')
|
||
|
|
owner = row.get('BENEFICIAL_OWNER', '?')
|
||
|
|
print(f" [{i+1}] {name} | MMSI={mmsi} | IMO={imo} | GT={gt} | DWT={dwt} | Owner={owner}")
|
||
|
|
return data
|
||
|
|
else:
|
||
|
|
print(f" Response: {str(data)[:200]}")
|
||
|
|
return None
|
||
|
|
except (json.JSONDecodeError, ValueError):
|
||
|
|
print(f" Not JSON: {body[:200]}")
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
async def main():
|
||
|
|
from playwright.async_api import async_playwright
|
||
|
|
|
||
|
|
async with async_playwright() as p:
|
||
|
|
browser = await p.chromium.launch(
|
||
|
|
headless=False,
|
||
|
|
args=['--no-sandbox', '--disable-blink-features=AutomationControlled']
|
||
|
|
)
|
||
|
|
context = await browser.new_context(
|
||
|
|
viewport={'width': 1440, 'height': 900},
|
||
|
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||
|
|
'(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||
|
|
)
|
||
|
|
page = await context.new_page()
|
||
|
|
|
||
|
|
if not await do_login(page):
|
||
|
|
await browser.close()
|
||
|
|
return
|
||
|
|
|
||
|
|
# Warm up data page
|
||
|
|
await page.goto('https://www.marinetraffic.com/en/data/?asset_type=vessels',
|
||
|
|
wait_until='load', timeout=40000)
|
||
|
|
await asyncio.sleep(5)
|
||
|
|
|
||
|
|
base = 'https://www.marinetraffic.com/en/reports/?asset_type=vessels'
|
||
|
|
cols = 'shipname,imo,mmsi,flag,ship_type,dwt,beneficial_owner,operator,registered_owner'
|
||
|
|
|
||
|
|
# Test 1: gt_shiptype_in with pipe separator
|
||
|
|
await probe_reports(page,
|
||
|
|
f'{base}&columns={cols}>_shiptype_in|6|',
|
||
|
|
'gt_shiptype_in|6| (pipe)')
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
|
||
|
|
# Test 2: ship_type_in with pipe separator
|
||
|
|
await probe_reports(page,
|
||
|
|
f'{base}&columns={cols}&ship_type_in|7|',
|
||
|
|
'ship_type_in|7| (pipe)')
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
|
||
|
|
# Test 3: Using column filter format (like MT data page)
|
||
|
|
await probe_reports(page,
|
||
|
|
f'{base}&columns={cols}&column:gt_shiptype_in|6|',
|
||
|
|
'column:gt_shiptype_in|6|')
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
|
||
|
|
# Test 4: Using quicksearch_ship_type_in (guessing MT param name)
|
||
|
|
await probe_reports(page,
|
||
|
|
f'{base}&columns={cols}&quicksearch_ship_type_in|7|',
|
||
|
|
'quicksearch_ship_type_in|7|')
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
|
||
|
|
# Test 5: Simple ship_type=7 (cargo)
|
||
|
|
await probe_reports(page,
|
||
|
|
f'{base}&columns={cols}&ship_type=7',
|
||
|
|
'ship_type=7')
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
|
||
|
|
# Test 6: No filter but with columns including gt_shiptype
|
||
|
|
cols2 = 'shipname,imo,mmsi,flag,ship_type,gt_shiptype,dwt,beneficial_owner,operator,registered_owner'
|
||
|
|
await probe_reports(page,
|
||
|
|
f'{base}&columns={cols2}',
|
||
|
|
'No filter, just columns with gt_shiptype')
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
|
||
|
|
# Test 7: With per_page
|
||
|
|
await probe_reports(page,
|
||
|
|
f'{base}&columns={cols2}&per_page=50',
|
||
|
|
'No filter, per_page=50')
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
|
||
|
|
# Test 8: With page=2
|
||
|
|
await probe_reports(page,
|
||
|
|
f'{base}&columns={cols2}&per_page=50&page=2',
|
||
|
|
'No filter, per_page=50, page=2')
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
|
||
|
|
# Test 9: With time_of_latest_position (active vessels only)
|
||
|
|
await probe_reports(page,
|
||
|
|
f'{base}&columns={cols2}&per_page=100&time_of_latest_position_between|60|',
|
||
|
|
'Active in last 60 min, per_page=100')
|
||
|
|
|
||
|
|
print("\n=== PROBE DONE ===")
|
||
|
|
await browser.close()
|
||
|
|
|
||
|
|
|
||
|
|
asyncio.run(main())
|