240 lines
10 KiB
Python
240 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
MT Async Probe v2 - use page.on('response') instead of route.fetch().
|
|
route.fetch() re-makes requests which Cloudflare blocks.
|
|
page.on('response') captures the browser's own natural request body.
|
|
"""
|
|
import asyncio, json, time, base64, struct, hmac, hashlib, math, os, sys
|
|
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
|
# Fix Windows console encoding
|
|
if hasattr(sys.stdout, 'reconfigure'):
|
|
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
|
if hasattr(sys.stderr, 'reconfigure'):
|
|
sys.stderr.reconfigure(encoding='utf-8', errors='replace')
|
|
|
|
EMAIL = "operation@mrlogisticcorp.com"
|
|
PASSWORD = "NKh9i8Z!7fU9jfi"
|
|
TOTP_SECRET = "MNWTEPTFJZBUC32GJFEWY6LVKQ2GGYKH"
|
|
|
|
|
|
def totp(secret):
|
|
s = secret.upper().replace(' ', '')
|
|
pad = (-len(s)) % 8
|
|
key = base64.b32decode(s + '=' * pad)
|
|
counter = int(time.time()) // 30
|
|
msg = struct.pack('>Q', counter)
|
|
h = hmac.new(key, msg, hashlib.sha1).digest()
|
|
offset = h[-1] & 0x0f
|
|
code = struct.unpack('>I', h[offset:offset + 4])[0] & 0x7fffffff
|
|
return str(code % 1000000).zfill(6)
|
|
|
|
|
|
async def do_login(page):
|
|
print("Step 1: Login to MT Pro...")
|
|
await page.goto('https://www.marinetraffic.com/en/users/login',
|
|
wait_until='domcontentloaded', timeout=30000)
|
|
await asyncio.sleep(3)
|
|
print(f" URL: {page.url}")
|
|
|
|
await page.fill('input[name="username"]', EMAIL)
|
|
await page.click('button[type="submit"]')
|
|
await asyncio.sleep(3)
|
|
print(f" After email: {page.url}")
|
|
|
|
await page.fill('input[type="password"]', PASSWORD)
|
|
await page.click('button[type="submit"]')
|
|
await asyncio.sleep(4)
|
|
print(f" After password: {page.url}")
|
|
|
|
if 'mfa-login-options' in page.url:
|
|
print(" 2FA: clicking Google Authenticator...")
|
|
await page.click('button:has-text("Google Authenticator")')
|
|
await asyncio.sleep(3)
|
|
otp = totp(TOTP_SECRET)
|
|
print(f" TOTP: {otp}")
|
|
await page.fill('input[name="code"]', otp)
|
|
await page.click('button[type="submit"]')
|
|
await asyncio.sleep(5)
|
|
|
|
logged_in = 'marinetraffic.com' in page.url and 'auth.kpler' not in page.url
|
|
print(f" Logged in: {logged_in} | URL: {page.url}")
|
|
return logged_in
|
|
|
|
|
|
async def main():
|
|
from playwright.async_api import async_playwright
|
|
|
|
tile_data = []
|
|
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch(
|
|
headless=False,
|
|
args=['--no-sandbox', '--disable-blink-features=AutomationControlled']
|
|
)
|
|
context = await browser.new_context(
|
|
viewport={'width': 1280, 'height': 900},
|
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
|
'(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
|
)
|
|
page = await context.new_page()
|
|
|
|
# === CAPTURE BROWSER'S OWN RESPONSES (no interception) ===
|
|
# Key insight: route.fetch() makes a NEW request which CF blocks.
|
|
# page.on('response') captures the BROWSER's natural request body - no CF issue.
|
|
async def on_response(response):
|
|
url = response.url
|
|
if 'getData' not in url:
|
|
return
|
|
skip = ['.js', '.css', '.png', '.jpg', '.gif']
|
|
if any(s in url for s in skip):
|
|
return
|
|
try:
|
|
body = await response.body()
|
|
body_text = body.decode('utf-8', errors='replace')
|
|
entry = {
|
|
'url': url,
|
|
'status': response.status,
|
|
'size': len(body),
|
|
'body': body_text[:5000],
|
|
}
|
|
tile_data.append(entry)
|
|
try:
|
|
parsed = json.loads(body_text)
|
|
rows = parsed.get('data', {}).get('rows', [])
|
|
area_ships = parsed.get('data', {}).get('areaShips', '?')
|
|
print(" TILE %d (%db): %d vessel rows, areaShips=%s" % (
|
|
response.status, len(body), len(rows), area_ships))
|
|
if rows:
|
|
print(" Keys: %s" % list(rows[0].keys()))
|
|
print(" First: %s" % json.dumps(rows[0])[:300])
|
|
except json.JSONDecodeError:
|
|
# Not JSON - check if it's the CF block page
|
|
if '<!DOCTYPE' in body_text:
|
|
print(" TILE %d (%db) -> CF block page" % (response.status, len(body)))
|
|
else:
|
|
print(" TILE %d (%db) non-JSON: %s" % (
|
|
response.status, len(body), body_text[:100]))
|
|
except Exception as e:
|
|
print(" Response body error: %s" % e)
|
|
|
|
page.on('response', on_response)
|
|
|
|
# Login
|
|
logged_in = await do_login(page)
|
|
if not logged_in:
|
|
print(" ERROR: Login failed!")
|
|
await page.screenshot(path='mt_async_login_fail.png')
|
|
await browser.close()
|
|
return
|
|
await page.screenshot(path='mt_async_logged_in.png')
|
|
|
|
# === WAIT FOR FULL MT SESSION TO BE ESTABLISHED ===
|
|
# After oauth callback, wait for MT to set auth cookies
|
|
await asyncio.sleep(5)
|
|
print("\nWaiting on MT home to let session stabilize...")
|
|
try:
|
|
await page.wait_for_url('**/marinetraffic.com/**', timeout=15000)
|
|
except Exception:
|
|
pass
|
|
print(" Final URL after stabilize: %s" % page.url)
|
|
|
|
# Show cookies
|
|
cookies = await context.cookies()
|
|
print(" Cookies: %d total" % len(cookies))
|
|
for c in cookies:
|
|
if c['name'] in ('authTokens', 'AUTH', 'CAKEPHP', 'mt_user[UserID]',
|
|
'cf_clearance', '__cf_bm'):
|
|
print(" %s @ %s = %s..." % (c['name'], c['domain'], str(c['value'])[:50]))
|
|
|
|
# === LOAD ROTTERDAM MAP ===
|
|
print("\nStep 2: Loading Rotterdam map (zoom 10)...")
|
|
tile_data.clear()
|
|
await page.goto(
|
|
'https://www.marinetraffic.com/en/ais/home/centerx:4.5/centery:51.9/zoom:10',
|
|
wait_until='load', timeout=30000)
|
|
print(" Map loaded, waiting 15s for tiles...")
|
|
await asyncio.sleep(15)
|
|
await page.screenshot(path='mt_async_rotterdam.png')
|
|
print(" Tile data captured: %d" % len(tile_data))
|
|
|
|
if tile_data:
|
|
with open('mt_tile_data_async.json', 'w') as f:
|
|
json.dump(tile_data, f, indent=2)
|
|
print(" Saved -> mt_tile_data_async.json")
|
|
else:
|
|
print(" No tile data captured!")
|
|
|
|
# === DIRECT PAGE.REQUEST TEST ===
|
|
print("\nStep 3: Direct tile requests via page.request (authenticated)...")
|
|
lat, lon = 51.9, 4.5
|
|
lat_rad = math.radians(lat)
|
|
|
|
for zoom in [8, 9, 10]:
|
|
n = 2.0 ** zoom
|
|
x = int((lon + 180.0) / 360.0 * n)
|
|
y = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n)
|
|
tile_url = ('https://www.marinetraffic.com/getData/get_data_json_4'
|
|
'/z:%d/X:%d/Y:%d/station:0' % (zoom, x, y))
|
|
print(" z=%d X=%d Y=%d" % (zoom, x, y))
|
|
try:
|
|
r = await page.request.get(tile_url, headers={
|
|
'Referer': ('https://www.marinetraffic.com/en/ais/home'
|
|
'/centerx:%s/centery:%s/zoom:%d' % (lon, lat, zoom)),
|
|
'X-Requested-With': 'XMLHttpRequest',
|
|
'Accept': 'application/json, text/javascript, */*',
|
|
})
|
|
body = await r.body()
|
|
body_text = body.decode('utf-8', errors='replace')
|
|
if body_text.startswith('{'):
|
|
parsed = json.loads(body_text)
|
|
rows = parsed.get('data', {}).get('rows', [])
|
|
area_ships = parsed.get('data', {}).get('areaShips', '?')
|
|
print(" status=%d, %d vessels, areaShips=%s" % (r.status, len(rows), area_ships))
|
|
if rows:
|
|
print(" Keys: %s" % list(rows[0].keys()))
|
|
print(" First: %s" % json.dumps(rows[0])[:400])
|
|
with open('mt_direct_z%d.json' % zoom, 'w') as f:
|
|
json.dump(parsed, f, indent=2)
|
|
print(" Saved -> mt_direct_z%d.json" % zoom)
|
|
else:
|
|
print(" status=%d (%db) CF/HTML block" % (r.status, len(body)))
|
|
print(" %s" % body_text[:100])
|
|
except Exception as e:
|
|
print(" Error: %s" % e)
|
|
|
|
# === CASPIAN TEST ===
|
|
print("\nStep 4: Caspian direct test (z=8, lat=41.5, lon=50.5)...")
|
|
lat2, lon2, z2 = 41.5, 50.5, 8
|
|
lr2 = math.radians(lat2)
|
|
n2 = 2.0 ** z2
|
|
x2 = int((lon2 + 180.0) / 360.0 * n2)
|
|
y2 = int((1.0 - math.asinh(math.tan(lr2)) / math.pi) / 2.0 * n2)
|
|
tile_url2 = ('https://www.marinetraffic.com/getData/get_data_json_4'
|
|
'/z:%d/X:%d/Y:%d/station:0' % (z2, x2, y2))
|
|
print(" URL: %s" % tile_url2)
|
|
try:
|
|
r2 = await page.request.get(tile_url2, headers={
|
|
'Referer': ('https://www.marinetraffic.com/en/ais/home'
|
|
'/centerx:%s/centery:%s/zoom:%d' % (lon2, lat2, z2)),
|
|
'X-Requested-With': 'XMLHttpRequest',
|
|
})
|
|
body2 = await r2.body()
|
|
body2_text = body2.decode('utf-8', errors='replace')
|
|
if body2_text.startswith('{'):
|
|
parsed2 = json.loads(body2_text)
|
|
rows2 = parsed2.get('data', {}).get('rows', [])
|
|
print(" status=%d, %d vessels" % (r2.status, len(rows2)))
|
|
if rows2:
|
|
print(" Keys: %s" % list(rows2[0].keys()))
|
|
print(" Sample: %s" % json.dumps(rows2[0])[:400])
|
|
else:
|
|
print(" status=%d CF/HTML block: %s" % (r2.status, body2_text[:100]))
|
|
except Exception as e:
|
|
print(" Error: %s" % e)
|
|
|
|
await browser.close()
|
|
print("\nDone!")
|
|
|
|
|
|
asyncio.run(main())
|