import { readMarketListRequestSnapshot, type MarketListRequestSnapshot } from "./market-list-request-snapshot"; import { parseMarketListResponse, readKnownPaginationNumber } from "./market-list-row"; import type { MarketExportTarget, MarketRecord } from "./types"; interface FetchResponseLike { json(): Promise; ok: boolean; } type FetchLike = (input: string, init?: RequestInit) => Promise; interface SilentExportControllerOptions { document: Document; fetchImpl?: FetchLike; onProgress?: (state: { currentPage: number; totalPages?: number }) => void; } type PageSource = "body" | "none" | "url"; const PAGE_NUMBER_KEYS = [ "currentPage", "page", "pageNo", "pageNum", "page_no", "page_num" ] as const; export function createSilentExportController( options: SilentExportControllerOptions ) { const fetchImpl = options.fetchImpl ?? defaultFetch; return { async exportRecords(target: MarketExportTarget): Promise { const snapshot = readMarketListRequestSnapshot(options.document); if (!snapshot) { return null; } const baseRequest = createPagedRequest(snapshot); if (!baseRequest) { return null; } const mergedRecords = new Map(); const maxPageCount = target.mode === "count" ? target.pageCount : 200; let totalPagesHint: number | undefined; for (let offset = 0; offset < maxPageCount; offset += 1) { const pageNumber = baseRequest.initialPage + offset; options.onProgress?.({ currentPage: offset + 1, totalPages: target.mode === "count" ? target.pageCount : totalPagesHint }); const payload = await fetchPagePayload(fetchImpl, baseRequest, pageNumber); const parsedResponse = parseMarketListResponse(payload); if (!parsedResponse) { return null; } totalPagesHint = parsedResponse.totalPages ?? totalPagesHint; if (parsedResponse.records.length === 0) { break; } parsedResponse.records.forEach((record) => { const existingRecord = mergedRecords.get(record.authorId); mergedRecords.set(record.authorId, mergeMarketRecord(existingRecord, record)); }); if (target.mode === "count" && offset + 1 >= target.pageCount) { break; } if (target.mode === "all") { if ( typeof parsedResponse.totalPages === "number" && pageNumber >= parsedResponse.totalPages ) { break; } if ( typeof parsedResponse.pageSize === "number" && parsedResponse.records.length < parsedResponse.pageSize ) { break; } } } return Array.from(mergedRecords.values()); } }; } function createPagedRequest( snapshot: MarketListRequestSnapshot ): { initialPage: number; pageSource: PageSource; snapshot: MarketListRequestSnapshot; } { const bodyPage = readPageFromBody(snapshot.body); if (bodyPage !== null) { return { initialPage: bodyPage, pageSource: "body", snapshot }; } const urlPage = readPageFromUrl(snapshot.url); if (urlPage !== null) { return { initialPage: urlPage, pageSource: "url", snapshot }; } return { initialPage: 1, pageSource: "none", snapshot }; } async function fetchPagePayload( fetchImpl: FetchLike, request: { pageSource: PageSource; snapshot: MarketListRequestSnapshot; }, pageNumber: number ): Promise { const nextUrl = request.pageSource === "url" ? mutateUrlPage(request.snapshot.url, pageNumber) : request.snapshot.url; const nextBody = mutateBodyPage(request.snapshot.body, pageNumber); const response = await fetchImpl(nextUrl, { body: nextBody, credentials: "include", headers: filterReplayHeaders(request.snapshot.headers, nextBody), method: request.snapshot.method }); if (!response.ok) { throw new Error("静默导出请求失败"); } return response.json(); } function readPageFromUrl(url: string): number | null { try { const parsedUrl = new URL(url); for (const key of PAGE_NUMBER_KEYS) { const value = readNumericString(parsedUrl.searchParams.get(key)); if (value !== null) { return value; } } } catch { return null; } return null; } function mutateUrlPage(url: string, pageNumber: number): string { try { const parsedUrl = new URL(url); for (const key of PAGE_NUMBER_KEYS) { if (!parsedUrl.searchParams.has(key)) { continue; } parsedUrl.searchParams.set(key, String(pageNumber)); return parsedUrl.toString(); } parsedUrl.searchParams.set("page", String(pageNumber)); return parsedUrl.toString(); } catch { return url; } } function readPageFromBody(body: string | undefined): number | null { const parsedBody = parseBody(body); if (!parsedBody) { return null; } return readKnownPaginationNumber(parsedBody, "page"); } function mutateBodyPage(body: string | undefined, pageNumber: number): string | undefined { if (!body) { return body; } const trimmedBody = body.trim(); if (!trimmedBody) { return body; } try { const parsedJson = JSON.parse(trimmedBody); if (!replacePageNumberInValue(parsedJson, pageNumber) && isRecord(parsedJson)) { parsedJson.page = pageNumber; } return JSON.stringify(parsedJson); } catch { const searchParams = new URLSearchParams(trimmedBody); for (const key of PAGE_NUMBER_KEYS) { if (!searchParams.has(key)) { continue; } searchParams.set(key, String(pageNumber)); return searchParams.toString(); } searchParams.set("page", String(pageNumber)); return searchParams.toString(); } } function parseBody(body: string | undefined): Record | null { if (!body) { return null; } const trimmedBody = body.trim(); if (!trimmedBody) { return null; } try { const parsedBody = JSON.parse(trimmedBody); return isRecord(parsedBody) ? parsedBody : null; } catch { const searchParams = new URLSearchParams(trimmedBody); const payload: Record = {}; searchParams.forEach((value, key) => { payload[key] = value; }); return payload; } } function replacePageNumberInValue(value: unknown, pageNumber: number): boolean { if (!isRecord(value)) { return false; } let replaced = false; PAGE_NUMBER_KEYS.forEach((key) => { if (!(key in value)) { return; } value[key] = pageNumber; replaced = true; }); if (replaced) { return true; } return Object.values(value).some((entry) => replacePageNumberInValue(entry, pageNumber)); } function filterReplayHeaders( headers: Record | undefined, body: string | undefined ): HeadersInit | undefined { const filteredHeaders = Object.fromEntries( Object.entries(headers ?? {}).filter(([key]) => { const normalizedKey = key.toLowerCase(); return normalizedKey !== "content-length" && normalizedKey !== "host"; }) ); if (body) { if (!hasHeader(filteredHeaders, "accept")) { filteredHeaders.Accept = "application/json, text/plain, */*"; } if (!hasHeader(filteredHeaders, "content-type")) { filteredHeaders["Content-Type"] = "application/json"; } if (!hasHeader(filteredHeaders, "x-login-source")) { filteredHeaders["x-login-source"] = "1"; } if (!hasHeader(filteredHeaders, "agw-js-conv")) { filteredHeaders["Agw-Js-Conv"] = "str"; } } return Object.keys(filteredHeaders).length > 0 ? filteredHeaders : undefined; } function hasHeader(headers: Record, key: string): boolean { return Object.keys(headers).some((headerKey) => headerKey.toLowerCase() === key); } function readNumericString(value: string | null): number | null { if (!value) { return null; } const parsedValue = Number(value); return Number.isFinite(parsedValue) ? parsedValue : null; } async function defaultFetch(input: string, init?: RequestInit) { return fetch(input, init); } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } function mergeMarketRecord( existingRecord: MarketRecord | undefined, incomingRecord: MarketRecord ): MarketRecord { if (!existingRecord) { return { ...incomingRecord, exportFields: mergeFieldMap(undefined, incomingRecord.exportFields), rates: mergeFieldMap(undefined, incomingRecord.rates), status: incomingRecord.status ?? "idle" }; } return { ...existingRecord, ...incomingRecord, authorName: mergeStringValue(existingRecord.authorName, incomingRecord.authorName) ?? "", coreUserId: mergeStringValue(existingRecord.coreUserId, incomingRecord.coreUserId), exportFields: mergeFieldMap( existingRecord.exportFields, incomingRecord.exportFields ), failureReason: incomingRecord.failureReason ?? existingRecord.failureReason, hasDirectRatesSource: existingRecord.hasDirectRatesSource || incomingRecord.hasDirectRatesSource, location: mergeStringValue(existingRecord.location, incomingRecord.location), price21To60s: mergeStringValue( existingRecord.price21To60s, incomingRecord.price21To60s ), rates: mergeFieldMap(existingRecord.rates, incomingRecord.rates), status: incomingRecord.status ?? existingRecord.status }; } function mergeFieldMap>( current: T | undefined, incoming: T | undefined ): T | undefined { if (!current && !incoming) { return undefined; } const merged = { ...(current ?? {}) } as Record; Object.entries(incoming ?? {}).forEach(([key, value]) => { const currentValue = merged[key]; if (hasTextValue(value) || !hasTextValue(currentValue)) { merged[key] = value; } }); return merged as T; } function mergeStringValue( current: string | undefined, incoming: string | undefined ): string | undefined { return hasTextValue(incoming) ? incoming : current; } function hasTextValue(value: string | undefined): boolean { return Boolean(value && value.trim().length > 0); }