402 lines
10 KiB
TypeScript
402 lines
10 KiB
TypeScript
import {
|
|
readMarketListRequestSnapshot,
|
|
type MarketListRequestSnapshot
|
|
} from "./market-list-request-snapshot";
|
|
import {
|
|
parseMarketListResponse,
|
|
readKnownPaginationNumber
|
|
} from "./market-list-row";
|
|
import type { MarketExportTarget, MarketRecord } from "./types";
|
|
|
|
interface FetchResponseLike {
|
|
json(): Promise<unknown>;
|
|
ok: boolean;
|
|
}
|
|
|
|
type FetchLike = (input: string, init?: RequestInit) => Promise<FetchResponseLike>;
|
|
|
|
interface SilentExportControllerOptions {
|
|
document: Document;
|
|
fetchImpl?: FetchLike;
|
|
onProgress?: (state: { currentPage: number; totalPages?: number }) => void;
|
|
}
|
|
|
|
type PageSource = "body" | "none" | "url";
|
|
|
|
const PAGE_NUMBER_KEYS = [
|
|
"currentPage",
|
|
"page",
|
|
"pageNo",
|
|
"pageNum",
|
|
"page_no",
|
|
"page_num"
|
|
] as const;
|
|
|
|
export function createSilentExportController(
|
|
options: SilentExportControllerOptions
|
|
) {
|
|
const fetchImpl = options.fetchImpl ?? defaultFetch;
|
|
|
|
return {
|
|
async exportRecords(target: MarketExportTarget): Promise<MarketRecord[] | null> {
|
|
const snapshot = readMarketListRequestSnapshot(options.document);
|
|
if (!snapshot) {
|
|
return null;
|
|
}
|
|
|
|
const baseRequest = createPagedRequest(snapshot);
|
|
if (!baseRequest) {
|
|
return null;
|
|
}
|
|
|
|
const mergedRecords = new Map<string, MarketRecord>();
|
|
const maxPageCount = target.mode === "count" ? target.pageCount : 200;
|
|
let totalPagesHint: number | undefined;
|
|
|
|
for (let offset = 0; offset < maxPageCount; offset += 1) {
|
|
const pageNumber = baseRequest.initialPage + offset;
|
|
options.onProgress?.({
|
|
currentPage: offset + 1,
|
|
totalPages: target.mode === "count" ? target.pageCount : totalPagesHint
|
|
});
|
|
|
|
const payload = await fetchPagePayload(fetchImpl, baseRequest, pageNumber);
|
|
const parsedResponse = parseMarketListResponse(payload);
|
|
if (!parsedResponse) {
|
|
return null;
|
|
}
|
|
|
|
totalPagesHint = parsedResponse.totalPages ?? totalPagesHint;
|
|
if (parsedResponse.records.length === 0) {
|
|
break;
|
|
}
|
|
|
|
parsedResponse.records.forEach((record) => {
|
|
const existingRecord = mergedRecords.get(record.authorId);
|
|
mergedRecords.set(record.authorId, mergeMarketRecord(existingRecord, record));
|
|
});
|
|
|
|
if (target.mode === "count" && offset + 1 >= target.pageCount) {
|
|
break;
|
|
}
|
|
|
|
if (target.mode === "all") {
|
|
if (
|
|
typeof parsedResponse.totalPages === "number" &&
|
|
pageNumber >= parsedResponse.totalPages
|
|
) {
|
|
break;
|
|
}
|
|
|
|
if (
|
|
typeof parsedResponse.pageSize === "number" &&
|
|
parsedResponse.records.length < parsedResponse.pageSize
|
|
) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return Array.from(mergedRecords.values());
|
|
}
|
|
};
|
|
}
|
|
|
|
function createPagedRequest(
|
|
snapshot: MarketListRequestSnapshot
|
|
): {
|
|
initialPage: number;
|
|
pageSource: PageSource;
|
|
snapshot: MarketListRequestSnapshot;
|
|
} {
|
|
const bodyPage = readPageFromBody(snapshot.body);
|
|
if (bodyPage !== null) {
|
|
return {
|
|
initialPage: bodyPage,
|
|
pageSource: "body",
|
|
snapshot
|
|
};
|
|
}
|
|
|
|
const urlPage = readPageFromUrl(snapshot.url);
|
|
if (urlPage !== null) {
|
|
return {
|
|
initialPage: urlPage,
|
|
pageSource: "url",
|
|
snapshot
|
|
};
|
|
}
|
|
|
|
return {
|
|
initialPage: 1,
|
|
pageSource: "none",
|
|
snapshot
|
|
};
|
|
}
|
|
|
|
async function fetchPagePayload(
|
|
fetchImpl: FetchLike,
|
|
request: {
|
|
pageSource: PageSource;
|
|
snapshot: MarketListRequestSnapshot;
|
|
},
|
|
pageNumber: number
|
|
): Promise<unknown> {
|
|
const nextUrl =
|
|
request.pageSource === "url"
|
|
? mutateUrlPage(request.snapshot.url, pageNumber)
|
|
: request.snapshot.url;
|
|
const nextBody = mutateBodyPage(request.snapshot.body, pageNumber);
|
|
|
|
const response = await fetchImpl(nextUrl, {
|
|
body: nextBody,
|
|
credentials: "include",
|
|
headers: filterReplayHeaders(request.snapshot.headers, nextBody),
|
|
method: request.snapshot.method
|
|
});
|
|
if (!response.ok) {
|
|
throw new Error("静默导出请求失败");
|
|
}
|
|
|
|
return response.json();
|
|
}
|
|
|
|
function readPageFromUrl(url: string): number | null {
|
|
try {
|
|
const parsedUrl = new URL(url);
|
|
for (const key of PAGE_NUMBER_KEYS) {
|
|
const value = readNumericString(parsedUrl.searchParams.get(key));
|
|
if (value !== null) {
|
|
return value;
|
|
}
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function mutateUrlPage(url: string, pageNumber: number): string {
|
|
try {
|
|
const parsedUrl = new URL(url);
|
|
for (const key of PAGE_NUMBER_KEYS) {
|
|
if (!parsedUrl.searchParams.has(key)) {
|
|
continue;
|
|
}
|
|
|
|
parsedUrl.searchParams.set(key, String(pageNumber));
|
|
return parsedUrl.toString();
|
|
}
|
|
|
|
parsedUrl.searchParams.set("page", String(pageNumber));
|
|
return parsedUrl.toString();
|
|
} catch {
|
|
return url;
|
|
}
|
|
}
|
|
|
|
function readPageFromBody(body: string | undefined): number | null {
|
|
const parsedBody = parseBody(body);
|
|
if (!parsedBody) {
|
|
return null;
|
|
}
|
|
|
|
return readKnownPaginationNumber(parsedBody, "page");
|
|
}
|
|
|
|
function mutateBodyPage(body: string | undefined, pageNumber: number): string | undefined {
|
|
if (!body) {
|
|
return body;
|
|
}
|
|
|
|
const trimmedBody = body.trim();
|
|
if (!trimmedBody) {
|
|
return body;
|
|
}
|
|
|
|
try {
|
|
const parsedJson = JSON.parse(trimmedBody);
|
|
if (!replacePageNumberInValue(parsedJson, pageNumber) && isRecord(parsedJson)) {
|
|
parsedJson.page = pageNumber;
|
|
}
|
|
|
|
return JSON.stringify(parsedJson);
|
|
} catch {
|
|
const searchParams = new URLSearchParams(trimmedBody);
|
|
for (const key of PAGE_NUMBER_KEYS) {
|
|
if (!searchParams.has(key)) {
|
|
continue;
|
|
}
|
|
|
|
searchParams.set(key, String(pageNumber));
|
|
return searchParams.toString();
|
|
}
|
|
|
|
searchParams.set("page", String(pageNumber));
|
|
return searchParams.toString();
|
|
}
|
|
}
|
|
|
|
function parseBody(body: string | undefined): Record<string, unknown> | null {
|
|
if (!body) {
|
|
return null;
|
|
}
|
|
|
|
const trimmedBody = body.trim();
|
|
if (!trimmedBody) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const parsedBody = JSON.parse(trimmedBody);
|
|
return isRecord(parsedBody) ? parsedBody : null;
|
|
} catch {
|
|
const searchParams = new URLSearchParams(trimmedBody);
|
|
const payload: Record<string, unknown> = {};
|
|
searchParams.forEach((value, key) => {
|
|
payload[key] = value;
|
|
});
|
|
return payload;
|
|
}
|
|
}
|
|
|
|
function replacePageNumberInValue(value: unknown, pageNumber: number): boolean {
|
|
if (!isRecord(value)) {
|
|
return false;
|
|
}
|
|
|
|
let replaced = false;
|
|
PAGE_NUMBER_KEYS.forEach((key) => {
|
|
if (!(key in value)) {
|
|
return;
|
|
}
|
|
|
|
value[key] = pageNumber;
|
|
replaced = true;
|
|
});
|
|
|
|
if (replaced) {
|
|
return true;
|
|
}
|
|
|
|
return Object.values(value).some((entry) => replacePageNumberInValue(entry, pageNumber));
|
|
}
|
|
|
|
function filterReplayHeaders(
|
|
headers: Record<string, string> | undefined,
|
|
body: string | undefined
|
|
): HeadersInit | undefined {
|
|
const filteredHeaders = Object.fromEntries(
|
|
Object.entries(headers ?? {}).filter(([key]) => {
|
|
const normalizedKey = key.toLowerCase();
|
|
return normalizedKey !== "content-length" && normalizedKey !== "host";
|
|
})
|
|
);
|
|
|
|
if (body) {
|
|
if (!hasHeader(filteredHeaders, "accept")) {
|
|
filteredHeaders.Accept = "application/json, text/plain, */*";
|
|
}
|
|
if (!hasHeader(filteredHeaders, "content-type")) {
|
|
filteredHeaders["Content-Type"] = "application/json";
|
|
}
|
|
if (!hasHeader(filteredHeaders, "x-login-source")) {
|
|
filteredHeaders["x-login-source"] = "1";
|
|
}
|
|
if (!hasHeader(filteredHeaders, "agw-js-conv")) {
|
|
filteredHeaders["Agw-Js-Conv"] = "str";
|
|
}
|
|
}
|
|
|
|
return Object.keys(filteredHeaders).length > 0 ? filteredHeaders : undefined;
|
|
}
|
|
|
|
function hasHeader(headers: Record<string, string>, key: string): boolean {
|
|
return Object.keys(headers).some((headerKey) => headerKey.toLowerCase() === key);
|
|
}
|
|
|
|
function readNumericString(value: string | null): number | null {
|
|
if (!value) {
|
|
return null;
|
|
}
|
|
|
|
const parsedValue = Number(value);
|
|
return Number.isFinite(parsedValue) ? parsedValue : null;
|
|
}
|
|
|
|
async function defaultFetch(input: string, init?: RequestInit) {
|
|
return fetch(input, init);
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null;
|
|
}
|
|
|
|
function mergeMarketRecord(
|
|
existingRecord: MarketRecord | undefined,
|
|
incomingRecord: MarketRecord
|
|
): MarketRecord {
|
|
if (!existingRecord) {
|
|
return {
|
|
...incomingRecord,
|
|
exportFields: mergeFieldMap(undefined, incomingRecord.exportFields),
|
|
rates: mergeFieldMap(undefined, incomingRecord.rates),
|
|
status: incomingRecord.status ?? "idle"
|
|
};
|
|
}
|
|
|
|
return {
|
|
...existingRecord,
|
|
...incomingRecord,
|
|
authorName: mergeStringValue(existingRecord.authorName, incomingRecord.authorName) ?? "",
|
|
exportFields: mergeFieldMap(
|
|
existingRecord.exportFields,
|
|
incomingRecord.exportFields
|
|
),
|
|
failureReason: incomingRecord.failureReason ?? existingRecord.failureReason,
|
|
hasDirectRatesSource:
|
|
existingRecord.hasDirectRatesSource || incomingRecord.hasDirectRatesSource,
|
|
location: mergeStringValue(existingRecord.location, incomingRecord.location),
|
|
price21To60s: mergeStringValue(
|
|
existingRecord.price21To60s,
|
|
incomingRecord.price21To60s
|
|
),
|
|
rates: mergeFieldMap(existingRecord.rates, incomingRecord.rates),
|
|
status: incomingRecord.status ?? existingRecord.status
|
|
};
|
|
}
|
|
|
|
function mergeFieldMap<T extends Record<string, string | undefined>>(
|
|
current: T | undefined,
|
|
incoming: T | undefined
|
|
): T | undefined {
|
|
if (!current && !incoming) {
|
|
return undefined;
|
|
}
|
|
|
|
const merged = {
|
|
...(current ?? {})
|
|
} as Record<string, string | undefined>;
|
|
|
|
Object.entries(incoming ?? {}).forEach(([key, value]) => {
|
|
const currentValue = merged[key];
|
|
if (hasTextValue(value) || !hasTextValue(currentValue)) {
|
|
merged[key] = value;
|
|
}
|
|
});
|
|
|
|
return merged as T;
|
|
}
|
|
|
|
function mergeStringValue(
|
|
current: string | undefined,
|
|
incoming: string | undefined
|
|
): string | undefined {
|
|
return hasTextValue(incoming) ? incoming : current;
|
|
}
|
|
|
|
function hasTextValue(value: string | undefined): boolean {
|
|
return Boolean(value && value.trim().length > 0);
|
|
}
|