Fix silent market CSV export
This commit is contained in:
@@ -0,0 +1,401 @@
|
||||
import {
|
||||
readMarketListRequestSnapshot,
|
||||
type MarketListRequestSnapshot
|
||||
} from "./market-list-request-snapshot";
|
||||
import {
|
||||
parseMarketListResponse,
|
||||
readKnownPaginationNumber
|
||||
} from "./market-list-row";
|
||||
import type { MarketExportTarget, MarketRecord } from "./types";
|
||||
|
||||
interface FetchResponseLike {
|
||||
json(): Promise<unknown>;
|
||||
ok: boolean;
|
||||
}
|
||||
|
||||
type FetchLike = (input: string, init?: RequestInit) => Promise<FetchResponseLike>;
|
||||
|
||||
interface SilentExportControllerOptions {
|
||||
document: Document;
|
||||
fetchImpl?: FetchLike;
|
||||
onProgress?: (state: { currentPage: number; totalPages?: number }) => void;
|
||||
}
|
||||
|
||||
type PageSource = "body" | "none" | "url";
|
||||
|
||||
const PAGE_NUMBER_KEYS = [
|
||||
"currentPage",
|
||||
"page",
|
||||
"pageNo",
|
||||
"pageNum",
|
||||
"page_no",
|
||||
"page_num"
|
||||
] as const;
|
||||
|
||||
export function createSilentExportController(
|
||||
options: SilentExportControllerOptions
|
||||
) {
|
||||
const fetchImpl = options.fetchImpl ?? defaultFetch;
|
||||
|
||||
return {
|
||||
async exportRecords(target: MarketExportTarget): Promise<MarketRecord[] | null> {
|
||||
const snapshot = readMarketListRequestSnapshot(options.document);
|
||||
if (!snapshot) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const baseRequest = createPagedRequest(snapshot);
|
||||
if (!baseRequest) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const mergedRecords = new Map<string, MarketRecord>();
|
||||
const maxPageCount = target.mode === "count" ? target.pageCount : 200;
|
||||
let totalPagesHint: number | undefined;
|
||||
|
||||
for (let offset = 0; offset < maxPageCount; offset += 1) {
|
||||
const pageNumber = baseRequest.initialPage + offset;
|
||||
options.onProgress?.({
|
||||
currentPage: offset + 1,
|
||||
totalPages: target.mode === "count" ? target.pageCount : totalPagesHint
|
||||
});
|
||||
|
||||
const payload = await fetchPagePayload(fetchImpl, baseRequest, pageNumber);
|
||||
const parsedResponse = parseMarketListResponse(payload);
|
||||
if (!parsedResponse) {
|
||||
return null;
|
||||
}
|
||||
|
||||
totalPagesHint = parsedResponse.totalPages ?? totalPagesHint;
|
||||
if (parsedResponse.records.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
parsedResponse.records.forEach((record) => {
|
||||
const existingRecord = mergedRecords.get(record.authorId);
|
||||
mergedRecords.set(record.authorId, mergeMarketRecord(existingRecord, record));
|
||||
});
|
||||
|
||||
if (target.mode === "count" && offset + 1 >= target.pageCount) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (target.mode === "all") {
|
||||
if (
|
||||
typeof parsedResponse.totalPages === "number" &&
|
||||
pageNumber >= parsedResponse.totalPages
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (
|
||||
typeof parsedResponse.pageSize === "number" &&
|
||||
parsedResponse.records.length < parsedResponse.pageSize
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(mergedRecords.values());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function createPagedRequest(
|
||||
snapshot: MarketListRequestSnapshot
|
||||
): {
|
||||
initialPage: number;
|
||||
pageSource: PageSource;
|
||||
snapshot: MarketListRequestSnapshot;
|
||||
} {
|
||||
const bodyPage = readPageFromBody(snapshot.body);
|
||||
if (bodyPage !== null) {
|
||||
return {
|
||||
initialPage: bodyPage,
|
||||
pageSource: "body",
|
||||
snapshot
|
||||
};
|
||||
}
|
||||
|
||||
const urlPage = readPageFromUrl(snapshot.url);
|
||||
if (urlPage !== null) {
|
||||
return {
|
||||
initialPage: urlPage,
|
||||
pageSource: "url",
|
||||
snapshot
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
initialPage: 1,
|
||||
pageSource: "none",
|
||||
snapshot
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchPagePayload(
|
||||
fetchImpl: FetchLike,
|
||||
request: {
|
||||
pageSource: PageSource;
|
||||
snapshot: MarketListRequestSnapshot;
|
||||
},
|
||||
pageNumber: number
|
||||
): Promise<unknown> {
|
||||
const nextUrl =
|
||||
request.pageSource === "url"
|
||||
? mutateUrlPage(request.snapshot.url, pageNumber)
|
||||
: request.snapshot.url;
|
||||
const nextBody = mutateBodyPage(request.snapshot.body, pageNumber);
|
||||
|
||||
const response = await fetchImpl(nextUrl, {
|
||||
body: nextBody,
|
||||
credentials: "include",
|
||||
headers: filterReplayHeaders(request.snapshot.headers, nextBody),
|
||||
method: request.snapshot.method
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error("静默导出请求失败");
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
function readPageFromUrl(url: string): number | null {
|
||||
try {
|
||||
const parsedUrl = new URL(url);
|
||||
for (const key of PAGE_NUMBER_KEYS) {
|
||||
const value = readNumericString(parsedUrl.searchParams.get(key));
|
||||
if (value !== null) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function mutateUrlPage(url: string, pageNumber: number): string {
|
||||
try {
|
||||
const parsedUrl = new URL(url);
|
||||
for (const key of PAGE_NUMBER_KEYS) {
|
||||
if (!parsedUrl.searchParams.has(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
parsedUrl.searchParams.set(key, String(pageNumber));
|
||||
return parsedUrl.toString();
|
||||
}
|
||||
|
||||
parsedUrl.searchParams.set("page", String(pageNumber));
|
||||
return parsedUrl.toString();
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
function readPageFromBody(body: string | undefined): number | null {
|
||||
const parsedBody = parseBody(body);
|
||||
if (!parsedBody) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return readKnownPaginationNumber(parsedBody, "page");
|
||||
}
|
||||
|
||||
function mutateBodyPage(body: string | undefined, pageNumber: number): string | undefined {
|
||||
if (!body) {
|
||||
return body;
|
||||
}
|
||||
|
||||
const trimmedBody = body.trim();
|
||||
if (!trimmedBody) {
|
||||
return body;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsedJson = JSON.parse(trimmedBody);
|
||||
if (!replacePageNumberInValue(parsedJson, pageNumber) && isRecord(parsedJson)) {
|
||||
parsedJson.page = pageNumber;
|
||||
}
|
||||
|
||||
return JSON.stringify(parsedJson);
|
||||
} catch {
|
||||
const searchParams = new URLSearchParams(trimmedBody);
|
||||
for (const key of PAGE_NUMBER_KEYS) {
|
||||
if (!searchParams.has(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
searchParams.set(key, String(pageNumber));
|
||||
return searchParams.toString();
|
||||
}
|
||||
|
||||
searchParams.set("page", String(pageNumber));
|
||||
return searchParams.toString();
|
||||
}
|
||||
}
|
||||
|
||||
function parseBody(body: string | undefined): Record<string, unknown> | null {
|
||||
if (!body) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const trimmedBody = body.trim();
|
||||
if (!trimmedBody) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsedBody = JSON.parse(trimmedBody);
|
||||
return isRecord(parsedBody) ? parsedBody : null;
|
||||
} catch {
|
||||
const searchParams = new URLSearchParams(trimmedBody);
|
||||
const payload: Record<string, unknown> = {};
|
||||
searchParams.forEach((value, key) => {
|
||||
payload[key] = value;
|
||||
});
|
||||
return payload;
|
||||
}
|
||||
}
|
||||
|
||||
function replacePageNumberInValue(value: unknown, pageNumber: number): boolean {
|
||||
if (!isRecord(value)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let replaced = false;
|
||||
PAGE_NUMBER_KEYS.forEach((key) => {
|
||||
if (!(key in value)) {
|
||||
return;
|
||||
}
|
||||
|
||||
value[key] = pageNumber;
|
||||
replaced = true;
|
||||
});
|
||||
|
||||
if (replaced) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return Object.values(value).some((entry) => replacePageNumberInValue(entry, pageNumber));
|
||||
}
|
||||
|
||||
function filterReplayHeaders(
|
||||
headers: Record<string, string> | undefined,
|
||||
body: string | undefined
|
||||
): HeadersInit | undefined {
|
||||
const filteredHeaders = Object.fromEntries(
|
||||
Object.entries(headers ?? {}).filter(([key]) => {
|
||||
const normalizedKey = key.toLowerCase();
|
||||
return normalizedKey !== "content-length" && normalizedKey !== "host";
|
||||
})
|
||||
);
|
||||
|
||||
if (body) {
|
||||
if (!hasHeader(filteredHeaders, "accept")) {
|
||||
filteredHeaders.Accept = "application/json, text/plain, */*";
|
||||
}
|
||||
if (!hasHeader(filteredHeaders, "content-type")) {
|
||||
filteredHeaders["Content-Type"] = "application/json";
|
||||
}
|
||||
if (!hasHeader(filteredHeaders, "x-login-source")) {
|
||||
filteredHeaders["x-login-source"] = "1";
|
||||
}
|
||||
if (!hasHeader(filteredHeaders, "agw-js-conv")) {
|
||||
filteredHeaders["Agw-Js-Conv"] = "str";
|
||||
}
|
||||
}
|
||||
|
||||
return Object.keys(filteredHeaders).length > 0 ? filteredHeaders : undefined;
|
||||
}
|
||||
|
||||
function hasHeader(headers: Record<string, string>, key: string): boolean {
|
||||
return Object.keys(headers).some((headerKey) => headerKey.toLowerCase() === key);
|
||||
}
|
||||
|
||||
function readNumericString(value: string | null): number | null {
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsedValue = Number(value);
|
||||
return Number.isFinite(parsedValue) ? parsedValue : null;
|
||||
}
|
||||
|
||||
async function defaultFetch(input: string, init?: RequestInit) {
|
||||
return fetch(input, init);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
|
||||
function mergeMarketRecord(
|
||||
existingRecord: MarketRecord | undefined,
|
||||
incomingRecord: MarketRecord
|
||||
): MarketRecord {
|
||||
if (!existingRecord) {
|
||||
return {
|
||||
...incomingRecord,
|
||||
exportFields: mergeFieldMap(undefined, incomingRecord.exportFields),
|
||||
rates: mergeFieldMap(undefined, incomingRecord.rates),
|
||||
status: incomingRecord.status ?? "idle"
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...existingRecord,
|
||||
...incomingRecord,
|
||||
authorName: mergeStringValue(existingRecord.authorName, incomingRecord.authorName) ?? "",
|
||||
exportFields: mergeFieldMap(
|
||||
existingRecord.exportFields,
|
||||
incomingRecord.exportFields
|
||||
),
|
||||
failureReason: incomingRecord.failureReason ?? existingRecord.failureReason,
|
||||
hasDirectRatesSource:
|
||||
existingRecord.hasDirectRatesSource || incomingRecord.hasDirectRatesSource,
|
||||
location: mergeStringValue(existingRecord.location, incomingRecord.location),
|
||||
price21To60s: mergeStringValue(
|
||||
existingRecord.price21To60s,
|
||||
incomingRecord.price21To60s
|
||||
),
|
||||
rates: mergeFieldMap(existingRecord.rates, incomingRecord.rates),
|
||||
status: incomingRecord.status ?? existingRecord.status
|
||||
};
|
||||
}
|
||||
|
||||
function mergeFieldMap<T extends Record<string, string | undefined>>(
|
||||
current: T | undefined,
|
||||
incoming: T | undefined
|
||||
): T | undefined {
|
||||
if (!current && !incoming) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const merged = {
|
||||
...(current ?? {})
|
||||
} as Record<string, string | undefined>;
|
||||
|
||||
Object.entries(incoming ?? {}).forEach(([key, value]) => {
|
||||
const currentValue = merged[key];
|
||||
if (hasTextValue(value) || !hasTextValue(currentValue)) {
|
||||
merged[key] = value;
|
||||
}
|
||||
});
|
||||
|
||||
return merged as T;
|
||||
}
|
||||
|
||||
function mergeStringValue(
|
||||
current: string | undefined,
|
||||
incoming: string | undefined
|
||||
): string | undefined {
|
||||
return hasTextValue(incoming) ? incoming : current;
|
||||
}
|
||||
|
||||
function hasTextValue(value: string | undefined): boolean {
|
||||
return Boolean(value && value.trim().length > 0);
|
||||
}
|
||||
Reference in New Issue
Block a user