diff --git a/src/content/index.ts b/src/content/index.ts index f6d1726..13f8af6 100644 --- a/src/content/index.ts +++ b/src/content/index.ts @@ -39,15 +39,18 @@ export async function bootContentScript( return null; } + installMarketPageBridge(currentDocument); + const authState = await readAuthState(sendAuthMessage); if (!authState?.isAuthenticated) { + await waitForBodyReady(currentDocument, currentWindow); renderMarketAuthGate(currentDocument, currentWindow); return { ready: Promise.resolve() }; } - installMarketPageBridge(currentDocument); + await waitForBodyReady(currentDocument, currentWindow); return controllerFactory({ document: currentDocument, @@ -144,6 +147,24 @@ function createRuntimeMessageSender(): (message: unknown) => Promise { }; } +async function waitForBodyReady(document: Document, currentWindow: Window): Promise { + if (document.body) { + return; + } + + await new Promise((resolve) => { + const handleReady = () => { + if (document.body) { + document.removeEventListener("DOMContentLoaded", handleReady); + resolve(); + } + }; + + document.addEventListener("DOMContentLoaded", handleReady); + currentWindow.setTimeout(handleReady, 0); + }); +} + function downloadCsv(document: Document, window: Window, csv: string): void { const blob = new Blob(["\uFEFF", csv], { type: "text/csv;charset=utf-8" diff --git a/src/content/market/dom-sync.ts b/src/content/market/dom-sync.ts index 1b8865f..b871179 100644 --- a/src/content/market/dom-sync.ts +++ b/src/content/market/dom-sync.ts @@ -2,6 +2,7 @@ import { normalizeFractionRateDisplay, normalizeRateDisplay } from "../../shared/rate-normalizer"; +import { mapMarketListRow } from "./market-list-row"; import type { AfterSearchRates, BackendMetrics, @@ -1253,33 +1254,9 @@ function readVueMarketRows( continue; } - return marketList.map((row) => { - const record = isRecord(row) ? row : {}; - const attributeDatas = readMarketAttributeDatas(record); - const singleVideoAfterSearchRate = normalizeMarketListRate( - readMarketFieldValue(record, attributeDatas, "avg_search_after_view_rate_30d") - ); - - return { - authorId: - readString(readMarketFieldValue(record, attributeDatas, "star_id")) ?? - readString(readMarketFieldValue(record, attributeDatas, "id")) ?? - "", - authorName: - readString(readMarketFieldValue(record, attributeDatas, "nickname")) ?? - readString(readMarketFieldValue(record, attributeDatas, "nick_name")) ?? - "", - exportFields: buildMarketExportFieldFallbacks(record, attributeDatas), - hasDirectRatesSource: true, - location: readMarketLocation(record, attributeDatas), - price21To60s: readMarketPrice21To60s(record, attributeDatas), - rates: singleVideoAfterSearchRate - ? { - singleVideoAfterSearchRate - } - : undefined - }; - }); + return marketList + .map((row) => (isRecord(row) ? mapMarketListRow(row) : null)) + .filter((row): row is MarketDataRow => row !== null); } } diff --git a/src/content/market/index.ts b/src/content/market/index.ts index 686e290..390f717 100644 --- a/src/content/market/index.ts +++ b/src/content/market/index.ts @@ -14,6 +14,7 @@ import { applyFilterAndSort } from "./filter-sort-controller"; import { createMarketApiClient } from "./api-client"; import { createExportRangeController } from "./export-range-controller"; import { ensurePluginToolbar, isPluginToolbarMounted } from "./plugin-toolbar"; +import { createSilentExportController } from "./silent-export-controller"; import { readToolbarExportTarget, setToolbarBusyState, @@ -93,6 +94,17 @@ export function createMarketController(options: CreateMarketControllerOptions) { readCurrentPageRowCount: () => countCurrentPageRows(options.document), window: options.window }); + const silentExportController = createSilentExportController({ + document: options.document, + onProgress: ({ currentPage, totalPages }) => { + setToolbarExportStatus( + toolbar, + totalPages + ? `导出中 ${currentPage}/${totalPages} 页...` + : `导出中 第${currentPage}页...` + ); + } + }); let activeSort: MarketSortState | undefined; let isDisposed = false; let isSyncRunning = false; @@ -239,69 +251,96 @@ export function createMarketController(options: CreateMarketControllerOptions) { rowSnapshot }); resultStore.upsertMarketRow(rowSnapshot); - const existingRecord = resultStore.getRecord(rowSnapshot.authorId); - if (existingRecord?.status === "success" && existingRecord.rates) { - renderMarketRowState(rowDom, existingRecord); - continue; - } - - if (existingRecord?.status === "failed") { - renderMarketRowState(rowDom, existingRecord); - continue; - } - - if (existingRecord?.status === "loading") { - renderMarketRowState(rowDom, { - ...rowSnapshot, - status: "loading" - }); - continue; - } - - if (rowSnapshot.hasDirectRatesSource) { - const directRates = rowSnapshot.rates ?? {}; - const hasAllRates = - Boolean(directRates.singleVideoAfterSearchRate) && - Boolean(directRates.personalVideoAfterSearchRate); - - resultStore.setAuthorSuccess(rowSnapshot.authorId, directRates); - renderMarketRowState(rowDom, { - ...rowSnapshot, - rates: directRates, - status: "success" - }); - if (hasAllRates) { - continue; - } - } - - resultStore.setAuthorLoading(rowSnapshot.authorId); - renderMarketRowState(rowDom, { - ...rowSnapshot, - rates: resultStore.getRecord(rowSnapshot.authorId)?.rates, - status: "loading" - }); - - const metricsResult = await loadAuthorMetrics(rowSnapshot.authorId); - if (metricsResult.success) { - resultStore.setAuthorSuccess(rowSnapshot.authorId, metricsResult.rates); - renderMarketRowState(rowDom, { - ...rowSnapshot, - status: "success", - rates: metricsResult.rates - }); - continue; - } - - resultStore.setAuthorFailed(rowSnapshot.authorId, metricsResult.reason); - renderMarketRowState(rowDom, { - ...rowSnapshot, - failureReason: metricsResult.reason, - status: "failed" - }); } - await hydrateBackendMetricsForPage(pageRows); + const pendingRateRows: typeof pageRows = []; + const rowsNeedingBackendMetrics: typeof pageRows = []; + + pageRows.forEach(({ rowDom, rowSnapshot }) => { + if (rowSnapshot.hasDirectRatesSource) { + resultStore.setAuthorSuccess(rowSnapshot.authorId, rowSnapshot.rates ?? {}); + } + + const existingRecord = resultStore.getRecord(rowSnapshot.authorId); + const needsRateFetch = + !hasSettledRateState(existingRecord) && + !hasCompleteRates(existingRecord?.rates); + const needsBackendMetrics = + Boolean(searchBackendMetrics) && + !hasSettledBackendMetricsState(existingRecord); + + if (needsRateFetch) { + resultStore.setAuthorLoading(rowSnapshot.authorId); + pendingRateRows.push({ + rowDom, + rowSnapshot + }); + } + + if (needsBackendMetrics) { + resultStore.setBackendMetricsLoading(rowSnapshot.authorId); + rowsNeedingBackendMetrics.push({ + rowDom, + rowSnapshot + }); + } + + if (needsRateFetch || needsBackendMetrics) { + renderMarketRowState(rowDom, { + ...(existingRecord ?? { + authorId: rowSnapshot.authorId, + authorName: rowSnapshot.authorName, + status: "idle" as const + }), + ...rowSnapshot, + backendMetricsStatus: needsBackendMetrics ? "loading" : existingRecord?.backendMetricsStatus, + rates: existingRecord?.rates, + status: needsRateFetch || needsBackendMetrics ? "loading" : existingRecord?.status ?? "idle" + }); + return; + } + + if (existingRecord) { + renderMarketRowState(rowDom, existingRecord); + } + }); + + await Promise.all([ + hydrateRatesForRows(pendingRateRows), + hydrateBackendMetricsForPage(rowsNeedingBackendMetrics) + ]); + + pageRows.forEach(({ rowDom, rowSnapshot }) => { + const record = resultStore.getRecord(rowSnapshot.authorId); + if (!record) { + return; + } + + renderMarketRowState(rowDom, record); + }); + } + + async function hydrateRatesForRows( + pageRows: Array<{ + rowDom: MarketRowDom; + rowSnapshot: MarketRowSnapshot; + }> + ): Promise { + if (pageRows.length === 0) { + return; + } + + await Promise.all( + pageRows.map(async ({ rowSnapshot }) => { + const metricsResult = await loadAuthorMetrics(rowSnapshot.authorId); + if (metricsResult.success) { + resultStore.setAuthorSuccess(rowSnapshot.authorId, metricsResult.rates); + return; + } + + resultStore.setAuthorFailed(rowSnapshot.authorId, metricsResult.reason); + }) + ); } async function hydrateBackendMetricsForPage( @@ -314,70 +353,23 @@ export function createMarketController(options: CreateMarketControllerOptions) { return; } - const pendingBackendRows = pageRows.filter(({ rowDom, rowSnapshot }) => { - const record = resultStore.getRecord(rowSnapshot.authorId); - if (!record) { - return false; - } - - if ( - record.backendMetricsStatus === "success" || - record.backendMetricsStatus === "missing" || - record.backendMetricsStatus === "failed" || - record.backendMetricsStatus === "loading" - ) { - renderMarketRowState(rowDom, record); - return false; - } - - resultStore.setBackendMetricsLoading(rowSnapshot.authorId); - renderMarketRowState(rowDom, { - ...record, - ...rowSnapshot, - backendMetricsStatus: "loading" - }); - return true; - }); - - if (pendingBackendRows.length === 0) { - return; - } - try { const rows = await searchBackendMetrics( - pendingBackendRows.map(({ rowSnapshot }) => rowSnapshot.authorId) + pageRows.map(({ rowSnapshot }) => rowSnapshot.authorId) ); const rowMap = new Map(rows.map((row) => [row.starId, row])); - pendingBackendRows.forEach(({ rowSnapshot }) => { + pageRows.forEach(({ rowSnapshot }) => { const backendMetrics = rowMap.get(rowSnapshot.authorId); if (backendMetrics) { resultStore.setBackendMetricsSuccess(rowSnapshot.authorId, backendMetrics); } else { resultStore.setBackendMetricsMissing(rowSnapshot.authorId); } - - const record = resultStore.getRecord(rowSnapshot.authorId); - if (!record) { - return; - } - - const pageRow = pendingBackendRows.find( - (candidate) => candidate.rowSnapshot.authorId === rowSnapshot.authorId - ); - if (pageRow) { - renderMarketRowState(pageRow.rowDom, record); - } }); } catch { - pendingBackendRows.forEach(({ rowDom, rowSnapshot }) => { + pageRows.forEach(({ rowSnapshot }) => { resultStore.setBackendMetricsFailed(rowSnapshot.authorId); - const record = resultStore.getRecord(rowSnapshot.authorId); - if (!record) { - return; - } - - renderMarketRowState(rowDom, record); }); } } @@ -487,12 +479,23 @@ export function createMarketController(options: CreateMarketControllerOptions) { target: MarketExportTarget, inProgressLabel = "导出中" ): Promise { + setToolbarExportStatus(toolbar, `${inProgressLabel}...`); + if (target.mode === "count" && target.pageCount <= 1) { - setToolbarExportStatus(toolbar, `${inProgressLabel}...`); await prepareCurrentPageForExport(); return getVisibleOrderedRecords(); } + const silentExportRecords = await silentExportController.exportRecords(target); + if (silentExportRecords) { + return hydrateExportRecords( + silentExportRecords.map((record) => ({ + ...record, + status: record.status ?? "idle" + })) + ); + } + return exportRangeController.exportRecords(target); } @@ -513,6 +516,75 @@ export function createMarketController(options: CreateMarketControllerOptions) { await runSyncCycle(); } + async function hydrateExportRecords(records: MarketRecord[]): Promise { + for (const record of records) { + resultStore.upsertMarketRow(record); + const existingRecord = resultStore.getRecord(record.authorId); + if (existingRecord?.status === "success" && existingRecord.rates) { + continue; + } + + if (record.hasDirectRatesSource) { + const directRates = record.rates ?? {}; + const hasAllRates = + Boolean(directRates.singleVideoAfterSearchRate) && + Boolean(directRates.personalVideoAfterSearchRate); + + resultStore.setAuthorSuccess(record.authorId, directRates); + if (hasAllRates) { + continue; + } + } else { + resultStore.setAuthorLoading(record.authorId); + } + + const metricsResult = await loadAuthorMetrics(record.authorId); + if (metricsResult.success) { + resultStore.setAuthorSuccess(record.authorId, metricsResult.rates); + } else { + resultStore.setAuthorFailed(record.authorId, metricsResult.reason); + } + } + + if (searchBackendMetrics) { + const backendTargetRecords = records.filter((record) => { + const existingRecord = resultStore.getRecord(record.authorId); + return !( + existingRecord?.backendMetricsStatus === "success" || + existingRecord?.backendMetricsStatus === "missing" + ); + }); + + if (backendTargetRecords.length > 0) { + backendTargetRecords.forEach((record) => { + resultStore.setBackendMetricsLoading(record.authorId); + }); + + try { + const backendRows = await searchBackendMetrics( + backendTargetRecords.map((record) => record.authorId) + ); + const backendRowMap = new Map(backendRows.map((row) => [row.starId, row])); + + backendTargetRecords.forEach((record) => { + const backendMetrics = backendRowMap.get(record.authorId); + if (backendMetrics) { + resultStore.setBackendMetricsSuccess(record.authorId, backendMetrics); + } else { + resultStore.setBackendMetricsMissing(record.authorId); + } + }); + } catch { + backendTargetRecords.forEach((record) => { + resultStore.setBackendMetricsFailed(record.authorId); + }); + } + } + } + + return records.map((record) => toMarketRecord(record)); + } + async function harvestCurrentPageForExport(): Promise { let hydrationSnapshot = await collectCurrentPageSnapshotsUntilSettled(); if ( @@ -574,40 +646,44 @@ export function createMarketController(options: CreateMarketControllerOptions) { return null; } - const existingRecord = resultStore.getRecord(rowSnapshot.authorId); - const authorName = - mergeStringValue(existingRecord?.authorName, rowSnapshot.authorName) ?? ""; - const location = mergeStringValue(existingRecord?.location, rowSnapshot.location); - const price21To60s = mergeStringValue( - existingRecord?.price21To60s, - rowSnapshot.price21To60s - ); - return { - ...existingRecord, - ...rowSnapshot, - authorName, - backendMetrics: mergeFieldMap( - existingRecord?.backendMetrics, - rowSnapshot.backendMetrics - ), - backendMetricsStatus: existingRecord?.backendMetricsStatus ?? "idle", - exportFields: withExportFieldFallbacks( - mergeFieldMap(existingRecord?.exportFields, rowSnapshot.exportFields), - { - authorName, - location, - price21To60s - } - ), - location, - price21To60s, - rates: mergeFieldMap(existingRecord?.rates, rowSnapshot.rates), - status: existingRecord?.status ?? "idle" - } satisfies MarketRecord; + return toMarketRecord(rowSnapshot); }) .filter((record): record is MarketRecord => record !== null); } + function toMarketRecord(rowSnapshot: MarketRowSnapshot): MarketRecord { + const existingRecord = resultStore.getRecord(rowSnapshot.authorId); + const authorName = + mergeStringValue(existingRecord?.authorName, rowSnapshot.authorName) ?? ""; + const location = mergeStringValue(existingRecord?.location, rowSnapshot.location); + const price21To60s = mergeStringValue( + existingRecord?.price21To60s, + rowSnapshot.price21To60s + ); + return { + ...existingRecord, + ...rowSnapshot, + authorName, + backendMetrics: mergeFieldMap( + existingRecord?.backendMetrics, + rowSnapshot.backendMetrics + ), + backendMetricsStatus: existingRecord?.backendMetricsStatus ?? "idle", + exportFields: withExportFieldFallbacks( + mergeFieldMap(existingRecord?.exportFields, rowSnapshot.exportFields), + { + authorName, + location, + price21To60s + } + ), + location, + price21To60s, + rates: mergeFieldMap(existingRecord?.rates, rowSnapshot.rates), + status: existingRecord?.status ?? "idle" + } satisfies MarketRecord; + } + function collectCurrentPageSnapshots(): void { readCurrentPageRows(options.document).forEach((rowSnapshot) => { resultStore.upsertMarketRow(rowSnapshot); @@ -937,6 +1013,39 @@ function getNextSortState( return undefined; } +function hasCompleteRates( + rates: + | { + personalVideoAfterSearchRate?: string; + singleVideoAfterSearchRate?: string; + } + | undefined +): boolean { + return Boolean( + rates?.singleVideoAfterSearchRate && rates?.personalVideoAfterSearchRate + ); +} + +function hasSettledRateState(record: MarketRecord | null): boolean { + if (!record) { + return false; + } + + return record.status === "failed" || hasCompleteRates(record.rates); +} + +function hasSettledBackendMetricsState(record: MarketRecord | null): boolean { + if (!record) { + return false; + } + + return ( + record.backendMetricsStatus === "success" || + record.backendMetricsStatus === "missing" || + record.backendMetricsStatus === "failed" + ); +} + function mergeFieldMap>( current: T | undefined, incoming: T | undefined diff --git a/src/content/market/market-list-request-snapshot.ts b/src/content/market/market-list-request-snapshot.ts new file mode 100644 index 0000000..d9f795e --- /dev/null +++ b/src/content/market/market-list-request-snapshot.ts @@ -0,0 +1,173 @@ +export const MARKET_REQUEST_SNAPSHOT_ATTRIBUTE = + "data-sces-market-request-snapshot"; +const MARKET_SEARCH_ENDPOINT_PATH = "/gw/api/gsearch/search_for_author_square"; + +export interface MarketListRequestSnapshot { + body?: string; + headers?: Record; + method: string; + url: string; +} + +export function readMarketListRequestSnapshot( + document: Document +): MarketListRequestSnapshot | null { + const serializedSnapshot = document.documentElement.getAttribute( + MARKET_REQUEST_SNAPSHOT_ATTRIBUTE + ); + if (!serializedSnapshot) { + return readMarketListRequestSnapshotFromPageState(document); + } + + try { + const parsedSnapshot = normalizeMarketListRequestSnapshot( + JSON.parse(serializedSnapshot) + ); + if (!parsedSnapshot) { + return readMarketListRequestSnapshotFromPageState(document); + } + + return parsedSnapshot; + } catch { + return readMarketListRequestSnapshotFromPageState(document); + } +} + +export function writeMarketListRequestSnapshot( + document: Document, + snapshot: MarketListRequestSnapshot +): void { + document.documentElement.setAttribute( + MARKET_REQUEST_SNAPSHOT_ATTRIBUTE, + JSON.stringify(snapshot) + ); +} + +function isMarketListRequestSnapshot( + value: unknown +): value is MarketListRequestSnapshot { + if (!value || typeof value !== "object") { + return false; + } + + const candidate = value as Partial; + return ( + typeof candidate.method === "string" && + typeof candidate.url === "string" && + (!("body" in candidate) || typeof candidate.body === "string") && + (!("headers" in candidate) || isStringRecord(candidate.headers)) + ); +} + +function normalizeMarketListRequestSnapshot( + value: unknown +): MarketListRequestSnapshot | null { + if (!value || typeof value !== "object") { + return null; + } + + const candidate = value as Partial & { + headers?: Record; + }; + const normalizedSnapshot: Partial = { + body: typeof candidate.body === "string" ? candidate.body : undefined, + method: typeof candidate.method === "string" ? candidate.method : undefined, + url: typeof candidate.url === "string" ? candidate.url : undefined + }; + + if (candidate.headers && typeof candidate.headers === "object") { + normalizedSnapshot.headers = Object.fromEntries( + Object.entries(candidate.headers) + .filter(([, entry]) => + ["string", "number", "boolean"].includes(typeof entry) + ) + .map(([key, entry]) => [key, String(entry)]) + ); + } + + return isMarketListRequestSnapshot(normalizedSnapshot) + ? normalizedSnapshot + : null; +} + +function isStringRecord(value: unknown): value is Record { + if (!value || typeof value !== "object") { + return false; + } + + return Object.values(value).every((entry) => typeof entry === "string"); +} + +function readMarketListRequestSnapshotFromPageState( + document: Document +): MarketListRequestSnapshot | null { + const reqParams = findMarketReqParams(document); + if (!reqParams) { + return null; + } + + return { + body: JSON.stringify(reqParams), + method: "POST", + url: buildMarketSearchUrl(document) + }; +} + +function findMarketReqParams(document: Document): Record | null { + const marketRoot = document.querySelector(".base-author-list") as + | (Element & { + __vue__?: { + _setupState?: Record; + }; + }) + | null; + const setupState = marketRoot?.__vue__?._setupState; + if (!setupState) { + return null; + } + + const queue: unknown[] = Object.values(setupState); + while (queue.length > 0) { + const current = unwrapVueRef(queue.shift()); + if (!isRecord(current)) { + continue; + } + + const reqParams = unwrapVueRef(current.reqParams); + if (isRecord(reqParams)) { + return reqParams; + } + + Object.values(current).forEach((value) => { + queue.push(value); + }); + } + + return null; +} + +function buildMarketSearchUrl(document: Document): string { + if ( + document.location?.origin && + document.location.origin !== "null" && + document.location.origin !== "about:blank" + ) { + return document.location.origin.includes("xingtu.cn") + ? MARKET_SEARCH_ENDPOINT_PATH + : new URL(MARKET_SEARCH_ENDPOINT_PATH, document.location.origin).toString(); + } + + return MARKET_SEARCH_ENDPOINT_PATH; +} + +function unwrapVueRef(value: unknown): unknown { + if (isRecord(value) && "value" in value) { + return value.value; + } + + return value; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} diff --git a/src/content/market/market-list-row.ts b/src/content/market/market-list-row.ts new file mode 100644 index 0000000..6713e73 --- /dev/null +++ b/src/content/market/market-list-row.ts @@ -0,0 +1,513 @@ +import { normalizeFractionRateDisplay } from "../../shared/rate-normalizer"; +import type { MarketRowSnapshot } from "./types"; + +export interface ParsedMarketListResponse { + currentPage?: number; + pageSize?: number; + records: MarketRowSnapshot[]; + totalCount?: number; + totalPages?: number; +} + +const PAGE_NUMBER_KEYS = [ + "currentPage", + "page", + "pageNo", + "pageNum", + "page_no", + "page_num" +] as const; + +const PAGE_SIZE_KEYS = [ + "limit", + "pageSize", + "page_size", + "size" +] as const; + +const TOTAL_COUNT_KEYS = [ + "total", + "totalCount", + "total_count" +] as const; + +const TOTAL_PAGE_KEYS = [ + "pageCount", + "page_count", + "totalPage", + "totalPages", + "total_page", + "total_pages" +] as const; + +export function mapMarketListRow( + row: Record +): MarketRowSnapshot { + const attributeDatas = readMarketAttributeDatas(row); + const singleVideoAfterSearchRate = normalizeMarketListRate( + readMarketFieldValue(row, attributeDatas, "avg_search_after_view_rate_30d") + ); + + return { + authorId: + readString(readMarketFieldValue(row, attributeDatas, "star_id")) ?? + readString(readMarketFieldValue(row, attributeDatas, "id")) ?? + "", + authorName: + readString(readMarketFieldValue(row, attributeDatas, "nickname")) ?? + readString(readMarketFieldValue(row, attributeDatas, "nick_name")) ?? + "", + exportFields: buildMarketExportFieldFallbacks(row, attributeDatas), + hasDirectRatesSource: true, + location: readMarketLocation(row, attributeDatas), + price21To60s: readMarketPrice21To60s(row, attributeDatas), + rates: singleVideoAfterSearchRate + ? { + singleVideoAfterSearchRate + } + : undefined + }; +} + +export function parseMarketListResponse( + payload: unknown +): ParsedMarketListResponse | null { + const container = findMarketListContainer(payload); + if (!container) { + return null; + } + + const marketList = readMarketListArray(container); + if (!marketList) { + return null; + } + + return { + currentPage: readKnownNumberDeep(container, PAGE_NUMBER_KEYS) ?? undefined, + pageSize: readKnownNumberDeep(container, PAGE_SIZE_KEYS) ?? undefined, + records: marketList + .map((row) => (isRecord(row) ? mapMarketListRow(row) : null)) + .filter( + (row): row is MarketRowSnapshot => + row !== null && Boolean(row.authorId || row.authorName) + ), + totalCount: readKnownNumberDeep(container, TOTAL_COUNT_KEYS) ?? undefined, + totalPages: readKnownNumberDeep(container, TOTAL_PAGE_KEYS) ?? undefined + }; +} + +export function readKnownPaginationNumber( + value: unknown, + kind: "page" | "pageSize" +): number | null { + if (!isRecord(value)) { + return null; + } + + return readKnownNumberDeep(value, kind === "page" ? PAGE_NUMBER_KEYS : PAGE_SIZE_KEYS); +} + +function findMarketListContainer(value: unknown): Record | null { + const queue: unknown[] = [value]; + + while (queue.length > 0) { + const current = queue.shift(); + if (!isRecord(current)) { + continue; + } + + if (readMarketListArray(current)) { + return current; + } + + Object.values(current).forEach((entry) => { + queue.push(unwrapVueRef(entry)); + }); + } + + return null; +} + +function readMarketListArray(record: Record): unknown[] | null { + const marketList = unwrapVueRef(record.marketList); + if (Array.isArray(marketList)) { + return marketList; + } + + const authors = unwrapVueRef(record.authors); + if (Array.isArray(authors)) { + return authors; + } + + return null; +} + +function unwrapVueRef(value: unknown): unknown { + if (isRecord(value) && "value" in value) { + return value.value; + } + + return value; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function readMarketAttributeDatas( + record: Record +): Record { + return isRecord(record.attribute_datas) ? record.attribute_datas : {}; +} + +function readMarketFieldValue( + record: Record, + attributeDatas: Record, + field: string +): unknown { + return record[field] ?? attributeDatas[field]; +} + +function readString(value: unknown): string | null { + return typeof value === "string" ? value : null; +} + +function normalizeMarketListRate(value: unknown): string | null { + if (typeof value === "number") { + return normalizeFractionRateDisplay(String(value)); + } + + return typeof value === "string" ? normalizeFractionRateDisplay(value) : null; +} + +function normalizeExportCellText(value: string | null | undefined): string { + return value?.replace(/\s+/g, " ").trim() ?? ""; +} + +function buildMarketExportFieldFallbacks( + record: Record, + attributeDatas: Record +): Record | undefined { + const exportFields: Record = {}; + const authorInfo = buildMarketAuthorInfo(record, attributeDatas); + const authorType = buildMarketAuthorType(record, attributeDatas); + const contentTheme = buildMarketContentTheme(record, attributeDatas); + const connectedUsers = formatWanValue( + readNumericValue(readMarketFieldValue(record, attributeDatas, "link_link_cnt_by_industry")) + ); + const followerCount = formatWanValue( + readNumericValue(readMarketFieldValue(record, attributeDatas, "follower")) + ); + const expectedCpm = formatDecimalDisplay( + readNumericValue(readMarketFieldValue(record, attributeDatas, "prospective_20_60_cpm")) + ); + const expectedPlayCount = formatWanValue( + readNumericValue(readMarketFieldValue(record, attributeDatas, "expected_play_num")) + ); + const interactionRate = formatFractionPercent( + readNumericValue(readMarketFieldValue(record, attributeDatas, "interact_rate_within_30d")) + ); + const finishRate = formatFractionPercent( + readNumericValue(readMarketFieldValue(record, attributeDatas, "play_over_rate_within_30d")) + ); + const burstRate = readBurstRateDisplay( + readNumericValue(readMarketFieldValue(record, attributeDatas, "burst_text_rate")) + ); + const price21To60s = readMarketPrice21To60s(record, attributeDatas); + const representativeVideo = readMarketRepresentativeVideo(record, attributeDatas); + + assignExportField(exportFields, "达人信息", authorInfo); + assignExportField(exportFields, "代表视频", representativeVideo); + assignExportField(exportFields, "达人类型", authorType); + assignExportField(exportFields, "内容主题", contentTheme); + assignExportField(exportFields, "连接用户数", connectedUsers); + assignExportField(exportFields, "粉丝数", followerCount); + assignExportField(exportFields, "预期CPM", expectedCpm); + assignExportField(exportFields, "预期播放量", expectedPlayCount); + assignExportField(exportFields, "互动率", interactionRate); + assignExportField(exportFields, "完播率", finishRate); + assignExportField(exportFields, "爆文率", burstRate); + assignExportField(exportFields, "21-60s报价", price21To60s); + + return Object.keys(exportFields).length > 0 ? exportFields : undefined; +} + +function assignExportField( + exportFields: Record, + key: string, + value: string | undefined +): void { + if (hasTextValue(value)) { + exportFields[key] = value; + } +} + +function hasTextValue(value: string | undefined | null): boolean { + return Boolean(value && value.trim().length > 0); +} + +function buildMarketAuthorInfo( + record: Record, + attributeDatas: Record +): string | undefined { + const nickname = + readString(readMarketFieldValue(record, attributeDatas, "nickname")) ?? + readString(readMarketFieldValue(record, attributeDatas, "nick_name")) ?? + ""; + const parts = [ + nickname, + readMarketGenderLabel(readMarketFieldValue(record, attributeDatas, "gender")), + readString(readMarketFieldValue(record, attributeDatas, "city")) ?? "" + ].filter((value) => Boolean(value)); + + return parts.length > 0 ? parts.join(" ") : undefined; +} + +function buildMarketAuthorType( + record: Record, + attributeDatas: Record +): string | undefined { + const tagsRelation = readRecordLike( + readMarketFieldValue(record, attributeDatas, "tags_relation") + ); + if (tagsRelation) { + const primaryTag = Object.keys(tagsRelation)[0]; + if (hasTextValue(primaryTag)) { + return primaryTag; + } + } + + return undefined; +} + +function buildMarketContentTheme( + record: Record, + attributeDatas: Record +): string | undefined { + const themes = readStringArray( + readMarketFieldValue(record, attributeDatas, "content_theme_labels_180d") + ); + if (themes.length === 0) { + return undefined; + } + + if (themes.length <= 2) { + return themes.join(" "); + } + + return `${themes.slice(0, 2).join(" ")} ${themes.length - 2}+`; +} + +function readMarketLocation( + record: Record, + attributeDatas: Record +): string | undefined { + return readString(readMarketFieldValue(record, attributeDatas, "city")) ?? undefined; +} + +function readMarketPrice21To60s( + record: Record, + attributeDatas: Record +): string | undefined { + return formatCurrencyValue( + readNumericValue(readMarketFieldValue(record, attributeDatas, "price_20_60")) + ); +} + +function readMarketRepresentativeVideo( + record: Record, + attributeDatas: Record +): string | undefined { + const items = readArrayLike(readMarketFieldValue(record, attributeDatas, "items")); + for (const item of items) { + if (!isRecord(item)) { + continue; + } + + const title = readString(item.title); + if (hasTextValue(title)) { + return normalizeExportCellText(title); + } + } + + return undefined; +} + +function readMarketGenderLabel(value: unknown): string | undefined { + const rawValue = typeof value === "number" ? String(value) : readString(value); + if (rawValue === "1") { + return "男"; + } + + if (rawValue === "2") { + return "女"; + } + + return undefined; +} + +function readBurstRateDisplay(value: number | null): string | undefined { + if (value === null) { + return undefined; + } + + if (value <= 0) { + return "-"; + } + + return formatFractionPercent(value); +} + +function formatCurrencyValue(value: number | null): string | undefined { + if (value === null) { + return undefined; + } + + return `¥${value.toLocaleString("en-US", { + maximumFractionDigits: 0 + })}`; +} + +function formatWanValue(value: number | null): string | undefined { + if (value === null) { + return undefined; + } + + return `${formatDecimalWithGrouping(value / 10000)}w`; +} + +function formatFractionPercent(value: number | null): string | undefined { + if (value === null) { + return undefined; + } + + return `${formatDecimalDisplay(value * 100)}%`; +} + +function formatDecimalDisplay(value: number | null): string | undefined { + if (value === null) { + return undefined; + } + + return value.toLocaleString("en-US", { + maximumFractionDigits: 1, + minimumFractionDigits: 0, + useGrouping: false + }); +} + +function formatDecimalWithGrouping(value: number): string { + return value.toLocaleString("en-US", { + maximumFractionDigits: 1, + minimumFractionDigits: 0 + }); +} + +function readNumericValue(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + + if (typeof value === "string") { + const trimmedValue = value.trim(); + if (!trimmedValue) { + return null; + } + + const parsedValue = Number(trimmedValue); + return Number.isFinite(parsedValue) ? parsedValue : null; + } + + return null; +} + +function readStringArray(value: unknown): string[] { + if (Array.isArray(value)) { + return value.filter((item): item is string => typeof item === "string"); + } + + if (typeof value === "string") { + try { + const parsedValue = JSON.parse(value); + return Array.isArray(parsedValue) + ? parsedValue.filter((item): item is string => typeof item === "string") + : []; + } catch { + return []; + } + } + + return []; +} + +function readArrayLike(value: unknown): unknown[] { + if (Array.isArray(value)) { + return value; + } + + if (typeof value === "string") { + try { + const parsedValue = JSON.parse(value); + return Array.isArray(parsedValue) ? parsedValue : []; + } catch { + return []; + } + } + + return []; +} + +function readRecordLike(value: unknown): Record | null { + if (isRecord(value)) { + return value; + } + + if (typeof value === "string") { + try { + const parsedValue = JSON.parse(value); + return isRecord(parsedValue) ? parsedValue : null; + } catch { + return null; + } + } + + return null; +} + +function readKnownNumber( + record: Record, + keys: readonly string[] +): number | undefined { + for (const key of keys) { + const value = readNumericValue(record[key]); + if (value !== null) { + return value; + } + } + + return undefined; +} + +function readKnownNumberDeep( + value: unknown, + keys: readonly string[] +): number | null { + if (!isRecord(value)) { + return null; + } + + const directValue = readKnownNumber(value, keys); + if (typeof directValue === "number") { + return directValue; + } + + for (const nestedValue of Object.values(value)) { + const candidate = + readKnownNumberDeep(unwrapVueRef(nestedValue), keys); + if (typeof candidate === "number") { + return candidate; + } + } + + return null; +} diff --git a/src/content/market/page-bridge.ts b/src/content/market/page-bridge.ts index b4801cd..b8495ce 100644 --- a/src/content/market/page-bridge.ts +++ b/src/content/market/page-bridge.ts @@ -1,6 +1,11 @@ import { normalizeFractionRateDisplay } from "../../shared/rate-normalizer"; +import { + writeMarketListRequestSnapshot +} from "./market-list-request-snapshot"; +import { parseMarketListResponse } from "./market-list-row"; const BRIDGE_MARKER = "__SCES_MARKET_PAGE_BRIDGE_INSTALLED__"; +const MARKET_SEARCH_REQUEST_PATH = "/gw/api/gsearch/search_for_author_square"; const SERIALIZED_MARKET_ROWS_ATTRIBUTE = "data-sces-market-rows"; type MarketRow = { @@ -24,6 +29,7 @@ function installMarketPageBridge() { } window[BRIDGE_MARKER] = true; + installMarketRequestSnapshotBridge(); syncSerializedMarketRows(); const observer = new MutationObserver(() => { @@ -39,7 +45,16 @@ function installMarketPageBridge() { }, 1000); } +function installMarketRequestSnapshotBridge() { + installFetchSnapshotBridge(); + installXmlHttpRequestSnapshotBridge(); +} + function syncSerializedMarketRows() { + if (typeof document === "undefined") { + return; + } + const nextSerializedRows = JSON.stringify(readSerializedMarketRows()); if ( document.documentElement.getAttribute(SERIALIZED_MARKET_ROWS_ATTRIBUTE) !== @@ -52,6 +67,131 @@ function syncSerializedMarketRows() { } } +function installFetchSnapshotBridge() { + if (typeof window.fetch !== "function") { + return; + } + + const originalFetch = window.fetch.bind(window); + window.fetch = async (input: RequestInfo | URL, init?: RequestInit) => { + const requestSnapshot = readFetchSnapshot(input, init); + const response = await originalFetch(input, init); + if (requestSnapshot) { + const clonedResponse = response.clone(); + void captureMarketSnapshotFromResponse(requestSnapshot, () => + clonedResponse.json() + ); + } + return response; + }; +} + +function installXmlHttpRequestSnapshotBridge() { + const OriginalXmlHttpRequest = window.XMLHttpRequest; + if (!OriginalXmlHttpRequest) { + return; + } + + const originalOpen = OriginalXmlHttpRequest.prototype.open; + const originalSend = OriginalXmlHttpRequest.prototype.send; + const originalSetRequestHeader = OriginalXmlHttpRequest.prototype.setRequestHeader; + + OriginalXmlHttpRequest.prototype.open = function ( + method: string, + url: string | URL, + ...rest: unknown[] + ) { + ( + this as XMLHttpRequest & { + __scesMarketSnapshot?: { + headers: Record; + method: string; + url: string; + }; + } + ).__scesMarketSnapshot = { + headers: {}, + method, + url: String(url) + }; + return originalOpen.call(this, method, url, ...(rest as [boolean?, string?, string?])); + }; + + OriginalXmlHttpRequest.prototype.setRequestHeader = function ( + name: string, + value: string + ) { + ( + this as XMLHttpRequest & { + __scesMarketSnapshot?: { + headers: Record; + }; + } + ).__scesMarketSnapshot?.headers && + ((this as XMLHttpRequest & { + __scesMarketSnapshot?: { + headers: Record; + }; + }).__scesMarketSnapshot!.headers[name] = value); + return originalSetRequestHeader.call(this, name, value); + }; + + OriginalXmlHttpRequest.prototype.send = function (body?: Document | XMLHttpRequestBodyInit | null) { + const snapshotState = ( + this as XMLHttpRequest & { + __scesMarketSnapshot?: { + body?: string; + headers: Record; + method: string; + url: string; + }; + } + ).__scesMarketSnapshot; + if (snapshotState) { + snapshotState.body = typeof body === "string" ? body : undefined; + this.addEventListener("load", () => { + if (this.status < 200 || this.status >= 300 || typeof this.responseText !== "string") { + return; + } + + void captureMarketSnapshotFromResponse(snapshotState, async () => + JSON.parse(this.responseText) + ); + }); + } + + return originalSend.call(this, body); + }; +} + +async function captureMarketSnapshotFromResponse( + snapshot: { + body?: string; + headers?: Record; + method: string; + url: string; + }, + readPayload: () => Promise +) { + if (!isMarketSearchRequest(snapshot.url)) { + return; + } + + try { + const payload = await readPayload(); + if (!parseMarketListResponse(payload)) { + return; + } + + writeMarketListRequestSnapshot(document, { + body: snapshot.body, + headers: snapshot.headers, + method: snapshot.method, + url: snapshot.url + }); + } catch {} +} + function readSerializedMarketRows() { const marketList = readMarketList(); return marketList @@ -71,7 +211,59 @@ function readSerializedMarketRows() { .filter((row) => Boolean(row.authorId || row.authorName)); } +function readFetchSnapshot( + input: RequestInfo | URL, + init?: RequestInit +): { + body?: string; + headers?: Record; + method: string; + url: string; +} | null { + const request = input instanceof Request ? input : null; + const method = init?.method ?? request?.method ?? "GET"; + const url = request?.url ?? String(input); + const body = + typeof init?.body === "string" + ? init.body + : typeof request?.bodyUsed === "boolean" && request.bodyUsed + ? undefined + : undefined; + const headers = serializeHeaders(init?.headers ?? request?.headers); + + return { + body, + headers, + method, + url + }; +} + +function serializeHeaders( + headers: HeadersInit | undefined +): Record | undefined { + if (!headers) { + return undefined; + } + + if (headers instanceof Headers) { + return Object.fromEntries(headers.entries()); + } + + if (Array.isArray(headers)) { + return Object.fromEntries(headers); + } + + return Object.fromEntries( + Object.entries(headers).map(([key, value]) => [key, String(value)]) + ); +} + function readMarketList(): MarketRow[] { + if (typeof document === "undefined") { + return []; + } + const marketRoot = document.querySelector(".base-author-list") as | (HTMLElement & { __vue__?: { @@ -102,6 +294,15 @@ function readMarketList(): MarketRow[] { return []; } +function isMarketSearchRequest(url: string): boolean { + return ( + url === MARKET_SEARCH_REQUEST_PATH || + url.startsWith(`${MARKET_SEARCH_REQUEST_PATH}?`) || + url.includes(`${MARKET_SEARCH_REQUEST_PATH}?`) || + url.endsWith(MARKET_SEARCH_REQUEST_PATH) + ); +} + function looksLikeMarketList(value: unknown[]): boolean { const firstRow = value[0]; return isRecord(firstRow) && ("star_id" in firstRow || "attribute_datas" in firstRow); diff --git a/src/content/market/silent-export-controller.ts b/src/content/market/silent-export-controller.ts new file mode 100644 index 0000000..76480ea --- /dev/null +++ b/src/content/market/silent-export-controller.ts @@ -0,0 +1,401 @@ +import { + readMarketListRequestSnapshot, + type MarketListRequestSnapshot +} from "./market-list-request-snapshot"; +import { + parseMarketListResponse, + readKnownPaginationNumber +} from "./market-list-row"; +import type { MarketExportTarget, MarketRecord } from "./types"; + +interface FetchResponseLike { + json(): Promise; + ok: boolean; +} + +type FetchLike = (input: string, init?: RequestInit) => Promise; + +interface SilentExportControllerOptions { + document: Document; + fetchImpl?: FetchLike; + onProgress?: (state: { currentPage: number; totalPages?: number }) => void; +} + +type PageSource = "body" | "none" | "url"; + +const PAGE_NUMBER_KEYS = [ + "currentPage", + "page", + "pageNo", + "pageNum", + "page_no", + "page_num" +] as const; + +export function createSilentExportController( + options: SilentExportControllerOptions +) { + const fetchImpl = options.fetchImpl ?? defaultFetch; + + return { + async exportRecords(target: MarketExportTarget): Promise { + const snapshot = readMarketListRequestSnapshot(options.document); + if (!snapshot) { + return null; + } + + const baseRequest = createPagedRequest(snapshot); + if (!baseRequest) { + return null; + } + + const mergedRecords = new Map(); + const maxPageCount = target.mode === "count" ? target.pageCount : 200; + let totalPagesHint: number | undefined; + + for (let offset = 0; offset < maxPageCount; offset += 1) { + const pageNumber = baseRequest.initialPage + offset; + options.onProgress?.({ + currentPage: offset + 1, + totalPages: target.mode === "count" ? target.pageCount : totalPagesHint + }); + + const payload = await fetchPagePayload(fetchImpl, baseRequest, pageNumber); + const parsedResponse = parseMarketListResponse(payload); + if (!parsedResponse) { + return null; + } + + totalPagesHint = parsedResponse.totalPages ?? totalPagesHint; + if (parsedResponse.records.length === 0) { + break; + } + + parsedResponse.records.forEach((record) => { + const existingRecord = mergedRecords.get(record.authorId); + mergedRecords.set(record.authorId, mergeMarketRecord(existingRecord, record)); + }); + + if (target.mode === "count" && offset + 1 >= target.pageCount) { + break; + } + + if (target.mode === "all") { + if ( + typeof parsedResponse.totalPages === "number" && + pageNumber >= parsedResponse.totalPages + ) { + break; + } + + if ( + typeof parsedResponse.pageSize === "number" && + parsedResponse.records.length < parsedResponse.pageSize + ) { + break; + } + } + } + + return Array.from(mergedRecords.values()); + } + }; +} + +function createPagedRequest( + snapshot: MarketListRequestSnapshot +): { + initialPage: number; + pageSource: PageSource; + snapshot: MarketListRequestSnapshot; +} { + const bodyPage = readPageFromBody(snapshot.body); + if (bodyPage !== null) { + return { + initialPage: bodyPage, + pageSource: "body", + snapshot + }; + } + + const urlPage = readPageFromUrl(snapshot.url); + if (urlPage !== null) { + return { + initialPage: urlPage, + pageSource: "url", + snapshot + }; + } + + return { + initialPage: 1, + pageSource: "none", + snapshot + }; +} + +async function fetchPagePayload( + fetchImpl: FetchLike, + request: { + pageSource: PageSource; + snapshot: MarketListRequestSnapshot; + }, + pageNumber: number +): Promise { + const nextUrl = + request.pageSource === "url" + ? mutateUrlPage(request.snapshot.url, pageNumber) + : request.snapshot.url; + const nextBody = mutateBodyPage(request.snapshot.body, pageNumber); + + const response = await fetchImpl(nextUrl, { + body: nextBody, + credentials: "include", + headers: filterReplayHeaders(request.snapshot.headers, nextBody), + method: request.snapshot.method + }); + if (!response.ok) { + throw new Error("静默导出请求失败"); + } + + return response.json(); +} + +function readPageFromUrl(url: string): number | null { + try { + const parsedUrl = new URL(url); + for (const key of PAGE_NUMBER_KEYS) { + const value = readNumericString(parsedUrl.searchParams.get(key)); + if (value !== null) { + return value; + } + } + } catch { + return null; + } + + return null; +} + +function mutateUrlPage(url: string, pageNumber: number): string { + try { + const parsedUrl = new URL(url); + for (const key of PAGE_NUMBER_KEYS) { + if (!parsedUrl.searchParams.has(key)) { + continue; + } + + parsedUrl.searchParams.set(key, String(pageNumber)); + return parsedUrl.toString(); + } + + parsedUrl.searchParams.set("page", String(pageNumber)); + return parsedUrl.toString(); + } catch { + return url; + } +} + +function readPageFromBody(body: string | undefined): number | null { + const parsedBody = parseBody(body); + if (!parsedBody) { + return null; + } + + return readKnownPaginationNumber(parsedBody, "page"); +} + +function mutateBodyPage(body: string | undefined, pageNumber: number): string | undefined { + if (!body) { + return body; + } + + const trimmedBody = body.trim(); + if (!trimmedBody) { + return body; + } + + try { + const parsedJson = JSON.parse(trimmedBody); + if (!replacePageNumberInValue(parsedJson, pageNumber) && isRecord(parsedJson)) { + parsedJson.page = pageNumber; + } + + return JSON.stringify(parsedJson); + } catch { + const searchParams = new URLSearchParams(trimmedBody); + for (const key of PAGE_NUMBER_KEYS) { + if (!searchParams.has(key)) { + continue; + } + + searchParams.set(key, String(pageNumber)); + return searchParams.toString(); + } + + searchParams.set("page", String(pageNumber)); + return searchParams.toString(); + } +} + +function parseBody(body: string | undefined): Record | null { + if (!body) { + return null; + } + + const trimmedBody = body.trim(); + if (!trimmedBody) { + return null; + } + + try { + const parsedBody = JSON.parse(trimmedBody); + return isRecord(parsedBody) ? parsedBody : null; + } catch { + const searchParams = new URLSearchParams(trimmedBody); + const payload: Record = {}; + searchParams.forEach((value, key) => { + payload[key] = value; + }); + return payload; + } +} + +function replacePageNumberInValue(value: unknown, pageNumber: number): boolean { + if (!isRecord(value)) { + return false; + } + + let replaced = false; + PAGE_NUMBER_KEYS.forEach((key) => { + if (!(key in value)) { + return; + } + + value[key] = pageNumber; + replaced = true; + }); + + if (replaced) { + return true; + } + + return Object.values(value).some((entry) => replacePageNumberInValue(entry, pageNumber)); +} + +function filterReplayHeaders( + headers: Record | undefined, + body: string | undefined +): HeadersInit | undefined { + const filteredHeaders = Object.fromEntries( + Object.entries(headers ?? {}).filter(([key]) => { + const normalizedKey = key.toLowerCase(); + return normalizedKey !== "content-length" && normalizedKey !== "host"; + }) + ); + + if (body) { + if (!hasHeader(filteredHeaders, "accept")) { + filteredHeaders.Accept = "application/json, text/plain, */*"; + } + if (!hasHeader(filteredHeaders, "content-type")) { + filteredHeaders["Content-Type"] = "application/json"; + } + if (!hasHeader(filteredHeaders, "x-login-source")) { + filteredHeaders["x-login-source"] = "1"; + } + if (!hasHeader(filteredHeaders, "agw-js-conv")) { + filteredHeaders["Agw-Js-Conv"] = "str"; + } + } + + return Object.keys(filteredHeaders).length > 0 ? filteredHeaders : undefined; +} + +function hasHeader(headers: Record, key: string): boolean { + return Object.keys(headers).some((headerKey) => headerKey.toLowerCase() === key); +} + +function readNumericString(value: string | null): number | null { + if (!value) { + return null; + } + + const parsedValue = Number(value); + return Number.isFinite(parsedValue) ? parsedValue : null; +} + +async function defaultFetch(input: string, init?: RequestInit) { + return fetch(input, init); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function mergeMarketRecord( + existingRecord: MarketRecord | undefined, + incomingRecord: MarketRecord +): MarketRecord { + if (!existingRecord) { + return { + ...incomingRecord, + exportFields: mergeFieldMap(undefined, incomingRecord.exportFields), + rates: mergeFieldMap(undefined, incomingRecord.rates), + status: incomingRecord.status ?? "idle" + }; + } + + return { + ...existingRecord, + ...incomingRecord, + authorName: mergeStringValue(existingRecord.authorName, incomingRecord.authorName) ?? "", + exportFields: mergeFieldMap( + existingRecord.exportFields, + incomingRecord.exportFields + ), + failureReason: incomingRecord.failureReason ?? existingRecord.failureReason, + hasDirectRatesSource: + existingRecord.hasDirectRatesSource || incomingRecord.hasDirectRatesSource, + location: mergeStringValue(existingRecord.location, incomingRecord.location), + price21To60s: mergeStringValue( + existingRecord.price21To60s, + incomingRecord.price21To60s + ), + rates: mergeFieldMap(existingRecord.rates, incomingRecord.rates), + status: incomingRecord.status ?? existingRecord.status + }; +} + +function mergeFieldMap>( + current: T | undefined, + incoming: T | undefined +): T | undefined { + if (!current && !incoming) { + return undefined; + } + + const merged = { + ...(current ?? {}) + } as Record; + + Object.entries(incoming ?? {}).forEach(([key, value]) => { + const currentValue = merged[key]; + if (hasTextValue(value) || !hasTextValue(currentValue)) { + merged[key] = value; + } + }); + + return merged as T; +} + +function mergeStringValue( + current: string | undefined, + incoming: string | undefined +): string | undefined { + return hasTextValue(incoming) ? incoming : current; +} + +function hasTextValue(value: string | undefined): boolean { + return Boolean(value && value.trim().length > 0); +} diff --git a/src/manifest.json b/src/manifest.json index fce0f3c..bb53cd5 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -23,7 +23,7 @@ "https://*.xingtu.cn/ad/creator/market*" ], "js": ["content/index.js"], - "run_at": "document_idle" + "run_at": "document_start" } ], "web_accessible_resources": [ diff --git a/tests/manifest.test.ts b/tests/manifest.test.ts index 9432162..97c0bac 100644 --- a/tests/manifest.test.ts +++ b/tests/manifest.test.ts @@ -9,6 +9,7 @@ describe("manifest", () => { expect.stringMatching(/^https:\/\/(\*\.|www\.)?xingtu\.cn\/ad\/creator\/market\*$/) ]) ); + expect(manifest.content_scripts?.[0]?.run_at).toBe("document_start"); }); test("declares the downloads and auth permissions plus background worker", () => { diff --git a/tests/market-content-entry.test.ts b/tests/market-content-entry.test.ts index 98592f6..8aef431 100644 --- a/tests/market-content-entry.test.ts +++ b/tests/market-content-entry.test.ts @@ -11,6 +11,7 @@ describe("market-content-entry", () => { beforeEach(() => { document.body.innerHTML = ""; document.documentElement.removeAttribute("data-sces-market-rows"); + document.documentElement.removeAttribute("data-sces-market-request-snapshot"); document.documentElement.removeAttribute("data-test-page-index"); window.history.replaceState({}, "", "/"); }); @@ -33,7 +34,13 @@ describe("market-content-entry", () => { __SCES_MARKET_PAGE_BRIDGE_INSTALLED__?: boolean; } ).__SCES_MARKET_PAGE_BRIDGE_INSTALLED__; + delete ( + globalThis as typeof globalThis & { + fetch?: unknown; + } + ).fetch; document.documentElement.removeAttribute("data-sces-market-rows"); + document.documentElement.removeAttribute("data-sces-market-request-snapshot"); document.documentElement.removeAttribute("data-test-page-index"); vi.resetModules(); @@ -95,6 +102,40 @@ describe("market-content-entry", () => { ).not.toBeNull(); }); + test("installs the market bridge before auth state resolves", async () => { + const createMarketController = vi.fn(() => ({ + ready: Promise.resolve() + })); + let resolveAuthState: ((value: unknown) => void) | null = null; + + window.history.replaceState({}, "", "/ad/creator/market"); + + const { bootContentScript } = await import("../src/content/index"); + const bootPromise = bootContentScript({ + createMarketController, + sendAuthMessage: vi.fn( + () => + new Promise((resolve) => { + resolveAuthState = resolve; + }) + ) + }); + + expect( + document.documentElement.querySelector('[data-sces-market-bridge="script"]') + ).not.toBeNull(); + expect(createMarketController).not.toHaveBeenCalled(); + + resolveAuthState?.({ + ok: true, + type: "auth:state", + value: { isAuthenticated: true } + }); + await bootPromise; + + expect(createMarketController).toHaveBeenCalledTimes(1); + }); + test("boots the market controller on the www Xingtu market URL", async () => { const createMarketController = vi.fn(() => ({ ready: Promise.resolve() @@ -657,6 +698,104 @@ describe("market-content-entry", () => { ]); }); + test("keeps all plugin columns in loading state until backend metrics are ready", async () => { + document.body.innerHTML = buildRealMarketFixture([ + { + authorId: "111", + authorName: "达人 A", + price21To60s: "¥450,000" + }, + { + authorId: "222", + authorName: "达人 B", + price21To60s: "¥20,000" + } + ]); + const backendDeferred = createDeferred< + Array<{ + afterViewSearchRate: string; + starId: string; + }> + >(); + + const { createMarketController } = await import("../src/content/market/index"); + const controller = trackController(createMarketController({ + document, + loadAuthorMetrics: async (authorId) => ({ + success: true, + rates: + authorId === "111" + ? { + singleVideoAfterSearchRate: "0.02%", + personalVideoAfterSearchRate: "0.03% - 0.2%" + } + : { + singleVideoAfterSearchRate: "0.5%-1%", + personalVideoAfterSearchRate: "0.01% - 0.1%" + } + }), + searchBackendMetrics: () => backendDeferred.promise, + window + })); + + await flushWithTimers(); + + expect(readDivPluginRowTexts(0)).toEqual([ + "加载中...", + "加载中...", + "加载中...", + "加载中...", + "加载中...", + "加载中...", + "加载中...", + "加载中..." + ]); + expect(readDivPluginRowTexts(1)).toEqual([ + "加载中...", + "加载中...", + "加载中...", + "加载中...", + "加载中...", + "加载中...", + "加载中...", + "加载中..." + ]); + + backendDeferred.resolve([ + { + afterViewSearchRate: "0.36%", + starId: "111" + }, + { + afterViewSearchRate: "1.4%", + starId: "222" + } + ]); + + await controller.ready; + + expect(readDivPluginRowTexts(0)).toEqual([ + "0.02%", + "0.03% - 0.2%", + "0.36%", + "", + "", + "", + "", + "" + ]); + expect(readDivPluginRowTexts(1)).toEqual([ + "0.5% - 1%", + "0.01% - 0.1%", + "1.4%", + "", + "", + "", + "", + "" + ]); + }); + test("hydrates real rows from serialized market rows when vue state is unavailable", async () => { document.body.innerHTML = buildRealMarketFixtureWithoutAuthorIds([ { @@ -1050,6 +1189,125 @@ describe("market-content-entry", () => { 15000 ); + test( + "default export replays captured market requests silently without paging the visible table", + async () => { + const pages = [ + [{ authorId: "111", authorName: "达人 A", price21To60s: "¥11,000" }], + [{ authorId: "222", authorName: "达人 B", price21To60s: "¥22,000" }], + [{ authorId: "333", authorName: "达人 C", price21To60s: "¥33,000" }], + [{ authorId: "444", authorName: "达人 D", price21To60s: "¥44,000" }], + [{ authorId: "555", authorName: "达人 E", price21To60s: "¥55,000" }] + ]; + + document.body.innerHTML = buildRealMarketFixture(pages[0]); + const pagination = installAsyncPaginationHarness(pages); + const buildCsv = vi.fn(() => "csv-output"); + const onCsvReady = vi.fn(); + const fetchMock = vi.fn(async (_input: string, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { page?: number }; + const pageIndex = Math.max((body.page ?? 1) - 1, 0); + return { + json: async () => ({ + data: { + marketList: buildMarketListResponseRows(pages[pageIndex] ?? []) + } + }), + ok: true + }; + }); + + ( + globalThis as typeof globalThis & { + fetch?: typeof fetchMock; + } + ).fetch = fetchMock; + document.documentElement.setAttribute( + "data-sces-market-request-snapshot", + JSON.stringify({ + body: JSON.stringify({ + page: 1 + }), + method: "POST", + url: "https://xingtu.cn/api/mock-market-search" + }) + ); + + const { createMarketController } = await import("../src/content/market/index"); + const controller = trackController(createMarketController({ + buildCsv, + document, + loadAuthorMetrics: async () => ({ + success: false, + reason: "request-failed" + }), + onCsvReady, + window + })); + + await controller.ready; + click('[data-plugin-export="button"]'); + await waitForMockCall(buildCsv, 120, 100); + + expect(pagination.getClicks()).toBe(0); + expect(fetchMock).toHaveBeenCalledTimes(5); + expect(buildCsv.mock.calls[0][0].map((record) => record.authorId)).toEqual([ + "111", + "222", + "333", + "444", + "555" + ]); + expect(onCsvReady).toHaveBeenCalledWith("csv-output"); + }, + 15000 + ); + + test( + "default export falls back to visible pagination when no captured market request is available", + async () => { + const pages = [ + [{ authorId: "111", authorName: "达人 A", price21To60s: "¥11,000" }], + [{ authorId: "222", authorName: "达人 B", price21To60s: "¥22,000" }], + [{ authorId: "333", authorName: "达人 C", price21To60s: "¥33,000" }], + [{ authorId: "444", authorName: "达人 D", price21To60s: "¥44,000" }], + [{ authorId: "555", authorName: "达人 E", price21To60s: "¥55,000" }] + ]; + + document.body.innerHTML = buildRealMarketFixture(pages[0]); + const pagination = installAsyncPaginationHarness(pages); + const buildCsv = vi.fn(() => "csv-output"); + + document.documentElement.removeAttribute("data-sces-market-request-snapshot"); + + const { createMarketController } = await import("../src/content/market/index"); + const controller = trackController(createMarketController({ + buildCsv, + document, + loadAuthorMetrics: async () => ({ + success: false, + reason: "request-failed" + }), + onCsvReady: vi.fn(), + window + })); + + await controller.ready; + click('[data-plugin-export="button"]'); + await waitForMockCall(buildCsv, 120, 100); + + expect(pagination.getClicks()).toBe(4); + expect(buildCsv.mock.calls[0][0].map((record) => record.authorId)).toEqual([ + "111", + "222", + "333", + "444", + "555" + ]); + }, + 15000 + ); + test( "default export waits for the next page rows instead of only the pager state", async () => { @@ -2663,6 +2921,29 @@ function attachMarketListState( }); } +function buildMarketListResponseRows( + rows: Array<{ + authorId: string; + authorName: string; + price21To60s: string; + }> +): Array> { + return rows.map((row) => ({ + attribute_datas: { + items: JSON.stringify([ + { + title: `代表视频${row.authorName}` + } + ]), + nick_name: row.authorName, + nickname: row.authorName, + price_20_60: Number(row.price21To60s.replace(/[^\d]/g, "")) + }, + nick_name: row.authorName, + star_id: row.authorId + })); +} + function installPaginationHarness( pages: Array< Array<{ @@ -3638,3 +3919,18 @@ async function waitForMockCall( await flushWithTimers(); } } + +function createDeferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((nextResolve, nextReject) => { + resolve = nextResolve; + reject = nextReject; + }); + + return { + promise, + reject, + resolve + }; +} diff --git a/tests/market-page-bridge.test.ts b/tests/market-page-bridge.test.ts new file mode 100644 index 0000000..49d5237 --- /dev/null +++ b/tests/market-page-bridge.test.ts @@ -0,0 +1,129 @@ +// @vitest-environment jsdom +// @vitest-environment-options {"url":"https://www.xingtu.cn/ad/creator/market"} + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +describe("market-page-bridge", () => { + beforeEach(() => { + document.body.innerHTML = '
'; + document.documentElement.removeAttribute("data-sces-market-request-snapshot"); + vi.spyOn(window, "setInterval").mockReturnValue(0 as unknown as number); + }); + + afterEach(() => { + delete ( + window as Window & { + __SCES_MARKET_PAGE_BRIDGE_INSTALLED__?: boolean; + } + ).__SCES_MARKET_PAGE_BRIDGE_INSTALLED__; + vi.restoreAllMocks(); + vi.resetModules(); + }); + + test("ignores non-search market list responses when capturing the export snapshot", async () => { + const originalFetch = vi.fn(async () => ({ + clone() { + return this; + }, + json: async () => ({ + authors: [ + { + attribute_datas: { + nickname: "推荐达人" + }, + star_id: "recommend-1" + } + ], + pagination: { + limit: 20, + page: 1, + total_count: 20 + } + }) + })); + + ( + window as Window & { + fetch: typeof fetch; + } + ).fetch = originalFetch as unknown as typeof fetch; + + await import("../src/content/market/page-bridge"); + + await window.fetch("/gw/api/gauthor/demander_get_recommend_author_lists_v2", { + headers: { + Accept: "application/json, text/plain, */*" + }, + method: "GET" + }); + await Promise.resolve(); + await Promise.resolve(); + + expect( + document.documentElement.getAttribute("data-sces-market-request-snapshot") + ).toBeNull(); + }); + + test("captures the search_for_author_square request for silent export", async () => { + const originalFetch = vi.fn(async () => ({ + clone() { + return this; + }, + json: async () => ({ + authors: [ + { + attribute_datas: { + nickname: "搜索达人" + }, + star_id: "search-1" + } + ], + pagination: { + limit: 20, + page: 1, + total_count: 20 + } + }) + })); + + ( + window as Window & { + fetch: typeof fetch; + } + ).fetch = originalFetch as unknown as typeof fetch; + + await import("../src/content/market/page-bridge"); + + await window.fetch("/gw/api/gsearch/search_for_author_square", { + body: JSON.stringify({ + page_param: { + page: "1" + } + }), + headers: { + "Content-Type": "application/json" + }, + method: "POST" + }); + await Promise.resolve(); + await Promise.resolve(); + + expect( + JSON.parse( + document.documentElement.getAttribute( + "data-sces-market-request-snapshot" + ) ?? "null" + ) + ).toEqual( + expect.objectContaining({ + body: JSON.stringify({ + page_param: { + page: "1" + } + }), + method: "POST", + url: "/gw/api/gsearch/search_for_author_square" + }) + ); + }); +}); diff --git a/tests/silent-export-controller.test.ts b/tests/silent-export-controller.test.ts new file mode 100644 index 0000000..c6e9655 --- /dev/null +++ b/tests/silent-export-controller.test.ts @@ -0,0 +1,263 @@ +// @vitest-environment jsdom + +import { describe, expect, test } from "vitest"; + +import { createSilentExportController } from "../src/content/market/silent-export-controller"; + +describe("silent-export-controller", () => { + test("replays exports from the live market page state when the request snapshot attribute is missing", async () => { + document.body.innerHTML = '
'; + const marketRoot = document.querySelector(".base-author-list") as HTMLElement & { + __vue__?: { + _setupState?: Record; + }; + }; + marketRoot.__vue__ = { + _setupState: { + __$temp_1: { + reqParams: { + scene_param: { + platform_source: 1 + }, + page_param: { + limit: "20", + page: "2" + }, + search_param: { + seach_type: 3 + } + } + } + } + }; + + const requestedPages: number[] = []; + const requestedHeaders: Array = []; + const requestedUrls: string[] = []; + const controller = createSilentExportController({ + document, + fetchImpl: async (url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { + page_param?: { page?: number | string }; + }; + const pageNo = Number(body.page_param?.page ?? 0); + requestedPages.push(pageNo); + requestedHeaders.push(init?.headers); + requestedUrls.push(url); + + return { + json: async () => ({ + authors: [ + { + attribute_datas: { + nickname: `达人${pageNo}`, + price_20_60: pageNo * 1000 + }, + star_id: String(pageNo) + } + ], + pagination: { + limit: 20, + page: pageNo, + total_count: 100 + } + }), + ok: true + }; + } + }); + + const records = await controller.exportRecords({ + mode: "count", + pageCount: 2 + }); + + expect(requestedPages).toEqual([2, 3]); + expect( + requestedUrls.every((url) => + url.endsWith("/gw/api/gsearch/search_for_author_square") + ) + ).toBe(true); + expect(requestedHeaders).toEqual([ + { + Accept: "application/json, text/plain, */*", + "Agw-Js-Conv": "str", + "Content-Type": "application/json", + "x-login-source": "1" + }, + { + Accept: "application/json, text/plain, */*", + "Agw-Js-Conv": "str", + "Content-Type": "application/json", + "x-login-source": "1" + } + ]); + expect(records?.map((record) => record.authorId)).toEqual(["2", "3"]); + }); + + test("replays paged exports when the page number is nested inside the request body", async () => { + document.documentElement.setAttribute( + "data-sces-market-request-snapshot", + JSON.stringify({ + body: JSON.stringify({ + page_param: { + page: 2 + } + }), + method: "POST", + url: "https://xingtu.cn/api/mock-market-search" + }) + ); + + const requestedPages: number[] = []; + const controller = createSilentExportController({ + document, + fetchImpl: async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { + page_param?: { page?: number }; + }; + const pageNo = body.page_param?.page ?? 0; + requestedPages.push(pageNo); + + return { + json: async () => ({ + authors: [ + { + attribute_datas: { + nickname: `达人${pageNo}`, + price_20_60: pageNo * 1000 + }, + star_id: String(pageNo) + } + ], + pagination: { + limit: 20, + page: pageNo, + total_count: 100 + } + }), + ok: true + }; + } + }); + + const records = await controller.exportRecords({ + mode: "count", + pageCount: 2 + }); + + expect(requestedPages).toEqual([2, 3]); + expect(records?.map((record) => record.authorId)).toEqual(["2", "3"]); + }); + + test("starts from page 1 when the captured request omits an explicit page number", async () => { + document.documentElement.setAttribute( + "data-sces-market-request-snapshot", + JSON.stringify({ + body: JSON.stringify({ + filters: { + keyword: "test" + } + }), + method: "POST", + url: "https://xingtu.cn/api/mock-market-search" + }) + ); + + const requestedPages: number[] = []; + const controller = createSilentExportController({ + document, + fetchImpl: async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { page?: number }; + const page = body.page ?? 0; + requestedPages.push(page); + + return { + json: async () => ({ + data: { + marketList: [ + { + attribute_datas: { + nickname: `达人${page}`, + price_20_60: page * 1000 + }, + star_id: String(page) + } + ] + } + }), + ok: true + }; + } + }); + + const records = await controller.exportRecords({ + mode: "count", + pageCount: 2 + }); + + expect(requestedPages).toEqual([1, 2]); + expect(records?.map((record) => record.authorId)).toEqual(["1", "2"]); + }); + + test("accepts snapshot headers that contain non-string values from the live XHR capture", async () => { + document.body.innerHTML = ""; + document.documentElement.setAttribute( + "data-sces-market-request-snapshot", + JSON.stringify({ + body: JSON.stringify({ + page_param: { + page: "1", + limit: "20" + } + }), + headers: { + Accept: "application/json, text/plain, */*", + "Content-Type": "application/json", + "x-login-source": 1, + "Agw-Js-Conv": "str" + }, + method: "POST", + url: "/gw/api/gsearch/search_for_author_square" + }) + ); + + const requestedPages: number[] = []; + const controller = createSilentExportController({ + document, + fetchImpl: async (_url, init) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { + page_param?: { page?: number | string }; + }; + requestedPages.push(Number(body.page_param?.page ?? 0)); + + return { + json: async () => ({ + authors: [ + { + attribute_datas: { + nickname: "达人1" + }, + star_id: "1" + } + ], + pagination: { + limit: 20, + page: 1, + total_count: 20 + } + }), + ok: true + }; + } + }); + + const records = await controller.exportRecords({ + mode: "count", + pageCount: 1 + }); + + expect(requestedPages).toEqual([1]); + expect(records?.map((record) => record.authorId)).toEqual(["1"]); + }); +});