Fix silent market CSV export
This commit is contained in:
+22
-1
@@ -39,15 +39,18 @@ export async function bootContentScript(
|
||||
return null;
|
||||
}
|
||||
|
||||
installMarketPageBridge(currentDocument);
|
||||
|
||||
const authState = await readAuthState(sendAuthMessage);
|
||||
if (!authState?.isAuthenticated) {
|
||||
await waitForBodyReady(currentDocument, currentWindow);
|
||||
renderMarketAuthGate(currentDocument, currentWindow);
|
||||
return {
|
||||
ready: Promise.resolve()
|
||||
};
|
||||
}
|
||||
|
||||
installMarketPageBridge(currentDocument);
|
||||
await waitForBodyReady(currentDocument, currentWindow);
|
||||
|
||||
return controllerFactory({
|
||||
document: currentDocument,
|
||||
@@ -144,6 +147,24 @@ function createRuntimeMessageSender(): (message: unknown) => Promise<unknown> {
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForBodyReady(document: Document, currentWindow: Window): Promise<void> {
|
||||
if (document.body) {
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
const handleReady = () => {
|
||||
if (document.body) {
|
||||
document.removeEventListener("DOMContentLoaded", handleReady);
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
document.addEventListener("DOMContentLoaded", handleReady);
|
||||
currentWindow.setTimeout(handleReady, 0);
|
||||
});
|
||||
}
|
||||
|
||||
function downloadCsv(document: Document, window: Window, csv: string): void {
|
||||
const blob = new Blob(["\uFEFF", csv], {
|
||||
type: "text/csv;charset=utf-8"
|
||||
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
normalizeFractionRateDisplay,
|
||||
normalizeRateDisplay
|
||||
} from "../../shared/rate-normalizer";
|
||||
import { mapMarketListRow } from "./market-list-row";
|
||||
import type {
|
||||
AfterSearchRates,
|
||||
BackendMetrics,
|
||||
@@ -1253,33 +1254,9 @@ function readVueMarketRows(
|
||||
continue;
|
||||
}
|
||||
|
||||
return marketList.map((row) => {
|
||||
const record = isRecord(row) ? row : {};
|
||||
const attributeDatas = readMarketAttributeDatas(record);
|
||||
const singleVideoAfterSearchRate = normalizeMarketListRate(
|
||||
readMarketFieldValue(record, attributeDatas, "avg_search_after_view_rate_30d")
|
||||
);
|
||||
|
||||
return {
|
||||
authorId:
|
||||
readString(readMarketFieldValue(record, attributeDatas, "star_id")) ??
|
||||
readString(readMarketFieldValue(record, attributeDatas, "id")) ??
|
||||
"",
|
||||
authorName:
|
||||
readString(readMarketFieldValue(record, attributeDatas, "nickname")) ??
|
||||
readString(readMarketFieldValue(record, attributeDatas, "nick_name")) ??
|
||||
"",
|
||||
exportFields: buildMarketExportFieldFallbacks(record, attributeDatas),
|
||||
hasDirectRatesSource: true,
|
||||
location: readMarketLocation(record, attributeDatas),
|
||||
price21To60s: readMarketPrice21To60s(record, attributeDatas),
|
||||
rates: singleVideoAfterSearchRate
|
||||
? {
|
||||
singleVideoAfterSearchRate
|
||||
}
|
||||
: undefined
|
||||
};
|
||||
});
|
||||
return marketList
|
||||
.map((row) => (isRecord(row) ? mapMarketListRow(row) : null))
|
||||
.filter((row): row is MarketDataRow => row !== null);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+251
-142
@@ -14,6 +14,7 @@ import { applyFilterAndSort } from "./filter-sort-controller";
|
||||
import { createMarketApiClient } from "./api-client";
|
||||
import { createExportRangeController } from "./export-range-controller";
|
||||
import { ensurePluginToolbar, isPluginToolbarMounted } from "./plugin-toolbar";
|
||||
import { createSilentExportController } from "./silent-export-controller";
|
||||
import {
|
||||
readToolbarExportTarget,
|
||||
setToolbarBusyState,
|
||||
@@ -93,6 +94,17 @@ export function createMarketController(options: CreateMarketControllerOptions) {
|
||||
readCurrentPageRowCount: () => countCurrentPageRows(options.document),
|
||||
window: options.window
|
||||
});
|
||||
const silentExportController = createSilentExportController({
|
||||
document: options.document,
|
||||
onProgress: ({ currentPage, totalPages }) => {
|
||||
setToolbarExportStatus(
|
||||
toolbar,
|
||||
totalPages
|
||||
? `导出中 ${currentPage}/${totalPages} 页...`
|
||||
: `导出中 第${currentPage}页...`
|
||||
);
|
||||
}
|
||||
});
|
||||
let activeSort: MarketSortState | undefined;
|
||||
let isDisposed = false;
|
||||
let isSyncRunning = false;
|
||||
@@ -239,69 +251,96 @@ export function createMarketController(options: CreateMarketControllerOptions) {
|
||||
rowSnapshot
|
||||
});
|
||||
resultStore.upsertMarketRow(rowSnapshot);
|
||||
const existingRecord = resultStore.getRecord(rowSnapshot.authorId);
|
||||
if (existingRecord?.status === "success" && existingRecord.rates) {
|
||||
renderMarketRowState(rowDom, existingRecord);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existingRecord?.status === "failed") {
|
||||
renderMarketRowState(rowDom, existingRecord);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existingRecord?.status === "loading") {
|
||||
renderMarketRowState(rowDom, {
|
||||
...rowSnapshot,
|
||||
status: "loading"
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (rowSnapshot.hasDirectRatesSource) {
|
||||
const directRates = rowSnapshot.rates ?? {};
|
||||
const hasAllRates =
|
||||
Boolean(directRates.singleVideoAfterSearchRate) &&
|
||||
Boolean(directRates.personalVideoAfterSearchRate);
|
||||
|
||||
resultStore.setAuthorSuccess(rowSnapshot.authorId, directRates);
|
||||
renderMarketRowState(rowDom, {
|
||||
...rowSnapshot,
|
||||
rates: directRates,
|
||||
status: "success"
|
||||
});
|
||||
if (hasAllRates) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
resultStore.setAuthorLoading(rowSnapshot.authorId);
|
||||
renderMarketRowState(rowDom, {
|
||||
...rowSnapshot,
|
||||
rates: resultStore.getRecord(rowSnapshot.authorId)?.rates,
|
||||
status: "loading"
|
||||
});
|
||||
|
||||
const metricsResult = await loadAuthorMetrics(rowSnapshot.authorId);
|
||||
if (metricsResult.success) {
|
||||
resultStore.setAuthorSuccess(rowSnapshot.authorId, metricsResult.rates);
|
||||
renderMarketRowState(rowDom, {
|
||||
...rowSnapshot,
|
||||
status: "success",
|
||||
rates: metricsResult.rates
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
resultStore.setAuthorFailed(rowSnapshot.authorId, metricsResult.reason);
|
||||
renderMarketRowState(rowDom, {
|
||||
...rowSnapshot,
|
||||
failureReason: metricsResult.reason,
|
||||
status: "failed"
|
||||
});
|
||||
}
|
||||
|
||||
await hydrateBackendMetricsForPage(pageRows);
|
||||
const pendingRateRows: typeof pageRows = [];
|
||||
const rowsNeedingBackendMetrics: typeof pageRows = [];
|
||||
|
||||
pageRows.forEach(({ rowDom, rowSnapshot }) => {
|
||||
if (rowSnapshot.hasDirectRatesSource) {
|
||||
resultStore.setAuthorSuccess(rowSnapshot.authorId, rowSnapshot.rates ?? {});
|
||||
}
|
||||
|
||||
const existingRecord = resultStore.getRecord(rowSnapshot.authorId);
|
||||
const needsRateFetch =
|
||||
!hasSettledRateState(existingRecord) &&
|
||||
!hasCompleteRates(existingRecord?.rates);
|
||||
const needsBackendMetrics =
|
||||
Boolean(searchBackendMetrics) &&
|
||||
!hasSettledBackendMetricsState(existingRecord);
|
||||
|
||||
if (needsRateFetch) {
|
||||
resultStore.setAuthorLoading(rowSnapshot.authorId);
|
||||
pendingRateRows.push({
|
||||
rowDom,
|
||||
rowSnapshot
|
||||
});
|
||||
}
|
||||
|
||||
if (needsBackendMetrics) {
|
||||
resultStore.setBackendMetricsLoading(rowSnapshot.authorId);
|
||||
rowsNeedingBackendMetrics.push({
|
||||
rowDom,
|
||||
rowSnapshot
|
||||
});
|
||||
}
|
||||
|
||||
if (needsRateFetch || needsBackendMetrics) {
|
||||
renderMarketRowState(rowDom, {
|
||||
...(existingRecord ?? {
|
||||
authorId: rowSnapshot.authorId,
|
||||
authorName: rowSnapshot.authorName,
|
||||
status: "idle" as const
|
||||
}),
|
||||
...rowSnapshot,
|
||||
backendMetricsStatus: needsBackendMetrics ? "loading" : existingRecord?.backendMetricsStatus,
|
||||
rates: existingRecord?.rates,
|
||||
status: needsRateFetch || needsBackendMetrics ? "loading" : existingRecord?.status ?? "idle"
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (existingRecord) {
|
||||
renderMarketRowState(rowDom, existingRecord);
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all([
|
||||
hydrateRatesForRows(pendingRateRows),
|
||||
hydrateBackendMetricsForPage(rowsNeedingBackendMetrics)
|
||||
]);
|
||||
|
||||
pageRows.forEach(({ rowDom, rowSnapshot }) => {
|
||||
const record = resultStore.getRecord(rowSnapshot.authorId);
|
||||
if (!record) {
|
||||
return;
|
||||
}
|
||||
|
||||
renderMarketRowState(rowDom, record);
|
||||
});
|
||||
}
|
||||
|
||||
async function hydrateRatesForRows(
|
||||
pageRows: Array<{
|
||||
rowDom: MarketRowDom;
|
||||
rowSnapshot: MarketRowSnapshot;
|
||||
}>
|
||||
): Promise<void> {
|
||||
if (pageRows.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
pageRows.map(async ({ rowSnapshot }) => {
|
||||
const metricsResult = await loadAuthorMetrics(rowSnapshot.authorId);
|
||||
if (metricsResult.success) {
|
||||
resultStore.setAuthorSuccess(rowSnapshot.authorId, metricsResult.rates);
|
||||
return;
|
||||
}
|
||||
|
||||
resultStore.setAuthorFailed(rowSnapshot.authorId, metricsResult.reason);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async function hydrateBackendMetricsForPage(
|
||||
@@ -314,70 +353,23 @@ export function createMarketController(options: CreateMarketControllerOptions) {
|
||||
return;
|
||||
}
|
||||
|
||||
const pendingBackendRows = pageRows.filter(({ rowDom, rowSnapshot }) => {
|
||||
const record = resultStore.getRecord(rowSnapshot.authorId);
|
||||
if (!record) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (
|
||||
record.backendMetricsStatus === "success" ||
|
||||
record.backendMetricsStatus === "missing" ||
|
||||
record.backendMetricsStatus === "failed" ||
|
||||
record.backendMetricsStatus === "loading"
|
||||
) {
|
||||
renderMarketRowState(rowDom, record);
|
||||
return false;
|
||||
}
|
||||
|
||||
resultStore.setBackendMetricsLoading(rowSnapshot.authorId);
|
||||
renderMarketRowState(rowDom, {
|
||||
...record,
|
||||
...rowSnapshot,
|
||||
backendMetricsStatus: "loading"
|
||||
});
|
||||
return true;
|
||||
});
|
||||
|
||||
if (pendingBackendRows.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const rows = await searchBackendMetrics(
|
||||
pendingBackendRows.map(({ rowSnapshot }) => rowSnapshot.authorId)
|
||||
pageRows.map(({ rowSnapshot }) => rowSnapshot.authorId)
|
||||
);
|
||||
const rowMap = new Map(rows.map((row) => [row.starId, row]));
|
||||
|
||||
pendingBackendRows.forEach(({ rowSnapshot }) => {
|
||||
pageRows.forEach(({ rowSnapshot }) => {
|
||||
const backendMetrics = rowMap.get(rowSnapshot.authorId);
|
||||
if (backendMetrics) {
|
||||
resultStore.setBackendMetricsSuccess(rowSnapshot.authorId, backendMetrics);
|
||||
} else {
|
||||
resultStore.setBackendMetricsMissing(rowSnapshot.authorId);
|
||||
}
|
||||
|
||||
const record = resultStore.getRecord(rowSnapshot.authorId);
|
||||
if (!record) {
|
||||
return;
|
||||
}
|
||||
|
||||
const pageRow = pendingBackendRows.find(
|
||||
(candidate) => candidate.rowSnapshot.authorId === rowSnapshot.authorId
|
||||
);
|
||||
if (pageRow) {
|
||||
renderMarketRowState(pageRow.rowDom, record);
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
pendingBackendRows.forEach(({ rowDom, rowSnapshot }) => {
|
||||
pageRows.forEach(({ rowSnapshot }) => {
|
||||
resultStore.setBackendMetricsFailed(rowSnapshot.authorId);
|
||||
const record = resultStore.getRecord(rowSnapshot.authorId);
|
||||
if (!record) {
|
||||
return;
|
||||
}
|
||||
|
||||
renderMarketRowState(rowDom, record);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -487,12 +479,23 @@ export function createMarketController(options: CreateMarketControllerOptions) {
|
||||
target: MarketExportTarget,
|
||||
inProgressLabel = "导出中"
|
||||
): Promise<MarketRecord[]> {
|
||||
setToolbarExportStatus(toolbar, `${inProgressLabel}...`);
|
||||
|
||||
if (target.mode === "count" && target.pageCount <= 1) {
|
||||
setToolbarExportStatus(toolbar, `${inProgressLabel}...`);
|
||||
await prepareCurrentPageForExport();
|
||||
return getVisibleOrderedRecords();
|
||||
}
|
||||
|
||||
const silentExportRecords = await silentExportController.exportRecords(target);
|
||||
if (silentExportRecords) {
|
||||
return hydrateExportRecords(
|
||||
silentExportRecords.map((record) => ({
|
||||
...record,
|
||||
status: record.status ?? "idle"
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
return exportRangeController.exportRecords(target);
|
||||
}
|
||||
|
||||
@@ -513,6 +516,75 @@ export function createMarketController(options: CreateMarketControllerOptions) {
|
||||
await runSyncCycle();
|
||||
}
|
||||
|
||||
async function hydrateExportRecords(records: MarketRecord[]): Promise<MarketRecord[]> {
|
||||
for (const record of records) {
|
||||
resultStore.upsertMarketRow(record);
|
||||
const existingRecord = resultStore.getRecord(record.authorId);
|
||||
if (existingRecord?.status === "success" && existingRecord.rates) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (record.hasDirectRatesSource) {
|
||||
const directRates = record.rates ?? {};
|
||||
const hasAllRates =
|
||||
Boolean(directRates.singleVideoAfterSearchRate) &&
|
||||
Boolean(directRates.personalVideoAfterSearchRate);
|
||||
|
||||
resultStore.setAuthorSuccess(record.authorId, directRates);
|
||||
if (hasAllRates) {
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
resultStore.setAuthorLoading(record.authorId);
|
||||
}
|
||||
|
||||
const metricsResult = await loadAuthorMetrics(record.authorId);
|
||||
if (metricsResult.success) {
|
||||
resultStore.setAuthorSuccess(record.authorId, metricsResult.rates);
|
||||
} else {
|
||||
resultStore.setAuthorFailed(record.authorId, metricsResult.reason);
|
||||
}
|
||||
}
|
||||
|
||||
if (searchBackendMetrics) {
|
||||
const backendTargetRecords = records.filter((record) => {
|
||||
const existingRecord = resultStore.getRecord(record.authorId);
|
||||
return !(
|
||||
existingRecord?.backendMetricsStatus === "success" ||
|
||||
existingRecord?.backendMetricsStatus === "missing"
|
||||
);
|
||||
});
|
||||
|
||||
if (backendTargetRecords.length > 0) {
|
||||
backendTargetRecords.forEach((record) => {
|
||||
resultStore.setBackendMetricsLoading(record.authorId);
|
||||
});
|
||||
|
||||
try {
|
||||
const backendRows = await searchBackendMetrics(
|
||||
backendTargetRecords.map((record) => record.authorId)
|
||||
);
|
||||
const backendRowMap = new Map(backendRows.map((row) => [row.starId, row]));
|
||||
|
||||
backendTargetRecords.forEach((record) => {
|
||||
const backendMetrics = backendRowMap.get(record.authorId);
|
||||
if (backendMetrics) {
|
||||
resultStore.setBackendMetricsSuccess(record.authorId, backendMetrics);
|
||||
} else {
|
||||
resultStore.setBackendMetricsMissing(record.authorId);
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
backendTargetRecords.forEach((record) => {
|
||||
resultStore.setBackendMetricsFailed(record.authorId);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return records.map((record) => toMarketRecord(record));
|
||||
}
|
||||
|
||||
async function harvestCurrentPageForExport(): Promise<void> {
|
||||
let hydrationSnapshot = await collectCurrentPageSnapshotsUntilSettled();
|
||||
if (
|
||||
@@ -574,40 +646,44 @@ export function createMarketController(options: CreateMarketControllerOptions) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const existingRecord = resultStore.getRecord(rowSnapshot.authorId);
|
||||
const authorName =
|
||||
mergeStringValue(existingRecord?.authorName, rowSnapshot.authorName) ?? "";
|
||||
const location = mergeStringValue(existingRecord?.location, rowSnapshot.location);
|
||||
const price21To60s = mergeStringValue(
|
||||
existingRecord?.price21To60s,
|
||||
rowSnapshot.price21To60s
|
||||
);
|
||||
return {
|
||||
...existingRecord,
|
||||
...rowSnapshot,
|
||||
authorName,
|
||||
backendMetrics: mergeFieldMap(
|
||||
existingRecord?.backendMetrics,
|
||||
rowSnapshot.backendMetrics
|
||||
),
|
||||
backendMetricsStatus: existingRecord?.backendMetricsStatus ?? "idle",
|
||||
exportFields: withExportFieldFallbacks(
|
||||
mergeFieldMap(existingRecord?.exportFields, rowSnapshot.exportFields),
|
||||
{
|
||||
authorName,
|
||||
location,
|
||||
price21To60s
|
||||
}
|
||||
),
|
||||
location,
|
||||
price21To60s,
|
||||
rates: mergeFieldMap(existingRecord?.rates, rowSnapshot.rates),
|
||||
status: existingRecord?.status ?? "idle"
|
||||
} satisfies MarketRecord;
|
||||
return toMarketRecord(rowSnapshot);
|
||||
})
|
||||
.filter((record): record is MarketRecord => record !== null);
|
||||
}
|
||||
|
||||
function toMarketRecord(rowSnapshot: MarketRowSnapshot): MarketRecord {
|
||||
const existingRecord = resultStore.getRecord(rowSnapshot.authorId);
|
||||
const authorName =
|
||||
mergeStringValue(existingRecord?.authorName, rowSnapshot.authorName) ?? "";
|
||||
const location = mergeStringValue(existingRecord?.location, rowSnapshot.location);
|
||||
const price21To60s = mergeStringValue(
|
||||
existingRecord?.price21To60s,
|
||||
rowSnapshot.price21To60s
|
||||
);
|
||||
return {
|
||||
...existingRecord,
|
||||
...rowSnapshot,
|
||||
authorName,
|
||||
backendMetrics: mergeFieldMap(
|
||||
existingRecord?.backendMetrics,
|
||||
rowSnapshot.backendMetrics
|
||||
),
|
||||
backendMetricsStatus: existingRecord?.backendMetricsStatus ?? "idle",
|
||||
exportFields: withExportFieldFallbacks(
|
||||
mergeFieldMap(existingRecord?.exportFields, rowSnapshot.exportFields),
|
||||
{
|
||||
authorName,
|
||||
location,
|
||||
price21To60s
|
||||
}
|
||||
),
|
||||
location,
|
||||
price21To60s,
|
||||
rates: mergeFieldMap(existingRecord?.rates, rowSnapshot.rates),
|
||||
status: existingRecord?.status ?? "idle"
|
||||
} satisfies MarketRecord;
|
||||
}
|
||||
|
||||
function collectCurrentPageSnapshots(): void {
|
||||
readCurrentPageRows(options.document).forEach((rowSnapshot) => {
|
||||
resultStore.upsertMarketRow(rowSnapshot);
|
||||
@@ -937,6 +1013,39 @@ function getNextSortState(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function hasCompleteRates(
|
||||
rates:
|
||||
| {
|
||||
personalVideoAfterSearchRate?: string;
|
||||
singleVideoAfterSearchRate?: string;
|
||||
}
|
||||
| undefined
|
||||
): boolean {
|
||||
return Boolean(
|
||||
rates?.singleVideoAfterSearchRate && rates?.personalVideoAfterSearchRate
|
||||
);
|
||||
}
|
||||
|
||||
function hasSettledRateState(record: MarketRecord | null): boolean {
|
||||
if (!record) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return record.status === "failed" || hasCompleteRates(record.rates);
|
||||
}
|
||||
|
||||
function hasSettledBackendMetricsState(record: MarketRecord | null): boolean {
|
||||
if (!record) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
record.backendMetricsStatus === "success" ||
|
||||
record.backendMetricsStatus === "missing" ||
|
||||
record.backendMetricsStatus === "failed"
|
||||
);
|
||||
}
|
||||
|
||||
function mergeFieldMap<T extends Record<string, string | undefined>>(
|
||||
current: T | undefined,
|
||||
incoming: T | undefined
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
export const MARKET_REQUEST_SNAPSHOT_ATTRIBUTE =
|
||||
"data-sces-market-request-snapshot";
|
||||
const MARKET_SEARCH_ENDPOINT_PATH = "/gw/api/gsearch/search_for_author_square";
|
||||
|
||||
export interface MarketListRequestSnapshot {
|
||||
body?: string;
|
||||
headers?: Record<string, string>;
|
||||
method: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export function readMarketListRequestSnapshot(
|
||||
document: Document
|
||||
): MarketListRequestSnapshot | null {
|
||||
const serializedSnapshot = document.documentElement.getAttribute(
|
||||
MARKET_REQUEST_SNAPSHOT_ATTRIBUTE
|
||||
);
|
||||
if (!serializedSnapshot) {
|
||||
return readMarketListRequestSnapshotFromPageState(document);
|
||||
}
|
||||
|
||||
try {
|
||||
const parsedSnapshot = normalizeMarketListRequestSnapshot(
|
||||
JSON.parse(serializedSnapshot)
|
||||
);
|
||||
if (!parsedSnapshot) {
|
||||
return readMarketListRequestSnapshotFromPageState(document);
|
||||
}
|
||||
|
||||
return parsedSnapshot;
|
||||
} catch {
|
||||
return readMarketListRequestSnapshotFromPageState(document);
|
||||
}
|
||||
}
|
||||
|
||||
export function writeMarketListRequestSnapshot(
|
||||
document: Document,
|
||||
snapshot: MarketListRequestSnapshot
|
||||
): void {
|
||||
document.documentElement.setAttribute(
|
||||
MARKET_REQUEST_SNAPSHOT_ATTRIBUTE,
|
||||
JSON.stringify(snapshot)
|
||||
);
|
||||
}
|
||||
|
||||
function isMarketListRequestSnapshot(
|
||||
value: unknown
|
||||
): value is MarketListRequestSnapshot {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const candidate = value as Partial<MarketListRequestSnapshot>;
|
||||
return (
|
||||
typeof candidate.method === "string" &&
|
||||
typeof candidate.url === "string" &&
|
||||
(!("body" in candidate) || typeof candidate.body === "string") &&
|
||||
(!("headers" in candidate) || isStringRecord(candidate.headers))
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeMarketListRequestSnapshot(
|
||||
value: unknown
|
||||
): MarketListRequestSnapshot | null {
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const candidate = value as Partial<MarketListRequestSnapshot> & {
|
||||
headers?: Record<string, unknown>;
|
||||
};
|
||||
const normalizedSnapshot: Partial<MarketListRequestSnapshot> = {
|
||||
body: typeof candidate.body === "string" ? candidate.body : undefined,
|
||||
method: typeof candidate.method === "string" ? candidate.method : undefined,
|
||||
url: typeof candidate.url === "string" ? candidate.url : undefined
|
||||
};
|
||||
|
||||
if (candidate.headers && typeof candidate.headers === "object") {
|
||||
normalizedSnapshot.headers = Object.fromEntries(
|
||||
Object.entries(candidate.headers)
|
||||
.filter(([, entry]) =>
|
||||
["string", "number", "boolean"].includes(typeof entry)
|
||||
)
|
||||
.map(([key, entry]) => [key, String(entry)])
|
||||
);
|
||||
}
|
||||
|
||||
return isMarketListRequestSnapshot(normalizedSnapshot)
|
||||
? normalizedSnapshot
|
||||
: null;
|
||||
}
|
||||
|
||||
function isStringRecord(value: unknown): value is Record<string, string> {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
|
||||
return Object.values(value).every((entry) => typeof entry === "string");
|
||||
}
|
||||
|
||||
function readMarketListRequestSnapshotFromPageState(
|
||||
document: Document
|
||||
): MarketListRequestSnapshot | null {
|
||||
const reqParams = findMarketReqParams(document);
|
||||
if (!reqParams) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
body: JSON.stringify(reqParams),
|
||||
method: "POST",
|
||||
url: buildMarketSearchUrl(document)
|
||||
};
|
||||
}
|
||||
|
||||
function findMarketReqParams(document: Document): Record<string, unknown> | null {
|
||||
const marketRoot = document.querySelector(".base-author-list") as
|
||||
| (Element & {
|
||||
__vue__?: {
|
||||
_setupState?: Record<string, unknown>;
|
||||
};
|
||||
})
|
||||
| null;
|
||||
const setupState = marketRoot?.__vue__?._setupState;
|
||||
if (!setupState) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const queue: unknown[] = Object.values(setupState);
|
||||
while (queue.length > 0) {
|
||||
const current = unwrapVueRef(queue.shift());
|
||||
if (!isRecord(current)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const reqParams = unwrapVueRef(current.reqParams);
|
||||
if (isRecord(reqParams)) {
|
||||
return reqParams;
|
||||
}
|
||||
|
||||
Object.values(current).forEach((value) => {
|
||||
queue.push(value);
|
||||
});
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildMarketSearchUrl(document: Document): string {
|
||||
if (
|
||||
document.location?.origin &&
|
||||
document.location.origin !== "null" &&
|
||||
document.location.origin !== "about:blank"
|
||||
) {
|
||||
return document.location.origin.includes("xingtu.cn")
|
||||
? MARKET_SEARCH_ENDPOINT_PATH
|
||||
: new URL(MARKET_SEARCH_ENDPOINT_PATH, document.location.origin).toString();
|
||||
}
|
||||
|
||||
return MARKET_SEARCH_ENDPOINT_PATH;
|
||||
}
|
||||
|
||||
function unwrapVueRef(value: unknown): unknown {
|
||||
if (isRecord(value) && "value" in value) {
|
||||
return value.value;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
@@ -0,0 +1,513 @@
|
||||
import { normalizeFractionRateDisplay } from "../../shared/rate-normalizer";
|
||||
import type { MarketRowSnapshot } from "./types";
|
||||
|
||||
export interface ParsedMarketListResponse {
|
||||
currentPage?: number;
|
||||
pageSize?: number;
|
||||
records: MarketRowSnapshot[];
|
||||
totalCount?: number;
|
||||
totalPages?: number;
|
||||
}
|
||||
|
||||
const PAGE_NUMBER_KEYS = [
|
||||
"currentPage",
|
||||
"page",
|
||||
"pageNo",
|
||||
"pageNum",
|
||||
"page_no",
|
||||
"page_num"
|
||||
] as const;
|
||||
|
||||
const PAGE_SIZE_KEYS = [
|
||||
"limit",
|
||||
"pageSize",
|
||||
"page_size",
|
||||
"size"
|
||||
] as const;
|
||||
|
||||
const TOTAL_COUNT_KEYS = [
|
||||
"total",
|
||||
"totalCount",
|
||||
"total_count"
|
||||
] as const;
|
||||
|
||||
const TOTAL_PAGE_KEYS = [
|
||||
"pageCount",
|
||||
"page_count",
|
||||
"totalPage",
|
||||
"totalPages",
|
||||
"total_page",
|
||||
"total_pages"
|
||||
] as const;
|
||||
|
||||
export function mapMarketListRow(
|
||||
row: Record<string, unknown>
|
||||
): MarketRowSnapshot {
|
||||
const attributeDatas = readMarketAttributeDatas(row);
|
||||
const singleVideoAfterSearchRate = normalizeMarketListRate(
|
||||
readMarketFieldValue(row, attributeDatas, "avg_search_after_view_rate_30d")
|
||||
);
|
||||
|
||||
return {
|
||||
authorId:
|
||||
readString(readMarketFieldValue(row, attributeDatas, "star_id")) ??
|
||||
readString(readMarketFieldValue(row, attributeDatas, "id")) ??
|
||||
"",
|
||||
authorName:
|
||||
readString(readMarketFieldValue(row, attributeDatas, "nickname")) ??
|
||||
readString(readMarketFieldValue(row, attributeDatas, "nick_name")) ??
|
||||
"",
|
||||
exportFields: buildMarketExportFieldFallbacks(row, attributeDatas),
|
||||
hasDirectRatesSource: true,
|
||||
location: readMarketLocation(row, attributeDatas),
|
||||
price21To60s: readMarketPrice21To60s(row, attributeDatas),
|
||||
rates: singleVideoAfterSearchRate
|
||||
? {
|
||||
singleVideoAfterSearchRate
|
||||
}
|
||||
: undefined
|
||||
};
|
||||
}
|
||||
|
||||
export function parseMarketListResponse(
|
||||
payload: unknown
|
||||
): ParsedMarketListResponse | null {
|
||||
const container = findMarketListContainer(payload);
|
||||
if (!container) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const marketList = readMarketListArray(container);
|
||||
if (!marketList) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
currentPage: readKnownNumberDeep(container, PAGE_NUMBER_KEYS) ?? undefined,
|
||||
pageSize: readKnownNumberDeep(container, PAGE_SIZE_KEYS) ?? undefined,
|
||||
records: marketList
|
||||
.map((row) => (isRecord(row) ? mapMarketListRow(row) : null))
|
||||
.filter(
|
||||
(row): row is MarketRowSnapshot =>
|
||||
row !== null && Boolean(row.authorId || row.authorName)
|
||||
),
|
||||
totalCount: readKnownNumberDeep(container, TOTAL_COUNT_KEYS) ?? undefined,
|
||||
totalPages: readKnownNumberDeep(container, TOTAL_PAGE_KEYS) ?? undefined
|
||||
};
|
||||
}
|
||||
|
||||
export function readKnownPaginationNumber(
|
||||
value: unknown,
|
||||
kind: "page" | "pageSize"
|
||||
): number | null {
|
||||
if (!isRecord(value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return readKnownNumberDeep(value, kind === "page" ? PAGE_NUMBER_KEYS : PAGE_SIZE_KEYS);
|
||||
}
|
||||
|
||||
function findMarketListContainer(value: unknown): Record<string, unknown> | null {
|
||||
const queue: unknown[] = [value];
|
||||
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
if (!isRecord(current)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (readMarketListArray(current)) {
|
||||
return current;
|
||||
}
|
||||
|
||||
Object.values(current).forEach((entry) => {
|
||||
queue.push(unwrapVueRef(entry));
|
||||
});
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function readMarketListArray(record: Record<string, unknown>): unknown[] | null {
|
||||
const marketList = unwrapVueRef(record.marketList);
|
||||
if (Array.isArray(marketList)) {
|
||||
return marketList;
|
||||
}
|
||||
|
||||
const authors = unwrapVueRef(record.authors);
|
||||
if (Array.isArray(authors)) {
|
||||
return authors;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function unwrapVueRef(value: unknown): unknown {
|
||||
if (isRecord(value) && "value" in value) {
|
||||
return value.value;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
|
||||
function readMarketAttributeDatas(
|
||||
record: Record<string, unknown>
|
||||
): Record<string, unknown> {
|
||||
return isRecord(record.attribute_datas) ? record.attribute_datas : {};
|
||||
}
|
||||
|
||||
function readMarketFieldValue(
|
||||
record: Record<string, unknown>,
|
||||
attributeDatas: Record<string, unknown>,
|
||||
field: string
|
||||
): unknown {
|
||||
return record[field] ?? attributeDatas[field];
|
||||
}
|
||||
|
||||
function readString(value: unknown): string | null {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
function normalizeMarketListRate(value: unknown): string | null {
|
||||
if (typeof value === "number") {
|
||||
return normalizeFractionRateDisplay(String(value));
|
||||
}
|
||||
|
||||
return typeof value === "string" ? normalizeFractionRateDisplay(value) : null;
|
||||
}
|
||||
|
||||
function normalizeExportCellText(value: string | null | undefined): string {
|
||||
return value?.replace(/\s+/g, " ").trim() ?? "";
|
||||
}
|
||||
|
||||
function buildMarketExportFieldFallbacks(
|
||||
record: Record<string, unknown>,
|
||||
attributeDatas: Record<string, unknown>
|
||||
): Record<string, string> | undefined {
|
||||
const exportFields: Record<string, string> = {};
|
||||
const authorInfo = buildMarketAuthorInfo(record, attributeDatas);
|
||||
const authorType = buildMarketAuthorType(record, attributeDatas);
|
||||
const contentTheme = buildMarketContentTheme(record, attributeDatas);
|
||||
const connectedUsers = formatWanValue(
|
||||
readNumericValue(readMarketFieldValue(record, attributeDatas, "link_link_cnt_by_industry"))
|
||||
);
|
||||
const followerCount = formatWanValue(
|
||||
readNumericValue(readMarketFieldValue(record, attributeDatas, "follower"))
|
||||
);
|
||||
const expectedCpm = formatDecimalDisplay(
|
||||
readNumericValue(readMarketFieldValue(record, attributeDatas, "prospective_20_60_cpm"))
|
||||
);
|
||||
const expectedPlayCount = formatWanValue(
|
||||
readNumericValue(readMarketFieldValue(record, attributeDatas, "expected_play_num"))
|
||||
);
|
||||
const interactionRate = formatFractionPercent(
|
||||
readNumericValue(readMarketFieldValue(record, attributeDatas, "interact_rate_within_30d"))
|
||||
);
|
||||
const finishRate = formatFractionPercent(
|
||||
readNumericValue(readMarketFieldValue(record, attributeDatas, "play_over_rate_within_30d"))
|
||||
);
|
||||
const burstRate = readBurstRateDisplay(
|
||||
readNumericValue(readMarketFieldValue(record, attributeDatas, "burst_text_rate"))
|
||||
);
|
||||
const price21To60s = readMarketPrice21To60s(record, attributeDatas);
|
||||
const representativeVideo = readMarketRepresentativeVideo(record, attributeDatas);
|
||||
|
||||
assignExportField(exportFields, "达人信息", authorInfo);
|
||||
assignExportField(exportFields, "代表视频", representativeVideo);
|
||||
assignExportField(exportFields, "达人类型", authorType);
|
||||
assignExportField(exportFields, "内容主题", contentTheme);
|
||||
assignExportField(exportFields, "连接用户数", connectedUsers);
|
||||
assignExportField(exportFields, "粉丝数", followerCount);
|
||||
assignExportField(exportFields, "预期CPM", expectedCpm);
|
||||
assignExportField(exportFields, "预期播放量", expectedPlayCount);
|
||||
assignExportField(exportFields, "互动率", interactionRate);
|
||||
assignExportField(exportFields, "完播率", finishRate);
|
||||
assignExportField(exportFields, "爆文率", burstRate);
|
||||
assignExportField(exportFields, "21-60s报价", price21To60s);
|
||||
|
||||
return Object.keys(exportFields).length > 0 ? exportFields : undefined;
|
||||
}
|
||||
|
||||
function assignExportField(
|
||||
exportFields: Record<string, string>,
|
||||
key: string,
|
||||
value: string | undefined
|
||||
): void {
|
||||
if (hasTextValue(value)) {
|
||||
exportFields[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
function hasTextValue(value: string | undefined | null): boolean {
|
||||
return Boolean(value && value.trim().length > 0);
|
||||
}
|
||||
|
||||
function buildMarketAuthorInfo(
|
||||
record: Record<string, unknown>,
|
||||
attributeDatas: Record<string, unknown>
|
||||
): string | undefined {
|
||||
const nickname =
|
||||
readString(readMarketFieldValue(record, attributeDatas, "nickname")) ??
|
||||
readString(readMarketFieldValue(record, attributeDatas, "nick_name")) ??
|
||||
"";
|
||||
const parts = [
|
||||
nickname,
|
||||
readMarketGenderLabel(readMarketFieldValue(record, attributeDatas, "gender")),
|
||||
readString(readMarketFieldValue(record, attributeDatas, "city")) ?? ""
|
||||
].filter((value) => Boolean(value));
|
||||
|
||||
return parts.length > 0 ? parts.join(" ") : undefined;
|
||||
}
|
||||
|
||||
function buildMarketAuthorType(
|
||||
record: Record<string, unknown>,
|
||||
attributeDatas: Record<string, unknown>
|
||||
): string | undefined {
|
||||
const tagsRelation = readRecordLike(
|
||||
readMarketFieldValue(record, attributeDatas, "tags_relation")
|
||||
);
|
||||
if (tagsRelation) {
|
||||
const primaryTag = Object.keys(tagsRelation)[0];
|
||||
if (hasTextValue(primaryTag)) {
|
||||
return primaryTag;
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function buildMarketContentTheme(
|
||||
record: Record<string, unknown>,
|
||||
attributeDatas: Record<string, unknown>
|
||||
): string | undefined {
|
||||
const themes = readStringArray(
|
||||
readMarketFieldValue(record, attributeDatas, "content_theme_labels_180d")
|
||||
);
|
||||
if (themes.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (themes.length <= 2) {
|
||||
return themes.join(" ");
|
||||
}
|
||||
|
||||
return `${themes.slice(0, 2).join(" ")} ${themes.length - 2}+`;
|
||||
}
|
||||
|
||||
function readMarketLocation(
|
||||
record: Record<string, unknown>,
|
||||
attributeDatas: Record<string, unknown>
|
||||
): string | undefined {
|
||||
return readString(readMarketFieldValue(record, attributeDatas, "city")) ?? undefined;
|
||||
}
|
||||
|
||||
function readMarketPrice21To60s(
|
||||
record: Record<string, unknown>,
|
||||
attributeDatas: Record<string, unknown>
|
||||
): string | undefined {
|
||||
return formatCurrencyValue(
|
||||
readNumericValue(readMarketFieldValue(record, attributeDatas, "price_20_60"))
|
||||
);
|
||||
}
|
||||
|
||||
function readMarketRepresentativeVideo(
|
||||
record: Record<string, unknown>,
|
||||
attributeDatas: Record<string, unknown>
|
||||
): string | undefined {
|
||||
const items = readArrayLike(readMarketFieldValue(record, attributeDatas, "items"));
|
||||
for (const item of items) {
|
||||
if (!isRecord(item)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const title = readString(item.title);
|
||||
if (hasTextValue(title)) {
|
||||
return normalizeExportCellText(title);
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function readMarketGenderLabel(value: unknown): string | undefined {
|
||||
const rawValue = typeof value === "number" ? String(value) : readString(value);
|
||||
if (rawValue === "1") {
|
||||
return "男";
|
||||
}
|
||||
|
||||
if (rawValue === "2") {
|
||||
return "女";
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function readBurstRateDisplay(value: number | null): string | undefined {
|
||||
if (value === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (value <= 0) {
|
||||
return "-";
|
||||
}
|
||||
|
||||
return formatFractionPercent(value);
|
||||
}
|
||||
|
||||
function formatCurrencyValue(value: number | null): string | undefined {
|
||||
if (value === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return `¥${value.toLocaleString("en-US", {
|
||||
maximumFractionDigits: 0
|
||||
})}`;
|
||||
}
|
||||
|
||||
function formatWanValue(value: number | null): string | undefined {
|
||||
if (value === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return `${formatDecimalWithGrouping(value / 10000)}w`;
|
||||
}
|
||||
|
||||
function formatFractionPercent(value: number | null): string | undefined {
|
||||
if (value === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return `${formatDecimalDisplay(value * 100)}%`;
|
||||
}
|
||||
|
||||
function formatDecimalDisplay(value: number | null): string | undefined {
|
||||
if (value === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return value.toLocaleString("en-US", {
|
||||
maximumFractionDigits: 1,
|
||||
minimumFractionDigits: 0,
|
||||
useGrouping: false
|
||||
});
|
||||
}
|
||||
|
||||
function formatDecimalWithGrouping(value: number): string {
|
||||
return value.toLocaleString("en-US", {
|
||||
maximumFractionDigits: 1,
|
||||
minimumFractionDigits: 0
|
||||
});
|
||||
}
|
||||
|
||||
function readNumericValue(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
const trimmedValue = value.trim();
|
||||
if (!trimmedValue) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsedValue = Number(trimmedValue);
|
||||
return Number.isFinite(parsedValue) ? parsedValue : null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function readStringArray(value: unknown): string[] {
|
||||
if (Array.isArray(value)) {
|
||||
return value.filter((item): item is string => typeof item === "string");
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
try {
|
||||
const parsedValue = JSON.parse(value);
|
||||
return Array.isArray(parsedValue)
|
||||
? parsedValue.filter((item): item is string => typeof item === "string")
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
function readArrayLike(value: unknown): unknown[] {
|
||||
if (Array.isArray(value)) {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
try {
|
||||
const parsedValue = JSON.parse(value);
|
||||
return Array.isArray(parsedValue) ? parsedValue : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
function readRecordLike(value: unknown): Record<string, unknown> | null {
|
||||
if (isRecord(value)) {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
try {
|
||||
const parsedValue = JSON.parse(value);
|
||||
return isRecord(parsedValue) ? parsedValue : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function readKnownNumber(
|
||||
record: Record<string, unknown>,
|
||||
keys: readonly string[]
|
||||
): number | undefined {
|
||||
for (const key of keys) {
|
||||
const value = readNumericValue(record[key]);
|
||||
if (value !== null) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function readKnownNumberDeep(
|
||||
value: unknown,
|
||||
keys: readonly string[]
|
||||
): number | null {
|
||||
if (!isRecord(value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const directValue = readKnownNumber(value, keys);
|
||||
if (typeof directValue === "number") {
|
||||
return directValue;
|
||||
}
|
||||
|
||||
for (const nestedValue of Object.values(value)) {
|
||||
const candidate =
|
||||
readKnownNumberDeep(unwrapVueRef(nestedValue), keys);
|
||||
if (typeof candidate === "number") {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -1,6 +1,11 @@
|
||||
import { normalizeFractionRateDisplay } from "../../shared/rate-normalizer";
|
||||
import {
|
||||
writeMarketListRequestSnapshot
|
||||
} from "./market-list-request-snapshot";
|
||||
import { parseMarketListResponse } from "./market-list-row";
|
||||
|
||||
const BRIDGE_MARKER = "__SCES_MARKET_PAGE_BRIDGE_INSTALLED__";
|
||||
const MARKET_SEARCH_REQUEST_PATH = "/gw/api/gsearch/search_for_author_square";
|
||||
const SERIALIZED_MARKET_ROWS_ATTRIBUTE = "data-sces-market-rows";
|
||||
|
||||
type MarketRow = {
|
||||
@@ -24,6 +29,7 @@ function installMarketPageBridge() {
|
||||
}
|
||||
|
||||
window[BRIDGE_MARKER] = true;
|
||||
installMarketRequestSnapshotBridge();
|
||||
syncSerializedMarketRows();
|
||||
|
||||
const observer = new MutationObserver(() => {
|
||||
@@ -39,7 +45,16 @@ function installMarketPageBridge() {
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
function installMarketRequestSnapshotBridge() {
|
||||
installFetchSnapshotBridge();
|
||||
installXmlHttpRequestSnapshotBridge();
|
||||
}
|
||||
|
||||
function syncSerializedMarketRows() {
|
||||
if (typeof document === "undefined") {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextSerializedRows = JSON.stringify(readSerializedMarketRows());
|
||||
if (
|
||||
document.documentElement.getAttribute(SERIALIZED_MARKET_ROWS_ATTRIBUTE) !==
|
||||
@@ -52,6 +67,131 @@ function syncSerializedMarketRows() {
|
||||
}
|
||||
}
|
||||
|
||||
function installFetchSnapshotBridge() {
|
||||
if (typeof window.fetch !== "function") {
|
||||
return;
|
||||
}
|
||||
|
||||
const originalFetch = window.fetch.bind(window);
|
||||
window.fetch = async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const requestSnapshot = readFetchSnapshot(input, init);
|
||||
const response = await originalFetch(input, init);
|
||||
if (requestSnapshot) {
|
||||
const clonedResponse = response.clone();
|
||||
void captureMarketSnapshotFromResponse(requestSnapshot, () =>
|
||||
clonedResponse.json()
|
||||
);
|
||||
}
|
||||
return response;
|
||||
};
|
||||
}
|
||||
|
||||
function installXmlHttpRequestSnapshotBridge() {
|
||||
const OriginalXmlHttpRequest = window.XMLHttpRequest;
|
||||
if (!OriginalXmlHttpRequest) {
|
||||
return;
|
||||
}
|
||||
|
||||
const originalOpen = OriginalXmlHttpRequest.prototype.open;
|
||||
const originalSend = OriginalXmlHttpRequest.prototype.send;
|
||||
const originalSetRequestHeader = OriginalXmlHttpRequest.prototype.setRequestHeader;
|
||||
|
||||
OriginalXmlHttpRequest.prototype.open = function (
|
||||
method: string,
|
||||
url: string | URL,
|
||||
...rest: unknown[]
|
||||
) {
|
||||
(
|
||||
this as XMLHttpRequest & {
|
||||
__scesMarketSnapshot?: {
|
||||
headers: Record<string, string>;
|
||||
method: string;
|
||||
url: string;
|
||||
};
|
||||
}
|
||||
).__scesMarketSnapshot = {
|
||||
headers: {},
|
||||
method,
|
||||
url: String(url)
|
||||
};
|
||||
return originalOpen.call(this, method, url, ...(rest as [boolean?, string?, string?]));
|
||||
};
|
||||
|
||||
OriginalXmlHttpRequest.prototype.setRequestHeader = function (
|
||||
name: string,
|
||||
value: string
|
||||
) {
|
||||
(
|
||||
this as XMLHttpRequest & {
|
||||
__scesMarketSnapshot?: {
|
||||
headers: Record<string, string>;
|
||||
};
|
||||
}
|
||||
).__scesMarketSnapshot?.headers &&
|
||||
((this as XMLHttpRequest & {
|
||||
__scesMarketSnapshot?: {
|
||||
headers: Record<string, string>;
|
||||
};
|
||||
}).__scesMarketSnapshot!.headers[name] = value);
|
||||
return originalSetRequestHeader.call(this, name, value);
|
||||
};
|
||||
|
||||
OriginalXmlHttpRequest.prototype.send = function (body?: Document | XMLHttpRequestBodyInit | null) {
|
||||
const snapshotState = (
|
||||
this as XMLHttpRequest & {
|
||||
__scesMarketSnapshot?: {
|
||||
body?: string;
|
||||
headers: Record<string, string>;
|
||||
method: string;
|
||||
url: string;
|
||||
};
|
||||
}
|
||||
).__scesMarketSnapshot;
|
||||
if (snapshotState) {
|
||||
snapshotState.body = typeof body === "string" ? body : undefined;
|
||||
this.addEventListener("load", () => {
|
||||
if (this.status < 200 || this.status >= 300 || typeof this.responseText !== "string") {
|
||||
return;
|
||||
}
|
||||
|
||||
void captureMarketSnapshotFromResponse(snapshotState, async () =>
|
||||
JSON.parse(this.responseText)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
return originalSend.call(this, body);
|
||||
};
|
||||
}
|
||||
|
||||
async function captureMarketSnapshotFromResponse(
|
||||
snapshot: {
|
||||
body?: string;
|
||||
headers?: Record<string, string>;
|
||||
method: string;
|
||||
url: string;
|
||||
},
|
||||
readPayload: () => Promise<unknown>
|
||||
) {
|
||||
if (!isMarketSearchRequest(snapshot.url)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const payload = await readPayload();
|
||||
if (!parseMarketListResponse(payload)) {
|
||||
return;
|
||||
}
|
||||
|
||||
writeMarketListRequestSnapshot(document, {
|
||||
body: snapshot.body,
|
||||
headers: snapshot.headers,
|
||||
method: snapshot.method,
|
||||
url: snapshot.url
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
|
||||
function readSerializedMarketRows() {
|
||||
const marketList = readMarketList();
|
||||
return marketList
|
||||
@@ -71,7 +211,59 @@ function readSerializedMarketRows() {
|
||||
.filter((row) => Boolean(row.authorId || row.authorName));
|
||||
}
|
||||
|
||||
function readFetchSnapshot(
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit
|
||||
): {
|
||||
body?: string;
|
||||
headers?: Record<string, string>;
|
||||
method: string;
|
||||
url: string;
|
||||
} | null {
|
||||
const request = input instanceof Request ? input : null;
|
||||
const method = init?.method ?? request?.method ?? "GET";
|
||||
const url = request?.url ?? String(input);
|
||||
const body =
|
||||
typeof init?.body === "string"
|
||||
? init.body
|
||||
: typeof request?.bodyUsed === "boolean" && request.bodyUsed
|
||||
? undefined
|
||||
: undefined;
|
||||
const headers = serializeHeaders(init?.headers ?? request?.headers);
|
||||
|
||||
return {
|
||||
body,
|
||||
headers,
|
||||
method,
|
||||
url
|
||||
};
|
||||
}
|
||||
|
||||
function serializeHeaders(
|
||||
headers: HeadersInit | undefined
|
||||
): Record<string, string> | undefined {
|
||||
if (!headers) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (headers instanceof Headers) {
|
||||
return Object.fromEntries(headers.entries());
|
||||
}
|
||||
|
||||
if (Array.isArray(headers)) {
|
||||
return Object.fromEntries(headers);
|
||||
}
|
||||
|
||||
return Object.fromEntries(
|
||||
Object.entries(headers).map(([key, value]) => [key, String(value)])
|
||||
);
|
||||
}
|
||||
|
||||
function readMarketList(): MarketRow[] {
|
||||
if (typeof document === "undefined") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const marketRoot = document.querySelector(".base-author-list") as
|
||||
| (HTMLElement & {
|
||||
__vue__?: {
|
||||
@@ -102,6 +294,15 @@ function readMarketList(): MarketRow[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
function isMarketSearchRequest(url: string): boolean {
|
||||
return (
|
||||
url === MARKET_SEARCH_REQUEST_PATH ||
|
||||
url.startsWith(`${MARKET_SEARCH_REQUEST_PATH}?`) ||
|
||||
url.includes(`${MARKET_SEARCH_REQUEST_PATH}?`) ||
|
||||
url.endsWith(MARKET_SEARCH_REQUEST_PATH)
|
||||
);
|
||||
}
|
||||
|
||||
function looksLikeMarketList(value: unknown[]): boolean {
|
||||
const firstRow = value[0];
|
||||
return isRecord(firstRow) && ("star_id" in firstRow || "attribute_datas" in firstRow);
|
||||
|
||||
@@ -0,0 +1,401 @@
|
||||
import {
|
||||
readMarketListRequestSnapshot,
|
||||
type MarketListRequestSnapshot
|
||||
} from "./market-list-request-snapshot";
|
||||
import {
|
||||
parseMarketListResponse,
|
||||
readKnownPaginationNumber
|
||||
} from "./market-list-row";
|
||||
import type { MarketExportTarget, MarketRecord } from "./types";
|
||||
|
||||
interface FetchResponseLike {
|
||||
json(): Promise<unknown>;
|
||||
ok: boolean;
|
||||
}
|
||||
|
||||
type FetchLike = (input: string, init?: RequestInit) => Promise<FetchResponseLike>;
|
||||
|
||||
interface SilentExportControllerOptions {
|
||||
document: Document;
|
||||
fetchImpl?: FetchLike;
|
||||
onProgress?: (state: { currentPage: number; totalPages?: number }) => void;
|
||||
}
|
||||
|
||||
type PageSource = "body" | "none" | "url";
|
||||
|
||||
const PAGE_NUMBER_KEYS = [
|
||||
"currentPage",
|
||||
"page",
|
||||
"pageNo",
|
||||
"pageNum",
|
||||
"page_no",
|
||||
"page_num"
|
||||
] as const;
|
||||
|
||||
export function createSilentExportController(
|
||||
options: SilentExportControllerOptions
|
||||
) {
|
||||
const fetchImpl = options.fetchImpl ?? defaultFetch;
|
||||
|
||||
return {
|
||||
async exportRecords(target: MarketExportTarget): Promise<MarketRecord[] | null> {
|
||||
const snapshot = readMarketListRequestSnapshot(options.document);
|
||||
if (!snapshot) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const baseRequest = createPagedRequest(snapshot);
|
||||
if (!baseRequest) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const mergedRecords = new Map<string, MarketRecord>();
|
||||
const maxPageCount = target.mode === "count" ? target.pageCount : 200;
|
||||
let totalPagesHint: number | undefined;
|
||||
|
||||
for (let offset = 0; offset < maxPageCount; offset += 1) {
|
||||
const pageNumber = baseRequest.initialPage + offset;
|
||||
options.onProgress?.({
|
||||
currentPage: offset + 1,
|
||||
totalPages: target.mode === "count" ? target.pageCount : totalPagesHint
|
||||
});
|
||||
|
||||
const payload = await fetchPagePayload(fetchImpl, baseRequest, pageNumber);
|
||||
const parsedResponse = parseMarketListResponse(payload);
|
||||
if (!parsedResponse) {
|
||||
return null;
|
||||
}
|
||||
|
||||
totalPagesHint = parsedResponse.totalPages ?? totalPagesHint;
|
||||
if (parsedResponse.records.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
parsedResponse.records.forEach((record) => {
|
||||
const existingRecord = mergedRecords.get(record.authorId);
|
||||
mergedRecords.set(record.authorId, mergeMarketRecord(existingRecord, record));
|
||||
});
|
||||
|
||||
if (target.mode === "count" && offset + 1 >= target.pageCount) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (target.mode === "all") {
|
||||
if (
|
||||
typeof parsedResponse.totalPages === "number" &&
|
||||
pageNumber >= parsedResponse.totalPages
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (
|
||||
typeof parsedResponse.pageSize === "number" &&
|
||||
parsedResponse.records.length < parsedResponse.pageSize
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(mergedRecords.values());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function createPagedRequest(
|
||||
snapshot: MarketListRequestSnapshot
|
||||
): {
|
||||
initialPage: number;
|
||||
pageSource: PageSource;
|
||||
snapshot: MarketListRequestSnapshot;
|
||||
} {
|
||||
const bodyPage = readPageFromBody(snapshot.body);
|
||||
if (bodyPage !== null) {
|
||||
return {
|
||||
initialPage: bodyPage,
|
||||
pageSource: "body",
|
||||
snapshot
|
||||
};
|
||||
}
|
||||
|
||||
const urlPage = readPageFromUrl(snapshot.url);
|
||||
if (urlPage !== null) {
|
||||
return {
|
||||
initialPage: urlPage,
|
||||
pageSource: "url",
|
||||
snapshot
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
initialPage: 1,
|
||||
pageSource: "none",
|
||||
snapshot
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchPagePayload(
|
||||
fetchImpl: FetchLike,
|
||||
request: {
|
||||
pageSource: PageSource;
|
||||
snapshot: MarketListRequestSnapshot;
|
||||
},
|
||||
pageNumber: number
|
||||
): Promise<unknown> {
|
||||
const nextUrl =
|
||||
request.pageSource === "url"
|
||||
? mutateUrlPage(request.snapshot.url, pageNumber)
|
||||
: request.snapshot.url;
|
||||
const nextBody = mutateBodyPage(request.snapshot.body, pageNumber);
|
||||
|
||||
const response = await fetchImpl(nextUrl, {
|
||||
body: nextBody,
|
||||
credentials: "include",
|
||||
headers: filterReplayHeaders(request.snapshot.headers, nextBody),
|
||||
method: request.snapshot.method
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error("静默导出请求失败");
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
function readPageFromUrl(url: string): number | null {
|
||||
try {
|
||||
const parsedUrl = new URL(url);
|
||||
for (const key of PAGE_NUMBER_KEYS) {
|
||||
const value = readNumericString(parsedUrl.searchParams.get(key));
|
||||
if (value !== null) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function mutateUrlPage(url: string, pageNumber: number): string {
|
||||
try {
|
||||
const parsedUrl = new URL(url);
|
||||
for (const key of PAGE_NUMBER_KEYS) {
|
||||
if (!parsedUrl.searchParams.has(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
parsedUrl.searchParams.set(key, String(pageNumber));
|
||||
return parsedUrl.toString();
|
||||
}
|
||||
|
||||
parsedUrl.searchParams.set("page", String(pageNumber));
|
||||
return parsedUrl.toString();
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
function readPageFromBody(body: string | undefined): number | null {
|
||||
const parsedBody = parseBody(body);
|
||||
if (!parsedBody) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return readKnownPaginationNumber(parsedBody, "page");
|
||||
}
|
||||
|
||||
function mutateBodyPage(body: string | undefined, pageNumber: number): string | undefined {
|
||||
if (!body) {
|
||||
return body;
|
||||
}
|
||||
|
||||
const trimmedBody = body.trim();
|
||||
if (!trimmedBody) {
|
||||
return body;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsedJson = JSON.parse(trimmedBody);
|
||||
if (!replacePageNumberInValue(parsedJson, pageNumber) && isRecord(parsedJson)) {
|
||||
parsedJson.page = pageNumber;
|
||||
}
|
||||
|
||||
return JSON.stringify(parsedJson);
|
||||
} catch {
|
||||
const searchParams = new URLSearchParams(trimmedBody);
|
||||
for (const key of PAGE_NUMBER_KEYS) {
|
||||
if (!searchParams.has(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
searchParams.set(key, String(pageNumber));
|
||||
return searchParams.toString();
|
||||
}
|
||||
|
||||
searchParams.set("page", String(pageNumber));
|
||||
return searchParams.toString();
|
||||
}
|
||||
}
|
||||
|
||||
function parseBody(body: string | undefined): Record<string, unknown> | null {
|
||||
if (!body) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const trimmedBody = body.trim();
|
||||
if (!trimmedBody) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsedBody = JSON.parse(trimmedBody);
|
||||
return isRecord(parsedBody) ? parsedBody : null;
|
||||
} catch {
|
||||
const searchParams = new URLSearchParams(trimmedBody);
|
||||
const payload: Record<string, unknown> = {};
|
||||
searchParams.forEach((value, key) => {
|
||||
payload[key] = value;
|
||||
});
|
||||
return payload;
|
||||
}
|
||||
}
|
||||
|
||||
function replacePageNumberInValue(value: unknown, pageNumber: number): boolean {
|
||||
if (!isRecord(value)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let replaced = false;
|
||||
PAGE_NUMBER_KEYS.forEach((key) => {
|
||||
if (!(key in value)) {
|
||||
return;
|
||||
}
|
||||
|
||||
value[key] = pageNumber;
|
||||
replaced = true;
|
||||
});
|
||||
|
||||
if (replaced) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return Object.values(value).some((entry) => replacePageNumberInValue(entry, pageNumber));
|
||||
}
|
||||
|
||||
function filterReplayHeaders(
|
||||
headers: Record<string, string> | undefined,
|
||||
body: string | undefined
|
||||
): HeadersInit | undefined {
|
||||
const filteredHeaders = Object.fromEntries(
|
||||
Object.entries(headers ?? {}).filter(([key]) => {
|
||||
const normalizedKey = key.toLowerCase();
|
||||
return normalizedKey !== "content-length" && normalizedKey !== "host";
|
||||
})
|
||||
);
|
||||
|
||||
if (body) {
|
||||
if (!hasHeader(filteredHeaders, "accept")) {
|
||||
filteredHeaders.Accept = "application/json, text/plain, */*";
|
||||
}
|
||||
if (!hasHeader(filteredHeaders, "content-type")) {
|
||||
filteredHeaders["Content-Type"] = "application/json";
|
||||
}
|
||||
if (!hasHeader(filteredHeaders, "x-login-source")) {
|
||||
filteredHeaders["x-login-source"] = "1";
|
||||
}
|
||||
if (!hasHeader(filteredHeaders, "agw-js-conv")) {
|
||||
filteredHeaders["Agw-Js-Conv"] = "str";
|
||||
}
|
||||
}
|
||||
|
||||
return Object.keys(filteredHeaders).length > 0 ? filteredHeaders : undefined;
|
||||
}
|
||||
|
||||
function hasHeader(headers: Record<string, string>, key: string): boolean {
|
||||
return Object.keys(headers).some((headerKey) => headerKey.toLowerCase() === key);
|
||||
}
|
||||
|
||||
function readNumericString(value: string | null): number | null {
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsedValue = Number(value);
|
||||
return Number.isFinite(parsedValue) ? parsedValue : null;
|
||||
}
|
||||
|
||||
async function defaultFetch(input: string, init?: RequestInit) {
|
||||
return fetch(input, init);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
|
||||
function mergeMarketRecord(
|
||||
existingRecord: MarketRecord | undefined,
|
||||
incomingRecord: MarketRecord
|
||||
): MarketRecord {
|
||||
if (!existingRecord) {
|
||||
return {
|
||||
...incomingRecord,
|
||||
exportFields: mergeFieldMap(undefined, incomingRecord.exportFields),
|
||||
rates: mergeFieldMap(undefined, incomingRecord.rates),
|
||||
status: incomingRecord.status ?? "idle"
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...existingRecord,
|
||||
...incomingRecord,
|
||||
authorName: mergeStringValue(existingRecord.authorName, incomingRecord.authorName) ?? "",
|
||||
exportFields: mergeFieldMap(
|
||||
existingRecord.exportFields,
|
||||
incomingRecord.exportFields
|
||||
),
|
||||
failureReason: incomingRecord.failureReason ?? existingRecord.failureReason,
|
||||
hasDirectRatesSource:
|
||||
existingRecord.hasDirectRatesSource || incomingRecord.hasDirectRatesSource,
|
||||
location: mergeStringValue(existingRecord.location, incomingRecord.location),
|
||||
price21To60s: mergeStringValue(
|
||||
existingRecord.price21To60s,
|
||||
incomingRecord.price21To60s
|
||||
),
|
||||
rates: mergeFieldMap(existingRecord.rates, incomingRecord.rates),
|
||||
status: incomingRecord.status ?? existingRecord.status
|
||||
};
|
||||
}
|
||||
|
||||
function mergeFieldMap<T extends Record<string, string | undefined>>(
|
||||
current: T | undefined,
|
||||
incoming: T | undefined
|
||||
): T | undefined {
|
||||
if (!current && !incoming) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const merged = {
|
||||
...(current ?? {})
|
||||
} as Record<string, string | undefined>;
|
||||
|
||||
Object.entries(incoming ?? {}).forEach(([key, value]) => {
|
||||
const currentValue = merged[key];
|
||||
if (hasTextValue(value) || !hasTextValue(currentValue)) {
|
||||
merged[key] = value;
|
||||
}
|
||||
});
|
||||
|
||||
return merged as T;
|
||||
}
|
||||
|
||||
function mergeStringValue(
|
||||
current: string | undefined,
|
||||
incoming: string | undefined
|
||||
): string | undefined {
|
||||
return hasTextValue(incoming) ? incoming : current;
|
||||
}
|
||||
|
||||
function hasTextValue(value: string | undefined): boolean {
|
||||
return Boolean(value && value.trim().length > 0);
|
||||
}
|
||||
+1
-1
@@ -23,7 +23,7 @@
|
||||
"https://*.xingtu.cn/ad/creator/market*"
|
||||
],
|
||||
"js": ["content/index.js"],
|
||||
"run_at": "document_idle"
|
||||
"run_at": "document_start"
|
||||
}
|
||||
],
|
||||
"web_accessible_resources": [
|
||||
|
||||
Reference in New Issue
Block a user