From e506e7d9c2fb95d9262951877e07a6949f0e04b4 Mon Sep 17 00:00:00 2001 From: renzhiye Date: Fri, 3 Apr 2026 13:58:38 +0800 Subject: [PATCH] =?UTF-8?q?feat(api):=20=E6=89=93=E9=80=9A=E5=8F=8C?= =?UTF-8?q?=E5=B9=B3=E5=8F=B0=20live=20=E6=8A=93=E5=8F=96=E4=B8=8E?= =?UTF-8?q?=E8=BF=90=E7=BB=B4=E4=BC=9A=E8=AF=9D=E4=B8=BB=E9=93=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/src/mock-data.ts | 6 +- apps/api/src/platforms/jd/keyword-preview.ts | 170 ++ .../api/src/platforms/jd/live-session.test.ts | 301 +++ apps/api/src/platforms/jd/live-session.ts | 540 ++++- .../src/platforms/jd/session-manager.test.ts | 367 ++++ apps/api/src/platforms/jd/session-manager.ts | 645 ++++++ apps/api/src/platforms/jd/types.ts | 105 +- apps/api/src/platforms/jd/utils.ts | 17 + .../src/platforms/tmall/live-session.test.ts | 356 +++ apps/api/src/platforms/tmall/live-session.ts | 705 ++++++ apps/api/src/platforms/tmall/parsers.test.ts | 94 + apps/api/src/platforms/tmall/parsers.ts | 918 ++++++++ .../platforms/tmall/session-manager.test.ts | 187 ++ .../src/platforms/tmall/session-manager.ts | 391 ++++ apps/api/src/platforms/tmall/types.ts | 182 ++ apps/api/src/platforms/tmall/utils.ts | 187 ++ apps/api/src/review-sampling.test.ts | 92 + apps/api/src/review-sampling.ts | 312 +++ .../api/src/server.dual-platform-live.test.ts | 976 +++++++++ apps/api/src/server.jd-live.test.ts | 48 +- .../api/src/server.jd-session-manager.test.ts | 341 +++ apps/api/src/server.test.ts | 1210 ++++++++++- apps/api/src/server.tmall-live.test.ts | 395 ++++ .../src/server.tmall-session-manager.test.ts | 310 +++ apps/api/src/server.ts | 538 ++++- apps/api/src/store.ts | 1901 ++++++++++++++++- 26 files changed, 11015 insertions(+), 279 deletions(-) create mode 100644 apps/api/src/platforms/jd/keyword-preview.ts create mode 100644 apps/api/src/platforms/jd/live-session.test.ts create mode 100644 apps/api/src/platforms/jd/session-manager.test.ts create mode 100644 apps/api/src/platforms/jd/session-manager.ts create mode 100644 apps/api/src/platforms/tmall/live-session.test.ts create mode 100644 apps/api/src/platforms/tmall/live-session.ts create mode 100644 apps/api/src/platforms/tmall/parsers.test.ts create mode 100644 apps/api/src/platforms/tmall/parsers.ts create mode 100644 apps/api/src/platforms/tmall/session-manager.test.ts create mode 100644 apps/api/src/platforms/tmall/session-manager.ts create mode 100644 apps/api/src/platforms/tmall/types.ts create mode 100644 apps/api/src/platforms/tmall/utils.ts create mode 100644 apps/api/src/review-sampling.test.ts create mode 100644 apps/api/src/review-sampling.ts create mode 100644 apps/api/src/server.dual-platform-live.test.ts create mode 100644 apps/api/src/server.jd-session-manager.test.ts create mode 100644 apps/api/src/server.tmall-live.test.ts create mode 100644 apps/api/src/server.tmall-session-manager.test.ts diff --git a/apps/api/src/mock-data.ts b/apps/api/src/mock-data.ts index 8bd2027..d16f70c 100644 --- a/apps/api/src/mock-data.ts +++ b/apps/api/src/mock-data.ts @@ -19,6 +19,7 @@ export function createMockCandidates( const storeName = platform === "tmall" ? "官方旗舰店" : "京东自营"; const basePrice = platform === "tmall" ? 7999 : 8099; const slug = slugify(query); + const tmallItemIds = ["833444005595", "833444005596", "833444005597"]; return Array.from({ length: 3 }, (_, index) => ({ candidateId: `${platform}-${slug}-${index + 1}`, @@ -27,7 +28,10 @@ export function createMockCandidates( price: basePrice + index * 120, priceLabel: `¥${basePrice + index * 120}`, storeName, - productUrl: `https://example.com/${platformName}/${slug}-${index + 1}`, + productUrl: + platform === "tmall" + ? `https://detail.tmall.com/item.htm?id=${tmallItemIds[index] ?? tmallItemIds[0]}` + : `https://example.com/${platformName}/${slug}-${index + 1}`, imageUrl: `https://placehold.co/640x480?text=${platformName.toUpperCase()}+${index + 1}`, salesHint: platform === "tmall" diff --git a/apps/api/src/platforms/jd/keyword-preview.ts b/apps/api/src/platforms/jd/keyword-preview.ts new file mode 100644 index 0000000..0efbbcc --- /dev/null +++ b/apps/api/src/platforms/jd/keyword-preview.ts @@ -0,0 +1,170 @@ +import type { CandidateRecord } from "@cross-ai/domain"; + +import type { JdProductPreviewResult, JdSearchMode } from "./types"; + +const NON_ALPHANUMERIC_PATTERN = /[^\p{L}\p{N}]+/gu; +const TOKEN_SPLIT_PATTERN = /[\s,，。/\\|+_:-]+/u; + +export interface RankedJdKeywordCandidate { + candidate: CandidateRecord; + skuId: string; + score: number; + summary: string; + matchedTokens: string[]; +} + +export interface JdKeywordPreviewResult { + query: string; + search: { + source: JdSearchMode; + candidateCount: number; + selected: RankedJdKeywordCandidate; + alternatives: RankedJdKeywordCandidate[]; + }; + product: JdProductPreviewResult; +} + +function normalizeText(value: string): string { + return value.toLowerCase().replace(NON_ALPHANUMERIC_PATTERN, ""); +} + +function tokenizeQuery(query: string): string[] { + const normalizedQuery = normalizeText(query); + const rawTokens = query + .split(TOKEN_SPLIT_PATTERN) + .map((token) => normalizeText(token)) + .filter(Boolean); + const uniqueTokens = new Set(); + + for (const token of rawTokens) { + if (token.length >= 2 || /[\u4e00-\u9fff]/u.test(token)) { + uniqueTokens.add(token); + } + } + + if (normalizedQuery) { + uniqueTokens.add(normalizedQuery); + } + + return [...uniqueTokens]; +} + +function extractSkuId(candidate: CandidateRecord): string | null { + const productUrlMatch = candidate.productUrl.match(/item\.jd\.com\/(\d+)\.html/i); + if (productUrlMatch?.[1]) { + return productUrlMatch[1]; + } + + const candidateIdMatch = candidate.candidateId.match(/^jd-(\d+)$/i); + return candidateIdMatch?.[1] ?? null; +} + +function isFallbackCandidate(candidate: CandidateRecord): boolean { + return ( + candidate.candidateId.startsWith("jd-fallback-") || + candidate.highlights.some((highlight) => highlight.includes("需要刷新搜索模板")) || + candidate.salesHint.includes("未解析出稳定商品卡片") + ); +} + +function buildSelectionSummary( + candidate: CandidateRecord, + matchedTokens: string[], + fullQueryMatched: boolean, + index: number, + hasSkuId: boolean +): string { + const parts: string[] = []; + + if (fullQueryMatched) { + parts.push("标题完整命中关键词"); + } else if (matchedTokens.length > 0) { + parts.push(`标题/卖点命中 ${matchedTokens.length} 个关键词片段`); + } + + if (index === 0) { + parts.push("位于搜索结果前列"); + } + + if (hasSkuId) { + parts.push("已解析出可回放 SKU"); + } + + if (candidate.storeName.includes("自营")) { + parts.push("店铺信息完整"); + } + + return parts.join("；") || "沿用搜索结果排序命中候选"; +} + +export function rankJdCandidatesForKeyword( + query: string, + candidates: CandidateRecord[] +): RankedJdKeywordCandidate[] { + const normalizedQuery = normalizeText(query); + const tokens = tokenizeQuery(query); + + return candidates + .map((candidate, index) => { + const skuId = extractSkuId(candidate); + if (!skuId || isFallbackCandidate(candidate)) { + return null; + } + + const title = normalizeText(candidate.title); + const storeName = normalizeText(candidate.storeName); + const highlights = candidate.highlights.map((item) => normalizeText(item)); + const matchedTokens = tokens.filter( + (token) => + title.includes(token) || + storeName.includes(token) || + highlights.some((highlight) => highlight.includes(token)) + ); + const fullQueryMatched = normalizedQuery.length > 0 && title.includes(normalizedQuery); + let score = 0; + + if (fullQueryMatched) { + score += 120; + } + + score += matchedTokens.reduce( + (sum, token) => sum + (token.length >= 4 || /[\u4e00-\u9fff]/u.test(token) ? 26 : 16), + 0 + ); + + if (matchedTokens.length === tokens.length && tokens.length > 1) { + score += 36; + } + + if (/item\.jd\.com\/\d+\.html/i.test(candidate.productUrl)) { + score += 8; + } + + if (candidate.price > 0) { + score += 4; + } + + if (candidate.storeName.includes("自营")) { + score += 4; + } + + score -= index * 2; + + return { + candidate, + skuId, + score, + summary: buildSelectionSummary(candidate, matchedTokens, fullQueryMatched, index, true), + matchedTokens + } satisfies RankedJdKeywordCandidate; + }) + .filter((candidate): candidate is RankedJdKeywordCandidate => Boolean(candidate)) + .sort((left, right) => right.score - left.score); +} + +export function selectJdCandidateForKeyword( + query: string, + candidates: CandidateRecord[] +): RankedJdKeywordCandidate | null { + return rankJdCandidatesForKeyword(query, candidates)[0] ?? null; +} diff --git a/apps/api/src/platforms/jd/live-session.test.ts b/apps/api/src/platforms/jd/live-session.test.ts new file mode 100644 index 0000000..bb78ebe --- /dev/null +++ b/apps/api/src/platforms/jd/live-session.test.ts @@ -0,0 +1,301 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { JdLiveSessionService } from "./live-session"; + +function buildResponse(body: Record): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { + "Content-Type": "application/json" + } + }); +} + +describe("JdLiveSessionService", () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("rebinds detail templates to the requested sku", async () => { + const fetchMock = vi.fn<(input: string, init?: RequestInit) => Promise>(async () => + buildResponse({ + skuHeadVO: { + skuTitle: "Apple iPhone 15" + }, + price: { + p: "4398.00" + } + }) + ); + vi.stubGlobal("fetch", fetchMock); + + const service = new JdLiveSessionService(); + service.importSession({ + cookieHeader: "thor=masked;", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22,%22area%22:%222_2813_61125_0%22%7D" + }); + + const preview = await service.previewDetail("100068388535"); + + expect(preview.detail).toMatchObject({ + skuId: "100068388535", + title: "Apple iPhone 15", + price: "4398.00" + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + const firstCall = fetchMock.mock.calls[0]; + expect(firstCall).toBeDefined(); + const requestUrl = firstCall?.[0]; + const requestInit = firstCall?.[1]; + if (!requestUrl || !requestInit) { + throw new Error("Expected fetch to receive both url and init."); + } + expect(decodeURIComponent(new URL(requestUrl).searchParams.get("body") ?? "")).toContain( + '"skuId":"100068388535"' + ); + expect(requestInit.headers).toMatchObject({ + Referer: "https://item.jd.com/100068388535.html" + }); + }); + + it("paginates and deduplicates JD reviews across multiple pages", async () => { + const fetchMock = vi + .fn<(input: string, init?: RequestInit) => Promise>() + .mockResolvedValueOnce( + buildResponse({ + allCnt: "10000", + goodRate: "95%", + pictureCnt: "500", + tagStatisticsinfoList: [ + { + tagId: "tag-1", + name: "拍照效果超清晰", + count: "9313" + } + ], + commentInfoList: [ + { + commentId: "comment-1", + commentData: "第一页评论一", + commentScore: 5 + }, + { + commentId: "comment-2", + commentData: "第一页评论二", + commentScore: 5 + } + ] + }) + ) + .mockResolvedValueOnce( + buildResponse({ + allCnt: "10000", + goodRate: "95%", + pictureCnt: "500", + commentInfoList: [ + { + commentId: "comment-2", + commentData: "第一页评论二", + commentScore: 5 + }, + { + commentId: "comment-3", + commentData: "第二页评论三", + commentScore: 4 + } + ] + }) + ); + vi.stubGlobal("fetch", fetchMock); + + const service = new JdLiveSessionService(); + service.importSession({ + cookieHeader: "thor=masked;", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22shopType%22:%220%22,%22sku%22:100068388533,%22commentNum%22:2,%22page%22:1,%22source%22:%22pc%22%7D" + }); + + const preview = await service.previewReviews("100068388535", { + commentCount: 2, + page: 1, + maxPages: 2 + }); + + expect(fetchMock).toHaveBeenCalledTimes(2); + const firstUrl = fetchMock.mock.calls[0]?.[0]; + const secondUrl = fetchMock.mock.calls[1]?.[0]; + if (!firstUrl || !secondUrl) { + throw new Error("Expected paged review requests to be issued."); + } + expect(decodeURIComponent(new URL(firstUrl).searchParams.get("body") ?? "")).toContain( + '"sku":100068388535' + ); + expect(decodeURIComponent(new URL(firstUrl).searchParams.get("body") ?? "")).toContain( + '"page":1' + ); + expect(decodeURIComponent(new URL(secondUrl).searchParams.get("body") ?? "")).toContain( + '"page":2' + ); + + expect(preview.pagination).toMatchObject({ + requestedPage: 1, + requestedCommentCount: 2, + maxPages: 2, + pagesFetched: 2, + pageKey: "page" + }); + expect(preview.reviews.tags).toHaveLength(1); + expect(preview.reviews.comments.map((comment) => comment.id)).toEqual([ + "comment-1", + "comment-2", + "comment-3" + ]); + }); + + it("rejects multi-page review replay when the imported template has no page field", async () => { + const fetchMock = vi.fn<(input: string, init?: RequestInit) => Promise>(); + vi.stubGlobal("fetch", fetchMock); + + const service = new JdLiveSessionService(); + service.importSession({ + cookieHeader: "thor=masked;", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22shopType%22:%220%22,%22sku%22:100068388533,%22commentNum%22:5,%22source%22:%22pc%22%7D" + }); + + await expect( + service.previewReviews("100068388535", { + commentCount: 5, + maxPages: 2 + }) + ).rejects.toMatchObject({ + message: expect.stringContaining("does not expose a page field") + }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("stops paged review replay when the next page contains no new comments", async () => { + const fetchMock = vi + .fn<(input: string, init?: RequestInit) => Promise>() + .mockResolvedValueOnce( + buildResponse({ + commentInfoList: [ + { + commentId: "comment-1", + commentData: "第一页评论一", + commentScore: 5 + }, + { + commentId: "comment-2", + commentData: "第一页评论二", + commentScore: 4 + } + ] + }) + ) + .mockResolvedValueOnce( + buildResponse({ + commentInfoList: [ + { + commentId: "comment-1", + commentData: "第一页评论一", + commentScore: 5 + }, + { + commentId: "comment-2", + commentData: "第一页评论二", + commentScore: 4 + } + ] + }) + ) + .mockResolvedValueOnce( + buildResponse({ + commentInfoList: [ + { + commentId: "comment-3", + commentData: "第三页评论三", + commentScore: 4 + } + ] + }) + ); + vi.stubGlobal("fetch", fetchMock); + + const service = new JdLiveSessionService(); + service.importSession({ + cookieHeader: "thor=masked;", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22shopType%22:%220%22,%22sku%22:100068388533,%22commentNum%22:2,%22page%22:1,%22source%22:%22pc%22%7D" + }); + + const preview = await service.previewReviews("100068388535", { + commentCount: 2, + page: 1, + maxPages: 3 + }); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(preview.pagination.pagesFetched).toBe(2); + expect(preview.reviews.comments.map((comment) => comment.id)).toEqual([ + "comment-1", + "comment-2" + ]); + }); + + it("surfaces JD verification pages as a risk-blocked error", async () => { + const fetchMock = vi.fn<(input: string, init?: RequestInit) => Promise>(async () => + new Response("请完成验证码验证后继续访问", { + status: 200, + headers: { + "Content-Type": "text/html" + } + }) + ); + vi.stubGlobal("fetch", fetchMock); + + const service = new JdLiveSessionService(); + service.importSession({ + cookieHeader: "thor=masked;", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22,%22area%22:%222_2813_61125_0%22%7D" + }); + + await expect(service.previewDetail("100068388535")).rejects.toMatchObject({ + statusCode: 423, + code: "RISK_BLOCKED" + }); + }); + + it("rejects a detail template that still resolves to another sku", async () => { + const fetchMock = vi.fn<(input: string, init?: RequestInit) => Promise>(async () => + buildResponse({ + wareInfo: { + skuId: "100068388533" + }, + skuHeadVO: { + skuTitle: "Apple iPhone 15" + }, + price: { + p: "4398.00" + } + }) + ); + vi.stubGlobal("fetch", fetchMock); + + const service = new JdLiveSessionService(); + service.importSession({ + cookieHeader: "thor=masked;", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22,%22area%22:%222_2813_61125_0%22%7D" + }); + + await expect(service.previewDetail("100068388535")).rejects.toMatchObject({ + statusCode: 409, + code: "TEMPLATE_EXPIRED", + message: expect.stringContaining("another sku") + }); + }); +}); diff --git a/apps/api/src/platforms/jd/live-session.ts b/apps/api/src/platforms/jd/live-session.ts index afcf321..40db654 100644 --- a/apps/api/src/platforms/jd/live-session.ts +++ b/apps/api/src/platforms/jd/live-session.ts @@ -7,19 +7,71 @@ import { import type { JdDetailPreviewResult, JdLiveService, + JdProductPreviewResult, JdLiveSessionInput, JdLiveSessionSummary, + JdReviewsPaginationSummary, + JdReviewsPreviewOptions, JdReviewsPreviewResult, JdSearchMode, JdSearchPreviewResult, JdTemplateSummary } from "./types"; -import { firstString, readQueryBody, withUpdatedQueryBody } from "./utils"; +import { + findFirstBodyKey, + firstString, + readQueryBody, + withUpdatedQueryBody +} from "./utils"; const DEFAULT_JD_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"; +const DETAIL_SKU_BODY_KEYS = ["skuId", "sku", "wareId", "itemSkuId"]; +const REVIEW_SKU_BODY_KEYS = ["sku", "skuId", "wareId", "itemSkuId"]; +const REVIEW_PAGE_BODY_KEYS = [ + "page", + "pageIndex", + "pageNum", + "currentPage", + "commentPage" +]; +const REVIEW_PAGE_SIZE_BODY_KEYS = ["commentNum", "pageSize", "pageLimit"]; +const SEARCH_FUNCTION_ID = "pc_search_searchWare"; +const DETAIL_FUNCTION_ID = "pc_detailpage_wareBusiness"; +const REVIEWS_FUNCTION_ID = "getLegoWareDetailComment"; +const JD_LOGIN_MARKERS = ["passport.jd.com", "请登录", "登录后可见"]; +const JD_RISK_MARKERS = [ + "验证码", + "安全验证", + "请完成验证", + "访问受限", + "异常访问", + "verify", + "captcha" +]; + +type JdLiveErrorCode = + | "INVALID_COOKIE" + | "MISSING_SESSION" + | "INVALID_TEMPLATE" + | "TEMPLATE_MISSING" + | "TEMPLATE_EXPIRED" + | "TEMPLATE_PAGE_FIELD_MISSING" + | "TEMPLATE_QUERY_LOCKED" + | "SESSION_REQUIRED" + | "RISK_BLOCKED" + | "NETWORK_ERROR" + | "HTTP_ERROR" + | "BAD_REQUEST"; + +type ResolvedJdReviewsPreviewOptions = { + commentCount: number; + page: number; + maxPages: number; +}; + type StoredJdLiveSession = { cookieHeader: string; importedAt: string; @@ -34,7 +86,8 @@ type StoredJdLiveSession = { class JdLiveError extends Error { constructor( message: string, - readonly statusCode: number = 400 + readonly statusCode: number = 400, + readonly code: JdLiveErrorCode = "BAD_REQUEST" ) { super(message); this.name = "JdLiveError"; @@ -72,7 +125,11 @@ function readEnvSession(): StoredJdLiveSession | null { function requireNonEmptyCookie(cookieHeader: string): string { const normalized = cookieHeader.trim(); if (!normalized) { - throw new JdLiveError("cookieHeader is required for JD live requests."); + throw new JdLiveError( + "cookieHeader is required for JD live requests.", + 400, + "INVALID_COOKIE" + ); } return normalized; @@ -83,9 +140,13 @@ function extractTemplateSkuId(templateUrl: string | undefined): string | undefin return undefined; } - const url = new URL(templateUrl); - const body = readQueryBody(url); - return firstString(body?.skuId, body?.sku) ?? undefined; + try { + const url = new URL(templateUrl); + const body = readQueryBody(url); + return firstString(body?.skuId, body?.sku) ?? undefined; + } catch { + return undefined; + } } function buildTemplateSummary(templateUrl: string | undefined): JdTemplateSummary { @@ -105,8 +166,226 @@ function templateMatchesQuery( return false; } - const templateKeyword = new URL(templateUrl).searchParams.get("keyword"); - return Boolean(templateKeyword && templateKeyword === query); + try { + const templateKeyword = new URL(templateUrl).searchParams.get("keyword"); + return Boolean(templateKeyword && templateKeyword === query); + } catch { + return false; + } +} + +function coerceBodyValue(existing: unknown, nextValue: number | string): number | string { + if (typeof existing === "number") { + return typeof nextValue === "number" ? nextValue : Number.parseInt(nextValue, 10); + } + + return String(nextValue); +} + +function normalizePositiveInteger( + value: number | undefined, + fieldName: string, + fallback: number, + max: number +): number { + if (value === undefined) { + return fallback; + } + + if (!Number.isInteger(value) || value < 1 || value > max) { + throw new JdLiveError(`${fieldName} must be an integer between 1 and ${max}.`); + } + + return value; +} + +function normalizeReviewsPreviewOptions( + options?: number | JdReviewsPreviewOptions +): ResolvedJdReviewsPreviewOptions { + if (typeof options === "number") { + return { + commentCount: normalizePositiveInteger(options, "commentCount", 5, 50), + page: 1, + maxPages: 1 + }; + } + + return { + commentCount: normalizePositiveInteger(options?.commentCount, "commentCount", 5, 50), + page: normalizePositiveInteger(options?.page, "page", 1, 1000), + maxPages: normalizePositiveInteger(options?.maxPages, "maxPages", 1, 10) + }; +} + +function parseTemplateUrlOrThrow( + templateUrl: string, + templateLabel: string +): URL { + try { + return new URL(templateUrl); + } catch { + throw new JdLiveError( + `${templateLabel} must be a valid URL.`, + 400, + "INVALID_TEMPLATE" + ); + } +} + +function validateTemplateFunctionId( + templateUrl: string, + templateLabel: string, + expectedFunctionId: string +): URL { + const url = parseTemplateUrlOrThrow(templateUrl, templateLabel); + const functionId = url.searchParams.get("functionId"); + + if (functionId !== expectedFunctionId) { + throw new JdLiveError( + `${templateLabel} must point to functionId=${expectedFunctionId}.`, + 400, + "INVALID_TEMPLATE" + ); + } + + return url; +} + +function validateTemplateBodyKey( + url: URL, + bodyKeys: string[], + _templateLabel: string, + errorMessage: string +): void { + const body = readQueryBody(url); + if (!findFirstBodyKey(body, bodyKeys)) { + throw new JdLiveError(errorMessage, 409, "TEMPLATE_EXPIRED"); + } +} + +function validateImportedTemplates(input: { + searchApiTemplateUrl?: string | undefined; + detailTemplateUrl?: string | undefined; + reviewsTemplateUrl?: string | undefined; +}): void { + if (input.searchApiTemplateUrl) { + validateTemplateFunctionId( + input.searchApiTemplateUrl, + "JD search API template", + SEARCH_FUNCTION_ID + ); + } + + if (input.detailTemplateUrl) { + const detailUrl = validateTemplateFunctionId( + input.detailTemplateUrl, + "JD detail template", + DETAIL_FUNCTION_ID + ); + validateTemplateBodyKey( + detailUrl, + DETAIL_SKU_BODY_KEYS, + "JD detail template", + "JD detail template does not expose a sku field. Capture a fresh pc_detailpage_wareBusiness request first." + ); + } + + if (input.reviewsTemplateUrl) { + const reviewsUrl = validateTemplateFunctionId( + input.reviewsTemplateUrl, + "JD reviews template", + REVIEWS_FUNCTION_ID + ); + validateTemplateBodyKey( + reviewsUrl, + REVIEW_SKU_BODY_KEYS, + "JD reviews template", + "JD reviews template does not expose a sku field. Capture a fresh getLegoWareDetailComment request first." + ); + } +} + +function buildDetailRequestUrl(templateUrl: string, skuId: string): string { + const template = new URL(templateUrl); + const body = readQueryBody(template); + const skuKey = findFirstBodyKey(body, DETAIL_SKU_BODY_KEYS) ?? "skuId"; + + return withUpdatedQueryBody(template, (currentBody) => ({ + ...currentBody, + [skuKey]: coerceBodyValue(currentBody[skuKey], skuId) + })); +} + +function buildReviewsRequestUrl( + templateUrl: string, + skuId: string, + options: ResolvedJdReviewsPreviewOptions, + page: number +): { + url: string; + pageKey?: string | undefined; +} { + const template = new URL(templateUrl); + const body = readQueryBody(template); + const skuKey = findFirstBodyKey(body, REVIEW_SKU_BODY_KEYS) ?? "sku"; + const pageKey = findFirstBodyKey(body, REVIEW_PAGE_BODY_KEYS) ?? undefined; + const pageSizeKey = findFirstBodyKey(body, REVIEW_PAGE_SIZE_BODY_KEYS) ?? "commentNum"; + + if ((options.maxPages > 1 || options.page > 1) && !pageKey) { + throw new JdLiveError( + "Imported reviews template does not expose a page field. Capture a paged getLegoWareDetailComment request first.", + 409, + "TEMPLATE_PAGE_FIELD_MISSING" + ); + } + + return { + url: withUpdatedQueryBody(template, (currentBody) => { + const nextBody: Record = { + ...currentBody, + [skuKey]: coerceBodyValue(currentBody[skuKey], skuId), + [pageSizeKey]: coerceBodyValue(currentBody[pageSizeKey], options.commentCount) + }; + + if (pageKey) { + nextBody[pageKey] = coerceBodyValue(currentBody[pageKey], page); + } + + return nextBody; + }), + ...(pageKey ? { pageKey } : {}) + }; +} + +function mergeReviewPages( + skuId: string, + pages: JdReviewsPreviewResult["reviews"][] +): JdReviewsPreviewResult["reviews"] { + const firstPage = pages[0]; + if (!firstPage) { + return { + skuId, + total: null, + goodRate: null, + pictureCount: null, + tags: [], + comments: [] + }; + } + + const commentsById = new Map(firstPage.comments.map((comment) => [comment.id, comment])); + for (const page of pages.slice(1)) { + for (const comment of page.comments) { + if (!commentsById.has(comment.id)) { + commentsById.set(comment.id, comment); + } + } + } + + return { + ...firstPage, + comments: Array.from(commentsById.values()) + }; } async function fetchTextOrThrow( @@ -126,19 +405,52 @@ async function fetchTextOrThrow( `JD live request failed before receiving a response: ${ error instanceof Error ? error.message : "unknown error" }`, - 502 + 502, + "NETWORK_ERROR" ); } const text = await response.text(); - if (response.url.includes("passport.jd.com") || text.includes("passport.jd.com")) { - throw new JdLiveError(sessionExpiredMessage, 409); + const normalizedResponseUrl = response.url.toLowerCase(); + const normalizedText = text.toLowerCase(); + + if ( + JD_LOGIN_MARKERS.some( + (marker) => + normalizedResponseUrl.includes(marker.toLowerCase()) || + normalizedText.includes(marker.toLowerCase()) + ) + ) { + throw new JdLiveError(sessionExpiredMessage, 409, "SESSION_REQUIRED"); + } + + if ( + JD_RISK_MARKERS.some( + (marker) => + normalizedResponseUrl.includes(marker.toLowerCase()) || + normalizedText.includes(marker.toLowerCase()) + ) + ) { + throw new JdLiveError( + "JD request hit verification or risk control. Refresh the session/template in the browser first.", + 423, + "RISK_BLOCKED" + ); } if (!response.ok) { + const errorCode: JdLiveErrorCode = + response.status === 401 + ? "SESSION_REQUIRED" + : response.status === 403 || response.status === 412 || response.status === 423 + ? "RISK_BLOCKED" + : "HTTP_ERROR"; + const errorStatus = + errorCode === "SESSION_REQUIRED" ? 409 : errorCode === "RISK_BLOCKED" ? 423 : 502; throw new JdLiveError( `JD live request failed with status ${response.status}.`, - 502 + errorStatus, + errorCode ); } @@ -148,10 +460,16 @@ async function fetchTextOrThrow( }; } -export function isJdLiveError(error: unknown): error is Error & { statusCode: number } { +export function isJdLiveError( + error: unknown +): error is Error & { statusCode: number; code?: JdLiveErrorCode } { return error instanceof Error && "statusCode" in error; } +export function getJdLiveErrorCode(error: unknown): JdLiveErrorCode | undefined { + return isJdLiveError(error) && typeof error.code === "string" ? error.code : undefined; +} + export class JdLiveSessionService implements JdLiveService { private session: StoredJdLiveSession | null = readEnvSession(); @@ -174,6 +492,12 @@ export class JdLiveSessionService implements JdLiveService { const searchReferer = input.searchReferer?.trim(); const detailReferer = input.detailReferer?.trim(); + validateImportedTemplates({ + searchApiTemplateUrl, + detailTemplateUrl, + reviewsTemplateUrl + }); + this.session = { cookieHeader: requireNonEmptyCookie(input.cookieHeader), importedAt: nowIso(), @@ -199,7 +523,7 @@ export class JdLiveSessionService implements JdLiveService { const session = this.requireSession(); const normalizedQuery = query.trim(); if (!normalizedQuery) { - throw new JdLiveError("query is required for JD live search preview."); + throw new JdLiveError("query is required for JD live search preview.", 400, "BAD_REQUEST"); } const resolvedMode = @@ -209,7 +533,9 @@ export class JdLiveSessionService implements JdLiveService { if (resolvedMode === "api") { if (!session.searchApiTemplateUrl) { throw new JdLiveError( - "JD search API template is missing. Import a fresh search request URL or use mode=html." + "JD search API template is missing. Import a fresh search request URL or use mode=html.", + 409, + "TEMPLATE_MISSING" ); } @@ -218,7 +544,9 @@ export class JdLiveSessionService implements JdLiveService { if (templateKeyword && templateKeyword !== normalizedQuery) { throw new JdLiveError( `Imported search API template is locked to query "${templateKeyword}". ` + - "Capture a fresh request for the target query or use mode=html." + "Capture a fresh request for the target query or use mode=html.", + 409, + "TEMPLATE_QUERY_LOCKED" ); } @@ -275,70 +603,18 @@ export class JdLiveSessionService implements JdLiveService { const session = this.requireSession(); const normalizedSkuId = skuId.trim(); if (!normalizedSkuId) { - throw new JdLiveError("skuId is required for JD detail preview."); + throw new JdLiveError("skuId is required for JD detail preview.", 400, "BAD_REQUEST"); } if (!session.detailTemplateUrl) { throw new JdLiveError( - "JD detail template is missing. Capture a fresh pc_detailpage_wareBusiness request and import it first." + "JD detail template is missing. Capture a fresh pc_detailpage_wareBusiness request and import it first.", + 409, + "TEMPLATE_MISSING" ); } - const templateSkuId = extractTemplateSkuId(session.detailTemplateUrl); - if (templateSkuId && templateSkuId !== normalizedSkuId) { - throw new JdLiveError( - `Imported detail template is bound to sku ${templateSkuId}. Open the matching JD item page and capture a fresh request for sku ${normalizedSkuId}.` - ); - } - - const response = await fetchTextOrThrow( - session.detailTemplateUrl, - { - headers: { - Accept: "application/json, text/plain, */*", - Cookie: session.cookieHeader, - Referer: session.detailReferer ?? `https://item.jd.com/${normalizedSkuId}.html`, - "User-Agent": session.userAgent - } - }, - "JD detail session appears invalid. Re-login in the browser and re-import the cookie/header." - ); - - return { - skuId: normalizedSkuId, - source: "api", - detail: parseJdDetailApiResponse(normalizedSkuId, { text: response.text }) - }; - } - - async previewReviews( - skuId: string, - commentCount = 5 - ): Promise { - const session = this.requireSession(); - const normalizedSkuId = skuId.trim(); - if (!normalizedSkuId) { - throw new JdLiveError("skuId is required for JD reviews preview."); - } - - if (!session.reviewsTemplateUrl) { - throw new JdLiveError( - "JD reviews template is missing. Capture a fresh getLegoWareDetailComment request and import it first." - ); - } - - const templateSkuId = extractTemplateSkuId(session.reviewsTemplateUrl); - if (templateSkuId && templateSkuId !== normalizedSkuId) { - throw new JdLiveError( - `Imported reviews template is bound to sku ${templateSkuId}. Open the matching JD item page and capture a fresh request for sku ${normalizedSkuId}.` - ); - } - - const templateUrl = new URL(session.reviewsTemplateUrl); - const requestUrl = withUpdatedQueryBody(templateUrl, (body) => ({ - ...body, - commentNum: commentCount - })); + const requestUrl = buildDetailRequestUrl(session.detailTemplateUrl, normalizedSkuId); const response = await fetchTextOrThrow( requestUrl, @@ -350,20 +626,132 @@ export class JdLiveSessionService implements JdLiveService { "User-Agent": session.userAgent } }, - "JD reviews session appears invalid. Re-login in the browser and re-import the cookie/header." + "JD detail session appears invalid. Re-login in the browser and re-import the cookie/header." ); + const detail = parseJdDetailApiResponse(normalizedSkuId, { text: response.text }); + if (detail.skuId !== normalizedSkuId) { + throw new JdLiveError( + `JD detail template appears bound to another sku (${detail.skuId}). Capture a fresh pc_detailpage_wareBusiness request first.`, + 409, + "TEMPLATE_EXPIRED" + ); + } + return { skuId: normalizedSkuId, source: "api", - reviews: parseJdReviewsApiResponse(normalizedSkuId, { text: response.text }) + detail + }; + } + + async previewReviews( + skuId: string, + options?: number | JdReviewsPreviewOptions + ): Promise { + const session = this.requireSession(); + const normalizedSkuId = skuId.trim(); + const resolvedOptions = normalizeReviewsPreviewOptions(options); + if (!normalizedSkuId) { + throw new JdLiveError("skuId is required for JD reviews preview.", 400, "BAD_REQUEST"); + } + + if (!session.reviewsTemplateUrl) { + throw new JdLiveError( + "JD reviews template is missing. Capture a fresh getLegoWareDetailComment request and import it first.", + 409, + "TEMPLATE_MISSING" + ); + } + + const reviewPages: JdReviewsPreviewResult["reviews"][] = []; + let pageKey: string | undefined; + const seenCommentIds = new Set(); + + for (let pageOffset = 0; pageOffset < resolvedOptions.maxPages; pageOffset += 1) { + const currentPage = resolvedOptions.page + pageOffset; + const request = buildReviewsRequestUrl( + session.reviewsTemplateUrl, + normalizedSkuId, + resolvedOptions, + currentPage + ); + pageKey ??= request.pageKey; + + const response = await fetchTextOrThrow( + request.url, + { + headers: { + Accept: "application/json, text/plain, */*", + Cookie: session.cookieHeader, + Referer: session.detailReferer ?? `https://item.jd.com/${normalizedSkuId}.html`, + "User-Agent": session.userAgent + } + }, + "JD reviews session appears invalid. Re-login in the browser and re-import the cookie/header." + ); + + const parsedPage = parseJdReviewsApiResponse(normalizedSkuId, { text: response.text }); + reviewPages.push(parsedPage); + let newCommentCount = 0; + for (const comment of parsedPage.comments) { + if (seenCommentIds.has(comment.id)) { + continue; + } + + seenCommentIds.add(comment.id); + newCommentCount += 1; + } + + if ( + parsedPage.comments.length === 0 || + newCommentCount === 0 || + parsedPage.comments.length < resolvedOptions.commentCount + ) { + break; + } + } + + const pagination: JdReviewsPaginationSummary = { + requestedPage: resolvedOptions.page, + requestedCommentCount: resolvedOptions.commentCount, + maxPages: resolvedOptions.maxPages, + pagesFetched: reviewPages.length, + ...(pageKey ? { pageKey } : {}) + }; + + return { + skuId: normalizedSkuId, + source: "api", + pagination, + reviews: mergeReviewPages(normalizedSkuId, reviewPages) + }; + } + + async previewProduct( + skuId: string, + options?: number | JdReviewsPreviewOptions + ): Promise { + const [detailPreview, reviewsPreview] = await Promise.all([ + this.previewDetail(skuId), + this.previewReviews(skuId, options) + ]); + + return { + skuId: detailPreview.skuId, + source: "api", + detail: detailPreview.detail, + pagination: reviewsPreview.pagination, + reviews: reviewsPreview.reviews }; } private requireSession(): StoredJdLiveSession { if (!this.session?.cookieHeader) { throw new JdLiveError( - "JD live session is not configured. Import a browser cookie/header first." + "JD live session is not configured. Import a browser cookie/header first.", + 409, + "MISSING_SESSION" ); } diff --git a/apps/api/src/platforms/jd/session-manager.test.ts b/apps/api/src/platforms/jd/session-manager.test.ts new file mode 100644 index 0000000..7af700f --- /dev/null +++ b/apps/api/src/platforms/jd/session-manager.test.ts @@ -0,0 +1,367 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { JdSessionManagerService } from "./session-manager"; +import type { + JdDetailPreviewResult, + JdLiveService, + JdLiveSessionInput, + JdLiveSessionSummary, + JdProductPreviewResult, + JdReviewsPreviewOptions, + JdReviewsPreviewResult, + JdSearchPreviewResult +} from "./types"; + +function createConfiguredSummary(): JdLiveSessionSummary { + return { + configured: true, + importedAt: "2026-04-03T08:00:00.000Z", + hasCookie: true, + userAgent: "stub-user-agent", + searchApiTemplate: { + available: true, + skuId: "100068388533" + }, + detailTemplate: { + available: true, + skuId: "100068388533" + }, + reviewsTemplate: { + available: true, + skuId: "100068388533" + } + }; +} + +function createEmptySummary(): JdLiveSessionSummary { + return { + configured: false, + hasCookie: false, + searchApiTemplate: { + available: false + }, + detailTemplate: { + available: false + }, + reviewsTemplate: { + available: false + } + }; +} + +function createJdLiveServiceStub( + overrides: Partial = {} +): JdLiveService { + let summary = createEmptySummary(); + + return { + getSessionSummary() { + return overrides.getSessionSummary?.() ?? summary; + }, + importSession(input: JdLiveSessionInput) { + if (overrides.importSession) { + return overrides.importSession(input); + } + + summary = { + ...createConfiguredSummary(), + importedAt: "2026-04-03T08:30:00.000Z", + userAgent: input.userAgent ?? "stub-user-agent", + searchApiTemplate: { + available: Boolean(input.searchApiTemplateUrl), + skuId: input.searchApiTemplateUrl ? "100068388533" : undefined + }, + detailTemplate: { + available: Boolean(input.detailTemplateUrl), + skuId: input.detailTemplateUrl ? "100068388533" : undefined + }, + reviewsTemplate: { + available: Boolean(input.reviewsTemplateUrl), + skuId: input.reviewsTemplateUrl ? "100068388533" : undefined + } + }; + + return summary; + }, + clearSession() { + if (overrides.clearSession) { + overrides.clearSession(); + return; + } + + summary = createEmptySummary(); + }, + async previewSearch(query, mode) { + if (overrides.previewSearch) { + return overrides.previewSearch(query, mode); + } + + const preview: JdSearchPreviewResult = { + query, + source: mode ?? "html", + candidateCount: 1, + candidates: [] + }; + + return preview; + }, + async previewDetail(skuId) { + if (overrides.previewDetail) { + return overrides.previewDetail(skuId); + } + + const preview: JdDetailPreviewResult = { + skuId, + source: "api", + detail: { + skuId, + title: "Apple iPhone 15", + price: "4398.00", + originalPrice: "4599.00", + estimatedPrice: "4398.00", + shopName: "JD Self Operated", + vendorId: null, + categoryPath: ["phones"], + stockState: "in stock", + mainImage: "https://img14.360buyimg.com/n2/jfs/t1/example.jpg", + averageScore: "4.9" + } + }; + + return preview; + }, + async previewReviews(skuId, options) { + if (overrides.previewReviews) { + return overrides.previewReviews(skuId, options); + } + + const preview: JdReviewsPreviewResult = { + skuId, + source: "api", + pagination: { + requestedPage: typeof options === "object" ? (options?.page ?? 1) : 1, + requestedCommentCount: + typeof options === "number" ? options : (options?.commentCount ?? 1), + maxPages: typeof options === "object" ? (options?.maxPages ?? 1) : 1, + pagesFetched: 1, + pageKey: "page" + }, + reviews: { + skuId, + total: "1000", + goodRate: "96%", + pictureCount: "120", + tags: [], + comments: [ + { + id: "comment-1", + content: "great product", + score: "5", + creationTime: "2026-04-03 08:00:00", + userLevelName: "PLUS" + } + ] + } + }; + + return preview; + }, + async previewProduct(skuId, options?: number | JdReviewsPreviewOptions) { + if (overrides.previewProduct) { + return overrides.previewProduct(skuId, options); + } + + const detail = await this.previewDetail(skuId); + const reviews = await this.previewReviews(skuId, options); + const preview: JdProductPreviewResult = { + skuId, + source: "api", + detail: detail.detail, + pagination: reviews.pagination, + reviews: reviews.reviews + }; + + return preview; + } + }; +} + +describe("JdSessionManagerService", () => { + const managers: JdSessionManagerService[] = []; + + afterEach(() => { + for (const manager of managers.splice(0)) { + manager.shutdown(); + } + + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + + it("accepts manual session imports and marks the manager healthy", async () => { + const onSessionReady = vi.fn(); + const manager = new JdSessionManagerService(createJdLiveServiceStub(), { + onSessionReady + }); + managers.push(manager); + + const state = await manager.importManualSession({ + cookieHeader: "thor=masked; pin=masked;", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22%7D", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22sku%22:100068388533%7D" + }); + + expect(onSessionReady).toHaveBeenCalledOnce(); + expect(state.status).toBe("healthy"); + expect(state.pendingManualAction).toBe(false); + expect(state.session).toMatchObject({ + configured: true, + detailTemplate: { + available: true, + skuId: "100068388533" + }, + reviewsTemplate: { + available: true, + skuId: "100068388533" + } + }); + }); + + it("passes health checks for a configured and valid session", async () => { + const previewSearch = vi.fn< + (query: string, mode?: "html" | "api") => Promise + >(async (query, mode) => ({ + query, + source: mode ?? "html", + candidateCount: 1, + candidates: [] + })); + const previewDetail = vi.fn<(skuId: string) => Promise>(async (skuId) => ({ + skuId, + source: "api", + detail: { + skuId, + title: "Apple iPhone 15", + price: "4398.00", + originalPrice: "4599.00", + estimatedPrice: "4398.00", + shopName: "JD Self Operated", + vendorId: null, + categoryPath: ["phones"], + stockState: "in stock", + mainImage: "https://img14.360buyimg.com/n2/jfs/t1/example.jpg", + averageScore: "4.9" + } + })); + const previewReviews = vi.fn< + (skuId: string, options?: number | JdReviewsPreviewOptions) => Promise + >(async (skuId, options) => ({ + skuId, + source: "api", + pagination: { + requestedPage: typeof options === "object" ? (options?.page ?? 1) : 1, + requestedCommentCount: + typeof options === "number" ? options : (options?.commentCount ?? 1), + maxPages: typeof options === "object" ? (options?.maxPages ?? 1) : 1, + pagesFetched: 1, + pageKey: "page" + }, + reviews: { + skuId, + total: "1000", + goodRate: "96%", + pictureCount: "120", + tags: [], + comments: [] + } + })); + const onSessionReady = vi.fn(); + const liveService = createJdLiveServiceStub({ + getSessionSummary: () => createConfiguredSummary(), + previewSearch, + previewDetail, + previewReviews + }); + const manager = new JdSessionManagerService(liveService, { + onSessionReady + }); + managers.push(manager); + + const result = await manager.runHealthCheck("ops"); + + expect(result.recovered).toBe(false); + expect(result.state.status).toBe("healthy"); + expect(onSessionReady).toHaveBeenCalledOnce(); + expect(previewSearch).toHaveBeenCalledWith("iPhone 15", "html"); + expect(previewDetail).toHaveBeenCalledWith("100068388533"); + expect(previewReviews).toHaveBeenCalledWith("100068388533", { + commentCount: 1, + maxPages: 1 + }); + }); + + it("auto-recovers a missing session via the configured recovery command", async () => { + const executeRecoveryCommand = vi.fn(async () => ({ + cookieHeader: "thor=masked; pin=masked;", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22%7D", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22sku%22:100068388533%7D" + })); + const onSessionReady = vi.fn(); + const manager = new JdSessionManagerService( + createJdLiveServiceStub(), + { + onSessionReady + }, + { + executeRecoveryCommand + } + ); + managers.push(manager); + + manager.configure({ + enabled: true, + autoLoginMode: "command", + loginCommand: "node scripts/jd-login-ops.mjs" + }); + + const result = await manager.runHealthCheck("ops"); + + expect(executeRecoveryCommand).toHaveBeenCalledOnce(); + expect(onSessionReady).toHaveBeenCalledOnce(); + expect(result.recovered).toBe(true); + expect(result.state.status).toBe("healthy"); + expect(result.state.session.configured).toBe(true); + }); + + it("switches to manual_action_required when JD triggers risk verification", async () => { + const riskBlockedError = Object.assign(new Error("captcha required"), { + statusCode: 423, + code: "RISK_BLOCKED" as const + }); + const onSessionUnavailable = vi.fn(); + const manager = new JdSessionManagerService( + createJdLiveServiceStub({ + getSessionSummary: () => createConfiguredSummary(), + previewSearch: async () => { + throw riskBlockedError; + } + }), + { + onSessionUnavailable + } + ); + managers.push(manager); + + const result = await manager.runHealthCheck("ops"); + + expect(result.recovered).toBe(false); + expect(result.state.status).toBe("manual_action_required"); + expect(result.state.pendingManualAction).toBe(true); + expect(result.state.lastFailureCode).toBe("RISK_BLOCKED"); + expect(onSessionUnavailable).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/api/src/platforms/jd/session-manager.ts b/apps/api/src/platforms/jd/session-manager.ts new file mode 100644 index 0000000..0bad1c4 --- /dev/null +++ b/apps/api/src/platforms/jd/session-manager.ts @@ -0,0 +1,645 @@ +import { spawn } from "node:child_process"; + +import { getJdLiveErrorCode, isJdLiveError } from "./live-session"; +import type { + JdLiveService, + JdLiveSessionInput, + JdSessionManager, + JdSessionManagerAutoMode, + JdSessionManagerConfigInput, + JdSessionManagerRunResult, + JdSessionManagerState, + JdSessionManagerStatus +} from "./types"; + +const DEFAULT_HEARTBEAT_QUERY = "iPhone 15"; +const DEFAULT_CHECK_INTERVAL_MS = 10 * 60 * 1000; +const DEFAULT_RUNNER_TIMEOUT_MS = 5 * 60 * 1000; + +type StoredJdSessionManagerConfig = { + enabled: boolean; + autoLoginMode: JdSessionManagerAutoMode; + loginCommand?: string | undefined; + browserProfilePath?: string | undefined; + heartbeatQuery: string; + account?: string | undefined; + password?: string | undefined; + checkIntervalMs: number; + runnerTimeoutMs: number; + configuredAt?: string | undefined; +}; + +type JdSessionManagerCallbacks = { + onSessionReady?: () => void; + onSessionUnavailable?: () => void; +}; + +type JdSessionManagerDependencies = { + executeRecoveryCommand?: ( + config: StoredJdSessionManagerConfig + ) => Promise; +}; + +function nowIso(): string { + return new Date().toISOString(); +} + +function readBooleanEnv(name: string, fallback: boolean): boolean { + const value = process.env[name]?.trim().toLowerCase(); + if (!value) { + return fallback; + } + + return value === "1" || value === "true" || value === "yes" || value === "on"; +} + +function readPositiveIntegerEnv(name: string, fallback: number): number { + const value = process.env[name]?.trim(); + if (!value) { + return fallback; + } + + const parsed = Number.parseInt(value, 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function normalizeOptionalString(value: string | null | undefined): string | undefined { + const normalized = value?.trim(); + return normalized ? normalized : undefined; +} + +function maskAccount(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + + if (value.length <= 4) { + return "*".repeat(value.length); + } + + return `${value.slice(0, 2)}***${value.slice(-2)}`; +} + +function buildConfigFromEnv(): StoredJdSessionManagerConfig { + const autoLoginMode = + normalizeOptionalString(process.env.JD_OPS_LOGIN_COMMAND) ? "command" : "disabled"; + + return { + enabled: readBooleanEnv("JD_OPS_AUTO_ENABLED", true), + autoLoginMode, + loginCommand: normalizeOptionalString(process.env.JD_OPS_LOGIN_COMMAND), + browserProfilePath: normalizeOptionalString(process.env.JD_OPS_BROWSER_PROFILE_DIR), + heartbeatQuery: + normalizeOptionalString(process.env.JD_OPS_HEARTBEAT_QUERY) ?? DEFAULT_HEARTBEAT_QUERY, + account: normalizeOptionalString(process.env.JD_OPS_ACCOUNT), + password: normalizeOptionalString(process.env.JD_OPS_PASSWORD), + checkIntervalMs: readPositiveIntegerEnv( + "JD_OPS_CHECK_INTERVAL_MS", + DEFAULT_CHECK_INTERVAL_MS + ), + runnerTimeoutMs: readPositiveIntegerEnv( + "JD_OPS_RUNNER_TIMEOUT_MS", + DEFAULT_RUNNER_TIMEOUT_MS + ), + configuredAt: + normalizeOptionalString(process.env.JD_OPS_LOGIN_COMMAND) || + normalizeOptionalString(process.env.JD_OPS_ACCOUNT) || + normalizeOptionalString(process.env.JD_OPS_BROWSER_PROFILE_DIR) + ? nowIso() + : undefined + }; +} + +function createBaseState( + liveService: JdLiveService, + config: StoredJdSessionManagerConfig +): JdSessionManagerState { + return { + status: liveService.getSessionSummary().configured ? "degraded" : "idle", + enabled: config.enabled, + autoLoginMode: config.autoLoginMode, + commandConfigured: Boolean(config.loginCommand), + accountConfigured: Boolean(config.account), + passwordConfigured: Boolean(config.password), + accountLabel: maskAccount(config.account), + browserProfilePath: config.browserProfilePath, + heartbeatQuery: config.heartbeatQuery, + checkIntervalMs: config.checkIntervalMs, + runnerTimeoutMs: config.runnerTimeoutMs, + pendingManualAction: false, + note: liveService.getSessionSummary().configured + ? "京东会话已导入，等待后台健康检查确认。" + : "京东会话管理器已初始化，等待首次会话注入或自动恢复。", + publicNote: liveService.getSessionSummary().configured + ? "京东会话正在后台校验。" + : "京东会话由运维后台维护，当前尚未就绪。", + configuredAt: config.configuredAt, + session: liveService.getSessionSummary() + }; +} + +function isRecoverableFailureCode(code: string | undefined): boolean { + return ( + code === "MISSING_SESSION" || + code === "SESSION_REQUIRED" || + code === "INVALID_COOKIE" || + code === "TEMPLATE_MISSING" || + code === "TEMPLATE_EXPIRED" || + code === "TEMPLATE_PAGE_FIELD_MISSING" || + code === "TEMPLATE_QUERY_LOCKED" || + code === "INVALID_TEMPLATE" || + code === "RISK_BLOCKED" + ); +} + +export class JdSessionManagerService implements JdSessionManager { + private config = buildConfigFromEnv(); + private state: JdSessionManagerState; + private timer: NodeJS.Timeout | null = null; + private recoveryInFlight: Promise | null = null; + + constructor( + private readonly liveService: JdLiveService, + private readonly callbacks: JdSessionManagerCallbacks = {}, + private readonly dependencies: JdSessionManagerDependencies = {} + ) { + this.state = createBaseState(liveService, this.config); + this.restartScheduler(); + } + + getState(): JdSessionManagerState { + return { + ...this.state, + session: this.liveService.getSessionSummary() + }; + } + + configure(input: JdSessionManagerConfigInput): JdSessionManagerState { + const nextConfig: StoredJdSessionManagerConfig = { + ...this.config, + ...(typeof input.enabled === "boolean" ? { enabled: input.enabled } : {}), + ...(input.autoLoginMode ? { autoLoginMode: input.autoLoginMode } : {}), + ...(input.loginCommand !== undefined + ? { loginCommand: normalizeOptionalString(input.loginCommand) } + : {}), + ...(input.browserProfilePath !== undefined + ? { browserProfilePath: normalizeOptionalString(input.browserProfilePath) } + : {}), + ...(input.heartbeatQuery !== undefined + ? { + heartbeatQuery: + normalizeOptionalString(input.heartbeatQuery) ?? DEFAULT_HEARTBEAT_QUERY + } + : {}), + ...(input.account !== undefined ? { account: normalizeOptionalString(input.account) } : {}), + ...(input.password !== undefined + ? { password: normalizeOptionalString(input.password) } + : {}), + ...(input.checkIntervalMs !== undefined && input.checkIntervalMs !== null + ? { + checkIntervalMs: Number.isInteger(input.checkIntervalMs) && input.checkIntervalMs > 0 + ? input.checkIntervalMs + : this.config.checkIntervalMs + } + : {}), + ...(input.runnerTimeoutMs !== undefined && input.runnerTimeoutMs !== null + ? { + runnerTimeoutMs: + Number.isInteger(input.runnerTimeoutMs) && input.runnerTimeoutMs > 0 + ? input.runnerTimeoutMs + : this.config.runnerTimeoutMs + } + : {}), + configuredAt: nowIso() + }; + + if (!nextConfig.loginCommand) { + nextConfig.autoLoginMode = "disabled"; + } + + this.config = nextConfig; + this.state = { + ...this.state, + enabled: nextConfig.enabled, + autoLoginMode: nextConfig.autoLoginMode, + commandConfigured: Boolean(nextConfig.loginCommand), + accountConfigured: Boolean(nextConfig.account), + passwordConfigured: Boolean(nextConfig.password), + accountLabel: maskAccount(nextConfig.account), + browserProfilePath: nextConfig.browserProfilePath, + heartbeatQuery: nextConfig.heartbeatQuery, + checkIntervalMs: nextConfig.checkIntervalMs, + runnerTimeoutMs: nextConfig.runnerTimeoutMs, + configuredAt: nextConfig.configuredAt, + note: "京东运维配置已更新。", + publicNote: this.state.publicNote, + session: this.liveService.getSessionSummary() + }; + this.restartScheduler(); + return this.getState(); + } + + clearConfig(): JdSessionManagerState { + this.config = { + enabled: false, + autoLoginMode: "disabled", + heartbeatQuery: DEFAULT_HEARTBEAT_QUERY, + checkIntervalMs: DEFAULT_CHECK_INTERVAL_MS, + runnerTimeoutMs: DEFAULT_RUNNER_TIMEOUT_MS + }; + this.state = { + ...this.state, + enabled: false, + autoLoginMode: "disabled", + commandConfigured: false, + accountConfigured: false, + passwordConfigured: false, + accountLabel: undefined, + browserProfilePath: undefined, + heartbeatQuery: DEFAULT_HEARTBEAT_QUERY, + checkIntervalMs: DEFAULT_CHECK_INTERVAL_MS, + runnerTimeoutMs: DEFAULT_RUNNER_TIMEOUT_MS, + configuredAt: nowIso(), + note: "京东运维配置已清空，自动恢复已停用。", + publicNote: this.liveService.getSessionSummary().configured + ? "京东会话已存在，但后台自动恢复已停用。" + : "京东会话由运维后台维护，当前自动恢复未启用。", + pendingManualAction: false, + session: this.liveService.getSessionSummary() + }; + this.restartScheduler(); + return this.getState(); + } + + async importManualSession( + input: JdLiveSessionInput, + source = "ops-manual" + ): Promise { + this.liveService.importSession(input); + this.callbacks.onSessionReady?.(); + this.state = { + ...this.state, + status: "healthy", + pendingManualAction: false, + note: `京东会话已通过 ${source} 更新，等待下一轮健康检查。`, + publicNote: "京东会话由运维后台维护，当前可用。", + lastRecoveredAt: nowIso(), + lastHealthyAt: nowIso(), + lastFailureCode: undefined, + lastFailureMessage: undefined, + session: this.liveService.getSessionSummary() + }; + return this.getState(); + } + + clearManagedSession(reason = "ops-manual-clear"): JdSessionManagerState { + this.liveService.clearSession(); + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: "idle", + pendingManualAction: false, + note: `京东会话已清理：${reason}。`, + publicNote: "京东会话由运维后台维护，当前尚未就绪。", + session: this.liveService.getSessionSummary() + }; + return this.getState(); + } + + async runHealthCheck(trigger = "manual"): Promise { + this.state = { + ...this.state, + lastCheckAt: nowIso(), + session: this.liveService.getSessionSummary() + }; + + const summary = this.liveService.getSessionSummary(); + if (!summary.configured) { + if (this.canAutoRecover()) { + return this.runAutoRecovery(`${trigger}:missing_session`); + } + + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: "manual_action_required", + pendingManualAction: true, + note: "当前没有可用京东会话，等待运维注入或自动恢复命令。", + publicNote: "京东会话由运维后台维护，当前尚未就绪。", + session: summary + }; + return { + state: this.getState(), + recovered: false + }; + } + + if (!summary.detailTemplate.available || !summary.reviewsTemplate.available) { + if (this.canAutoRecover()) { + return this.runAutoRecovery(`${trigger}:template_missing`); + } + + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: "manual_action_required", + pendingManualAction: true, + note: "京东详情或评论模板缺失，需要运维刷新模板。", + publicNote: "京东会话缺少有效模板，运维后台正在处理。", + session: summary + }; + return { + state: this.getState(), + recovered: false + }; + } + + try { + await this.verifyCurrentSession(); + this.callbacks.onSessionReady?.(); + this.state = { + ...this.state, + status: "healthy", + pendingManualAction: false, + note: `京东会话健康检查通过：${trigger}。`, + publicNote: "京东会话由运维后台维护，当前可用。", + lastHealthyAt: nowIso(), + lastFailureCode: undefined, + lastFailureMessage: undefined, + session: this.liveService.getSessionSummary() + }; + return { + state: this.getState(), + recovered: false + }; + } catch (error) { + const recovered = await this.handleFailure(error, `${trigger}:health_check`); + return { + state: this.getState(), + recovered + }; + } + } + + async runAutoRecovery(trigger = "manual"): Promise { + const recovered = await this.attemptAutoRecovery(trigger); + return { + state: this.getState(), + recovered + }; + } + + async handleLiveFailure( + error: unknown, + context: { + capability?: "search" | "detail" | "reviews"; + taskId?: string | undefined; + trigger?: string | undefined; + } = {} + ): Promise { + const contextLabel = [ + context.trigger ?? "system", + context.capability ?? "unknown", + context.taskId ?? "no-task" + ].join(":"); + return this.handleFailure(error, contextLabel); + } + + shutdown(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + } + + private restartScheduler(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + + if (!this.config.enabled) { + return; + } + + this.timer = setInterval(() => { + void this.runHealthCheck("scheduler"); + }, this.config.checkIntervalMs); + this.timer.unref?.(); + } + + private canAutoRecover(): boolean { + return ( + this.config.enabled && + this.config.autoLoginMode === "command" && + Boolean(this.config.loginCommand) + ); + } + + private async handleFailure(error: unknown, trigger: string): Promise { + const code = getJdLiveErrorCode(error); + const message = error instanceof Error ? error.message : "unknown error"; + + this.state = { + ...this.state, + status: isRecoverableFailureCode(code) ? "degraded" : "manual_action_required", + pendingManualAction: code === "RISK_BLOCKED" || !this.canAutoRecover(), + lastFailureCode: code, + lastFailureMessage: message, + note: `京东会话检测失败：${message}`, + publicNote: + code === "RISK_BLOCKED" + ? "京东触发了风控验证，运维后台需要人工恢复。" + : "京东会话正在后台恢复，请稍后自动重试。", + session: this.liveService.getSessionSummary() + }; + + if (!isRecoverableFailureCode(code)) { + this.callbacks.onSessionUnavailable?.(); + return false; + } + + if (code === "RISK_BLOCKED" && !this.canAutoRecover()) { + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: "manual_action_required", + pendingManualAction: true, + note: "京东触发风控验证，需要运维人工处理。", + publicNote: "京东触发了风控验证，运维后台需要人工恢复。" + }; + return false; + } + + return this.attemptAutoRecovery(trigger); + } + + private async attemptAutoRecovery(trigger: string): Promise { + if (!this.canAutoRecover()) { + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: "manual_action_required", + pendingManualAction: true, + note: "未配置自动恢复命令，等待运维人工更新京东会话。", + publicNote: "京东会话需要运维人工恢复，系统会在恢复后自动继续。" + }; + return false; + } + + if (this.recoveryInFlight) { + return this.recoveryInFlight; + } + + this.recoveryInFlight = this.runRecoveryCommand(trigger).finally(() => { + this.recoveryInFlight = null; + }); + return this.recoveryInFlight; + } + + private async runRecoveryCommand(trigger: string): Promise { + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: "recovering", + pendingManualAction: false, + lastRecoverAttemptAt: nowIso(), + note: `正在执行京东自动恢复：${trigger}`, + publicNote: "京东会话正在后台恢复，请稍后自动重试。", + session: this.liveService.getSessionSummary() + }; + + try { + const payload = await this.executeRecoveryCommand(); + this.liveService.importSession(payload); + await this.verifyCurrentSession(); + this.callbacks.onSessionReady?.(); + this.state = { + ...this.state, + status: "healthy", + pendingManualAction: false, + note: `京东自动恢复成功：${trigger}`, + publicNote: "京东会话由运维后台维护，当前可用。", + lastRecoveredAt: nowIso(), + lastHealthyAt: nowIso(), + lastFailureCode: undefined, + lastFailureMessage: undefined, + session: this.liveService.getSessionSummary() + }; + return true; + } catch (error) { + const message = error instanceof Error ? error.message : "unknown error"; + const code = + getJdLiveErrorCode(error) ?? + (isJdLiveError(error) ? String(error.statusCode) : "OPS_RECOVERY_FAILED"); + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: "manual_action_required", + pendingManualAction: true, + note: `京东自动恢复失败：${message}`, + publicNote: "京东会话需要运维人工恢复，系统会在恢复后自动继续。", + lastFailureCode: code, + lastFailureMessage: message, + session: this.liveService.getSessionSummary() + }; + return false; + } + } + + private async verifyCurrentSession(): Promise { + const summary = this.liveService.getSessionSummary(); + const detailSkuId = summary.detailTemplate.skuId; + const reviewsSkuId = summary.reviewsTemplate.skuId ?? detailSkuId; + + if (!summary.configured || !detailSkuId || !reviewsSkuId) { + throw new Error("京东会话缺少可校验的详情/评论模板。"); + } + + await this.liveService.previewSearch(this.config.heartbeatQuery, "html"); + await this.liveService.previewDetail(detailSkuId); + await this.liveService.previewReviews(reviewsSkuId, { + commentCount: 1, + maxPages: 1 + }); + } + + private executeRecoveryCommand(): Promise { + if (this.dependencies.executeRecoveryCommand) { + return this.dependencies.executeRecoveryCommand(this.config); + } + + return new Promise((resolve, reject) => { + const command = this.config.loginCommand; + if (!command) { + reject(new Error("JD ops login command is not configured.")); + return; + } + + const child = spawn(command, { + cwd: process.cwd(), + env: { + ...process.env, + JD_OPS_ACCOUNT: this.config.account ?? "", + JD_OPS_PASSWORD: this.config.password ?? "", + JD_OPS_BROWSER_PROFILE_DIR: this.config.browserProfilePath ?? "", + JD_OPS_HEARTBEAT_QUERY: this.config.heartbeatQuery + }, + shell: true, + stdio: ["ignore", "pipe", "pipe"] + }); + + let stdout = ""; + let stderr = ""; + const timeout = setTimeout(() => { + child.kill(); + reject( + new Error( + `JD ops login command timed out after ${this.config.runnerTimeoutMs}ms.` + ) + ); + }, this.config.runnerTimeoutMs); + + child.stdout.on("data", (chunk) => { + stdout += String(chunk); + }); + child.stderr.on("data", (chunk) => { + stderr += String(chunk); + }); + + child.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + + child.on("close", (code) => { + clearTimeout(timeout); + if (code !== 0) { + reject( + new Error( + `JD ops login command failed with code ${code}. ${stderr.trim() || stdout.trim()}` + ) + ); + return; + } + + const trimmed = stdout.trim(); + if (!trimmed) { + reject(new Error("JD ops login command returned an empty payload.")); + return; + } + + try { + const payload = JSON.parse(trimmed) as JdLiveSessionInput; + resolve(payload); + } catch { + reject( + new Error( + `JD ops login command did not return valid JSON. Output: ${trimmed.slice(-500)}` + ) + ); + } + }); + }); + } +} diff --git a/apps/api/src/platforms/jd/types.ts b/apps/api/src/platforms/jd/types.ts index aa77fa4..e50d33d 100644 --- a/apps/api/src/platforms/jd/types.ts +++ b/apps/api/src/platforms/jd/types.ts @@ -27,6 +27,57 @@ export interface JdLiveSessionSummary { reviewsTemplate: JdTemplateSummary; } +export type JdSessionManagerStatus = + | "idle" + | "healthy" + | "degraded" + | "recovering" + | "manual_action_required"; + +export type JdSessionManagerAutoMode = "disabled" | "command"; + +export interface JdSessionManagerConfigInput { + enabled?: boolean | undefined; + autoLoginMode?: JdSessionManagerAutoMode | undefined; + loginCommand?: string | null | undefined; + browserProfilePath?: string | null | undefined; + heartbeatQuery?: string | null | undefined; + account?: string | null | undefined; + password?: string | null | undefined; + checkIntervalMs?: number | null | undefined; + runnerTimeoutMs?: number | null | undefined; +} + +export interface JdSessionManagerState { + status: JdSessionManagerStatus; + enabled: boolean; + autoLoginMode: JdSessionManagerAutoMode; + commandConfigured: boolean; + accountConfigured: boolean; + passwordConfigured: boolean; + accountLabel?: string | undefined; + browserProfilePath?: string | undefined; + heartbeatQuery: string; + checkIntervalMs: number; + runnerTimeoutMs: number; + pendingManualAction: boolean; + note: string; + publicNote: string; + configuredAt?: string | undefined; + lastCheckAt?: string | undefined; + lastHealthyAt?: string | undefined; + lastRecoverAttemptAt?: string | undefined; + lastRecoveredAt?: string | undefined; + lastFailureCode?: string | undefined; + lastFailureMessage?: string | undefined; + session: JdLiveSessionSummary; +} + +export interface JdSessionManagerRunResult { + state: JdSessionManagerState; + recovered: boolean; +} + export interface JdSearchPreviewResult { query: string; source: JdSearchMode; @@ -71,6 +122,20 @@ export interface JdProductReviewsSnapshot { comments: JdReviewCommentSnapshot[]; } +export interface JdReviewsPreviewOptions { + commentCount?: number | undefined; + page?: number | undefined; + maxPages?: number | undefined; +} + +export interface JdReviewsPaginationSummary { + requestedPage: number; + requestedCommentCount: number; + maxPages: number; + pagesFetched: number; + pageKey?: string | undefined; +} + export interface JdDetailPreviewResult { skuId: string; source: "api"; @@ -80,6 +145,15 @@ export interface JdDetailPreviewResult { export interface JdReviewsPreviewResult { skuId: string; source: "api"; + pagination: JdReviewsPaginationSummary; + reviews: JdProductReviewsSnapshot; +} + +export interface JdProductPreviewResult { + skuId: string; + source: "api"; + detail: JdProductDetailSnapshot; + pagination: JdReviewsPaginationSummary; reviews: JdProductReviewsSnapshot; } @@ -89,5 +163,34 @@ export interface JdLiveService { clearSession(): void; previewSearch(query: string, mode?: JdSearchMode): Promise; previewDetail(skuId: string): Promise; - previewReviews(skuId: string, commentCount?: number): Promise; + previewReviews( + skuId: string, + options?: number | JdReviewsPreviewOptions + ): Promise; + previewProduct( + skuId: string, + options?: number | JdReviewsPreviewOptions + ): Promise; +} + +export interface JdSessionManager { + getState(): JdSessionManagerState; + configure(input: JdSessionManagerConfigInput): JdSessionManagerState; + clearConfig(): JdSessionManagerState; + importManualSession( + input: JdLiveSessionInput, + source?: string + ): Promise; + clearManagedSession(reason?: string): JdSessionManagerState; + runHealthCheck(trigger?: string): Promise; + runAutoRecovery(trigger?: string): Promise; + handleLiveFailure( + error: unknown, + context?: { + capability?: "search" | "detail" | "reviews"; + taskId?: string | undefined; + trigger?: string | undefined; + } + ): Promise; + shutdown(): void; } diff --git a/apps/api/src/platforms/jd/utils.ts b/apps/api/src/platforms/jd/utils.ts index dbaa242..cce1c7a 100644 --- a/apps/api/src/platforms/jd/utils.ts +++ b/apps/api/src/platforms/jd/utils.ts @@ -103,6 +103,23 @@ export function readQueryBody(url: URL): Record | null { return parseEmbeddedJson(url.searchParams.get("body")); } +export function findFirstBodyKey( + body: Record | null, + candidates: string[] +): string | null { + if (!body) { + return null; + } + + for (const candidate of candidates) { + if (candidate in body) { + return candidate; + } + } + + return null; +} + export function withUpdatedQueryBody( url: URL, updater: (body: Record) => Record diff --git a/apps/api/src/platforms/tmall/live-session.test.ts b/apps/api/src/platforms/tmall/live-session.test.ts new file mode 100644 index 0000000..b3ce67a --- /dev/null +++ b/apps/api/src/platforms/tmall/live-session.test.ts @@ -0,0 +1,356 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { TmallLiveSessionService } from "./live-session"; + +function buildResponse(body: Record): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { + "Content-Type": "application/json" + } + }); +} + +function buildHtmlResponse(body: string): Response { + return new Response(body, { + status: 200, + headers: { + "Content-Type": "text/html" + } + }); +} + +function buildTextResponse(body: string): Response { + return new Response(body, { + status: 200, + headers: { + "Content-Type": "application/javascript" + } + }); +} + +describe("TmallLiveSessionService", () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("fetches the mall search page and parses candidates from embedded state", async () => { + const html = ``; + const fetchMock = vi.fn<(input: string, init?: RequestInit) => Promise>(async () => + buildHtmlResponse(html) + ); + vi.stubGlobal("fetch", fetchMock); + + const service = new TmallLiveSessionService(); + service.importSession({ + cookieHeader: "_tb_token_=masked;" + }); + + const preview = await service.previewSearch("iPhone 15"); + + expect(preview).toMatchObject({ + query: "iPhone 15", + source: "html", + candidateCount: 1, + candidates: [ + expect.objectContaining({ + candidateId: "tmall-934454505228", + productUrl: "https://detail.tmall.com/item.htm?id=934454505228" + }) + ] + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0]?.[0]).toBe( + "https://s.taobao.com/search?q=iPhone%2015&tab=mall" + ); + expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({ + headers: { + Referer: "https://www.tmall.com/" + } + }); + }); + + it("parses detail from the embedded Tmall SSR page state", async () => { + const pageState = { + appData: null, + loaderData: { + home: { + data: { + res: { + seller: { + shopName: "Apple Flagship Store", + pcShopUrl: "//apple.tmall.com", + sellerType: "B" + }, + item: { + itemId: "833444005596", + title: "Apple iPhone 15", + images: ["//img.alicdn.com/example.jpg"], + vagueSellCount: "sold 10k+" + }, + componentsVO: { + titleVO: { + title: { + title: "Apple iPhone 15" + }, + salesDesc: "sold 10k+" + }, + priceVO: { + extraPrice: { + priceText: "4399.00" + }, + price: { + priceText: "4999.00" + } + }, + rateVO: { + totalCount: "20k+" + }, + headImageVO: { + images: ["//img.alicdn.com/example.jpg"] + } + } + } + } + } + } + }; + const html = [ + "detail", + "", + "" + ].join(""); + const fetchMock = vi.fn<(input: string, init?: RequestInit) => Promise>(async () => + buildHtmlResponse(html) + ); + vi.stubGlobal("fetch", fetchMock); + + const service = new TmallLiveSessionService(); + service.importSession({ + cookieHeader: "_tb_token_=masked;" + }); + + const preview = await service.previewDetail("833444005596"); + + expect(preview.detail).toMatchObject({ + itemId: "833444005596", + title: "Apple iPhone 15", + shopName: "Apple Flagship Store", + price: "4399.00", + originalPrice: "4999.00", + salesDesc: "sold 10k+", + commentCount: "20k+", + sellerType: "tmall" + }); + expect(preview.source).toBe("html"); + expect(fetchMock).toHaveBeenCalledTimes(1); + const requestUrl = fetchMock.mock.calls[0]?.[0]; + const requestInit = fetchMock.mock.calls[0]?.[1]; + if (!requestUrl || !requestInit) { + throw new Error("Expected fetch to receive both url and init."); + } + + expect(requestUrl).toBe("https://detail.tmall.com/item.htm?id=833444005596"); + expect(requestInit.headers).toMatchObject({ + Referer: "https://detail.tmall.com/" + }); + }); + + it("paginates and deduplicates Tmall PC reviews across multiple pages", async () => { + vi.spyOn(Date, "now").mockReturnValue(1775186257185); + const fetchMock = vi + .fn<(input: string, init?: RequestInit) => Promise>() + .mockResolvedValueOnce( + buildResponse({ + ret: ["SUCCESS::ok"], + data: { + module: { + hasNext: "true", + reviewVOList: [ + { + id: "review-1", + userNick: "Alice", + reviewDate: "2026-04-03", + reviewWordContent: "First review", + reviewPicPathList: ["//img.alicdn.com/review-1.jpg"] + }, + { + id: "review-2", + userNick: "Bob", + reviewDate: "2026-04-03", + reviewWordContent: "Second review" + } + ] + } + } + }) + ) + .mockResolvedValueOnce( + buildResponse({ + ret: ["SUCCESS::ok"], + data: { + module: { + hasNext: "false", + reviewVOList: [ + { + id: "review-2", + userNick: "Bob", + reviewDate: "2026-04-03", + reviewWordContent: "Second review" + }, + { + id: "review-3", + userNick: "Cathy", + reviewDate: "2026-04-04", + reviewWordContent: "Third review" + } + ] + } + } + }) + ); + vi.stubGlobal("fetch", fetchMock); + + const service = new TmallLiveSessionService(); + service.importSession({ + cookieHeader: + "_tb_token_=masked; _m_h5_tk=fcce0507fd586e928c94a9d54fec6a5c_1775194955427;", + reviewsTemplateUrl: + "https://h5api.m.taobao.com/h5/mtop.alibaba.review.list.for.new.pc.detail/1.0/?" + + "api=mtop.alibaba.review.list.for.new.pc.detail&v=1.0&" + + "data=%7B%22itemId%22%3A%22833444005595%22%2C%22bizCode%22%3A%22ali.china.tmall%22%2C%22channel%22%3A%22pc_detail%22%2C%22pageNum%22%3A1%2C%22pageSize%22%3A2%7D" + }); + + const preview = await service.previewReviews("833444005596", { + commentCount: 2, + page: 1, + maxPages: 2 + }); + + expect(fetchMock).toHaveBeenCalledTimes(2); + const firstUrl = fetchMock.mock.calls[0]?.[0]; + const secondUrl = fetchMock.mock.calls[1]?.[0]; + if (!firstUrl || !secondUrl) { + throw new Error("Expected paged reviews requests to be issued."); + } + + const firstData = JSON.parse( + decodeURIComponent(new URL(firstUrl).searchParams.get("data") ?? "") + ) as Record; + const secondData = JSON.parse( + decodeURIComponent(new URL(secondUrl).searchParams.get("data") ?? "") + ) as Record; + + expect(firstData.itemId).toBe("833444005596"); + expect(firstData.pageNum).toBe(1); + expect(secondData.pageNum).toBe(2); + expect(new URL(firstUrl).searchParams.get("t")).toBe("1775186257185"); + expect(new URL(firstUrl).searchParams.get("sign")).toBe( + "b8fa133d502fd87c9d5bc9dbf95032d7" + ); + expect(new URL(secondUrl).searchParams.get("sign")).toBe( + "8eda756f901c9f588209b0498e898928" + ); + + expect(preview.pagination).toMatchObject({ + requestedPage: 1, + requestedCommentCount: 2, + maxPages: 2, + pagesFetched: 2, + pageKey: "pageNum" + }); + expect(preview.reviews.comments.map((comment) => comment.id)).toEqual([ + "review-1", + "review-2", + "review-3" + ]); + }); + + it("parses H5 review JSONP responses from mtop.taobao.rate.detaillist.get", async () => { + const fetchMock = vi.fn<(input: string, init?: RequestInit) => Promise>(async () => + buildTextResponse( + 'mtopjsonp13({"api":"mtop.taobao.rate.detaillist.get","data":{"hasNext":"true","fuzzyRateCount":"5万+","feedPicCountFuzzy":"7000+","feedAppendCountFuzzy":"800+","imprItemVOS":[{"title":"发货速度快","count":"139"}],"rateList":[{"id":"1300870633402","feedback":"挺好的，外观高级","feedbackDate":"2026年3月27日","userNick":"可**月","headPicUrl":"//sns.m.taobao.com/avatar/example","skuMap":{"颜色分类":"小米手环10 银色"},"feedPicPathList":["//img.alicdn.com/review-1.jpg"],"interactInfo":{"likeCount":"1"},"reply":"感谢您的认可"}]}})' + ) + ); + vi.stubGlobal("fetch", fetchMock); + + const service = new TmallLiveSessionService(); + service.importSession({ + cookieHeader: + "_tb_token_=masked; _m_h5_tk=41fb9bae259f3b525fa0be5bfa989739_1775196962107;", + reviewsTemplateUrl: + "https://h5api.m.tmall.com/h5/mtop.taobao.rate.detaillist.get/6.0/?" + + "api=mtop.taobao.rate.detaillist.get&v=6.0&" + + "data=%7B%22auctionNumId%22%3A%22934454505228%22%2C%22pageNo%22%3A1%2C%22pageSize%22%3A20%7D" + }); + + const preview = await service.previewReviews("934454505228"); + + expect(preview.reviews.tags).toEqual([{ name: "发货速度快", count: "139" }]); + expect(preview.reviews.comments).toHaveLength(1); + expect(preview.reviews.comments[0]).toMatchObject({ + id: "1300870633402", + content: "挺好的，外观高级", + date: "2026年3月27日", + userNick: "可**月", + likeCount: "1", + reply: "感谢您的认可" + }); + expect(preview.reviews.comments[0]?.pictureUrls).toEqual([ + "https://img.alicdn.com/review-1.jpg" + ]); + expect(preview.reviews.comments[0]?.skuText).toEqual(["小米手环10 银色"]); + }); + + it("rejects review replay with a login redirect response", async () => { + const fetchMock = vi.fn<(input: string, init?: RequestInit) => Promise>(async () => + buildResponse({ + ret: ["RGV587_ERROR::SM::session required"], + data: { + url: "https://login.taobao.com/member/login.jhtml?redirectURL=masked" + } + }) + ); + vi.stubGlobal("fetch", fetchMock); + + const service = new TmallLiveSessionService(); + service.importSession({ + cookieHeader: + "_tb_token_=masked; _m_h5_tk=fcce0507fd586e928c94a9d54fec6a5c_1775194955427;", + reviewsTemplateUrl: + "https://h5api.m.taobao.com/h5/mtop.taobao.rate.detaillist.get/6.0/?" + + "api=mtop.taobao.rate.detaillist.get&v=6.0&data=%7B%22auctionNumId%22%3A%22833444005595%22%2C%22pageNo%22%3A1%2C%22pageSize%22%3A20%7D" + }); + + await expect(service.previewReviews("833444005596")).rejects.toMatchObject({ + statusCode: 409, + message: expect.stringContaining("Re-login") + }); + }); +}); diff --git a/apps/api/src/platforms/tmall/live-session.ts b/apps/api/src/platforms/tmall/live-session.ts new file mode 100644 index 0000000..d75f76a --- /dev/null +++ b/apps/api/src/platforms/tmall/live-session.ts @@ -0,0 +1,705 @@ +import { createHash } from "node:crypto"; + +import { + extractTmallRetCodes, + hasTmallSearchNoResultMarker, + parseTmallDetailApiResponse, + parseTmallDetailHtmlResponse, + parseTmallSearchHtml, + parseTmallReviewsApiResponse +} from "./parsers"; +import type { + TmallDetailPreviewResult, + TmallLiveService, + TmallLiveSessionInput, + TmallLiveSessionSummary, + TmallProductPreviewResult, + TmallReviewsPaginationSummary, + TmallReviewsPreviewOptions, + TmallReviewsPreviewResult, + TmallSearchPreviewResult, + TmallTemplateSummary +} from "./types"; +import { + asRecord, + findFirstRecordKey, + firstString, + parseEmbeddedJson, + readNestedJsonRecord, + readQueryData, + stringFrom, + withUpdatedQueryData +} from "./utils"; + +const DEFAULT_TMALL_USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"; + +const DETAIL_ITEM_DATA_KEYS = ["id", "itemId", "itemNumId"]; +const REVIEW_PC_ITEM_DATA_KEYS = ["itemId", "id"]; +const REVIEW_H5_ITEM_DATA_KEYS = ["auctionNumId", "itemId", "id"]; +const REVIEW_PAGE_DATA_KEYS = ["pageNum", "pageNo", "page", "currentPage"]; +const REVIEW_PAGE_SIZE_DATA_KEYS = ["pageSize", "pageLimit", "commentSize"]; +const DEFAULT_LOGIC_VERSION = "2025031302"; + +type ResolvedTmallReviewsPreviewOptions = { + commentCount: number; + page: number; + maxPages: number; +}; + +type StoredTmallLiveSession = { + cookieHeader: string; + importedAt: string; + userAgent: string; + detailTemplateUrl?: string | undefined; + reviewsTemplateUrl?: string | undefined; + detailReferer?: string | undefined; +}; + +class TmallLiveError extends Error { + constructor( + message: string, + readonly statusCode: number = 400 + ) { + super(message); + this.name = "TmallLiveError"; + } +} + +function nowIso(): string { + return new Date().toISOString(); +} + +function readEnvSession(): StoredTmallLiveSession | null { + const cookieHeader = process.env.TMALL_COOKIE_HEADER?.trim(); + if (!cookieHeader) { + return null; + } + + const detailTemplateUrl = process.env.TMALL_DETAIL_TEMPLATE_URL?.trim(); + const reviewsTemplateUrl = process.env.TMALL_REVIEWS_TEMPLATE_URL?.trim(); + const detailReferer = process.env.TMALL_DETAIL_REFERER?.trim(); + + return { + cookieHeader, + importedAt: nowIso(), + userAgent: process.env.TMALL_USER_AGENT?.trim() || DEFAULT_TMALL_USER_AGENT, + ...(detailTemplateUrl ? { detailTemplateUrl } : {}), + ...(reviewsTemplateUrl ? { reviewsTemplateUrl } : {}), + ...(detailReferer ? { detailReferer } : {}) + }; +} + +function requireNonEmptyCookie(cookieHeader: string): string { + const normalized = cookieHeader.trim(); + if (!normalized) { + throw new TmallLiveError("cookieHeader is required for Tmall live requests."); + } + + return normalized; +} + +function extractCookieValue(cookieHeader: string, name: string): string | null { + let matched: string | null = null; + + for (const part of cookieHeader.split(";")) { + const trimmed = part.trim(); + if (!trimmed) { + continue; + } + + const separatorIndex = trimmed.indexOf("="); + if (separatorIndex < 0) { + continue; + } + + const key = trimmed.slice(0, separatorIndex).trim(); + if (key !== name) { + continue; + } + + matched = trimmed.slice(separatorIndex + 1).trim(); + } + + return matched; +} + +function extractMtopToken(cookieHeader: string): string { + const tokenCookie = extractCookieValue(cookieHeader, "_m_h5_tk"); + const token = tokenCookie?.split("_")[0]?.trim(); + if (!token) { + throw new TmallLiveError( + "Tmall cookie header is missing _m_h5_tk, so signed mtop replay cannot be rebuilt." + ); + } + + return token; +} + +function signMtopData(token: string, timestamp: string, appKey: string, data: string): string { + return createHash("md5").update(`${token}&${timestamp}&${appKey}&${data}`).digest("hex"); +} + +function refreshMtopSignature(url: string, cookieHeader: string): string { + const nextUrl = new URL(url); + const data = nextUrl.searchParams.get("data") ?? nextUrl.searchParams.get("body"); + if (!data) { + return nextUrl.toString(); + } + + const appKey = nextUrl.searchParams.get("appKey") ?? "12574478"; + const timestamp = String(Date.now()); + const sign = signMtopData(extractMtopToken(cookieHeader), timestamp, appKey, data); + + nextUrl.searchParams.set("appKey", appKey); + nextUrl.searchParams.set("t", timestamp); + nextUrl.searchParams.set("sign", sign); + return nextUrl.toString(); +} + +function extractTemplateApi(templateUrl: string | undefined): string | undefined { + if (!templateUrl) { + return undefined; + } + + const url = new URL(templateUrl); + const direct = url.searchParams.get("api"); + if (direct) { + return direct; + } + + const match = url.pathname.match(/\/h5\/([^/]+)\/[^/]+\/$/i); + return match?.[1] ? match[1] : undefined; +} + +function extractTemplateItemId(templateUrl: string | undefined): string | undefined { + if (!templateUrl) { + return undefined; + } + + const data = readQueryData(new URL(templateUrl)); + const direct = + firstString(data?.id, data?.itemId, data?.itemNumId, data?.auctionNumId) ?? undefined; + if (direct) { + return direct; + } + + const exParams = readNestedJsonRecord(data?.exParams); + const queryParams = stringFrom(exParams?.queryParams); + if (!queryParams) { + return undefined; + } + + const params = new URLSearchParams(queryParams); + return firstString( + params.get("itemId"), + params.get("id"), + params.get("itemNumId"), + params.get("auctionNumId") + ) ?? undefined; +} + +function buildTemplateSummary(templateUrl: string | undefined): TmallTemplateSummary { + const api = extractTemplateApi(templateUrl); + const itemId = extractTemplateItemId(templateUrl); + + return { + available: Boolean(templateUrl), + ...(api ? { api } : {}), + ...(itemId ? { itemId } : {}) + }; +} + +function coerceValue(existing: unknown, nextValue: number | string): number | string { + if (typeof existing === "number") { + return typeof nextValue === "number" ? nextValue : Number.parseInt(nextValue, 10); + } + + return String(nextValue); +} + +function normalizePositiveInteger( + value: number | undefined, + fieldName: string, + fallback: number, + max: number +): number { + if (value === undefined) { + return fallback; + } + + if (!Number.isInteger(value) || value < 1 || value > max) { + throw new TmallLiveError(`${fieldName} must be an integer between 1 and ${max}.`); + } + + return value; +} + +function normalizeReviewsPreviewOptions( + options?: number | TmallReviewsPreviewOptions +): ResolvedTmallReviewsPreviewOptions { + if (typeof options === "number") { + return { + commentCount: normalizePositiveInteger(options, "commentCount", 20, 50), + page: 1, + maxPages: 1 + }; + } + + return { + commentCount: normalizePositiveInteger(options?.commentCount, "commentCount", 20, 50), + page: normalizePositiveInteger(options?.page, "page", 1, 1000), + maxPages: normalizePositiveInteger(options?.maxPages, "maxPages", 1, 10) + }; +} + +function buildSecurityNonce(itemId: string, logicVer: string): string { + const hash = createHash("sha256").update(`${itemId}${logicVer}`).digest("hex"); + const prefix = ( + 14802254 ^ + Number.parseInt(hash.slice(0, 6), 16) ^ + Number.parseInt(hash.slice(0, 6), 16) + ) + .toString(16) + .padStart(6, "0"); + + return `${prefix}${hash.slice(6)}`; +} + +function updateDetailExParams( + currentData: Record, + nextData: Record, + itemId: string +): void { + const exParams = readNestedJsonRecord(currentData.exParams); + if (!exParams) { + return; + } + + const queryParams = new URLSearchParams(stringFrom(exParams.queryParams) ?? ""); + for (const key of ["itemId", "id", "itemNumId", "auctionNumId"]) { + if (queryParams.has(key)) { + queryParams.set(key, itemId); + } + } + + if (![...queryParams.keys()].some((key) => key === "itemId" || key === "id")) { + queryParams.set("itemId", itemId); + queryParams.set("id", itemId); + } + + const logicVer = stringFrom(exParams.logicVer) ?? DEFAULT_LOGIC_VERSION; + const nextExParams: Record = { + ...exParams, + queryParams: queryParams.toString() + }; + + if ("logicVer" in exParams || "nonce" in exParams) { + nextExParams.logicVer = logicVer; + nextExParams.nonce = buildSecurityNonce(itemId, logicVer); + } + + nextData.exParams = JSON.stringify(nextExParams); +} + +function buildDetailRequestUrl(templateUrl: string, itemId: string): string { + const template = new URL(templateUrl); + const data = readQueryData(template); + + return withUpdatedQueryData(template, (currentData) => { + const nextData: Record = { + ...currentData + }; + let foundItemKey = false; + + for (const key of DETAIL_ITEM_DATA_KEYS) { + if (key in currentData) { + nextData[key] = coerceValue(currentData[key], itemId); + foundItemKey = true; + } + } + + if (!foundItemKey) { + const fallbackKey = findFirstRecordKey(data, DETAIL_ITEM_DATA_KEYS) ?? "id"; + nextData[fallbackKey] = itemId; + } + + updateDetailExParams(currentData, nextData, itemId); + return nextData; + }); +} + +function buildReviewsRequestUrl( + templateUrl: string, + itemId: string, + options: ResolvedTmallReviewsPreviewOptions, + page: number, + cookieHeader: string +): { url: string; pageKey?: string | undefined } { + const template = new URL(templateUrl); + const data = readQueryData(template); + const api = extractTemplateApi(templateUrl) ?? ""; + const itemKeys = api.endsWith("mtop.taobao.rate.detaillist.get") + ? REVIEW_H5_ITEM_DATA_KEYS + : REVIEW_PC_ITEM_DATA_KEYS; + const pageKey = findFirstRecordKey(data, REVIEW_PAGE_DATA_KEYS) ?? undefined; + const pageSizeKey = findFirstRecordKey(data, REVIEW_PAGE_SIZE_DATA_KEYS) ?? "pageSize"; + + if ((options.maxPages > 1 || options.page > 1) && !pageKey) { + throw new TmallLiveError( + "Imported Tmall reviews template does not expose a page field. Capture a paged reviews request first." + ); + } + + return { + url: refreshMtopSignature( + withUpdatedQueryData(template, (currentData) => { + const nextData: Record = { + ...currentData + }; + let foundItemKey = false; + + for (const key of itemKeys) { + if (key in currentData) { + nextData[key] = coerceValue(currentData[key], itemId); + foundItemKey = true; + } + } + + if (!foundItemKey) { + nextData[itemKeys[0] ?? "itemId"] = itemId; + } + + nextData[pageSizeKey] = coerceValue(currentData[pageSizeKey], options.commentCount); + if (pageKey) { + nextData[pageKey] = coerceValue(currentData[pageKey], page); + } + + return nextData; + }), + cookieHeader + ), + ...(pageKey ? { pageKey } : {}) + }; +} + +function mergeReviewPages( + itemId: string, + pages: TmallReviewsPreviewResult["reviews"][] +): TmallReviewsPreviewResult["reviews"] { + const firstPage = pages[0]; + if (!firstPage) { + return { + itemId, + total: null, + hasNext: false, + allCount: null, + pictureCount: null, + appendCount: null, + tags: [], + comments: [] + }; + } + + const commentsById = new Map(firstPage.comments.map((comment) => [comment.id, comment])); + for (const page of pages.slice(1)) { + for (const comment of page.comments) { + if (!commentsById.has(comment.id)) { + commentsById.set(comment.id, comment); + } + } + } + + return { + ...firstPage, + comments: Array.from(commentsById.values()) + }; +} + +function extractJsonRecord(text: string): Record | null { + return parseEmbeddedJson(text); +} + +function extractLoginUrl(text: string): string | null { + if (text.includes("login.taobao.com") || text.includes("login.m.taobao.com")) { + return "login"; + } + + const payload = extractJsonRecord(text); + const data = asRecord(payload?.data); + return firstString(data?.url, data?.h5url, asRecord(data?.pcTrade)?.redirectUrl); +} + +function summarizeRetCodes(text: string): string[] { + const payload = extractJsonRecord(text); + if (!payload) { + return []; + } + + return extractTmallRetCodes(payload); +} + +function isSuccessfulRet(retCode: string): boolean { + return retCode.includes("SUCCESS::"); +} + +async function fetchTextOrThrow( + url: string, + init: RequestInit, + sessionExpiredMessage: string +): Promise<{ finalUrl: string; text: string }> { + let response: Response; + + try { + response = await fetch(url, { + ...init, + redirect: "follow" + }); + } catch (error) { + throw new TmallLiveError( + `Tmall live request failed before receiving a response: ${ + error instanceof Error ? error.message : "unknown error" + }`, + 502 + ); + } + + const text = await response.text(); + const loginUrl = extractLoginUrl(text); + const retCodes = summarizeRetCodes(text); + + if (response.url.includes("login.taobao.com") || loginUrl) { + throw new TmallLiveError(sessionExpiredMessage, 409); + } + + if (!response.ok) { + throw new TmallLiveError( + `Tmall live request failed with status ${response.status}.`, + 502 + ); + } + + if (retCodes.length > 0 && retCodes.every((code) => !isSuccessfulRet(code))) { + throw new TmallLiveError( + `Tmall live request returned ${retCodes.join(" | ")}.`, + 502 + ); + } + + return { + finalUrl: response.url, + text + }; +} + +export function isTmallLiveError(error: unknown): error is Error & { statusCode: number } { + return error instanceof Error && "statusCode" in error; +} + +export class TmallLiveSessionService implements TmallLiveService { + private session: StoredTmallLiveSession | null = readEnvSession(); + + getSessionSummary(): TmallLiveSessionSummary { + return { + configured: Boolean(this.session), + hasCookie: Boolean(this.session?.cookieHeader), + ...(this.session?.importedAt ? { importedAt: this.session.importedAt } : {}), + ...(this.session?.userAgent ? { userAgent: this.session.userAgent } : {}), + detailTemplate: buildTemplateSummary(this.session?.detailTemplateUrl), + reviewsTemplate: buildTemplateSummary(this.session?.reviewsTemplateUrl) + }; + } + + importSession(input: TmallLiveSessionInput): TmallLiveSessionSummary { + const detailTemplateUrl = input.detailTemplateUrl?.trim(); + const reviewsTemplateUrl = input.reviewsTemplateUrl?.trim(); + const detailReferer = input.detailReferer?.trim(); + + this.session = { + cookieHeader: requireNonEmptyCookie(input.cookieHeader), + importedAt: nowIso(), + userAgent: input.userAgent?.trim() || DEFAULT_TMALL_USER_AGENT, + ...(detailTemplateUrl ? { detailTemplateUrl } : {}), + ...(reviewsTemplateUrl ? { reviewsTemplateUrl } : {}), + ...(detailReferer ? { detailReferer } : {}) + }; + + return this.getSessionSummary(); + } + + clearSession(): void { + this.session = readEnvSession(); + } + + async previewSearch(query: string): Promise { + const session = this.requireSession(); + const normalizedQuery = query.trim(); + if (!normalizedQuery) { + throw new TmallLiveError("query is required for Tmall live search preview."); + } + + const requestUrl = `https://s.taobao.com/search?q=${encodeURIComponent(normalizedQuery)}&tab=mall`; + const response = await fetchTextOrThrow( + requestUrl, + { + headers: { + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + Cookie: session.cookieHeader, + Referer: "https://www.tmall.com/", + "User-Agent": session.userAgent + } + }, + "Tmall search session appears invalid. Re-login in the browser and re-import the cookie/header." + ); + + const candidates = parseTmallSearchHtml(normalizedQuery, response.text); + if (candidates.length === 0 && !hasTmallSearchNoResultMarker(response.text)) { + throw new TmallLiveError( + "Tmall search page fetched successfully, but no stable mall candidates could be parsed.", + 502 + ); + } + + return { + query: normalizedQuery, + source: "html", + candidateCount: candidates.length, + candidates + }; + } + + async previewDetail(itemId: string): Promise { + const session = this.requireSession(); + const normalizedItemId = itemId.trim(); + if (!normalizedItemId) { + throw new TmallLiveError("itemId is required for Tmall detail preview."); + } + + const requestUrl = `https://detail.tmall.com/item.htm?id=${normalizedItemId}`; + const response = await fetchTextOrThrow( + requestUrl, + { + headers: { + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + Cookie: session.cookieHeader, + Referer: session.detailReferer ?? "https://detail.tmall.com/", + "User-Agent": session.userAgent + } + }, + "Tmall detail session appears invalid. Re-login in the browser and re-import the cookie/header." + ); + + const detail = parseTmallDetailHtmlResponse(normalizedItemId, { text: response.text }); + if (!detail.title && !detail.shopName && !detail.mainImage) { + throw new TmallLiveError( + "Tmall detail page fetched successfully, but the embedded page state could not be parsed.", + 502 + ); + } + + return { + itemId: normalizedItemId, + source: "html", + detail + }; + } + + async previewReviews( + itemId: string, + options?: number | TmallReviewsPreviewOptions + ): Promise { + const session = this.requireSession(); + const normalizedItemId = itemId.trim(); + const resolvedOptions = normalizeReviewsPreviewOptions(options); + if (!normalizedItemId) { + throw new TmallLiveError("itemId is required for Tmall reviews preview."); + } + + if (!session.reviewsTemplateUrl) { + throw new TmallLiveError( + "Tmall reviews template is missing. Capture a fresh reviews request and import it first." + ); + } + + const reviewPages = []; + let pageKey: string | undefined; + + for (let pageOffset = 0; pageOffset < resolvedOptions.maxPages; pageOffset += 1) { + const currentPage = resolvedOptions.page + pageOffset; + const request = buildReviewsRequestUrl( + session.reviewsTemplateUrl, + normalizedItemId, + resolvedOptions, + currentPage, + session.cookieHeader + ); + pageKey ??= request.pageKey; + + const response = await fetchTextOrThrow( + request.url, + { + headers: { + Accept: "application/json, text/plain, */*", + Cookie: session.cookieHeader, + Referer: + session.detailReferer ?? + `https://detail.tmall.com/wow/z/app/tbpc/pc-detail-ssr-2025/home?itemId=${normalizedItemId}`, + "User-Agent": session.userAgent + } + }, + "Tmall reviews session appears invalid. Re-login in the browser and re-import the cookie/header." + ); + + const parsedPage = parseTmallReviewsApiResponse(normalizedItemId, { text: response.text }); + reviewPages.push(parsedPage); + + if (parsedPage.comments.length < resolvedOptions.commentCount || !parsedPage.hasNext) { + break; + } + } + + const pagination: TmallReviewsPaginationSummary = { + requestedPage: resolvedOptions.page, + requestedCommentCount: resolvedOptions.commentCount, + maxPages: resolvedOptions.maxPages, + pagesFetched: reviewPages.length, + ...(pageKey ? { pageKey } : {}) + }; + + return { + itemId: normalizedItemId, + source: "api", + pagination, + reviews: mergeReviewPages(normalizedItemId, reviewPages) + }; + } + + async previewProduct( + itemId: string, + options?: number | TmallReviewsPreviewOptions + ): Promise { + const [detailPreview, reviewsPreview] = await Promise.all([ + this.previewDetail(itemId), + this.previewReviews(itemId, options) + ]); + + return { + itemId: detailPreview.itemId, + source: detailPreview.source === reviewsPreview.source ? detailPreview.source : "hybrid", + detail: detailPreview.detail, + pagination: reviewsPreview.pagination, + reviews: reviewsPreview.reviews + }; + } + + private requireSession(): StoredTmallLiveSession { + if (!this.session?.cookieHeader) { + throw new TmallLiveError( + "Tmall live session is not configured. Import a browser cookie/header first." + ); + } + + return this.session; + } +} diff --git a/apps/api/src/platforms/tmall/parsers.test.ts b/apps/api/src/platforms/tmall/parsers.test.ts new file mode 100644 index 0000000..8426310 --- /dev/null +++ b/apps/api/src/platforms/tmall/parsers.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest"; + +import { hasTmallSearchNoResultMarker, parseTmallSearchHtml } from "./parsers"; + +describe("Tmall search parsers", () => { + it("parses mall candidates from g_page_config payloads", () => { + const payload = { + mods: { + itemlist: { + data: { + auctions: [ + { + nid: "934454505228", + raw_title: "Apple iPhone 15 Natural Titanium 128GB", + view_price: "4399.00", + nick: "Apple Flagship Store", + detail_url: "//detail.tmall.com/item.htm?id=934454505228", + pic_url: "//img.alicdn.com/example-1.jpg", + view_sales: "sold 10k+", + comment_count: "20k+", + isTmall: "true", + iconList: [{ text: "Free shipping" }] + }, + { + nid: "934454505229", + raw_title: "Apple iPhone 15 Pink 256GB", + view_price: "4799.00", + nick: "Apple Flagship Store", + detail_url: "//detail.tmall.com/item.htm?id=934454505229", + pic_url: "//img.alicdn.com/example-2.jpg", + view_sales: "sold 5k+", + comment_count: "12k+", + isTmall: true + } + ] + } + } + } + }; + const html = ``; + + const candidates = parseTmallSearchHtml("iPhone 15", html); + + expect(candidates).toHaveLength(2); + expect(candidates[0]).toMatchObject({ + candidateId: "tmall-934454505228", + platform: "tmall", + title: "Apple iPhone 15 Natural Titanium 128GB", + price: 4399, + priceLabel: "CNY 4399", + storeName: "Apple Flagship Store", + productUrl: "https://detail.tmall.com/item.htm?id=934454505228", + imageUrl: "https://img.alicdn.com/example-1.jpg", + specLabel: "128GB" + }); + expect(candidates[0]?.highlights).toEqual(["Free shipping", "天猫"]); + }); + + it("parses fallback mall candidate blocks when embedded JSON is missing", () => { + const html = [ + "
", + 'item', + '', + '", + "
" + ].join(""); + + const candidates = parseTmallSearchHtml("iPhone 15", html); + + expect(candidates).toEqual([ + expect.objectContaining({ + candidateId: "tmall-934454505230", + productUrl: "https://detail.tmall.com/item.htm?id=934454505230", + title: "Apple iPhone 15 Blue 512GB", + price: 5799, + storeName: "Apple Flagship Store", + imageUrl: "https://img.alicdn.com/example-3.jpg" + }) + ]); + }); + + it("detects no-result markers from the search page", () => { + expect(hasTmallSearchNoResultMarker("
很抱歉，没有找到与 iPhone 15 相关的商品
")).toBe( + true + ); + expect(hasTmallSearchNoResultMarker("
search results ready
")).toBe(false); + }); +}); diff --git a/apps/api/src/platforms/tmall/parsers.ts b/apps/api/src/platforms/tmall/parsers.ts new file mode 100644 index 0000000..f89af24 --- /dev/null +++ b/apps/api/src/platforms/tmall/parsers.ts @@ -0,0 +1,918 @@ +import type { CandidateRecord } from "@cross-ai/domain"; + +import type { + TmallProductDetailSnapshot, + TmallProductReviewsSnapshot, + TmallReviewCommentSnapshot, + TmallReviewTagSnapshot +} from "./types"; +import { + absolutizeTmallUrl, + asArray, + asRecord, + firstString, + normalizeWhitespace, + parseBooleanish, + parseEmbeddedJson, + pickFirstStringByPaths, + readPath, + stringFrom, + uniqueStrings +} from "./utils"; + +type PathSegment = string | number; + +const TMALL_SEARCH_RESULT_PATHS: PathSegment[][] = [ + ["mods", "itemlist", "data", "auctions"], + ["root", "fields", "mods", "itemlist", "data", "auctions"], + ["data", "root", "fields", "mods", "itemlist", "data", "auctions"], + ["props", "pageProps", "mods", "itemlist", "data", "auctions"], + ["itemlist", "data", "auctions"], + ["itemList"], + ["items"] +]; + +const TMALL_SEARCH_JSON_MARKERS = [ + "g_page_config =", + "window.g_page_config =", + "window.__SEARCH_DATA__ =", + "window.__INITIAL_STATE__ =", + "window.__PRELOADED_STATE__ =" +]; + +const TMALL_SEARCH_NO_RESULT_MARKERS = [ + "没有找到相关宝贝", + "没有找到相应的宝贝", + "很抱歉，没有找到与", + "未找到相关商品", + "暂无相关商品" +]; + +function extractSpecLabel(title: string): string { + const storageMatch = title.match(/\b\d+(?:GB|TB)\b/i); + if (storageMatch) { + return storageMatch[0].toUpperCase(); + } + + const colorMatch = title.match( + /(黑色|白色|蓝色|粉色|绿色|黄色|紫色|原色|银色|金色|灰色|红色|深空黑|午夜色|星光色|natural titanium)/i + ); + if (colorMatch?.[0]) { + return normalizeWhitespace(colorMatch[0]); + } + + return "标准款"; +} + +function normalizePriceText(value: string | null): { value: number; label: string } | null { + if (!value) { + return null; + } + + const stripped = value.replace(/[^\d.]/g, ""); + if (!stripped) { + return null; + } + + const parsed = Number.parseFloat(stripped); + if (Number.isNaN(parsed)) { + return null; + } + + const normalizedLabel = Number.isInteger(parsed) ? parsed.toFixed(0) : parsed.toFixed(2); + return { + value: parsed, + label: `CNY ${normalizedLabel}` + }; +} + +function normalizeInlineText(value: string | null): string | null { + if (!value) { + return null; + } + + return normalizeWhitespace(value) || null; +} + +function matchFirst(value: string, pattern: RegExp): string | null { + const match = pattern.exec(value); + return match?.[1] ? normalizeWhitespace(match[1]) : null; +} + +function unwrapCapturedPayload(input: unknown): Record | null { + const record = asRecord(input); + const text = stringFrom(record?.text); + if (text) { + try { + return asRecord(JSON.parse(text)); + } catch { + return parseEmbeddedJson(text) ?? record; + } + } + + return record; +} + +function unwrapTmallData(input: unknown): Record | null { + const payload = unwrapCapturedPayload(input); + return asRecord(payload?.data) ?? payload; +} + +function extractTextPayload(input: unknown): string | null { + if (typeof input === "string") { + return input; + } + + const record = asRecord(input); + return stringFrom(record?.text); +} + +function extractJsonObjectLiteral( + source: string, + startIndex: number +): Record | null { + let depth = 0; + let inString = false; + let isEscaped = false; + let objectStart = -1; + + for (let index = startIndex; index < source.length; index += 1) { + const current = source[index]; + if (!current) { + continue; + } + + if (objectStart < 0) { + if (current === "{") { + objectStart = index; + depth = 1; + } + continue; + } + + if (inString) { + if (isEscaped) { + isEscaped = false; + } else if (current === "\\") { + isEscaped = true; + } else if (current === "\"") { + inString = false; + } + continue; + } + + if (current === "\"") { + inString = true; + continue; + } + + if (current === "{") { + depth += 1; + continue; + } + + if (current === "}") { + depth -= 1; + if (depth === 0) { + return parseEmbeddedJson(source.slice(objectStart, index + 1)); + } + } + } + + return null; +} + +function extractSearchPayloads(html: string): Record[] { + const payloads: Record[] = []; + + for (const marker of TMALL_SEARCH_JSON_MARKERS) { + let searchIndex = 0; + while (searchIndex < html.length) { + const markerIndex = html.indexOf(marker, searchIndex); + if (markerIndex < 0) { + break; + } + + const payload = extractJsonObjectLiteral(html, markerIndex + marker.length); + if (payload) { + payloads.push(payload); + } + + searchIndex = markerIndex + marker.length; + } + } + + const jsonScriptMatches = html.matchAll( + /]*type="application\/json"[^>]*>([\s\S]*?)<\/script>/gi + ); + for (const match of jsonScriptMatches) { + const payload = parseEmbeddedJson(match[1] ?? null); + if (payload) { + payloads.push(payload); + } + } + + return payloads; +} + +function looksLikeTmallSearchItem(input: Record): boolean { + const detailUrl = normalizeInlineText( + firstString( + input.detail_url, + input.detailUrl, + input.itemUrl, + input.url, + input.item_url + ) + ); + const itemId = firstString( + input.nid, + input.itemId, + input.item_id, + input.id, + input.auctionNumId + ); + const title = normalizeInlineText( + firstString( + input.raw_title, + input.title, + input.name, + input.itemTitle, + asRecord(input.item)?.title + ) + ); + + if (!title) { + return false; + } + + if (detailUrl && /detail\.tmall\.com\/item\.htm/i.test(detailUrl)) { + return true; + } + + return Boolean(itemId && (detailUrl || "view_price" in input || "price" in input)); +} + +function collectSearchItems( + root: unknown, + depth = 0, + seenCollections = new Set() +): Record[] { + if (depth > 8) { + return []; + } + + const directList = TMALL_SEARCH_RESULT_PATHS.flatMap((path) => { + const value = readPath(root, path); + const collection = asArray(value) + .map((entry) => asRecord(entry)) + .filter((entry): entry is Record => Boolean(entry)) + .filter((entry) => looksLikeTmallSearchItem(entry)); + + return collection.length > 0 ? [collection] : []; + }); + if (directList.length > 0) { + return directList[0] ?? []; + } + + const record = asRecord(root); + if (record) { + const signature = Object.keys(record).sort().join("|"); + if (signature) { + seenCollections.add(signature); + } + + const nestedCollections = Object.values(record).flatMap((value) => { + const entries = asArray(value) + .map((entry) => asRecord(entry)) + .filter((entry): entry is Record => Boolean(entry)); + if (entries.length > 0 && entries.some((entry) => looksLikeTmallSearchItem(entry))) { + const collectionSignature = entries + .map((entry) => + firstString(entry.nid, entry.itemId, entry.item_id, entry.id, entry.auctionNumId) ?? + firstString(entry.detail_url, entry.detailUrl, entry.url) ?? + "" + ) + .filter(Boolean) + .join("|"); + if (collectionSignature && !seenCollections.has(collectionSignature)) { + seenCollections.add(collectionSignature); + return [entries]; + } + } + + return []; + }); + if (nestedCollections.length > 0) { + return nestedCollections[0] ?? []; + } + + for (const value of Object.values(record)) { + const nested = collectSearchItems(value, depth + 1, seenCollections); + if (nested.length > 0) { + return nested; + } + } + } + + const array = asArray(root); + for (const value of array) { + const nested = collectSearchItems(value, depth + 1, seenCollections); + if (nested.length > 0) { + return nested; + } + } + + return []; +} + +function extractTmallDetailPageState(input: unknown): Record | null { + const text = extractTextPayload(input); + if (!text) { + return null; + } + + const anchor = text.indexOf("window.__ICE_APP_CONTEXT__"); + if (anchor < 0) { + return null; + } + + const marker = "var b = "; + const markerIndex = text.indexOf(marker, anchor); + if (markerIndex < 0) { + return null; + } + + const context = extractJsonObjectLiteral(text, markerIndex + marker.length); + const loaderData = asRecord(context?.loaderData); + const home = asRecord(loaderData?.home); + const data = asRecord(home?.data); + return asRecord(data?.res); +} + +export function extractTmallRetCodes(input: unknown): string[] { + const payload = unwrapCapturedPayload(input); + const ret = payload?.ret; + if (Array.isArray(ret)) { + return ret + .map((value) => stringFrom(value)) + .filter((value): value is string => Boolean(value)); + } + + const asString = stringFrom(ret); + return asString ? [asString] : []; +} + +function pickFirstImageByPaths(root: unknown, paths: PathSegment[][]): string | null { + for (const path of paths) { + const value = readPath(root, path); + const candidate = + stringFrom(value) ?? + firstString( + asRecord(value)?.url, + asRecord(value)?.image, + asRecord(value)?.imageUrl, + asRecord(value)?.picUrl + ); + + if (candidate) { + return absolutizeTmallUrl(candidate); + } + } + + return null; +} + +function pickCategoryPath(root: unknown): string[] { + const candidates: PathSegment[][] = [ + ["item", "categoryPath"], + ["item", "categoryNamePath"], + ["item", "categoryPathNames"], + ["vertical", "categoryPath"], + ["categoryPath"], + ["categoryNamePath"] + ]; + + for (const path of candidates) { + const value = readPath(root, path); + if (typeof value === "string") { + const parts = value + .split(/[>/|]/g) + .map((part) => normalizeWhitespace(part)) + .filter(Boolean); + if (parts.length > 0) { + return parts; + } + } + + const items = asArray(value) + .map((entry) => { + const record = asRecord(entry); + return firstString(entry, record?.name, record?.text, record?.title); + }) + .filter((entry): entry is string => Boolean(entry)) + .map((entry) => normalizeWhitespace(entry)); + + if (items.length > 0) { + return items; + } + } + + return []; +} + +function parseSkuText(value: unknown): string[] { + if (typeof value === "string") { + return uniqueStrings( + value + .split(/[;/]/g) + .map((entry) => normalizeWhitespace(entry)) + .filter(Boolean) + ); + } + + const record = asRecord(value); + if (!record) { + return []; + } + + return uniqueStrings( + Object.values(record).map((entry) => + typeof entry === "string" ? normalizeWhitespace(entry) : null + ) + ); +} + +function collectImageUrls(value: unknown): string[] { + return uniqueStrings( + asArray(value).flatMap((entry) => { + const record = asRecord(entry); + const direct = absolutizeTmallUrl(stringFrom(entry)); + if (direct) { + return [direct]; + } + + return uniqueStrings([ + absolutizeTmallUrl(firstString(record?.url, record?.image, record?.picPath)), + absolutizeTmallUrl(firstString(record?.sourceUrl, record?.cloudVideoUrl)) + ]); + }) + ); +} + +function parseReviewTag(input: unknown): TmallReviewTagSnapshot | null { + const tag = asRecord(input); + const name = firstString(tag?.name, tag?.imprName, tag?.label, tag?.title); + if (!name) { + return null; + } + + return { + name: normalizeWhitespace(name), + count: firstString(tag?.count, tag?.showCount, tag?.num) + }; +} + +function parseH5ReviewComment(input: unknown): TmallReviewCommentSnapshot | null { + const comment = asRecord(input); + const appended = asRecord(comment?.appendedFeed); + const video = asRecord(comment?.video); + const content = firstString(comment?.feedback, comment?.reviewWordContent, comment?.content); + const id = firstString(comment?.id, comment?.commentId, comment?.rateId); + + if (!content || !id) { + return null; + } + + return { + id, + content: normalizeWhitespace(content), + date: firstString(comment?.feedbackDate, comment?.reviewDate, comment?.date), + userNick: firstString(comment?.userNick, asRecord(comment?.userInfo)?.nickName), + userAvatar: absolutizeTmallUrl( + firstString(comment?.headPicUrl, asRecord(comment?.userInfo)?.image) + ), + skuText: parseSkuText(comment?.skuMap ?? comment?.skuText), + pictureUrls: collectImageUrls(comment?.feedPicPathList ?? comment?.reviewPicPathList), + videoUrls: collectImageUrls(video?.videoId ? [video] : []), + likeCount: firstString(asRecord(comment?.interactInfo)?.likeCount), + reply: firstString(comment?.reply), + appendContent: firstString(appended?.appendedFeedback), + appendPictureUrls: collectImageUrls(appended?.appendFeedPicPathList) + }; +} + +function parsePcReviewComment(input: unknown): TmallReviewCommentSnapshot | null { + const comment = asRecord(input); + const append = asRecord(comment?.reviewAppendVO); + const content = firstString(comment?.reviewWordContent, comment?.feedback, comment?.content); + const id = firstString(comment?.id, comment?.commentId); + + if (!content || !id) { + return null; + } + + return { + id, + content: normalizeWhitespace(content), + date: firstString(comment?.reviewDate, comment?.feedbackDate, comment?.date), + userNick: firstString(comment?.userNick, asRecord(comment?.userInfo)?.nickName), + userAvatar: absolutizeTmallUrl( + firstString(comment?.headPicUrl, asRecord(comment?.userInfo)?.image) + ), + skuText: parseSkuText(comment?.skuText ?? comment?.skuMap), + pictureUrls: collectImageUrls(comment?.reviewPicPathList), + videoUrls: collectImageUrls(comment?.videoVOList), + likeCount: firstString(asRecord(comment?.interactionVO)?.likeCount), + reply: firstString(comment?.reply), + appendContent: firstString(append?.appendedWordContent), + appendPictureUrls: collectImageUrls(append?.reviewPicPathList) + }; +} + +function parseSearchItem(input: Record): CandidateRecord | null { + const nestedItem = asRecord(input.item); + const record = nestedItem ? { ...nestedItem, ...input } : input; + const detailUrl = absolutizeTmallUrl( + firstString(record.detail_url, record.detailUrl, record.itemUrl, record.url, record.item_url) + ); + const itemId = + firstString(record.nid, record.itemId, record.item_id, record.id, record.auctionNumId) ?? + detailUrl?.match(/[?&](?:id|itemId|itemNumId|auctionNumId)=(\d+)/i)?.[1] ?? + null; + const title = normalizeInlineText( + firstString( + record.raw_title, + record.title, + record.name, + record.itemTitle, + asRecord(record.item)?.title + ) + ); + const price = normalizePriceText( + firstString( + record.view_price, + record.price, + record.priceText, + asRecord(record.priceInfo)?.priceText, + asRecord(record.priceInfo)?.price + ) + ); + const storeName = + normalizeInlineText( + firstString( + record.nick, + record.shopName, + record.sellerNick, + asRecord(record.shopcard)?.shopName + ) + ) ?? "天猫店铺"; + const salesHints = uniqueStrings([ + normalizeInlineText( + firstString( + record.view_sales, + record.sold, + record.salesDesc, + record.soldText, + record.comment_count ? `累计评价 ${stringFrom(record.comment_count)}` : null + ) + ), + normalizeInlineText(firstString(record.commentCount, record.comment_count)) + ]); + const featureFlags = asArray(record.iconList) + .map((entry) => + normalizeInlineText(firstString(entry, asRecord(entry)?.text, asRecord(entry)?.name)) + ) + .filter((entry): entry is string => Boolean(entry)); + const highlights = uniqueStrings([ + ...featureFlags, + parseBooleanish(record.isTmall) ? "天猫" : null, + normalizeInlineText(firstString(record.shopIcon, record.shopType)) + ]).slice(0, 4); + + if (!itemId || !title || !detailUrl || !/detail\.tmall\.com\/item\.htm/i.test(detailUrl)) { + return null; + } + + return { + candidateId: `tmall-${itemId}`, + platform: "tmall", + title, + price: price?.value ?? 0, + priceLabel: price?.label ?? "CNY 0", + storeName, + productUrl: detailUrl, + imageUrl: + absolutizeTmallUrl( + firstString( + record.pic_url, + record.image, + record.imageUrl, + asRecord(record.pic)?.url + ) + ) ?? "https://placehold.co/640x480?text=TMALL", + salesHint: salesHints.join(" | ") || "搜索页已返回，但缺少稳定销量文案", + specLabel: extractSpecLabel(title), + highlights: highlights.length > 0 ? highlights : ["天猫搜索结果候选"] + }; +} + +function parseSearchCandidateBlocks(query: string, html: string): CandidateRecord[] { + const blocks = html.matchAll( + /((?:https?:)?\/\/detail\.tmall\.com\/item\.htm\?[^"'<>\\\s]*\bid=\d+[\s\S]{0,1200})/gi + ); + const seen = new Set(); + const candidates: CandidateRecord[] = []; + + for (const match of blocks) { + const block = match[1] ?? ""; + const itemId = block.match(/[?&](?:id|itemId|itemNumId|auctionNumId)=(\d+)/i)?.[1]; + if (!itemId || seen.has(itemId)) { + continue; + } + + seen.add(itemId); + const detailUrlMatch = block.match( + /((?:https?:)?\/\/detail\.tmall\.com\/item\.htm\?[^"'<>\\\s]*\bid=\d+)/i + ); + const title = + matchFirst(block, /"(?:raw_title|title|itemTitle)"\s*:\s*"([^"]+)"/i) ?? + matchFirst(block, /title="([^"]{4,200})"/i) ?? + query; + const price = normalizePriceText( + matchFirst(block, /"(?:view_price|price|priceText)"\s*:\s*"([^"]+)"/i) ?? + matchFirst(block, />\s*([0-9]+(?:\.[0-9]+)?)\s*]+src="([^"]+)"/i) + ) ?? "https://placehold.co/640x480?text=TMALL"; + const salesHint = + matchFirst(block, /"(?:view_sales|sold|salesDesc|comment_count)"\s*:\s*"([^"]+)"/i) ?? + "搜索页已返回，但未抽取到稳定销量文案"; + + candidates.push({ + candidateId: `tmall-${itemId}`, + platform: "tmall", + title, + price: price?.value ?? 0, + priceLabel: price?.label ?? "CNY 0", + storeName, + productUrl: absolutizeTmallUrl(detailUrlMatch?.[1] ?? null) ?? + `https://detail.tmall.com/item.htm?id=${itemId}`, + imageUrl, + salesHint, + specLabel: extractSpecLabel(title), + highlights: ["搜索页候选回退解析"] + }); + } + + return candidates; +} + +function normalizeTmallSellerType(value: string | null): string | null { + if (!value) { + return null; + } + + const normalized = value.trim().toLowerCase(); + if (normalized === "b" || normalized.includes("tmall")) { + return "tmall"; + } + + if (normalized === "c" || normalized.includes("taobao")) { + return "taobao"; + } + + return value.trim(); +} + +export function hasTmallSearchNoResultMarker(html: string): boolean { + return TMALL_SEARCH_NO_RESULT_MARKERS.some((marker) => html.includes(marker)); +} + +export function parseTmallSearchHtml(query: string, html: string): CandidateRecord[] { + const seen = new Set(); + const candidates: CandidateRecord[] = []; + + for (const payload of extractSearchPayloads(html)) { + const items = collectSearchItems(payload); + for (const item of items) { + const candidate = parseSearchItem(item); + if (!candidate || seen.has(candidate.candidateId)) { + continue; + } + + seen.add(candidate.candidateId); + candidates.push(candidate); + } + } + + if (candidates.length > 0) { + return candidates; + } + + for (const candidate of parseSearchCandidateBlocks(query, html)) { + if (seen.has(candidate.candidateId)) { + continue; + } + + seen.add(candidate.candidateId); + candidates.push(candidate); + } + + return candidates; +} + +export function parseTmallDetailHtmlResponse( + itemId: string, + input: unknown +): TmallProductDetailSnapshot { + const payload = extractTmallDetailPageState(input) ?? {}; + + return { + itemId: + pickFirstStringByPaths(payload, [["item", "itemId"], ["itemId"], ["id"]]) ?? itemId, + title: + pickFirstStringByPaths(payload, [ + ["componentsVO", "titleVO", "title", "title"], + ["item", "title"] + ]) ?? null, + subtitle: + pickFirstStringByPaths(payload, [["item", "subtitle"], ["item", "subTitle"]]) ?? null, + price: + pickFirstStringByPaths(payload, [ + ["componentsVO", "priceVO", "extraPrice", "priceText"], + ["componentsVO", "priceVO", "price", "priceText"] + ]) ?? null, + originalPrice: + pickFirstStringByPaths(payload, [["componentsVO", "priceVO", "price", "priceText"]]) ?? + null, + shopName: + pickFirstStringByPaths(payload, [ + ["seller", "shopName"], + ["componentsVO", "storeCardVO", "shopName"] + ]) ?? null, + shopUrl: + pickFirstImageByPaths(payload, [ + ["seller", "pcShopUrl"], + ["componentsVO", "storeCardVO", "shopUrl"] + ]) ?? null, + sellerType: normalizeTmallSellerType( + pickFirstStringByPaths(payload, [ + ["seller", "sellerType"], + ["componentsVO", "storeCardVO", "sellerType"] + ]) + ), + categoryPath: pickCategoryPath(payload), + mainImage: + pickFirstImageByPaths(payload, [ + ["componentsVO", "headImageVO", "images", 0], + ["item", "images", 0] + ]) ?? null, + salesDesc: + pickFirstStringByPaths(payload, [ + ["componentsVO", "titleVO", "salesDesc"], + ["item", "vagueSellCount"] + ]) ?? null, + commentCount: + pickFirstStringByPaths(payload, [ + ["componentsVO", "rateVO", "totalCount"], + ["componentsVO", "rateVO", "total"], + ["item", "commentCount"] + ]) ?? null + }; +} + +export function parseTmallDetailApiResponse( + itemId: string, + input: unknown +): TmallProductDetailSnapshot { + const payload = unwrapTmallData(input) ?? {}; + const price = asRecord(payload.price) ?? asRecord(payload.priceInfo) ?? {}; + + return { + itemId: + pickFirstStringByPaths(payload, [["item", "itemId"], ["itemId"], ["id"]]) ?? itemId, + title: + pickFirstStringByPaths(payload, [ + ["item", "title"], + ["item", "itemTitle"], + ["itemInfo", "title"], + ["itemInfoModel", "title"], + ["title"] + ]) ?? null, + subtitle: + pickFirstStringByPaths(payload, [ + ["item", "subtitle"], + ["item", "subTitle"], + ["subTitle"] + ]) ?? null, + price: + pickFirstStringByPaths(payload, [ + ["price", "componentsVO", "priceVO", "priceText"], + ["price", "priceText"], + ["price", "price"], + ["priceInfo", "priceText"], + ["vertical", "price", "priceText"] + ]) ?? + pickFirstStringByPaths(price, [["priceText"], ["price"]]) ?? + null, + originalPrice: + pickFirstStringByPaths(payload, [ + ["price", "componentsVO", "priceVO", "crossPriceText"], + ["price", "componentsVO", "priceVO", "subPriceText"], + ["price", "originPriceText"], + ["price", "originalPrice"], + ["priceInfo", "originPriceText"] + ]) ?? null, + shopName: + pickFirstStringByPaths(payload, [ + ["seller", "shopName"], + ["seller", "shopTitle"], + ["shop", "shopName"], + ["shopInfo", "shopName"] + ]) ?? null, + shopUrl: + pickFirstImageByPaths(payload, [ + ["seller", "shopUrl"], + ["shop", "shopUrl"], + ["shopInfo", "shopUrl"] + ]) ?? null, + sellerType: normalizeTmallSellerType( + pickFirstStringByPaths(payload, [ + ["seller", "sellerType"], + ["shop", "sellerType"], + ["feature", "sellerType"] + ]) + ), + categoryPath: pickCategoryPath(payload), + mainImage: + pickFirstImageByPaths(payload, [ + ["item", "mainPic"], + ["item", "mainImage"], + ["item", "images", 0], + ["item", "images", 0, "url"], + ["item", "picGallery", 0], + ["item", "picGallery", 0, "image"], + ["skuBase", "images", 0], + ["skuBase", "images", 0, "url"] + ]) ?? null, + salesDesc: + pickFirstStringByPaths(payload, [ + ["item", "sellCount"], + ["item", "sellCountText"], + ["item", "soldCount"], + ["item", "soldCountText"], + ["vertical", "salesDesc"] + ]) ?? null, + commentCount: + pickFirstStringByPaths(payload, [ + ["rate", "total"], + ["rate", "commentCount"], + ["review", "total"], + ["item", "commentCount"] + ]) ?? null + }; +} + +export function parseTmallReviewsApiResponse( + itemId: string, + input: unknown +): TmallProductReviewsSnapshot { + const payload = unwrapTmallData(input) ?? {}; + const module = asRecord(payload.module); + const h5Comments = asArray(payload.rateList) + .map((comment) => parseH5ReviewComment(comment)) + .filter((comment): comment is TmallReviewCommentSnapshot => Boolean(comment)); + const pcComments = asArray(module?.reviewVOList) + .map((comment) => parsePcReviewComment(comment)) + .filter((comment): comment is TmallReviewCommentSnapshot => Boolean(comment)); + const tags = asArray(payload.imprItemList ?? payload.imprItemVOS ?? module?.imprItemList) + .map((tag) => parseReviewTag(tag)) + .filter((tag): tag is TmallReviewTagSnapshot => Boolean(tag)); + + return { + itemId, + total: firstString(payload.totalFuzzy, payload.total, module?.totalFuzzy, module?.total), + hasNext: parseBooleanish(firstString(payload.hasNext, module?.hasNext)), + allCount: firstString(payload.fuzzyRateCount, payload.feedAllCount, module?.feedAllCount), + pictureCount: firstString( + payload.feedMediaCountFuzzy, + payload.feedMediaCount, + module?.feedMediaCountFuzzy, + module?.feedMediaCount + ), + appendCount: firstString( + payload.feedAppendCountFuzzy, + payload.feedAppendCount, + module?.feedAppendCountFuzzy, + module?.feedAppendCount + ), + tags, + comments: h5Comments.length > 0 ? h5Comments : pcComments + }; +} diff --git a/apps/api/src/platforms/tmall/session-manager.test.ts b/apps/api/src/platforms/tmall/session-manager.test.ts new file mode 100644 index 0000000..c570f19 --- /dev/null +++ b/apps/api/src/platforms/tmall/session-manager.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it, vi } from "vitest"; + +import { TmallSessionManagerService } from "./session-manager"; +import type { TmallLiveService, TmallLiveSessionSummary } from "./types"; + +function createSessionSummary(configured: boolean): TmallLiveSessionSummary { + return configured + ? { + configured: true, + importedAt: "2026-04-03T10:00:00.000Z", + hasCookie: true, + userAgent: "stub-user-agent", + detailTemplate: { + available: true, + api: "mtop.taobao.pcdetail.data.get", + itemId: "934454505228" + }, + reviewsTemplate: { + available: true, + api: "mtop.taobao.rate.detaillist.get", + itemId: "934454505228" + } + } + : { + configured: false, + hasCookie: false, + detailTemplate: { + available: false + }, + reviewsTemplate: { + available: false + } + }; +} + +function createLiveService(): TmallLiveService { + let summary = createSessionSummary(false); + + return { + getSessionSummary() { + return summary; + }, + importSession() { + summary = createSessionSummary(true); + return summary; + }, + clearSession() { + summary = createSessionSummary(false); + }, + async previewSearch(query) { + return { + query, + source: "html", + candidateCount: 1, + candidates: [ + { + candidateId: "tmall-934454505228", + platform: "tmall", + title: "Apple iPhone 15", + price: 4399, + priceLabel: "CNY 4399", + storeName: "Apple 官方旗舰店", + productUrl: "https://detail.tmall.com/item.htm?id=934454505228", + imageUrl: "https://img.alicdn.com/example.jpg", + salesHint: "已售 70万+", + specLabel: "128GB", + highlights: ["天猫"] + } + ] + }; + }, + async previewDetail(itemId) { + return { + itemId, + source: "html", + detail: { + itemId, + title: "Apple iPhone 15", + subtitle: null, + price: "4399.00", + originalPrice: "4999.00", + shopName: "Apple 官方旗舰店", + shopUrl: null, + sellerType: "tmall", + categoryPath: [], + mainImage: null, + salesDesc: "已售 70万+", + commentCount: "20万+" + } + }; + }, + async previewReviews(itemId) { + return { + itemId, + source: "api", + pagination: { + requestedPage: 1, + requestedCommentCount: 1, + maxPages: 1, + pagesFetched: 1 + }, + reviews: { + itemId, + total: "20万+", + hasNext: false, + allCount: "20万+", + pictureCount: "1万+", + appendCount: "5000+", + tags: [], + comments: [] + } + }; + }, + async previewProduct(itemId, options) { + const detail = await this.previewDetail(itemId); + const reviews = await this.previewReviews(itemId, options); + return { + itemId, + source: "hybrid", + detail: detail.detail, + pagination: reviews.pagination, + reviews: reviews.reviews + }; + } + }; +} + +describe("TmallSessionManagerService", () => { + it("imports a manual session and exposes healthy ops state", async () => { + const onSessionReady = vi.fn(); + const service = createLiveService(); + const manager = new TmallSessionManagerService(service, { + onSessionReady + }); + + const state = await manager.importManualSession({ + cookieHeader: "_m_h5_tk=masked_token_123;", + detailTemplateUrl: "https://detail.tmall.com/item.htm?id=934454505228", + reviewsTemplateUrl: + "https://h5api.m.tmall.com/h5/mtop.taobao.rate.detaillist.get/6.0/?data=%7B%22auctionNumId%22%3A%22934454505228%22%7D" + }); + + expect(onSessionReady).toHaveBeenCalledTimes(1); + expect(state).toMatchObject({ + status: "healthy", + publicNote: "天猫会话由运维后台维护，当前可用。", + session: { + configured: true + } + }); + }); + + it("marks the manager for manual action when the session has expired", async () => { + const onSessionUnavailable = vi.fn(); + const service = createLiveService(); + service.importSession({ + cookieHeader: "_m_h5_tk=masked_token_123;", + detailTemplateUrl: "https://detail.tmall.com/item.htm?id=934454505228", + reviewsTemplateUrl: + "https://h5api.m.tmall.com/h5/mtop.taobao.rate.detaillist.get/6.0/?data=%7B%22auctionNumId%22%3A%22934454505228%22%7D" + }); + service.previewDetail = vi.fn(async () => { + const error = new Error("Tmall detail session appears invalid.") as Error & { + statusCode: number; + }; + error.statusCode = 409; + throw error; + }); + + const manager = new TmallSessionManagerService(service, { + onSessionUnavailable + }); + + const result = await manager.runHealthCheck("ops"); + + expect(onSessionUnavailable).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + recovered: false, + state: { + status: "manual_action_required", + pendingManualAction: true, + lastFailureCode: "SESSION_REQUIRED", + publicNote: "天猫会话需要运维重新登录并更新 Cookie。" + } + }); + }); +}); diff --git a/apps/api/src/platforms/tmall/session-manager.ts b/apps/api/src/platforms/tmall/session-manager.ts new file mode 100644 index 0000000..8a6c660 --- /dev/null +++ b/apps/api/src/platforms/tmall/session-manager.ts @@ -0,0 +1,391 @@ +import { isTmallLiveError } from "./live-session"; +import type { + TmallLiveService, + TmallLiveSessionInput, + TmallSessionManager, + TmallSessionManagerConfigInput, + TmallSessionManagerRunResult, + TmallSessionManagerState, + TmallSessionManagerStatus +} from "./types"; + +const DEFAULT_CHECK_INTERVAL_MS = 10 * 60 * 1000; +const DEFAULT_HEARTBEAT_ITEM_ID = "934454505228"; + +type StoredTmallSessionManagerConfig = { + enabled: boolean; + heartbeatItemId?: string | undefined; + checkIntervalMs: number; + configuredAt?: string | undefined; +}; + +type TmallSessionManagerCallbacks = { + onSessionReady?: () => void; + onSessionUnavailable?: () => void; +}; + +function nowIso(): string { + return new Date().toISOString(); +} + +function readBooleanEnv(name: string, fallback: boolean): boolean { + const value = process.env[name]?.trim().toLowerCase(); + if (!value) { + return fallback; + } + + return value === "1" || value === "true" || value === "yes" || value === "on"; +} + +function readPositiveIntegerEnv(name: string, fallback: number): number { + const value = process.env[name]?.trim(); + if (!value) { + return fallback; + } + + const parsed = Number.parseInt(value, 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function normalizeOptionalString(value: string | null | undefined): string | undefined { + const normalized = value?.trim(); + return normalized ? normalized : undefined; +} + +function buildConfigFromEnv(): StoredTmallSessionManagerConfig { + const heartbeatItemId = + normalizeOptionalString(process.env.TMALL_OPS_HEARTBEAT_ITEM_ID) ?? DEFAULT_HEARTBEAT_ITEM_ID; + + return { + enabled: readBooleanEnv("TMALL_OPS_ENABLED", true), + heartbeatItemId, + checkIntervalMs: readPositiveIntegerEnv( + "TMALL_OPS_CHECK_INTERVAL_MS", + DEFAULT_CHECK_INTERVAL_MS + ), + configuredAt: heartbeatItemId ? nowIso() : undefined + }; +} + +function createBaseState( + liveService: TmallLiveService, + config: StoredTmallSessionManagerConfig +): TmallSessionManagerState { + return { + status: liveService.getSessionSummary().configured ? "degraded" : "idle", + enabled: config.enabled, + heartbeatItemId: config.heartbeatItemId, + checkIntervalMs: config.checkIntervalMs, + pendingManualAction: false, + note: liveService.getSessionSummary().configured + ? "天猫会话已导入，等待后台健康检查确认。" + : "天猫会话管理器已初始化，等待首次会话注入。", + publicNote: liveService.getSessionSummary().configured + ? "天猫会话正在后台校验。" + : "天猫会话由运维后台维护，当前尚未就绪。", + configuredAt: config.configuredAt, + session: liveService.getSessionSummary() + }; +} + +function resolveFailureDescriptor(error: unknown): { + code: string; + status: TmallSessionManagerStatus; + pendingManualAction: boolean; + publicNote: string; +} { + const message = error instanceof Error ? error.message : "unknown error"; + const normalizedMessage = message.toLowerCase(); + + if ( + (isTmallLiveError(error) && error.statusCode === 409) || + normalizedMessage.includes("session appears invalid") || + normalizedMessage.includes("live session is not configured") || + normalizedMessage.includes("cookie/header") + ) { + return { + code: "SESSION_REQUIRED", + status: "manual_action_required", + pendingManualAction: true, + publicNote: "天猫会话需要运维重新登录并更新 Cookie。" + }; + } + + if ( + normalizedMessage.includes("template is missing") || + normalizedMessage.includes("capture a fresh") || + normalizedMessage.includes("does not expose a page field") + ) { + return { + code: "TEMPLATE_REQUIRED", + status: "manual_action_required", + pendingManualAction: true, + publicNote: "天猫会话缺少有效模板，运维后台需要刷新模板。" + }; + } + + return { + code: + isTmallLiveError(error) && typeof error.statusCode === "number" + ? `TMALL_${error.statusCode}` + : "TMALL_HEALTH_CHECK_FAILED", + status: "degraded", + pendingManualAction: false, + publicNote: "天猫会话健康检查失败，运维后台需要复核。" + }; +} + +export class TmallSessionManagerService implements TmallSessionManager { + private config = buildConfigFromEnv(); + private state: TmallSessionManagerState; + private timer: NodeJS.Timeout | null = null; + + constructor( + private readonly liveService: TmallLiveService, + private readonly callbacks: TmallSessionManagerCallbacks = {} + ) { + this.state = createBaseState(liveService, this.config); + this.restartScheduler(); + } + + getState(): TmallSessionManagerState { + return { + ...this.state, + session: this.liveService.getSessionSummary() + }; + } + + configure(input: TmallSessionManagerConfigInput): TmallSessionManagerState { + const nextConfig: StoredTmallSessionManagerConfig = { + ...this.config, + ...(typeof input.enabled === "boolean" ? { enabled: input.enabled } : {}), + ...(input.heartbeatItemId !== undefined + ? { heartbeatItemId: normalizeOptionalString(input.heartbeatItemId) } + : {}), + ...(input.checkIntervalMs !== undefined && input.checkIntervalMs !== null + ? { + checkIntervalMs: + Number.isInteger(input.checkIntervalMs) && input.checkIntervalMs > 0 + ? input.checkIntervalMs + : this.config.checkIntervalMs + } + : {}), + configuredAt: nowIso() + }; + + this.config = nextConfig; + this.state = { + ...this.state, + enabled: nextConfig.enabled, + heartbeatItemId: nextConfig.heartbeatItemId, + checkIntervalMs: nextConfig.checkIntervalMs, + configuredAt: nextConfig.configuredAt, + note: "天猫运维配置已更新。", + publicNote: this.state.publicNote, + session: this.liveService.getSessionSummary() + }; + this.restartScheduler(); + return this.getState(); + } + + clearConfig(): TmallSessionManagerState { + this.config = { + enabled: false, + heartbeatItemId: DEFAULT_HEARTBEAT_ITEM_ID, + checkIntervalMs: DEFAULT_CHECK_INTERVAL_MS + }; + this.state = { + ...this.state, + enabled: false, + heartbeatItemId: DEFAULT_HEARTBEAT_ITEM_ID, + checkIntervalMs: DEFAULT_CHECK_INTERVAL_MS, + configuredAt: nowIso(), + note: "天猫运维配置已清空，自动巡检已停用。", + publicNote: this.liveService.getSessionSummary().configured + ? "天猫会话已存在，但后台自动巡检已停用。" + : "天猫会话由运维后台维护，当前自动巡检未启用。", + pendingManualAction: false, + session: this.liveService.getSessionSummary() + }; + this.restartScheduler(); + return this.getState(); + } + + async importManualSession( + input: TmallLiveSessionInput, + source = "ops-manual" + ): Promise { + this.liveService.importSession(input); + this.callbacks.onSessionReady?.(); + this.state = { + ...this.state, + status: "healthy", + pendingManualAction: false, + note: `天猫会话已通过 ${source} 更新，等待下一轮健康检查。`, + publicNote: "天猫会话由运维后台维护，当前可用。", + lastHealthyAt: nowIso(), + lastFailureCode: undefined, + lastFailureMessage: undefined, + session: this.liveService.getSessionSummary() + }; + return this.getState(); + } + + clearManagedSession(reason = "ops-manual-clear"): TmallSessionManagerState { + this.liveService.clearSession(); + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: "idle", + pendingManualAction: false, + note: `天猫会话已清理：${reason}。`, + publicNote: "天猫会话由运维后台维护，当前尚未就绪。", + session: this.liveService.getSessionSummary() + }; + return this.getState(); + } + + async runHealthCheck(trigger = "manual"): Promise { + this.state = { + ...this.state, + lastCheckAt: nowIso(), + session: this.liveService.getSessionSummary() + }; + + const summary = this.liveService.getSessionSummary(); + if (!summary.configured) { + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: "manual_action_required", + pendingManualAction: true, + note: "当前没有可用天猫会话，等待运维注入。", + publicNote: "天猫会话由运维后台维护，当前尚未就绪。", + session: summary + }; + return { + state: this.getState(), + recovered: false + }; + } + + if (!summary.detailTemplate.available || !summary.reviewsTemplate.available) { + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: "manual_action_required", + pendingManualAction: true, + note: "天猫详情或评论模板缺失，需要运维刷新模板。", + publicNote: "天猫会话缺少有效模板，运维后台正在处理。", + session: summary + }; + return { + state: this.getState(), + recovered: false + }; + } + + try { + await this.verifyCurrentSession(); + this.callbacks.onSessionReady?.(); + this.state = { + ...this.state, + status: "healthy", + pendingManualAction: false, + note: `天猫会话健康检查通过：${trigger}。`, + publicNote: "天猫会话由运维后台维护，当前可用。", + lastHealthyAt: nowIso(), + lastFailureCode: undefined, + lastFailureMessage: undefined, + session: this.liveService.getSessionSummary() + }; + return { + state: this.getState(), + recovered: false + }; + } catch (error) { + await this.handleFailure(error); + return { + state: this.getState(), + recovered: false + }; + } + } + + async handleLiveFailure( + error: unknown, + _context: { + capability?: "search" | "detail" | "reviews"; + taskId?: string | undefined; + trigger?: string | undefined; + } = {} + ): Promise { + await this.handleFailure(error); + return false; + } + + shutdown(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + } + + private restartScheduler(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + + if (!this.config.enabled) { + return; + } + + this.timer = setInterval(() => { + void this.runHealthCheck("scheduler"); + }, this.config.checkIntervalMs); + this.timer.unref?.(); + } + + private resolveHeartbeatItemId(): string | null { + const summary = this.liveService.getSessionSummary(); + return ( + this.config.heartbeatItemId ?? + summary.detailTemplate.itemId ?? + summary.reviewsTemplate.itemId ?? + null + ); + } + + private async verifyCurrentSession(): Promise { + const summary = this.liveService.getSessionSummary(); + const heartbeatItemId = this.resolveHeartbeatItemId(); + + if (!summary.configured || !heartbeatItemId) { + throw new Error("天猫会话缺少可校验的详情/评论模板。"); + } + + await this.liveService.previewDetail(heartbeatItemId); + await this.liveService.previewReviews(heartbeatItemId, { + commentCount: 1, + maxPages: 1 + }); + } + + private async handleFailure(error: unknown): Promise { + const message = error instanceof Error ? error.message : "unknown error"; + const failure = resolveFailureDescriptor(error); + + this.callbacks.onSessionUnavailable?.(); + this.state = { + ...this.state, + status: failure.status, + pendingManualAction: failure.pendingManualAction, + lastFailureCode: failure.code, + lastFailureMessage: message, + note: `天猫会话检测失败：${message}`, + publicNote: failure.publicNote, + session: this.liveService.getSessionSummary() + }; + } +} diff --git a/apps/api/src/platforms/tmall/types.ts b/apps/api/src/platforms/tmall/types.ts new file mode 100644 index 0000000..c645794 --- /dev/null +++ b/apps/api/src/platforms/tmall/types.ts @@ -0,0 +1,182 @@ +import type { CandidateRecord } from "@cross-ai/domain"; + +export interface TmallTemplateSummary { + available: boolean; + api?: string | undefined; + itemId?: string | undefined; +} + +export interface TmallLiveSessionInput { + cookieHeader: string; + userAgent?: string | undefined; + detailTemplateUrl?: string | undefined; + reviewsTemplateUrl?: string | undefined; + detailReferer?: string | undefined; +} + +export interface TmallLiveSessionSummary { + configured: boolean; + importedAt?: string | undefined; + hasCookie: boolean; + userAgent?: string | undefined; + detailTemplate: TmallTemplateSummary; + reviewsTemplate: TmallTemplateSummary; +} + +export type TmallSessionManagerStatus = + | "idle" + | "healthy" + | "degraded" + | "manual_action_required"; + +export interface TmallSessionManagerConfigInput { + enabled?: boolean | undefined; + heartbeatItemId?: string | null | undefined; + checkIntervalMs?: number | null | undefined; +} + +export interface TmallSessionManagerState { + status: TmallSessionManagerStatus; + enabled: boolean; + heartbeatItemId?: string | undefined; + checkIntervalMs: number; + pendingManualAction: boolean; + note: string; + publicNote: string; + configuredAt?: string | undefined; + lastCheckAt?: string | undefined; + lastHealthyAt?: string | undefined; + lastFailureCode?: string | undefined; + lastFailureMessage?: string | undefined; + session: TmallLiveSessionSummary; +} + +export interface TmallSessionManagerRunResult { + state: TmallSessionManagerState; + recovered: boolean; +} + +export interface TmallProductDetailSnapshot { + itemId: string; + title: string | null; + subtitle: string | null; + price: string | null; + originalPrice: string | null; + shopName: string | null; + shopUrl: string | null; + sellerType: string | null; + categoryPath: string[]; + mainImage: string | null; + salesDesc: string | null; + commentCount: string | null; +} + +export interface TmallReviewTagSnapshot { + name: string; + count: string | null; +} + +export interface TmallReviewCommentSnapshot { + id: string; + content: string; + date: string | null; + userNick: string | null; + userAvatar: string | null; + skuText: string[]; + pictureUrls: string[]; + videoUrls: string[]; + likeCount: string | null; + reply: string | null; + appendContent: string | null; + appendPictureUrls: string[]; +} + +export interface TmallProductReviewsSnapshot { + itemId: string; + total: string | null; + hasNext: boolean; + allCount: string | null; + pictureCount: string | null; + appendCount: string | null; + tags: TmallReviewTagSnapshot[]; + comments: TmallReviewCommentSnapshot[]; +} + +export interface TmallReviewsPreviewOptions { + commentCount?: number | undefined; + page?: number | undefined; + maxPages?: number | undefined; +} + +export interface TmallReviewsPaginationSummary { + requestedPage: number; + requestedCommentCount: number; + maxPages: number; + pagesFetched: number; + pageKey?: string | undefined; +} + +export interface TmallSearchPreviewResult { + query: string; + source: "html"; + candidateCount: number; + candidates: CandidateRecord[]; +} + +export interface TmallDetailPreviewResult { + itemId: string; + source: "api" | "html"; + detail: TmallProductDetailSnapshot; +} + +export interface TmallReviewsPreviewResult { + itemId: string; + source: "api"; + pagination: TmallReviewsPaginationSummary; + reviews: TmallProductReviewsSnapshot; +} + +export interface TmallProductPreviewResult { + itemId: string; + source: "api" | "html" | "hybrid"; + detail: TmallProductDetailSnapshot; + pagination: TmallReviewsPaginationSummary; + reviews: TmallProductReviewsSnapshot; +} + +export interface TmallLiveService { + getSessionSummary(): TmallLiveSessionSummary; + importSession(input: TmallLiveSessionInput): TmallLiveSessionSummary; + clearSession(): void; + previewSearch(query: string): Promise; + previewDetail(itemId: string): Promise; + previewReviews( + itemId: string, + options?: number | TmallReviewsPreviewOptions + ): Promise; + previewProduct( + itemId: string, + options?: number | TmallReviewsPreviewOptions + ): Promise; +} + +export interface TmallSessionManager { + getState(): TmallSessionManagerState; + configure(input: TmallSessionManagerConfigInput): TmallSessionManagerState; + clearConfig(): TmallSessionManagerState; + importManualSession( + input: TmallLiveSessionInput, + source?: string + ): Promise; + clearManagedSession(reason?: string): TmallSessionManagerState; + runHealthCheck(trigger?: string): Promise; + handleLiveFailure( + error: unknown, + context?: { + capability?: "search" | "detail" | "reviews"; + taskId?: string | undefined; + trigger?: string | undefined; + } + ): Promise; + shutdown(): void; +} diff --git a/apps/api/src/platforms/tmall/utils.ts b/apps/api/src/platforms/tmall/utils.ts new file mode 100644 index 0000000..c742aca --- /dev/null +++ b/apps/api/src/platforms/tmall/utils.ts @@ -0,0 +1,187 @@ +type PathSegment = string | number; + +export function asRecord(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + + return value as Record; +} + +export function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +export function stringFrom(value: unknown): string | null { + if (typeof value === "string") { + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; + } + + if (typeof value === "number" || typeof value === "boolean") { + return String(value); + } + + return null; +} + +export function firstString(...values: unknown[]): string | null { + for (const value of values) { + const resolved = stringFrom(value); + if (resolved) { + return resolved; + } + } + + return null; +} + +export function parseBooleanish(value: unknown): boolean { + const normalized = stringFrom(value)?.toLowerCase(); + return normalized === "true" || normalized === "1" || normalized === "yes"; +} + +export function decodeHtmlEntities(value: string): string { + return value + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/</g, "<") + .replace(/>/g, ">"); +} + +export function stripTags(value: string): string { + return value.replace(/<[^>]+>/g, " "); +} + +export function normalizeWhitespace(value: string): string { + return decodeHtmlEntities(stripTags(value)).replace(/\s+/g, " ").trim(); +} + +export function uniqueStrings(values: Array): string[] { + return Array.from( + new Set( + values + .map((value) => value?.trim()) + .filter((value): value is string => Boolean(value)) + ) + ); +} + +export function absolutizeTmallUrl(value: string | null | undefined): string | null { + if (!value) { + return null; + } + + if (value.startsWith("http://") || value.startsWith("https://")) { + return value; + } + + if (value.startsWith("//")) { + return `https:${value}`; + } + + if (value.startsWith("/")) { + return `https://detail.tmall.com${value}`; + } + + return `https://${value}`; +} + +export function parseEmbeddedJson(value: string | null): Record | null { + if (!value) { + return null; + } + + try { + const parsed = JSON.parse(value) as unknown; + return asRecord(parsed); + } catch { + const jsonpMatch = value.match(/^[^(]+\(([\s\S]*)\)\s*;?\s*$/); + if (!jsonpMatch?.[1]) { + return null; + } + + try { + const parsed = JSON.parse(jsonpMatch[1]) as unknown; + return asRecord(parsed); + } catch { + return null; + } + } +} + +export function readNestedJsonRecord(value: unknown): Record | null { + if (typeof value === "string") { + return parseEmbeddedJson(value); + } + + return asRecord(value); +} + +export function readQueryData(url: URL): Record | null { + return ( + parseEmbeddedJson(url.searchParams.get("data")) ?? + parseEmbeddedJson(url.searchParams.get("body")) + ); +} + +export function findFirstRecordKey( + record: Record | null, + candidates: string[] +): string | null { + if (!record) { + return null; + } + + for (const candidate of candidates) { + if (candidate in record) { + return candidate; + } + } + + return null; +} + +export function withUpdatedQueryData( + url: URL, + updater: (data: Record) => Record +): string { + const nextUrl = new URL(url.toString()); + const paramName = nextUrl.searchParams.has("data") ? "data" : "body"; + const currentData = readQueryData(nextUrl) ?? {}; + nextUrl.searchParams.set(paramName, JSON.stringify(updater(currentData))); + return nextUrl.toString(); +} + +export function readPath(root: unknown, path: PathSegment[]): unknown { + let current: unknown = root; + + for (const segment of path) { + if (typeof segment === "number") { + const currentArray = asArray(current); + current = currentArray[segment]; + continue; + } + + const currentRecord = asRecord(current); + current = currentRecord?.[segment]; + } + + return current; +} + +export function pickFirstStringByPaths( + root: unknown, + paths: PathSegment[][] +): string | null { + for (const path of paths) { + const value = stringFrom(readPath(root, path)); + if (value) { + return normalizeWhitespace(value); + } + } + + return null; +} diff --git a/apps/api/src/review-sampling.test.ts b/apps/api/src/review-sampling.test.ts new file mode 100644 index 0000000..0a0b0e1 --- /dev/null +++ b/apps/api/src/review-sampling.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; + +import { + buildReviewBudgetPlan, + sampleReviewComments, + type ReviewSamplingComment +} from "./review-sampling"; + +function buildComment( + id: string, + content: string, + score: string, + authorLabel = "PLUS" +): ReviewSamplingComment { + return { + id, + content, + score, + createdAt: "2026-04-03 10:00:00", + authorLabel + }; +} + +describe("review-sampling", () => { + it("distributes the task-level review budget across selected links", () => { + const budgets = buildReviewBudgetPlan(["a", "b", "c"], 100, 5); + + expect(Array.from(budgets.entries())).toEqual([ + ["a", 2], + ["b", 2], + ["c", 1] + ]); + expect(Array.from(budgets.values()).reduce((sum, value) => sum + value, 0)).toBe(5); + }); + + it("allows zero-review allocations when the task total limit is below selected link count", () => { + const budgets = buildReviewBudgetPlan(["a", "b", "c", "d", "e"], 100, 2); + + expect(Array.from(budgets.entries())).toEqual([ + ["a", 1], + ["b", 1], + ["c", 0], + ["d", 0], + ["e", 0] + ]); + }); + + it("samples comments with 40/30/30 latest-hot-negative targets when all buckets are available", () => { + const comments = [ + buildComment("latest-1", "物流很快，系统很流畅。", "5"), + buildComment("latest-2", "拍照效果不错，续航也稳定。", "5"), + buildComment("latest-3", "屏幕亮度高，外观好看。", "5"), + buildComment("latest-4", "音质清晰，做工扎实。", "5"), + buildComment("hot-1", "这是我写过最长的一条好评，细节描述非常完整，便于归入热评桶。", "5"), + buildComment("hot-2", "另一条较长的高分评论，包含更多使用场景和优点描述。", "5"), + buildComment("hot-3", "第三条高信息量评论，强调拍照、手感和续航表现。", "4"), + buildComment("negative-1", "发热明显，打游戏会卡顿。", "2"), + buildComment("negative-2", "续航一般，晚上掉电有点快。", "3"), + buildComment("negative-3", "边框有瑕疵，体验一般。", "2") + ]; + + const sampled = sampleReviewComments(comments, 10, ["发热", "卡顿", "掉电", "瑕疵"]); + + expect(sampled.actualCount).toBe(10); + expect(sampled.sampleInsufficient).toBe(false); + expect(sampled.bucketCounts).toEqual({ + latest: 4, + hot: 3, + negative: 3 + }); + }); + + it("redistributes missing negative quota to the remaining buckets", () => { + const comments = [ + buildComment("c-1", "外观好看。", "5"), + buildComment("c-2", "系统流畅。", "5"), + buildComment("c-3", "续航不错。", "5"), + buildComment("c-4", "拍照清晰。", "5"), + buildComment("c-5", "做工扎实。", "4") + ]; + + const sampled = sampleReviewComments(comments, 5, ["发热", "卡顿"]); + + expect(sampled.actualCount).toBe(5); + expect(sampled.sampleInsufficient).toBe(false); + expect(sampled.bucketCounts).toEqual({ + latest: 3, + hot: 2, + negative: 0 + }); + }); +}); diff --git a/apps/api/src/review-sampling.ts b/apps/api/src/review-sampling.ts new file mode 100644 index 0000000..8cfe1d3 --- /dev/null +++ b/apps/api/src/review-sampling.ts @@ -0,0 +1,312 @@ +export type ReviewSamplingBucket = "latest" | "hot" | "negative"; + +export interface ReviewSamplingComment { + id: string; + content: string; + score: string | null; + createdAt: string | null; + authorLabel: string | null; +} + +export interface SampledReviewComment { + bucket: ReviewSamplingBucket; + comment: T; +} + +export interface ReviewSamplingResult { + targetCount: number; + actualCount: number; + sampleInsufficient: boolean; + bucketCounts: Record; + comments: Array>; +} + +const BUCKET_WEIGHTS: Record = { + latest: 0.4, + hot: 0.3, + negative: 0.3 +}; + +const QUOTA_PRIORITY: ReviewSamplingBucket[] = ["latest", "hot", "negative"]; +const SELECTION_PRIORITY: ReviewSamplingBucket[] = ["negative", "hot", "latest"]; +const FALLBACK_PRIORITY: ReviewSamplingBucket[] = ["latest", "hot", "negative"]; + +function normalizeBudget(value: number): number { + if (!Number.isFinite(value) || value <= 0) { + return 0; + } + + return Math.floor(value); +} + +function parseScore(score: string | null): number | null { + if (!score) { + return null; + } + + const parsed = Number.parseInt(score, 10); + return Number.isNaN(parsed) ? null : parsed; +} + +function isNegativeComment( + comment: ReviewSamplingComment, + negativeKeywords: string[] +): boolean { + const score = parseScore(comment.score); + if (score !== null && score <= 3) { + return true; + } + + const normalizedContent = comment.content.toLowerCase(); + return negativeKeywords.some((keyword) => normalizedContent.includes(keyword.toLowerCase())); +} + +function getHotScore(comment: ReviewSamplingComment): number { + const score = parseScore(comment.score); + const scoreBonus = score !== null && score >= 4 ? 16 : score === 3 ? 8 : 0; + const authorBonus = comment.authorLabel ? 12 : 0; + const timestampBonus = comment.createdAt ? 4 : 0; + + return Math.min(comment.content.length, 160) + scoreBonus + authorBonus + timestampBonus; +} + +function dedupeComments(comments: T[]): T[] { + const deduped = new Map(); + for (const comment of comments) { + if (!deduped.has(comment.id)) { + deduped.set(comment.id, comment); + } + } + + return Array.from(deduped.values()); +} + +function buildBucketTargets( + targetCount: number, + availability: Record +): Record { + const normalizedTarget = normalizeBudget(targetCount); + const activeBuckets = QUOTA_PRIORITY.filter((bucket) => availability[bucket]); + const targets: Record = { + latest: 0, + hot: 0, + negative: 0 + }; + + if (normalizedTarget === 0 || activeBuckets.length === 0) { + return targets; + } + + const activeWeight = activeBuckets.reduce((sum, bucket) => sum + BUCKET_WEIGHTS[bucket], 0); + let allocated = 0; + const fractions = activeBuckets.map((bucket) => { + const raw = (normalizedTarget * BUCKET_WEIGHTS[bucket]) / activeWeight; + const base = Math.floor(raw); + targets[bucket] = base; + allocated += base; + + return { + bucket, + fraction: raw - base + }; + }); + + let remainder = normalizedTarget - allocated; + fractions + .sort( + (left, right) => + right.fraction - left.fraction || + QUOTA_PRIORITY.indexOf(left.bucket) - QUOTA_PRIORITY.indexOf(right.bucket) + ) + .forEach(({ bucket }) => { + if (remainder <= 0) { + return; + } + + targets[bucket] += 1; + remainder -= 1; + }); + + return targets; +} + +function hasUnusedComment(pool: T[], usedIds: Set): boolean { + return pool.some((comment) => !usedIds.has(comment.id)); +} + +function pickNextComment( + bucket: ReviewSamplingBucket, + pool: T[], + usedIds: Set, + selected: Array>, + bucketCounts: Record +): boolean { + for (const comment of pool) { + if (usedIds.has(comment.id)) { + continue; + } + + usedIds.add(comment.id); + selected.push({ + bucket, + comment + }); + bucketCounts[bucket] += 1; + return true; + } + + return false; +} + +export function buildReviewBudgetPlan( + candidateIds: string[], + perLinkLimit: number, + taskTotalLimit: number +): Map { + const uniqueCandidateIds = Array.from(new Set(candidateIds)); + const budgets = new Map( + uniqueCandidateIds.map((candidateId) => [candidateId, 0] as const) + ); + const normalizedPerLinkLimit = normalizeBudget(perLinkLimit); + const normalizedTaskTotalLimit = normalizeBudget(taskTotalLimit); + + if ( + uniqueCandidateIds.length === 0 || + normalizedPerLinkLimit === 0 || + normalizedTaskTotalLimit === 0 + ) { + return budgets; + } + + const totalBudget = Math.min( + normalizedTaskTotalLimit, + uniqueCandidateIds.length * normalizedPerLinkLimit + ); + const baseAllocation = Math.min( + normalizedPerLinkLimit, + Math.floor(totalBudget / uniqueCandidateIds.length) + ); + + for (const candidateId of uniqueCandidateIds) { + budgets.set(candidateId, baseAllocation); + } + + let remainder = totalBudget - baseAllocation * uniqueCandidateIds.length; + while (remainder > 0) { + let assigned = false; + + for (const candidateId of uniqueCandidateIds) { + const currentBudget = budgets.get(candidateId) ?? 0; + if (currentBudget >= normalizedPerLinkLimit) { + continue; + } + + budgets.set(candidateId, currentBudget + 1); + remainder -= 1; + assigned = true; + + if (remainder === 0) { + break; + } + } + + if (!assigned) { + break; + } + } + + return budgets; +} + +export function sampleReviewComments( + comments: T[], + targetCount: number, + negativeKeywords: string[] +): ReviewSamplingResult { + const uniqueComments = dedupeComments(comments); + const normalizedTarget = normalizeBudget(targetCount); + const bucketCounts: Record = { + latest: 0, + hot: 0, + negative: 0 + }; + + if (normalizedTarget === 0 || uniqueComments.length === 0) { + return { + targetCount: normalizedTarget, + actualCount: 0, + sampleInsufficient: normalizedTarget > 0, + bucketCounts, + comments: [] + }; + } + + const negativeCommentIds = new Set( + uniqueComments + .filter((comment) => isNegativeComment(comment, negativeKeywords)) + .map((comment) => comment.id) + ); + const negativePool = uniqueComments.filter((comment) => negativeCommentIds.has(comment.id)); + const latestPool = uniqueComments.filter((comment) => !negativeCommentIds.has(comment.id)); + const hotPool = [...latestPool].sort( + (left, right) => + getHotScore(right) - getHotScore(left) || + latestPool.findIndex((comment) => comment.id === left.id) - + latestPool.findIndex((comment) => comment.id === right.id) + ); + const bucketPools: Record = { + latest: latestPool, + hot: hotPool, + negative: negativePool + }; + const bucketTargets = buildBucketTargets(normalizedTarget, { + latest: latestPool.length > 0, + hot: hotPool.length > 0, + negative: negativePool.length > 0 + }); + const selected: Array> = []; + const usedIds = new Set(); + + for (const bucket of SELECTION_PRIORITY) { + while ( + bucketCounts[bucket] < bucketTargets[bucket] && + pickNextComment(bucket, bucketPools[bucket], usedIds, selected, bucketCounts) + ) { + // Continue until this bucket reaches its target or runs out of comments. + } + } + + while (selected.length < normalizedTarget) { + const deficitBucket = SELECTION_PRIORITY.find( + (bucket) => + bucketCounts[bucket] < bucketTargets[bucket] && + hasUnusedComment(bucketPools[bucket], usedIds) + ); + + if (deficitBucket) { + if (pickNextComment(deficitBucket, bucketPools[deficitBucket], usedIds, selected, bucketCounts)) { + continue; + } + } + + const fallbackBucket = FALLBACK_PRIORITY.find((bucket) => + hasUnusedComment(bucketPools[bucket], usedIds) + ); + + if (!fallbackBucket) { + break; + } + + if (!pickNextComment(fallbackBucket, bucketPools[fallbackBucket], usedIds, selected, bucketCounts)) { + break; + } + } + + return { + targetCount: normalizedTarget, + actualCount: selected.length, + sampleInsufficient: selected.length < normalizedTarget, + bucketCounts, + comments: selected + }; +} diff --git a/apps/api/src/server.dual-platform-live.test.ts b/apps/api/src/server.dual-platform-live.test.ts new file mode 100644 index 0000000..521d7ff --- /dev/null +++ b/apps/api/src/server.dual-platform-live.test.ts @@ -0,0 +1,976 @@ +import { describe, expect, it } from "vitest"; + +import type { + JdDetailPreviewResult, + JdLiveService, + JdLiveSessionSummary, + JdProductPreviewResult, + JdReviewsPreviewOptions, + JdReviewsPreviewResult, + JdSearchPreviewResult +} from "./platforms/jd/types"; +import type { + TmallDetailPreviewResult, + TmallLiveService, + TmallLiveSessionSummary, + TmallProductPreviewResult, + TmallReviewsPreviewResult, + TmallSearchPreviewResult +} from "./platforms/tmall/types"; +import { createServer } from "./server"; + +async function createTask(app: ReturnType, query: string) { + const response = await app.inject({ + method: "POST", + url: "/api/tasks", + payload: { + query, + perLinkLimit: 100, + taskTotalLimit: 500 + } + }); + + return response.json().task; +} + +async function importDualLiveSessions(app: ReturnType) { + const jdResponse = await app.inject({ + method: "POST", + url: "/api/platforms/jd/live-session", + payload: { + cookieHeader: "thor=masked; pin=masked;", + searchApiTemplateUrl: + "https://api.m.jd.com/?functionId=pc_search_searchWare&body=%7B%22keyword%22:%22iphone%2015%22%7D", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22%7D", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22sku%22:100068388533%7D" + } + }); + const tmallResponse = await app.inject({ + method: "POST", + url: "/api/platforms/tmall/live-session", + payload: { + cookieHeader: "_tb_token_=masked;", + detailTemplateUrl: "https://detail.tmall.com/item.htm?id=934454505228", + reviewsTemplateUrl: + "https://h5api.m.tmall.com/h5/mtop.taobao.rate.detaillist.get/6.0/?data=%7B%22auctionNumId%22%3A%22934454505228%22%7D" + } + }); + + expect(jdResponse.statusCode).toBe(200); + expect(tmallResponse.statusCode).toBe(200); +} + +function createJdLiveServiceStub(overrides: Partial = {}): JdLiveService { + let summary: JdLiveSessionSummary = { + configured: false, + hasCookie: false, + searchApiTemplate: { available: false }, + detailTemplate: { available: false }, + reviewsTemplate: { available: false } + }; + + return { + getSessionSummary() { + return overrides.getSessionSummary?.() ?? summary; + }, + importSession(input) { + if (overrides.importSession) { + return overrides.importSession(input); + } + + summary = { + configured: true, + importedAt: "2026-04-03T10:00:00.000Z", + hasCookie: true, + userAgent: input.userAgent ?? "stub-user-agent", + searchApiTemplate: { available: Boolean(input.searchApiTemplateUrl) }, + detailTemplate: { available: Boolean(input.detailTemplateUrl) }, + reviewsTemplate: { available: Boolean(input.reviewsTemplateUrl) } + }; + return summary; + }, + clearSession() { + if (overrides.clearSession) { + overrides.clearSession(); + return; + } + + summary = { + configured: false, + hasCookie: false, + searchApiTemplate: { available: false }, + detailTemplate: { available: false }, + reviewsTemplate: { available: false } + }; + }, + async previewSearch(query) { + if (overrides.previewSearch) { + return overrides.previewSearch(query); + } + + const preview: JdSearchPreviewResult = { + query, + source: "html", + candidateCount: 1, + candidates: [ + { + candidateId: "jd-100068388533", + platform: "jd", + title: "Apple/苹果 iPhone 15", + price: 3898, + priceLabel: "CNY 3898", + storeName: "Apple产品京东自营旗舰店", + productUrl: "https://item.jd.com/100068388533.html", + imageUrl: "https://img14.360buyimg.com/n2/jfs/t1/example.jpg", + salesHint: "已售500万+", + specLabel: "128GB", + highlights: ["A16仿生芯片"] + } + ] + }; + + return preview; + }, + async previewDetail(skuId) { + if (overrides.previewDetail) { + return overrides.previewDetail(skuId); + } + + const preview: JdDetailPreviewResult = { + skuId, + source: "api", + detail: { + skuId, + title: "Apple/苹果 iPhone 15", + price: "4398.00", + originalPrice: "4599.00", + estimatedPrice: "3898.00", + shopName: "Apple产品京东自营旗舰店", + vendorId: null, + categoryPath: ["手机通讯", "手机", "Apple"], + stockState: "现货", + mainImage: "https://img14.360buyimg.com/n2/jfs/t1/example.jpg", + averageScore: "4.9" + } + }; + + return preview; + }, + async previewReviews(skuId, options) { + if (overrides.previewReviews) { + return overrides.previewReviews(skuId, options); + } + + const requestedCommentCount = + typeof options === "number" ? options : (options?.commentCount ?? 20); + const preview: JdReviewsPreviewResult = { + skuId, + source: "api", + pagination: { + requestedPage: typeof options === "object" ? (options?.page ?? 1) : 1, + requestedCommentCount, + maxPages: typeof options === "object" ? (options?.maxPages ?? 1) : 1, + pagesFetched: 1 + }, + reviews: { + skuId, + total: String(requestedCommentCount), + goodRate: "95%", + pictureCount: "500", + tags: [{ tagId: "jd-tag-1", name: "拍照清晰", count: "1200" }], + comments: [ + { + id: "jd-comment-1", + content: "系统流畅，拍照清晰。", + score: "5", + creationTime: "2026-04-03 10:00:00", + userLevelName: "PLUS会员" + } + ] + } + }; + + return preview; + }, + async previewProduct(skuId, options?: number | JdReviewsPreviewOptions) { + if (overrides.previewProduct) { + return overrides.previewProduct(skuId, options); + } + + const detail = await this.previewDetail(skuId); + const reviews = await this.previewReviews(skuId, options); + const preview: JdProductPreviewResult = { + skuId, + source: "api", + detail: detail.detail, + pagination: reviews.pagination, + reviews: reviews.reviews + }; + + return preview; + } + }; +} + +function createTmallLiveServiceStub(overrides: Partial = {}): TmallLiveService { + let summary: TmallLiveSessionSummary = { + configured: false, + hasCookie: false, + detailTemplate: { available: false }, + reviewsTemplate: { available: false } + }; + + return { + getSessionSummary() { + return overrides.getSessionSummary?.() ?? summary; + }, + importSession(input) { + if (overrides.importSession) { + return overrides.importSession(input); + } + + summary = { + configured: true, + importedAt: "2026-04-03T10:05:00.000Z", + hasCookie: true, + userAgent: input.userAgent ?? "stub-user-agent", + detailTemplate: { available: Boolean(input.detailTemplateUrl), itemId: "934454505228" }, + reviewsTemplate: { available: Boolean(input.reviewsTemplateUrl), itemId: "934454505228" } + }; + return summary; + }, + clearSession() { + if (overrides.clearSession) { + overrides.clearSession(); + return; + } + + summary = { + configured: false, + hasCookie: false, + detailTemplate: { available: false }, + reviewsTemplate: { available: false } + }; + }, + async previewSearch(query) { + if (overrides.previewSearch) { + return overrides.previewSearch(query); + } + + const preview: TmallSearchPreviewResult = { + query, + source: "html", + candidateCount: 1, + candidates: [ + { + candidateId: "tmall-934454505228", + platform: "tmall", + title: "Apple iPhone 15", + price: 4399, + priceLabel: "CNY 4399", + storeName: "Apple 官方旗舰店", + productUrl: "https://detail.tmall.com/item.htm?id=934454505228", + imageUrl: "https://img.alicdn.com/example.jpg", + salesHint: "已售 70万+", + specLabel: "128GB", + highlights: ["天猫", "官方旗舰店"] + } + ] + }; + + return preview; + }, + async previewDetail(itemId) { + if (overrides.previewDetail) { + return overrides.previewDetail(itemId); + } + + const preview: TmallDetailPreviewResult = { + itemId, + source: "html", + detail: { + itemId, + title: "Apple iPhone 15", + subtitle: null, + price: "4399.00", + originalPrice: "4999.00", + shopName: "Apple 官方旗舰店", + shopUrl: null, + sellerType: "tmall", + categoryPath: ["手机", "Apple"], + mainImage: "https://img.alicdn.com/example.jpg", + salesDesc: "已售 70万+", + commentCount: "20万+" + } + }; + + return preview; + }, + async previewReviews(itemId, options) { + if (overrides.previewReviews) { + return overrides.previewReviews(itemId, options); + } + + const requestedCommentCount = + typeof options === "number" ? options : (options?.commentCount ?? 20); + const preview: TmallReviewsPreviewResult = { + itemId, + source: "api", + pagination: { + requestedPage: typeof options === "object" ? (options?.page ?? 1) : 1, + requestedCommentCount, + maxPages: typeof options === "object" ? (options?.maxPages ?? 1) : 1, + pagesFetched: 1, + pageKey: "pageNum" + }, + reviews: { + itemId, + total: String(requestedCommentCount), + hasNext: false, + allCount: "20万+", + pictureCount: "5000+", + appendCount: "1200+", + tags: [{ name: "手感好", count: "5313" }], + comments: [ + { + id: "tmall-review-1", + content: "外观精致，物流很快。", + date: "2026-04-03", + userNick: "Alice", + userAvatar: null, + skuText: ["黑色", "128GB"], + pictureUrls: [], + videoUrls: [], + likeCount: "3", + reply: null, + appendContent: null, + appendPictureUrls: [] + } + ] + } + }; + + return preview; + }, + async previewProduct(itemId, options) { + if (overrides.previewProduct) { + return overrides.previewProduct(itemId, options); + } + + const detail = await this.previewDetail(itemId); + const reviews = await this.previewReviews(itemId, options); + const preview: TmallProductPreviewResult = { + itemId, + source: "hybrid", + detail: detail.detail, + pagination: reviews.pagination, + reviews: reviews.reviews + }; + + return preview; + } + }; +} + +describe("Dual-platform live loop", () => { + it("closes the JD + Tmall live loop and publishes a combined report", async () => { + const app = createServer({ + jdLiveService: createJdLiveServiceStub(), + tmallLiveService: createTmallLiveServiceStub() + }); + await app.ready(); + + await importDualLiveSessions(app); + + const createdTask = await createTask(app, "iPhone 15"); + + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + expect(candidatesResponse.statusCode).toBe(200); + const candidates = candidatesResponse.json().candidates; + expect(candidates.jd[0]).toMatchObject({ + candidateId: "jd-100068388533", + productUrl: "https://item.jd.com/100068388533.html" + }); + expect(candidates.tmall[0]).toMatchObject({ + candidateId: "tmall-934454505228", + productUrl: "https://detail.tmall.com/item.htm?id=934454505228" + }); + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "jd", + candidateIds: [candidates.jd[0].candidateId] + }, + { + platform: "tmall", + candidateIds: [candidates.tmall[0].candidateId] + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + expect(confirmResponse.json().task).toMatchObject({ + taskStatus: "Completed", + defaultReportVersion: 1 + }); + expect(confirmResponse.json().task.platformRuns).toEqual( + expect.arrayContaining([ + expect.objectContaining({ platform: "jd", status: "Completed" }), + expect.objectContaining({ platform: "tmall", status: "Completed" }) + ]) + ); + + const reportResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/report` + }); + + expect(reportResponse.statusCode).toBe(200); + expect(reportResponse.json().report.task_status).toBe("Completed"); + expect(reportResponse.json().report.evidence_index).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + platform: "jd", + source_type: "product", + source_url: "https://item.jd.com/100068388533.html" + }), + expect.objectContaining({ + platform: "tmall", + source_type: "product", + source_url: "https://detail.tmall.com/item.htm?id=934454505228" + }), + expect.objectContaining({ + platform: "jd", + source_type: "review", + review_ref: "jd-comment-1" + }), + expect.objectContaining({ + platform: "tmall", + source_type: "review", + review_ref: "tmall-review-1" + }) + ]) + ); + + await app.close(); + }); + + it("keeps the task confirmable when Tmall live search is blocked but JD succeeds", async () => { + const tmallLiveService = createTmallLiveServiceStub({ + async previewSearch() { + const error = new Error("Tmall search session appears invalid.") as Error & { + statusCode: number; + }; + error.statusCode = 409; + throw error; + } + }); + const app = createServer({ + jdLiveService: createJdLiveServiceStub(), + tmallLiveService + }); + await app.ready(); + await importDualLiveSessions(app); + + const createdTask = await createTask(app, "iPhone 15"); + expect( + createdTask.platformRuns.find((run: { platform: string }) => run.platform === "tmall") + ).toMatchObject({ + platform: "tmall", + status: "SearchBlocked" + }); + expect( + createdTask.platformRuns.find((run: { platform: string }) => run.platform === "jd") + ).toMatchObject({ + platform: "jd", + status: "AwaitingSelection" + }); + + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const candidates = candidatesResponse.json().candidates; + expect(candidates.tmall).toEqual([]); + expect(candidates.jd).toHaveLength(1); + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "jd", + candidateIds: [candidates.jd[0].candidateId] + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + expect(confirmResponse.json().task.taskStatus).toBe("Completed"); + + const reportResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/report` + }); + expect(reportResponse.statusCode).toBe(200); + expect(reportResponse.json().report.quality_flags.blocked_platforms).toEqual(["tmall"]); + + await app.close(); + }); + + it("keeps the task confirmable when Tmall live search returns no result but JD succeeds", async () => { + const tmallLiveService = createTmallLiveServiceStub({ + async previewSearch(query) { + return { + query, + source: "html", + candidateCount: 0, + candidates: [] + }; + } + }); + const app = createServer({ + jdLiveService: createJdLiveServiceStub(), + tmallLiveService + }); + await app.ready(); + await importDualLiveSessions(app); + + const createdTask = await createTask(app, "iPhone 15"); + expect( + createdTask.platformRuns.find((run: { platform: string }) => run.platform === "tmall") + ).toMatchObject({ + platform: "tmall", + status: "NoResult" + }); + expect( + createdTask.platformRuns.find((run: { platform: string }) => run.platform === "jd") + ).toMatchObject({ + platform: "jd", + status: "AwaitingSelection" + }); + + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const candidates = candidatesResponse.json().candidates; + expect(candidates.tmall).toEqual([]); + expect(candidates.jd).toHaveLength(1); + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "jd", + candidateIds: [candidates.jd[0].candidateId] + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + expect(confirmResponse.json().task.taskStatus).toBe("Completed"); + + await app.close(); + }); + + it("publishes a partial report when Tmall crawl is blocked after both platforms are selected", async () => { + const tmallLiveService = createTmallLiveServiceStub({ + async previewProduct() { + const error = new Error("Tmall detail session appears invalid.") as Error & { + statusCode: number; + }; + error.statusCode = 409; + throw error; + } + }); + const app = createServer({ + jdLiveService: createJdLiveServiceStub(), + tmallLiveService + }); + await app.ready(); + await importDualLiveSessions(app); + + const createdTask = await createTask(app, "iPhone 15"); + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const candidates = candidatesResponse.json().candidates; + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "jd", + candidateIds: [candidates.jd[0].candidateId] + }, + { + platform: "tmall", + candidateIds: [candidates.tmall[0].candidateId] + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + expect(confirmResponse.json().task.taskStatus).toBe("PartialCompleted"); + expect(confirmResponse.json().task.platformRuns).toEqual( + expect.arrayContaining([ + expect.objectContaining({ platform: "jd", status: "Completed" }), + expect.objectContaining({ platform: "tmall", status: "Blocked" }) + ]) + ); + + const reportResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/report` + }); + + expect(reportResponse.statusCode).toBe(200); + expect(reportResponse.json().report.task_status).toBe("PartialCompleted"); + expect(reportResponse.json().report.quality_flags.blocked_platforms).toEqual(["tmall"]); + expect(reportResponse.json().report.evidence_index).toEqual( + expect.not.arrayContaining([ + expect.objectContaining({ + platform: "tmall", + source_type: "review" + }) + ]) + ); + expect(reportResponse.json().report.evidence_index).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + platform: "tmall", + source_type: "product", + source_url: "https://detail.tmall.com/item.htm?id=934454505228" + }), + expect.objectContaining({ + platform: "jd", + source_type: "product", + source_url: "https://item.jd.com/100068388533.html" + }) + ]) + ); + + await app.close(); + }); + + it("publishes a new report version when a blocked Tmall platform is retried successfully", async () => { + let allowTmallProduct = false; + const tmallLiveService = createTmallLiveServiceStub(); + const originalPreviewProduct = tmallLiveService.previewProduct.bind(tmallLiveService); + tmallLiveService.previewProduct = async (itemId, options) => { + if (!allowTmallProduct) { + const error = new Error("Tmall detail session appears invalid.") as Error & { + statusCode: number; + }; + error.statusCode = 409; + throw error; + } + + return originalPreviewProduct(itemId, options); + }; + + const app = createServer({ + jdLiveService: createJdLiveServiceStub(), + tmallLiveService + }); + await app.ready(); + await importDualLiveSessions(app); + + const createdTask = await createTask(app, "iPhone 15"); + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const candidates = candidatesResponse.json().candidates; + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "jd", + candidateIds: [candidates.jd[0].candidateId] + }, + { + platform: "tmall", + candidateIds: [candidates.tmall[0].candidateId] + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + expect(confirmResponse.json().task.taskStatus).toBe("PartialCompleted"); + expect(confirmResponse.json().task.reportVersions).toEqual([1]); + + allowTmallProduct = true; + + const retryResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/platforms/tmall/retry` + }); + + expect(retryResponse.statusCode).toBe(200); + expect(retryResponse.json().task.taskStatus).toBe("Completed"); + expect(retryResponse.json().task.reportVersions).toEqual([1, 2]); + expect(retryResponse.json().task.defaultReportVersion).toBe(2); + expect(retryResponse.json().task.platformRuns).toEqual( + expect.arrayContaining([ + expect.objectContaining({ platform: "tmall", status: "Completed" }) + ]) + ); + + const secondReportResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/report?version=2` + }); + + expect(secondReportResponse.statusCode).toBe(200); + expect(secondReportResponse.json().report.task_status).toBe("Completed"); + expect(secondReportResponse.json().report.quality_flags.blocked_platforms).toEqual([]); + expect(secondReportResponse.json().report.evidence_index).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + platform: "tmall", + source_type: "review", + review_ref: "tmall-review-1" + }) + ]) + ); + + await app.close(); + }); + + it("executes only the newly recovered Tmall platform after search recovery", async () => { + let allowTmallSearch = false; + let jdPreviewProductCalls = 0; + let tmallPreviewProductCalls = 0; + + const jdLiveService = createJdLiveServiceStub(); + const originalJdPreviewProduct = jdLiveService.previewProduct.bind(jdLiveService); + jdLiveService.previewProduct = async (skuId, options) => { + jdPreviewProductCalls += 1; + return originalJdPreviewProduct(skuId, options); + }; + + const tmallLiveService = createTmallLiveServiceStub(); + const originalTmallPreviewSearch = tmallLiveService.previewSearch.bind(tmallLiveService); + const originalTmallPreviewProduct = tmallLiveService.previewProduct.bind(tmallLiveService); + tmallLiveService.previewSearch = async (query) => { + if (!allowTmallSearch) { + const error = new Error("Tmall search session appears invalid.") as Error & { + statusCode: number; + }; + error.statusCode = 409; + throw error; + } + + return originalTmallPreviewSearch(query); + }; + tmallLiveService.previewProduct = async (itemId, options) => { + tmallPreviewProductCalls += 1; + return originalTmallPreviewProduct(itemId, options); + }; + + const app = createServer({ + jdLiveService, + tmallLiveService + }); + await app.ready(); + await importDualLiveSessions(app); + + const createdTask = await createTask(app, "iPhone 15"); + expect( + createdTask.platformRuns.find((run: { platform: string }) => run.platform === "tmall") + ).toMatchObject({ + platform: "tmall", + status: "SearchBlocked" + }); + + const firstCandidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const firstCandidates = firstCandidatesResponse.json().candidates; + + const firstConfirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "jd", + candidateIds: [firstCandidates.jd[0].candidateId] + } + ] + } + }); + + expect(firstConfirmResponse.statusCode).toBe(200); + expect(firstConfirmResponse.json().task.taskStatus).toBe("Completed"); + expect(firstConfirmResponse.json().task.reportVersions).toEqual([1]); + expect(jdPreviewProductCalls).toBe(1); + expect(tmallPreviewProductCalls).toBe(0); + + allowTmallSearch = true; + + const retryResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/platforms/tmall/retry` + }); + + expect(retryResponse.statusCode).toBe(200); + expect(retryResponse.json().task.taskStatus).toBe("AwaitingConfirmation"); + expect( + retryResponse + .json() + .task.platformRuns.find((run: { platform: string }) => run.platform === "tmall") + ).toMatchObject({ + platform: "tmall", + status: "AwaitingSelection" + }); + + const recoveredCandidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const recoveredCandidates = recoveredCandidatesResponse.json().candidates; + expect(recoveredCandidates.tmall).toHaveLength(1); + + const secondConfirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "tmall", + candidateIds: [recoveredCandidates.tmall[0].candidateId] + } + ] + } + }); + + expect(secondConfirmResponse.statusCode).toBe(200); + expect(secondConfirmResponse.json().task.taskStatus).toBe("Completed"); + expect(secondConfirmResponse.json().task.reportVersions).toEqual([1, 2]); + expect(secondConfirmResponse.json().task.defaultReportVersion).toBe(2); + expect(jdPreviewProductCalls).toBe(1); + expect(tmallPreviewProductCalls).toBe(1); + + const secondReportResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/report?version=2` + }); + + expect(secondReportResponse.statusCode).toBe(200); + expect(secondReportResponse.json().report.task_status).toBe("Completed"); + expect(secondReportResponse.json().report.evidence_index).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + platform: "jd", + source_type: "product", + source_url: "https://item.jd.com/100068388533.html" + }), + expect.objectContaining({ + platform: "tmall", + source_type: "review", + review_ref: "tmall-review-1" + }) + ]) + ); + + await app.close(); + }); + + it("keeps the current report version when a blocked Tmall retry does not recover", async () => { + const tmallLiveService = createTmallLiveServiceStub({ + async previewProduct() { + const error = new Error("Tmall detail session appears invalid.") as Error & { + statusCode: number; + }; + error.statusCode = 409; + throw error; + } + }); + const app = createServer({ + jdLiveService: createJdLiveServiceStub(), + tmallLiveService + }); + await app.ready(); + await importDualLiveSessions(app); + + const createdTask = await createTask(app, "iPhone 15"); + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const candidates = candidatesResponse.json().candidates; + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "jd", + candidateIds: [candidates.jd[0].candidateId] + }, + { + platform: "tmall", + candidateIds: [candidates.tmall[0].candidateId] + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + expect(confirmResponse.json().task.taskStatus).toBe("PartialCompleted"); + expect(confirmResponse.json().task.reportVersions).toEqual([1]); + + const retryResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/platforms/tmall/retry` + }); + + expect(retryResponse.statusCode).toBe(200); + expect(retryResponse.json().task.taskStatus).toBe("PartialCompleted"); + expect(retryResponse.json().task.reportVersions).toEqual([1]); + expect(retryResponse.json().task.defaultReportVersion).toBe(1); + expect(retryResponse.json().task.platformRuns).toEqual( + expect.arrayContaining([ + expect.objectContaining({ platform: "tmall", status: "Blocked" }) + ]) + ); + + const reportResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/report` + }); + + expect(reportResponse.statusCode).toBe(200); + expect(reportResponse.json().report.task_status).toBe("PartialCompleted"); + + await app.close(); + }); +}); diff --git a/apps/api/src/server.jd-live.test.ts b/apps/api/src/server.jd-live.test.ts index 90f07f1..930d9cf 100644 --- a/apps/api/src/server.jd-live.test.ts +++ b/apps/api/src/server.jd-live.test.ts @@ -3,7 +3,9 @@ import { describe, expect, it } from "vitest"; import type { JdDetailPreviewResult, JdLiveService, + JdProductPreviewResult, JdLiveSessionSummary, + JdReviewsPreviewOptions, JdReviewsPreviewResult, JdSearchPreviewResult } from "./platforms/jd/types"; @@ -107,14 +109,24 @@ function createJdLiveServiceStub( return preview; }, - async previewReviews(skuId, commentCount) { + async previewReviews(skuId, options) { if (overrides.previewReviews) { - return overrides.previewReviews(skuId, commentCount); + return overrides.previewReviews(skuId, options); } const preview: JdReviewsPreviewResult = { skuId, source: "api", + pagination: { + requestedPage: typeof options === "object" ? (options?.page ?? 1) : 1, + requestedCommentCount: + typeof options === "number" + ? options + : (options?.commentCount ?? 5), + maxPages: typeof options === "object" ? (options?.maxPages ?? 1) : 1, + pagesFetched: typeof options === "object" ? (options?.maxPages ?? 1) : 1, + pageKey: typeof options === "object" && options?.maxPages ? "page" : undefined + }, reviews: { skuId, total: "10000", @@ -139,6 +151,23 @@ function createJdLiveServiceStub( } }; + return preview; + }, + async previewProduct(skuId, options?: number | JdReviewsPreviewOptions) { + if (overrides.previewProduct) { + return overrides.previewProduct(skuId, options); + } + + const detail = (await this.previewDetail(skuId)) as JdDetailPreviewResult; + const reviews = (await this.previewReviews(skuId, options)) as JdReviewsPreviewResult; + const preview: JdProductPreviewResult = { + skuId, + source: "api", + detail: detail.detail, + pagination: reviews.pagination, + reviews: reviews.reviews + }; + return preview; } }; @@ -256,6 +285,21 @@ describe("JD live server endpoints", () => { skuId: "100068388533", goodRate: "95%" }); + expect(reviewsResponse.json().preview.pagination).toMatchObject({ + requestedCommentCount: 3 + }); + + const productResponse = await app.inject({ + method: "GET", + url: "/api/platforms/jd/live-product-preview?skuId=100068388533&commentCount=3&maxPages=2" + }); + expect(productResponse.statusCode).toBe(200); + expect(productResponse.json().preview).toMatchObject({ + skuId: "100068388533", + pagination: { + maxPages: 2 + } + }); await app.close(); }); diff --git a/apps/api/src/server.jd-session-manager.test.ts b/apps/api/src/server.jd-session-manager.test.ts new file mode 100644 index 0000000..a4639bd --- /dev/null +++ b/apps/api/src/server.jd-session-manager.test.ts @@ -0,0 +1,341 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { createServer } from "./server"; +import type { JdLiveService, JdSessionManager, JdSessionManagerState } from "./platforms/jd/types"; + +function createSessionSummary(configured: boolean) { + return configured + ? { + configured: true, + importedAt: "2026-04-03T08:30:00.000Z", + hasCookie: true, + userAgent: "stub-user-agent", + searchApiTemplate: { + available: true, + skuId: "100068388533" + }, + detailTemplate: { + available: true, + skuId: "100068388533" + }, + reviewsTemplate: { + available: true, + skuId: "100068388533" + } + } + : { + configured: false, + hasCookie: false, + searchApiTemplate: { + available: false + }, + detailTemplate: { + available: false + }, + reviewsTemplate: { + available: false + } + }; +} + +function createManagerState( + overrides: Partial = {} +): JdSessionManagerState { + return { + status: "idle", + enabled: true, + autoLoginMode: "disabled", + commandConfigured: false, + accountConfigured: false, + passwordConfigured: false, + heartbeatQuery: "iPhone 15", + checkIntervalMs: 600000, + runnerTimeoutMs: 300000, + pendingManualAction: false, + note: "京东会话等待运维注入。", + publicNote: "京东会话由运维后台维护，当前尚未就绪。", + session: createSessionSummary(false), + ...overrides + }; +} + +describe("JD ops session manager routes", () => { + const apps: Array>> = []; + + afterEach(async () => { + while (apps.length > 0) { + await apps.pop()!.close(); + } + }); + + it("exposes ops endpoints and keeps readiness/live-session state in sync", async () => { + let state = createManagerState(); + const manager: JdSessionManager = { + getState() { + return state; + }, + configure(input) { + state = createManagerState({ + ...state, + enabled: input.enabled ?? state.enabled, + autoLoginMode: input.autoLoginMode ?? state.autoLoginMode, + commandConfigured: Boolean(input.loginCommand), + heartbeatQuery: input.heartbeatQuery ?? state.heartbeatQuery, + checkIntervalMs: input.checkIntervalMs ?? state.checkIntervalMs, + runnerTimeoutMs: input.runnerTimeoutMs ?? state.runnerTimeoutMs, + note: "京东运维配置已更新。" + }); + return state; + }, + clearConfig() { + state = createManagerState({ + ...state, + enabled: false, + autoLoginMode: "disabled", + commandConfigured: false, + note: "京东运维配置已清空。" + }); + return state; + }, + async importManualSession() { + state = createManagerState({ + ...state, + status: "healthy", + note: "京东会话已通过 ops-manual 更新。", + publicNote: "京东会话由运维后台维护，当前可用。", + session: createSessionSummary(true) + }); + return state; + }, + clearManagedSession() { + state = createManagerState({ + ...state, + status: "idle", + note: "京东会话已清理。", + publicNote: "京东会话由运维后台维护，当前尚未就绪。", + session: createSessionSummary(false) + }); + return state; + }, + async runHealthCheck() { + state = createManagerState({ + ...state, + status: "healthy", + note: "京东会话健康检查通过。", + publicNote: "京东会话由运维后台维护，当前可用。", + session: createSessionSummary(true) + }); + return { + state, + recovered: false + }; + }, + async runAutoRecovery() { + state = createManagerState({ + ...state, + status: "healthy", + autoLoginMode: "command", + commandConfigured: true, + note: "京东自动恢复成功。", + publicNote: "京东会话由运维后台维护，当前可用。", + session: createSessionSummary(true) + }); + return { + state, + recovered: true + }; + }, + async handleLiveFailure() { + return true; + }, + shutdown() {} + }; + const jdLiveService: JdLiveService = { + getSessionSummary() { + return state.session; + }, + importSession() { + return state.session; + }, + clearSession() {}, + async previewSearch() { + return { + query: "iPhone 15", + source: "html", + candidateCount: 0, + candidates: [] + }; + }, + async previewDetail(skuId) { + return { + skuId, + source: "api", + detail: { + skuId, + title: "Apple iPhone 15", + price: "4398.00", + originalPrice: "4599.00", + estimatedPrice: "4398.00", + shopName: "JD Self Operated", + vendorId: null, + categoryPath: [], + stockState: "in stock", + mainImage: null, + averageScore: "4.9" + } + }; + }, + async previewReviews(skuId) { + return { + skuId, + source: "api", + pagination: { + requestedPage: 1, + requestedCommentCount: 1, + maxPages: 1, + pagesFetched: 1 + }, + reviews: { + skuId, + total: "1000", + goodRate: "96%", + pictureCount: "120", + tags: [], + comments: [] + } + }; + }, + async previewProduct(skuId) { + const detail = await this.previewDetail(skuId); + const reviews = await this.previewReviews(skuId); + return { + skuId, + source: "api", + detail: detail.detail, + pagination: reviews.pagination, + reviews: reviews.reviews + }; + } + }; + const app = createServer({ + jdLiveService, + jdSessionManager: manager + }); + apps.push(app); + await app.ready(); + + const getManagerResponse = await app.inject({ + method: "GET", + url: "/api/ops/jd/session-manager" + }); + + expect(getManagerResponse.statusCode).toBe(200); + expect(getManagerResponse.json().manager).toMatchObject({ + status: "idle", + publicNote: "京东会话由运维后台维护，当前尚未就绪。" + }); + + const readinessResponse = await app.inject({ + method: "GET", + url: "/api/platforms/readiness" + }); + + expect( + readinessResponse + .json() + .platforms.find((platform: { platform: string }) => platform.platform === "jd") + ).toMatchObject({ + platform: "jd", + reason: "京东会话由运维后台维护，当前尚未就绪。" + }); + + const configResponse = await app.inject({ + method: "POST", + url: "/api/ops/jd/session-manager/config", + payload: { + enabled: true, + autoLoginMode: "command", + loginCommand: "node scripts/jd-login-ops.mjs" + } + }); + + expect(configResponse.statusCode).toBe(200); + expect(configResponse.json().manager).toMatchObject({ + autoLoginMode: "command", + commandConfigured: true + }); + + const healthCheckResponse = await app.inject({ + method: "POST", + url: "/api/ops/jd/session-manager/check" + }); + + expect(healthCheckResponse.statusCode).toBe(200); + expect(healthCheckResponse.json()).toMatchObject({ + recovered: false, + state: { + status: "healthy" + } + }); + + const recoverResponse = await app.inject({ + method: "POST", + url: "/api/ops/jd/session-manager/recover" + }); + + expect(recoverResponse.statusCode).toBe(200); + expect(recoverResponse.json()).toMatchObject({ + recovered: true, + state: { + status: "healthy", + autoLoginMode: "command" + } + }); + + const importResponse = await app.inject({ + method: "POST", + url: "/api/ops/jd/session-manager/session", + payload: { + cookieHeader: "thor=masked; pin=masked;", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22%7D", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22sku%22:100068388533%7D" + } + }); + + expect(importResponse.statusCode).toBe(200); + expect(importResponse.json().manager).toMatchObject({ + status: "healthy", + session: { + configured: true + } + }); + + const liveSessionResponse = await app.inject({ + method: "GET", + url: "/api/platforms/jd/live-session" + }); + + expect(liveSessionResponse.statusCode).toBe(200); + expect(liveSessionResponse.json().session).toMatchObject({ + configured: true, + detailTemplate: { + available: true + } + }); + + const clearResponse = await app.inject({ + method: "DELETE", + url: "/api/ops/jd/session-manager/session" + }); + + expect(clearResponse.statusCode).toBe(200); + expect(clearResponse.json().manager).toMatchObject({ + status: "idle", + session: { + configured: false + } + }); + }); +}); diff --git a/apps/api/src/server.test.ts b/apps/api/src/server.test.ts index d1d6116..be50f71 100644 --- a/apps/api/src/server.test.ts +++ b/apps/api/src/server.test.ts @@ -1,7 +1,21 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import type { + JdDetailPreviewResult, + JdLiveService, + JdProductPreviewResult, + JdLiveSessionSummary, + JdReviewsPreviewOptions, + JdReviewsPreviewResult, + JdSearchPreviewResult +} from "./platforms/jd/types"; import { createServer } from "./server"; +const DAY_MS = 24 * 60 * 60 * 1000; + async function createTask(app: ReturnType, query: string) { const response = await app.inject({ method: "POST", @@ -23,6 +37,274 @@ async function preparePlatform(app: ReturnType, platform: " }); } +async function createCompletedTask(app: ReturnType, query: string) { + const createdTask = await createTask(app, query); + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const firstCandidateId = candidatesResponse.json().candidates.tmall[0].candidateId; + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "tmall", + candidateIds: [firstCandidateId] + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + + return confirmResponse.json().task; +} + +function backdateStoredTask(storagePath: string, taskId: string, ageDays: number) { + const timestamp = new Date(Date.now() - ageDays * DAY_MS).toISOString(); + const snapshot = JSON.parse(readFileSync(storagePath, "utf8")) as { + tasks: Array>; + reports: Array<[string, Array>]>; + strategyAttempts: Array<[string, Array>]>; + platformRunMetrics: Array<[string, Record>]>; + reportMetrics: Array>; + }; + + snapshot.tasks = snapshot.tasks.map((task) => + task.taskId === taskId + ? { + ...task, + createdAt: timestamp, + updatedAt: timestamp, + platformRuns: ((task.platformRuns as Array> | undefined) ?? []).map( + (run) => ({ + ...run, + lastUpdatedAt: timestamp + }) + ), + events: ((task.events as Array> | undefined) ?? []).map( + (event) => ({ + ...event, + createdAt: timestamp + }) + ) + } + : task + ); + + snapshot.reports = snapshot.reports.map(([currentTaskId, reports]) => [ + currentTaskId, + currentTaskId === taskId + ? reports.map((report) => ({ + ...report, + generated_at: timestamp + })) + : reports + ]); + + snapshot.strategyAttempts = snapshot.strategyAttempts.map(([currentTaskId, attempts]) => [ + currentTaskId, + currentTaskId === taskId + ? attempts.map((attempt) => ({ + ...attempt, + startedAt: timestamp, + finishedAt: timestamp + })) + : attempts + ]); + + snapshot.platformRunMetrics = snapshot.platformRunMetrics.map(([currentTaskId, metrics]) => [ + currentTaskId, + currentTaskId === taskId + ? Object.fromEntries( + Object.entries(metrics).map(([platform, metric]) => [ + platform, + { + ...metric, + lastUpdatedAt: timestamp + } + ]) + ) + : metrics + ]); + + snapshot.reportMetrics = snapshot.reportMetrics.map((metric) => + metric.taskId === taskId + ? { + ...metric, + generatedAt: timestamp + } + : metric + ); + + writeFileSync(storagePath, JSON.stringify(snapshot, null, 2), "utf8"); +} + +function createJdLiveServiceStub( + overrides: Partial = {} +): JdLiveService { + let summary: JdLiveSessionSummary = { + configured: false, + hasCookie: false, + searchApiTemplate: { available: false }, + detailTemplate: { available: false }, + reviewsTemplate: { available: false } + }; + + return { + getSessionSummary() { + return overrides.getSessionSummary?.() ?? summary; + }, + importSession(input) { + if (overrides.importSession) { + return overrides.importSession(input); + } + + summary = { + configured: true, + importedAt: "2026-04-02T12:00:00.000Z", + hasCookie: true, + userAgent: input.userAgent ?? "stub-user-agent", + searchApiTemplate: { available: Boolean(input.searchApiTemplateUrl) }, + detailTemplate: { available: Boolean(input.detailTemplateUrl) }, + reviewsTemplate: { available: Boolean(input.reviewsTemplateUrl) } + }; + return summary; + }, + clearSession() { + if (overrides.clearSession) { + overrides.clearSession(); + return; + } + + summary = { + configured: false, + hasCookie: false, + searchApiTemplate: { available: false }, + detailTemplate: { available: false }, + reviewsTemplate: { available: false } + }; + }, + async previewSearch(query) { + if (overrides.previewSearch) { + return overrides.previewSearch(query); + } + + const preview: JdSearchPreviewResult = { + query, + source: "html", + candidateCount: 1, + candidates: [ + { + candidateId: "jd-100068388533", + platform: "jd", + title: "Apple/苹果 iPhone 15", + price: 3898, + priceLabel: "¥3898", + storeName: "Apple产品京东自营旗舰店", + productUrl: "https://item.jd.com/100068388533.html", + imageUrl: "https://img14.360buyimg.com/n2/jfs/t1/example.jpg", + salesHint: "已售500万+", + specLabel: "128GB", + highlights: ["A16仿生芯片"] + } + ] + }; + + return preview; + }, + async previewDetail(skuId) { + if (overrides.previewDetail) { + return overrides.previewDetail(skuId); + } + + const preview: JdDetailPreviewResult = { + skuId, + source: "api", + detail: { + skuId, + title: "Apple/苹果 iPhone 15", + price: "4398.00", + originalPrice: "4599.00", + estimatedPrice: "3898", + shopName: "Apple产品京东自营旗舰店", + vendorId: null, + categoryPath: ["手机通讯", "手机", "Apple"], + stockState: "有货，仅剩318件", + mainImage: "https://img14.360buyimg.com/n2/jfs/t1/example.jpg", + averageScore: null + } + }; + + return preview; + }, + async previewReviews(skuId, options) { + if (overrides.previewReviews) { + return overrides.previewReviews(skuId, options); + } + + const preview: JdReviewsPreviewResult = { + skuId, + source: "api", + pagination: { + requestedPage: typeof options === "object" ? (options?.page ?? 1) : 1, + requestedCommentCount: + typeof options === "number" + ? options + : (options?.commentCount ?? 5), + maxPages: typeof options === "object" ? (options?.maxPages ?? 1) : 1, + pagesFetched: typeof options === "object" ? (options?.maxPages ?? 1) : 1, + pageKey: typeof options === "object" && options?.maxPages ? "page" : undefined + }, + reviews: { + skuId, + total: "10000", + goodRate: "95%", + pictureCount: "500", + tags: [ + { + tagId: "tag-1", + name: "拍照效果超清晰", + count: "9313" + } + ], + comments: [ + { + id: "comment-1", + content: "系统流畅，拍照清晰。", + score: "5", + creationTime: "2026-04-02 19:23:16", + userLevelName: "PLUS会员" + } + ] + } + }; + + return preview; + }, + async previewProduct(skuId, options?: number | JdReviewsPreviewOptions) { + if (overrides.previewProduct) { + return overrides.previewProduct(skuId, options); + } + + const detail = (await this.previewDetail(skuId)) as JdDetailPreviewResult; + const reviews = (await this.previewReviews(skuId, options)) as JdReviewsPreviewResult; + const preview: JdProductPreviewResult = { + skuId, + source: "api", + detail: detail.detail, + pagination: reviews.pagination, + reviews: reviews.reviews + }; + + return preview; + } + }; +} + describe("API server", () => { it("returns platform readiness with jd blocked by default", async () => { const app = createServer(); @@ -218,6 +500,628 @@ describe("API server", () => { await app.close(); }); + it("imports a JD live session without persisting raw cookies into the task store", async () => { + const jdLiveService = createJdLiveServiceStub(); + const app = createServer({ jdLiveService }); + await app.ready(); + + const response = await app.inject({ + method: "POST", + url: "/api/platforms/jd/live-session", + payload: { + cookieHeader: "thor=masked; pin=masked;", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22%7D", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22sku%22:100068388533%7D" + } + }); + + expect(response.statusCode).toBe(200); + expect(response.json().session).toMatchObject({ + configured: true, + hasCookie: true, + detailTemplate: { + available: true + }, + reviewsTemplate: { + available: true + } + }); + + const sessionSummary = await app.inject({ + method: "GET", + url: "/api/platforms/jd/live-session" + }); + expect(sessionSummary.statusCode).toBe(200); + expect(sessionSummary.json().session).toMatchObject({ + configured: true, + hasCookie: true + }); + + const readinessResponse = await app.inject({ + method: "GET", + url: "/api/platforms/readiness" + }); + expect( + readinessResponse + .json() + .platforms.find((platform: { platform: string }) => platform.platform === "jd") + ).toMatchObject({ + platform: "jd", + ready: true, + status: "ready" + }); + + await app.close(); + }); + + it("exposes JD live preview endpoints through the injected live service", async () => { + const jdLiveService = createJdLiveServiceStub(); + const app = createServer({ jdLiveService }); + await app.ready(); + + const searchResponse = await app.inject({ + method: "GET", + url: "/api/platforms/jd/live-search-preview?query=iPhone%2015" + }); + expect(searchResponse.statusCode).toBe(200); + expect(searchResponse.json().preview).toMatchObject({ + query: "iPhone 15", + source: "html", + candidateCount: 1 + }); + + const detailResponse = await app.inject({ + method: "GET", + url: "/api/platforms/jd/live-detail-preview?skuId=100068388533" + }); + expect(detailResponse.statusCode).toBe(200); + expect(detailResponse.json().preview.detail).toMatchObject({ + skuId: "100068388533", + shopName: "Apple产品京东自营旗舰店" + }); + + const reviewsResponse = await app.inject({ + method: "GET", + url: "/api/platforms/jd/live-reviews-preview?skuId=100068388533&commentCount=3" + }); + expect(reviewsResponse.statusCode).toBe(200); + expect(reviewsResponse.json().preview.reviews).toMatchObject({ + skuId: "100068388533", + goodRate: "95%" + }); + expect(reviewsResponse.json().preview.pagination).toMatchObject({ + requestedCommentCount: 3 + }); + + const productResponse = await app.inject({ + method: "GET", + url: "/api/platforms/jd/live-product-preview?skuId=100068388533&commentCount=3&maxPages=2" + }); + expect(productResponse.statusCode).toBe(200); + expect(productResponse.json().preview).toMatchObject({ + skuId: "100068388533", + pagination: { + maxPages: 2 + } + }); + + await app.close(); + }); + + it("closes a JD keyword-only crawl loop by auto-selecting the best live candidate", async () => { + const previewProductCalls: Array<{ + skuId: string; + options: number | JdReviewsPreviewOptions | undefined; + }> = []; + const jdLiveService = createJdLiveServiceStub({ + async previewSearch(query) { + return { + query, + source: "html", + candidateCount: 3, + candidates: [ + { + candidateId: "jd-111111111111", + platform: "jd", + title: "小米手环9 NFC版 智能手环", + price: 249, + priceLabel: "¥249", + storeName: "小米京东自营旗舰店", + productUrl: "https://item.jd.com/111111111111.html", + imageUrl: "https://img14.360buyimg.com/n2/jfs/t1/example-9.jpg", + salesHint: "已售20万+", + specLabel: "NFC版", + highlights: ["健康监测"] + }, + { + candidateId: "jd-222222222222", + platform: "jd", + title: "小米手环10 标准版 智能手环 血氧监测", + price: 269, + priceLabel: "¥269", + storeName: "小米京东自营旗舰店", + productUrl: "https://item.jd.com/222222222222.html", + imageUrl: "https://img14.360buyimg.com/n2/jfs/t1/example-10.jpg", + salesHint: "已售50万+", + specLabel: "标准版", + highlights: ["全天候健康监测", "14天续航"] + }, + { + candidateId: "jd-fallback-%E5%B0%8F%E7%B1%B3%E6%89%8B%E7%8E%AF10", + platform: "jd", + title: "小米手环10", + price: 0, + priceLabel: "¥0", + storeName: "京东", + productUrl: + "https://search.jd.com/Search?keyword=%E5%B0%8F%E7%B1%B3%E6%89%8B%E7%8E%AF10", + imageUrl: "https://placehold.co/640x480?text=JD", + salesHint: "页面已返回，但未解析出稳定商品卡片", + specLabel: "待确认", + highlights: ["需要刷新搜索模板或调整解析器"] + } + ] + }; + }, + async previewProduct(skuId, options) { + previewProductCalls.push({ skuId, options }); + return { + skuId, + source: "api", + detail: { + skuId, + title: "小米手环10 标准版 智能手环 血氧监测", + price: "269.00", + originalPrice: "299.00", + estimatedPrice: "269.00", + shopName: "小米京东自营旗舰店", + vendorId: null, + categoryPath: ["智能设备", "智能手环"], + stockState: "现货", + mainImage: "https://img14.360buyimg.com/n2/jfs/t1/example-10.jpg", + averageScore: "4.9" + }, + pagination: { + requestedPage: 1, + requestedCommentCount: + typeof options === "number" ? options : (options?.commentCount ?? 8), + maxPages: typeof options === "number" ? 1 : (options?.maxPages ?? 2), + pagesFetched: typeof options === "number" ? 1 : (options?.maxPages ?? 2), + pageKey: "page" + }, + reviews: { + skuId, + total: "10000+", + goodRate: "97%", + pictureCount: "800", + tags: [ + { + tagId: "tag-1", + name: "续航很久", + count: "5300" + } + ], + comments: [ + { + id: "comment-1", + content: "表带舒适，睡眠监测比较准。", + score: "5", + creationTime: "2026-04-03 09:00:00", + userLevelName: "PLUS会员" + } + ] + } + }; + } + }); + const app = createServer({ jdLiveService }); + await app.ready(); + + const response = await app.inject({ + method: "GET", + url: "/api/platforms/jd/live-keyword-preview?query=%E5%B0%8F%E7%B1%B3%E6%89%8B%E7%8E%AF10&commentCount=8&maxPages=2" + }); + + expect(response.statusCode).toBe(200); + expect(previewProductCalls).toEqual([ + { + skuId: "222222222222", + options: { + commentCount: 8, + maxPages: 2 + } + } + ]); + expect(response.json().preview).toMatchObject({ + query: "小米手环10", + search: { + source: "html", + candidateCount: 3, + selected: { + skuId: "222222222222", + candidate: { + candidateId: "jd-222222222222", + title: "小米手环10 标准版 智能手环 血氧监测" + } + } + }, + product: { + detail: { + title: "小米手环10 标准版 智能手环 血氧监测" + }, + reviews: { + goodRate: "97%" + } + } + }); + + await app.close(); + }); + + it("returns 404 when JD keyword preview has no valid item candidate", async () => { + const jdLiveService = createJdLiveServiceStub({ + async previewSearch(query) { + return { + query, + source: "html", + candidateCount: 1, + candidates: [ + { + candidateId: "jd-fallback-test", + platform: "jd", + title: query, + price: 0, + priceLabel: "¥0", + storeName: "京东", + productUrl: "https://search.jd.com/Search?keyword=test", + imageUrl: "https://placehold.co/640x480?text=JD", + salesHint: "页面已返回，但未解析出稳定商品卡片", + specLabel: "待确认", + highlights: ["需要刷新搜索模板或调整解析器"] + } + ] + }; + } + }); + const app = createServer({ jdLiveService }); + await app.ready(); + + const response = await app.inject({ + method: "GET", + url: "/api/platforms/jd/live-keyword-preview?query=test" + }); + + expect(response.statusCode).toBe(404); + expect(response.json()).toMatchObject({ + message: expect.stringContaining("no valid item candidates") + }); + + await app.close(); + }); + + it("closes the jd minimal loop with live search, crawl, and report publishing", async () => { + const jdLiveService = createJdLiveServiceStub(); + const app = createServer({ jdLiveService }); + await app.ready(); + + const importResponse = await app.inject({ + method: "POST", + url: "/api/platforms/jd/live-session", + payload: { + cookieHeader: "thor=masked; pin=masked;", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22%7D", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22sku%22:100068388533%7D" + } + }); + + expect(importResponse.statusCode).toBe(200); + + const createdTask = await createTask(app, "iPhone 15"); + const jdRun = createdTask.platformRuns.find((run: { platform: string }) => run.platform === "jd"); + expect(jdRun).toMatchObject({ + platform: "jd", + status: "AwaitingSelection" + }); + + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const firstJdCandidate = candidatesResponse.json().candidates.jd[0]; + expect(firstJdCandidate).toMatchObject({ + candidateId: "jd-100068388533", + productUrl: "https://item.jd.com/100068388533.html" + }); + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "jd", + candidateIds: [firstJdCandidate.candidateId] + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + expect(confirmResponse.json().task).toMatchObject({ + taskStatus: "Completed", + defaultReportVersion: 1 + }); + + const reportResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/report` + }); + + expect(reportResponse.statusCode).toBe(200); + expect(reportResponse.json().report.summary.headline).toContain("京东"); + expect(reportResponse.json().report.product_snapshot.review_sample_count).toBe(1); + expect(reportResponse.json().report.evidence_index).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + platform: "jd", + source_type: "product", + source_url: "https://item.jd.com/100068388533.html" + }), + expect.objectContaining({ + platform: "jd", + source_type: "review", + review_ref: "comment-1" + }) + ]) + ); + + await app.close(); + }); + + it("shrinks JD review budgets across multiple selected links under the task total limit", async () => { + const previewProductCalls: Array<{ + skuId: string; + options: number | JdReviewsPreviewOptions | undefined; + }> = []; + const jdLiveService = createJdLiveServiceStub({ + async previewSearch(query) { + return { + query, + source: "html", + candidateCount: 2, + candidates: [ + { + candidateId: "jd-100068388533", + platform: "jd", + title: "Apple/苹果 iPhone 15 128GB", + price: 3898, + priceLabel: "¥3898", + storeName: "Apple产品京东自营旗舰店", + productUrl: "https://item.jd.com/100068388533.html", + imageUrl: "https://img14.360buyimg.com/n2/jfs/t1/example-1.jpg", + salesHint: "已售500万+", + specLabel: "128GB", + highlights: ["A16仿生芯片"] + }, + { + candidateId: "jd-100068388535", + platform: "jd", + title: "Apple/苹果 iPhone 15 256GB", + price: 4398, + priceLabel: "¥4398", + storeName: "Apple产品京东自营旗舰店", + productUrl: "https://item.jd.com/100068388535.html", + imageUrl: "https://img14.360buyimg.com/n2/jfs/t1/example-2.jpg", + salesHint: "已售300万+", + specLabel: "256GB", + highlights: ["灵动岛"] + } + ] + }; + }, + async previewProduct(skuId, options) { + previewProductCalls.push({ skuId, options }); + const requestedCommentCount = + typeof options === "number" ? options : (options?.commentCount ?? 0); + + return { + skuId, + source: "api", + detail: { + skuId, + title: `Apple/苹果 iPhone 15 ${skuId}`, + price: skuId === "100068388533" ? "3898.00" : "4398.00", + originalPrice: null, + estimatedPrice: null, + shopName: "Apple产品京东自营旗舰店", + vendorId: null, + categoryPath: ["手机通讯", "手机", "Apple"], + stockState: "现货", + mainImage: "https://img14.360buyimg.com/n2/jfs/t1/example.jpg", + averageScore: null + }, + pagination: { + requestedPage: 1, + requestedCommentCount, + maxPages: requestedCommentCount > 0 ? 1 : 0, + pagesFetched: requestedCommentCount > 0 ? 1 : 0 + }, + reviews: { + skuId, + total: String(requestedCommentCount), + goodRate: "95%", + pictureCount: "1", + tags: [], + comments: Array.from({ length: requestedCommentCount }, (_, index) => ({ + id: `${skuId}-comment-${index + 1}`, + content: + index === requestedCommentCount - 1 + ? "发热有点明显，需要观察续航。" + : `第 ${index + 1} 条评论，整体体验稳定。`, + score: index === requestedCommentCount - 1 ? "3" : "5", + creationTime: `2026-04-0${index + 1} 10:00:00`, + userLevelName: "PLUS会员" + })) + } + }; + } + }); + const app = createServer({ jdLiveService }); + await app.ready(); + + const importResponse = await app.inject({ + method: "POST", + url: "/api/platforms/jd/live-session", + payload: { + cookieHeader: "thor=masked; pin=masked;", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22%7D", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22sku%22:100068388533,%22page%22:1%7D" + } + }); + expect(importResponse.statusCode).toBe(200); + + const createResponse = await app.inject({ + method: "POST", + url: "/api/tasks", + payload: { + query: "iPhone 15", + perLinkLimit: 4, + taskTotalLimit: 5 + } + }); + const createdTask = createResponse.json().task; + + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const jdCandidates = candidatesResponse.json().candidates.jd; + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "jd", + candidateIds: jdCandidates.map((candidate: { candidateId: string }) => candidate.candidateId) + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + expect(previewProductCalls).toHaveLength(2); + expect( + previewProductCalls.map((call) => + typeof call.options === "number" ? call.options : call.options?.commentCount ?? 0 + ) + ).toEqual([3, 2]); + + const reportResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/report` + }); + + expect(reportResponse.statusCode).toBe(200); + expect(reportResponse.json().report.product_snapshot.review_sample_count).toBe(5); + + await app.close(); + }); + + it("maps JD risk-blocked crawl failures to a blocked platform status", async () => { + const jdLiveService = createJdLiveServiceStub({ + async previewProduct() { + const error = new Error("JD request hit verification or risk control.") as Error & { + statusCode: number; + code: string; + }; + error.statusCode = 423; + error.code = "RISK_BLOCKED"; + throw error; + } + }); + const app = createServer({ jdLiveService }); + await app.ready(); + + await app.inject({ + method: "POST", + url: "/api/platforms/jd/live-session", + payload: { + cookieHeader: "thor=masked; pin=masked;", + detailTemplateUrl: + "https://api.m.jd.com/?functionId=pc_detailpage_wareBusiness&body=%7B%22skuId%22:%22100068388533%22%7D", + reviewsTemplateUrl: + "https://api.m.jd.com/?functionId=getLegoWareDetailComment&body=%7B%22sku%22:100068388533,%22page%22:1%7D" + } + }); + + const createdTask = await createTask(app, "iPhone 15"); + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const firstJdCandidate = candidatesResponse.json().candidates.jd[0]; + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "jd", + candidateIds: [firstJdCandidate.candidateId] + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + expect( + confirmResponse + .json() + .task.platformRuns.find((run: { platform: string }) => run.platform === "jd") + ).toMatchObject({ + platform: "jd", + status: "Blocked", + reason: expect.stringContaining("风控") + }); + + await app.close(); + }); + + it("surfaces JD live preview failures with service-provided status codes", async () => { + const jdLiveService = createJdLiveServiceStub({ + async previewDetail() { + const error = new Error("Imported detail template is bound to another sku.") as Error & { + statusCode: number; + }; + error.statusCode = 409; + throw error; + } + }); + const app = createServer({ jdLiveService }); + await app.ready(); + + const response = await app.inject({ + method: "GET", + url: "/api/platforms/jd/live-detail-preview?skuId=100068388533" + }); + + expect(response.statusCode).toBe(409); + expect(response.json()).toMatchObject({ + message: "Imported detail template is bound to another sku." + }); + + await app.close(); + }); + it("supports NoSelection terminal state when user confirms nothing", async () => { const app = createServer(); await app.ready(); @@ -317,97 +1221,6 @@ describe("API server", () => { await app.close(); }); - it("does not rerun completed platforms when confirming a newly recovered platform", async () => { - const app = createServer(); - await app.ready(); - - const createdTask = await createTask(app, "iPhone 15 Pro"); - const firstCandidatesResponse = await app.inject({ - method: "GET", - url: `/api/tasks/${createdTask.taskId}/candidates` - }); - const firstCandidates = firstCandidatesResponse.json().candidates; - - const firstConfirmResponse = await app.inject({ - method: "POST", - url: `/api/tasks/${createdTask.taskId}/confirm`, - payload: { - selections: [ - { - platform: "tmall", - candidateIds: [firstCandidates.tmall[0].candidateId] - } - ] - } - }); - - expect(firstConfirmResponse.statusCode).toBe(200); - expect(firstConfirmResponse.json().task.taskStatus).toBe("Completed"); - expect(firstConfirmResponse.json().task.reportVersions).toEqual([1]); - - const attemptsAfterFirstConfirm = await app.inject({ - method: "GET", - url: `/api/tasks/${createdTask.taskId}/strategy-attempts` - }); - const firstTmallExecutionAttempts = attemptsAfterFirstConfirm - .json() - .attempts.filter( - (attempt: { platform: string; capability: string }) => - attempt.platform === "tmall" && - (attempt.capability === "detail" || attempt.capability === "reviews") - ); - expect(firstTmallExecutionAttempts).toHaveLength(2); - - await preparePlatform(app, "jd"); - const retryResponse = await app.inject({ - method: "POST", - url: `/api/tasks/${createdTask.taskId}/platforms/jd/retry` - }); - - expect(retryResponse.statusCode).toBe(200); - - const recoveredCandidatesResponse = await app.inject({ - method: "GET", - url: `/api/tasks/${createdTask.taskId}/candidates` - }); - const recoveredCandidates = recoveredCandidatesResponse.json().candidates; - - const secondConfirmResponse = await app.inject({ - method: "POST", - url: `/api/tasks/${createdTask.taskId}/confirm`, - payload: { - selections: [ - { - platform: "jd", - candidateIds: [recoveredCandidates.jd[0].candidateId] - } - ] - } - }); - - expect(secondConfirmResponse.statusCode).toBe(200); - expect(secondConfirmResponse.json().task.taskStatus).toBe("Completed"); - expect(secondConfirmResponse.json().task.reportVersions).toEqual([1, 2]); - - const attemptsAfterSecondConfirm = await app.inject({ - method: "GET", - url: `/api/tasks/${createdTask.taskId}/strategy-attempts` - }); - const executionAttempts = attemptsAfterSecondConfirm.json().attempts.filter( - (attempt: { platform: string; capability: string }) => - attempt.capability === "detail" || attempt.capability === "reviews" - ); - - expect( - executionAttempts.filter((attempt: { platform: string }) => attempt.platform === "tmall") - ).toHaveLength(2); - expect( - executionAttempts.filter((attempt: { platform: string }) => attempt.platform === "jd") - ).toHaveLength(2); - - await app.close(); - }); - it("records recovery audit entries and retry metrics for recovered platforms", async () => { const app = createServer(); await app.ready(); @@ -635,4 +1448,219 @@ describe("API server", () => { await app.close(); }); + + it("supports retention cleanup dry runs without mutating stored artifacts", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "cross-ai-task-store-")); + const storagePath = join(tempDir, "task-store.json"); + const app = createServer({ storagePath }); + await app.ready(); + + const completedTask = await createCompletedTask(app, "Lenovo Legion Go"); + await app.close(); + + backdateStoredTask(storagePath, completedTask.taskId, 45); + + const reopenedApp = createServer({ storagePath }); + await reopenedApp.ready(); + + const cleanupResponse = await reopenedApp.inject({ + method: "POST", + url: "/api/retention/cleanup", + payload: { + dryRun: true + } + }); + + expect(cleanupResponse.statusCode).toBe(200); + expect(cleanupResponse.json().cleanup).toMatchObject({ + dryRun: true, + cleanedTaskCount: 1, + rawCleanedTaskCount: 1, + reportCleanedTaskCount: 0, + deletedReportCount: 0 + }); + + const taskResponse = await reopenedApp.inject({ + method: "GET", + url: `/api/tasks/${completedTask.taskId}` + }); + expect(taskResponse.json().task.events.length).toBeGreaterThan(0); + expect(taskResponse.json().task.platformCandidates.tmall).toHaveLength(3); + + const overviewResponse = await reopenedApp.inject({ + method: "GET", + url: "/api/observability/overview" + }); + expect(overviewResponse.json().overview.retention.cleanupRuns).toBe(0); + + await reopenedApp.close(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + it("applies 30/90-day retention cleanup and prunes expired artifacts", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "cross-ai-task-store-")); + const storagePath = join(tempDir, "task-store.json"); + const app = createServer({ storagePath }); + await app.ready(); + + const rawExpiredTask = await createCompletedTask(app, "Steam Deck OLED"); + const reportExpiredTask = await createCompletedTask(app, "PlayStation Portal"); + await app.close(); + + backdateStoredTask(storagePath, rawExpiredTask.taskId, 45); + backdateStoredTask(storagePath, reportExpiredTask.taskId, 95); + + const reopenedApp = createServer({ storagePath }); + await reopenedApp.ready(); + + const cleanupResponse = await reopenedApp.inject({ + method: "POST", + url: "/api/retention/cleanup" + }); + + expect(cleanupResponse.statusCode).toBe(200); + expect(cleanupResponse.json().cleanup).toMatchObject({ + dryRun: false, + cleanedTaskCount: 2, + rawCleanedTaskCount: 2, + reportCleanedTaskCount: 1, + deletedReportCount: 1, + residualArtifactCount: 0 + }); + expect(cleanupResponse.json().cleanup.deletedArtifactCount).toBeGreaterThan(0); + expect(cleanupResponse.json().cleanup.affectedTaskIds).toEqual( + expect.arrayContaining([rawExpiredTask.taskId, reportExpiredTask.taskId]) + ); + + const rawTaskResponse = await reopenedApp.inject({ + method: "GET", + url: `/api/tasks/${rawExpiredTask.taskId}` + }); + expect(rawTaskResponse.json().task.events).toHaveLength(0); + expect(rawTaskResponse.json().task.platformCandidates.tmall).toHaveLength(1); + expect(rawTaskResponse.json().task.reportVersions).toEqual([1]); + + const rawAttemptsResponse = await reopenedApp.inject({ + method: "GET", + url: `/api/tasks/${rawExpiredTask.taskId}/strategy-attempts` + }); + expect(rawAttemptsResponse.statusCode).toBe(200); + expect(rawAttemptsResponse.json().attempts).toHaveLength(0); + + const rawReportResponse = await reopenedApp.inject({ + method: "GET", + url: `/api/tasks/${rawExpiredTask.taskId}/report` + }); + expect(rawReportResponse.statusCode).toBe(200); + + const expiredTaskResponse = await reopenedApp.inject({ + method: "GET", + url: `/api/tasks/${reportExpiredTask.taskId}` + }); + expect(expiredTaskResponse.json().task.platformCandidates.tmall).toHaveLength(0); + expect(expiredTaskResponse.json().task.reportVersions).toEqual([]); + expect(expiredTaskResponse.json().task.defaultReportVersion).toBeUndefined(); + + const expiredReportResponse = await reopenedApp.inject({ + method: "GET", + url: `/api/tasks/${reportExpiredTask.taskId}/report` + }); + expect(expiredReportResponse.statusCode).toBe(404); + + const historyResponse = await reopenedApp.inject({ + method: "GET", + url: "/api/history" + }); + expect( + historyResponse + .json() + .tasks.find((task: { taskId: string }) => task.taskId === reportExpiredTask.taskId) + ).toMatchObject({ + taskId: reportExpiredTask.taskId, + hasReport: false + }); + + const overviewResponse = await reopenedApp.inject({ + method: "GET", + url: "/api/observability/overview" + }); + expect(overviewResponse.json().overview.retention).toMatchObject({ + taskDeletes: 0, + cleanupRuns: 1, + deletedReports: 1, + residualArtifacts: 0 + }); + + await reopenedApp.close(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + it("persists sessions, tasks, and reports across server restarts when storage is enabled", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "cross-ai-task-store-")); + const storagePath = join(tempDir, "task-store.json"); + const app = createServer({ storagePath }); + await app.ready(); + + await preparePlatform(app, "jd"); + const createdTask = await createTask(app, "Steam Deck OLED"); + + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const firstCandidateId = candidatesResponse.json().candidates.tmall[0].candidateId; + + await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "tmall", + candidateIds: [firstCandidateId] + } + ] + } + }); + + await app.close(); + + const reopenedApp = createServer({ storagePath }); + await reopenedApp.ready(); + + const historyResponse = await reopenedApp.inject({ + method: "GET", + url: "/api/history" + }); + expect(historyResponse.statusCode).toBe(200); + expect( + historyResponse + .json() + .tasks.find((task: { taskId: string }) => task.taskId === createdTask.taskId) + ).toMatchObject({ + taskId: createdTask.taskId, + hasReport: true + }); + + const sessionResponse = await reopenedApp.inject({ + method: "GET", + url: "/api/sessions/jd" + }); + expect(sessionResponse.statusCode).toBe(200); + expect(sessionResponse.json().session).toMatchObject({ + platform: "jd", + ready: true, + status: "ready" + }); + + const reportResponse = await reopenedApp.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/report` + }); + expect(reportResponse.statusCode).toBe(200); + expect(reportResponse.json().report.task_id).toBe(createdTask.taskId); + + await reopenedApp.close(); + rmSync(tempDir, { recursive: true, force: true }); + }); }); diff --git a/apps/api/src/server.tmall-live.test.ts b/apps/api/src/server.tmall-live.test.ts new file mode 100644 index 0000000..ce411cb --- /dev/null +++ b/apps/api/src/server.tmall-live.test.ts @@ -0,0 +1,395 @@ +import { describe, expect, it } from "vitest"; + +import type { + TmallDetailPreviewResult, + TmallLiveService, + TmallLiveSessionSummary, + TmallProductPreviewResult, + TmallReviewsPreviewResult, + TmallSearchPreviewResult +} from "./platforms/tmall/types"; +import { createServer } from "./server"; + +async function createTask(app: ReturnType, query: string) { + const response = await app.inject({ + method: "POST", + url: "/api/tasks", + payload: { + query, + perLinkLimit: 100, + taskTotalLimit: 500 + } + }); + + return response.json().task; +} + +function createTmallLiveServiceStub( + overrides: Partial = {} +): TmallLiveService { + let summary: TmallLiveSessionSummary = { + configured: false, + hasCookie: false, + detailTemplate: { available: false }, + reviewsTemplate: { available: false } + }; + + return { + getSessionSummary() { + return overrides.getSessionSummary?.() ?? summary; + }, + importSession(input) { + if (overrides.importSession) { + return overrides.importSession(input); + } + + summary = { + configured: true, + importedAt: "2026-04-03T02:30:00.000Z", + hasCookie: true, + userAgent: input.userAgent ?? "stub-user-agent", + detailTemplate: { available: Boolean(input.detailTemplateUrl), itemId: "833444005595" }, + reviewsTemplate: { available: Boolean(input.reviewsTemplateUrl), itemId: "833444005595" } + }; + return summary; + }, + clearSession() { + if (overrides.clearSession) { + overrides.clearSession(); + return; + } + + summary = { + configured: false, + hasCookie: false, + detailTemplate: { available: false }, + reviewsTemplate: { available: false } + }; + }, + async previewSearch(query) { + if (overrides.previewSearch) { + return overrides.previewSearch(query); + } + + const preview: TmallSearchPreviewResult = { + query, + source: "html", + candidateCount: 1, + candidates: [ + { + candidateId: "tmall-833444005595", + platform: "tmall", + title: "Apple iPhone 15", + price: 4399, + priceLabel: "CNY 4399", + storeName: "Apple Flagship Store", + productUrl: "https://detail.tmall.com/item.htm?id=833444005595", + imageUrl: "https://img.alicdn.com/example.jpg", + salesHint: "sold 10k+", + specLabel: "128GB", + highlights: ["Tmall", "Fast shipping"] + } + ] + }; + + return preview; + }, + async previewDetail(itemId) { + if (overrides.previewDetail) { + return overrides.previewDetail(itemId); + } + + const preview: TmallDetailPreviewResult = { + itemId, + source: "api", + detail: { + itemId, + title: "Apple iPhone 15", + subtitle: null, + price: "4399.00", + originalPrice: "4999.00", + shopName: "Apple Flagship Store", + shopUrl: null, + sellerType: "tmall", + categoryPath: ["Phones", "Mobile", "Apple"], + mainImage: "https://img.alicdn.com/example.jpg", + salesDesc: "sold 10k+", + commentCount: "20k+" + } + }; + + return preview; + }, + async previewReviews(itemId, options) { + if (overrides.previewReviews) { + return overrides.previewReviews(itemId, options); + } + + const commentCount = + typeof options === "number" ? options : (options?.commentCount ?? 20); + const page = typeof options === "number" ? 1 : (options?.page ?? 1); + const maxPages = typeof options === "number" ? 1 : (options?.maxPages ?? 1); + + const preview: TmallReviewsPreviewResult = { + itemId, + source: "api", + pagination: { + requestedPage: page, + requestedCommentCount: commentCount, + maxPages, + pagesFetched: 1, + pageKey: "pageNum" + }, + reviews: { + itemId, + total: "20k+", + hasNext: false, + allCount: "20k+", + pictureCount: "5000+", + appendCount: "1200+", + tags: [ + { + name: "good looking", + count: "5313" + } + ], + comments: [ + { + id: "review-1", + content: "Smooth and well built.", + date: "2026-04-03", + userNick: "Alice", + userAvatar: null, + skuText: ["Black", "128GB"], + pictureUrls: [], + videoUrls: [], + likeCount: "3", + reply: null, + appendContent: null, + appendPictureUrls: [] + } + ] + } + }; + + return preview; + }, + async previewProduct(itemId, options) { + if (overrides.previewProduct) { + return overrides.previewProduct(itemId, options); + } + + const detail = await this.previewDetail(itemId); + const reviews = await this.previewReviews(itemId, options); + const preview: TmallProductPreviewResult = { + itemId, + source: "api", + detail: detail.detail, + pagination: reviews.pagination, + reviews: reviews.reviews + }; + + return preview; + } + }; +} + +describe("Tmall live session endpoints", () => { + it("imports a Tmall live session and updates readiness", async () => { + const tmallLiveService = createTmallLiveServiceStub(); + const app = createServer({ tmallLiveService }); + await app.ready(); + + const response = await app.inject({ + method: "POST", + url: "/api/platforms/tmall/live-session", + payload: { + cookieHeader: "_tb_token_=masked;", + detailTemplateUrl: + "https://h5api.m.taobao.com/h5/mtop.taobao.pcdetail.data.get/1.0/?data=%7B%22id%22%3A%22833444005595%22%7D", + reviewsTemplateUrl: + "https://h5api.m.taobao.com/h5/mtop.alibaba.review.list.for.new.pc.detail/1.0/?data=%7B%22itemId%22%3A%22833444005595%22%7D" + } + }); + + expect(response.statusCode).toBe(200); + expect(response.json().session).toMatchObject({ + configured: true, + hasCookie: true, + detailTemplate: { + available: true + }, + reviewsTemplate: { + available: true + } + }); + + const readinessResponse = await app.inject({ + method: "GET", + url: "/api/platforms/readiness" + }); + expect( + readinessResponse + .json() + .platforms.find((platform: { platform: string }) => platform.platform === "tmall") + ).toMatchObject({ + platform: "tmall", + ready: true, + status: "ready" + }); + + await app.close(); + }); + + it("exposes Tmall live preview endpoints through the injected service", async () => { + const tmallLiveService = createTmallLiveServiceStub(); + const app = createServer({ tmallLiveService }); + await app.ready(); + + const searchResponse = await app.inject({ + method: "GET", + url: "/api/platforms/tmall/live-search-preview?query=iPhone%2015" + }); + expect(searchResponse.statusCode).toBe(200); + expect(searchResponse.json().preview).toMatchObject({ + query: "iPhone 15", + source: "html", + candidateCount: 1 + }); + + const detailResponse = await app.inject({ + method: "GET", + url: "/api/platforms/tmall/live-detail-preview?itemId=833444005595" + }); + expect(detailResponse.statusCode).toBe(200); + expect(detailResponse.json().preview.detail).toMatchObject({ + itemId: "833444005595", + shopName: "Apple Flagship Store" + }); + + const reviewsResponse = await app.inject({ + method: "GET", + url: "/api/platforms/tmall/live-reviews-preview?itemId=833444005595&commentCount=20&page=1&maxPages=2" + }); + expect(reviewsResponse.statusCode).toBe(200); + expect(reviewsResponse.json().preview.pagination).toMatchObject({ + requestedCommentCount: 20, + maxPages: 2 + }); + + const productResponse = await app.inject({ + method: "GET", + url: "/api/platforms/tmall/live-product-preview?itemId=833444005595&commentCount=20&page=1&maxPages=2" + }); + expect(productResponse.statusCode).toBe(200); + expect(productResponse.json().preview).toMatchObject({ + itemId: "833444005595", + pagination: { + requestedCommentCount: 20, + maxPages: 2 + } + }); + + await app.close(); + }); + + it("closes the tmall minimal loop with live detail/reviews replay and report publishing", async () => { + const tmallLiveService = createTmallLiveServiceStub(); + const app = createServer({ tmallLiveService }); + await app.ready(); + + const importResponse = await app.inject({ + method: "POST", + url: "/api/platforms/tmall/live-session", + payload: { + cookieHeader: "_tb_token_=masked;", + detailTemplateUrl: + "https://h5api.m.taobao.com/h5/mtop.taobao.pcdetail.data.get/1.0/?data=%7B%22id%22%3A%22833444005595%22%7D", + reviewsTemplateUrl: + "https://h5api.m.taobao.com/h5/mtop.alibaba.review.list.for.new.pc.detail/1.0/?data=%7B%22itemId%22%3A%22833444005595%22%7D" + } + }); + expect(importResponse.statusCode).toBe(200); + + const createdTask = await createTask(app, "iPhone 15"); + + const candidatesResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/candidates` + }); + const firstTmallCandidate = candidatesResponse.json().candidates.tmall[0]; + expect(firstTmallCandidate).toMatchObject({ + productUrl: "https://detail.tmall.com/item.htm?id=833444005595" + }); + + const confirmResponse = await app.inject({ + method: "POST", + url: `/api/tasks/${createdTask.taskId}/confirm`, + payload: { + selections: [ + { + platform: "tmall", + candidateIds: [firstTmallCandidate.candidateId] + } + ] + } + }); + + expect(confirmResponse.statusCode).toBe(200); + expect(confirmResponse.json().task).toMatchObject({ + taskStatus: "Completed", + defaultReportVersion: 1 + }); + + const reportResponse = await app.inject({ + method: "GET", + url: `/api/tasks/${createdTask.taskId}/report` + }); + + expect(reportResponse.statusCode).toBe(200); + expect(reportResponse.json().report.summary.headline).toContain("天猫"); + expect(reportResponse.json().report.product_snapshot.review_sample_count).toBe(1); + expect(reportResponse.json().report.evidence_index).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + platform: "tmall", + source_type: "product", + source_url: "https://detail.tmall.com/item.htm?id=833444005595" + }), + expect.objectContaining({ + platform: "tmall", + source_type: "review", + review_ref: "review-1" + }) + ]) + ); + + await app.close(); + }); + + it("surfaces Tmall live preview failures with service-provided status codes", async () => { + const tmallLiveService = createTmallLiveServiceStub({ + async previewDetail() { + const error = new Error("Tmall detail session appears invalid.") as Error & { + statusCode: number; + }; + error.statusCode = 409; + throw error; + } + }); + const app = createServer({ tmallLiveService }); + await app.ready(); + + const response = await app.inject({ + method: "GET", + url: "/api/platforms/tmall/live-detail-preview?itemId=833444005595" + }); + + expect(response.statusCode).toBe(409); + expect(response.json()).toMatchObject({ + message: "Tmall detail session appears invalid." + }); + + await app.close(); + }); +}); diff --git a/apps/api/src/server.tmall-session-manager.test.ts b/apps/api/src/server.tmall-session-manager.test.ts new file mode 100644 index 0000000..47ec348 --- /dev/null +++ b/apps/api/src/server.tmall-session-manager.test.ts @@ -0,0 +1,310 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { createServer } from "./server"; +import type { + TmallLiveService, + TmallSessionManager, + TmallSessionManagerState +} from "./platforms/tmall/types"; + +function createSessionSummary(configured: boolean) { + return configured + ? { + configured: true, + importedAt: "2026-04-03T08:30:00.000Z", + hasCookie: true, + userAgent: "stub-user-agent", + detailTemplate: { + available: true, + api: "mtop.taobao.pcdetail.data.get", + itemId: "934454505228" + }, + reviewsTemplate: { + available: true, + api: "mtop.taobao.rate.detaillist.get", + itemId: "934454505228" + } + } + : { + configured: false, + hasCookie: false, + detailTemplate: { + available: false + }, + reviewsTemplate: { + available: false + } + }; +} + +function createManagerState( + overrides: Partial = {} +): TmallSessionManagerState { + return { + status: "idle", + enabled: true, + heartbeatItemId: "934454505228", + checkIntervalMs: 600000, + pendingManualAction: false, + note: "天猫会话等待运维注入。", + publicNote: "天猫会话由运维后台维护，当前尚未就绪。", + session: createSessionSummary(false), + ...overrides + }; +} + +describe("Tmall ops session manager routes", () => { + const apps: Array>> = []; + + afterEach(async () => { + while (apps.length > 0) { + await apps.pop()!.close(); + } + }); + + it("exposes ops endpoints and keeps readiness/live-session state in sync", async () => { + let state = createManagerState(); + const manager: TmallSessionManager = { + getState() { + return state; + }, + configure(input) { + state = createManagerState({ + ...state, + enabled: input.enabled ?? state.enabled, + heartbeatItemId: input.heartbeatItemId ?? state.heartbeatItemId, + checkIntervalMs: input.checkIntervalMs ?? state.checkIntervalMs, + note: "天猫运维配置已更新。" + }); + return state; + }, + clearConfig() { + state = createManagerState({ + ...state, + enabled: false, + note: "天猫运维配置已清空。" + }); + return state; + }, + async importManualSession() { + state = createManagerState({ + ...state, + status: "healthy", + note: "天猫会话已通过 ops-manual 更新。", + publicNote: "天猫会话由运维后台维护，当前可用。", + session: createSessionSummary(true) + }); + return state; + }, + clearManagedSession() { + state = createManagerState({ + ...state, + status: "idle", + note: "天猫会话已清理。", + publicNote: "天猫会话由运维后台维护，当前尚未就绪。", + session: createSessionSummary(false) + }); + return state; + }, + async runHealthCheck() { + state = createManagerState({ + ...state, + status: "healthy", + note: "天猫会话健康检查通过。", + publicNote: "天猫会话由运维后台维护，当前可用。", + session: createSessionSummary(true) + }); + return { + state, + recovered: false + }; + }, + async handleLiveFailure() { + return false; + }, + shutdown() {} + }; + + const tmallLiveService: TmallLiveService = { + getSessionSummary() { + return state.session; + }, + importSession() { + return state.session; + }, + clearSession() {}, + async previewSearch(query) { + return { + query, + source: "html", + candidateCount: 1, + candidates: [ + { + candidateId: "tmall-934454505228", + platform: "tmall", + title: "Apple iPhone 15", + price: 4399, + priceLabel: "CNY 4399", + storeName: "Apple 官方旗舰店", + productUrl: "https://detail.tmall.com/item.htm?id=934454505228", + imageUrl: "https://img.alicdn.com/example.jpg", + salesHint: "已售 70万+", + specLabel: "128GB", + highlights: ["天猫"] + } + ] + }; + }, + async previewDetail(itemId) { + return { + itemId, + source: "html", + detail: { + itemId, + title: "Apple iPhone 15", + subtitle: null, + price: "4399.00", + originalPrice: "4999.00", + shopName: "Apple 官方旗舰店", + shopUrl: null, + sellerType: "tmall", + categoryPath: [], + mainImage: null, + salesDesc: "已售 70万+", + commentCount: "20万+" + } + }; + }, + async previewReviews(itemId) { + return { + itemId, + source: "api", + pagination: { + requestedPage: 1, + requestedCommentCount: 1, + maxPages: 1, + pagesFetched: 1 + }, + reviews: { + itemId, + total: "20万+", + hasNext: false, + allCount: "20万+", + pictureCount: "1万+", + appendCount: "5000+", + tags: [], + comments: [] + } + }; + }, + async previewProduct(itemId, options) { + const detail = await this.previewDetail(itemId); + const reviews = await this.previewReviews(itemId, options); + return { + itemId, + source: "hybrid", + detail: detail.detail, + pagination: reviews.pagination, + reviews: reviews.reviews + }; + } + }; + + const app = createServer({ + tmallLiveService, + tmallSessionManager: manager + }); + apps.push(app); + await app.ready(); + + const getManagerResponse = await app.inject({ + method: "GET", + url: "/api/ops/tmall/session-manager" + }); + expect(getManagerResponse.statusCode).toBe(200); + expect(getManagerResponse.json().manager).toMatchObject({ + status: "idle", + publicNote: "天猫会话由运维后台维护，当前尚未就绪。" + }); + + const readinessResponse = await app.inject({ + method: "GET", + url: "/api/platforms/readiness" + }); + expect( + readinessResponse + .json() + .platforms.find((platform: { platform: string }) => platform.platform === "tmall") + ).toMatchObject({ + platform: "tmall", + reason: "天猫会话由运维后台维护，当前尚未就绪。" + }); + + const configResponse = await app.inject({ + method: "POST", + url: "/api/ops/tmall/session-manager/config", + payload: { + enabled: true, + heartbeatItemId: "934454505228" + } + }); + expect(configResponse.statusCode).toBe(200); + expect(configResponse.json().manager).toMatchObject({ + heartbeatItemId: "934454505228" + }); + + const healthCheckResponse = await app.inject({ + method: "POST", + url: "/api/ops/tmall/session-manager/check" + }); + expect(healthCheckResponse.statusCode).toBe(200); + expect(healthCheckResponse.json()).toMatchObject({ + recovered: false, + state: { + status: "healthy" + } + }); + + const importResponse = await app.inject({ + method: "POST", + url: "/api/ops/tmall/session-manager/session", + payload: { + cookieHeader: "_m_h5_tk=masked_token_123;", + detailTemplateUrl: "https://detail.tmall.com/item.htm?id=934454505228", + reviewsTemplateUrl: + "https://h5api.m.tmall.com/h5/mtop.taobao.rate.detaillist.get/6.0/?data=%7B%22auctionNumId%22%3A%22934454505228%22%7D" + } + }); + expect(importResponse.statusCode).toBe(200); + expect(importResponse.json().manager).toMatchObject({ + status: "healthy", + session: { + configured: true + } + }); + + const liveSessionResponse = await app.inject({ + method: "GET", + url: "/api/platforms/tmall/live-session" + }); + expect(liveSessionResponse.statusCode).toBe(200); + expect(liveSessionResponse.json().session).toMatchObject({ + configured: true, + detailTemplate: { + available: true + } + }); + + const clearResponse = await app.inject({ + method: "DELETE", + url: "/api/ops/tmall/session-manager/session" + }); + expect(clearResponse.statusCode).toBe(200); + expect(clearResponse.json().manager).toMatchObject({ + status: "idle", + session: { + configured: false + } + }); + }); +}); diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index a70ad24..349685a 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -5,15 +5,79 @@ import type { } from "@cross-ai/domain"; import cors from "@fastify/cors"; import Fastify from "fastify"; +import { resolve } from "node:path"; +import { rankJdCandidatesForKeyword } from "./platforms/jd/keyword-preview"; import { JdLiveSessionService, isJdLiveError } from "./platforms/jd/live-session"; -import type { JdLiveService, JdSearchMode } from "./platforms/jd/types"; +import { JdSessionManagerService } from "./platforms/jd/session-manager"; +import type { + JdLiveService, + JdSearchMode, + JdSessionManager, + JdSessionManagerAutoMode +} from "./platforms/jd/types"; +import { TmallLiveSessionService, isTmallLiveError } from "./platforms/tmall/live-session"; +import { TmallSessionManagerService } from "./platforms/tmall/session-manager"; +import type { + TmallLiveService, + TmallSessionManager, + TmallSessionManagerConfigInput +} from "./platforms/tmall/types"; import { InMemoryTaskStore } from "./store"; -export function createServer(options: { jdLiveService?: JdLiveService } = {}) { +export function createServer( + options: { + storagePath?: string; + jdLiveService?: JdLiveService; + jdSessionManager?: JdSessionManager; + tmallLiveService?: TmallLiveService; + tmallSessionManager?: TmallSessionManager; + } = {} +) { const app = Fastify({ logger: false }); - const store = new InMemoryTaskStore(); const jdLiveService = options.jdLiveService ?? new JdLiveSessionService(); + const tmallLiveService = options.tmallLiveService ?? new TmallLiveSessionService(); + const storagePath = + options.storagePath ?? + process.env.TASK_STORE_PATH ?? + (process.env.NODE_ENV === "test" + ? undefined + : resolve(process.cwd(), ".data", "task-store.json")); + const store = new InMemoryTaskStore({ storagePath, jdLiveService, tmallLiveService }); + const jdSessionManager = + options.jdSessionManager ?? + new JdSessionManagerService(jdLiveService, { + onSessionReady: () => { + store.preparePlatform("jd"); + }, + onSessionUnavailable: () => { + store.clearPlatformSession("jd"); + } + }); + const tmallSessionManager = + options.tmallSessionManager ?? + new TmallSessionManagerService(tmallLiveService, { + onSessionReady: () => { + store.preparePlatform("tmall"); + }, + onSessionUnavailable: () => { + store.clearPlatformSession("tmall"); + } + }); + store.setJdSessionManager(jdSessionManager); + store.setTmallSessionManager(tmallSessionManager); + + if (jdLiveService.getSessionSummary().configured && !store.getSession("jd").ready) { + store.preparePlatform("jd"); + } + if (tmallLiveService.getSessionSummary().configured && !store.getSession("tmall").ready) { + store.preparePlatform("tmall"); + } + + app.addHook("onClose", async () => { + jdSessionManager.shutdown(); + tmallSessionManager.shutdown(); + }); app.register(cors, { origin: true }); @@ -23,7 +87,19 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { })); app.get("/api/platforms/readiness", async () => ({ - platforms: store.getPlatformReadiness() + platforms: store.getPlatformReadiness().map((platform) => + platform.platform === "jd" + ? { + ...platform, + reason: jdSessionManager.getState().publicNote + } + : platform.platform === "tmall" + ? { + ...platform, + reason: tmallSessionManager.getState().publicNote + } + : platform + ) })); app.get("/api/sessions", async () => ({ @@ -61,6 +137,163 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { session: jdLiveService.getSessionSummary() })); + app.get("/api/ops/jd/session-manager", async () => ({ + manager: jdSessionManager.getState() + })); + + app.post<{ + Body: { + enabled?: boolean; + autoLoginMode?: JdSessionManagerAutoMode; + loginCommand?: string | null; + browserProfilePath?: string | null; + heartbeatQuery?: string | null; + account?: string | null; + password?: string | null; + checkIntervalMs?: number | null; + runnerTimeoutMs?: number | null; + }; + }>("/api/ops/jd/session-manager/config", async (request, reply) => { + try { + const manager = jdSessionManager.configure(request.body); + reply.code(200); + return { manager }; + } catch (error) { + reply.code(400); + return { + message: + error instanceof Error ? error.message : "Invalid JD ops session manager config." + }; + } + }); + + app.delete("/api/ops/jd/session-manager/config", async () => ({ + manager: jdSessionManager.clearConfig() + })); + + app.post("/api/ops/jd/session-manager/check", async (_request, reply) => { + try { + const result = await jdSessionManager.runHealthCheck("ops"); + reply.code(200); + return result; + } catch (error) { + reply.code(502); + return { + message: + error instanceof Error ? error.message : "JD ops health check failed." + }; + } + }); + + app.post("/api/ops/jd/session-manager/recover", async (_request, reply) => { + try { + const result = await jdSessionManager.runAutoRecovery("ops"); + reply.code(200); + return result; + } catch (error) { + reply.code(502); + return { + message: + error instanceof Error ? error.message : "JD ops auto recovery failed." + }; + } + }); + + app.post<{ + Body: { + cookieHeader: string; + userAgent?: string; + searchApiTemplateUrl?: string; + detailTemplateUrl?: string; + reviewsTemplateUrl?: string; + searchReferer?: string; + detailReferer?: string; + }; + }>("/api/ops/jd/session-manager/session", async (request, reply) => { + try { + const manager = await jdSessionManager.importManualSession(request.body, "ops-manual"); + reply.code(200); + return { manager }; + } catch (error) { + reply.code(isJdLiveError(error) ? error.statusCode : 400); + return { + message: error instanceof Error ? error.message : "Invalid JD ops live session payload." + }; + } + }); + + app.delete("/api/ops/jd/session-manager/session", async () => ({ + manager: jdSessionManager.clearManagedSession("ops-manual-clear") + })); + + app.get("/api/ops/tmall/session-manager", async () => ({ + manager: tmallSessionManager.getState() + })); + + app.post<{ + Body: TmallSessionManagerConfigInput; + }>("/api/ops/tmall/session-manager/config", async (request, reply) => { + try { + const manager = tmallSessionManager.configure(request.body); + reply.code(200); + return { manager }; + } catch (error) { + reply.code(400); + return { + message: + error instanceof Error ? error.message : "Invalid Tmall ops session manager config." + }; + } + }); + + app.delete("/api/ops/tmall/session-manager/config", async () => ({ + manager: tmallSessionManager.clearConfig() + })); + + app.post("/api/ops/tmall/session-manager/check", async (_request, reply) => { + try { + const result = await tmallSessionManager.runHealthCheck("ops"); + reply.code(200); + return result; + } catch (error) { + reply.code(502); + return { + message: + error instanceof Error ? error.message : "Tmall ops health check failed." + }; + } + }); + + app.post<{ + Body: { + cookieHeader: string; + userAgent?: string; + detailTemplateUrl?: string; + reviewsTemplateUrl?: string; + detailReferer?: string; + }; + }>("/api/ops/tmall/session-manager/session", async (request, reply) => { + try { + const manager = await tmallSessionManager.importManualSession(request.body, "ops-manual"); + reply.code(200); + return { manager }; + } catch (error) { + reply.code(isTmallLiveError(error) ? error.statusCode : 400); + return { + message: + error instanceof Error ? error.message : "Invalid Tmall ops live session payload." + }; + } + }); + + app.delete("/api/ops/tmall/session-manager/session", async () => ({ + manager: tmallSessionManager.clearManagedSession("ops-manual-clear") + })); + + app.get("/api/platforms/tmall/live-session", async () => ({ + session: tmallLiveService.getSessionSummary() + })); + app.post<{ Body: { cookieHeader: string; @@ -73,10 +306,9 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { }; }>("/api/platforms/jd/live-session", async (request, reply) => { try { - const session = jdLiveService.importSession(request.body); - store.preparePlatform("jd"); + await jdSessionManager.importManualSession(request.body, "legacy-live-session"); reply.code(200); - return { session }; + return { session: jdLiveService.getSessionSummary() }; } catch (error) { reply.code(isJdLiveError(error) ? error.statusCode : 400); return { @@ -85,9 +317,36 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { } }); + app.post<{ + Body: { + cookieHeader: string; + userAgent?: string; + detailTemplateUrl?: string; + reviewsTemplateUrl?: string; + detailReferer?: string; + }; + }>("/api/platforms/tmall/live-session", async (request, reply) => { + try { + await tmallSessionManager.importManualSession(request.body, "legacy-live-session"); + const session = tmallLiveService.getSessionSummary(); + reply.code(200); + return { session }; + } catch (error) { + reply.code(isTmallLiveError(error) ? error.statusCode : 400); + return { + message: error instanceof Error ? error.message : "Invalid Tmall live session payload." + }; + } + }); + app.delete("/api/platforms/jd/live-session", async (_request, reply) => { - jdLiveService.clearSession(); - store.clearPlatformSession("jd"); + jdSessionManager.clearManagedSession("legacy-live-session-clear"); + reply.code(204); + return null; + }); + + app.delete("/api/platforms/tmall/live-session", async (_request, reply) => { + tmallSessionManager.clearManagedSession("legacy-live-session-clear"); reply.code(204); return null; }); @@ -100,6 +359,9 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { if (request.params.platform === "jd") { jdLiveService.clearSession(); } + if (request.params.platform === "tmall") { + tmallLiveService.clearSession(); + } reply.code(204); return null; } catch { @@ -111,7 +373,7 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { app.post<{ Body: CreateTaskInput; }>("/api/tasks", async (request, reply) => { - const task = store.createTask(request.body); + const task = await store.createTask(request.body); reply.code(201); return { task }; }); @@ -180,7 +442,7 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { Body: ConfirmTaskPayload; }>("/api/tasks/:taskId/confirm", async (request, reply) => { try { - const task = store.confirmTask(request.params.taskId, request.body); + const task = await store.confirmTask(request.params.taskId, request.body); return { task }; } catch { reply.code(404); @@ -192,7 +454,7 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { Params: { taskId: string; platform: PlatformId }; }>("/api/tasks/:taskId/platforms/:platform/retry", async (request, reply) => { try { - const task = store.retryPlatform(request.params.taskId, request.params.platform); + const task = await store.retryPlatform(request.params.taskId, request.params.platform); return { task }; } catch { reply.code(404); @@ -256,7 +518,71 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { }); app.get<{ - Querystring: { skuId?: string; commentCount?: string }; + Querystring: { + query?: string; + commentCount?: string; + maxPages?: string; + mode?: JdSearchMode; + }; + }>("/api/platforms/jd/live-keyword-preview", async (request, reply) => { + try { + const query = request.query.query?.trim(); + if (!query) { + reply.code(400); + return { message: "query is required." }; + } + + const commentCount = request.query.commentCount + ? Number.parseInt(request.query.commentCount, 10) + : undefined; + const maxPages = request.query.maxPages + ? Number.parseInt(request.query.maxPages, 10) + : undefined; + const searchPreview = await jdLiveService.previewSearch(query, request.query.mode); + const rankedCandidates = rankJdCandidatesForKeyword(query, searchPreview.candidates); + const selectedCandidate = rankedCandidates[0]; + + if (!selectedCandidate) { + reply.code(404); + return { + message: + "JD live search returned no valid item candidates for this keyword. Capture a fresher search session or refine the query." + }; + } + + const preview = await jdLiveService.previewProduct(selectedCandidate.skuId, { + commentCount, + maxPages: Number.isNaN(maxPages ?? Number.NaN) ? undefined : maxPages + }); + + return { + preview: { + query, + search: { + source: searchPreview.source, + candidateCount: searchPreview.candidateCount, + selected: selectedCandidate, + alternatives: rankedCandidates.slice(1, 4) + }, + product: preview + } + }; + } catch (error) { + reply.code(isJdLiveError(error) ? error.statusCode : 502); + return { + message: + error instanceof Error ? error.message : "JD live keyword preview failed." + }; + } + }); + + app.get<{ + Querystring: { + skuId?: string; + commentCount?: string; + page?: string; + maxPages?: string; + }; }>("/api/platforms/jd/live-reviews-preview", async (request, reply) => { try { const skuId = request.query.skuId?.trim(); @@ -268,7 +594,15 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { const commentCount = request.query.commentCount ? Number.parseInt(request.query.commentCount, 10) : undefined; - const preview = await jdLiveService.previewReviews(skuId, commentCount); + const page = request.query.page ? Number.parseInt(request.query.page, 10) : undefined; + const maxPages = request.query.maxPages + ? Number.parseInt(request.query.maxPages, 10) + : undefined; + const preview = await jdLiveService.previewReviews(skuId, { + commentCount, + page: Number.isNaN(page ?? Number.NaN) ? undefined : page, + maxPages: Number.isNaN(maxPages ?? Number.NaN) ? undefined : maxPages + }); return { preview }; } catch (error) { reply.code(isJdLiveError(error) ? error.statusCode : 502); @@ -279,6 +613,182 @@ export function createServer(options: { jdLiveService?: JdLiveService } = {}) { } }); + app.get<{ + Querystring: { + skuId?: string; + commentCount?: string; + page?: string; + maxPages?: string; + }; + }>("/api/platforms/jd/live-product-preview", async (request, reply) => { + try { + const skuId = request.query.skuId?.trim(); + if (!skuId) { + reply.code(400); + return { message: "skuId is required." }; + } + + const commentCount = request.query.commentCount + ? Number.parseInt(request.query.commentCount, 10) + : undefined; + const page = request.query.page ? Number.parseInt(request.query.page, 10) : undefined; + const maxPages = request.query.maxPages + ? Number.parseInt(request.query.maxPages, 10) + : undefined; + + const preview = await jdLiveService.previewProduct(skuId, { + commentCount, + page: Number.isNaN(page ?? Number.NaN) ? undefined : page, + maxPages: Number.isNaN(maxPages ?? Number.NaN) ? undefined : maxPages + }); + return { preview }; + } catch (error) { + reply.code(isJdLiveError(error) ? error.statusCode : 502); + return { + message: + error instanceof Error ? error.message : "JD live product preview failed." + }; + } + }); + + app.get<{ + Querystring: { itemId?: string }; + }>("/api/platforms/tmall/live-detail-preview", async (request, reply) => { + try { + const itemId = request.query.itemId?.trim(); + if (!itemId) { + reply.code(400); + return { message: "itemId is required." }; + } + + const preview = await tmallLiveService.previewDetail(itemId); + return { preview }; + } catch (error) { + reply.code(isTmallLiveError(error) ? error.statusCode : 502); + return { + message: + error instanceof Error ? error.message : "Tmall live detail preview failed." + }; + } + }); + + app.get<{ + Querystring: { query?: string }; + }>("/api/platforms/tmall/live-search-preview", async (request, reply) => { + try { + const query = request.query.query?.trim(); + if (!query) { + reply.code(400); + return { message: "query is required." }; + } + + const preview = await tmallLiveService.previewSearch(query); + return { preview }; + } catch (error) { + reply.code(isTmallLiveError(error) ? error.statusCode : 502); + return { + message: + error instanceof Error ? error.message : "Tmall live search preview failed." + }; + } + }); + + app.get<{ + Querystring: { + itemId?: string; + commentCount?: string; + page?: string; + maxPages?: string; + }; + }>("/api/platforms/tmall/live-reviews-preview", async (request, reply) => { + try { + const itemId = request.query.itemId?.trim(); + if (!itemId) { + reply.code(400); + return { message: "itemId is required." }; + } + + const commentCount = request.query.commentCount + ? Number.parseInt(request.query.commentCount, 10) + : undefined; + const page = request.query.page ? Number.parseInt(request.query.page, 10) : undefined; + const maxPages = request.query.maxPages + ? Number.parseInt(request.query.maxPages, 10) + : undefined; + + const preview = await tmallLiveService.previewReviews(itemId, { + commentCount, + page: Number.isNaN(page ?? Number.NaN) ? undefined : page, + maxPages: Number.isNaN(maxPages ?? Number.NaN) ? undefined : maxPages + }); + return { preview }; + } catch (error) { + reply.code(isTmallLiveError(error) ? error.statusCode : 502); + return { + message: + error instanceof Error ? error.message : "Tmall live reviews preview failed." + }; + } + }); + + app.get<{ + Querystring: { + itemId?: string; + commentCount?: string; + page?: string; + maxPages?: string; + }; + }>("/api/platforms/tmall/live-product-preview", async (request, reply) => { + try { + const itemId = request.query.itemId?.trim(); + if (!itemId) { + reply.code(400); + return { message: "itemId is required." }; + } + + const commentCount = request.query.commentCount + ? Number.parseInt(request.query.commentCount, 10) + : undefined; + const page = request.query.page ? Number.parseInt(request.query.page, 10) : undefined; + const maxPages = request.query.maxPages + ? Number.parseInt(request.query.maxPages, 10) + : undefined; + + const preview = await tmallLiveService.previewProduct(itemId, { + commentCount, + page: Number.isNaN(page ?? Number.NaN) ? undefined : page, + maxPages: Number.isNaN(maxPages ?? Number.NaN) ? undefined : maxPages + }); + return { preview }; + } catch (error) { + reply.code(isTmallLiveError(error) ? error.statusCode : 502); + return { + message: + error instanceof Error ? error.message : "Tmall live product preview failed." + }; + } + }); + + app.post<{ + Body?: { + dryRun?: boolean; + asOf?: string; + rawRetentionDays?: number; + reportRetentionDays?: number; + }; + }>("/api/retention/cleanup", async (request, reply) => { + try { + const cleanup = store.runRetentionCleanup(request.body ?? {}); + return { cleanup }; + } catch (error) { + reply.code(400); + return { + message: + error instanceof Error ? error.message : "Invalid retention cleanup request." + }; + } + }); + app.get("/api/history", async () => ({ tasks: store.listHistory() })); diff --git a/apps/api/src/store.ts b/apps/api/src/store.ts index 60812ee..5439128 100644 --- a/apps/api/src/store.ts +++ b/apps/api/src/store.ts @@ -32,8 +32,38 @@ import { } from "@cross-ai/domain"; import { type ReportSnapshot, parseReportSnapshot } from "@cross-ai/report-schema"; import { randomUUID } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; import { createMockCandidates } from "./mock-data"; +import { getJdLiveErrorCode, isJdLiveError } from "./platforms/jd/live-session"; +import type { + JdLiveService, + JdProductDetailSnapshot, + JdProductPreviewResult, + JdProductReviewsSnapshot, + JdReviewCommentSnapshot, + JdSessionManager, + JdReviewsPaginationSummary, + JdReviewsPreviewOptions +} from "./platforms/jd/types"; +import { isTmallLiveError } from "./platforms/tmall/live-session"; +import type { + TmallLiveService, + TmallProductDetailSnapshot, + TmallProductPreviewResult, + TmallProductReviewsSnapshot, + TmallReviewCommentSnapshot, + TmallSessionManager, + TmallReviewsPaginationSummary, + TmallReviewsPreviewOptions +} from "./platforms/tmall/types"; +import { + buildReviewBudgetPlan, + sampleReviewComments, + type ReviewSamplingBucket, + type ReviewSamplingComment +} from "./review-sampling"; function nowIso(): string { return new Date().toISOString(); @@ -148,6 +178,30 @@ function getMockDurationMs( const SESSION_TTL_HOURS = 24; const SESSION_TTL_MS = SESSION_TTL_HOURS * 60 * 60 * 1000; +const DEFAULT_RAW_RETENTION_DAYS = 30; +const DEFAULT_REPORT_RETENTION_DAYS = 90; + +type RetentionCleanupOptions = { + dryRun?: boolean; + asOf?: string; + rawRetentionDays?: number; + reportRetentionDays?: number; +}; + +export type RetentionCleanupResult = { + dryRun: boolean; + asOf: string; + rawCutoffAt: string; + reportCutoffAt: string; + scannedTaskCount: number; + cleanedTaskCount: number; + rawCleanedTaskCount: number; + reportCleanedTaskCount: number; + deletedReportCount: number; + deletedArtifactCount: number; + residualArtifactCount: number; + affectedTaskIds: string[]; +}; type StoredSessionState = Omit & { encryptedSnapshot: string | null; @@ -228,6 +282,130 @@ function isReportableTaskStatus( return status === "Completed" || status === "PartialCompleted"; } +type InMemoryTaskStoreOptions = { + storagePath?: string | undefined; + jdLiveService?: JdLiveService | undefined; + jdSessionManager?: JdSessionManager | undefined; + tmallLiveService?: TmallLiveService | undefined; + tmallSessionManager?: TmallSessionManager | undefined; +}; + +type TaskExecutionArtifact = { + platform: PlatformId; + candidateId: string; + source: "jd-live" | "tmall-live"; + capturedAt: string; + detail: JdProductDetailSnapshot | TmallProductDetailSnapshot; + pagination: JdReviewsPaginationSummary | TmallReviewsPaginationSummary; + reviews: JdProductReviewsSnapshot | TmallProductReviewsSnapshot; +}; + +type LiveExecutionFailure = { + status: Extract; + errorType: string; + reason: string; + detail: string; +}; + +type ExecutionReviewComment = JdReviewCommentSnapshot | TmallReviewCommentSnapshot; +type CandidateReviewSample = { + candidateId: string; + targetCount: number; + actualCount: number; + sampleInsufficient: boolean; + bucketCounts: Record; + comments: Array<{ + bucket: ReviewSamplingBucket; + comment: ExecutionReviewComment; + }>; +}; + +const NEGATIVE_REVIEW_KEYWORDS = [ + "发热", + "卡顿", + "掉电", + "噪音", + "划痕", + "瑕疵", + "慢", + "差", + "bug", + "heat", + "slow", + "lag", + "scratches" +]; + +function isJdArtifact( + artifact: TaskExecutionArtifact | undefined +): artifact is TaskExecutionArtifact & { + source: "jd-live"; + detail: JdProductDetailSnapshot; + pagination: JdReviewsPaginationSummary; + reviews: JdProductReviewsSnapshot; +} { + return artifact?.source === "jd-live"; +} + +function getArtifactEstimatedPrice(artifact: TaskExecutionArtifact | undefined): string | null { + return isJdArtifact(artifact) ? artifact.detail.estimatedPrice : null; +} + +function getArtifactStockState(artifact: TaskExecutionArtifact | undefined): string | null { + return isJdArtifact(artifact) ? artifact.detail.stockState : null; +} + +function getArtifactGoodRate(artifact: TaskExecutionArtifact | undefined): string | null { + return isJdArtifact(artifact) ? artifact.reviews.goodRate : null; +} + +function getExecutionCommentScore(comment: ExecutionReviewComment): string | null { + return "score" in comment ? comment.score : null; +} + +function getExecutionCommentDate(comment: ExecutionReviewComment): string | null { + return "creationTime" in comment ? comment.creationTime : comment.date; +} + +function getExecutionCommentUserLabel(comment: ExecutionReviewComment): string | null { + return "userLevelName" in comment ? comment.userLevelName : comment.userNick; +} + +function toReviewSamplingComment(comment: ExecutionReviewComment): ReviewSamplingComment { + return { + id: comment.id, + content: comment.content, + score: getExecutionCommentScore(comment), + createdAt: getExecutionCommentDate(comment), + authorLabel: getExecutionCommentUserLabel(comment) + }; +} + +function createEmptyJdReviewsSnapshot(skuId: string): JdProductReviewsSnapshot { + return { + skuId, + total: null, + goodRate: null, + pictureCount: null, + tags: [], + comments: [] + }; +} + +type PersistedTaskStoreState = { + tasks: TaskRecord[]; + reports: Array<[string, ReportSnapshot[]]>; + reportFingerprints: Array<[string, string[]]>; + readiness: StoredSessionState[]; + strategyAttempts: Array<[string, StrategyAttemptRecord[]]>; + platformRunMetrics: Array<[string, Record]>; + reportMetrics: ReportMetricRecord[]; + retentionMetrics: RetentionMetricRecord[]; + auditLogs: AuditLogRecord[]; + executionScenarios: Array<[string, Partial>]>; + executionArtifacts?: Array<[string, TaskExecutionArtifact[]]>; +}; + export class InMemoryTaskStore { private readonly tasks = new Map(); private readonly reports = new Map(); @@ -245,11 +423,36 @@ export class InMemoryTaskStore { string, Partial> >(); + private readonly executionArtifacts = new Map(); + private readonly storagePath: string | undefined; + private readonly jdLiveService: JdLiveService | undefined; + private jdSessionManager: JdSessionManager | undefined; + private readonly tmallLiveService: TmallLiveService | undefined; + private tmallSessionManager: TmallSessionManager | undefined; - constructor() { - const timestamp = nowIso(); - this.readiness.set("tmall", createPreparedSession("tmall", timestamp)); - this.readiness.set("jd", createMissingSession("jd")); + constructor(options: InMemoryTaskStoreOptions = {}) { + this.storagePath = options.storagePath; + this.jdLiveService = options.jdLiveService; + this.jdSessionManager = options.jdSessionManager; + this.tmallLiveService = options.tmallLiveService; + this.tmallSessionManager = options.tmallSessionManager; + + const persistedState = this.loadPersistedState(); + if (persistedState) { + this.hydrate(persistedState); + return; + } + + this.initializeDefaultState(); + this.persistState(); + } + + setJdSessionManager(manager: JdSessionManager | undefined): void { + this.jdSessionManager = manager; + } + + setTmallSessionManager(manager: TmallSessionManager | undefined): void { + this.tmallSessionManager = manager; } getPlatformReadiness(): SessionReadinessRecord[] { @@ -276,6 +479,7 @@ export class InMemoryTaskStore { expires_at: next.expiresAt ?? null } }); + this.persistState(); return this.toSessionRecord(next); } @@ -289,9 +493,10 @@ export class InMemoryTaskStore { search_requirement: cleared.searchRequirement } }); + this.persistState(); } - createTask(input: CreateTaskInput): TaskRecord { + async createTask(input: CreateTaskInput): Promise { const timestamp = nowIso(); const taskId = randomUUID(); const executionScenarios = parseMockExecutionScenarios(input.query); @@ -331,8 +536,9 @@ export class InMemoryTaskStore { task.taskStage = "precheck"; this.pushEvent(task, "task.searching", "系统已开始执行平台预检查。"); - this.runSearch(task); + await this.runSearch(task); this.tasks.set(task.taskId, task); + this.persistState(); return task; } @@ -452,7 +658,7 @@ export class InMemoryTaskStore { }; } - confirmTask(taskId: string, payload: ConfirmTaskPayload): TaskRecord { + async confirmTask(taskId: string, payload: ConfirmTaskPayload): Promise { const task = this.requireTask(taskId); const selectionMap = new Map( payload.selections.map((selection) => [selection.platform, selection.candidateIds]) @@ -490,6 +696,7 @@ export class InMemoryTaskStore { if (confirmedRuns.length === 0) { task.taskStatus = "NoSelection"; this.pushEvent(task, "task.no_selection", "用户未确认任何商品链接，任务结束。"); + this.persistState(); return task; } @@ -499,6 +706,7 @@ export class InMemoryTaskStore { task.taskStatus = deriveTaskStatusFromConfirmedPlatforms(task.platformRuns); task.updatedAt = nowIso(); this.publishReportIfNeeded(task); + this.persistState(); return task; } @@ -506,15 +714,16 @@ export class InMemoryTaskStore { task.taskStage = "session_check"; this.pushEvent(task, "task.running", "系统开始执行抓取前校验。"); - this.executeSelectedPlatforms(task, selectedRuns); + await this.executeSelectedPlatforms(task, selectedRuns); task.taskStatus = deriveTaskStatusFromConfirmedPlatforms(task.platformRuns); task.updatedAt = nowIso(); this.publishReportIfNeeded(task); + this.persistState(); return task; } - retryPlatform(taskId: string, platform: PlatformId): TaskRecord { + async retryPlatform(taskId: string, platform: PlatformId): Promise { const task = this.requireTask(taskId); const run = task.platformRuns.find((item) => item.platform === platform); @@ -534,6 +743,7 @@ export class InMemoryTaskStore { `platform.${platform}.retry_blocked`, `${platformCatalogMap[platform].label} 仍缺少有效会话，无法重新搜索。` ); + this.persistState(); return task; } @@ -557,7 +767,7 @@ export class InMemoryTaskStore { platform, message: `${platformCatalogMap[platform].label} 已完成恢复，准备重新搜索。` }); - this.runSearchForPlatform(task, run, "recovery"); + await this.runSearchForPlatform(task, run, "recovery"); const recoveredCandidateCount = task.platformCandidates[platform].length; task.taskStage = "confirmation"; task.taskStatus = "AwaitingConfirmation"; @@ -581,6 +791,7 @@ export class InMemoryTaskStore { next_status: run.status } }); + this.persistState(); return task; } @@ -590,6 +801,7 @@ export class InMemoryTaskStore { `platform.${platform}.retry_skipped`, `${platformCatalogMap[platform].label} 当前没有已确认链接，无法执行定向重试。` ); + this.persistState(); return task; } @@ -619,7 +831,11 @@ export class InMemoryTaskStore { } }); - this.executeSelectedPlatforms(task, [run], previousStatus === "Blocked" ? "recovery" : "retry"); + await this.executeSelectedPlatforms( + task, + [run], + previousStatus === "Blocked" ? "recovery" : "retry" + ); task.taskStatus = deriveTaskStatusFromConfirmedPlatforms(task.platformRuns); task.updatedAt = nowIso(); this.publishReportIfNeeded(task); @@ -632,6 +848,7 @@ export class InMemoryTaskStore { task_status: task.taskStatus } }); + this.persistState(); return task; } @@ -652,12 +869,13 @@ export class InMemoryTaskStore { deleteTask(taskId: string): void { const task = this.requireTask(taskId); const deletedReportCount = this.reports.get(taskId)?.length ?? 0; + const deletedArtifactCount = this.clearExecutionArtifacts(taskId); this.retentionMetrics.push({ metricId: randomUUID(), action: "task_deleted", taskId, deletedReportCount, - deletedArtifactCount: 0, + deletedArtifactCount, residualArtifactCount: 0, recordedAt: nowIso() }); @@ -674,13 +892,770 @@ export class InMemoryTaskStore { this.strategyAttempts.delete(taskId); this.platformRunMetrics.delete(taskId); this.executionScenarios.delete(taskId); + this.persistState(); } - private runSearch(task: TaskRecord): void { + runRetentionCleanup(options: RetentionCleanupOptions = {}): RetentionCleanupResult { + const dryRun = options.dryRun ?? false; + const asOf = options.asOf ?? nowIso(); + const rawRetentionDays = options.rawRetentionDays ?? DEFAULT_RAW_RETENTION_DAYS; + const reportRetentionDays = + options.reportRetentionDays ?? DEFAULT_REPORT_RETENTION_DAYS; + + if (Number.isNaN(Date.parse(asOf))) { + throw new Error("Invalid cleanup timestamp."); + } + + if (!Number.isInteger(rawRetentionDays) || rawRetentionDays < 0) { + throw new Error("rawRetentionDays must be a non-negative integer."); + } + + if (!Number.isInteger(reportRetentionDays) || reportRetentionDays < 0) { + throw new Error("reportRetentionDays must be a non-negative integer."); + } + + if (rawRetentionDays > reportRetentionDays) { + throw new Error("rawRetentionDays cannot exceed reportRetentionDays."); + } + + const rawCutoffAt = new Date( + Date.parse(asOf) - rawRetentionDays * 24 * 60 * 60 * 1000 + ).toISOString(); + const reportCutoffAt = new Date( + Date.parse(asOf) - reportRetentionDays * 24 * 60 * 60 * 1000 + ).toISOString(); + const rawCutoffMs = Date.parse(rawCutoffAt); + const reportCutoffMs = Date.parse(reportCutoffAt); + + let deletedReportCount = 0; + let deletedArtifactCount = 0; + let rawCleanedTaskCount = 0; + let reportCleanedTaskCount = 0; + const affectedTaskIds = new Set(); + + for (const task of this.tasks.values()) { + const taskUpdatedAt = Date.parse(task.updatedAt); + if (Number.isNaN(taskUpdatedAt)) { + continue; + } + + let rawTouched = false; + let reportTouched = false; + + if (taskUpdatedAt < rawCutoffMs) { + const deletedCandidateCount = this.pruneTaskCandidatesToSelected(task, !dryRun); + const deletedEventCount = task.events.length; + const deletedAttemptCount = this.strategyAttempts.get(task.taskId)?.length ?? 0; + const deletedScenarioCount = this.executionScenarios.has(task.taskId) ? 1 : 0; + const deletedExecutionArtifactCount = dryRun + ? this.getExecutionArtifacts(task.taskId).length + : this.clearExecutionArtifacts(task.taskId); + + if (!dryRun) { + task.events = []; + this.strategyAttempts.delete(task.taskId); + this.executionScenarios.delete(task.taskId); + } + + const rawDeletedCount = + deletedCandidateCount + + deletedEventCount + + deletedAttemptCount + + deletedScenarioCount + + deletedExecutionArtifactCount; + + if (rawDeletedCount > 0) { + deletedArtifactCount += rawDeletedCount; + rawTouched = true; + } + } + + if (taskUpdatedAt < reportCutoffMs) { + const deletedEvidenceCount = this.clearTaskCandidates(task, !dryRun); + if (deletedEvidenceCount > 0) { + deletedArtifactCount += deletedEvidenceCount; + reportTouched = true; + } + } + + const expiredReportsCleanup = this.clearExpiredReports( + task, + reportCutoffMs, + !dryRun + ); + if (expiredReportsCleanup.deletedReportCount > 0) { + deletedReportCount += expiredReportsCleanup.deletedReportCount; + reportTouched = true; + } + + if (rawTouched) { + rawCleanedTaskCount += 1; + } + + if (reportTouched) { + reportCleanedTaskCount += 1; + } + + if (rawTouched || reportTouched) { + affectedTaskIds.add(task.taskId); + } + } + + const cleanup: RetentionCleanupResult = { + dryRun, + asOf, + rawCutoffAt, + reportCutoffAt, + scannedTaskCount: this.tasks.size, + cleanedTaskCount: affectedTaskIds.size, + rawCleanedTaskCount, + reportCleanedTaskCount, + deletedReportCount, + deletedArtifactCount, + residualArtifactCount: 0, + affectedTaskIds: Array.from(affectedTaskIds).sort() + }; + + if (!dryRun) { + this.retentionMetrics.push({ + metricId: randomUUID(), + action: "retention_cleanup", + deletedReportCount, + deletedArtifactCount, + residualArtifactCount: cleanup.residualArtifactCount, + recordedAt: asOf + }); + this.persistState(); + } + + return cleanup; + } + + private initializeDefaultState(): void { + const timestamp = nowIso(); + this.readiness.set("tmall", createPreparedSession("tmall", timestamp)); + this.readiness.set("jd", createMissingSession("jd")); + } + + private hydrate(state: PersistedTaskStoreState): void { + for (const task of state.tasks ?? []) { + this.tasks.set(task.taskId, task); + } + + for (const [taskId, reports] of state.reports ?? []) { + this.reports.set( + taskId, + reports.map((report) => parseReportSnapshot(report)) + ); + } + + for (const [taskId, fingerprints] of state.reportFingerprints ?? []) { + this.reportFingerprints.set(taskId, fingerprints); + } + + const readinessByPlatform = new Map( + (state.readiness ?? []).map((session) => [session.platform, session] as const) + ); + const fallbackTimestamp = nowIso(); + this.readiness.set( + "tmall", + readinessByPlatform.get("tmall") ?? createPreparedSession("tmall", fallbackTimestamp) + ); + this.readiness.set("jd", readinessByPlatform.get("jd") ?? createMissingSession("jd")); + + for (const [taskId, attempts] of state.strategyAttempts ?? []) { + this.strategyAttempts.set(taskId, attempts); + } + + for (const [taskId, metrics] of state.platformRunMetrics ?? []) { + this.platformRunMetrics.set(taskId, metrics); + } + + this.reportMetrics.push(...(state.reportMetrics ?? [])); + this.retentionMetrics.push(...(state.retentionMetrics ?? [])); + this.auditLogs.push(...(state.auditLogs ?? [])); + + for (const [taskId, scenarios] of state.executionScenarios ?? []) { + this.executionScenarios.set(taskId, scenarios); + } + + for (const [taskId, artifacts] of state.executionArtifacts ?? []) { + this.executionArtifacts.set(taskId, artifacts); + } + } + + private loadPersistedState(): PersistedTaskStoreState | null { + if (!this.storagePath) { + return null; + } + + try { + const raw = readFileSync(this.storagePath, "utf8"); + return JSON.parse(raw) as PersistedTaskStoreState; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return null; + } + + throw error; + } + } + + private persistState(): void { + if (!this.storagePath) { + return; + } + + mkdirSync(dirname(this.storagePath), { recursive: true }); + const snapshot: PersistedTaskStoreState = { + tasks: Array.from(this.tasks.values()), + reports: Array.from(this.reports.entries()), + reportFingerprints: Array.from(this.reportFingerprints.entries()), + readiness: Array.from(this.readiness.values()), + strategyAttempts: Array.from(this.strategyAttempts.entries()), + platformRunMetrics: Array.from(this.platformRunMetrics.entries()), + reportMetrics: this.reportMetrics, + retentionMetrics: this.retentionMetrics, + auditLogs: this.auditLogs, + executionScenarios: Array.from(this.executionScenarios.entries()), + executionArtifacts: Array.from(this.executionArtifacts.entries()) + }; + + writeFileSync(this.storagePath, JSON.stringify(snapshot, null, 2), "utf8"); + } + + private hasConfiguredJdLiveSession(): boolean { + return this.jdLiveService?.getSessionSummary().configured ?? false; + } + + private hasConfiguredTmallLiveSession(): boolean { + return this.tmallLiveService?.getSessionSummary().configured ?? false; + } + + private getSelectedCandidates( + task: TaskRecord, + platform?: PlatformId + ): CandidateRecord[] { + const selectedCandidates = task.platformRuns.flatMap((run) => { + if (platform && run.platform !== platform) { + return []; + } + + const selectedIds = new Set(run.selectedCandidateIds); + return task.platformCandidates[run.platform].filter((candidate) => + selectedIds.has(candidate.candidateId) + ); + }); + + return selectedCandidates; + } + + private getExecutionArtifacts( + taskId: string, + platform?: PlatformId + ): TaskExecutionArtifact[] { + const artifacts = this.executionArtifacts.get(taskId) ?? []; + return platform ? artifacts.filter((artifact) => artifact.platform === platform) : artifacts; + } + + private setExecutionArtifacts( + taskId: string, + platform: PlatformId, + nextArtifacts: TaskExecutionArtifact[] + ): void { + const currentArtifacts = this.executionArtifacts.get(taskId) ?? []; + const retainedArtifacts = currentArtifacts.filter((artifact) => artifact.platform !== platform); + const mergedArtifacts = [...retainedArtifacts, ...nextArtifacts]; + + if (mergedArtifacts.length > 0) { + this.executionArtifacts.set(taskId, mergedArtifacts); + return; + } + + this.executionArtifacts.delete(taskId); + } + + private clearExecutionArtifacts(taskId: string, platform?: PlatformId): number { + const currentArtifacts = this.executionArtifacts.get(taskId) ?? []; + if (!platform) { + this.executionArtifacts.delete(taskId); + return currentArtifacts.length; + } + + const retainedArtifacts = currentArtifacts.filter((artifact) => artifact.platform !== platform); + const deletedCount = currentArtifacts.length - retainedArtifacts.length; + + if (retainedArtifacts.length > 0) { + this.executionArtifacts.set(taskId, retainedArtifacts); + } else { + this.executionArtifacts.delete(taskId); + } + + return deletedCount; + } + + private extractJdSkuId(candidate: CandidateRecord): string | null { + const productUrlMatch = candidate.productUrl.match(/item\.jd\.com\/(\d+)\.html/i); + if (productUrlMatch?.[1]) { + return productUrlMatch[1]; + } + + const candidateIdMatch = candidate.candidateId.match(/^jd-(\d+)$/i); + return candidateIdMatch?.[1] ?? null; + } + + private extractTmallItemId(candidate: CandidateRecord): string | null { + const productUrlMatch = candidate.productUrl.match( + /[?&](?:id|itemId|itemNumId|auctionNumId)=(\d+)/i + ); + if (productUrlMatch?.[1]) { + return productUrlMatch[1]; + } + + try { + const url = new URL(candidate.productUrl); + const direct = + url.searchParams.get("id") ?? + url.searchParams.get("itemId") ?? + url.searchParams.get("itemNumId") ?? + url.searchParams.get("auctionNumId"); + if (direct) { + return direct; + } + } catch { + // Ignore malformed candidate URLs and continue trying other fallbacks. + } + + const candidateIdMatch = candidate.candidateId.match(/^tmall-(\d+)$/i); + return candidateIdMatch?.[1] ?? null; + } + + private buildSelectedCandidateReviewBudgets(task: TaskRecord): Map { + return buildReviewBudgetPlan( + this.getSelectedCandidates(task).map((candidate) => candidate.candidateId), + task.perLinkLimit, + task.taskTotalLimit + ); + } + + private createJdReviewsPreviewOptions(requestedReviewCount: number): JdReviewsPreviewOptions { + const commentCount = Math.min(50, requestedReviewCount); + + return { + commentCount, + page: 1, + maxPages: Math.max(1, Math.min(10, Math.ceil(requestedReviewCount / commentCount))) + }; + } + + private createTmallReviewsPreviewOptions(task: TaskRecord): TmallReviewsPreviewOptions { + const requestedReviewCount = Math.max(1, Math.min(task.perLinkLimit, task.taskTotalLimit)); + const commentCount = Math.min(50, requestedReviewCount); + + return { + commentCount, + page: 1, + maxPages: Math.max(1, Math.min(10, Math.ceil(requestedReviewCount / commentCount))) + }; + } + + private isJdPagingTemplateError(error: unknown): boolean { + return getJdLiveErrorCode(error) === "TEMPLATE_PAGE_FIELD_MISSING"; + } + + private isTmallPagingTemplateError(error: unknown): boolean { + return ( + error instanceof Error && + error.message.includes("Imported Tmall reviews template does not expose a page field") + ); + } + + private getJdPublicSessionReason(fallback: string): string { + const note = this.jdSessionManager?.getState().publicNote?.trim(); + return note && note.length > 0 ? note : fallback; + } + + private getTmallPublicSessionReason(fallback: string): string { + const note = this.tmallSessionManager?.getState().publicNote?.trim(); + return note && note.length > 0 ? note : fallback; + } + + private getPlatformSessionReason(platform: PlatformId, fallback: string): string { + if (platform === "jd") { + return this.getJdPublicSessionReason(fallback); + } + + if (platform === "tmall") { + return this.getTmallPublicSessionReason(fallback); + } + + return fallback; + } + + private async attemptJdAutoRecovery( + error: unknown, + capability: "search" | "detail" | "reviews", + taskId: string, + trigger: StrategyAttemptTrigger + ): Promise { + if (!this.jdSessionManager) { + return false; + } + + return this.jdSessionManager.handleLiveFailure(error, { + capability, + taskId, + trigger + }); + } + + private async notifyTmallSessionManager( + error: unknown, + capability: "search" | "detail" | "reviews", + taskId: string, + trigger: StrategyAttemptTrigger + ): Promise { + if (!this.tmallSessionManager) { + return; + } + + await this.tmallSessionManager.handleLiveFailure(error, { + capability, + taskId, + trigger + }); + } + + private classifyJdExecutionFailure( + capability: "search" | "detail" | "reviews", + error: unknown + ): LiveExecutionFailure { + const message = error instanceof Error ? error.message : "unknown error"; + const errorCode = getJdLiveErrorCode(error); + const capabilityLabelMap = { + search: "搜索", + detail: "详情", + reviews: "评论" + } as const; + const capabilityLabel = capabilityLabelMap[capability]; + + if ( + errorCode === "MISSING_SESSION" || + errorCode === "SESSION_REQUIRED" || + errorCode === "INVALID_COOKIE" || + ((isJdLiveError(error) && error.statusCode === 409) && + errorCode !== "TEMPLATE_MISSING" && + errorCode !== "TEMPLATE_EXPIRED" && + errorCode !== "TEMPLATE_PAGE_FIELD_MISSING" && + errorCode !== "TEMPLATE_QUERY_LOCKED") + ) { + return { + status: capability === "search" ? "SearchBlocked" : "Blocked", + errorType: "session_required", + reason: this.getJdPublicSessionReason( + "京东运维会话当前不可用，系统会在后台继续恢复。" + ), + detail: `京东${capabilityLabel}抓取因会话问题被阻塞：${message}` + }; + } + + if ( + errorCode === "TEMPLATE_MISSING" || + errorCode === "TEMPLATE_EXPIRED" || + errorCode === "TEMPLATE_PAGE_FIELD_MISSING" || + errorCode === "TEMPLATE_QUERY_LOCKED" || + errorCode === "INVALID_TEMPLATE" + ) { + return { + status: capability === "search" ? "SearchBlocked" : "Blocked", + errorType: + errorCode === "TEMPLATE_MISSING" || errorCode === "INVALID_TEMPLATE" + ? "template_required" + : "template_expired", + reason: this.getJdPublicSessionReason( + errorCode === "TEMPLATE_MISSING" || errorCode === "INVALID_TEMPLATE" + ? "京东运维模板当前不可用，后台正在刷新。" + : "京东运维模板已失效，后台正在刷新并重试。" + ), + detail: `京东${capabilityLabel}抓取因模板问题被阻塞：${message}` + }; + } + + if (errorCode === "RISK_BLOCKED") { + return { + status: capability === "search" ? "SearchBlocked" : "Blocked", + errorType: "risk_intercepted", + reason: this.getJdPublicSessionReason( + "京东触发了验证或风控拦截，已转入运维后台处理。" + ), + detail: `京东${capabilityLabel}抓取因验证或风控被阻塞：${message}` + }; + } + + return { + status: "Failed", + errorType: + isJdLiveError(error) && typeof error.statusCode === "number" + ? `jd_live_${error.statusCode}` + : "jd_live_failed", + reason: `京东${capabilityLabel}抓取失败，请稍后重试。`, + detail: `京东${capabilityLabel}抓取失败：${message}` + }; + } + + private classifyTmallExecutionFailure( + capability: "search" | "detail" | "reviews", + error: unknown + ): LiveExecutionFailure { + const message = error instanceof Error ? error.message : "unknown error"; + const normalizedMessage = message.toLowerCase(); + const capabilityLabelMap = { + search: "搜索", + detail: "详情", + reviews: "评论" + } as const; + const capabilityLabel = capabilityLabelMap[capability]; + + if ( + (isTmallLiveError(error) && error.statusCode === 409) || + normalizedMessage.includes("session appears invalid") || + normalizedMessage.includes("live session is not configured") || + normalizedMessage.includes("cookie/header") + ) { + return { + status: capability === "search" ? "SearchBlocked" : "Blocked", + errorType: "session_required", + reason: this.getTmallPublicSessionReason( + "天猫会话由运维后台维护，当前需要重新登录并更新 Cookie。" + ), + detail: `天猫${capabilityLabel}抓取因会话问题被阻塞：${message}` + }; + } + + if ( + normalizedMessage.includes("template is missing") || + normalizedMessage.includes("capture a fresh") || + normalizedMessage.includes("does not expose a page field") + ) { + return { + status: capability === "search" ? "SearchBlocked" : "Blocked", + errorType: "template_required", + reason: this.getTmallPublicSessionReason( + "天猫会话缺少有效模板，运维后台需要刷新模板。" + ), + detail: `天猫${capabilityLabel}抓取因模板问题被阻塞：${message}` + }; + } + + return { + status: "Failed", + errorType: + isTmallLiveError(error) && typeof error.statusCode === "number" + ? `tmall_live_${error.statusCode}` + : "tmall_live_failed", + reason: `天猫${capabilityLabel}抓取失败，请稍后重试。`, + detail: `天猫${capabilityLabel}抓取失败：${message}` + }; + } + + private async previewJdProduct( + task: TaskRecord, + skuId: string, + requestedReviewCount: number, + trigger: StrategyAttemptTrigger = "system" + ): Promise { + if (!this.jdLiveService) { + throw new Error("JD live service is unavailable."); + } + + if (requestedReviewCount <= 0) { + const detailPreview = await this.jdLiveService.previewDetail(skuId); + return { + skuId: detailPreview.skuId, + source: "api", + detail: detailPreview.detail, + pagination: { + requestedPage: 1, + requestedCommentCount: 0, + maxPages: 0, + pagesFetched: 0 + }, + reviews: createEmptyJdReviewsSnapshot(detailPreview.skuId) + }; + } + + const options = this.createJdReviewsPreviewOptions(requestedReviewCount); + + try { + return await this.jdLiveService.previewProduct(skuId, options); + } catch (error) { + const recovered = await this.attemptJdAutoRecovery(error, "detail", task.taskId, trigger); + if (recovered) { + return this.jdLiveService.previewProduct(skuId, options); + } + + if (!this.isJdPagingTemplateError(error) || options.maxPages === 1) { + throw error; + } + + try { + return await this.jdLiveService.previewProduct(skuId, { + ...options, + maxPages: 1 + }); + } catch (fallbackError) { + const fallbackRecovered = await this.attemptJdAutoRecovery( + fallbackError, + "reviews", + task.taskId, + trigger + ); + if (fallbackRecovered) { + return this.jdLiveService.previewProduct(skuId, { + ...options, + maxPages: 1 + }); + } + + throw fallbackError; + } + } + } + + private async previewTmallProduct( + task: TaskRecord, + itemId: string + ): Promise { + if (!this.tmallLiveService) { + throw new Error("Tmall live service is unavailable."); + } + + const options = this.createTmallReviewsPreviewOptions(task); + + try { + return await this.tmallLiveService.previewProduct(itemId, options); + } catch (error) { + if (!this.isTmallPagingTemplateError(error) || options.maxPages === 1) { + throw error; + } + + return this.tmallLiveService.previewProduct(itemId, { + ...options, + maxPages: 1 + }); + } + } + + private parsePriceValue(price: string | null | undefined): number | null { + if (!price) { + return null; + } + + const normalized = price.replace(/[^\d.]/g, ""); + if (!normalized) { + return null; + } + + const parsed = Number.parseFloat(normalized); + return Number.isNaN(parsed) ? null : parsed; + } + + private buildCandidateReviewSamples( + task: TaskRecord, + selectedCandidates: CandidateRecord[], + executionArtifactsByCandidateId: Map + ): { + byCandidateId: Map; + liveTargetCount: number; + liveActualCount: number; + } { + const reviewBudgetsByCandidateId = this.buildSelectedCandidateReviewBudgets(task); + const byCandidateId = new Map(); + let liveTargetCount = 0; + let liveActualCount = 0; + + for (const candidate of selectedCandidates) { + const targetCount = reviewBudgetsByCandidateId.get(candidate.candidateId) ?? 0; + const artifact = executionArtifactsByCandidateId.get(candidate.candidateId); + const executionComments = artifact?.reviews.comments ?? []; + const sampled = sampleReviewComments( + executionComments.map((comment) => toReviewSamplingComment(comment)), + targetCount, + NEGATIVE_REVIEW_KEYWORDS + ); + const commentsById = new Map(executionComments.map((comment) => [comment.id, comment] as const)); + const sampledComments = sampled.comments + .map(({ bucket, comment }) => { + const executionComment = commentsById.get(comment.id); + if (!executionComment) { + return null; + } + + return { + bucket, + comment: executionComment + }; + }) + .filter( + ( + comment + ): comment is { + bucket: ReviewSamplingBucket; + comment: ExecutionReviewComment; + } => Boolean(comment) + ); + + if (artifact) { + liveTargetCount += sampled.targetCount; + liveActualCount += sampled.actualCount; + } + + byCandidateId.set(candidate.candidateId, { + candidateId: candidate.candidateId, + targetCount: sampled.targetCount, + actualCount: sampled.actualCount, + sampleInsufficient: sampled.sampleInsufficient, + bucketCounts: sampled.bucketCounts, + comments: sampledComments + }); + } + + return { + byCandidateId, + liveTargetCount, + liveActualCount + }; + } + + private buildInsightCard( + title: string, + statement: string, + evidenceIds: string[], + platforms: PlatformId[], + linkCount: number, + reviewCount: number + ) { + return { + card_id: randomUUID(), + title, + statement, + confidence: evidenceIds.length > 0 ? ("high" as const) : ("medium" as const), + sample_flag: + evidenceIds.length > 0 ? ("sufficient" as const) : ("insufficient" as const), + source_scope: { + platforms: platforms.length > 0 ? platforms : (["tmall"] as const), + link_count: linkCount, + review_count: reviewCount + }, + evidence_ids: evidenceIds + }; + } + + private async runSearch(task: TaskRecord): Promise { task.taskStage = "search"; for (const run of task.platformRuns) { - this.runSearchForPlatform(task, run); + await this.runSearchForPlatform(task, run); } task.taskStage = "confirmation"; @@ -690,7 +1665,7 @@ export class InMemoryTaskStore { } private buildReport(task: TaskRecord): ReportSnapshot { - const reportVersion = (this.reports.get(task.taskId)?.length ?? 0) + 1; + const reportVersion = this.getNextReportVersion(task); const completedRuns = task.platformRuns.filter((run) => run.status === "Completed"); const blockedPlatforms = task.platformRuns .filter((run) => run.status === "Blocked" || run.status === "SearchBlocked") @@ -698,42 +1673,113 @@ export class InMemoryTaskStore { const failedPlatforms = task.platformRuns .filter((run) => run.status === "Failed") .map((run) => run.platform); - const selectedCandidates = task.platformRuns.flatMap((run) => - task.platformCandidates[run.platform].filter((candidate) => - run.selectedCandidateIds.includes(candidate.candidateId) + const selectedCandidates = this.getSelectedCandidates(task); + const executionArtifacts = this.getExecutionArtifacts(task.taskId); + const executionArtifactsByCandidateId = new Map( + executionArtifacts.map((artifact) => [artifact.candidateId, artifact] as const) + ); + const livePlatforms = Array.from( + new Set(executionArtifacts.map((artifact) => artifact.platform)) + ); + const livePlatformLabels = livePlatforms.map((platform) => platformCatalogMap[platform].label); + const livePlatformLabelText = + livePlatformLabels.length > 0 ? livePlatformLabels.join("、") : "实时"; + const candidateReviewSamples = this.buildCandidateReviewSamples( + task, + selectedCandidates, + executionArtifactsByCandidateId + ); + const reviewSampleCount = + candidateReviewSamples.liveActualCount > 0 + ? candidateReviewSamples.liveActualCount + : Math.min(selectedCandidates.length * task.perLinkLimit, task.taskTotalLimit); + const sampleInsufficient = + candidateReviewSamples.liveTargetCount > 0 + ? candidateReviewSamples.liveActualCount < candidateReviewSamples.liveTargetCount + : reviewSampleCount < 30; + const insightPlatforms = Array.from( + new Set( + (completedRuns.length > 0 + ? completedRuns.map((run) => run.platform) + : selectedCandidates.map((candidate) => candidate.platform)) as PlatformId[] ) ); - const reviewSampleCount = Math.min( - selectedCandidates.length * task.perLinkLimit, - task.taskTotalLimit - ); - const evidenceIndex = selectedCandidates.map((candidate, index) => ({ - evidence_id: `evidence-${task.taskId}-${index + 1}`, - platform: candidate.platform, - source_type: "product" as const, - source_url: candidate.productUrl, - review_ref: null, - snippet: `${candidate.title} 在 ${candidate.storeName} 下展示 ${candidate.specLabel}，当前价格 ${candidate.priceLabel}。`, - captured_at: nowIso() - })); + const sourcePlatforms: PlatformId[] = + insightPlatforms.length > 0 ? insightPlatforms : ["tmall"]; + const evidenceIndex: ReportSnapshot["evidence_index"] = []; + const evidenceIdsByPlatform = new Map(); + let evidenceCounter = 0; - const sharedInsight = (title: string, statement: string, evidenceIds: string[]) => ({ - card_id: randomUUID(), - title, - statement, - confidence: evidenceIds.length > 0 ? ("high" as const) : ("medium" as const), - sample_flag: - evidenceIds.length > 0 ? ("sufficient" as const) : ("insufficient" as const), - source_scope: { - platforms: - completedRuns.length > 0 - ? completedRuns.map((run) => run.platform) - : (["tmall"] as const), - link_count: selectedCandidates.length, - review_count: reviewSampleCount - }, - evidence_ids: evidenceIds - }); + const truncate = (value: string, maxLength: number) => + value.length > maxLength ? `${value.slice(0, maxLength - 1)}…` : value; + + const addEvidence = ( + platform: PlatformId, + sourceType: "product" | "review", + sourceUrl: string, + snippet: string, + reviewRef: string | null + ) => { + const evidenceId = `evidence-${task.taskId}-${++evidenceCounter}`; + evidenceIndex.push({ + evidence_id: evidenceId, + platform, + source_type: sourceType, + source_url: sourceUrl, + review_ref: reviewRef, + snippet, + captured_at: nowIso() + }); + const platformEvidenceIds = evidenceIdsByPlatform.get(platform) ?? []; + platformEvidenceIds.push(evidenceId); + evidenceIdsByPlatform.set(platform, platformEvidenceIds); + return evidenceId; + }; + + for (const candidate of selectedCandidates) { + const artifact = executionArtifactsByCandidateId.get(candidate.candidateId); + const detail = artifact?.detail; + const sampledComments = + candidateReviewSamples.byCandidateId.get(candidate.candidateId)?.comments ?? []; + const priceLabel = + detail?.price ?? + getArtifactEstimatedPrice(artifact) ?? + detail?.originalPrice ?? + candidate.priceLabel; + const headline = detail?.title ?? candidate.title; + const storeName = detail?.shopName ?? candidate.storeName; + const detailSummary = [ + `${headline}`, + `店铺 ${storeName}`, + `价格 ${priceLabel}`, + getArtifactStockState(artifact) + ? `库存 ${getArtifactStockState(artifact)}` + : `规格 ${candidate.specLabel}` + ] + .filter(Boolean) + .join(" | "); + + addEvidence(candidate.platform, "product", candidate.productUrl, detailSummary, null); + + for (const { bucket, comment } of sampledComments.slice(0, 2)) { + const commentSummary = [ + `样本 ${bucket}`, + getExecutionCommentScore(comment) ? `评分 ${getExecutionCommentScore(comment)}` : null, + getExecutionCommentUserLabel(comment) ? `${getExecutionCommentUserLabel(comment)}` : null, + truncate(comment.content, 72) + ] + .filter(Boolean) + .join(" | "); + + addEvidence( + candidate.platform, + "review", + candidate.productUrl, + commentSummary, + comment.id + ); + } + } return parseReportSnapshot({ report_id: `report-${task.taskId}-${reportVersion}`, @@ -743,19 +1789,28 @@ export class InMemoryTaskStore { task_status: task.taskStatus, summary: { headline: - completedRuns.length > 1 - ? "双平台已形成可回看报告。" - : "已基于当前可用平台生成首版结构化报告。", + executionArtifacts.length > 0 + ? `${livePlatformLabelText}真实详情与评论链路已形成可回看报告。` + : completedRuns.length > 1 + ? "双平台已形成可回看报告。" + : "已基于当前可用平台生成首版结构化报告。", key_points: [ - `已确认 ${selectedCandidates.length} 个商品链接，纳入 ${reviewSampleCount} 条评论样本。`, + executionArtifacts.length > 0 + ? `已抓取 ${executionArtifacts.length} 个实时商品详情，纳入 ${reviewSampleCount} 条真实评论样本。` + : `已确认 ${selectedCandidates.length} 个商品链接，纳入 ${reviewSampleCount} 条评论样本。`, blockedPlatforms.length + failedPlatforms.length > 0 ? `仍有 ${blockedPlatforms.length + failedPlatforms.length} 个平台未完成，其中 ${blockedPlatforms.length} 个阻塞、${failedPlatforms.length} 个失败。` - : "当前已确认平台全部处理完成。" + : "当前已确认平台全部处理完成。", + executionArtifacts.length > 0 + ? `结果页证据已切换为${livePlatformLabelText}实时详情与评论，不再只依赖样板候选摘要。` + : "当前报告仍以候选摘要和规则化洞察为主。" ], limitations: [ blockedPlatforms.length + failedPlatforms.length > 0 ? "未完成平台不进入当前可发布洞察范围。" - : "当前为第一批开发样板数据，后续将替换为真实采集链路。" + : executionArtifacts.length > 0 + ? `当前仅${livePlatformLabelText}已接入真实详情与评论抓取，其他平台仍沿用样板链路。` + : "当前为第一批开发样板数据，后续将替换为真实采集链路。" ] }, product_snapshot: { @@ -770,13 +1825,42 @@ export class InMemoryTaskStore { } }, platform_insights: task.platformRuns.map((run) => { - const platformEvidenceIds = evidenceIndex - .filter((evidence) => evidence.platform === run.platform) - .map((evidence) => evidence.evidence_id); + const platformEvidenceIds = evidenceIdsByPlatform.get(run.platform) ?? []; const selectedForPlatform = selectedCandidates.filter( (candidate) => candidate.platform === run.platform ); - const prices = selectedForPlatform.map((candidate) => candidate.price); + const artifactsForPlatform = executionArtifacts.filter( + (artifact) => artifact.platform === run.platform + ); + const prices = selectedForPlatform + .map((candidate) => { + const artifact = executionArtifactsByCandidateId.get(candidate.candidateId); + return this.parsePriceValue( + artifact?.detail.price ?? getArtifactEstimatedPrice(artifact) ?? null + ) ?? candidate.price; + }) + .filter((price) => Number.isFinite(price)); + const platformSampledComments = selectedForPlatform.flatMap( + (candidate) => candidateReviewSamples.byCandidateId.get(candidate.candidateId)?.comments ?? [] + ); + const platformTargetReviewCount = selectedForPlatform.reduce( + (sum, candidate) => + sum + (candidateReviewSamples.byCandidateId.get(candidate.candidateId)?.targetCount ?? 0), + 0 + ); + const platformReviewCount = platformSampledComments.length; + const topTags = Array.from( + new Set( + artifactsForPlatform.flatMap((artifact) => + artifact.reviews.tags.map((tag) => tag.name.trim()).filter(Boolean) + ) + ) + ).slice(0, 3); + const negativeComments = platformSampledComments + .filter((sampledComment) => sampledComment.bucket === "negative") + .map((sampledComment) => sampledComment.comment); + const representativeSampledComment = platformSampledComments[0]?.comment; + const primaryArtifact = artifactsForPlatform[0]; return { platform: run.platform, execution_status: mapPlatformStatusToExecutionStatus(run.status), @@ -789,70 +1873,152 @@ export class InMemoryTaskStore { } : null, selling_points: - selectedForPlatform.length > 0 + primaryArtifact ? [ - sharedInsight( + this.buildInsightCard( + `${platformCatalogMap[run.platform].label} 实时商品卡片已落地`, + [ + `${primaryArtifact.detail.title ?? selectedForPlatform[0]?.title ?? "商品"}`, + `店铺 ${primaryArtifact.detail.shopName ?? selectedForPlatform[0]?.storeName ?? "未知"}`, + primaryArtifact.detail.price + ? `当前价格 ${primaryArtifact.detail.price}` + : null, + getArtifactStockState(primaryArtifact) + ? `库存状态 ${getArtifactStockState(primaryArtifact)}` + : null, + topTags.length > 0 ? `高频标签 ${topTags.join("、")}` : null + ] + .filter(Boolean) + .join("；"), + platformEvidenceIds, + sourcePlatforms, + selectedForPlatform.length, + platformReviewCount + ) + ] + : selectedForPlatform.length > 0 + ? [ + this.buildInsightCard( `${platformCatalogMap[run.platform].label} 卖点稳定`, `${platformCatalogMap[run.platform].label} 当前候选集中强调 ${selectedForPlatform[0]?.highlights[0] ?? "基础卖点"}。`, - platformEvidenceIds + platformEvidenceIds, + sourcePlatforms, + selectedForPlatform.length, + platformReviewCount ) ] : [], positive_themes: - selectedForPlatform.length > 0 + primaryArtifact ? [ - sharedInsight( + this.buildInsightCard( + `${platformCatalogMap[run.platform].label} 评论反馈已采样`, + [ + `已采样 ${platformReviewCount} / ${platformTargetReviewCount} 条评论`, + getArtifactGoodRate(primaryArtifact) + ? `好评率 ${getArtifactGoodRate(primaryArtifact)}` + : null, + topTags.length > 0 ? `高频提到 ${topTags.join("、")}` : null, + representativeSampledComment?.content + ? `代表性评论：${truncate(representativeSampledComment.content, 48)}` + : null + ] + .filter(Boolean) + .join("；"), + platformEvidenceIds, + sourcePlatforms, + selectedForPlatform.length, + platformReviewCount + ) + ] + : selectedForPlatform.length > 0 + ? [ + this.buildInsightCard( `${platformCatalogMap[run.platform].label} 正向反馈集中`, "模拟样本显示用户更关注核心功能与履约确定性。", - platformEvidenceIds + platformEvidenceIds, + sourcePlatforms, + selectedForPlatform.length, + platformReviewCount ) ] : [], negative_themes: run.status === "SearchBlocked" || run.status === "Blocked" || run.status === "Failed" ? [ - sharedInsight( + this.buildInsightCard( `${platformCatalogMap[run.platform].label} 当前未完成执行`, run.reason ?? "当前平台需要先恢复会话或稍后重试。", - [] + [], + sourcePlatforms, + selectedForPlatform.length, + platformReviewCount ) ] + : negativeComments.length > 0 + ? [ + this.buildInsightCard( + `${platformCatalogMap[run.platform].label} 已识别负向反馈`, + `采样评论中出现了“${truncate(negativeComments[0]!.content, 48)}”等负向表达，建议在后续版本加入更细的评论归因。`, + platformEvidenceIds, + sourcePlatforms, + selectedForPlatform.length, + platformReviewCount + ) + ] : [], store_diff_notes: selectedForPlatform.length > 1 ? [ - sharedInsight( + this.buildInsightCard( `${platformCatalogMap[run.platform].label} 候选存在规格差异`, "同平台已确认多个链接，报告保留差异而不强行合并。", - platformEvidenceIds + platformEvidenceIds, + sourcePlatforms, + selectedForPlatform.length, + platformReviewCount ) ] : [] }; }), cross_platform_insights: [ - sharedInsight( - "当前报告以平台级摘要为主", - completedRuns.length > 1 - ? "系统保留链接级差异，同时默认从平台级组织结论。" - : "当前仅部分平台完成执行，跨平台对比视角仍然保留但覆盖有限。", - evidenceIndex.map((evidence) => evidence.evidence_id) + this.buildInsightCard( + executionArtifacts.length > 0 + ? `${livePlatformLabelText}实时证据已进入报告主链` + : "当前报告以平台级摘要为主", + executionArtifacts.length > 0 + ? `${livePlatformLabelText}已切换到真实详情与评论回放，报告证据直接引用实时抓取结果；其他平台仍保持平台级摘要。` + : completedRuns.length > 1 + ? "系统保留链接级差异，同时默认从平台级组织结论。" + : "当前仅部分平台完成执行，跨平台对比视角仍然保留但覆盖有限。", + evidenceIndex.map((evidence) => evidence.evidence_id), + sourcePlatforms, + selectedCandidates.length, + reviewSampleCount ) ], recommendations: [ - sharedInsight( - "继续补齐受阻平台会话", + this.buildInsightCard( + blockedPlatforms.length + failedPlatforms.length > 0 + ? "继续补齐受阻平台会话" + : "继续推进模板刷新与真实 AI 总结", blockedPlatforms.length + failedPlatforms.length > 0 ? `建议优先处理 ${[...blockedPlatforms, ...failedPlatforms] .map((platform) => platformCatalogMap[platform].label) .join("、")} 的恢复或重试，再发起下一轮任务。` - : "建议基于当前样板链路继续扩展到真实采集策略。", - evidenceIndex.map((evidence) => evidence.evidence_id) + : executionArtifacts.length > 0 + ? `下一步应把模板失效归类、L2 模板刷新和真正的 AI 归纳接入这条已打通的${livePlatformLabelText}实时链路。` + : "建议基于当前样板链路继续扩展到真实采集策略。", + evidenceIndex.map((evidence) => evidence.evidence_id), + sourcePlatforms, + selectedCandidates.length, + reviewSampleCount ) ], evidence_index: evidenceIndex, quality_flags: { - sample_insufficient: reviewSampleCount < 30, + sample_insufficient: sampleInsufficient, partial_platform_failure: blockedPlatforms.length > 0 || failedPlatforms.length > 0, blocked_platforms: blockedPlatforms, @@ -861,15 +2027,19 @@ export class InMemoryTaskStore { }); } - private runSearchForPlatform( + private async runSearchForPlatform( task: TaskRecord, run: PlatformRunRecord, trigger: StrategyAttemptTrigger = "system" - ): void { + ): Promise { const readiness = this.requireReadiness(run.platform); if (readiness.searchRequirement === "required" && !readiness.ready) { + const blockedReason = this.getPlatformSessionReason( + run.platform, + platformCatalogMap[run.platform].recoveryHint + ); run.status = "SearchBlocked"; - run.reason = platformCatalogMap[run.platform].recoveryHint; + run.reason = blockedReason; run.candidateCount = 0; run.lastUpdatedAt = nowIso(); this.recordStrategyAttempt( @@ -886,7 +2056,7 @@ export class InMemoryTaskStore { this.pushEvent( task, `platform.${run.platform}.search_blocked`, - `${platformCatalogMap[run.platform].label} 缺少有效会话，搜索被阻塞。` + blockedReason ); return; } @@ -894,7 +2064,184 @@ export class InMemoryTaskStore { run.status = "Searching"; run.reason = undefined; run.lastUpdatedAt = nowIso(); - const candidates = createMockCandidates(task.query, run.platform); + let candidates: CandidateRecord[] = []; + + if (run.platform === "jd" && this.hasConfiguredJdLiveSession()) { + const startedAt = Date.now(); + let searchError: unknown = null; + + try { + candidates = await this.jdLiveService!.previewSearch(task.query).then( + (preview) => preview.candidates + ); + } catch (error) { + searchError = error; + const recovered = await this.attemptJdAutoRecovery( + error, + "search", + task.taskId, + trigger + ); + if (recovered) { + try { + candidates = await this.jdLiveService!.previewSearch(task.query).then( + (preview) => preview.candidates + ); + } catch (retryError) { + searchError = retryError; + } + } + } + + if (searchError) { + const failure = this.classifyJdExecutionFailure("search", searchError); + run.status = failure.status; + run.reason = failure.reason; + run.candidateCount = 0; + run.lastUpdatedAt = nowIso(); + this.recordStrategyAttempt( + task.taskId, + run.platform, + "search", + failure.status === "Failed" ? "failed" : "blocked", + trigger, + failure.detail, + { + errorType: failure.errorType, + durationMs: Date.now() - startedAt + } + ); + this.pushEvent(task, `platform.${run.platform}.search_failed`, failure.reason); + return; + } + + if (candidates.length === 0) { + const failureCandidateMessage = "京东实时搜索已执行，但当前查询没有返回可确认候选。"; + task.platformCandidates[run.platform] = candidates; + run.candidateCount = 0; + run.status = "NoResult"; + run.reason = failureCandidateMessage; + run.lastUpdatedAt = nowIso(); + this.recordStrategyAttempt( + task.taskId, + run.platform, + "search", + "no_result", + trigger, + "京东实时搜索未返回候选结果。", + { + errorType: "no_candidates", + durationMs: Date.now() - startedAt + } + ); + this.pushEvent(task, `platform.${run.platform}.searched`, "京东实时搜索未返回候选结果。"); + return; + } + + if (candidates.length > 0) { + task.platformCandidates[run.platform] = candidates; + run.candidateCount = candidates.length; + run.status = "AwaitingSelection"; + run.reason = undefined; + run.lastUpdatedAt = nowIso(); + this.recordStrategyAttempt( + task.taskId, + run.platform, + "search", + "succeeded", + trigger, + "京东实时搜索已返回候选列表。", + { + durationMs: Date.now() - startedAt + } + ); + this.pushEvent( + task, + `platform.${run.platform}.searched`, + `京东实时搜索返回 ${candidates.length} 个候选。` + ); + return; + } + } + + if (run.platform === "tmall" && this.hasConfiguredTmallLiveSession()) { + const startedAt = Date.now(); + + try { + candidates = await this.tmallLiveService!.previewSearch(task.query).then( + (preview) => preview.candidates + ); + } catch (error) { + await this.notifyTmallSessionManager(error, "search", task.taskId, trigger); + const failure = this.classifyTmallExecutionFailure("search", error); + run.status = failure.status; + run.reason = failure.reason; + run.candidateCount = 0; + run.lastUpdatedAt = nowIso(); + this.recordStrategyAttempt( + task.taskId, + run.platform, + "search", + failure.status === "Failed" ? "failed" : "blocked", + trigger, + failure.detail, + { + errorType: failure.errorType, + durationMs: Date.now() - startedAt + } + ); + this.pushEvent(task, `platform.${run.platform}.search_failed`, failure.reason); + return; + } + + if (candidates.length === 0) { + const noResultMessage = "天猫实时搜索已执行，但当前查询没有返回可确认候选。"; + task.platformCandidates[run.platform] = candidates; + run.candidateCount = 0; + run.status = "NoResult"; + run.reason = noResultMessage; + run.lastUpdatedAt = nowIso(); + this.recordStrategyAttempt( + task.taskId, + run.platform, + "search", + "no_result", + trigger, + "天猫实时搜索未返回候选结果。", + { + errorType: "no_candidates", + durationMs: Date.now() - startedAt + } + ); + this.pushEvent(task, `platform.${run.platform}.searched`, "天猫实时搜索未返回候选结果。"); + return; + } + + task.platformCandidates[run.platform] = candidates; + run.candidateCount = candidates.length; + run.status = "AwaitingSelection"; + run.reason = undefined; + run.lastUpdatedAt = nowIso(); + this.recordStrategyAttempt( + task.taskId, + run.platform, + "search", + "succeeded", + trigger, + "天猫实时搜索已返回候选列表。", + { + durationMs: Date.now() - startedAt + } + ); + this.pushEvent( + task, + `platform.${run.platform}.searched`, + `天猫实时搜索返回 ${candidates.length} 个候选。` + ); + return; + } + + candidates = createMockCandidates(task.query, run.platform); task.platformCandidates[run.platform] = candidates; run.candidateCount = candidates.length; run.status = candidates.length > 0 ? "AwaitingSelection" : "NoResult"; @@ -927,18 +2274,23 @@ export class InMemoryTaskStore { ); } - private executeSelectedPlatforms( + private async executeSelectedPlatforms( task: TaskRecord, runs: PlatformRunRecord[], trigger: StrategyAttemptTrigger = "system" - ): void { + ): Promise { task.taskStage = "crawl"; + const reviewBudgetsByCandidateId = this.buildSelectedCandidateReviewBudgets(task); for (const run of runs) { const readiness = this.requireReadiness(run.platform); if (readiness.searchRequirement === "required" && !readiness.ready) { + const blockedReason = this.getPlatformSessionReason( + run.platform, + platformCatalogMap[run.platform].recoveryHint + ); run.status = "Blocked"; - run.reason = platformCatalogMap[run.platform].recoveryHint; + run.reason = blockedReason; run.lastUpdatedAt = nowIso(); this.recordStrategyAttempt( task.taskId, @@ -955,11 +2307,12 @@ export class InMemoryTaskStore { this.pushEvent( task, `platform.${run.platform}.blocked`, - `${platformCatalogMap[run.platform].label} 抓取前校验失败，需要先恢复会话。` + blockedReason ); continue; } + const selectedCandidates = this.getSelectedCandidates(task, run.platform); const mockOutcome = this.consumeExecutionScenario(task.taskId, run.platform); if (mockOutcome === "Blocked") { run.status = "Blocked"; @@ -985,6 +2338,235 @@ export class InMemoryTaskStore { continue; } + if (run.platform === "jd" && this.hasConfiguredJdLiveSession()) { + const artifacts: TaskExecutionArtifact[] = []; + let liveFailure: LiveExecutionFailure | null = null; + + run.status = "Running"; + run.reason = undefined; + run.lastUpdatedAt = nowIso(); + this.pushEvent( + task, + `platform.${run.platform}.running`, + "京东实时链路开始抓取商品详情与评论。" + ); + + for (const candidate of selectedCandidates) { + const skuId = this.extractJdSkuId(candidate); + if (!skuId) { + liveFailure = { + status: "Failed", + errorType: "invalid_candidate", + reason: "已选京东候选缺少可解析 SKU，无法发起实时抓取。", + detail: `京东候选 ${candidate.candidateId} 缺少可解析 SKU。` + }; + break; + } + + const startedAt = Date.now(); + const requestedReviewCount = reviewBudgetsByCandidateId.get(candidate.candidateId) ?? 0; + + try { + const preview = await this.previewJdProduct( + task, + skuId, + requestedReviewCount, + trigger + ); + const durationMs = Date.now() - startedAt; + + artifacts.push({ + platform: "jd", + candidateId: candidate.candidateId, + source: "jd-live", + capturedAt: nowIso(), + detail: preview.detail, + pagination: preview.pagination, + reviews: preview.reviews + }); + this.recordStrategyAttempt( + task.taskId, + run.platform, + "detail", + "succeeded", + trigger, + `京东 SKU ${skuId} 商品详情已通过实时请求回放抓取。`, + { + durationMs + } + ); + this.recordStrategyAttempt( + task.taskId, + run.platform, + "reviews", + "succeeded", + trigger, + `京东 SKU ${skuId} 评论已抓取 ${preview.reviews.comments.length} 条，目标 ${requestedReviewCount} 条，分页 ${preview.pagination.pagesFetched}/${preview.pagination.maxPages}。`, + { + durationMs + } + ); + } catch (error) { + const failedCapability = + error instanceof Error && error.message.toLowerCase().includes("review") + ? "reviews" + : "detail"; + liveFailure = this.classifyJdExecutionFailure(failedCapability, error); + + if (liveFailure) { + this.recordStrategyAttempt( + task.taskId, + run.platform, + failedCapability, + liveFailure.status === "Failed" ? "failed" : "blocked", + trigger, + liveFailure.detail, + { + errorType: liveFailure.errorType, + durationMs: Date.now() - startedAt + } + ); + } + break; + } + } + + this.setExecutionArtifacts(task.taskId, run.platform, artifacts); + + if (liveFailure) { + run.status = liveFailure.status === "SearchBlocked" ? "Blocked" : liveFailure.status; + run.reason = liveFailure.reason; + run.lastUpdatedAt = nowIso(); + this.pushEvent( + task, + `platform.${run.platform}.${run.status === "Blocked" ? "blocked" : "failed"}`, + liveFailure.reason + ); + continue; + } + + run.status = "Completed"; + run.reason = undefined; + run.lastUpdatedAt = nowIso(); + this.pushEvent( + task, + `platform.${run.platform}.completed`, + `京东已完成 ${artifacts.length} 个已选商品的实时抓取。` + ); + continue; + } + + if (run.platform === "tmall" && this.hasConfiguredTmallLiveSession()) { + const artifacts: TaskExecutionArtifact[] = []; + let liveFailure: LiveExecutionFailure | null = null; + + run.status = "Running"; + run.reason = undefined; + run.lastUpdatedAt = nowIso(); + this.pushEvent( + task, + `platform.${run.platform}.running`, + "天猫实时链路开始抓取商品详情与评论。" + ); + + for (const candidate of selectedCandidates) { + const itemId = this.extractTmallItemId(candidate); + if (!itemId) { + liveFailure = { + status: "Failed", + errorType: "invalid_candidate", + reason: "已选天猫候选缺少可解析 itemId，无法发起实时抓取。", + detail: `天猫候选 ${candidate.candidateId} 缺少可解析 itemId。` + }; + break; + } + + const startedAt = Date.now(); + + try { + const preview = await this.previewTmallProduct(task, itemId); + const durationMs = Date.now() - startedAt; + + artifacts.push({ + platform: "tmall", + candidateId: candidate.candidateId, + source: "tmall-live", + capturedAt: nowIso(), + detail: preview.detail, + pagination: preview.pagination, + reviews: preview.reviews + }); + this.recordStrategyAttempt( + task.taskId, + run.platform, + "detail", + "succeeded", + trigger, + `天猫 item ${itemId} 商品详情已通过实时请求回放抓取。`, + { + durationMs + } + ); + this.recordStrategyAttempt( + task.taskId, + run.platform, + "reviews", + "succeeded", + trigger, + `天猫 item ${itemId} 评论已抓取 ${preview.reviews.comments.length} 条，分页 ${preview.pagination.pagesFetched}/${preview.pagination.maxPages}。`, + { + durationMs + } + ); + } catch (error) { + const failedCapability = + error instanceof Error && error.message.toLowerCase().includes("review") + ? "reviews" + : "detail"; + await this.notifyTmallSessionManager(error, failedCapability, task.taskId, trigger); + liveFailure = this.classifyTmallExecutionFailure(failedCapability, error); + + this.recordStrategyAttempt( + task.taskId, + run.platform, + failedCapability, + liveFailure.status === "Failed" ? "failed" : "blocked", + trigger, + liveFailure.detail, + { + errorType: liveFailure.errorType, + durationMs: Date.now() - startedAt + } + ); + break; + } + } + + this.setExecutionArtifacts(task.taskId, run.platform, artifacts); + + if (liveFailure) { + run.status = liveFailure.status === "SearchBlocked" ? "Blocked" : liveFailure.status; + run.reason = liveFailure.reason; + run.lastUpdatedAt = nowIso(); + this.pushEvent( + task, + `platform.${run.platform}.${run.status === "Blocked" ? "blocked" : "failed"}`, + liveFailure.reason + ); + continue; + } + + run.status = "Completed"; + run.reason = undefined; + run.lastUpdatedAt = nowIso(); + this.pushEvent( + task, + `platform.${run.platform}.completed`, + `天猫已完成 ${artifacts.length} 个已选商品的实时抓取。` + ); + continue; + } + run.status = "Running"; run.reason = undefined; run.lastUpdatedAt = nowIso(); @@ -993,6 +2575,7 @@ export class InMemoryTaskStore { `platform.${run.platform}.running`, `${platformCatalogMap[run.platform].label} 开始抓取商品与评论。` ); + this.clearExecutionArtifacts(task.taskId, run.platform); this.recordStrategyAttempt( task.taskId, run.platform, @@ -1122,12 +2705,23 @@ export class InMemoryTaskStore { (sum, run) => sum + run.selectedCandidateIds.length, 0 ); + const executionArtifacts = this.getExecutionArtifacts(task.taskId) + .map((artifact) => ({ + platform: artifact.platform, + candidateId: artifact.candidateId, + title: artifact.detail.title, + price: artifact.detail.price ?? getArtifactEstimatedPrice(artifact), + goodRate: getArtifactGoodRate(artifact), + commentIds: artifact.reviews.comments.map((comment) => comment.id) + })) + .sort((left, right) => left.candidateId.localeCompare(right.candidateId)); return JSON.stringify({ taskStatus: task.taskStatus, perLinkLimit: task.perLinkLimit, taskTotalLimit: task.taskTotalLimit, selectedLinkCount, + executionArtifacts, platformRuns: task.platformRuns.map((run) => ({ platform: run.platform, status: run.status, @@ -1137,6 +2731,114 @@ export class InMemoryTaskStore { }); } + private getNextReportVersion(task: TaskRecord): number { + const persistedMaxVersion = Math.max( + 0, + ...(this.reports.get(task.taskId) ?? []).map((report) => report.report_version) + ); + const historicalMaxVersion = task.latestSuccessfulReportVersion ?? 0; + + return Math.max(persistedMaxVersion, historicalMaxVersion) + 1; + } + + private pruneTaskCandidatesToSelected(task: TaskRecord, commit: boolean): number { + let deletedCandidateCount = 0; + const nextCandidates = createPlatformCandidatesRecord(); + + for (const platform of platforms) { + const selectedIds = new Set( + task.platformRuns + .find((run) => run.platform === platform) + ?.selectedCandidateIds ?? [] + ); + const currentCandidates = task.platformCandidates[platform] ?? []; + const keptCandidates = currentCandidates.filter((candidate) => + selectedIds.has(candidate.candidateId) + ); + + deletedCandidateCount += currentCandidates.length - keptCandidates.length; + nextCandidates[platform] = keptCandidates; + } + + if (commit) { + task.platformCandidates = nextCandidates; + } + + return deletedCandidateCount; + } + + private clearTaskCandidates(task: TaskRecord, commit: boolean): number { + const deletedCandidateCount = platforms.reduce( + (sum, platform) => sum + (task.platformCandidates[platform]?.length ?? 0), + 0 + ); + + if (commit) { + task.platformCandidates = createPlatformCandidatesRecord(); + } + + return deletedCandidateCount; + } + + private clearExpiredReports( + task: TaskRecord, + reportCutoffMs: number, + commit: boolean + ): { + deletedReportCount: number; + } { + const reports = this.reports.get(task.taskId) ?? []; + if (reports.length === 0) { + return { + deletedReportCount: 0 + }; + } + + const fingerprints = this.reportFingerprints.get(task.taskId) ?? []; + const keptEntries = reports + .map((report, index) => ({ + report, + fingerprint: fingerprints[index] + })) + .filter(({ report }) => { + const generatedAt = Date.parse(report.generated_at); + return Number.isNaN(generatedAt) || generatedAt >= reportCutoffMs; + }); + + const deletedReportCount = reports.length - keptEntries.length; + if (deletedReportCount === 0) { + return { + deletedReportCount: 0 + }; + } + + if (commit) { + const keptReports = keptEntries.map(({ report }) => report); + const keptFingerprints = keptEntries + .map(({ fingerprint }) => fingerprint) + .filter((fingerprint): fingerprint is string => typeof fingerprint === "string"); + + if (keptReports.length > 0) { + this.reports.set(task.taskId, keptReports); + } else { + this.reports.delete(task.taskId); + } + + if (keptFingerprints.length > 0) { + this.reportFingerprints.set(task.taskId, keptFingerprints); + } else { + this.reportFingerprints.delete(task.taskId); + } + + task.reportVersions = keptReports.map((report) => report.report_version); + task.defaultReportVersion = keptReports[keptReports.length - 1]?.report_version; + } + + return { + deletedReportCount + }; + } + private consumeExecutionScenario( taskId: string, platform: PlatformId @@ -1168,9 +2870,11 @@ export class InMemoryTaskStore { options?: { layer?: StrategyLayer; errorType?: string; + durationMs?: number; } ): StrategyAttemptRecord { - const durationMs = getMockDurationMs(platform, capability, outcome); + const durationMs = + options?.durationMs ?? getMockDurationMs(platform, capability, outcome); const finishedAt = nowIso(); const startedAt = new Date(Date.parse(finishedAt) - durationMs).toISOString(); const attempt: StrategyAttemptRecord = { @@ -1309,13 +3013,21 @@ export class InMemoryTaskStore { private getSessionReason(session: StoredSessionState): string { switch (session.status) { case "ready": - return "当前工作区存在可复用会话，创建任务时会再次校验。"; + return session.platform === "jd" + ? this.getJdPublicSessionReason("当前工作区存在可复用会话，创建任务时会再次校验。") + : "当前工作区存在可复用会话，创建任务时会再次校验。"; case "expired": - return "最近一次会话已过期，请重新完成会话准备。"; + return this.getPlatformSessionReason( + session.platform, + "最近一次会话已过期，请重新完成会话准备。" + ); default: - return session.searchRequirement === "required" - ? platformCatalogMap[session.platform].recoveryHint - : "当前没有可复用会话，建议先预热以提升搜索稳定性。"; + return this.getPlatformSessionReason( + session.platform, + session.searchRequirement === "required" + ? platformCatalogMap[session.platform].recoveryHint + : "当前没有可复用会话，建议先预热以提升搜索稳定性。" + ); } } @@ -1338,6 +3050,7 @@ export class InMemoryTaskStore { cipherLabel: undefined }; this.readiness.set(platform, expired); + this.persistState(); return expired; }