diff --git a/pugongying/src/xhs-pgy-export-core.js b/pugongying/src/xhs-pgy-export-core.js index e1241ef..8422141 100644 --- a/pugongying/src/xhs-pgy-export-core.js +++ b/pugongying/src/xhs-pgy-export-core.js @@ -506,10 +506,37 @@ async function fetchMergedBloggerRecord(id, fetchImpl) { return mergedPayload; } +async function mapWithConcurrency(items, limit, mapper) { + const list = Array.isArray(items) ? items : []; + if (!list.length) { + return []; + } + const size = Math.max(1, Number(limit) || 1); + const workerCount = Math.min(size, list.length); + const results = new Array(list.length); + let nextIndex = 0; + + const worker = async () => { + while (true) { + const index = nextIndex; + nextIndex += 1; + if (index >= list.length) { + return; + } + results[index] = await mapper(list[index], index); + } + }; + + const workers = Array.from({ length: workerCount }, () => worker()); + await Promise.all(workers); + return results; +} + function createExportController(options) { const settings = options || {}; const now = settings.now || (() => new Date()); const fetchImpl = settings.fetchImpl; + const concurrency = Math.max(1, Number(settings.concurrency) || 4); let cachedRecords = []; let cachedFields = []; @@ -520,15 +547,14 @@ function createExportController(options) { throw new Error("请输入至少一个有效的达人主页链接或达人 ID。"); } - const records = []; - for (const id of ids) { + const records = await mapWithConcurrency(ids, concurrency, async (id) => { const raw = await fetchMergedBloggerRecord(id, fetchImpl); - records.push({ + return { id, raw, flattened: flattenRecord(raw), - }); - } + }; + }); cachedRecords = records; cachedFields = buildFieldOptions(records); diff --git a/pugongying/xhs-pgy-export.user.js b/pugongying/xhs-pgy-export.user.js index 8cb5f17..4885b48 100644 --- a/pugongying/xhs-pgy-export.user.js +++ b/pugongying/xhs-pgy-export.user.js @@ -402,11 +402,43 @@ return mergedPayload; } + async function mapWithConcurrency(items, limit, mapper, onDone) { + const list = Array.isArray(items) ? items : []; + if (!list.length) { + return []; + } + const size = Math.max(1, Number(limit) || 1); + const workerCount = Math.min(size, list.length); + const results = new Array(list.length); + let nextIndex = 0; + let doneCount = 0; + + const worker = async () => { + while (true) { + const index = nextIndex; + nextIndex += 1; + if (index >= list.length) { + return; + } + results[index] = await mapper(list[index], index); + doneCount += 1; + if (typeof onDone === "function") { + onDone(doneCount, list.length); + } + } + }; + + const workers = Array.from({ length: workerCount }, () => worker()); + await Promise.all(workers); + return results; + } + function createExportController(options) { const settings = options || {}; const now = settings.now || (() => new Date()); const fetchImpl = settings.fetchImpl || (typeof root.fetch === "function" ? root.fetch.bind(root) : null); + const concurrency = Math.max(1, Number(settings.concurrency) || 4); let cachedRecords = []; let cachedFields = []; @@ -423,17 +455,20 @@ } }; - const records = []; report(0, ids.length); - for (const id of ids) { - const raw = await fetchMergedBloggerRecord(id, fetchImpl); - records.push({ - id, - raw, - flattened: flattenRecord(raw), - }); - report(records.length, ids.length); - } + const records = await mapWithConcurrency( + ids, + concurrency, + async (id) => { + const raw = await fetchMergedBloggerRecord(id, fetchImpl); + return { + id, + raw, + flattened: flattenRecord(raw), + }; + }, + (done, total) => report(done, total), + ); cachedRecords = records; cachedFields = buildFieldOptions(records);