scriptCat/douyinLive/liveRoomHourData/liveRoomHourData.user.js
intelligrow c5fe080aa6 feat: 优化人群画像批量下载功能,支持多种格式导出
对人群画像批量下载功能进行了优化,新增支持导出为多种格式(如JSON、Excel等),提升用户体验。同时,改进了数据请求与处理逻辑,确保下载过程更加高效稳定。
2025-06-20 22:38:49 +08:00

704 lines
24 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// ==UserScript==
// @name 抖音直播间数据监控(全自动)半小时
// @namespace https://bbs.tampermonkey.net.cn/
// @version 2.3
// @description 后台定时获取直播间数据,并在多维表格中显示
// @author 汪喜
// @require https://cdn.staticfile.net/sweetalert2/11.10.3/sweetalert2.all.min.js
// @match https://compass.jinritemai.com/screen/live/shop?live_room_id=*
// @match https://compass.jinritemai.com/screen/live/*
// @match https://compass.jinritemai.com/shop/live-detail?live_room_id=*
// @grant GM_xmlhttpRequest
// @grant GM_setValue
// @grant GM_getValue
// @grant GM_deleteValue
// @crontab */30 * * * *
// @connect open.feishu.cn
// @connect compass.jinritemai.com
// @connect compass.jinritemai.com
// @connect qianchuan.jinritemai.com
// ==/UserScript==
/*
更新日志:
1. 支持多个直播间
*/
const CONFIG = {
rooms: [
{
name: "夸迪官方旗舰店",
aadvid: "1794468035923978",
type: "official",
},/*
{
name: "夸迪官方旗舰店直播间",
aadvid: "1798388578394176",
type: "shop",
},*/
],
id: "cli_a73b734d0564d01c",
secret: "xeKSIOpQiVW6oHeJPH4EvlZMZ1QmnCxx",
appId: "W4jbbkRRbaOdFCsB6D2czBovn7d",
tableId: "tblLYkzpzy8TqZtP",
logTableId: "tbld90KC73YBSu0B",
rawDataTableId: "tblbzx4nG0PbZ6US",
urls: {
tenantAccessToken:
"https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal",
bitableUpdateRecords: (appId, tableId) =>
`https://open.feishu.cn/open-apis/bitable/v1/apps/${appId}/tables/${tableId}/records`,
fetchLiveRoomId:
"https://compass.jinritemai.com/compass_api/shop/live/live_list/realtime?page_no=1&page_size=100",
/*fetchLiveRoomData: (roomId) => {
const indexSelected = encodeURIComponent("pay_ucnt,live_show_watch_cnt_ratio,avg_watch_duration,online_user_cnt,live_show_cnt,real_refund_amt");
return `https://compass.jinritemai.com/compass_api/content_live/shop_official/live_screen/core_data?room_id=${roomId}&index_selected=${indexSelected}`;
},
*/
fetchQianchuanData: (aadvid) =>
`https://qianchuan.jinritemai.com/ad/api/pmc/v1/standard/get_summary_info?aavid=${aadvid}`,
fetchCommentsData: (roomId) =>
`https://compass.jinritemai.com/compass_api/content_live/shop_official/live_screen/five_min_data?room_id=${roomId}`, //使用五分钟数据来加总评论次数
fetchConvertData: (roomId) =>
`https://compass.jinritemai.com/business_api/shop/live_room/flow/gmv_interaction?live_room_id=${roomId}`,
},
};
let logs = [];
let liveRoomId = null;
let intervalId = null;
let timeoutId = null;
//let commentCounts = []; // 用于存储每五分钟获取到的评论次数
function log(message) {
logs.push(message);
console.log(message);
}
const retry = async (fn, retries = 3, delay = 10000) => {
for (let i = 0; i < retries; i++) {
try {
return await fn();
} catch (error) {
if (i < retries - 1) {
console.warn(`Retrying... (${i + 1}/${retries})`);
await new Promise((resolve) => setTimeout(resolve, delay));
} else {
throw error;
}
}
}
};
async function sendHttpRequest(
method,
url,
body = null,
headers = {
"Content-Type": "application/json",
}
) {
return retry(
() =>
new Promise((resolve, reject) => {
GM_xmlhttpRequest({
method: method,
url: url,
data: body,
headers: headers,
onload: function (response) {
if (response.status >= 200 && response.status < 300) {
try {
const jsonResponse = JSON.parse(response.responseText);
if (jsonResponse.msg) {
log(jsonResponse.msg);
}
resolve(jsonResponse);
} catch (e) {
reject(`Failed to parse JSON: ${e}`);
}
} else {
reject(`Error: ${response.status} ${response.statusText}`);
}
},
onerror: function (error) {
reject(`Network Error: ${error}`);
},
});
})
);
}
async function fetchTenantAccessToken(id, secret) {
try {
const response = await sendHttpRequest(
"POST",
CONFIG.urls.tenantAccessToken,
JSON.stringify({
app_id: id,
app_secret: secret,
})
);
return response.tenant_access_token;
} catch (error) {
throw new Error(`Error fetching Tenant Access Token: ${error}`);
}
}
// 更新记录
async function updateBitableRecords(accessToken, appId, tableId, items) {
try {
const response = await sendHttpRequest(
"POST",
CONFIG.urls.bitableUpdateRecords(appId, tableId),
JSON.stringify(items),
{
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json",
}
);
log("Updated record: " + JSON.stringify(response));
return response;
} catch (error) {
throw new Error(`Failed to update record in Bitable: ${error}`);
}
}
//获取正在直播的直播间id
async function fetchLiveRoomId(authorNickName) {
const url = CONFIG.urls.fetchLiveRoomId;
try {
const response = await sendHttpRequest("GET", url);
const liveRooms = response.data.card_list;
const targetRoom = liveRooms.find(
(room) => room.author.author_nick_name === authorNickName
);
if (targetRoom) {
return targetRoom.live_room_id;
} else {
throw new Error("未找到符合条件的直播间");
}
} catch (error) {
log("Error fetching live room list: " + error.message);
return null;
}
}
// 获取直播间大屏数据
async function fetchLiveRoomData(roomId, roomType) {
try {
let url;
if (roomType === "official") {
const indexSelected = encodeURIComponent(
"pay_ucnt,live_show_watch_cnt_ratio,avg_watch_duration,online_user_cnt,live_show_cnt,real_refund_amt"
);
url = `https://compass.jinritemai.com/compass_api/content_live/shop_official/live_screen/core_data?room_id=${roomId}&index_selected=${indexSelected}`;
} else if (roomType === "shop") {
const indexSelected = encodeURIComponent(
"pay_ucnt,watch_cnt_show_ratio,avg_watch_duration,current_cnt,live_show_cnt,real_refund_amt"
);
url = `https://compass.jinritemai.com/compass_api/shop/live/live_screen/core_data?room_id=${roomId}&index_selected=${indexSelected}`;
} else {
throw new Error(`Unknown room type: ${roomType}`);
}
const headers = {
accept: "application/json, text/plain, */*",
};
const response = await sendHttpRequest("GET", url, null, headers);
const data = response.data;
if (roomType === "official") {
return {
pay_amt: data.pay_amt.value,
pay_ucnt: data.core_data.find((d) => d.index_display === "成交人数")
.value.value,
real_refund_amt: data.core_data.find(
(d) => d.index_display === "退款金额"
).value.value,
avg_watch_duration: data.core_data.find(
(d) => d.index_display === "人均观看时长"
).value.value,
current_cnt: data.core_data.find(
(d) => d.index_display === "实时在线人数"
).value.value,
live_show_watch_cnt_ratio: data.core_data.find(
(d) => d.index_display === "曝光-观看率(次数)"
).value.value,
live_show_cnt: data.core_data.find(
(d) => d.index_display === "曝光次数"
).value.value,
};
} else if (roomType === "shop") {
// 根据实际返回的数据结构,提取所需字段
// 这里假设返回的数据结构与官方直播间类似
return {
pay_amt: data.pay_amt.value,
pay_ucnt: data.core_data.find((d) => d.index_display === "成交人数")
.value.value,
real_refund_amt: data.core_data.find(
(d) => d.index_display === "退款金额"
).value.value,
avg_watch_duration: data.core_data.find(
(d) => d.index_display === "人均观看时长"
).value.value,
current_cnt: data.core_data.find(
(d) => d.index_display === "实时在线人数"
).value.value,
live_show_watch_cnt_ratio: data.core_data.find(
(d) => d.index_display === "曝光-观看率(次数)"
).value.value,
live_show_cnt: data.core_data.find(
(d) => d.index_display === "曝光次数"
).value.value,
};
}
} catch (error) {
log("Error fetching live room data: " + error.message);
return {
pay_amt: 0,
pay_ucnt: 0,
real_refund_amt: 0,
avg_watch_duration: 0,
current_cnt: 0,
live_show_watch_cnt_ratio: 0,
live_show_cnt: 0,
};
}
}
/*
//获取评论次数的数据
async function fetchCommentsData(roomId) {
try {
const url = CONFIG.urls.fetchCommentsData(roomId);
const response = await sendHttpRequest('GET', url);
const comments = response.data.card.find(d => d.index_display === '评论次数').value.value;
const updateTimeStr = response.data.update_time.replace("更新", "").trim();
const updateTime = new Date(updateTimeStr).getTime();
return { count: comments, timestamp: updateTime };
} catch (error) {
log('Error fetching interaction data: ' + error.message);
return { count: 0, timestamp: new Date().getTime() };
}
}
// 每五分钟获取一次评论数据
async function fetchComments() {
try {
log('Fetching comments data...');
const commentsData = await fetchCommentsData(liveRoomId);
commentCounts.push(commentsData);
log(commentCounts)
log(`Fetched comments count: ${commentsData.count} at ${new Date(commentsData.timestamp).toLocaleString()}`);
} catch (error) {
log('Error fetching comments data: ' + error.message);
}
}
// 初始化评论次数数组并设置定时器
async function initializeCommentFetching() {
if (intervalId !== null) {
clearInterval(intervalId);
}
commentCounts = [];
await fetchComments(); // 立即执行一次
intervalId = setInterval(fetchComments, 5 * 60 * 1000); // 每五分钟重新获取一次评论数据
}
// 计算过去一小时的评论次数
function getCommentsCountLastHour() {
log(`Current commentCounts array: ${JSON.stringify(commentCounts)}`);
const oneHourAgo = new Date().getTime() - (60 * 60 * 1000);
const filteredComments = commentCounts.filter(comment => comment.timestamp >= oneHourAgo);
log(`Filtered comments array: ${JSON.stringify(filteredComments)}`);
const totalComments = filteredComments.reduce((total, comment) => total + comment.count, 0);
log(`Total comments in the last hour: ${totalComments}`);
return totalComments;
}
*/
//获取直播间开始时间数据
async function fetchLiveRoomStartTime(roomId) {
const url = `https://compass.jinritemai.com/compass_api/shop/live/live_screen/live_base_info?room_id=${roomId}`;
try {
const response = await sendHttpRequest("GET", url);
return response.data.live_start_time;
} catch (error) {
log("Error fetching live room start time: " + error.message);
return null; // 默认返回 null
}
}
// 获取千川消耗金额数据
async function fetchQianchuanData(aadvid) {
const now = new Date();
const formatNumber = (num) => num.toString().padStart(2, "0");
// 结束时间为动态的当前时间
const year = now.getFullYear();
const month = formatNumber(now.getMonth() + 1); // 月份从0开始
const day = formatNumber(now.getDate());
const hours = formatNumber(now.getHours());
const minutes = formatNumber(now.getMinutes());
const seconds = formatNumber(now.getSeconds());
const end_time = `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
let start_time;
log("Failed to fetch start time, using default start time.");
if (now.getHours() >= 6) {
// 时间超过6点则使用当天6点作为开始时间
start_time = `${year}-${month}-${day} 06:00:00`;
} else {
// 如果当前时间在0点到6点之间则开始时间为昨天的6点
const yesterday = new Date(now);
yesterday.setDate(now.getDate() - 1);
const yesterdayYear = yesterday.getFullYear();
const yesterdayMonth = formatNumber(yesterday.getMonth() + 1);
const yesterdayDay = formatNumber(yesterday.getDate());
start_time = `${yesterdayYear}-${yesterdayMonth}-${yesterdayDay} 06:00:00`;
}
const body = {
DataSetKey: "home_cost_total_prom",
Dimensions: ["adlab_mode", "pricing_category", "qcpx_category"],
Metrics: ["stat_cost"],
StartTime: start_time,
EndTime: end_time,
Filters: {
Conditions: [
{
Field: "advertiser_id",
Values: [aadvid],
Operator: 7,
},
{
Field: "ecp_app_id",
Values: ["1", "2"],
Operator: 7,
},
{
Field: "query_self_data",
Values: ["off"],
Operator: 7,
},
],
ConditionRelationshipType: 1,
},
};
try {
const response = await sendHttpRequest(
"POST",
`https://qianchuan.jinritemai.com/ad/api/data/v1/common/statQuery?reqFrom=data-summary&aavid=${aadvid}`,
JSON.stringify(body),
{
"Content-Type": "application/json",
}
);
return response.data.StatsData.Totals.stat_cost.Value;
} catch (error) {
log("Error fetching Qianchuan data: " + error.message);
return 0;
}
}
// 获取直播间转化漏斗数据
async function fetchConvertData(roomId) {
try {
const url = CONFIG.urls.fetchConvertData(roomId);
const response = await sendHttpRequest("GET", url);
return {
exposure_ucnt: response.data.gmv_change.find(
(d) => d.index_name === "直播间曝光人数"
).value,
watch_ucnt: response.data.gmv_change.find(
(d) => d.index_name === "直播间观看人数"
).value,
product_exposure_ucnt: response.data.gmv_change.find(
(d) => d.index_name === "商品曝光人数"
).value,
product_click_ucnt: response.data.gmv_change.find(
(d) => d.index_name === "商品点击人数"
).value,
};
} catch (error) {
log("Error fetching flow data: " + error.message);
return {
exposure_ucnt: 0,
watch_ucnt: 0,
product_exposure_ucnt: 0,
product_click_ucnt: 0,
};
}
}
// 存储数据每个直播间单独一个key不覆盖之前的数据
function saveDataUsingGM(roomId, data) {
let existingData = GM_getValue(roomId, []);
if (!Array.isArray(existingData)) {
existingData = [];
}
existingData.push(data); // 将新的数据推入到现有数据数组中
GM_setValue(roomId, existingData);
}
// 读取数据
function getDataUsingGM(roomId) {
let data = GM_getValue(roomId, []);
return Array.isArray(data) ? data : [];
}
//延迟函数
function delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
//计算直播场次所在的日期
function calculateSessionDate(timestamp) {
const date = new Date(timestamp);
// 直播时间在凌晨2点前则认为是前一天的场次
if (date.getHours() <= 2) {
date.setDate(date.getDate() - 1);
}
// 创建一个新的日期对象,表示计算后的日期,并返回其时间戳
const sessionDate = new Date(
date.getFullYear(),
date.getMonth(),
date.getDate()
);
return sessionDate.getTime();
}
async function fetchAndUploadLiveRoomData() {
for (const room of CONFIG.rooms) {
try {
log(`Fetching live room id for ${room.name}`);
const liveRoomId = await fetchLiveRoomId(room.name);
if (!liveRoomId) {
log(`未找到符合条件的直播间 ${room.name},跳过执行`);
continue; // 未找到符合条件的直播间,跳过执行
}
log(`Fetching live room start time for ${room.name}...`);
const start_time = await fetchLiveRoomStartTime(liveRoomId);
if (!start_time) {
Swal.fire({
icon: "error",
title: `无法获取直播间 ${room.name} 开始时间请检查直播间ID`,
text: "使用默认开始时间 06:00:00",
});
}
log(`Fetching live room data for ${room.name}...`);
const liveRoomData = await fetchLiveRoomData(liveRoomId, room.type);
log("Live room data fetched successfully.");
await delay(2000);
log(`Fetching Qianchuan data for ${room.name}...`);
const qianchuanData = await fetchQianchuanData(room.aadvid);
log("Qianchuan data fetched successfully.");
await delay(2000);
log(`Fetching flow data for ${room.name}...`);
const convertData = await fetchConvertData(liveRoomId);
log("Flow data fetched successfully.");
const previousData = getDataUsingGM(liveRoomId);
const currentTime = new Date().toLocaleString("zh-CN", {
timeZone: "Asia/Shanghai",
});
let incrementData = {};
if (previousData.length > 0) {
const lastData = previousData[previousData.length - 1];
incrementData = {
直播间: room.name, // 添加直播间名字字段
GMV: liveRoomData.pay_amt / 100 - lastData.pay_amt,
商品成交人数: liveRoomData.pay_ucnt - lastData.pay_ucnt,
实时在线人数: liveRoomData.current_cnt,
退款金额:
liveRoomData.real_refund_amt / 100 - lastData.real_refund_amt,
千川消耗: qianchuanData - lastData.qianchuan_cost,
直播间曝光人数: convertData.exposure_ucnt - lastData.exposure_ucnt,
直播观看人数: convertData.watch_ucnt - lastData.watch_ucnt,
商品曝光人数:
convertData.product_exposure_ucnt - lastData.product_exposure_ucnt,
商品点击人数:
convertData.product_click_ucnt - lastData.product_click_ucnt,
直播间ID: liveRoomId,
时间: new Date(currentTime).getTime(),
直播间曝光次数: liveRoomData.live_show_cnt - lastData.live_show_cnt,
直播观看次数:
liveRoomData.live_show_watch_cnt_ratio *
liveRoomData.live_show_cnt -
lastData.live_show_cnt * lastData.live_show_watch_cnt_ratio,
人均停留时长:
(liveRoomData.avg_watch_duration * convertData.watch_ucnt -
lastData.avg_watch_duration * lastData.watch_ucnt) /
(convertData.watch_ucnt - lastData.watch_ucnt),
直播场次日期: calculateSessionDate(new Date(currentTime).getTime()),
差值时间: lastData.timestamp,
};
} else {
incrementData = {
直播间: room.name, // 添加直播间名字字段
GMV: liveRoomData.pay_amt / 100,
商品成交人数: liveRoomData.pay_ucnt,
实时在线人数: liveRoomData.current_cnt,
退款金额: liveRoomData.real_refund_amt / 100,
千川消耗: qianchuanData,
直播间曝光人数: convertData.exposure_ucnt,
直播观看人数: convertData.watch_ucnt,
商品曝光人数: convertData.product_exposure_ucnt,
商品点击人数: convertData.product_click_ucnt,
直播间ID: liveRoomId,
时间: new Date(currentTime).getTime(),
直播间曝光次数: liveRoomData.live_show_cnt,
直播观看次数:
liveRoomData.live_show_watch_cnt_ratio * liveRoomData.live_show_cnt,
人均停留时长: liveRoomData.avg_watch_duration,
直播场次日期: calculateSessionDate(new Date(currentTime).getTime()),
差值时间: new Date(start_time).getTime(),
};
}
log("Data extracted successfully");
log(incrementData);
// 获取tenant access token
log("Fetching tenant access token...");
const accessToken = await fetchTenantAccessToken(
CONFIG.id,
CONFIG.secret
);
log("Tenant access token fetched successfully.");
// 把计算后的数据写入到多维表格
log(`Uploading live room data to bitable for ${room.name}...`);
const itemsToWrite = {
fields: incrementData,
};
await updateBitableRecords(
accessToken,
CONFIG.appId,
CONFIG.tableId,
itemsToWrite
);
log("Live room data uploaded successfully.");
log(`Uploading live room data to localstorage for ${room.name}...`);
// 写入localStorage
const dataToSave = {
room_name: room.name,
pay_amt: liveRoomData.pay_amt / 100,
pay_ucnt: liveRoomData.pay_ucnt,
real_refund_amt: liveRoomData.real_refund_amt / 100,
avg_watch_duration: liveRoomData.avg_watch_duration,
current_cnt: liveRoomData.current_cnt,
qianchuan_cost: qianchuanData,
exposure_ucnt: convertData.exposure_ucnt,
watch_ucnt: convertData.watch_ucnt,
product_exposure_ucnt: convertData.product_exposure_ucnt,
product_click_ucnt: convertData.product_click_ucnt,
liveRoomId: liveRoomId,
timestamp: new Date(currentTime).getTime(),
timestring: currentTime,
live_show_watch_cnt_ratio: liveRoomData.live_show_watch_cnt_ratio,
live_show_cnt: liveRoomData.live_show_cnt,
};
log(dataToSave);
saveDataUsingGM(liveRoomId, dataToSave);
log("Writing to localstorage successfully");
// 写入rawdata多维表格 - 暂时取消此功能
/*
log(`Uploading saved data to another bitable table for ${room.name}...`);
await updateBitableRecords(
accessToken,
CONFIG.appId,
CONFIG.rawDataTableId,
{
fields: dataToSave,
}
);
log("Saved data uploaded to another bitable table successfully.");
*/
} catch (error) {
log(`Error for ${room.name}: ` + error.message);
Swal.fire({
icon: "error",
title: `直播间 ${room.name} 出错了,请联系@汪喜`,
text: error.message,
});
}
}
}
(() => {
let runResult = "成功"; // 默认值为成功
let logs = [];
const log = (message) => {
logs.push(message);
console.log(message);
};
async function logRunResult() {
try {
/*
const accessToken = await fetchTenantAccessToken(
CONFIG.id,
CONFIG.secret
);
const runTime = new Date().getTime();
const logData = {
fields: {
表名: "直播间数据",
表格id: CONFIG.tableId,
最近运行时间: runTime,
运行结果: runResult,
日志: logs.join("\n"),
},
};
await updateBitableRecords(
accessToken,
CONFIG.appId,
CONFIG.logTableId,
logData
);
log("Run result logged successfully.");
*/
// 暂时取消日志上传功能,仅在控制台记录
log("Run result logging disabled.");
} catch (error) {
log("Failed to log run result: " + error.message);
}
}
(async function () {
try {
log("Script started");
await fetchAndUploadLiveRoomData();
log("Data fetched and uploaded");
} catch (error) {
runResult = "失败";
log("Script execution failed: " + error.message);
} finally {
await logRunResult();
log("Script execution completed");
}
})();
})();