feat: 新增多个脚本用于监控抖音直播、店铺评价、售后及体验分数据
新增了多个脚本文件，用于监控抖音直播间的弹幕、店铺评价、售后数据及商家体验分。这些脚本通过飞书多维表格进行数据存储，并支持定时任务自动更新数据。具体包括： 1. 直播间弹幕监控脚本 2. 店铺评价监控脚本 3. 售后数据监控脚本 4. 商家体验分监控脚本 5. 竞品、行业及跨行业热门千川素材获取脚本 这些脚本通过飞书API进行数据写入，并支持去重和定时任务调度。
This commit is contained in:
intelligrow
@@ -0,0 +1,616 @@
|
||||
// ==UserScript==
|
||||
// @name 低流量成本达人监控
|
||||
// @namespace https://bbs.tampermonkey.net.cn/
|
||||
// @version 2.3
|
||||
// @description 每天获取T-2的低CPM CPsearch内容，并给到达人列表，写入多维表格
|
||||
// @author 汪喜
|
||||
// @crontab 53 8 once * *
|
||||
// @grant GM_xmlhttpRequest
|
||||
// @connect open.feishu.cn
|
||||
// @connect yuntu.oceanengine.com
|
||||
// ==/UserScript==
|
||||
|
||||
const CONFIG = {
|
||||
feishu: {
|
||||
id: "cli_a6f25876ea28100d",
|
||||
secret: "raLC56ZLIara07nKigpysfoDxHTAeyJf",
|
||||
appId: "Z6Icb88DsaKyLBsOwYhcjHRinTc",
|
||||
videoTableId: "tblfpGtDGUPhteRH",
|
||||
starTableId: "tblrqiSDmdtmBhXu",
|
||||
urls: {
|
||||
tenantAccessToken:
|
||||
"https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal",
|
||||
bitableBatchCreateRecords: (appId, tableId) =>
|
||||
`https://open.feishu.cn/open-apis/bitable/v1/apps/${appId}/tables/${tableId}/records/batch_create`,
|
||||
bitableSearchRecords: (appId, tableId) =>
|
||||
`https://open.feishu.cn/open-apis/bitable/v1/apps/${appId}/tables/${tableId}/records/search`,
|
||||
},
|
||||
},
|
||||
yuntu: {
|
||||
hotItemApiUrl:
|
||||
"https://yuntu.oceanengine.com/yuntu_ng/api/v1/DoubleHotContentListByTag",
|
||||
aadvid: "1710507483282439",
|
||||
starDataUrl:
|
||||
"https://yuntu.oceanengine.com/yuntu_ng/api/v2/get_talent_filter_v3",
|
||||
industryObj: [
|
||||
{
|
||||
industry_id: 12,
|
||||
industry_name: "美妆",
|
||||
},
|
||||
{
|
||||
industry_id: 13,
|
||||
industry_name: "个护清洁（日化）",
|
||||
},
|
||||
{
|
||||
industry_id: 14,
|
||||
industry_name: "食品饮料",
|
||||
},
|
||||
{
|
||||
industry_id: 17,
|
||||
industry_name: "3C数码",
|
||||
},
|
||||
{
|
||||
industry_id: 18,
|
||||
industry_name: "宠物",
|
||||
},
|
||||
{
|
||||
industry_id: 20,
|
||||
industry_name: "母婴",
|
||||
},
|
||||
{
|
||||
industry_id: 27,
|
||||
industry_name: "医疗保健",
|
||||
},
|
||||
{
|
||||
industry_id: 28,
|
||||
industry_name: "家用电器",
|
||||
},
|
||||
],
|
||||
},
|
||||
retry: {
|
||||
retries: 3,
|
||||
delay: 10000,
|
||||
},
|
||||
batchSize: 500,
|
||||
requestDelay: Math.floor(Math.random() * 8001) + 2000, // 2-10秒随机延迟
|
||||
webhook: {
|
||||
url: "https://bloomagebiotech.feishu.cn/base/automation/webhook/event/ZFVgaDgM9wwBUuhkma5cTYSOnIU", // Webhook URL
|
||||
},
|
||||
};
|
||||
|
||||
// 重试函数
|
||||
const retry = async (fn, retries = 3, delay = 10000) => {
|
||||
for (let i = 0; i < retries; i++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error) {
|
||||
if (i < retries - 1) {
|
||||
console.warn(`Retrying... (${i + 1}/${retries})`);
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 延迟函数
|
||||
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
// HTTP 请求函数
|
||||
async function sendHttpRequest(
|
||||
method,
|
||||
url,
|
||||
body = null,
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
errorHandler = null
|
||||
) {
|
||||
return retry(
|
||||
() =>
|
||||
new Promise((resolve, reject) => {
|
||||
GM_xmlhttpRequest({
|
||||
method: method,
|
||||
url: url,
|
||||
data: body,
|
||||
headers: headers,
|
||||
onload: function (response) {
|
||||
if (response.status >= 200 && response.status < 300) {
|
||||
try {
|
||||
const jsonResponse = JSON.parse(response.responseText);
|
||||
if (jsonResponse.msg) {
|
||||
console.log(jsonResponse.msg);
|
||||
}
|
||||
// 使用自定义的错误处理函数
|
||||
if (errorHandler && !errorHandler(jsonResponse)) {
|
||||
reject(`API Error: ${JSON.stringify(jsonResponse)}`);
|
||||
} else {
|
||||
resolve(jsonResponse);
|
||||
}
|
||||
} catch (e) {
|
||||
reject(`Failed to parse JSON: ${e}`);
|
||||
}
|
||||
} else {
|
||||
reject(`Error: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
},
|
||||
onerror: function (error) {
|
||||
reject(`Network Error: ${error}`);
|
||||
},
|
||||
});
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// 计算日期
|
||||
function formatDate(date) {
|
||||
const d = new Date(date);
|
||||
let month = "" + (d.getMonth() + 1);
|
||||
let day = "" + d.getDate();
|
||||
const year = d.getFullYear();
|
||||
|
||||
if (month.length < 2) month = "0" + month;
|
||||
if (day.length < 2) day = "0" + day;
|
||||
|
||||
return [year, month, day].join("-");
|
||||
}
|
||||
|
||||
// 获取 T-N 的数据
|
||||
function getDynamicDates(n) {
|
||||
const today = new Date();
|
||||
const T_minus_2 = new Date(today);
|
||||
T_minus_2.setDate(today.getDate() - 2);
|
||||
|
||||
const T_minus_n = new Date(T_minus_2);
|
||||
T_minus_n.setDate(T_minus_2.getDate() - n);
|
||||
|
||||
return {
|
||||
start_date: formatDate(T_minus_n),
|
||||
end_date: formatDate(T_minus_2),
|
||||
};
|
||||
}
|
||||
|
||||
// 处理云图 API 返回的数据
|
||||
function processYuntuData(item, industryName) {
|
||||
return {
|
||||
视频id: item.item_id,
|
||||
视频链接文本: item.item_link,
|
||||
达人昵称: item.nickname,
|
||||
达人id: item.aweme_id,
|
||||
达人星图id: item.star_uid_id,
|
||||
视频标题: item.title,
|
||||
视频后台id: item.video_id,
|
||||
视频时长: Number(item.video_duration),
|
||||
自然播放次数: item.index_map["80101"] || 0,
|
||||
自然互动次数: item.index_map["80105"] || 0,
|
||||
回搜次数: item.index_map["110001"] || 0,
|
||||
回搜人数: item.index_map["110002"] || 0,
|
||||
看后搜次数: item.index_map["10010"] || 0,
|
||||
看后搜人数: Number(item.key_word_after_search_info.search_uv) || 0,
|
||||
回搜率: item.index_map["110003"] || 0,
|
||||
看后搜率: item.key_word_after_search_info.search_rate || 0,
|
||||
//"search_cost": item.search_cost || 0,
|
||||
发布时间: new Date(
|
||||
item.create_date.slice(0, 4),
|
||||
item.create_date.slice(4, 6) - 1,
|
||||
item.create_date.slice(6, 8)
|
||||
).getTime(),
|
||||
品牌ID: Number(item.brand_id),
|
||||
行业名称: industryName,
|
||||
视频类型: item.content_type, //数组类型 多选
|
||||
搜索关键词:
|
||||
item.key_word_after_search_info.keywords === undefined
|
||||
? ""
|
||||
: item.key_word_after_search_info.keywords.join(", "),
|
||||
完播数: item.index_map["30019"] || 0,
|
||||
总播放次数: item.index_map["80001"] || 0,
|
||||
新增A3率: item.index_map["100002"] || 0,
|
||||
};
|
||||
}
|
||||
|
||||
// 云图 API 类
|
||||
class YuntuAPI {
|
||||
constructor(config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
handleYuntuResponse(response) {
|
||||
// 检查 status 字段
|
||||
return response.status === 0;
|
||||
}
|
||||
|
||||
async fetchDoubleHotContentListByTag(body, industryName) {
|
||||
try {
|
||||
const url = `${this.config.hotItemApiUrl}?aadvid=${this.config.aadvid}`;
|
||||
const response = await sendHttpRequest(
|
||||
"POST",
|
||||
url,
|
||||
JSON.stringify(body),
|
||||
{
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
this.handleYuntuResponse
|
||||
);
|
||||
await delay(CONFIG.requestDelay); // 添加延迟
|
||||
return response.data.item_list.map((item) =>
|
||||
processYuntuData(item, industryName)
|
||||
);
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch data from Yuntu API", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
createRequestBody(industryId, industryName) {
|
||||
const dates = getDynamicDates(7);
|
||||
return {
|
||||
general_cond: {
|
||||
industry_id: industryId,
|
||||
brand_id: "10209634",
|
||||
time_range: {
|
||||
date_type: 0,
|
||||
start_date: dates.start_date,
|
||||
end_date: dates.end_date,
|
||||
},
|
||||
time_type: 2,
|
||||
task_type: 1,
|
||||
},
|
||||
hot_type: [1, 0, 2, 3],
|
||||
item_type: 1, // 0 本品牌 1行业 2竞品
|
||||
industry_list: [
|
||||
{
|
||||
industry_id: industryId,
|
||||
industry_name: industryName,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async fetchAllIndustries() {
|
||||
const results = [];
|
||||
for (const industry of this.config.industryObj) {
|
||||
const { industry_id, industry_name } = industry;
|
||||
const requestBody = this.createRequestBody(industry_id, industry_name);
|
||||
const result = await this.fetchDoubleHotContentListByTag(
|
||||
requestBody,
|
||||
industry_name
|
||||
);
|
||||
results.push(...result);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// 获取达人星图数据
|
||||
async fetchStarData(starIds) {
|
||||
const results = [];
|
||||
await batchProcess(starIds, CONFIG.batchSize, async (batch) => {
|
||||
const body = {
|
||||
mkt_scene: 1,
|
||||
task_type: 1,
|
||||
brand_id: "10209634",
|
||||
industry_id: "12",
|
||||
exact_match_filter: {
|
||||
talent_filter_cond: 1,
|
||||
exact_match_list: batch,
|
||||
},
|
||||
gender: -1,
|
||||
limit: CONFIG.batchSize,
|
||||
};
|
||||
const url = `${this.config.starDataUrl}?aadvid=${this.config.aadvid}`;
|
||||
const response = await sendHttpRequest(
|
||||
"POST",
|
||||
url,
|
||||
JSON.stringify(body),
|
||||
{
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
this.handleYuntuResponse
|
||||
);
|
||||
await delay(CONFIG.requestDelay); // 添加延迟
|
||||
results.push(
|
||||
...response.data.info_item_list.map((item) => ({
|
||||
达人星图id: item.star_uid,
|
||||
达人昵称: item.nick_name,
|
||||
"20s以下视频报价": Number(item.price),
|
||||
"20-60s视频报价": Number(item.price_20_60),
|
||||
"60s以上视频报价": Number(item.price_60),
|
||||
达人uid: item.aweme_id,
|
||||
粉丝量: Number(item.fans_cnt),
|
||||
}))
|
||||
);
|
||||
});
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
// 飞书 API 类
|
||||
class FeishuAPI {
|
||||
constructor(config) {
|
||||
this.config = config;
|
||||
this.accessToken = null;
|
||||
}
|
||||
|
||||
//获取token
|
||||
async fetchTenantAccessToken() {
|
||||
try {
|
||||
const response = await sendHttpRequest(
|
||||
"POST",
|
||||
this.config.urls.tenantAccessToken,
|
||||
JSON.stringify({
|
||||
app_id: this.config.id,
|
||||
app_secret: this.config.secret,
|
||||
})
|
||||
);
|
||||
this.accessToken = response.tenant_access_token;
|
||||
await delay(CONFIG.requestDelay); // 添加延迟
|
||||
return this.accessToken;
|
||||
} catch (error) {
|
||||
throw new Error(`Error fetching Tenant Access Token: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
//批量更新记录
|
||||
async batchCreateBitableRecords(tableId, items) {
|
||||
if (!this.accessToken) {
|
||||
await this.fetchTenantAccessToken();
|
||||
}
|
||||
const results = [];
|
||||
await batchProcess(items, CONFIG.batchSize, async (batch) => {
|
||||
try {
|
||||
const response = await sendHttpRequest(
|
||||
"POST",
|
||||
this.config.urls.bitableBatchCreateRecords(
|
||||
this.config.appId,
|
||||
tableId
|
||||
),
|
||||
JSON.stringify({
|
||||
records: batch,
|
||||
}),
|
||||
{
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
);
|
||||
await delay(CONFIG.requestDelay); // 添加延迟
|
||||
console.log("Batch created records: " + JSON.stringify(response));
|
||||
results.push(response);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to batch create records in Bitable: ${error}`);
|
||||
}
|
||||
});
|
||||
return results;
|
||||
}
|
||||
|
||||
//返回视频id、recordid和视频标签
|
||||
async fetchRecordByVideoId(videoId) {
|
||||
if (!this.accessToken) {
|
||||
await this.fetchTenantAccessToken();
|
||||
}
|
||||
|
||||
const body = {
|
||||
field_names: ["视频id", "标签"],
|
||||
filter: {
|
||||
conjunction: "and",
|
||||
conditions: [
|
||||
{
|
||||
field_name: "视频id",
|
||||
operator: "is",
|
||||
value: [videoId],
|
||||
},
|
||||
],
|
||||
},
|
||||
sort: [
|
||||
{
|
||||
field_name: "记录时间",
|
||||
desc: true, //倒序
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const url = CONFIG.feishu.urls.bitableSearchRecords(
|
||||
CONFIG.feishu.appId,
|
||||
CONFIG.feishu.videoTableId
|
||||
);
|
||||
|
||||
const response = await sendHttpRequest("POST", url, JSON.stringify(body), {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
});
|
||||
|
||||
return response.data.items;
|
||||
}
|
||||
}
|
||||
|
||||
// 批量处理函数
|
||||
async function batchProcess(items, batchSize, processFunction) {
|
||||
for (let i = 0; i < items.length; i += batchSize) {
|
||||
const batch = items.slice(i, i + batchSize);
|
||||
await processFunction(batch);
|
||||
}
|
||||
}
|
||||
|
||||
// 发送统计结果到 Webhook 服务器
|
||||
async function sendStatisticsToWebhook(lowCostCount, explosiveCount) {
|
||||
const payload = {
|
||||
lowCostCount: lowCostCount,
|
||||
explosiveCount: explosiveCount,
|
||||
};
|
||||
|
||||
try {
|
||||
await sendHttpRequest("POST", CONFIG.webhook.url, JSON.stringify(payload), {
|
||||
"Content-Type": "application/json",
|
||||
});
|
||||
console.log("Statistics sent to webhook successfully.");
|
||||
} catch (error) {
|
||||
console.error("Failed to send statistics to webhook:", error);
|
||||
}
|
||||
}
|
||||
|
||||
// 主函数
|
||||
async function processAndUploadData() {
|
||||
try {
|
||||
console.log("Start processing and uploading data...");
|
||||
|
||||
// 1. 请求所有的视频数据
|
||||
const yuntuAPI = new YuntuAPI(CONFIG.yuntu);
|
||||
const allData = await yuntuAPI.fetchAllIndustries();
|
||||
console.log("Fetched all video data.");
|
||||
|
||||
// 2. 根据视频里面的达人星图id字段，请求所有达人的星图报价数据
|
||||
const starIds = allData.map((data) => data["达人星图id"]);
|
||||
const starData = await yuntuAPI.fetchStarData(starIds);
|
||||
console.log("Fetched all star data.");
|
||||
|
||||
// 3. 获取已存在的多维表格记录的视频id和标签信息
|
||||
const feishuAPI = new FeishuAPI(CONFIG.feishu);
|
||||
|
||||
// 4. 根据视频时长，选择合适的报价，如果计算出来的CPM<40，则把视频数据和达人数据push到两个json数组中
|
||||
const filteredVideoData = [];
|
||||
const filteredStarData = [];
|
||||
|
||||
// 记录标签推送次数
|
||||
let lowCostCount = 0;
|
||||
let explosiveCount = 0;
|
||||
|
||||
// 遍历所有视频数据
|
||||
for (const videoData of allData) {
|
||||
// 找到当前视频对应的达人信息
|
||||
const currentStarInfo = starData.find(
|
||||
(star) => star["达人星图id"] === videoData["达人星图id"]
|
||||
);
|
||||
|
||||
// 如果找到了对应的达人信息
|
||||
if (currentStarInfo) {
|
||||
// 根据视频时长选择合适的报价
|
||||
let price;
|
||||
if (videoData["视频时长"] < 20) {
|
||||
price = currentStarInfo["20s以下视频报价"];
|
||||
} else if (videoData["视频时长"] <= 60) {
|
||||
price = currentStarInfo["20-60s视频报价"];
|
||||
} else {
|
||||
price = currentStarInfo["60s以上视频报价"];
|
||||
}
|
||||
|
||||
// 数据过滤部分
|
||||
// 计算自然流量的每千次曝光成本（CPM）
|
||||
const cpm = (price / videoData["自然播放次数"]) * 1000;
|
||||
|
||||
// 计算预估的自然看后搜人数
|
||||
const nativeSearchUv =
|
||||
(videoData["自然播放次数"] / videoData["总播放次数"]) *
|
||||
videoData["看后搜人数"];
|
||||
|
||||
// 计算预估的自然看后搜成本
|
||||
const CPNativeSearchUv = price / nativeSearchUv;
|
||||
|
||||
// 初始化标识字段为数组
|
||||
videoData["标签"] = [];
|
||||
|
||||
// 筛选条件：CPM小于20，CPNativeSearchUv小于10，且报价大于0，或者自然播放次数大于1000万
|
||||
if (
|
||||
(cpm < 20 && CPNativeSearchUv < 10 && price > 0) ||
|
||||
(videoData["自然播放次数"] > 10000000 && nativeSearchUv > 3000)
|
||||
) {
|
||||
// 为视频数据添加新的字段标识“低成本”或“爆量”
|
||||
if (cpm < 20 && CPNativeSearchUv < 10 && price > 0) {
|
||||
if (!videoData["标签"].includes("低成本")) {
|
||||
videoData["标签"].push("低成本");
|
||||
//lowCostCount++;
|
||||
}
|
||||
}
|
||||
if (
|
||||
videoData["自然播放次数"] > 10000000 &&
|
||||
nativeSearchUv > 3000 &&
|
||||
price > 0
|
||||
) {
|
||||
if (!videoData["标签"].includes("爆量")) {
|
||||
videoData["标签"].push("爆量");
|
||||
// explosiveCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// 添加当前时间字段
|
||||
videoData["记录时间"] = Date.now();
|
||||
|
||||
// 查找多维表格中的记录
|
||||
const existingRecords = await feishuAPI.fetchRecordByVideoId(
|
||||
videoData["视频id"]
|
||||
);
|
||||
const existingRecord =
|
||||
existingRecords.length > 0 ? existingRecords[0] : null;
|
||||
|
||||
// 如果视频ID已经存在，则判断标签是否不同
|
||||
if (existingRecord) {
|
||||
const existingTags = existingRecord.fields["标签"] || [];
|
||||
const newTags = videoData["标签"];
|
||||
// 使用集合比较标签
|
||||
const tagsAreDifferent =
|
||||
new Set([...existingTags, ...newTags]).size !==
|
||||
existingTags.length;
|
||||
|
||||
// 如果标签不同，则更新记录
|
||||
if (tagsAreDifferent) {
|
||||
filteredVideoData.push(videoData);
|
||||
//标签不同不需要再push达人数据，表格中已经有了
|
||||
//filteredStarData.push(currentStarInfo);
|
||||
// 增加计数器
|
||||
if (videoData["标签"].includes("低成本")) {
|
||||
lowCostCount++;
|
||||
}
|
||||
if (videoData["标签"].includes("爆量")) {
|
||||
explosiveCount++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 如果视频ID不存在，则新增记录
|
||||
filteredVideoData.push(videoData);
|
||||
filteredStarData.push(currentStarInfo);
|
||||
// 增加计数器
|
||||
if (videoData["标签"].includes("低成本")) {
|
||||
lowCostCount++;
|
||||
}
|
||||
if (videoData["标签"].includes("爆量")) {
|
||||
explosiveCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 错误处理：如果没有满足条件的视频，记录日志
|
||||
if (filteredVideoData.length === 0) {
|
||||
console.warn("No videos met the criteria.");
|
||||
return;
|
||||
}
|
||||
|
||||
const videoRecordsToUpload = filteredVideoData.map((data) => ({
|
||||
fields: data,
|
||||
}));
|
||||
const starRecordsToUpload = filteredStarData.map((data) => ({
|
||||
fields: data,
|
||||
}));
|
||||
|
||||
// 4. 将两个json数组的结果，分别写入达人表和视频表
|
||||
await feishuAPI.batchCreateBitableRecords(
|
||||
CONFIG.feishu.videoTableId,
|
||||
videoRecordsToUpload
|
||||
);
|
||||
await feishuAPI.batchCreateBitableRecords(
|
||||
CONFIG.feishu.starTableId,
|
||||
starRecordsToUpload
|
||||
);
|
||||
|
||||
// 在数据成功写入多维表格后调用
|
||||
await sendStatisticsToWebhook(lowCostCount, explosiveCount);
|
||||
|
||||
console.log("Data successfully uploaded to Bitable.");
|
||||
} catch (error) {
|
||||
console.error("Failed to process and upload data: ", error);
|
||||
}
|
||||
}
|
||||
|
||||
// 自动启动脚本
|
||||
(async () => {
|
||||
await processAndUploadData();
|
||||
})();
|
||||
Reference in New Issue
Block a user