3c3ecf8947
新增了多个脚本文件，用于监控抖音直播间的弹幕、店铺评价、售后数据及商家体验分。这些脚本通过飞书多维表格进行数据存储，并支持定时任务自动更新数据。具体包括： 1. 直播间弹幕监控脚本 2. 店铺评价监控脚本 3. 售后数据监控脚本 4. 商家体验分监控脚本 5. 竞品、行业及跨行业热门千川素材获取脚本 这些脚本通过飞书API进行数据写入，并支持去重和定时任务调度。
617 lines
18 KiB
JavaScript
617 lines
18 KiB
JavaScript
// ==UserScript==
|
||
// @name 低流量成本达人监控
|
||
// @namespace https://bbs.tampermonkey.net.cn/
|
||
// @version 2.3
|
||
// @description 每天获取T-2的低CPM CPsearch内容，并给到达人列表，写入多维表格
|
||
// @author 汪喜
|
||
// @crontab 53 8 once * *
|
||
// @grant GM_xmlhttpRequest
|
||
// @connect open.feishu.cn
|
||
// @connect yuntu.oceanengine.com
|
||
// ==/UserScript==
|
||
|
||
const CONFIG = {
|
||
feishu: {
|
||
id: "cli_a6f25876ea28100d",
|
||
secret: "raLC56ZLIara07nKigpysfoDxHTAeyJf",
|
||
appId: "Z6Icb88DsaKyLBsOwYhcjHRinTc",
|
||
videoTableId: "tblfpGtDGUPhteRH",
|
||
starTableId: "tblrqiSDmdtmBhXu",
|
||
urls: {
|
||
tenantAccessToken:
|
||
"https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal",
|
||
bitableBatchCreateRecords: (appId, tableId) =>
|
||
`https://open.feishu.cn/open-apis/bitable/v1/apps/${appId}/tables/${tableId}/records/batch_create`,
|
||
bitableSearchRecords: (appId, tableId) =>
|
||
`https://open.feishu.cn/open-apis/bitable/v1/apps/${appId}/tables/${tableId}/records/search`,
|
||
},
|
||
},
|
||
yuntu: {
|
||
hotItemApiUrl:
|
||
"https://yuntu.oceanengine.com/yuntu_ng/api/v1/DoubleHotContentListByTag",
|
||
aadvid: "1710507483282439",
|
||
starDataUrl:
|
||
"https://yuntu.oceanengine.com/yuntu_ng/api/v2/get_talent_filter_v3",
|
||
industryObj: [
|
||
{
|
||
industry_id: 12,
|
||
industry_name: "美妆",
|
||
},
|
||
{
|
||
industry_id: 13,
|
||
industry_name: "个护清洁（日化）",
|
||
},
|
||
{
|
||
industry_id: 14,
|
||
industry_name: "食品饮料",
|
||
},
|
||
{
|
||
industry_id: 17,
|
||
industry_name: "3C数码",
|
||
},
|
||
{
|
||
industry_id: 18,
|
||
industry_name: "宠物",
|
||
},
|
||
{
|
||
industry_id: 20,
|
||
industry_name: "母婴",
|
||
},
|
||
{
|
||
industry_id: 27,
|
||
industry_name: "医疗保健",
|
||
},
|
||
{
|
||
industry_id: 28,
|
||
industry_name: "家用电器",
|
||
},
|
||
],
|
||
},
|
||
retry: {
|
||
retries: 3,
|
||
delay: 10000,
|
||
},
|
||
batchSize: 500,
|
||
requestDelay: Math.floor(Math.random() * 8001) + 2000, // 2-10秒随机延迟
|
||
webhook: {
|
||
url: "https://bloomagebiotech.feishu.cn/base/automation/webhook/event/ZFVgaDgM9wwBUuhkma5cTYSOnIU", // Webhook URL
|
||
},
|
||
};
|
||
|
||
// 重试函数
|
||
const retry = async (fn, retries = 3, delay = 10000) => {
|
||
for (let i = 0; i < retries; i++) {
|
||
try {
|
||
return await fn();
|
||
} catch (error) {
|
||
if (i < retries - 1) {
|
||
console.warn(`Retrying... (${i + 1}/${retries})`);
|
||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||
} else {
|
||
throw error;
|
||
}
|
||
}
|
||
}
|
||
};
|
||
|
||
// 延迟函数
|
||
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||
|
||
// HTTP 请求函数
|
||
async function sendHttpRequest(
|
||
method,
|
||
url,
|
||
body = null,
|
||
headers = {
|
||
"Content-Type": "application/json",
|
||
},
|
||
errorHandler = null
|
||
) {
|
||
return retry(
|
||
() =>
|
||
new Promise((resolve, reject) => {
|
||
GM_xmlhttpRequest({
|
||
method: method,
|
||
url: url,
|
||
data: body,
|
||
headers: headers,
|
||
onload: function (response) {
|
||
if (response.status >= 200 && response.status < 300) {
|
||
try {
|
||
const jsonResponse = JSON.parse(response.responseText);
|
||
if (jsonResponse.msg) {
|
||
console.log(jsonResponse.msg);
|
||
}
|
||
// 使用自定义的错误处理函数
|
||
if (errorHandler && !errorHandler(jsonResponse)) {
|
||
reject(`API Error: ${JSON.stringify(jsonResponse)}`);
|
||
} else {
|
||
resolve(jsonResponse);
|
||
}
|
||
} catch (e) {
|
||
reject(`Failed to parse JSON: ${e}`);
|
||
}
|
||
} else {
|
||
reject(`Error: ${response.status} ${response.statusText}`);
|
||
}
|
||
},
|
||
onerror: function (error) {
|
||
reject(`Network Error: ${error}`);
|
||
},
|
||
});
|
||
})
|
||
);
|
||
}
|
||
|
||
// 计算日期
|
||
function formatDate(date) {
|
||
const d = new Date(date);
|
||
let month = "" + (d.getMonth() + 1);
|
||
let day = "" + d.getDate();
|
||
const year = d.getFullYear();
|
||
|
||
if (month.length < 2) month = "0" + month;
|
||
if (day.length < 2) day = "0" + day;
|
||
|
||
return [year, month, day].join("-");
|
||
}
|
||
|
||
// 获取 T-N 的数据
|
||
function getDynamicDates(n) {
|
||
const today = new Date();
|
||
const T_minus_2 = new Date(today);
|
||
T_minus_2.setDate(today.getDate() - 2);
|
||
|
||
const T_minus_n = new Date(T_minus_2);
|
||
T_minus_n.setDate(T_minus_2.getDate() - n);
|
||
|
||
return {
|
||
start_date: formatDate(T_minus_n),
|
||
end_date: formatDate(T_minus_2),
|
||
};
|
||
}
|
||
|
||
// 处理云图 API 返回的数据
|
||
function processYuntuData(item, industryName) {
|
||
return {
|
||
视频id: item.item_id,
|
||
视频链接文本: item.item_link,
|
||
达人昵称: item.nickname,
|
||
达人id: item.aweme_id,
|
||
达人星图id: item.star_uid_id,
|
||
视频标题: item.title,
|
||
视频后台id: item.video_id,
|
||
视频时长: Number(item.video_duration),
|
||
自然播放次数: item.index_map["80101"] || 0,
|
||
自然互动次数: item.index_map["80105"] || 0,
|
||
回搜次数: item.index_map["110001"] || 0,
|
||
回搜人数: item.index_map["110002"] || 0,
|
||
看后搜次数: item.index_map["10010"] || 0,
|
||
看后搜人数: Number(item.key_word_after_search_info.search_uv) || 0,
|
||
回搜率: item.index_map["110003"] || 0,
|
||
看后搜率: item.key_word_after_search_info.search_rate || 0,
|
||
//"search_cost": item.search_cost || 0,
|
||
发布时间: new Date(
|
||
item.create_date.slice(0, 4),
|
||
item.create_date.slice(4, 6) - 1,
|
||
item.create_date.slice(6, 8)
|
||
).getTime(),
|
||
品牌ID: Number(item.brand_id),
|
||
行业名称: industryName,
|
||
视频类型: item.content_type, //数组类型 多选
|
||
搜索关键词:
|
||
item.key_word_after_search_info.keywords === undefined
|
||
? ""
|
||
: item.key_word_after_search_info.keywords.join(", "),
|
||
完播数: item.index_map["30019"] || 0,
|
||
总播放次数: item.index_map["80001"] || 0,
|
||
新增A3率: item.index_map["100002"] || 0,
|
||
};
|
||
}
|
||
|
||
// 云图 API 类
|
||
class YuntuAPI {
|
||
constructor(config) {
|
||
this.config = config;
|
||
}
|
||
|
||
handleYuntuResponse(response) {
|
||
// 检查 status 字段
|
||
return response.status === 0;
|
||
}
|
||
|
||
async fetchDoubleHotContentListByTag(body, industryName) {
|
||
try {
|
||
const url = `${this.config.hotItemApiUrl}?aadvid=${this.config.aadvid}`;
|
||
const response = await sendHttpRequest(
|
||
"POST",
|
||
url,
|
||
JSON.stringify(body),
|
||
{
|
||
"Content-Type": "application/json",
|
||
},
|
||
this.handleYuntuResponse
|
||
);
|
||
await delay(CONFIG.requestDelay); // 添加延迟
|
||
return response.data.item_list.map((item) =>
|
||
processYuntuData(item, industryName)
|
||
);
|
||
} catch (error) {
|
||
console.error("Failed to fetch data from Yuntu API", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
createRequestBody(industryId, industryName) {
|
||
const dates = getDynamicDates(7);
|
||
return {
|
||
general_cond: {
|
||
industry_id: industryId,
|
||
brand_id: "10209634",
|
||
time_range: {
|
||
date_type: 0,
|
||
start_date: dates.start_date,
|
||
end_date: dates.end_date,
|
||
},
|
||
time_type: 2,
|
||
task_type: 1,
|
||
},
|
||
hot_type: [1, 0, 2, 3],
|
||
item_type: 1, // 0 本品牌 1行业 2竞品
|
||
industry_list: [
|
||
{
|
||
industry_id: industryId,
|
||
industry_name: industryName,
|
||
},
|
||
],
|
||
};
|
||
}
|
||
|
||
async fetchAllIndustries() {
|
||
const results = [];
|
||
for (const industry of this.config.industryObj) {
|
||
const { industry_id, industry_name } = industry;
|
||
const requestBody = this.createRequestBody(industry_id, industry_name);
|
||
const result = await this.fetchDoubleHotContentListByTag(
|
||
requestBody,
|
||
industry_name
|
||
);
|
||
results.push(...result);
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// 获取达人星图数据
|
||
async fetchStarData(starIds) {
|
||
const results = [];
|
||
await batchProcess(starIds, CONFIG.batchSize, async (batch) => {
|
||
const body = {
|
||
mkt_scene: 1,
|
||
task_type: 1,
|
||
brand_id: "10209634",
|
||
industry_id: "12",
|
||
exact_match_filter: {
|
||
talent_filter_cond: 1,
|
||
exact_match_list: batch,
|
||
},
|
||
gender: -1,
|
||
limit: CONFIG.batchSize,
|
||
};
|
||
const url = `${this.config.starDataUrl}?aadvid=${this.config.aadvid}`;
|
||
const response = await sendHttpRequest(
|
||
"POST",
|
||
url,
|
||
JSON.stringify(body),
|
||
{
|
||
"Content-Type": "application/json",
|
||
},
|
||
this.handleYuntuResponse
|
||
);
|
||
await delay(CONFIG.requestDelay); // 添加延迟
|
||
results.push(
|
||
...response.data.info_item_list.map((item) => ({
|
||
达人星图id: item.star_uid,
|
||
达人昵称: item.nick_name,
|
||
"20s以下视频报价": Number(item.price),
|
||
"20-60s视频报价": Number(item.price_20_60),
|
||
"60s以上视频报价": Number(item.price_60),
|
||
达人uid: item.aweme_id,
|
||
粉丝量: Number(item.fans_cnt),
|
||
}))
|
||
);
|
||
});
|
||
return results;
|
||
}
|
||
}
|
||
|
||
// 飞书 API 类
|
||
class FeishuAPI {
|
||
constructor(config) {
|
||
this.config = config;
|
||
this.accessToken = null;
|
||
}
|
||
|
||
//获取token
|
||
async fetchTenantAccessToken() {
|
||
try {
|
||
const response = await sendHttpRequest(
|
||
"POST",
|
||
this.config.urls.tenantAccessToken,
|
||
JSON.stringify({
|
||
app_id: this.config.id,
|
||
app_secret: this.config.secret,
|
||
})
|
||
);
|
||
this.accessToken = response.tenant_access_token;
|
||
await delay(CONFIG.requestDelay); // 添加延迟
|
||
return this.accessToken;
|
||
} catch (error) {
|
||
throw new Error(`Error fetching Tenant Access Token: ${error}`);
|
||
}
|
||
}
|
||
|
||
//批量更新记录
|
||
async batchCreateBitableRecords(tableId, items) {
|
||
if (!this.accessToken) {
|
||
await this.fetchTenantAccessToken();
|
||
}
|
||
const results = [];
|
||
await batchProcess(items, CONFIG.batchSize, async (batch) => {
|
||
try {
|
||
const response = await sendHttpRequest(
|
||
"POST",
|
||
this.config.urls.bitableBatchCreateRecords(
|
||
this.config.appId,
|
||
tableId
|
||
),
|
||
JSON.stringify({
|
||
records: batch,
|
||
}),
|
||
{
|
||
Authorization: `Bearer ${this.accessToken}`,
|
||
"Content-Type": "application/json",
|
||
}
|
||
);
|
||
await delay(CONFIG.requestDelay); // 添加延迟
|
||
console.log("Batch created records: " + JSON.stringify(response));
|
||
results.push(response);
|
||
} catch (error) {
|
||
throw new Error(`Failed to batch create records in Bitable: ${error}`);
|
||
}
|
||
});
|
||
return results;
|
||
}
|
||
|
||
//返回视频id、recordid和视频标签
|
||
async fetchRecordByVideoId(videoId) {
|
||
if (!this.accessToken) {
|
||
await this.fetchTenantAccessToken();
|
||
}
|
||
|
||
const body = {
|
||
field_names: ["视频id", "标签"],
|
||
filter: {
|
||
conjunction: "and",
|
||
conditions: [
|
||
{
|
||
field_name: "视频id",
|
||
operator: "is",
|
||
value: [videoId],
|
||
},
|
||
],
|
||
},
|
||
sort: [
|
||
{
|
||
field_name: "记录时间",
|
||
desc: true, //倒序
|
||
},
|
||
],
|
||
};
|
||
|
||
const url = CONFIG.feishu.urls.bitableSearchRecords(
|
||
CONFIG.feishu.appId,
|
||
CONFIG.feishu.videoTableId
|
||
);
|
||
|
||
const response = await sendHttpRequest("POST", url, JSON.stringify(body), {
|
||
Authorization: `Bearer ${this.accessToken}`,
|
||
"Content-Type": "application/json",
|
||
});
|
||
|
||
return response.data.items;
|
||
}
|
||
}
|
||
|
||
// 批量处理函数
|
||
async function batchProcess(items, batchSize, processFunction) {
|
||
for (let i = 0; i < items.length; i += batchSize) {
|
||
const batch = items.slice(i, i + batchSize);
|
||
await processFunction(batch);
|
||
}
|
||
}
|
||
|
||
// 发送统计结果到 Webhook 服务器
|
||
async function sendStatisticsToWebhook(lowCostCount, explosiveCount) {
|
||
const payload = {
|
||
lowCostCount: lowCostCount,
|
||
explosiveCount: explosiveCount,
|
||
};
|
||
|
||
try {
|
||
await sendHttpRequest("POST", CONFIG.webhook.url, JSON.stringify(payload), {
|
||
"Content-Type": "application/json",
|
||
});
|
||
console.log("Statistics sent to webhook successfully.");
|
||
} catch (error) {
|
||
console.error("Failed to send statistics to webhook:", error);
|
||
}
|
||
}
|
||
|
||
// 主函数
|
||
async function processAndUploadData() {
|
||
try {
|
||
console.log("Start processing and uploading data...");
|
||
|
||
// 1. 请求所有的视频数据
|
||
const yuntuAPI = new YuntuAPI(CONFIG.yuntu);
|
||
const allData = await yuntuAPI.fetchAllIndustries();
|
||
console.log("Fetched all video data.");
|
||
|
||
// 2. 根据视频里面的达人星图id字段，请求所有达人的星图报价数据
|
||
const starIds = allData.map((data) => data["达人星图id"]);
|
||
const starData = await yuntuAPI.fetchStarData(starIds);
|
||
console.log("Fetched all star data.");
|
||
|
||
// 3. 获取已存在的多维表格记录的视频id和标签信息
|
||
const feishuAPI = new FeishuAPI(CONFIG.feishu);
|
||
|
||
// 4. 根据视频时长，选择合适的报价，如果计算出来的CPM<40，则把视频数据和达人数据push到两个json数组中
|
||
const filteredVideoData = [];
|
||
const filteredStarData = [];
|
||
|
||
// 记录标签推送次数
|
||
let lowCostCount = 0;
|
||
let explosiveCount = 0;
|
||
|
||
// 遍历所有视频数据
|
||
for (const videoData of allData) {
|
||
// 找到当前视频对应的达人信息
|
||
const currentStarInfo = starData.find(
|
||
(star) => star["达人星图id"] === videoData["达人星图id"]
|
||
);
|
||
|
||
// 如果找到了对应的达人信息
|
||
if (currentStarInfo) {
|
||
// 根据视频时长选择合适的报价
|
||
let price;
|
||
if (videoData["视频时长"] < 20) {
|
||
price = currentStarInfo["20s以下视频报价"];
|
||
} else if (videoData["视频时长"] <= 60) {
|
||
price = currentStarInfo["20-60s视频报价"];
|
||
} else {
|
||
price = currentStarInfo["60s以上视频报价"];
|
||
}
|
||
|
||
// 数据过滤部分
|
||
// 计算自然流量的每千次曝光成本（CPM）
|
||
const cpm = (price / videoData["自然播放次数"]) * 1000;
|
||
|
||
// 计算预估的自然看后搜人数
|
||
const nativeSearchUv =
|
||
(videoData["自然播放次数"] / videoData["总播放次数"]) *
|
||
videoData["看后搜人数"];
|
||
|
||
// 计算预估的自然看后搜成本
|
||
const CPNativeSearchUv = price / nativeSearchUv;
|
||
|
||
// 初始化标识字段为数组
|
||
videoData["标签"] = [];
|
||
|
||
// 筛选条件：CPM小于20，CPNativeSearchUv小于10，且报价大于0，或者自然播放次数大于1000万
|
||
if (
|
||
(cpm < 20 && CPNativeSearchUv < 10 && price > 0) ||
|
||
(videoData["自然播放次数"] > 10000000 && nativeSearchUv > 3000)
|
||
) {
|
||
// 为视频数据添加新的字段标识“低成本”或“爆量”
|
||
if (cpm < 20 && CPNativeSearchUv < 10 && price > 0) {
|
||
if (!videoData["标签"].includes("低成本")) {
|
||
videoData["标签"].push("低成本");
|
||
//lowCostCount++;
|
||
}
|
||
}
|
||
if (
|
||
videoData["自然播放次数"] > 10000000 &&
|
||
nativeSearchUv > 3000 &&
|
||
price > 0
|
||
) {
|
||
if (!videoData["标签"].includes("爆量")) {
|
||
videoData["标签"].push("爆量");
|
||
// explosiveCount++;
|
||
}
|
||
}
|
||
|
||
// 添加当前时间字段
|
||
videoData["记录时间"] = Date.now();
|
||
|
||
// 查找多维表格中的记录
|
||
const existingRecords = await feishuAPI.fetchRecordByVideoId(
|
||
videoData["视频id"]
|
||
);
|
||
const existingRecord =
|
||
existingRecords.length > 0 ? existingRecords[0] : null;
|
||
|
||
// 如果视频ID已经存在，则判断标签是否不同
|
||
if (existingRecord) {
|
||
const existingTags = existingRecord.fields["标签"] || [];
|
||
const newTags = videoData["标签"];
|
||
// 使用集合比较标签
|
||
const tagsAreDifferent =
|
||
new Set([...existingTags, ...newTags]).size !==
|
||
existingTags.length;
|
||
|
||
// 如果标签不同，则更新记录
|
||
if (tagsAreDifferent) {
|
||
filteredVideoData.push(videoData);
|
||
//标签不同不需要再push达人数据，表格中已经有了
|
||
//filteredStarData.push(currentStarInfo);
|
||
// 增加计数器
|
||
if (videoData["标签"].includes("低成本")) {
|
||
lowCostCount++;
|
||
}
|
||
if (videoData["标签"].includes("爆量")) {
|
||
explosiveCount++;
|
||
}
|
||
}
|
||
} else {
|
||
// 如果视频ID不存在，则新增记录
|
||
filteredVideoData.push(videoData);
|
||
filteredStarData.push(currentStarInfo);
|
||
// 增加计数器
|
||
if (videoData["标签"].includes("低成本")) {
|
||
lowCostCount++;
|
||
}
|
||
if (videoData["标签"].includes("爆量")) {
|
||
explosiveCount++;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 错误处理：如果没有满足条件的视频，记录日志
|
||
if (filteredVideoData.length === 0) {
|
||
console.warn("No videos met the criteria.");
|
||
return;
|
||
}
|
||
|
||
const videoRecordsToUpload = filteredVideoData.map((data) => ({
|
||
fields: data,
|
||
}));
|
||
const starRecordsToUpload = filteredStarData.map((data) => ({
|
||
fields: data,
|
||
}));
|
||
|
||
// 4. 将两个json数组的结果，分别写入达人表和视频表
|
||
await feishuAPI.batchCreateBitableRecords(
|
||
CONFIG.feishu.videoTableId,
|
||
videoRecordsToUpload
|
||
);
|
||
await feishuAPI.batchCreateBitableRecords(
|
||
CONFIG.feishu.starTableId,
|
||
starRecordsToUpload
|
||
);
|
||
|
||
// 在数据成功写入多维表格后调用
|
||
await sendStatisticsToWebhook(lowCostCount, explosiveCount);
|
||
|
||
console.log("Data successfully uploaded to Bitable.");
|
||
} catch (error) {
|
||
console.error("Failed to process and upload data: ", error);
|
||
}
|
||
}
|
||
|
||
// 自动启动脚本
|
||
(async () => {
|
||
await processAndUploadData();
|
||
})();
|