intelligrow/scriptCat
15
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: improve yuntu report sync flow

Browse Source
This commit is contained in:
wxs
2026-04-13 21:08:30 +08:00
parent c8b5d27078
commit bcc5afa291
8 changed files with 383 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yuntu/yuntuReportFilling/server/sql/schema.sql
+9 -53
View File
@@ -1,55 +1,11 @@
DROP TABLE IF EXISTS public.yuntu_report_info;
DROP TABLE IF EXISTS public.segmented_market_reports;
CREATE TABLE public.yuntu_report_info (
id BIGSERIAL PRIMARY KEY,
report_id VARCHAR(64) NOT NULL,
source_type VARCHAR(32) NOT NULL,
source_report_id VARCHAR(64) NULL,
aadvid VARCHAR(32) NOT NULL,
name TEXT NOT NULL,
price JSONB NOT NULL DEFAULT '[]'::jsonb,
rules JSONB NOT NULL DEFAULT '[]'::jsonb,
analysis_dims JSONB NOT NULL DEFAULT '[]'::jsonb,
categories JSONB NOT NULL DEFAULT '[]'::jsonb,
transaction_channels JSONB NOT NULL DEFAULT '[]'::jsonb,
start_time DATE NOT NULL,
end_time DATE NOT NULL,
period_type VARCHAR(32) NULL,
user_name TEXT NULL,
payload JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT uq_yuntu_report_info_report_id UNIQUE (report_id),
CONSTRAINT chk_yuntu_report_info_source_type
CHECK (source_type IN ('MANUAL_CAPTURE', 'AUTO_COPY')),
CONSTRAINT chk_yuntu_report_info_copy_source
CHECK (
(source_type = 'MANUAL_CAPTURE' AND source_report_id IS NULL)
OR
(source_type = 'AUTO_COPY' AND source_report_id IS NOT NULL)
)
CREATE TABLE public.segmented_market_reports (
report_id TEXT NOT NULL,
report_info JSONB NOT NULL,
completion_notified_at TIMESTAMP WITH TIME ZONE NULL,
details_synced_at TIMESTAMP WITH TIME ZONE NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
CONSTRAINT segmented_market_reports_pkey PRIMARY KEY (report_id)
);
CREATE INDEX idx_yuntu_report_info_source_report_id
ON public.yuntu_report_info (source_report_id);
CREATE INDEX idx_yuntu_report_info_created_at
ON public.yuntu_report_info (created_at DESC);
CREATE INDEX idx_yuntu_report_info_aadvid_created_at
ON public.yuntu_report_info (aadvid, created_at DESC);
CREATE OR REPLACE FUNCTION public.set_yuntu_report_info_updated_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS trg_yuntu_report_info_updated_at ON public.yuntu_report_info;
CREATE TRIGGER trg_yuntu_report_info_updated_at
BEFORE UPDATE ON public.yuntu_report_info
FOR EACH ROW
EXECUTE FUNCTION public.set_yuntu_report_info_updated_at();
yuntu/yuntuReportFilling/server/src/report-repository.js
+7 -37
View File
@@ -1,33 +1,18 @@
const INSERT_REPORT_SQL = `
INSERT INTO public.yuntu_report_info (
INSERT INTO public.segmented_market_reports (
report_id,
source_type,
source_report_id,
aadvid,
name,
price,
rules,
analysis_dims,
categories,
transaction_channels,
start_time,
end_time,
period_type,
user_name,
payload
report_info
)
VALUES (
$1, $2, $3, $4, $5,
$6::jsonb, $7::jsonb, $8::jsonb, $9::jsonb, $10::jsonb,
$11, $12, $13, $14, $15::jsonb
$1, $2::jsonb
)
ON CONFLICT (report_id) DO NOTHING
RETURNING id, report_id
RETURNING report_id
`;
const FIND_REPORT_SQL = `
SELECT id, report_id
FROM public.yuntu_report_info
SELECT report_id
FROM public.segmented_market_reports
WHERE report_id = $1
`;
@@ -36,20 +21,7 @@ function createRepository(pool) {
async save(record) {
const insertValues = [
record.report_id,
record.source_type,
record.source_report_id,
record.aadvid,
record.name,
JSON.stringify(record.price),
JSON.stringify(record.rules),
JSON.stringify(record.analysis_dims),
JSON.stringify(record.categories),
JSON.stringify(record.transaction_channels),
record.start_time,
record.end_time,
record.period_type,
record.user_name,
JSON.stringify(record.payload),
JSON.stringify(record.report_info),
];
const insertResult = await pool.query(INSERT_REPORT_SQL, insertValues);
@@ -57,7 +29,6 @@ function createRepository(pool) {
if (insertResult.rows.length > 0) {
const row = insertResult.rows[0];
return {
id: row.id,
reportId: row.report_id,
created: true,
};
@@ -71,7 +42,6 @@ function createRepository(pool) {
}
return {
id: existingRow.id,
reportId: existingRow.report_id,
created: false,
};
yuntu/yuntuReportFilling/server/src/report-service.js
+34 -76
View File
@@ -1,6 +1,3 @@
const ALLOWED_SOURCE_TYPES = new Set(['MANUAL_CAPTURE', 'AUTO_COPY']);
const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
class ValidationError extends Error {
constructor(message) {
super(message);
@@ -9,30 +6,6 @@ class ValidationError extends Error {
}
}
function ensureNonEmptyString(value, fieldName) {
if (typeof value !== 'string' || value.trim() === '') {
throw new ValidationError(`${fieldName} is required`);
}
return value.trim();
}
function ensureOptionalString(value, fieldName) {
if (value == null || value === '') {
return null;
}
return ensureNonEmptyString(value, fieldName);
}
function ensureArray(value, fieldName) {
if (!Array.isArray(value)) {
throw new ValidationError(`${fieldName} must be an array`);
}
return value;
}
function ensureObject(value, fieldName) {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new ValidationError(`${fieldName} must be an object`);
@@ -41,75 +14,60 @@ function ensureObject(value, fieldName) {
return value;
}
function ensureDateString(value, fieldName) {
const normalized = ensureNonEmptyString(value, fieldName);
if (!DATE_PATTERN.test(normalized)) {
throw new ValidationError(`${fieldName} must match YYYY-MM-DD`);
function normalizeReportId(value) {
if (typeof value === 'string' && value.trim() !== '') {
return value.trim();
}
return normalized;
if (typeof value === 'number' && Number.isFinite(value)) {
return String(value);
}
return null;
}
function extractReportId(reportInfo) {
const directReportId =
normalizeReportId(reportInfo.report_id) ||
normalizeReportId(reportInfo.reportId);
if (directReportId) {
return directReportId;
}
if (!reportInfo.data || typeof reportInfo.data !== 'object' || Array.isArray(reportInfo.data)) {
return null;
}
return (
normalizeReportId(reportInfo.data.report_id) ||
normalizeReportId(reportInfo.data.reportId)
);
}
function validateAndNormalizeReportInput(input) {
const payload = ensureObject(input, 'body');
const sourceType = ensureNonEmptyString(payload.sourceType, 'sourceType');
const reportInfo = ensureObject(input, 'body');
const reportId = extractReportId(reportInfo);
if (!ALLOWED_SOURCE_TYPES.has(sourceType)) {
throw new ValidationError('sourceType must be MANUAL_CAPTURE or AUTO_COPY');
}
const sourceReportId = ensureOptionalString(payload.sourceReportId, 'sourceReportId');
if (sourceType === 'MANUAL_CAPTURE' && sourceReportId !== null) {
throw new ValidationError('sourceReportId must be empty for MANUAL_CAPTURE');
}
if (sourceType === 'AUTO_COPY' && sourceReportId === null) {
throw new ValidationError('sourceReportId is required when sourceType is AUTO_COPY');
if (!reportId) {
throw new ValidationError('report_id is required in response info');
}
return {
reportId: ensureNonEmptyString(payload.reportId, 'reportId'),
sourceType,
sourceReportId,
aadvid: ensureNonEmptyString(payload.aadvid, 'aadvid'),
name: ensureNonEmptyString(payload.name, 'name'),
price: ensureArray(payload.price, 'price'),
rules: ensureArray(payload.rules, 'rules'),
analysisDims: ensureArray(payload.analysisDims, 'analysisDims'),
categories: ensureArray(payload.categories, 'categories'),
channels: ensureArray(payload.channels, 'channels'),
startTime: ensureDateString(payload.startTime, 'startTime'),
endTime: ensureDateString(payload.endTime, 'endTime'),
periodType: ensureOptionalString(payload.periodType, 'periodType'),
userName: ensureOptionalString(payload.userName, 'userName'),
payload: ensureObject(payload.payload, 'payload'),
reportId,
reportInfo,
};
}
function toDatabaseRecord(normalizedReport) {
return {
report_id: normalizedReport.reportId,
source_type: normalizedReport.sourceType,
source_report_id: normalizedReport.sourceReportId,
aadvid: normalizedReport.aadvid,
name: normalizedReport.name,
price: normalizedReport.price,
rules: normalizedReport.rules,
analysis_dims: normalizedReport.analysisDims,
categories: normalizedReport.categories,
transaction_channels: normalizedReport.channels,
start_time: normalizedReport.startTime,
end_time: normalizedReport.endTime,
period_type: normalizedReport.periodType,
user_name: normalizedReport.userName,
payload: normalizedReport.payload,
report_info: normalizedReport.reportInfo,
};
}
module.exports = {
ValidationError,
extractReportId,
validateAndNormalizeReportInput,
toDatabaseRecord,
};
yuntu/yuntuReportFilling/server/src/server.js
-1
View File
@@ -35,7 +35,6 @@ function createApp({ repository, allowedOrigin }) {
response.status(saved.created ? 201 : 200).json({
success: true,
data: {
id: saved.id,
reportId: saved.reportId,
created: saved.created,
},
yuntu/yuntuReportFilling/server/test/report-service.test.js
+38 -57
View File
@@ -6,90 +6,71 @@ const {
toDatabaseRecord,
} = require('../src/report-service');
function createManualPayload() {
function createResponseInfo() {
return {
reportId: 'report-001',
sourceType: 'MANUAL_CAPTURE',
sourceReportId: null,
aadvid: '1648829117232140',
name: '测试报告',
price: ['1,100', '101,100000'],
rules: [{ keywords: ['奶粉'], op: 'INCLUDE' }],
analysisDims: ['MARKETOVERVIEW'],
categories: [{ id: '20028', name: '奶粉类目' }],
channels: ['ALL'],
startTime: '2025-03-01',
endTime: '2026-02-28',
periodType: 'MONTH',
userName: 'tester@example.com',
payload: {
code: 0,
message: 'success',
data: {
reportId: 'report-001',
status: 'SUCCESS',
name: '测试报告',
startTime: '2025-03-01',
endTime: '2026-02-28',
},
};
}
test('validateAndNormalizeReportInput accepts a valid manual report payload', () => {
const input = createManualPayload();
test('validateAndNormalizeReportInput accepts response info with data.reportId', () => {
const input = createResponseInfo();
const result = validateAndNormalizeReportInput(input);
assert.equal(result.reportId, 'report-001');
assert.equal(result.sourceType, 'MANUAL_CAPTURE');
assert.equal(result.sourceReportId, null);
assert.equal(result.aadvid, '1648829117232140');
assert.equal(result.startTime, '2025-03-01');
assert.equal(result.endTime, '2026-02-28');
assert.deepEqual(result.reportInfo, input);
});
test('validateAndNormalizeReportInput rejects AUTO_COPY without sourceReportId', () => {
test('validateAndNormalizeReportInput accepts response info with data.report_id', () => {
const input = {
...createManualPayload(),
reportId: 'report-002',
sourceType: 'AUTO_COPY',
sourceReportId: '',
code: 0,
data: {
report_id: 'report-002',
},
};
const result = validateAndNormalizeReportInput(input);
assert.equal(result.reportId, 'report-002');
assert.deepEqual(result.reportInfo, input);
});
test('validateAndNormalizeReportInput rejects response info without a report id', () => {
assert.throws(
() => validateAndNormalizeReportInput(input),
() =>
validateAndNormalizeReportInput({
code: 0,
data: {},
}),
(error) => {
assert.equal(error.code, 'VALIDATION_ERROR');
assert.match(error.message, /sourceReportId/i);
assert.match(error.message, /report[_ ]?id/i);
return true;
},
);
});
test('toDatabaseRecord maps API payload fields into database-ready values', () => {
const normalized = validateAndNormalizeReportInput({
...createManualPayload(),
reportId: 'report-003',
sourceType: 'AUTO_COPY',
sourceReportId: 'report-001',
});
test('toDatabaseRecord maps response info into segmented_market_reports columns', () => {
const normalized = validateAndNormalizeReportInput(createResponseInfo());
const record = toDatabaseRecord(normalized);
assert.deepEqual(record, {
report_id: 'report-003',
source_type: 'AUTO_COPY',
source_report_id: 'report-001',
aadvid: '1648829117232140',
name: '测试报告',
price: ['1,100', '101,100000'],
rules: [{ keywords: ['奶粉'], op: 'INCLUDE' }],
analysis_dims: ['MARKETOVERVIEW'],
categories: [{ id: '20028', name: '奶粉类目' }],
transaction_channels: ['ALL'],
start_time: '2025-03-01',
end_time: '2026-02-28',
period_type: 'MONTH',
user_name: 'tester@example.com',
payload: {
name: '测试报告',
startTime: '2025-03-01',
endTime: '2026-02-28',
report_id: 'report-001',
report_info: {
code: 0,
message: 'success',
data: {
reportId: 'report-001',
status: 'SUCCESS',
name: '测试报告',
},
},
});
});
yuntu/yuntuReportFilling/server/test/server.test.js
+16 -29
View File
@@ -3,26 +3,13 @@ const assert = require('node:assert/strict');
const { createApp } = require('../src/server');
function createManualPayload() {
function createResponseInfo() {
return {
reportId: 'report-001',
sourceType: 'MANUAL_CAPTURE',
sourceReportId: null,
aadvid: '1648829117232140',
name: '测试报告',
price: ['1,100', '101,100000'],
rules: [{ keywords: ['奶粉'], op: 'INCLUDE' }],
analysisDims: ['MARKETOVERVIEW'],
categories: [{ id: '20028', name: '奶粉类目' }],
channels: ['ALL'],
startTime: '2025-03-01',
endTime: '2026-02-28',
periodType: 'MONTH',
userName: 'tester@example.com',
payload: {
name: '测试报告',
startTime: '2025-03-01',
endTime: '2026-02-28',
code: 0,
message: 'success',
data: {
reportId: 'report-001',
status: 'SUCCESS',
},
};
}
@@ -78,7 +65,7 @@ test('GET /health returns ok status', async () => {
);
});
test('POST /api/reports returns 400 for invalid auto copy payload', async () => {
test('POST /api/reports returns 400 when response info does not contain a report id', async () => {
await withServer(
{
async save() {
@@ -92,9 +79,8 @@ test('POST /api/reports returns 400 for invalid auto copy payload', async () =>
'content-type': 'application/json',
},
body: JSON.stringify({
...createManualPayload(),
sourceType: 'AUTO_COPY',
sourceReportId: '',
code: 0,
data: {},
}),
});
const body = await response.json();
@@ -106,7 +92,7 @@ test('POST /api/reports returns 400 for invalid auto copy payload', async () =>
);
});
test('POST /api/reports persists a valid payload and returns created response', async () => {
test('POST /api/reports persists raw response info and returns created response', async () => {
let savedRecord = null;
await withServer(
@@ -114,19 +100,19 @@ test('POST /api/reports persists a valid payload and returns created response',
async save(record) {
savedRecord = record;
return {
id: 12,
reportId: record.report_id,
created: true,
};
},
},
async (baseUrl) => {
const responseInfo = createResponseInfo();
const response = await fetch(`${baseUrl}/api/reports`, {
method: 'POST',
headers: {
'content-type': 'application/json',
},
body: JSON.stringify(createManualPayload()),
body: JSON.stringify(responseInfo),
});
const body = await response.json();
@@ -134,13 +120,14 @@ test('POST /api/reports persists a valid payload and returns created response',
assert.deepEqual(body, {
success: true,
data: {
id: 12,
reportId: 'report-001',
created: true,
},
});
assert.equal(savedRecord.report_id, 'report-001');
assert.equal(savedRecord.source_type, 'MANUAL_CAPTURE');
assert.deepEqual(savedRecord, {
report_id: 'report-001',
report_info: responseInfo,
});
},
);
});