402 lines
21 KiB
C#
402 lines
21 KiB
C#
using System.Diagnostics;
|
|
using System.Net;
|
|
using System.Net.Sockets;
|
|
using System.Text.Json;
|
|
using Dada.Supervisor;
|
|
|
|
internal static class Program
|
|
{
|
|
private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true };
|
|
|
|
[STAThread]
|
|
private static async Task<int> Main(string[] args)
|
|
{
|
|
if (args.FirstOrDefault() == "--credential-child")
|
|
{
|
|
return await RunCredentialChildAsync();
|
|
}
|
|
|
|
if (args.FirstOrDefault() == "--credential-echo")
|
|
{
|
|
Console.Write(await Console.In.ReadToEndAsync());
|
|
return 0;
|
|
}
|
|
|
|
if (args.FirstOrDefault() == "--instance-probe")
|
|
{
|
|
using var instance = await SingleInstanceCoordinator.TryAcquireAsync(args[1], args[2]);
|
|
Console.Write(instance.IsPrimary ? "primary" : "secondary");
|
|
return 0;
|
|
}
|
|
|
|
if (args.FirstOrDefault() == "--managed-child")
|
|
{
|
|
return await RunManagedChildAsync(args);
|
|
}
|
|
|
|
try
|
|
{
|
|
var security = await TestCredentialBoundaryAsync();
|
|
var supervisor = await TestSupervisorLifecycleAsync();
|
|
TestSecureConfigurationPersistence();
|
|
TestStructuredLogging();
|
|
WriteEvidence(Environment.GetEnvironmentVariable("DADA_EVIDENCE_DIR_SEC"), security);
|
|
WriteEvidence(Environment.GetEnvironmentVariable("DADA_EVIDENCE_DIR_SUP"), supervisor);
|
|
Console.WriteLine(JsonSerializer.Serialize(new { security = "passed", supervisor = "passed" }, JsonOptions));
|
|
return 0;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
Console.Error.WriteLine(exception);
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
private static void TestStructuredLogging()
|
|
{
|
|
Equal(10 * 1024 * 1024L, StructuredJsonlLogger.SizeLimitBytes, "Supervisor log byte limit");
|
|
Equal(10, StructuredJsonlLogger.FileLimit, "Supervisor log file limit");
|
|
Equal(30, StructuredJsonlLogger.RetentionDays, "Supervisor log retention");
|
|
var directory = Path.Combine(Path.GetTempPath(), $"dada-supervisor-log-{Guid.NewGuid():N}");
|
|
try
|
|
{
|
|
var logger = new StructuredJsonlLogger(directory, "supervisor", sizeLimitBytes: 512, fileLimit: 10);
|
|
for (var index = 0; index < 40; index++)
|
|
{
|
|
logger.Write(new StructuredLogEvent("completed", $"corr_{index}", $"obj_{index}", index, "none"));
|
|
}
|
|
var files = Directory.GetFiles(directory, "*.jsonl");
|
|
True(files.Length <= StructuredJsonlLogger.FileLimit, "Supervisor log file cap");
|
|
True(files.All(path => new FileInfo(path).Length <= 512), "Supervisor log rotation byte boundary");
|
|
foreach (var line in files.SelectMany(File.ReadLines))
|
|
{
|
|
using var document = JsonDocument.Parse(line);
|
|
var keys = document.RootElement.EnumerateObject().Select(property => property.Name).ToHashSet(StringComparer.Ordinal);
|
|
True(keys.IsSubsetOf(["schema_version", "timestamp", "component", "status_category", "correlation_id", "object_id", "duration_ms", "error_category"]), "Supervisor log field allowlist");
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
if (Directory.Exists(directory)) Directory.Delete(directory, recursive: true);
|
|
}
|
|
}
|
|
|
|
private static void TestSecureConfigurationPersistence()
|
|
{
|
|
var directory = Path.Combine(Path.GetTempPath(), $"dada-secure-config-{Guid.NewGuid():N}");
|
|
var path = Path.Combine(directory, "instance.json");
|
|
var email = "synthetic-admin@example.invalid";
|
|
var pepper = $"synthetic-pepper-{Guid.NewGuid():N}";
|
|
var digest = Convert.ToHexString(System.Security.Cryptography.HMACSHA256.HashData(
|
|
System.Text.Encoding.UTF8.GetBytes(pepper),
|
|
System.Text.Encoding.UTF8.GetBytes(email)));
|
|
try
|
|
{
|
|
var store = new InstanceConfigurationStore(path);
|
|
store.Save(new InstanceConfiguration(1, 7, null, null, [digest], [digest]));
|
|
var raw = File.ReadAllText(path);
|
|
True(raw.Contains("\"secure_config_revision\": 7", StringComparison.Ordinal), "secure config revision field");
|
|
True(raw.Contains("\"schema_version\": 1", StringComparison.Ordinal), "secure config schema field");
|
|
False(raw.Contains(email, StringComparison.OrdinalIgnoreCase), "allowlist email persisted");
|
|
False(raw.Contains(pepper, StringComparison.Ordinal), "admin pepper persisted");
|
|
False(File.Exists(path + ".tmp"), "secure config temporary file residual");
|
|
var loaded = store.Load();
|
|
Equal(7, loaded.Revision, "secure config revision round trip");
|
|
EqualSequence(new[] { digest }, loaded.AdminRecoveryHashes, "secure config recovery marker");
|
|
}
|
|
finally
|
|
{
|
|
if (Directory.Exists(directory)) Directory.Delete(directory, recursive: true);
|
|
}
|
|
}
|
|
|
|
private static async Task<object> TestCredentialBoundaryAsync()
|
|
{
|
|
var store = new TestCredentialStore();
|
|
var marker = $"wp0-{Guid.NewGuid():N}";
|
|
store.Write(CredentialCatalog.ApiResend, marker + "-mail");
|
|
store.Write(CredentialCatalog.ApiAmap, marker + "-map");
|
|
store.Write(CredentialCatalog.AdminPepper, marker + "-admin");
|
|
store.Write(CredentialCatalog.WorkerAiGateway, marker + "-ai");
|
|
|
|
var apiProbe = await LaunchCredentialProbeAsync(ChildRole.Api, store);
|
|
EqualSequence(new[] { CredentialCatalog.AdminPepper, CredentialCatalog.ApiAmap, CredentialCatalog.ApiResend }, apiProbe.Names.Order().ToArray(), "API credential scope");
|
|
False(apiProbe.EnvironmentContainsMarker, "credential leaked into child environment");
|
|
False(apiProbe.ArgumentsContainMarker, "credential leaked into child arguments");
|
|
|
|
var workerProbe = await LaunchCredentialProbeAsync(ChildRole.Worker, store);
|
|
EqualSequence(new[] { CredentialCatalog.WorkerAiGateway }, workerProbe.Names, "Worker credential scope");
|
|
var leakProbe = await CredentialProcessLauncher.RunToCompletionAsync(
|
|
new ProcessStartInfo(Environment.ProcessPath!, "--credential-echo"), ChildRole.Worker, store);
|
|
True(leakProbe.SensitiveOutputDetected, "credential echo must be detected");
|
|
Equal(string.Empty, leakProbe.StandardOutput, "credential echo output discarded");
|
|
Equal(string.Empty, leakProbe.StandardError, "credential echo error discarded");
|
|
|
|
var externalArguments = new[]
|
|
{
|
|
"--service", "ai-gateway-service-id",
|
|
"--model", "gpt-image-2",
|
|
"--run-id", "wp7-02-supervisor-probe",
|
|
"--max-real-calls", "120",
|
|
"--confirm-controlled-real",
|
|
"--execute-controlled-real",
|
|
};
|
|
EqualSequence(externalArguments, ControlledExternalValidationLauncher.ValidateArguments(externalArguments), "controlled external argument allowlist");
|
|
Throws<ArgumentException>(
|
|
() => ControlledExternalValidationLauncher.ValidateArguments(externalArguments.Where(value => value != "--confirm-controlled-real").ToArray()),
|
|
"controlled external confirmation required");
|
|
|
|
store.Delete(CredentialCatalog.WorkerAiGateway);
|
|
await ThrowsAsync<MissingCredentialException>(
|
|
() => LaunchCredentialProbeAsync(ChildRole.Worker, store),
|
|
"cleared credential must block the next child start");
|
|
|
|
var report = DiagnosticReport.Create(
|
|
SupervisorState.WorkerDegraded,
|
|
new[]
|
|
{
|
|
DiagnosticCheck.Pass("fixed_port", "loopback_ready"),
|
|
DiagnosticCheck.Warning("worker_health", "worker_unavailable"),
|
|
},
|
|
new DiagnosticStorageSummary(1024, 2048, 512),
|
|
new[] { new CredentialStatus("worker_ai_gateway", false) });
|
|
var diagnosticJson = JsonSerializer.Serialize(report, JsonOptions);
|
|
False(diagnosticJson.Contains(marker, StringComparison.Ordinal), "diagnostic report contains credential data");
|
|
False(diagnosticJson.Contains("stack", StringComparison.OrdinalIgnoreCase), "diagnostic report exposes stack data");
|
|
|
|
return new
|
|
{
|
|
processEnvironmentScan = new
|
|
{
|
|
api = apiProbe,
|
|
worker = workerProbe,
|
|
marker_absent = true,
|
|
status = "passed",
|
|
},
|
|
pipeAcl = new
|
|
{
|
|
channel = "inherited_anonymous_standard_input",
|
|
child_only = true,
|
|
one_shot = true,
|
|
status = "passed",
|
|
},
|
|
redaction = new
|
|
{
|
|
allowed_fields = report.GetType().GetProperties().Select(property => property.Name).Order().ToArray(),
|
|
marker_absent = true,
|
|
status = "passed",
|
|
},
|
|
};
|
|
}
|
|
|
|
private static async Task<CredentialProbe> LaunchCredentialProbeAsync(ChildRole role, ICredentialStore store)
|
|
{
|
|
var result = await CredentialProcessLauncher.RunToCompletionAsync(new ProcessStartInfo(Environment.ProcessPath!, "--credential-child"), role, store);
|
|
Equal(0, result.ExitCode, "credential child exit code");
|
|
False(result.SensitiveOutputDetected, "credential child output contains injected value");
|
|
return JsonSerializer.Deserialize<CredentialProbe>(result.StandardOutput, new JsonSerializerOptions { PropertyNameCaseInsensitive = true })
|
|
?? throw new InvalidOperationException("Credential child returned invalid JSON.");
|
|
}
|
|
|
|
private static async Task<int> RunCredentialChildAsync()
|
|
{
|
|
using var input = new StreamReader(Console.OpenStandardInput());
|
|
var json = await input.ReadToEndAsync();
|
|
using var document = JsonDocument.Parse(json);
|
|
var names = document.RootElement.EnumerateObject().Select(property => property.Name).Order().ToArray();
|
|
var values = document.RootElement.EnumerateObject().Select(property => property.Value.GetString() ?? string.Empty).ToArray();
|
|
var markerPrefix = values.FirstOrDefault()?.Split('-').Take(2).Aggregate((left, right) => left + "-" + right) ?? string.Empty;
|
|
var environmentContainsMarker = !string.IsNullOrEmpty(markerPrefix) && Environment.GetEnvironmentVariables().Values.Cast<object?>().Any(value => value?.ToString()?.Contains(markerPrefix, StringComparison.Ordinal) == true);
|
|
var argumentsContainMarker = !string.IsNullOrEmpty(markerPrefix) && Environment.GetCommandLineArgs().Any(value => value.Contains(markerPrefix, StringComparison.Ordinal));
|
|
Console.Write(JsonSerializer.Serialize(new CredentialProbe(names, environmentContainsMarker, argumentsContainMarker)));
|
|
return 0;
|
|
}
|
|
|
|
private static async Task<object> TestSupervisorLifecycleAsync()
|
|
{
|
|
EqualSequence(new[] { 1, 5, 15 }, RestartPolicy.Delays.Select(delay => (int)delay.TotalSeconds).ToArray(), "restart backoff");
|
|
Equal(3, RestartPolicy.MaximumRestarts, "restart limit");
|
|
False(RestartPolicy.CanRestart(3, TimeSpan.FromMinutes(1)), "fourth restart must be blocked");
|
|
|
|
using var occupied = new TcpListener(IPAddress.Loopback, LoopbackEndpoint.Port);
|
|
occupied.Start();
|
|
var portState = LoopbackPortGuard.Check();
|
|
Equal(SupervisorState.PortInUse, portState, "fixed occupied port state");
|
|
occupied.Stop();
|
|
Equal(SupervisorState.Starting, LoopbackPortGuard.Check(), "fixed free port state");
|
|
|
|
var mutexName = $"Dada.P0A.Instance.Tests.{Guid.NewGuid():N}";
|
|
var pipeName = $"Dada.P0A.Open.Tests.{Guid.NewGuid():N}";
|
|
using var primary = await SingleInstanceCoordinator.TryAcquireAsync(mutexName, pipeName);
|
|
True(primary.IsPrimary, "first instance must be primary");
|
|
var notification = primary.WaitForOpenRequestAsync(TimeSpan.FromSeconds(5));
|
|
using var secondary = Process.Start(new ProcessStartInfo(Environment.ProcessPath!, $"--instance-probe {mutexName} {pipeName}")
|
|
{
|
|
RedirectStandardOutput = true,
|
|
UseShellExecute = false,
|
|
}) ?? throw new InvalidOperationException("Unable to start second-instance probe.");
|
|
var probeResult = await secondary.StandardOutput.ReadToEndAsync();
|
|
await secondary.WaitForExitAsync();
|
|
Equal("secondary", probeResult, "second instance result");
|
|
True(await notification, "primary did not receive second-instance open request");
|
|
|
|
var childStore = new TestCredentialStore();
|
|
childStore.Write(CredentialCatalog.ApiResend, $"probe-{Guid.NewGuid():N}-mail");
|
|
childStore.Write(CredentialCatalog.ApiAmap, $"probe-{Guid.NewGuid():N}-map");
|
|
childStore.Write(CredentialCatalog.AdminPepper, $"probe-{Guid.NewGuid():N}-admin");
|
|
var childStart = new ProcessStartInfo(Environment.ProcessPath!);
|
|
childStart.ArgumentList.Add("--managed-child");
|
|
var managed = await ManagedChildProcess.StartAsync(childStart, ChildRole.Api, childStore);
|
|
var managedPid = managed.Process.Id;
|
|
False(managed.Process.HasExited, "managed child should be running after ready");
|
|
await managed.StopAsync();
|
|
True(managed.Process.HasExited, "managed child did not stop through the control pipe");
|
|
await managed.DisposeAsync();
|
|
var residual = Process.GetProcesses().Count(process => process.Id == managedPid);
|
|
Equal(0, residual, "managed child residual process");
|
|
|
|
var readyActions = SupervisorActions.For(SupervisorState.Ready);
|
|
True(readyActions.Contains(SupervisorAction.OpenProduct), "ready state open action");
|
|
var failedActions = SupervisorActions.For(SupervisorState.StartupFailed);
|
|
EqualSequence(new[] { SupervisorAction.Retry, SupervisorAction.OpenDiagnostics, SupervisorAction.Exit }, failedActions, "startup failure actions");
|
|
False(SupervisorActions.For(SupervisorState.StorageFull).Contains(SupervisorAction.RestartServices), "storage-full state must not offer service restart");
|
|
False(SupervisorActions.For(SupervisorState.StorageUnavailable).Contains(SupervisorAction.OpenProduct), "storage-unavailable state must not open the product");
|
|
|
|
using (var menuForm = new SupervisorForm(SupervisorState.Ready))
|
|
{
|
|
EqualSequence(new[] { "打开 Dada", "运行状态", "打开诊断", "重新启动服务", "退出 Dada" }, menuForm.TrayMenuLabels, "tray menu order");
|
|
}
|
|
using (var diagnostics = new DiagnosticsForm(SupervisorState.StorageUnavailable))
|
|
{
|
|
True(diagnostics.CopyPayload.Contains("log_write_failed", StringComparison.Ordinal), "diagnostic log status");
|
|
False(diagnostics.CopyPayload.Contains("CredentialBlob", StringComparison.Ordinal), "diagnostic credential redaction");
|
|
}
|
|
|
|
var screenshotPath = Path.Combine(Environment.GetEnvironmentVariable("DADA_EVIDENCE_DIR_SUP") ?? Path.GetTempPath(), "screenshots", "system-ui.png");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(screenshotPath)!);
|
|
CaptureWindow(new SupervisorForm(SupervisorState.WorkerDegraded), screenshotPath);
|
|
CaptureWindow(new SupervisorForm(SupervisorState.Starting), Path.Combine(Path.GetDirectoryName(screenshotPath)!, "starting.png"));
|
|
CaptureWindow(new SupervisorForm(SupervisorState.StartupFailed), Path.Combine(Path.GetDirectoryName(screenshotPath)!, "startup-failed.png"));
|
|
CaptureWindow(new SupervisorForm(SupervisorState.PortInUse), Path.Combine(Path.GetDirectoryName(screenshotPath)!, "port-in-use.png"));
|
|
CaptureWindow(new SupervisorForm(SupervisorState.ApiDegraded), Path.Combine(Path.GetDirectoryName(screenshotPath)!, "api-degraded.png"));
|
|
CaptureWindow(new SupervisorForm(SupervisorState.StorageFull), Path.Combine(Path.GetDirectoryName(screenshotPath)!, "storage-full.png"));
|
|
CaptureWindow(new SupervisorForm(SupervisorState.StorageUnavailable), Path.Combine(Path.GetDirectoryName(screenshotPath)!, "storage-unavailable.png"));
|
|
CaptureWindow(new DiagnosticsForm(SupervisorState.WorkerDegraded), Path.Combine(Path.GetDirectoryName(screenshotPath)!, "diagnostics.png"));
|
|
|
|
return new
|
|
{
|
|
port = new { host = LoopbackEndpoint.Host, port = LoopbackEndpoint.Port, alternate_port_attempted = false, status = "passed" },
|
|
processTree = new { second_instance = "rejected_and_notified", managed_child_pid = managedPid, residual_processes = residual, shutdown_deadline_seconds = (int)ManagedChildProcess.ShutdownDeadline.TotalSeconds, status = "passed" },
|
|
supervisorEvents = new { restart_delays_seconds = new[] { 1, 5, 15 }, maximum_restarts = 3, system_ui_states = new[] { "starting", "ready", "startup_failed", "port_in_use", "api_degraded", "worker_degraded", "storage_full", "storage_unavailable", "diagnostics_running", "diagnostics_ready" }, terminal_state = "worker_degraded", status = "passed" },
|
|
};
|
|
}
|
|
|
|
private static void CaptureWindow(Form form, string path)
|
|
{
|
|
using (form)
|
|
using (var bitmap = new Bitmap(form.ClientSize.Width, form.ClientSize.Height))
|
|
{
|
|
form.Show();
|
|
Application.DoEvents();
|
|
form.DrawToBitmap(bitmap, form.ClientRectangle);
|
|
bitmap.Save(path);
|
|
form.Hide();
|
|
}
|
|
}
|
|
|
|
private static async Task<int> RunManagedChildAsync(string[] args)
|
|
{
|
|
using var input = new StreamReader(Console.OpenStandardInput());
|
|
var payload = await input.ReadToEndAsync();
|
|
using var credentials = JsonDocument.Parse(payload);
|
|
var pipeIndex = Array.IndexOf(args, "--dada-control-pipe");
|
|
if (pipeIndex < 0 || pipeIndex + 1 >= args.Length) return 4;
|
|
await using var pipe = new System.IO.Pipes.NamedPipeClientStream(".", args[pipeIndex + 1], System.IO.Pipes.PipeDirection.InOut, System.IO.Pipes.PipeOptions.Asynchronous);
|
|
await pipe.ConnectAsync(5000);
|
|
using var reader = new StreamReader(pipe, leaveOpen: true);
|
|
await using var writer = new StreamWriter(pipe, leaveOpen: true) { AutoFlush = true };
|
|
await writer.WriteLineAsync("ready");
|
|
var command = await reader.ReadLineAsync();
|
|
return command == "shutdown" ? 0 : 5;
|
|
}
|
|
|
|
private static void WriteEvidence(string? directory, object result)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(directory)) return;
|
|
Directory.CreateDirectory(directory);
|
|
var root = JsonSerializer.SerializeToElement(result, JsonOptions);
|
|
foreach (var property in root.EnumerateObject())
|
|
{
|
|
var fileName = property.Name switch
|
|
{
|
|
"processEnvironmentScan" => "process-env-scan.json",
|
|
"pipeAcl" => "pipe-acl.json",
|
|
"processTree" => "process-tree.json",
|
|
"supervisorEvents" => "supervisor-events.json",
|
|
_ => property.Name + ".json",
|
|
};
|
|
File.WriteAllText(Path.Combine(directory, fileName), JsonSerializer.Serialize(property.Value, JsonOptions) + Environment.NewLine);
|
|
}
|
|
}
|
|
|
|
private static void True(bool value, string message)
|
|
{
|
|
if (!value) throw new InvalidOperationException(message);
|
|
}
|
|
|
|
private static void False(bool value, string message) => True(!value, message);
|
|
|
|
private static void Equal<T>(T expected, T actual, string message) where T : notnull
|
|
{
|
|
if (!EqualityComparer<T>.Default.Equals(expected, actual))
|
|
{
|
|
throw new InvalidOperationException($"{message}: expected {expected}, got {actual}");
|
|
}
|
|
}
|
|
|
|
private static void EqualSequence<T>(IReadOnlyList<T> expected, IReadOnlyList<T> actual, string message)
|
|
{
|
|
if (!expected.SequenceEqual(actual))
|
|
{
|
|
throw new InvalidOperationException($"{message}: expected [{string.Join(",", expected)}], got [{string.Join(",", actual)}]");
|
|
}
|
|
}
|
|
|
|
private static async Task ThrowsAsync<TException>(Func<Task> action, string message) where TException : Exception
|
|
{
|
|
try
|
|
{
|
|
await action();
|
|
}
|
|
catch (TException)
|
|
{
|
|
return;
|
|
}
|
|
throw new InvalidOperationException(message);
|
|
}
|
|
|
|
private static void Throws<TException>(Action action, string message) where TException : Exception
|
|
{
|
|
try
|
|
{
|
|
action();
|
|
}
|
|
catch (TException)
|
|
{
|
|
return;
|
|
}
|
|
throw new InvalidOperationException(message);
|
|
}
|
|
|
|
private sealed record CredentialProbe(string[] Names, bool EnvironmentContainsMarker, bool ArgumentsContainMarker);
|
|
|
|
private sealed class TestCredentialStore : ICredentialStore
|
|
{
|
|
private readonly Dictionary<string, string> values = new(StringComparer.Ordinal);
|
|
public void Delete(string target) => values.Remove(target);
|
|
public bool IsConfigured(string target) => values.ContainsKey(target);
|
|
public string? Read(string target) => values.GetValueOrDefault(target);
|
|
public void Write(string target, string value) => values[target] = value;
|
|
}
|
|
}
|