This commit is contained in:
2026-06-25 18:02:16 -05:00
parent 402bf6782a
commit da16e5ff56
17 changed files with 1475 additions and 69 deletions

View File

@@ -14,6 +14,8 @@ import { path } from '../utils/pathUtils.js';
import { analyzePowerShellCompatibility, compatibilityLogLines } from './powershellCompatibilityService.js';
import { normalizeOsFamily } from '../utils/osFamily.js';
const activeJobs = new Map();
// The execution kill-switch can be set at boot via ALLOW_SCRIPT_EXECUTION and
// flipped at runtime via the Config screen (allow_script_execution). The DB
// value, when present, takes precedence so the UI toggle is actually effective.
@@ -181,16 +183,33 @@ export function executeScriptTest(scriptId, payload, userId) {
async function runJob(jobId, script, hosts, parallel, executionContext) {
// RunPlans can fan out in parallel or execute serially while preserving identical log semantics.
logCompatibilityPreflight(jobId, script, hosts);
const runner = async (host) => runHost(jobId, script, host, executionContext);
if (parallel) {
await Promise.all(hosts.map(runner));
} else {
for (const host of hosts) await runner(host);
}
const state = { canceled: false, children: new Map() };
activeJobs.set(jobId, state);
try {
logCompatibilityPreflight(jobId, script, hosts);
const runner = async (host) => runHost(jobId, script, host, executionContext);
if (parallel) {
await Promise.all(hosts.map(runner));
} else {
for (const host of hosts) {
if (isJobCanceled(jobId)) {
cancelQueuedHosts(jobId);
break;
}
await runner(host);
}
}
const failed = db.prepare('SELECT COUNT(*) AS count FROM job_hosts WHERE job_id = ? AND status = ?').get(jobId, 'failed').count;
db.prepare('UPDATE jobs SET status = ?, ended_at = ? WHERE id = ?').run(failed ? 'failed' : 'completed', now(), jobId);
if (isJobCanceled(jobId)) {
cancelQueuedHosts(jobId);
db.prepare('UPDATE jobs SET status = ?, ended_at = COALESCE(ended_at, ?) WHERE id = ?').run('canceled', now(), jobId);
return;
}
const failed = db.prepare('SELECT COUNT(*) AS count FROM job_hosts WHERE job_id = ? AND status = ?').get(jobId, 'failed').count;
db.prepare('UPDATE jobs SET status = ?, ended_at = ? WHERE id = ?').run(failed ? 'failed' : 'completed', now(), jobId);
} finally {
activeJobs.delete(jobId);
}
}
function logCompatibilityPreflight(jobId, script, hosts) {
@@ -204,6 +223,10 @@ function logCompatibilityPreflight(jobId, script, hosts) {
async function runHost(jobId, script, host, executionContext) {
// Each host owns its own temp wrapper and job_host lifecycle for per-target evidence.
if (isJobCanceled(jobId)) {
finishHost(jobId, host.id, 'canceled', 130);
return;
}
db.prepare('UPDATE job_hosts SET status = ?, started_at = ? WHERE job_id = ? AND host_id = ?').run('running', now(), jobId, host.id);
appendLog(jobId, host.id, 'system', `Starting ${script.name} on ${host.name} (${host.transport})`);
appendLog(jobId, host.id, 'system', `Target OS type: ${normalizeOsFamily(host.os_family || host.osFamily, 'other')}`);
@@ -233,6 +256,7 @@ async function runHost(jobId, script, host, executionContext) {
async function vCenterPowerPreflight(jobId, host) {
if ((host.source_type || 'manual') !== 'vmware') return true;
if (isJobCanceled(jobId)) return false;
if (!host.source_connection_id || !host.source_ref) {
appendLog(jobId, host.id, 'system', 'Host is marked as VMware-sourced, but no vCenter connection or VM reference is stored. Continuing without a power-state gate.');
return true;
@@ -328,11 +352,22 @@ function spawnPowerShell(jobId, host, file, secret, executionContext = {}) {
},
shell: false
});
const active = activeJobs.get(jobId);
if (active) {
active.children.set(host.id, child);
if (active.canceled) {
child.kill('SIGTERM');
setTimeout(() => {
if (!child.killed) child.kill('SIGKILL');
}, 5000).unref?.();
}
}
let settled = false;
function settle(status, exitCode) {
if (settled) return;
settled = true;
activeJobs.get(jobId)?.children.delete(host.id);
finishHost(jobId, host.id, status, exitCode);
resolve();
}
@@ -346,6 +381,10 @@ function spawnPowerShell(jobId, host, file, secret, executionContext = {}) {
child.on('close', (code, signal) => {
if (settled) return;
if (signal) appendLog(jobId, host.id, 'stderr', `PowerShell terminated by signal ${signal}`);
if (isJobCanceled(jobId)) {
settle('canceled', 130);
return;
}
const exitCode = code ?? 1;
settle(exitCode === 0 ? 'completed' : 'failed', exitCode);
});
@@ -366,6 +405,10 @@ function appendLog(jobId, hostId, stream, line) {
`).run(id('log'), jobId, hostId, stream, line, now());
}
export function appendJobLog(jobId, stream, line, hostId = null) {
appendLog(jobId, hostId, stream, line);
}
function finishHost(jobId, hostId, status, exitCode) {
db.prepare(`
UPDATE job_hosts SET status = ?, exit_code = ?, ended_at = ?
@@ -374,6 +417,52 @@ function finishHost(jobId, hostId, status, exitCode) {
appendLog(jobId, hostId, 'system', `Finished with status ${status} and exit code ${exitCode}`);
}
function cancelQueuedHosts(jobId) {
const timestamp = now();
const rows = db.prepare('SELECT host_id FROM job_hosts WHERE job_id = ? AND status IN (?, ?)').all(jobId, 'queued', 'running');
db.prepare(`
UPDATE job_hosts
SET status = 'canceled', exit_code = COALESCE(exit_code, 130), ended_at = COALESCE(ended_at, ?)
WHERE job_id = ? AND status IN ('queued', 'running')
`).run(timestamp, jobId);
for (const row of rows) appendLog(jobId, row.host_id, 'system', 'Host execution canceled by job management.');
}
function isJobCanceled(jobId) {
if (activeJobs.get(jobId)?.canceled) return true;
const row = db.prepare('SELECT status FROM jobs WHERE id = ?').get(jobId);
return row?.status === 'canceled';
}
export function activeExecutionJobIds() {
return [...activeJobs.keys()];
}
export function cancelExecutionJob(jobId, { reason = '', actorUserId = '' } = {}) {
const job = db.prepare('SELECT * FROM jobs WHERE id = ?').get(jobId);
if (!job) return { canceled: false, reason: 'Job not found.' };
if (!['queued', 'running'].includes(job.status)) {
return { canceled: false, reason: `Job is ${job.status} and cannot be canceled.` };
}
const active = activeJobs.get(jobId);
if (active) active.canceled = true;
const children = active ? [...active.children.values()] : [];
appendLog(jobId, null, 'system', `Cancel requested${actorUserId ? ` by ${actorUserId}` : ''}${reason ? `: ${reason}` : '.'}`);
db.prepare('UPDATE jobs SET status = ?, ended_at = COALESCE(ended_at, ?) WHERE id = ?').run('canceled', now(), jobId);
cancelQueuedHosts(jobId);
for (const child of children) {
try {
child.kill('SIGTERM');
setTimeout(() => {
if (!child.killed) child.kill('SIGKILL');
}, 5000).unref?.();
} catch (error) {
logger.warn('failed to signal child process during job cancel', { jobId, error: error.message });
}
}
return { canceled: true, signaledProcesses: children.length };
}
function runtimeVariableValues(host, executionContext) {
const assetManifest = JSON.stringify(executionContext.assets || []);
return [