Initial
This commit is contained in:
307
server/services/jobOutputService.js
Normal file
307
server/services/jobOutputService.js
Normal file
@@ -0,0 +1,307 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import PDFDocument from 'pdfkit';
|
||||
import { config } from '../config.js';
|
||||
import { db, id, json, now } from '../db.js';
|
||||
import { path, sanitizeFileName } from '../utils/pathUtils.js';
|
||||
import { logger } from './logger.js';
|
||||
|
||||
const terminalStatuses = new Set(['completed', 'failed', 'canceled']);
|
||||
|
||||
function stripAnsi(value = '') {
|
||||
return String(value).replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, '');
|
||||
}
|
||||
|
||||
function artifactDir(jobId) {
|
||||
return path.join(config.fileLockerDir, 'job-output', sanitizeFileName(jobId, 'job'));
|
||||
}
|
||||
|
||||
function checksumBuffer(buffer) {
|
||||
return crypto.createHash('sha256').update(buffer).digest('hex');
|
||||
}
|
||||
|
||||
function safeArtifactName(job, suffix) {
|
||||
const script = sanitizeFileName(job.script_name || 'script', 'script');
|
||||
return sanitizeFileName(`${job.id}-${script}-${suffix}`, `${job.id}-${suffix}`);
|
||||
}
|
||||
|
||||
function getJobRecord(jobId) {
|
||||
return db.prepare(`
|
||||
SELECT j.*, r.name AS runplan_name, s.name AS script_name
|
||||
FROM jobs j
|
||||
LEFT JOIN runplans r ON r.id = j.runplan_id
|
||||
LEFT JOIN scripts s ON s.id = j.script_id
|
||||
WHERE j.id = ?
|
||||
`).get(jobId);
|
||||
}
|
||||
|
||||
function getHostRows(jobId) {
|
||||
return db.prepare(`
|
||||
SELECT jh.*, h.name AS host_name, h.address AS host_address
|
||||
FROM job_hosts jh
|
||||
LEFT JOIN hosts h ON h.id = jh.host_id
|
||||
WHERE jh.job_id = ?
|
||||
ORDER BY COALESCE(h.name, jh.host_id, jh.id)
|
||||
`).all(jobId);
|
||||
}
|
||||
|
||||
function getLogs(jobId) {
|
||||
return db.prepare(`
|
||||
SELECT jl.*, h.name AS host_name
|
||||
FROM job_logs jl
|
||||
LEFT JOIN hosts h ON h.id = jl.host_id
|
||||
WHERE jl.job_id = ?
|
||||
ORDER BY jl.created_at, jl.id
|
||||
`).all(jobId);
|
||||
}
|
||||
|
||||
function stdoutByHost(hosts, logs) {
|
||||
const buckets = new Map(hosts.map((host) => [host.host_id || host.id, {
|
||||
hostId: host.host_id || '',
|
||||
hostName: host.host_name || host.host_id || 'Unassigned host',
|
||||
hostAddress: host.host_address || '',
|
||||
status: host.status,
|
||||
exitCode: host.exit_code,
|
||||
lines: []
|
||||
}]));
|
||||
for (const log of logs.filter((entry) => entry.stream === 'stdout')) {
|
||||
const key = log.host_id || 'job';
|
||||
if (!buckets.has(key)) {
|
||||
buckets.set(key, {
|
||||
hostId: log.host_id || '',
|
||||
hostName: log.host_name || 'Job',
|
||||
hostAddress: '',
|
||||
status: '',
|
||||
exitCode: null,
|
||||
lines: []
|
||||
});
|
||||
}
|
||||
buckets.get(key).lines.push(stripAnsi(log.line));
|
||||
}
|
||||
return [...buckets.values()];
|
||||
}
|
||||
|
||||
function terminalTranscript(logs) {
|
||||
return logs.map((log) => {
|
||||
const host = log.host_name || log.host_id || 'system';
|
||||
return `[${log.created_at}] [${String(log.stream || '').toUpperCase()}] [${host}] ${stripAnsi(log.line)}`;
|
||||
}).join('\n') + '\n';
|
||||
}
|
||||
|
||||
function jsonRows(lines) {
|
||||
const text = lines.join('\n').trim();
|
||||
if (!text) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
const rows = Array.isArray(parsed) ? parsed : [parsed];
|
||||
if (rows.every((row) => row && typeof row === 'object' && !Array.isArray(row))) return { rows, parser: 'json' };
|
||||
} catch {
|
||||
// Try line-delimited JSON next.
|
||||
}
|
||||
const parsedLines = [];
|
||||
for (const line of lines.map((item) => item.trim()).filter(Boolean)) {
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null;
|
||||
parsedLines.push(parsed);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return parsedLines.length ? { rows: parsedLines, parser: 'json-lines' } : null;
|
||||
}
|
||||
|
||||
function splitDelimited(line, delimiter) {
|
||||
if (delimiter === '\t') return line.split('\t').map((part) => part.trim());
|
||||
return line.split(delimiter).map((part) => part.trim());
|
||||
}
|
||||
|
||||
function delimitedRows(lines) {
|
||||
const clean = lines.map((line) => line.trim()).filter(Boolean);
|
||||
if (clean.length < 2) return null;
|
||||
const delimiter = ['\t', ',', '|'].find((candidate) => splitDelimited(clean[0], candidate).length > 1);
|
||||
if (!delimiter) return null;
|
||||
const headers = splitDelimited(clean[0], delimiter).filter(Boolean);
|
||||
if (headers.length < 2) return null;
|
||||
const rows = clean.slice(1).map((line) => {
|
||||
const values = splitDelimited(line, delimiter);
|
||||
if (values.length < 2) return null;
|
||||
return Object.fromEntries(headers.map((header, index) => [header, values[index] ?? '']));
|
||||
}).filter(Boolean);
|
||||
return rows.length ? { rows, parser: delimiter === '\t' ? 'tsv' : delimiter === ',' ? 'csv' : 'pipe-delimited' } : null;
|
||||
}
|
||||
|
||||
function powershellTableRows(lines) {
|
||||
const clean = lines.map((line) => line.replace(/\s+$/g, '')).filter((line) => line.trim());
|
||||
if (clean.length < 3) return null;
|
||||
const separatorIndex = clean.findIndex((line, index) => index > 0 && /^[-\s]+$/.test(line) && line.trim().includes('-'));
|
||||
if (separatorIndex < 1) return null;
|
||||
const headers = clean[separatorIndex - 1].trim().split(/\s{2,}/).filter(Boolean);
|
||||
if (headers.length < 2) return null;
|
||||
const rows = clean.slice(separatorIndex + 1).map((line) => {
|
||||
const values = line.trim().split(/\s{2,}/);
|
||||
if (!values.length) return null;
|
||||
return Object.fromEntries(headers.map((header, index) => [header, values[index] ?? '']));
|
||||
}).filter(Boolean);
|
||||
return rows.length ? { rows, parser: 'powershell-table' } : null;
|
||||
}
|
||||
|
||||
function powershellObjectRows(lines) {
|
||||
const clean = lines.map((line) => line.trim()).filter(Boolean);
|
||||
if (!clean.length || !clean.every((line) => /^[^:]+:\s*.*$/.test(line))) return null;
|
||||
const row = {};
|
||||
for (const line of clean) {
|
||||
const index = line.indexOf(':');
|
||||
const key = line.slice(0, index).trim();
|
||||
if (!key) return null;
|
||||
row[key] = line.slice(index + 1).trim();
|
||||
}
|
||||
return Object.keys(row).length > 1 ? { rows: [row], parser: 'powershell-object' } : null;
|
||||
}
|
||||
|
||||
function normalizeCellValue(value) {
|
||||
if (value === null || value === undefined) return '';
|
||||
if (typeof value === 'object') return JSON.stringify(value);
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function parseHostRows(host) {
|
||||
const parsed = jsonRows(host.lines) || delimitedRows(host.lines) || powershellTableRows(host.lines) || powershellObjectRows(host.lines);
|
||||
if (parsed) {
|
||||
return {
|
||||
parser: parsed.parser,
|
||||
rows: parsed.rows.map((row) => ({
|
||||
Host: host.hostName,
|
||||
HostStatus: host.status || '',
|
||||
...Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeCellValue(value)]))
|
||||
}))
|
||||
};
|
||||
}
|
||||
const lastLine = [...host.lines].reverse().find((line) => line.trim()) || '';
|
||||
return {
|
||||
parser: 'stdout-last-line',
|
||||
rows: [{
|
||||
Host: host.hostName,
|
||||
HostStatus: host.status || '',
|
||||
Output: lastLine
|
||||
}]
|
||||
};
|
||||
}
|
||||
|
||||
function tablePayload(job, hosts) {
|
||||
const parsedHosts = hosts.map(parseHostRows);
|
||||
const rows = parsedHosts.flatMap((entry) => entry.rows).map((row, index) => ({ id: `row-${index + 1}`, ...row }));
|
||||
const keys = [...new Set(rows.flatMap((row) => Object.keys(row).filter((key) => key !== 'id')))];
|
||||
const preferred = ['Host', 'HostStatus', 'Output'];
|
||||
const ordered = [...preferred.filter((key) => keys.includes(key)), ...keys.filter((key) => !preferred.includes(key))];
|
||||
return {
|
||||
jobId: job.id,
|
||||
runplanName: job.runplan_name || '',
|
||||
scriptName: job.script_name || '',
|
||||
generatedAt: now(),
|
||||
parsers: [...new Set(parsedHosts.map((entry) => entry.parser))],
|
||||
columns: ordered.map((key) => ({ key, label: key })),
|
||||
rows
|
||||
};
|
||||
}
|
||||
|
||||
async function pdfBuffer(job, hosts) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
const doc = new PDFDocument({ autoFirstPage: false, margins: { top: 54, bottom: 48, left: 48, right: 48 } });
|
||||
doc.on('data', (chunk) => chunks.push(chunk));
|
||||
doc.on('error', reject);
|
||||
doc.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
|
||||
doc.addPage();
|
||||
doc.font('Helvetica-Bold').fontSize(18).fillColor('#101827').text('POSHManager Job STDOUT Report');
|
||||
doc.moveDown(0.35);
|
||||
doc.font('Helvetica').fontSize(9).fillColor('#45556c')
|
||||
.text(`Job: ${job.id}`)
|
||||
.text(`Script: ${job.script_name || 'Unknown script'}`)
|
||||
.text(`RunPlan: ${job.runplan_name || 'Script test / ad hoc run'}`)
|
||||
.text(`Status: ${job.status}`)
|
||||
.text(`Generated: ${new Date().toISOString()}`);
|
||||
doc.moveDown(1);
|
||||
|
||||
for (const host of hosts) {
|
||||
if (doc.y > 680) doc.addPage();
|
||||
doc.font('Helvetica-Bold').fontSize(12).fillColor('#101827').text(host.hostName || 'Host');
|
||||
doc.font('Helvetica').fontSize(8).fillColor('#64748b')
|
||||
.text(`Address: ${host.hostAddress || '-'} Status: ${host.status || '-'} Exit code: ${host.exitCode ?? '-'}`);
|
||||
doc.moveDown(0.4);
|
||||
const lines = host.lines.length ? host.lines : ['(No STDOUT captured for this host.)'];
|
||||
doc.font('Courier').fontSize(8).fillColor('#111827');
|
||||
for (const line of lines) {
|
||||
if (doc.y > 730) {
|
||||
doc.addPage();
|
||||
doc.font('Courier').fontSize(8).fillColor('#111827');
|
||||
}
|
||||
doc.text(line || ' ', { width: 500 });
|
||||
}
|
||||
doc.moveDown(0.9);
|
||||
}
|
||||
doc.end();
|
||||
});
|
||||
}
|
||||
|
||||
function writeArtifact(job, kind, name, mimeType, content, metadata) {
|
||||
const buffer = Buffer.isBuffer(content) ? content : Buffer.from(String(content), 'utf8');
|
||||
const dir = artifactDir(job.id);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const storagePath = path.join(dir, sanitizeFileName(name));
|
||||
fs.writeFileSync(storagePath, buffer);
|
||||
const checksum = checksumBuffer(buffer);
|
||||
const timestamp = now();
|
||||
const outputId = db.prepare('SELECT id FROM job_outputs WHERE job_id = ? AND kind = ?').get(job.id, kind)?.id || id('jout');
|
||||
db.prepare(`
|
||||
INSERT INTO job_outputs (id, job_id, kind, name, mime_type, storage_path, size_bytes, checksum, metadata_json, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(job_id, kind) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
mime_type = excluded.mime_type,
|
||||
storage_path = excluded.storage_path,
|
||||
size_bytes = excluded.size_bytes,
|
||||
checksum = excluded.checksum,
|
||||
metadata_json = excluded.metadata_json,
|
||||
updated_at = excluded.updated_at
|
||||
`).run(outputId, job.id, kind, name, mimeType, storagePath, buffer.length, checksum, json(metadata, {}), timestamp, timestamp);
|
||||
}
|
||||
|
||||
export async function generateJobOutputs(jobId) {
|
||||
const job = getJobRecord(jobId);
|
||||
if (!job || !terminalStatuses.has(job.status)) return [];
|
||||
const hosts = stdoutByHost(getHostRows(jobId), getLogs(jobId));
|
||||
const logs = getLogs(jobId);
|
||||
const table = tablePayload(job, hosts);
|
||||
const pdf = await pdfBuffer(job, hosts);
|
||||
const stdoutLineCount = hosts.reduce((sum, host) => sum + host.lines.length, 0);
|
||||
const metadata = {
|
||||
hostCount: hosts.length,
|
||||
stdoutLineCount,
|
||||
generatedAt: now()
|
||||
};
|
||||
|
||||
writeArtifact(job, 'stdout-pdf', safeArtifactName(job, 'stdout.pdf'), 'application/pdf', pdf, metadata);
|
||||
writeArtifact(job, 'datatable', safeArtifactName(job, 'datatable.json'), 'application/json', JSON.stringify(table, null, 2), {
|
||||
...metadata,
|
||||
rowCount: table.rows.length,
|
||||
columns: table.columns.map((column) => column.key),
|
||||
parsers: table.parsers
|
||||
});
|
||||
writeArtifact(job, 'terminal', safeArtifactName(job, 'terminal.log'), 'text/plain', terminalTranscript(logs), {
|
||||
...metadata,
|
||||
logLineCount: logs.length
|
||||
});
|
||||
|
||||
return db.prepare('SELECT * FROM job_outputs WHERE job_id = ? ORDER BY kind').all(jobId);
|
||||
}
|
||||
|
||||
export async function generateJobOutputsSafe(jobId) {
|
||||
try {
|
||||
return await generateJobOutputs(jobId);
|
||||
} catch (error) {
|
||||
logger.error('failed to generate job outputs', { jobId, error });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import { getVCenterVmPowerState } from './vcenterService.js';
|
||||
import { path } from '../utils/pathUtils.js';
|
||||
import { analyzePowerShellCompatibility, compatibilityLogLines } from './powershellCompatibilityService.js';
|
||||
import { normalizeOsFamily } from '../utils/osFamily.js';
|
||||
import { generateJobOutputsSafe } from './jobOutputService.js';
|
||||
|
||||
const activeJobs = new Map();
|
||||
|
||||
@@ -42,6 +43,19 @@ export function powerShellSpawnFailureLines(error, executable) {
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function powerShellSignalFailureLines(signal, executable) {
|
||||
const lines = [
|
||||
`PowerShell executable "${executable}" terminated by signal ${signal}.`,
|
||||
`API process platform: ${process.platform}; PATH: ${process.env.PATH || '(empty)'}`
|
||||
];
|
||||
if (signal === 'SIGABRT') {
|
||||
lines.push('SIGABRT usually means the PowerShell/.NET process aborted after launch, not that POSHManager could not find the executable.');
|
||||
lines.push('On Linux API hosts, verify the configured pwsh can run non-interactively with: pwsh -NoProfile -Command "$PSVersionTable".');
|
||||
lines.push('For Windows remoting targets, also verify PowerShell remoting/WinRM connectivity from the API host using the same credential and target address.');
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function executeRunPlan(runplanId, userId) {
|
||||
// Create the durable job record before async execution so the UI can attach to logs immediately.
|
||||
const runplan = db.prepare('SELECT * FROM runplans WHERE id = ?').get(runplanId);
|
||||
@@ -88,9 +102,10 @@ export function executeRunPlan(runplanId, userId) {
|
||||
assets: listExecutableAssets(userId)
|
||||
};
|
||||
|
||||
void runJob(jobId, script, hosts, Boolean(runplan.parallel), executionContext).catch((error) => {
|
||||
void runJob(jobId, script, hosts, Boolean(runplan.parallel), executionContext).catch(async (error) => {
|
||||
logger.error('job runner crashed', { jobId, error });
|
||||
db.prepare('UPDATE jobs SET status = ?, ended_at = ? WHERE id = ?').run('failed', now(), jobId);
|
||||
await generateJobOutputsSafe(jobId);
|
||||
});
|
||||
|
||||
return db.prepare('SELECT * FROM jobs WHERE id = ?').get(jobId);
|
||||
@@ -154,7 +169,7 @@ export function executeScriptTest(scriptId, payload, userId) {
|
||||
INSERT INTO jobs (id, runplan_id, script_id, status, triggered_by, started_at, created_at)
|
||||
VALUES (?, NULL, ?, 'running', ?, ?, ?)
|
||||
`).run(jobId, script.id, userId, now(), now());
|
||||
db.prepare(`
|
||||
const insertJobHost = db.prepare(`
|
||||
INSERT INTO job_hosts (id, job_id, host_id, status, started_at)
|
||||
VALUES (?, ?, ?, 'queued', NULL)
|
||||
`);
|
||||
@@ -172,10 +187,11 @@ export function executeScriptTest(scriptId, payload, userId) {
|
||||
};
|
||||
|
||||
for (const host of hosts) appendLog(jobId, host.id, 'system', `Test run requested for ${script.name} using credential ${host.credential_name}`);
|
||||
void runJob(jobId, script, hosts, false, executionContext).catch((error) => {
|
||||
void runJob(jobId, script, hosts, false, executionContext).catch(async (error) => {
|
||||
logger.error('script test runner crashed', { jobId, error });
|
||||
for (const host of hosts) appendLog(jobId, host.id, 'stderr', `Script test runner crashed: ${error.message}`);
|
||||
db.prepare('UPDATE jobs SET status = ?, ended_at = ? WHERE id = ?').run('failed', now(), jobId);
|
||||
await generateJobOutputsSafe(jobId);
|
||||
});
|
||||
|
||||
return db.prepare('SELECT * FROM jobs WHERE id = ?').get(jobId);
|
||||
@@ -203,10 +219,12 @@ async function runJob(jobId, script, hosts, parallel, executionContext) {
|
||||
if (isJobCanceled(jobId)) {
|
||||
cancelQueuedHosts(jobId);
|
||||
db.prepare('UPDATE jobs SET status = ?, ended_at = COALESCE(ended_at, ?) WHERE id = ?').run('canceled', now(), jobId);
|
||||
await generateJobOutputsSafe(jobId);
|
||||
return;
|
||||
}
|
||||
const failed = db.prepare('SELECT COUNT(*) AS count FROM job_hosts WHERE job_id = ? AND status = ?').get(jobId, 'failed').count;
|
||||
db.prepare('UPDATE jobs SET status = ?, ended_at = ? WHERE id = ?').run(failed ? 'failed' : 'completed', now(), jobId);
|
||||
await generateJobOutputsSafe(jobId);
|
||||
} finally {
|
||||
activeJobs.delete(jobId);
|
||||
}
|
||||
@@ -238,6 +256,7 @@ async function runHost(jobId, script, host, executionContext) {
|
||||
}
|
||||
|
||||
if (!(await vCenterPowerPreflight(jobId, host))) return;
|
||||
logTransportPreflight(jobId, host);
|
||||
|
||||
const file = path.join(os.tmpdir(), `poshmanager-${jobId}-${host.id}.ps1`);
|
||||
|
||||
@@ -284,7 +303,13 @@ async function vCenterPowerPreflight(jobId, host) {
|
||||
return true;
|
||||
}
|
||||
|
||||
function buildWrapper(content, host, executionContext = {}) {
|
||||
function logTransportPreflight(jobId, host) {
|
||||
if (host.transport === 'winrm' && process.platform !== 'win32') {
|
||||
appendLog(jobId, host.id, 'system', 'WinRM target from a Linux API host: Invoke-Command -ComputerName requires WSMan client support inside the API environment. If PowerShell reports no supported WSMan client library, install/configure the Linux WSMan dependencies or use SSH transport for PowerShell 7 targets.');
|
||||
}
|
||||
}
|
||||
|
||||
export function buildWrapper(content, host, executionContext = {}) {
|
||||
// Build PowerShell wrappers without shell interpolation; host values are quoted as PS literals.
|
||||
const runtimeVariables = runtimeVariableValues(host, executionContext);
|
||||
const customVariables = executionContext.customVariables || [];
|
||||
@@ -296,37 +321,53 @@ function buildWrapper(content, host, executionContext = {}) {
|
||||
const rendered = renderTokens(content, [...runtimeVariables, ...customVariables, ...assetVariables]);
|
||||
const scriptWithPrelude = `${buildVariablePrelude(runtimeVariables, customVariables)}\n${rendered}`;
|
||||
const escaped = scriptWithPrelude.replace(/'@/g, "'`@");
|
||||
const scriptSetup = [
|
||||
'$ErrorActionPreference = "Stop"',
|
||||
'$ProgressPreference = "SilentlyContinue"',
|
||||
'if (Get-Variable -Name PSStyle -ErrorAction SilentlyContinue) { $PSStyle.OutputRendering = "PlainText" }',
|
||||
'try {'
|
||||
];
|
||||
const scriptExit = [
|
||||
' if ($global:LASTEXITCODE -is [int] -and $global:LASTEXITCODE -ne 0) { exit $global:LASTEXITCODE }',
|
||||
' exit 0',
|
||||
'} catch {',
|
||||
' $message = ($_ | Out-String).TrimEnd()',
|
||||
' if (-not $message) { $message = $_.Exception.Message }',
|
||||
' [Console]::Error.WriteLine($message)',
|
||||
' exit 1',
|
||||
'}'
|
||||
];
|
||||
const scriptTextBlock = [
|
||||
` $scriptText = @'`,
|
||||
escaped,
|
||||
"'@",
|
||||
' $scriptBlock = [ScriptBlock]::Create($scriptText)'
|
||||
];
|
||||
if (host.transport === 'local') {
|
||||
return [
|
||||
'$ErrorActionPreference = "Continue"',
|
||||
`$scriptText = @'`,
|
||||
escaped,
|
||||
"'@",
|
||||
'$scriptBlock = [ScriptBlock]::Create($scriptText)',
|
||||
'& $scriptBlock'
|
||||
...scriptSetup,
|
||||
...scriptTextBlock,
|
||||
' & $scriptBlock',
|
||||
...scriptExit
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
if (host.transport === 'ssh') {
|
||||
return [
|
||||
'$ErrorActionPreference = "Continue"',
|
||||
`$scriptText = @'`,
|
||||
escaped,
|
||||
"'@",
|
||||
'$scriptBlock = [ScriptBlock]::Create($scriptText)',
|
||||
`Invoke-Command -HostName ${psString(host.address)} -UserName $env:POSHM_USERNAME -ScriptBlock $scriptBlock`
|
||||
...scriptSetup,
|
||||
...scriptTextBlock,
|
||||
` Invoke-Command -HostName ${psString(host.address)} -UserName $env:POSHM_USERNAME -ScriptBlock $scriptBlock -ErrorAction Stop`,
|
||||
...scriptExit
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
return [
|
||||
'$ErrorActionPreference = "Continue"',
|
||||
'$securePassword = ConvertTo-SecureString $env:POSHM_SECRET -AsPlainText -Force',
|
||||
'$credential = [System.Management.Automation.PSCredential]::new($env:POSHM_USERNAME, $securePassword)',
|
||||
`$scriptText = @'`,
|
||||
escaped,
|
||||
"'@",
|
||||
'$scriptBlock = [ScriptBlock]::Create($scriptText)',
|
||||
`Invoke-Command -ComputerName ${psString(host.address)} -Credential $credential -ScriptBlock $scriptBlock`
|
||||
...scriptSetup,
|
||||
' $securePassword = ConvertTo-SecureString $env:POSHM_SECRET -AsPlainText -Force',
|
||||
' $credential = [System.Management.Automation.PSCredential]::new($env:POSHM_USERNAME, $securePassword)',
|
||||
...scriptTextBlock,
|
||||
` Invoke-Command -ComputerName ${psString(host.address)} -Credential $credential -ScriptBlock $scriptBlock -ErrorAction Stop`,
|
||||
...scriptExit
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
@@ -380,7 +421,9 @@ function spawnPowerShell(jobId, host, file, secret, executionContext = {}) {
|
||||
});
|
||||
child.on('close', (code, signal) => {
|
||||
if (settled) return;
|
||||
if (signal) appendLog(jobId, host.id, 'stderr', `PowerShell terminated by signal ${signal}`);
|
||||
if (signal) {
|
||||
for (const line of powerShellSignalFailureLines(signal, executable)) appendLog(jobId, host.id, 'stderr', line);
|
||||
}
|
||||
if (isJobCanceled(jobId)) {
|
||||
settle('canceled', 130);
|
||||
return;
|
||||
|
||||
71
server/services/runPlanScheduleWorker.js
Normal file
71
server/services/runPlanScheduleWorker.js
Normal file
@@ -0,0 +1,71 @@
|
||||
import { getStringSetting } from '../models/settingsModel.js';
|
||||
import {
|
||||
listDueRunPlanSchedules,
|
||||
recordRunPlanScheduleDispatch
|
||||
} from '../models/runPlanScheduleModel.js';
|
||||
import { executeRunPlan } from './powershellRunner.js';
|
||||
import { logger } from './logger.js';
|
||||
|
||||
let timer = null;
|
||||
let running = false;
|
||||
|
||||
export function runPlanScheduleIntervalMinutes() {
|
||||
const minutes = Number(getStringSetting('runplan_schedule_interval_minutes', process.env.RUNPLAN_SCHEDULE_INTERVAL_MINUTES || '1'));
|
||||
return Number.isFinite(minutes) ? minutes : 1;
|
||||
}
|
||||
|
||||
export async function dispatchDueRunPlanSchedules({ nowIso = new Date().toISOString(), signal } = {}) {
|
||||
const due = listDueRunPlanSchedules(nowIso);
|
||||
const summary = { evaluated: due.length, dispatched: 0, skipped: 0, errors: 0 };
|
||||
for (const schedule of due) {
|
||||
if (signal?.aborted) {
|
||||
summary.skipped += 1;
|
||||
break;
|
||||
}
|
||||
try {
|
||||
const job = executeRunPlan(schedule.runplanId, schedule.createdBy || schedule.ownerUserId || null);
|
||||
recordRunPlanScheduleDispatch(schedule, {
|
||||
jobId: job.id,
|
||||
plannedFor: schedule.nextRunAt || nowIso,
|
||||
status: 'queued',
|
||||
message: `Scheduled RunPlan ${schedule.runplanName || schedule.runplanId} queued as ${job.id}.`
|
||||
});
|
||||
summary.dispatched += 1;
|
||||
} catch (error) {
|
||||
recordRunPlanScheduleDispatch(schedule, {
|
||||
plannedFor: schedule.nextRunAt || nowIso,
|
||||
status: 'failed',
|
||||
message: error.message
|
||||
});
|
||||
summary.errors += 1;
|
||||
logger.error('runplan schedule dispatch failed', { scheduleId: schedule.id, error: error.message });
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
export function startRunPlanScheduleWorker() {
|
||||
const minutes = runPlanScheduleIntervalMinutes();
|
||||
if (minutes <= 0) return null;
|
||||
running = true;
|
||||
scheduleNext(minutes);
|
||||
logger.info(`runplan schedule dispatcher scheduled every ${minutes} minute(s)`);
|
||||
return timer;
|
||||
}
|
||||
|
||||
export function stopRunPlanScheduleWorker() {
|
||||
running = false;
|
||||
if (timer) clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
|
||||
function scheduleNext(minutes) {
|
||||
if (!running || minutes <= 0) return;
|
||||
timer = setTimeout(() => {
|
||||
dispatchDueRunPlanSchedules()
|
||||
.then((summary) => logger.info('runplan schedule dispatch complete', summary))
|
||||
.catch((error) => logger.error('runplan schedule dispatch failed', { error: error.message }))
|
||||
.finally(() => scheduleNext(runPlanScheduleIntervalMinutes()));
|
||||
}, minutes * 60 * 1000);
|
||||
timer.unref?.();
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { listSystemJobActions, recordSystemJobAction } from '../models/systemJob
|
||||
import { syncDynamicHostGroups } from '../models/hostGroupModel.js';
|
||||
import { runCatalogChecks } from './catalogWorker.js';
|
||||
import { runAutoPromotions } from './promotionWorker.js';
|
||||
import { dispatchDueRunPlanSchedules } from './runPlanScheduleWorker.js';
|
||||
import { cancelExecutionJob, appendJobLog, activeExecutionJobIds } from './powershellRunner.js';
|
||||
|
||||
const backgroundRuns = new Map();
|
||||
@@ -29,6 +30,13 @@ const BACKGROUND_JOBS = [
|
||||
category: 'Intune',
|
||||
description: 'Promotes deployment rings when configured soak windows elapse.',
|
||||
run: async ({ signal } = {}) => runAutoPromotions({ signal })
|
||||
},
|
||||
{
|
||||
id: 'worker:runplan-schedule-dispatch',
|
||||
name: 'RunPlan Schedule Dispatch',
|
||||
category: 'Scheduling',
|
||||
description: 'Queues due automated RunPlan schedules and records dispatch history.',
|
||||
run: async ({ signal } = {}) => dispatchDueRunPlanSchedules({ signal })
|
||||
}
|
||||
];
|
||||
|
||||
|
||||
@@ -784,6 +784,35 @@ function soapRefElement(name, ref, fallbackType) {
|
||||
return `<${name} type="${xmlEscape(ref?.type || fallbackType)}">${xmlEscape(ref?.value || '')}</${name}>`;
|
||||
}
|
||||
|
||||
function soapBoolElement(name, value) {
|
||||
return `<${name}>${value ? 'true' : 'false'}</${name}>`;
|
||||
}
|
||||
|
||||
function propertySpecXml(type, paths = []) {
|
||||
return `<propSet>` +
|
||||
`<type>${xmlEscape(type)}</type>` +
|
||||
soapBoolElement('all', false) +
|
||||
paths.map((pathSet) => `<pathSet>${xmlEscape(pathSet)}</pathSet>`).join('') +
|
||||
`</propSet>`;
|
||||
}
|
||||
|
||||
function objectSpecXml(ref, { skip = false, selectSet = '' } = {}) {
|
||||
return `<objectSet>` +
|
||||
soapRefElement('obj', ref, ref?.type || 'ManagedObjectReference') +
|
||||
soapBoolElement('skip', skip) +
|
||||
selectSet +
|
||||
`</objectSet>`;
|
||||
}
|
||||
|
||||
function traversalSpecXml() {
|
||||
return `<selectSet xsi:type="TraversalSpec">` +
|
||||
`<name>containerViewTraversal</name>` +
|
||||
`<type>ContainerView</type>` +
|
||||
`<path>view</path>` +
|
||||
soapBoolElement('skip', false) +
|
||||
`</selectSet>`;
|
||||
}
|
||||
|
||||
async function connectStandaloneSoap(settings, trace = null) {
|
||||
pushTraceInfo(trace, `Using ${WEB_SERVICES_PROFILE_LABEL}`, {
|
||||
mode: 'web-services',
|
||||
@@ -826,7 +855,7 @@ async function createVmContainerView(settings, refs, trace = null) {
|
||||
return extractManagedObject(response, 'returnval', 'ContainerView');
|
||||
}
|
||||
|
||||
function retrievePropertiesBody(propertyCollector, viewRef, limit = 100) {
|
||||
export function retrievePropertiesBody(propertyCollector, viewRef, limit = 100) {
|
||||
const paths = [
|
||||
'name',
|
||||
'guest.hostName',
|
||||
@@ -841,12 +870,10 @@ function retrievePropertiesBody(propertyCollector, viewRef, limit = 100) {
|
||||
'summary.runtime.powerState'
|
||||
];
|
||||
return `<RetrievePropertiesEx xmlns="urn:vim25" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">` +
|
||||
`${soapRefElement('_this', propertyCollector, 'PropertyCollector')}` +
|
||||
soapRefElement('_this', propertyCollector, 'PropertyCollector') +
|
||||
`<specSet>` +
|
||||
`<propSet><type>VirtualMachine</type><all>false</all>${paths.map((pathSet) => `<pathSet>${pathSet}</pathSet>`).join('')}</propSet>` +
|
||||
`<objectSet>${soapRefElement('obj', viewRef, 'ContainerView')}<skip>true</skip>` +
|
||||
`<selectSet xsi:type="TraversalSpec"><name>containerViewTraversal</name><type>ContainerView</type><path>view</path><skip>false</skip></selectSet>` +
|
||||
`</objectSet>` +
|
||||
propertySpecXml('VirtualMachine', paths) +
|
||||
objectSpecXml(viewRef, { skip: true, selectSet: traversalSpecXml() }) +
|
||||
`</specSet>` +
|
||||
`<options><maxObjects>${Math.max(1, Math.min(Number(limit) || 100, 500))}</maxObjects></options>` +
|
||||
`</RetrievePropertiesEx>`;
|
||||
@@ -888,9 +915,11 @@ async function retrieveStandaloneVmPowerState(settings, vmId, trace = null) {
|
||||
const refs = await connectStandaloneSoap(settings, trace);
|
||||
const vmRef = { type: 'VirtualMachine', value: vmId };
|
||||
const body = `<RetrievePropertiesEx xmlns="urn:vim25" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">` +
|
||||
`${soapRefElement('_this', refs.propertyCollector, 'PropertyCollector')}` +
|
||||
`<specSet><propSet><type>VirtualMachine</type><all>false><pathSet>runtime.powerState</pathSet></propSet>` +
|
||||
`<objectSet>${soapRefElement('obj', vmRef, 'VirtualMachine')}<skip>false</skip></objectSet></specSet>` +
|
||||
soapRefElement('_this', refs.propertyCollector, 'PropertyCollector') +
|
||||
`<specSet>` +
|
||||
propertySpecXml('VirtualMachine', ['runtime.powerState']) +
|
||||
objectSpecXml(vmRef, { skip: false }) +
|
||||
`</specSet>` +
|
||||
`<options><maxObjects>1</maxObjects></options></RetrievePropertiesEx>`;
|
||||
const response = await vSphereSoapFetch(settings, 'RetrievePropertiesEx', body, trace, { cookie: refs.sessionCookie, endpoint: refs.endpoint });
|
||||
const object = parseVmPropertyObjects(response)[0];
|
||||
|
||||
Reference in New Issue
Block a user