import fs from 'node:fs/promises'; import os from 'node:os'; import { spawn } from 'node:child_process'; import { db, id, now } from '../db.js'; import { decryptSecret } from './cryptoStore.js'; import { config } from '../config.js'; import { logger } from './logger.js'; import { listExecutableAssets } from '../models/assetModel.js'; import { listExecutableCustomVariables } from '../models/variableModel.js'; import { getBooleanSetting, getStringSetting } from '../models/settingsModel.js'; import { getVCenterConnectionSystem } from '../models/vcenterConnectionModel.js'; import { getVCenterVmPowerState } from './vcenterService.js'; import { path } from '../utils/pathUtils.js'; import { analyzePowerShellCompatibility, compatibilityLogLines } from './powershellCompatibilityService.js'; import { normalizeOsFamily } from '../utils/osFamily.js'; import { generateJobOutputsSafe } from './jobOutputService.js'; const activeJobs = new Map(); // The execution kill-switch can be set at boot via ALLOW_SCRIPT_EXECUTION and // flipped at runtime via the Config screen (allow_script_execution). The DB // value, when present, takes precedence so the UI toggle is actually effective. function scriptExecutionAllowed() { return getBooleanSetting('allow_script_execution', config.allowScriptExecution); } export function resolvePowerShellExecutable() { return getStringSetting('powershell_bin', config.powershellBin).trim(); } export function powerShellSpawnFailureLines(error, executable) { const code = error?.code ? ` (${error.code})` : ''; const lines = [ `Unable to start configured PowerShell executable "${executable}"${code}.`, `Node spawn error: ${error?.message || 'unknown process start failure'}` ]; if (error?.code === 'ENOENT') { lines.push('The executable was not found on the API server PATH.'); lines.push('Windows: set Config > PowerShell Binary to "powershell.exe" or the full path "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe".'); lines.push('PowerShell 7: set it to "pwsh.exe" or the full pwsh install path.'); } lines.push(`API process platform: ${process.platform}; PATH: ${process.env.PATH || '(empty)'}`); return lines; } export function powerShellSignalFailureLines(signal, executable) { const lines = [ `PowerShell executable "${executable}" terminated by signal ${signal}.`, `API process platform: ${process.platform}; PATH: ${process.env.PATH || '(empty)'}` ]; if (signal === 'SIGABRT') { lines.push('SIGABRT usually means the PowerShell/.NET process aborted after launch, not that POSHManager could not find the executable.'); lines.push('On Linux API hosts, verify the configured pwsh can run non-interactively with: pwsh -NoProfile -Command "$PSVersionTable".'); lines.push('For Windows remoting targets, also verify PowerShell remoting/WinRM connectivity from the API host using the same credential and target address.'); } return lines; } export function executeRunPlan(runplanId, userId) { // Create the durable job record before async execution so the UI can attach to logs immediately. const runplan = db.prepare('SELECT * FROM runplans WHERE id = ?').get(runplanId); if (!runplan) throw new Error('RunPlan not found'); const script = db.prepare('SELECT * FROM scripts WHERE id = ?').get(runplan.script_id); if (!script) throw new Error('Script not found'); const hosts = db.prepare(` SELECT h.*, c.name AS credential_name, c.kind AS credential_kind, c.username AS credential_username, c.secret_cipher, c.secret_iv, c.secret_tag FROM hosts h LEFT JOIN credentials c ON c.id = h.credential_id WHERE h.id IN ( SELECT host_id FROM runplan_hosts WHERE runplan_id = ? UNION SELECT hgm.host_id FROM runplan_host_groups rhg JOIN host_group_members hgm ON hgm.host_group_id = rhg.host_group_id WHERE rhg.runplan_id = ? ) ORDER BY h.name `).all(runplanId, runplanId); if (hosts.length === 0) throw new Error('RunPlan has no hosts'); const jobId = id('job'); db.prepare(` INSERT INTO jobs (id, runplan_id, script_id, status, triggered_by, started_at, created_at) VALUES (?, ?, ?, 'running', ?, ?, ?) `).run(jobId, runplan.id, script.id, userId, now(), now()); for (const host of hosts) { db.prepare(` INSERT INTO job_hosts (id, job_id, host_id, status, started_at) VALUES (?, ?, ?, 'queued', NULL) `).run(id('jhost'), jobId, host.id); } const executionContext = { jobId, runplanId: runplan.id, scriptId: script.id, scriptName: script.name, triggeredBy: userId, customVariables: listExecutableCustomVariables(userId), assets: listExecutableAssets(userId) }; void runJob(jobId, script, hosts, Boolean(runplan.parallel), executionContext).catch(async (error) => { logger.error('job runner crashed', { jobId, error }); db.prepare('UPDATE jobs SET status = ?, ended_at = ? WHERE id = ?').run('failed', now(), jobId); await generateJobOutputsSafe(jobId); }); return db.prepare('SELECT * FROM jobs WHERE id = ?').get(jobId); } export function executeScriptTest(scriptId, payload, userId) { const script = db.prepare(` SELECT * FROM scripts WHERE id = ? AND ( visibility = 'shared' OR owner_user_id = ? OR group_id IN (SELECT group_id FROM user_groups WHERE user_id = ?) ) `).get(scriptId, userId, userId); if (!script) throw new Error('Script not found'); const hosts = payload.hostGroupId ? db.prepare(` SELECT h.*, c.name AS credential_name, c.kind AS credential_kind, c.username AS credential_username, c.secret_cipher, c.secret_iv, c.secret_tag FROM host_group_members hgm JOIN host_groups hg ON hg.id = hgm.host_group_id AND ( hg.visibility = 'shared' OR hg.owner_user_id = ? OR hg.group_id IN (SELECT group_id FROM user_groups WHERE user_id = ?) ) JOIN hosts h ON h.id = hgm.host_id AND ( h.visibility = 'shared' OR h.owner_user_id = ? OR h.group_id IN (SELECT group_id FROM user_groups WHERE user_id = ?) ) JOIN credentials c ON c.id = ? AND ( c.visibility = 'shared' OR c.owner_user_id = ? OR c.group_id IN (SELECT group_id FROM user_groups WHERE user_id = ?) ) WHERE hgm.host_group_id = ? ORDER BY h.name `).all(userId, userId, userId, userId, payload.credentialId, userId, userId, payload.hostGroupId) : [db.prepare(` SELECT h.*, c.name AS credential_name, c.kind AS credential_kind, c.username AS credential_username, c.secret_cipher, c.secret_iv, c.secret_tag FROM hosts h JOIN credentials c ON c.id = ? AND ( c.visibility = 'shared' OR c.owner_user_id = ? OR c.group_id IN (SELECT group_id FROM user_groups WHERE user_id = ?) ) WHERE h.id = ? AND ( h.visibility = 'shared' OR h.owner_user_id = ? OR h.group_id IN (SELECT group_id FROM user_groups WHERE user_id = ?) ) `).get(payload.credentialId, userId, userId, payload.hostId, userId, userId)].filter(Boolean); if (!hosts.length) throw new Error('Visible host or host group members and credential are required'); const jobId = id('job'); db.prepare(` INSERT INTO jobs (id, runplan_id, script_id, status, triggered_by, started_at, created_at) VALUES (?, NULL, ?, 'running', ?, ?, ?) `).run(jobId, script.id, userId, now(), now()); const insertJobHost = db.prepare(` INSERT INTO job_hosts (id, job_id, host_id, status, started_at) VALUES (?, ?, ?, 'queued', NULL) `); for (const host of hosts) insertJobHost.run(id('jhost'), jobId, host.id); const executionContext = { jobId, runplanId: 'script-test', scriptId: script.id, scriptName: script.name, triggeredBy: userId, testRun: true, customVariables: listExecutableCustomVariables(userId), assets: listExecutableAssets(userId) }; for (const host of hosts) appendLog(jobId, host.id, 'system', `Test run requested for ${script.name} using credential ${host.credential_name}`); void runJob(jobId, script, hosts, false, executionContext).catch(async (error) => { logger.error('script test runner crashed', { jobId, error }); for (const host of hosts) appendLog(jobId, host.id, 'stderr', `Script test runner crashed: ${error.message}`); db.prepare('UPDATE jobs SET status = ?, ended_at = ? WHERE id = ?').run('failed', now(), jobId); await generateJobOutputsSafe(jobId); }); return db.prepare('SELECT * FROM jobs WHERE id = ?').get(jobId); } async function runJob(jobId, script, hosts, parallel, executionContext) { // RunPlans can fan out in parallel or execute serially while preserving identical log semantics. const state = { canceled: false, children: new Map() }; activeJobs.set(jobId, state); try { logCompatibilityPreflight(jobId, script, hosts); const runner = async (host) => runHost(jobId, script, host, executionContext); if (parallel) { await Promise.all(hosts.map(runner)); } else { for (const host of hosts) { if (isJobCanceled(jobId)) { cancelQueuedHosts(jobId); break; } await runner(host); } } if (isJobCanceled(jobId)) { cancelQueuedHosts(jobId); db.prepare('UPDATE jobs SET status = ?, ended_at = COALESCE(ended_at, ?) WHERE id = ?').run('canceled', now(), jobId); await generateJobOutputsSafe(jobId); return; } const failed = db.prepare('SELECT COUNT(*) AS count FROM job_hosts WHERE job_id = ? AND status = ?').get(jobId, 'failed').count; db.prepare('UPDATE jobs SET status = ?, ended_at = ? WHERE id = ?').run(failed ? 'failed' : 'completed', now(), jobId); await generateJobOutputsSafe(jobId); } finally { activeJobs.delete(jobId); } } function logCompatibilityPreflight(jobId, script, hosts) { const analysis = analyzePowerShellCompatibility(script.content, hosts); const lines = compatibilityLogLines(analysis); if (!lines.length) return; for (const host of hosts.filter((target) => normalizeOsFamily(target.os_family || target.osFamily, 'other') === 'linux')) { for (const line of lines) appendLog(jobId, host.id, 'system', line); } } async function runHost(jobId, script, host, executionContext) { // Each host owns its own temp wrapper and job_host lifecycle for per-target evidence. if (isJobCanceled(jobId)) { finishHost(jobId, host.id, 'canceled', 130); return; } db.prepare('UPDATE job_hosts SET status = ?, started_at = ? WHERE job_id = ? AND host_id = ?').run('running', now(), jobId, host.id); appendLog(jobId, host.id, 'system', `Starting ${script.name} on ${host.name} (${host.transport})`); appendLog(jobId, host.id, 'system', `Target OS type: ${normalizeOsFamily(host.os_family || host.osFamily, 'other')}`); if (!scriptExecutionAllowed()) { appendLog(jobId, host.id, 'stderr', 'Script execution is disabled by ALLOW_SCRIPT_EXECUTION=false.'); finishHost(jobId, host.id, 'failed', 126); return; } if (!(await vCenterPowerPreflight(jobId, host))) return; logTransportPreflight(jobId, host); const file = path.join(os.tmpdir(), `poshmanager-${jobId}-${host.id}.ps1`); try { const secret = decryptSecret(host); const wrapper = buildWrapper(script.content, host, executionContext); await fs.writeFile(file, wrapper, { mode: 0o600 }); await spawnPowerShell(jobId, host, file, secret, executionContext); } catch (error) { appendLog(jobId, host.id, 'stderr', error.message); finishHost(jobId, host.id, 'failed', 1); } finally { await fs.rm(file, { force: true }).catch(() => {}); } } async function vCenterPowerPreflight(jobId, host) { if ((host.source_type || 'manual') !== 'vmware') return true; if (isJobCanceled(jobId)) return false; if (!host.source_connection_id || !host.source_ref) { appendLog(jobId, host.id, 'system', 'Host is marked as VMware-sourced, but no vCenter connection or VM reference is stored. Continuing without a power-state gate.'); return true; } const connection = getVCenterConnectionSystem(host.source_connection_id, true); if (!connection) { appendLog(jobId, host.id, 'system', `Host is VMware-sourced, but vCenter connection ${host.source_connection_id} no longer exists. Continuing without a power-state gate.`); return true; } try { const result = await getVCenterVmPowerState(connection, host.source_ref); if (!result.checked) { appendLog(jobId, host.id, 'system', `vCenter power-state check skipped: ${result.reason || 'not enough source metadata'}`); return true; } appendLog(jobId, host.id, 'system', `vCenter power state for VM ${host.source_ref}: ${result.state || 'unknown'} (${result.apiProfile || 'vCenter API'})`); if (result.state && result.state !== 'POWERED_ON') { appendLog(jobId, host.id, 'system', `Skipping ${host.name}; VMware VM is not powered on.`); finishHost(jobId, host.id, 'skipped', 0); return false; } } catch (err) { appendLog(jobId, host.id, 'system', `Unable to verify vCenter power state before execution: ${err.message}. Continuing without a power-state gate.`); } return true; } function logTransportPreflight(jobId, host) { if (host.transport === 'winrm' && process.platform !== 'win32') { appendLog(jobId, host.id, 'system', 'WinRM target from a Linux API host: Invoke-Command -ComputerName requires WSMan client support inside the API environment. If PowerShell reports no supported WSMan client library, install/configure the Linux WSMan dependencies or use SSH transport for PowerShell 7 targets.'); } } export function buildWrapper(content, host, executionContext = {}) { // Build PowerShell wrappers without shell interpolation; host values are quoted as PS literals. const runtimeVariables = runtimeVariableValues(host, executionContext); const customVariables = executionContext.customVariables || []; const assetVariables = (executionContext.assets || []).map((asset) => ({ name: `asset:${asset.id}`, value: asset.localPath, valueType: 'string' })); const rendered = renderTokens(content, [...runtimeVariables, ...customVariables, ...assetVariables]); const scriptWithPrelude = `${buildVariablePrelude(runtimeVariables, customVariables)}\n${rendered}`; const escaped = scriptWithPrelude.replace(/'@/g, "'`@"); const scriptSetup = [ '$ErrorActionPreference = "Stop"', '$ProgressPreference = "SilentlyContinue"', 'if (Get-Variable -Name PSStyle -ErrorAction SilentlyContinue) { $PSStyle.OutputRendering = "PlainText" }', 'try {' ]; const scriptExit = [ ' if ($global:LASTEXITCODE -is [int] -and $global:LASTEXITCODE -ne 0) { exit $global:LASTEXITCODE }', ' exit 0', '} catch {', ' $message = ($_ | Out-String).TrimEnd()', ' if (-not $message) { $message = $_.Exception.Message }', ' [Console]::Error.WriteLine($message)', ' exit 1', '}' ]; const scriptTextBlock = [ ` $scriptText = @'`, escaped, "'@", ' $scriptBlock = [ScriptBlock]::Create($scriptText)' ]; if (host.transport === 'local') { return [ ...scriptSetup, ...scriptTextBlock, ' & $scriptBlock', ...scriptExit ].join('\n'); } if (host.transport === 'ssh') { return [ ...scriptSetup, ...scriptTextBlock, ` Invoke-Command -HostName ${psString(host.address)} -UserName $env:POSHM_USERNAME -ScriptBlock $scriptBlock -ErrorAction Stop`, ...scriptExit ].join('\n'); } return [ ...scriptSetup, ' $securePassword = ConvertTo-SecureString $env:POSHM_SECRET -AsPlainText -Force', ' $credential = [System.Management.Automation.PSCredential]::new($env:POSHM_USERNAME, $securePassword)', ...scriptTextBlock, ` Invoke-Command -ComputerName ${psString(host.address)} -Credential $credential -ScriptBlock $scriptBlock -ErrorAction Stop`, ...scriptExit ].join('\n'); } function spawnPowerShell(jobId, host, file, secret, executionContext = {}) { // Secrets are passed through process environment variables and are never written to job logs. const runtimeEnv = Object.fromEntries(runtimeVariableValues(host, executionContext).map((variable) => [variable.name, String(variable.value ?? '')])); const assetManifest = JSON.stringify(executionContext.assets || []); const executable = resolvePowerShellExecutable(); const args = ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', file]; return new Promise((resolve) => { appendLog(jobId, host.id, 'system', `Launching PowerShell executable "${executable}"`); const child = spawn(executable, args, { env: { ...process.env, ...runtimeEnv, POSHM_HOST_ID: host.id, POSHM_HOST_NAME: host.name, POSHM_HOST_ADDRESS: host.address, POSHM_HOST_TRANSPORT: host.transport, POSHM_ASSET_MANIFEST: assetManifest, POSHM_USERNAME: host.credential_username || '', POSHM_SECRET: secret || '' }, shell: false }); const active = activeJobs.get(jobId); if (active) { active.children.set(host.id, child); if (active.canceled) { child.kill('SIGTERM'); setTimeout(() => { if (!child.killed) child.kill('SIGKILL'); }, 5000).unref?.(); } } let settled = false; function settle(status, exitCode) { if (settled) return; settled = true; activeJobs.get(jobId)?.children.delete(host.id); finishHost(jobId, host.id, status, exitCode); resolve(); } child.stdout.on('data', (chunk) => appendChunk(jobId, host.id, 'stdout', chunk)); child.stderr.on('data', (chunk) => appendChunk(jobId, host.id, 'stderr', chunk)); child.on('error', (error) => { for (const line of powerShellSpawnFailureLines(error, executable)) appendLog(jobId, host.id, 'stderr', line); settle('failed', 127); }); child.on('close', (code, signal) => { if (settled) return; if (signal) { for (const line of powerShellSignalFailureLines(signal, executable)) appendLog(jobId, host.id, 'stderr', line); } if (isJobCanceled(jobId)) { settle('canceled', 130); return; } const exitCode = code ?? 1; settle(exitCode === 0 ? 'completed' : 'failed', exitCode); }); }); } function appendChunk(jobId, hostId, stream, chunk) { String(chunk) .split(/\r?\n/) .filter(Boolean) .forEach((line) => appendLog(jobId, hostId, stream, line)); } function appendLog(jobId, hostId, stream, line) { db.prepare(` INSERT INTO job_logs (id, job_id, host_id, stream, line, created_at) VALUES (?, ?, ?, ?, ?, ?) `).run(id('log'), jobId, hostId, stream, line, now()); } export function appendJobLog(jobId, stream, line, hostId = null) { appendLog(jobId, hostId, stream, line); } function finishHost(jobId, hostId, status, exitCode) { db.prepare(` UPDATE job_hosts SET status = ?, exit_code = ?, ended_at = ? WHERE job_id = ? AND host_id = ? `).run(status, exitCode, now(), jobId, hostId); appendLog(jobId, hostId, 'system', `Finished with status ${status} and exit code ${exitCode}`); } function cancelQueuedHosts(jobId) { const timestamp = now(); const rows = db.prepare('SELECT host_id FROM job_hosts WHERE job_id = ? AND status IN (?, ?)').all(jobId, 'queued', 'running'); db.prepare(` UPDATE job_hosts SET status = 'canceled', exit_code = COALESCE(exit_code, 130), ended_at = COALESCE(ended_at, ?) WHERE job_id = ? AND status IN ('queued', 'running') `).run(timestamp, jobId); for (const row of rows) appendLog(jobId, row.host_id, 'system', 'Host execution canceled by job management.'); } function isJobCanceled(jobId) { if (activeJobs.get(jobId)?.canceled) return true; const row = db.prepare('SELECT status FROM jobs WHERE id = ?').get(jobId); return row?.status === 'canceled'; } export function activeExecutionJobIds() { return [...activeJobs.keys()]; } export function cancelExecutionJob(jobId, { reason = '', actorUserId = '' } = {}) { const job = db.prepare('SELECT * FROM jobs WHERE id = ?').get(jobId); if (!job) return { canceled: false, reason: 'Job not found.' }; if (!['queued', 'running'].includes(job.status)) { return { canceled: false, reason: `Job is ${job.status} and cannot be canceled.` }; } const active = activeJobs.get(jobId); if (active) active.canceled = true; const children = active ? [...active.children.values()] : []; appendLog(jobId, null, 'system', `Cancel requested${actorUserId ? ` by ${actorUserId}` : ''}${reason ? `: ${reason}` : '.'}`); db.prepare('UPDATE jobs SET status = ?, ended_at = COALESCE(ended_at, ?) WHERE id = ?').run('canceled', now(), jobId); cancelQueuedHosts(jobId); for (const child of children) { try { child.kill('SIGTERM'); setTimeout(() => { if (!child.killed) child.kill('SIGKILL'); }, 5000).unref?.(); } catch (error) { logger.warn('failed to signal child process during job cancel', { jobId, error: error.message }); } } return { canceled: true, signaledProcesses: children.length }; } function runtimeVariableValues(host, executionContext) { const assetManifest = JSON.stringify(executionContext.assets || []); return [ { name: 'POSHM_JOB_ID', value: executionContext.jobId || '', valueType: 'string' }, { name: 'POSHM_RUNPLAN_ID', value: executionContext.runplanId || '', valueType: 'string' }, { name: 'POSHM_SCRIPT_ID', value: executionContext.scriptId || '', valueType: 'string' }, { name: 'POSHM_SCRIPT_NAME', value: executionContext.scriptName || '', valueType: 'string' }, { name: 'POSHM_HOST_ID', value: host.id, valueType: 'string' }, { name: 'POSHM_HOST_NAME', value: host.name, valueType: 'string' }, { name: 'POSHM_HOST_ADDRESS', value: host.address, valueType: 'string' }, { name: 'POSHM_HOST_TRANSPORT', value: host.transport, valueType: 'string' }, { name: 'POSHM_HOST_OS_FAMILY', value: normalizeOsFamily(host.os_family || host.osFamily, 'other'), valueType: 'string' }, { name: 'POSHM_ASSET_MANIFEST', value: assetManifest, valueType: 'string' }, { name: 'POSHM_TRIGGERED_BY', value: executionContext.triggeredBy || '', valueType: 'string' } ]; } function buildVariablePrelude(runtimeVariables, customVariables) { const envLines = runtimeVariables.map((variable) => `$env:${variable.name} = ${psLiteral(variable.value, variable.valueType)}`); const setLines = [...runtimeVariables, ...customVariables] .filter((variable) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(variable.name)) .map((variable) => `Set-Variable -Name ${psString(variable.name)} -Value ${psLiteral(variable.value, variable.valueType)} -Scope Script`); return [ '# POSHManager runtime variables', ...envLines, ...setLines ].join('\n'); } function renderTokens(content, variables) { let rendered = String(content || ''); for (const variable of variables) { const token = `{{${variable.name}}}`; rendered = rendered.split(token).join(String(variable.value ?? '')); } return rendered; } function psLiteral(value, type = 'string') { if (type === 'boolean') return String(value).toLowerCase() === 'true' || value === true ? '$true' : '$false'; if (type === 'number' && value !== '' && Number.isFinite(Number(value))) return String(Number(value)); if (type === 'json') return `(ConvertFrom-Json -InputObject ${psString(value)})`; return psString(value); } function psString(value) { return `'${String(value || '').replace(/'/g, "''")}'`; }