Log Startdialog added
This commit is contained in:
+151
-111
@@ -21,6 +21,7 @@ from .software_packages import build_deployment_user_script, load_package_manife
|
||||
|
||||
_DISPATCH_SLOT_LOCK = threading.Lock()
|
||||
_NEXT_DISPATCH_MONOTONIC = 0.0
|
||||
_SOFTWARE_INVENTORY_MESHCTRL_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _wait_for_dispatch_slot() -> tuple[float, float]:
|
||||
@@ -744,134 +745,173 @@ def execute_job(job_id: int, token: str, callback_base: str) -> None:
|
||||
f'Execution timeout: {execution_timeout_seconds or timeout} seconds',
|
||||
]
|
||||
|
||||
if upload_required:
|
||||
suffix=_script_extension(interpreter)
|
||||
temp_dir=tempfile.mkdtemp(prefix=f'assetmanager-job-{job.id}-')
|
||||
temp_path=str(Path(temp_dir)/('run'+suffix))
|
||||
Path(temp_path).write_text(payload,encoding='utf-8',newline='\n')
|
||||
diagnostics += [f'Remote directory: {remote_dir}',f'Remote file: {remote_file}',f'Local staging bytes: {os.path.getsize(temp_path)}']
|
||||
inventory_meshctrl_lock_acquired = False
|
||||
if job.job_type == 'software_inventory':
|
||||
lock_started = time.monotonic()
|
||||
_SOFTWARE_INVENTORY_MESHCTRL_LOCK.acquire()
|
||||
inventory_meshctrl_lock_acquired = True
|
||||
diagnostics += [
|
||||
'MeshCtrl isolation: software inventory serialized',
|
||||
f'MeshCtrl isolation wait: {time.monotonic() - lock_started:.3f} seconds',
|
||||
]
|
||||
try:
|
||||
if upload_required:
|
||||
suffix=_script_extension(interpreter)
|
||||
temp_dir=tempfile.mkdtemp(prefix=f'assetmanager-job-{job.id}-')
|
||||
temp_path=str(Path(temp_dir)/('run'+suffix))
|
||||
Path(temp_path).write_text(payload,encoding='utf-8',newline='\n')
|
||||
diagnostics += [f'Remote directory: {remote_dir}',f'Remote file: {remote_file}',f'Local staging bytes: {os.path.getsize(temp_path)}']
|
||||
|
||||
if remote_cleanup_enabled:
|
||||
stale_cleanup=_cleanup_stale_remote_job_directories(
|
||||
cfg,asset,password,job.platform,remote_dir,remote_job_retention_hours,min(timeout,120)
|
||||
)
|
||||
diagnostics += [
|
||||
'--- Stale remote job cleanup stdout ---',stale_cleanup.stdout or '',
|
||||
'--- Stale remote job cleanup stderr ---',stale_cleanup.stderr or '',
|
||||
f'Stale remote job cleanup return code: {stale_cleanup.returncode}',
|
||||
]
|
||||
if remote_cleanup_enabled:
|
||||
stale_cleanup=_cleanup_stale_remote_job_directories(
|
||||
cfg,asset,password,job.platform,remote_dir,remote_job_retention_hours,min(timeout,120)
|
||||
)
|
||||
diagnostics += [
|
||||
'--- Stale remote job cleanup stdout ---',stale_cleanup.stdout or '',
|
||||
'--- Stale remote job cleanup stderr ---',stale_cleanup.stderr or '',
|
||||
f'Stale remote job cleanup return code: {stale_cleanup.returncode}',
|
||||
]
|
||||
|
||||
prepared=_prepare_remote_directory(cfg,asset,password,job.platform,remote_dir,timeout,remote_file)
|
||||
diagnostics += ['--- Prepare directory stdout ---',prepared.stdout or '','--- Prepare directory stderr ---',prepared.stderr or '',f'Prepare return code: {prepared.returncode}']
|
||||
if prepared.returncode!=0: raise RuntimeError(f'Remote Jobverzeichnis konnte nicht erstellt werden: {prepared.stderr or prepared.stdout}')
|
||||
prepared=_prepare_remote_directory(cfg,asset,password,job.platform,remote_dir,timeout,remote_file)
|
||||
diagnostics += ['--- Prepare directory stdout ---',prepared.stdout or '','--- Prepare directory stderr ---',prepared.stderr or '',f'Prepare return code: {prepared.returncode}']
|
||||
if prepared.returncode!=0: raise RuntimeError(f'Remote Jobverzeichnis konnte nicht erstellt werden: {prepared.stderr or prepared.stdout}')
|
||||
|
||||
upload_results=[]
|
||||
uploaded=_upload_script(cfg,asset,password,temp_path,remote_dir,timeout)
|
||||
upload_results.append(uploaded)
|
||||
diagnostics += ['--- Upload attempt 1 stdout ---',uploaded.stdout or '','--- Upload attempt 1 stderr ---',uploaded.stderr or '',f'Upload attempt 1 return code: {uploaded.returncode}']
|
||||
upload_text=(uploaded.stdout or '')+'\n'+(uploaded.stderr or '')
|
||||
upload_ok=uploaded.returncode==0 and 'Upload done' in upload_text and 'Upload error' not in upload_text
|
||||
if not upload_ok:
|
||||
# MeshCtrl can return code 0 together with "Upload error".
|
||||
# Remove only the stale target and retry once. Do not touch ACLs of already uploaded files.
|
||||
repair=_prepare_remote_directory(cfg,asset,password,job.platform,remote_dir,min(timeout,120),remote_file,reset_acl=False)
|
||||
diagnostics += ['--- Upload repair stdout ---',repair.stdout or '','--- Upload repair stderr ---',repair.stderr or '',f'Upload repair return code: {repair.returncode}']
|
||||
upload_results=[]
|
||||
uploaded=_upload_script(cfg,asset,password,temp_path,remote_dir,timeout)
|
||||
upload_results.append(uploaded)
|
||||
diagnostics += ['--- Upload attempt 2 stdout ---',uploaded.stdout or '','--- Upload attempt 2 stderr ---',uploaded.stderr or '',f'Upload attempt 2 return code: {uploaded.returncode}']
|
||||
diagnostics += ['--- Upload attempt 1 stdout ---',uploaded.stdout or '','--- Upload attempt 1 stderr ---',uploaded.stderr or '',f'Upload attempt 1 return code: {uploaded.returncode}']
|
||||
upload_text=(uploaded.stdout or '')+'\n'+(uploaded.stderr or '')
|
||||
upload_ok=uploaded.returncode==0 and 'Upload done' in upload_text and 'Upload error' not in upload_text
|
||||
if not upload_ok:
|
||||
combined='\n\n'.join(((item.stdout or '')+'\n'+(item.stderr or '')).strip() for item in upload_results if ((item.stdout or '')+(item.stderr or '')).strip())
|
||||
raise RuntimeError(f'MeshCentral-Dateiupload fehlgeschlagen nach 2 Versuchen: {combined.strip()}')
|
||||
if not upload_ok:
|
||||
# MeshCtrl can return code 0 together with "Upload error".
|
||||
# Remove only the stale target and retry once. Do not touch ACLs of already uploaded files.
|
||||
repair=_prepare_remote_directory(cfg,asset,password,job.platform,remote_dir,min(timeout,120),remote_file,reset_acl=False)
|
||||
diagnostics += ['--- Upload repair stdout ---',repair.stdout or '','--- Upload repair stderr ---',repair.stderr or '',f'Upload repair return code: {repair.returncode}']
|
||||
uploaded=_upload_script(cfg,asset,password,temp_path,remote_dir,timeout)
|
||||
upload_results.append(uploaded)
|
||||
diagnostics += ['--- Upload attempt 2 stdout ---',uploaded.stdout or '','--- Upload attempt 2 stderr ---',uploaded.stderr or '',f'Upload attempt 2 return code: {uploaded.returncode}']
|
||||
upload_text=(uploaded.stdout or '')+'\n'+(uploaded.stderr or '')
|
||||
upload_ok=uploaded.returncode==0 and 'Upload done' in upload_text and 'Upload error' not in upload_text
|
||||
if not upload_ok:
|
||||
combined='\n\n'.join(((item.stdout or '')+'\n'+(item.stderr or '')).strip() for item in upload_results if ((item.stdout or '')+(item.stderr or '')).strip())
|
||||
raise RuntimeError(f'MeshCentral-Dateiupload fehlgeschlagen nach 2 Versuchen: {combined.strip()}')
|
||||
|
||||
if job.job_type == 'software_deployment':
|
||||
payload_files = package_payload_files(job.package_id, job.action)
|
||||
diagnostics += [f'Package payload files: {len(payload_files)}']
|
||||
for package_file in payload_files:
|
||||
remote_package_file = remote_dir + '\\' + package_file.name
|
||||
package_uploads = []
|
||||
package_uploaded = _upload_script(cfg, asset, password, str(package_file), remote_dir, timeout)
|
||||
package_uploads.append(package_uploaded)
|
||||
diagnostics += [
|
||||
f'--- Package upload {package_file.name} attempt 1 stdout ---', package_uploaded.stdout or '',
|
||||
f'--- Package upload {package_file.name} attempt 1 stderr ---', package_uploaded.stderr or '',
|
||||
f'Package upload {package_file.name} attempt 1 return code: {package_uploaded.returncode}',
|
||||
]
|
||||
package_text = (package_uploaded.stdout or '') + '\n' + (package_uploaded.stderr or '')
|
||||
package_ok = package_uploaded.returncode == 0 and 'Upload done' in package_text and 'Upload error' not in package_text
|
||||
if not package_ok:
|
||||
package_repair = _prepare_remote_directory(
|
||||
cfg,
|
||||
asset,
|
||||
password,
|
||||
job.platform,
|
||||
remote_dir,
|
||||
min(timeout, 120),
|
||||
remote_package_file,
|
||||
reset_acl=False,
|
||||
)
|
||||
diagnostics += [
|
||||
f'--- Package upload {package_file.name} repair stdout ---', package_repair.stdout or '',
|
||||
f'--- Package upload {package_file.name} repair stderr ---', package_repair.stderr or '',
|
||||
f'Package upload {package_file.name} repair return code: {package_repair.returncode}',
|
||||
]
|
||||
if job.job_type == 'software_deployment':
|
||||
payload_files = package_payload_files(job.package_id, job.action)
|
||||
diagnostics += [f'Package payload files: {len(payload_files)}']
|
||||
for package_file in payload_files:
|
||||
remote_package_file = remote_dir + '\\' + package_file.name
|
||||
package_uploads = []
|
||||
package_uploaded = _upload_script(cfg, asset, password, str(package_file), remote_dir, timeout)
|
||||
package_uploads.append(package_uploaded)
|
||||
diagnostics += [
|
||||
f'--- Package upload {package_file.name} attempt 2 stdout ---', package_uploaded.stdout or '',
|
||||
f'--- Package upload {package_file.name} attempt 2 stderr ---', package_uploaded.stderr or '',
|
||||
f'Package upload {package_file.name} attempt 2 return code: {package_uploaded.returncode}',
|
||||
f'--- Package upload {package_file.name} attempt 1 stdout ---', package_uploaded.stdout or '',
|
||||
f'--- Package upload {package_file.name} attempt 1 stderr ---', package_uploaded.stderr or '',
|
||||
f'Package upload {package_file.name} attempt 1 return code: {package_uploaded.returncode}',
|
||||
]
|
||||
package_text = (package_uploaded.stdout or '') + '\n' + (package_uploaded.stderr or '')
|
||||
package_ok = package_uploaded.returncode == 0 and 'Upload done' in package_text and 'Upload error' not in package_text
|
||||
if not package_ok:
|
||||
combined = '\n\n'.join(
|
||||
((item.stdout or '') + '\n' + (item.stderr or '')).strip()
|
||||
for item in package_uploads
|
||||
if ((item.stdout or '') + (item.stderr or '')).strip()
|
||||
)
|
||||
raise RuntimeError(
|
||||
f'MeshCentral-Paketdateiupload fehlgeschlagen fuer {package_file.name}: {combined.strip()}'
|
||||
)
|
||||
if not package_ok:
|
||||
package_repair = _prepare_remote_directory(
|
||||
cfg,
|
||||
asset,
|
||||
password,
|
||||
job.platform,
|
||||
remote_dir,
|
||||
min(timeout, 120),
|
||||
remote_package_file,
|
||||
reset_acl=False,
|
||||
)
|
||||
diagnostics += [
|
||||
f'--- Package upload {package_file.name} repair stdout ---', package_repair.stdout or '',
|
||||
f'--- Package upload {package_file.name} repair stderr ---', package_repair.stderr or '',
|
||||
f'Package upload {package_file.name} repair return code: {package_repair.returncode}',
|
||||
]
|
||||
package_uploaded = _upload_script(cfg, asset, password, str(package_file), remote_dir, timeout)
|
||||
package_uploads.append(package_uploaded)
|
||||
diagnostics += [
|
||||
f'--- Package upload {package_file.name} attempt 2 stdout ---', package_uploaded.stdout or '',
|
||||
f'--- Package upload {package_file.name} attempt 2 stderr ---', package_uploaded.stderr or '',
|
||||
f'Package upload {package_file.name} attempt 2 return code: {package_uploaded.returncode}',
|
||||
]
|
||||
package_text = (package_uploaded.stdout or '') + '\n' + (package_uploaded.stderr or '')
|
||||
package_ok = package_uploaded.returncode == 0 and 'Upload done' in package_text and 'Upload error' not in package_text
|
||||
if not package_ok:
|
||||
combined = '\n\n'.join(
|
||||
((item.stdout or '') + '\n' + (item.stderr or '')).strip()
|
||||
for item in package_uploads
|
||||
if ((item.stdout or '') + (item.stderr or '')).strip()
|
||||
)
|
||||
raise RuntimeError(
|
||||
f'MeshCentral-Paketdateiupload fehlgeschlagen fuer {package_file.name}: {combined.strip()}'
|
||||
)
|
||||
|
||||
preflight=_remote_script_preflight(cfg,asset,password,job.platform,remote_file,min(timeout,120))
|
||||
diagnostics += [
|
||||
'--- Remote script preflight stdout ---', preflight.stdout or '',
|
||||
'--- Remote script preflight stderr ---', preflight.stderr or '',
|
||||
f'Remote script preflight return code: {preflight.returncode}',
|
||||
]
|
||||
preflight_text=(preflight.stdout or '')+'\n'+(preflight.stderr or '')
|
||||
if preflight.returncode!=0 or (job.platform=='windows' and ('File exists: True' not in preflight_text or 'Readable: True' not in preflight_text)):
|
||||
raise RuntimeError('Remote Jobskript ist nach dem Upload nicht lesbar. Siehe Remote script preflight im Dispatcherlog.')
|
||||
preflight_ok = False
|
||||
preflight_attempts = 3
|
||||
for preflight_attempt in range(1, preflight_attempts + 1):
|
||||
preflight = _remote_script_preflight(
|
||||
cfg,
|
||||
asset,
|
||||
password,
|
||||
job.platform,
|
||||
remote_file,
|
||||
min(timeout, 120),
|
||||
)
|
||||
diagnostics += [
|
||||
f'--- Remote script preflight attempt {preflight_attempt}/{preflight_attempts} stdout ---',
|
||||
preflight.stdout or '',
|
||||
f'--- Remote script preflight attempt {preflight_attempt}/{preflight_attempts} stderr ---',
|
||||
preflight.stderr or '',
|
||||
f'Remote script preflight attempt {preflight_attempt}/{preflight_attempts} return code: {preflight.returncode}',
|
||||
]
|
||||
preflight_text = (preflight.stdout or '') + '\n' + (preflight.stderr or '')
|
||||
preflight_ok = preflight.returncode == 0 and (
|
||||
job.platform != 'windows'
|
||||
or ('File exists: True' in preflight_text and 'Readable: True' in preflight_text)
|
||||
)
|
||||
if preflight_ok:
|
||||
break
|
||||
if preflight_attempt < preflight_attempts:
|
||||
diagnostics += [
|
||||
f'Remote script preflight attempt {preflight_attempt}/{preflight_attempts}: not readable',
|
||||
'Retrying remote script preflight in 1 second...',
|
||||
]
|
||||
time.sleep(1)
|
||||
|
||||
diagnostics += [f'Remote execution shell: {launch_shell}',f'Remote execution command: {launch_command}',f'MeshCtrl PowerShell mode: False']
|
||||
started=datetime.utcnow()
|
||||
result=_launch_uploaded_script(cfg,asset,password,job.platform,interpreter,remote_file,timeout)
|
||||
finished=datetime.utcnow()
|
||||
diagnostics += [f'Execution started UTC: {started.isoformat()}Z',f'Execution finished UTC: {finished.isoformat()}Z','--- Execution stdout ---',result.stdout or '','--- Execution stderr ---',result.stderr or '',f'Execution return code: {result.returncode}']
|
||||
else:
|
||||
started=datetime.utcnow()
|
||||
action=['RunCommand','--id',asset.mesh_node_id,'--run',payload,'--reply']
|
||||
if job.platform=='windows' and interpreter=='powershell': action += ['--powershell']
|
||||
result=_run_meshctrl(cfg,asset,password,action,timeout)
|
||||
finished=datetime.utcnow()
|
||||
diagnostics += ['--- Direct execution stdout ---',result.stdout or '','--- Direct execution stderr ---',result.stderr or '',f'Execution return code: {result.returncode}']
|
||||
if not preflight_ok:
|
||||
raise RuntimeError('Remote Jobskript ist nach dem Upload nicht lesbar. Siehe Remote script preflight im Dispatcherlog.')
|
||||
|
||||
now=datetime.utcnow()
|
||||
# MeshCtrl itself returns code 0 even for some remote PowerShell parser
|
||||
# errors. Only delete the remote files after a completed successful
|
||||
# callback, never merely because the MeshCtrl process exited cleanly.
|
||||
db.expire_all()
|
||||
callback_state=db.get(SoftwareJob,job.id)
|
||||
cleanup_result=None
|
||||
callback_success=bool(callback_state and callback_state.callback_completed_at is not None and callback_state.status=='success')
|
||||
diagnostics += [f'Callback completed before cleanup: {bool(callback_state and callback_state.callback_completed_at)}',f'Callback status before cleanup: {callback_state.status if callback_state else "unknown"}']
|
||||
if upload_required and callback_success:
|
||||
cleanup_result=_cleanup_remote_directory(cfg,asset,password,job.platform,remote_dir,min(timeout,120))
|
||||
diagnostics += ['--- Cleanup stdout ---',cleanup_result.stdout or '','--- Cleanup stderr ---',cleanup_result.stderr or '',f'Cleanup return code: {cleanup_result.returncode}']
|
||||
elif upload_required:
|
||||
diagnostics += ['Remote cleanup skipped: no successful completed callback; files retained for diagnosis.']
|
||||
diagnostics += [f'Remote execution shell: {launch_shell}',f'Remote execution command: {launch_command}',f'MeshCtrl PowerShell mode: False']
|
||||
started=datetime.utcnow()
|
||||
result=_launch_uploaded_script(cfg,asset,password,job.platform,interpreter,remote_file,timeout)
|
||||
finished=datetime.utcnow()
|
||||
diagnostics += [f'Execution started UTC: {started.isoformat()}Z',f'Execution finished UTC: {finished.isoformat()}Z','--- Execution stdout ---',result.stdout or '','--- Execution stderr ---',result.stderr or '',f'Execution return code: {result.returncode}']
|
||||
else:
|
||||
started=datetime.utcnow()
|
||||
action=['RunCommand','--id',asset.mesh_node_id,'--run',payload,'--reply']
|
||||
if job.platform=='windows' and interpreter=='powershell': action += ['--powershell']
|
||||
result=_run_meshctrl(cfg,asset,password,action,timeout)
|
||||
finished=datetime.utcnow()
|
||||
diagnostics += ['--- Direct execution stdout ---',result.stdout or '','--- Direct execution stderr ---',result.stderr or '',f'Execution return code: {result.returncode}']
|
||||
|
||||
now=datetime.utcnow()
|
||||
# MeshCtrl itself returns code 0 even for some remote PowerShell parser
|
||||
# errors. Only delete the remote files after a completed successful
|
||||
# callback, never merely because the MeshCtrl process exited cleanly.
|
||||
db.expire_all()
|
||||
callback_state=db.get(SoftwareJob,job.id)
|
||||
cleanup_result=None
|
||||
callback_success=bool(callback_state and callback_state.callback_completed_at is not None and callback_state.status=='success')
|
||||
diagnostics += [f'Callback completed before cleanup: {bool(callback_state and callback_state.callback_completed_at)}',f'Callback status before cleanup: {callback_state.status if callback_state else "unknown"}']
|
||||
if upload_required and callback_success:
|
||||
cleanup_result=_cleanup_remote_directory(cfg,asset,password,job.platform,remote_dir,min(timeout,120))
|
||||
diagnostics += ['--- Cleanup stdout ---',cleanup_result.stdout or '','--- Cleanup stderr ---',cleanup_result.stderr or '',f'Cleanup return code: {cleanup_result.returncode}']
|
||||
elif upload_required:
|
||||
diagnostics += ['Remote cleanup skipped: no successful completed callback; files retained for diagnosis.']
|
||||
|
||||
finally:
|
||||
if inventory_meshctrl_lock_acquired:
|
||||
_SOFTWARE_INVENTORY_MESHCTRL_LOCK.release()
|
||||
|
||||
db.query(SoftwareJob).filter(SoftwareJob.id==job.id).update({SoftwareJob.meshctrl_stdout:'\n'.join(diagnostics)[-30000:],SoftwareJob.meshctrl_stderr:(result.stderr or '')[-20000:],SoftwareJob.sent_at:now},synchronize_session=False)
|
||||
db.commit()
|
||||
|
||||
Reference in New Issue
Block a user