From 21d0ec13bf9f99e2ac3cfc7577057c60182b700e Mon Sep 17 00:00:00 2001 From: Francesco Bragagna Date: Mon, 7 Sep 2026 13:57:38 +0200 Subject: [PATCH 1/3] fix output redirection when a backup is written on the server Three problems with the file-redirecting mode of the Output class, all of them visible when running a server side backup: - ob_start() was called without a chunk size, so the whole dump was held in memory and only written to the file at the end. On a large database this hits memory_limit and the request dies with nothing to show for it. Flushing every 64 KB keeps memory flat and lets the file grow as it goes. - When the target file could not be opened (backup folder missing or not writable) the buffer was started anyway, so the error message the caller printed afterwards was swallowed by the callback and written nowhere. The user saw an empty page instead of the reason. Do not start buffering when there is no file to write to. - end() called gzclose() and then fell through to fclose() on the same handle, and ran ob_end_flush() again when invoked a second time by the destructor, ending a buffer it does not own. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GkPb2smVVLCiBbnTC3UcRZ --- lib/output.php | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/output.php b/lib/output.php index cf0158f..ce4b805 100644 --- a/lib/output.php +++ b/lib/output.php @@ -16,9 +16,13 @@ define("CLASS_OUTPUT_INCLUDED", "1"); class Output { + // size of the buffer that is written out to the file in one go + const CHUNK_SIZE = 65536; + public $file; public $compression; public $file_handle; + private $buffering = false; // controls output buffering public static function buffer() { @@ -78,7 +82,15 @@ public function __construct( $file, $compression = false ) { } else { $this->file_handle = fopen( $file, 'wb' ); } - ob_start( array( $this, 'output_callback' ) ); + + // nothing to redirect the output to, leave the buffering alone so that the + // caller can still report the failure to the browser + if ( !$this->file_handle ) + return; + + // flush every CHUNK_SIZE bytes instead of holding the whole dump in memory + ob_start( array( $this, 'output_callback' ), self::CHUNK_SIZE ); + $this->buffering = true; } public function __destruct() { @@ -91,12 +103,17 @@ public function is_valid() { // only works if output is being redirected with compression public function end() { - @ob_end_flush(); + // only close the buffer we started ourselves, end() is also called by the destructor + if ( $this->buffering ) { + @ob_end_flush(); + $this->buffering = false; + } + if ( $this->file_handle ) { if ( $this->compression == 'gz' ) { gzclose( $this->file_handle ); } - if ( $this->compression == 'bz' ) { + else if ( $this->compression == 'bz' ) { bzclose( $this->file_handle ); } else { fclose( $this->file_handle ); @@ -114,6 +131,7 @@ public function output_callback( $buffer ) { } else { fwrite( $this->file_handle, $buffer ); } + return ''; // nothing of this goes to the browser } } } From b9c8458d28115288b3bc84a7273478941287572b Mon Sep 17 00:00:00 2001 From: Francesco Bragagna Date: Mon, 7 Sep 2026 13:58:03 +0200 Subject: [PATCH 2/3] show progress while a server side backup is running A server side backup submits the dialog form into its own iframe and the whole dump is written to a file, so nothing is sent to the browser until php is done. The dialog therefore sits unchanged for as long as the backup takes - minutes on a large database - with no way to tell whether anything is happening, and a proxy timeout in between simply looks like a hang. The request is now sent in the background and the dialog polls its progress through status.php, using the getModuleStatus() hook that is already there for upload progress: - lib/backupstate.php records the state of a running backup in tmp/, keyed by a token the dialog generates. Updates are throttled to one write every 0.4s and written through a scratch file, so a poll never reads a half written state. Stale files are removed on the next run. - modules/backup.php answers the poll and only reports on a backup started by the session that is asking, releasing the session lock right away so polling never queues behind the backup itself. - modules/download.php registers the token before releasing the session, counts the selected objects, and updates the state per object and per batch of rows. It answers json when a token is given. ignore_user_abort() keeps a backup alive when the browser goes away, and a shutdown handler records fatal errors so the dialog reports them instead of polling forever. - lib/export/export.php gained an optional per batch progress callback. The dialog now shows a progress bar with the current object, the object and row counters, the bytes written and the elapsed time, and ends with the file name, its size and the duration. Because the state lives in a file, the poll - not the request - decides when the backup is over: a dropped connection no longer loses the result. Selecting no object is refused up front instead of silently producing an empty file. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GkPb2smVVLCiBbnTC3UcRZ --- lib/backupstate.php | 163 ++++++++++++++++++++++++++++++++++++ lib/export/export.php | 8 ++ lib/output.php | 7 ++ modules/backup.php | 51 ++++++++++++ modules/download.php | 101 ++++++++++++++++++++-- modules/views/backup.php | 175 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 496 insertions(+), 9 deletions(-) create mode 100644 lib/backupstate.php diff --git a/lib/backupstate.php b/lib/backupstate.php new file mode 100644 index 0000000..b09b17f --- /dev/null +++ b/lib/backupstate.php @@ -0,0 +1,163 @@ + $max_age ) + @unlink( $file ); + } + } + + public function __construct( $token ) { + $this->token = $token; + $this->path = self::statePath( $token ); + $this->data = array( + 'state' => 'running', + 'db' => '', + 'file' => '', + 'total' => 0, + 'done' => 0, + 'type' => '', + 'object' => '', + 'rows' => 0, + 'totalrows' => 0, + 'bytes' => 0, + 'started' => microtime(true), + 'updated' => microtime(true), + 'message' => '' + ); + } + + public function begin( $total, $db_name, $file_name ) { + $this->data['total'] = (int) $total; + $this->data['db'] = $db_name; + $this->data['file'] = $file_name; + $this->write( true ); + } + + // moves on to the next object of the backup + public function step( $type, $label ) { + $this->data['done']++; + $this->data['type'] = $type; + $this->data['object'] = $label; + $this->rows_base = $this->data['totalrows']; + $this->data['rows'] = 0; + $this->write( true ); + } + + // number of rows exported so far for the current object + public function rows( $count ) { + $this->data['rows'] = (int) $count; + $this->data['totalrows'] = $this->rows_base + (int) $count; + $this->write(); + } + + // called back by the Output class as the dump is written to disk + public function setBytes( $bytes ) { + $this->data['bytes'] = (int) $bytes; + $this->write(); + } + + public function isFinished() { + return $this->data['state'] != 'running'; + } + + public function finish( $state, $message, $extra = array() ) { + $this->data['state'] = $state; // 'done' or 'error' + $this->data['message'] = $message; + foreach( $extra as $key => $value ) + $this->data[$key] = $value; + $this->write( true ); + } + + public function getData() { + return $this->data; + } + + private function write( $force = false ) { + $now = microtime(true); + if ( !$force && ($now - $this->last_write) < $this->write_interval ) + return; + + $this->last_write = $now; + $this->data['updated'] = $now; + + $json = json_encode( $this->data ); + // write to a scratch file first, so a poll never sees a half written state + $tmp = $this->path . '.' . getmypid() . '.tmp'; + if ( @file_put_contents( $tmp, $json ) !== false && @rename( $tmp, $this->path ) ) + return; + + @unlink( $tmp ); + @file_put_contents( $this->path, $json, LOCK_EX ); + } + } +} +?> diff --git a/lib/export/export.php b/lib/export/export.php index b5c6f3e..a2f841d 100644 --- a/lib/export/export.php +++ b/lib/export/export.php @@ -55,6 +55,10 @@ function exportTable($sql, $options) { $id = 0; $field_info = NULL; + // optional callback, invoked after every batch with the row count exported so far + $progress = isset($options['progress']) ? $options['progress'] : NULL; + $rows_done = 0; + while(1) { $tempSql = $sql; if ($applyLimit) @@ -72,8 +76,12 @@ function exportTable($sql, $options) { while($row = $this->db->fetchRow("_temp", 'num')) { print $this->driver->createLine($row, $field_info); + $rows_done++; } + if ($progress) + call_user_func($progress, $rows_done); + if ($numRows == 0 || !$applyLimit) break; diff --git a/lib/output.php b/lib/output.php index ce4b805..7d9e763 100644 --- a/lib/output.php +++ b/lib/output.php @@ -22,6 +22,8 @@ class Output { public $file; public $compression; public $file_handle; + public $bytes = 0; // bytes handed over to the file so far + public $progress = null; // optional object with a setBytes() method private $buffering = false; // controls output buffering @@ -131,6 +133,11 @@ public function output_callback( $buffer ) { } else { fwrite( $this->file_handle, $buffer ); } + + $this->bytes += strlen( $buffer ); + if ( $this->progress ) + $this->progress->setBytes( $this->bytes ); + return ''; // nothing of this goes to the browser } } diff --git a/modules/backup.php b/modules/backup.php index 7e3fa18..3af7c4d 100644 --- a/modules/backup.php +++ b/modules/backup.php @@ -27,4 +27,55 @@ function processRequest(&$db) { echo view( array($folder.'/backup', 'backup'), $replace, $object_list); } + /** + * Progress of a running backup, polled by the dialog through status.php while the + * backup request itself is still busy writing the dump. + * + * NOTE: status.php is a minimal bootstrap, neither v() nor __() exist here. + */ + function getModuleStatus( $id ) { + include_once(BASE_PATH . "/lib/backupstate.php"); + + $status = array('c' => 0, 'r' => 0, 's' => 0, 'state' => 'unknown'); + + $token = BackupState::sanitizeToken( $id ); + $owned = ( $token !== false && $token === Session::get('backup', 'token') ); + + // this is polled once per second, do not sit on the session lock while doing so + Session::close(); + + // only report on the backup started by this very session + if ( !$owned ) + return $status; + + $data = BackupState::read( $token ); + if ( !is_array($data) ) { + // the backup request has not written its first update yet + $status['s'] = 1; + $status['state'] = 'starting'; + return $status; + } + + $percent = 0; + if ( $data['total'] > 0 ) + $percent = (int) floor( $data['done'] / $data['total'] * 100 ); + if ( $data['state'] == 'running' && $percent > 99 ) + $percent = 99; // the last object is still being written + if ( $data['state'] == 'done' ) + $percent = 100; + + $status['c'] = $percent; + $status['s'] = 1; + $status['r'] = $data['state'] == 'running' ? 0 : 1; + $status['state'] = $data['state']; + $status['elapsed'] = (int) ( microtime(true) - $data['started'] ); + + foreach( array('done', 'total', 'type', 'object', 'rows', 'totalrows', 'bytes', 'file', 'message') as $key ) + $status[$key] = isset($data[$key]) ? $data[$key] : ''; + + $status['size'] = isset($data['size']) ? $data['size'] : 0; + + return $status; + } + ?> \ No newline at end of file diff --git a/modules/download.php b/modules/download.php index 1f46ded..611f4bb 100644 --- a/modules/download.php +++ b/modules/download.php @@ -14,6 +14,19 @@ function processRequest(&$db) { set_time_limit(0); } + include_once(BASE_PATH . "/lib/backupstate.php"); + + // the browser polls status.php while this request is still running. The token has to + // be registered in the session before we release it, otherwise the poll cannot + // tell whether the backup belongs to the user asking about it + $backup_token = BackupState::sanitizeToken( v($_REQUEST['token']) ); + if ( $backup_token !== false ) { + Session::set('backup', 'token', $backup_token); + // the dump is written to a file, so a disconnected browser (dialog closed, + // proxy timeout) is no reason to throw away a backup that is halfway done + ignore_user_abort(true); + } + Session::close(); switch( $_REQUEST['id'] ) { @@ -22,20 +35,55 @@ function processRequest(&$db) { $compression = v($_REQUEST['compression']); $filename = v($_REQUEST['filename']); $file = get_backup_filename( $compression, $filename ); + + $state = false; + if ( $backup_token !== false ) { + BackupState::cleanup(); + $state = new BackupState( $backup_token ); + $state->begin( countBackupObjects($db), Session::get('db', 'name'), $file ? basename($file) : '' ); + // a fatal error (memory limit, killed worker) must not leave the dialog polling forever + register_shutdown_function( 'backupShutdown', $state ); + } + if ( $file ) { include_once(BASE_PATH . "/lib/output.php"); $output = new Output( $file, $compression ); $message = '
'.__('Database backup successfully created').'
'; if ( $output->is_valid() ) { - downloadDatabase($db, false); + $output->progress = $state ? $state : null; + $written = downloadDatabase($db, false, $state); $output->end(); + + if ( $written === false ) { + @unlink( $file ); // nothing was selected, do not leave an empty backup behind + $message = '
'.__('Select objects to include in backup').'
'; + if ( $state ) + $state->finish( 'error', __('Select objects to include in backup') ); + } + else if ( $state ) { + clearstatcache(); + $size = @filesize( $file ); + $state->finish( 'done', __('Database backup successfully created'), + array( 'size' => $size === false ? 0 : $size ) ); + } } else { $message = '
'.__('Failed to create database backup').'
'; + if ( $state ) + $state->finish( 'error', __('Backup folder does not exist or is not writable') ); } } else { $message = '
'.__('Invalid filename format').'
'; + if ( $state ) + $state->finish( 'error', __('Invalid filename format') ); + } + + if ( $state ) { + // the dialog stays where it is and shows the result, no page reload + header('Content-Type: application/json; charset=utf-8'); + echo json_encode( $state->getData() ); + } else { + echo view( 'backup', array( 'MESSAGE' => $message, 'FILENAME' => htmlspecialchars($filename) ), $db->getObjectList() ); } - echo view( 'backup', array( 'MESSAGE' => $message, 'FILENAME' => htmlspecialchars($filename) ), $db->getObjectList() ); } break; case 'exportres': { downloadResults($db); @@ -109,7 +157,41 @@ function downloadTable(&$db, $table) { } - function downloadDatabase(&$db, $headers = true) { + // number of objects the backup is going to write, used to show a meaningful progress bar + function countBackupObjects(&$db) { + $total = 0; + if ( is_array(v($_POST["tables"])) ) + $total += count($_POST["tables"]); + + $export_type = v($_REQUEST["exptype"]); + if ($export_type == "all" || $export_type == "struct") { + $object_types = $db->getObjectTypes(); + unset($object_types[0]); // tables are already counted above + foreach($object_types as $type) { + if ( is_array(v($_POST[$type])) ) + $total += count($_POST[$type]); + } + } + + return $total; + } + + // last resort reporting: if the request dies before the backup is marked as finished, + // record why, so the dialog shows an error instead of polling forever + function backupShutdown( $state ) { + if ( $state->isFinished() ) + return; + + $message = __('Backup was interrupted before it could complete'); + $error = error_get_last(); + $fatal = array(E_ERROR, E_PARSE, E_CORE_ERROR, E_COMPILE_ERROR, E_USER_ERROR); + if ( is_array($error) && in_array($error['type'], $fatal) ) + $message .= ': ' . $error['message']; + + $state->finish( 'error', $message ); + } + + function downloadDatabase(&$db, $headers = true, $state = false) { // don't make POST as REQUEST here. it won't work :P if ( !( is_array(v($_POST["tables"])) || is_array(v($_POST["views"])) || is_array(v($_POST["procs"])) ||is_array(v($_POST["funcs"])) || is_array(v($_POST["triggers"])) ||is_array(v($_POST["events"])) ) ) @@ -133,12 +215,18 @@ function downloadDatabase(&$db, $headers = true) { 'bulkinsert' => v($_REQUEST['bulkinsert']), 'bulksize' => v($_REQUEST['bulklimit']) == 'on' ? v($_REQUEST['bulksize'])*1024 : 0 ); + if ( $state ) + $options['progress'] = array($state, 'rows'); + foreach($tables as $table_name) { // is this table required in export? $key = array_search($table_name, $_POST["tables"]); if ($key === FALSE) continue; + if ( $state ) + $state->step('table', $table_name); + // -- -truncate command -- if (v($_REQUEST["emptycmd"]) == "on") { echo "\n" . $db->getTruncateCommand( $table_name ) . ";\n"; @@ -187,7 +275,7 @@ function downloadDatabase(&$db, $headers = true) { if (is_array(v($_POST[$type])) && count($_POST[$type]) > 0) { $func = 'get' . ucfirst( $type ); $name = substr($type, 0, -1); - exportObject($db, $name, $_POST[$type], $db->$func()); + exportObject($db, $name, $_POST[$type], $db->$func(), $state); } } } @@ -197,12 +285,15 @@ function downloadDatabase(&$db, $headers = true) { // ===================================== - function exportObject(&$db, $name, $list, $tables) { + function exportObject(&$db, $name, $list, $tables, $state = false) { foreach($tables as $table_name) { $key = array_search($table_name, $list); if ($key === FALSE) continue; + if ( $state ) + $state->step($name, $table_name); + if (v($_REQUEST["dropcmd"]) == "on") print "\ndrop $name if exists " . $db->quote($table_name) . ";\n"; diff --git a/modules/views/backup.php b/modules/views/backup.php index 58b6991..a54f906 100644 --- a/modules/views/backup.php +++ b/modules/views/backup.php @@ -5,18 +5,29 @@ div.objhead { background-color:#ececec; padding: 5px; margin: 0 0 3px 0 } span.toggler { display:inline-block; float:right; cursor: pointer; font-size:16px; margin: -5px 0 0 0 } div.obj { padding:5px; margin:0 0 0 20px } + + div#backup_progress { display:none; margin:3px 3px 6px 3px; padding:6px 10px } + div#backup_bar { height:14px } + div#backup_stage { margin:6px 0 0 0; font-weight:bold; white-space:nowrap; overflow:hidden; text-overflow:ellipsis } + div#backup_detail { margin:2px 0 0 0; color:#666 }