From ef86e0f017a28c7929067fedf43d385a9392942c Mon Sep 17 00:00:00 2001 From: Matej Bagar Date: Sat, 19 Sep 2026 14:43:10 +0200 Subject: [PATCH 1/3] Filter generated diff by size limits --- core/merginapi.cpp | 99 ++++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/core/merginapi.cpp b/core/merginapi.cpp index 4ca9067e9..d78ac9810 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -340,16 +340,16 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) } // Calculate local changes - transaction.diff = compareProjectFiles( - oldServerProject.files, - oldServerProject.files, - localFiles, - transaction.projectDir, - transaction.configAllowed, - transaction.config - ); + ProjectDiff localDiff = compareProjectFiles( + oldServerProject.files, + oldServerProject.files, + localFiles, + transaction.projectDir, + transaction.configAllowed, + transaction.config + ); - CoreUtils::log( "push " + projectFullName, transaction.diff.dump() ); + CoreUtils::log( "push " + projectFullName, localDiff.dump() ); if ( transaction.configAllowed && transaction.config.isValid && transaction.config.selectiveSyncEnabled ) { @@ -361,13 +361,13 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) CoreUtils::log( "push " + projectFullName, QStringLiteral( "Selective sync is not enabled" ) ); } - if ( !transaction.diff.remoteAdded.isEmpty() || - !transaction.diff.remoteUpdated.isEmpty() || - !transaction.diff.remoteDeleted.isEmpty() || - !transaction.diff.conflictRemoteAddedLocalAdded.isEmpty() || - !transaction.diff.conflictRemoteDeletedLocalUpdated.isEmpty() || - !transaction.diff.conflictRemoteUpdatedLocalDeleted.isEmpty() || - !transaction.diff.conflictRemoteUpdatedLocalUpdated.isEmpty() ) + if ( !localDiff.remoteAdded.isEmpty() || + !localDiff.remoteUpdated.isEmpty() || + !localDiff.remoteDeleted.isEmpty() || + !localDiff.conflictRemoteAddedLocalAdded.isEmpty() || + !localDiff.conflictRemoteDeletedLocalUpdated.isEmpty() || + !localDiff.conflictRemoteUpdatedLocalDeleted.isEmpty() || + !localDiff.conflictRemoteUpdatedLocalUpdated.isEmpty() ) { CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Detected remote changes when calculating local changes" ) ); // What to do here? @@ -376,31 +376,34 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) const bool useV2push = true; int fileCounter = 0; - for ( const QString &filePath : std::as_const( transaction.diff.localAdded ) ) + for ( auto filePath = localDiff.localAdded.begin(); filePath != localDiff.localAdded.end(); ++filePath ) { - MerginFile file = findFile( filePath, localFiles ); + if ( fileCounter >= MAX_UPLOAD_CHANGES ) + { + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Maximum amount of changed files reached, push will be split to multiple versions" ) ); + filePath = localDiff.localAdded.erase( filePath ); + continue; + } + + MerginFile file = findFile( *filePath, localFiles ); if ( isFileDiffable( file.path ) && file.size > MAX_UPLOAD_VERSIONED_SIZE ) { CoreUtils::log( "push " + projectFullName, QStringLiteral( "Versionable file \"%1\" exceeded maximum upload size" ).arg( file.path ) ); + filePath = localDiff.localAdded.erase( filePath ); continue; } if ( !isFileDiffable( file.path ) && file.size > MAX_UPLOAD_MEDIA_SIZE ) { CoreUtils::log( "push " + projectFullName, QStringLiteral( "Media file \"%1\" exceeded maximum upload size" ).arg( file.path ) ); + filePath = localDiff.localAdded.erase( filePath ); continue; } - if ( fileCounter >= MAX_UPLOAD_CHANGES ) - { - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Maximum amount of changed files reached, push will be split to multiple versions" ) ); - break; - } - file.chunks = generateChunkIdsForSize( file.size ); - if ( mSupportsSelectiveSync && filePath == sMerginConfigFile ) + if ( mSupportsSelectiveSync && *filePath == sMerginConfigFile ) { // problem, we changed selective sync config locally, this is undefined and sync must stop CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Selective sync config was added locally, this is undefined and might lead to data loss, aborting push..." ) ); @@ -411,37 +414,45 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) fileCounter++; } - for ( const QString &filePath : std::as_const( transaction.diff.localUpdated ) ) + if ( fileCounter >= MAX_UPLOAD_CHANGES ) + { + localDiff.localUpdated.clear(); + } + + for ( auto filePath = localDiff.localUpdated.begin(); filePath != localDiff.localUpdated.end(); ++filePath ) { - MerginFile file = findFile( filePath, localFiles ); + if ( fileCounter >= MAX_UPLOAD_CHANGES ) + { + CoreUtils::log( "push " + projectFullName, QStringLiteral( "Maximum amount of changed files reached, push will be split to multiple versions" ) ); + filePath = localDiff.localUpdated.erase( filePath ); + continue; + } + + MerginFile file = findFile( *filePath, localFiles ); if ( isFileDiffable( file.path ) && file.size > MAX_UPLOAD_VERSIONED_SIZE ) { CoreUtils::log( "push " + projectFullName, QStringLiteral( "Versionable file \"%1\" exceeded maximum upload size" ).arg( file.path ) ); + filePath = localDiff.localUpdated.erase( filePath ); continue; } if ( !isFileDiffable( file.path ) && file.size > MAX_UPLOAD_MEDIA_SIZE ) { CoreUtils::log( "push " + projectFullName, QStringLiteral( "Media file \"%1\" exceeded maximum upload size" ).arg( file.path ) ); + filePath = localDiff.localUpdated.erase( filePath ); continue; } - if ( fileCounter >= MAX_UPLOAD_CHANGES ) - { - CoreUtils::log( "push " + projectFullName, QStringLiteral( "Maximum amount of changed files exceeded" ) ); - break; - } - file.chunks = generateChunkIdsForSize( file.size ); - if ( MerginApi::isFileDiffable( filePath ) ) + if ( isFileDiffable( *filePath ) ) { // try to create a diff QString diffName; - int geodiffRes = GeodiffUtils::createChangeset( transaction.projectDir, filePath, diffName ); + int geodiffRes = GeodiffUtils::createChangeset( transaction.projectDir, *filePath, diffName ); QString diffPath = transaction.projectDir + "/.mergin/" + diffName; - QString basePath = transaction.projectDir + "/.mergin/" + filePath; + QString basePath = transaction.projectDir + "/.mergin/" + *filePath; if ( geodiffRes == GEODIFF_SUCCESS ) { @@ -449,7 +460,7 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) // TODO: this is ugly. our basefile may not need to have the same checksum as the server's // basefile (because each of them have applied the diff independently) so we have to fake it - QByteArray checksumBase = oldServerProject.fileInfo( filePath ).checksum.toLatin1(); + QByteArray checksumBase = oldServerProject.fileInfo( *filePath ).checksum.toLatin1(); file.diffName = diffName; file.diffChecksum = QString::fromLatin1( checksumDiff.data(), checksumDiff.size() ); @@ -457,16 +468,16 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) file.diffBaseChecksum = QString::fromLatin1( checksumBase.data(), checksumBase.size() ); file.chunks = generateChunkIdsForSize( file.diffSize ); - CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 successful: total size %2 bytes" ).arg( filePath ).arg( file.diffSize ) ); + CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 successful: total size %2 bytes" ).arg( *filePath ).arg( file.diffSize ) ); } else { // TODO: remove the diff file (if exists) - CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 FAILED with error %2 (will do full upload)" ).arg( filePath ).arg( geodiffRes ) ); + CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 FAILED with error %2 (will do full upload)" ).arg( *filePath ).arg( geodiffRes ) ); } } - if ( mSupportsSelectiveSync && filePath == sMerginConfigFile ) + if ( mSupportsSelectiveSync && *filePath == sMerginConfigFile ) { // problem, we changed selective sync config locally, this is undefined and sync must stop CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Selective sync config was updated locally, this is undefined and might lead to data loss, aborting push..." ) ); @@ -477,7 +488,9 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) fileCounter++; } - for ( QString filePath : transaction.diff.localDeleted ) + transaction.diff = localDiff; + + for ( const QString &filePath : transaction.diff.localDeleted ) { MerginFile file = findFile( filePath, oldServerProject.files ); transaction.pushChanges.removed.append( file ); @@ -490,12 +503,12 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) } qint64 totalSize = 0; - for ( MerginFile file : transaction.pushChanges.added ) + for ( const MerginFile &file : transaction.pushChanges.added ) { totalSize += file.size; } - for ( MerginFile file : transaction.pushChanges.updated ) + for ( const MerginFile &file : transaction.pushChanges.updated ) { if ( !file.diffName.isEmpty() ) { From ba33cf75980de20b4dc6f2664e828ceed7e5e81b Mon Sep 17 00:00:00 2001 From: Matej Bagar Date: Sat, 19 Sep 2026 15:35:08 +0200 Subject: [PATCH 2/3] Remove done TODO --- core/merginapi.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/merginapi.cpp b/core/merginapi.cpp index d78ac9810..4c1aed9f6 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -527,9 +527,6 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) .arg( transaction.pushQueue.count() ).arg( totalSize ) ); - // TODO: Check here if the total number of files to upload is not larger than accepted by the server (100) - // TODO: Check here if file size of the individual files to upload is not larger than accepted by server (5 GB for gpkg, 10 GB for other file types) - pushStart( projectFullName ); } From 309a5d82b1ccbc0f7da2b13dbf18afee5d715d5b Mon Sep 17 00:00:00 2001 From: Matej Bagar Date: Wed, 23 Sep 2026 22:01:48 +0200 Subject: [PATCH 3/3] Fix found issues --- core/merginapi.cpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/core/merginapi.cpp b/core/merginapi.cpp index 4c1aed9f6..c56348317 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -376,13 +376,17 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) const bool useV2push = true; int fileCounter = 0; - for ( auto filePath = localDiff.localAdded.begin(); filePath != localDiff.localAdded.end(); ++filePath ) + for ( auto filePath = localDiff.localAdded.begin(); filePath != localDiff.localAdded.end(); ) { if ( fileCounter >= MAX_UPLOAD_CHANGES ) { CoreUtils::log( "push " + projectFullName, QStringLiteral( "Maximum amount of changed files reached, push will be split to multiple versions" ) ); - filePath = localDiff.localAdded.erase( filePath ); - continue; + // remove remaining items of localDiff.localAdded + while ( filePath != localDiff.localAdded.end() ) + { + filePath = localDiff.localAdded.erase( filePath ); + } + break; } MerginFile file = findFile( *filePath, localFiles ); @@ -412,6 +416,7 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) transaction.pushChanges.added.append( file ); fileCounter++; + ++filePath; } if ( fileCounter >= MAX_UPLOAD_CHANGES ) @@ -419,13 +424,17 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) localDiff.localUpdated.clear(); } - for ( auto filePath = localDiff.localUpdated.begin(); filePath != localDiff.localUpdated.end(); ++filePath ) + for ( auto filePath = localDiff.localUpdated.begin(); filePath != localDiff.localUpdated.end(); ) { if ( fileCounter >= MAX_UPLOAD_CHANGES ) { CoreUtils::log( "push " + projectFullName, QStringLiteral( "Maximum amount of changed files reached, push will be split to multiple versions" ) ); - filePath = localDiff.localUpdated.erase( filePath ); - continue; + // remove remaining items of localDiff.localUpdated + while ( filePath != localDiff.localUpdated.end() ) + { + filePath = localDiff.localUpdated.erase( filePath ); + } + break; } MerginFile file = findFile( *filePath, localFiles ); @@ -486,11 +495,12 @@ void MerginApi::preparePushPayload( const QString &projectFullName ) transaction.pushChanges.updated.append( file ); fileCounter++; + ++filePath; } transaction.diff = localDiff; - for ( const QString &filePath : transaction.diff.localDeleted ) + for ( const QString &filePath : std::as_const( transaction.diff.localDeleted ) ) { MerginFile file = findFile( filePath, oldServerProject.files ); transaction.pushChanges.removed.append( file );