Skip to content

Filter generated diff by size limits #4709

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Withalion merged 3 commits into from
Sep 24, 2026
Merged

Filter generated diff by size limits #4709

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ef86e0f
Filter generated diff by size limits
Withalion Sep 19, 2026
ba33cf7
Remove done TODO
Withalion Sep 19, 2026
309a5d8
Fix found issues
Withalion Sep 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 66 additions & 46 deletions core/merginapi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,16 +340,16 @@ void MerginApi::preparePushPayload( const QString &projectFullName )
}

// Calculate local changes
transaction.diff = compareProjectFiles(
oldServerProject.files,
oldServerProject.files,
localFiles,
transaction.projectDir,
transaction.configAllowed,
transaction.config
);
ProjectDiff localDiff = compareProjectFiles(
oldServerProject.files,
oldServerProject.files,
localFiles,
transaction.projectDir,
transaction.configAllowed,
transaction.config
);

CoreUtils::log( "push " + projectFullName, transaction.diff.dump() );
CoreUtils::log( "push " + projectFullName, localDiff.dump() );

if ( transaction.configAllowed && transaction.config.isValid && transaction.config.selectiveSyncEnabled )
{
Expand All @@ -361,13 +361,13 @@ void MerginApi::preparePushPayload( const QString &projectFullName )
CoreUtils::log( "push " + projectFullName, QStringLiteral( "Selective sync is not enabled" ) );
}

if ( !transaction.diff.remoteAdded.isEmpty() ||
!transaction.diff.remoteUpdated.isEmpty() ||
!transaction.diff.remoteDeleted.isEmpty() ||
!transaction.diff.conflictRemoteAddedLocalAdded.isEmpty() ||
!transaction.diff.conflictRemoteDeletedLocalUpdated.isEmpty() ||
!transaction.diff.conflictRemoteUpdatedLocalDeleted.isEmpty() ||
!transaction.diff.conflictRemoteUpdatedLocalUpdated.isEmpty() )
if ( !localDiff.remoteAdded.isEmpty() ||
!localDiff.remoteUpdated.isEmpty() ||
!localDiff.remoteDeleted.isEmpty() ||
!localDiff.conflictRemoteAddedLocalAdded.isEmpty() ||
!localDiff.conflictRemoteDeletedLocalUpdated.isEmpty() ||
!localDiff.conflictRemoteUpdatedLocalDeleted.isEmpty() ||
!localDiff.conflictRemoteUpdatedLocalUpdated.isEmpty() )
{
CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Detected remote changes when calculating local changes" ) );
// What to do here?
Expand All @@ -376,31 +376,38 @@ void MerginApi::preparePushPayload( const QString &projectFullName )
const bool useV2push = true;
int fileCounter = 0;

for ( const QString &filePath : std::as_const( transaction.diff.localAdded ) )
for ( auto filePath = localDiff.localAdded.begin(); filePath != localDiff.localAdded.end(); )
{
MerginFile file = findFile( filePath, localFiles );
if ( fileCounter >= MAX_UPLOAD_CHANGES )
{
CoreUtils::log( "push " + projectFullName, QStringLiteral( "Maximum amount of changed files reached, push will be split to multiple versions" ) );
// remove remaining items of localDiff.localAdded
while ( filePath != localDiff.localAdded.end() )
{
filePath = localDiff.localAdded.erase( filePath );
}
break;
}
Comment thread
Withalion marked this conversation as resolved.

MerginFile file = findFile( *filePath, localFiles );

if ( isFileDiffable( file.path ) && file.size > MAX_UPLOAD_VERSIONED_SIZE )
{
CoreUtils::log( "push " + projectFullName, QStringLiteral( "Versionable file \"%1\" exceeded maximum upload size" ).arg( file.path ) );
filePath = localDiff.localAdded.erase( filePath );
continue;
}

if ( !isFileDiffable( file.path ) && file.size > MAX_UPLOAD_MEDIA_SIZE )
{
CoreUtils::log( "push " + projectFullName, QStringLiteral( "Media file \"%1\" exceeded maximum upload size" ).arg( file.path ) );
filePath = localDiff.localAdded.erase( filePath );
continue;
}

if ( fileCounter >= MAX_UPLOAD_CHANGES )
{
CoreUtils::log( "push " + projectFullName, QStringLiteral( "Maximum amount of changed files reached, push will be split to multiple versions" ) );
break;
}

file.chunks = generateChunkIdsForSize( file.size );

if ( mSupportsSelectiveSync && filePath == sMerginConfigFile )
if ( mSupportsSelectiveSync && *filePath == sMerginConfigFile )
{
// problem, we changed selective sync config locally, this is undefined and sync must stop
CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Selective sync config was added locally, this is undefined and might lead to data loss, aborting push..." ) );
Expand All @@ -409,64 +416,77 @@ void MerginApi::preparePushPayload( const QString &projectFullName )

transaction.pushChanges.added.append( file );
fileCounter++;
++filePath;
}

for ( const QString &filePath : std::as_const( transaction.diff.localUpdated ) )
if ( fileCounter >= MAX_UPLOAD_CHANGES )
{
MerginFile file = findFile( filePath, localFiles );
localDiff.localUpdated.clear();
}

for ( auto filePath = localDiff.localUpdated.begin(); filePath != localDiff.localUpdated.end(); )
{
if ( fileCounter >= MAX_UPLOAD_CHANGES )
{
CoreUtils::log( "push " + projectFullName, QStringLiteral( "Maximum amount of changed files reached, push will be split to multiple versions" ) );
Comment thread
Withalion marked this conversation as resolved.
// remove remaining items of localDiff.localUpdated
while ( filePath != localDiff.localUpdated.end() )
{
filePath = localDiff.localUpdated.erase( filePath );
}
break;
}

MerginFile file = findFile( *filePath, localFiles );

if ( isFileDiffable( file.path ) && file.size > MAX_UPLOAD_VERSIONED_SIZE )
{
CoreUtils::log( "push " + projectFullName, QStringLiteral( "Versionable file \"%1\" exceeded maximum upload size" ).arg( file.path ) );
filePath = localDiff.localUpdated.erase( filePath );
continue;
}

if ( !isFileDiffable( file.path ) && file.size > MAX_UPLOAD_MEDIA_SIZE )
{
CoreUtils::log( "push " + projectFullName, QStringLiteral( "Media file \"%1\" exceeded maximum upload size" ).arg( file.path ) );
filePath = localDiff.localUpdated.erase( filePath );
continue;
}

if ( fileCounter >= MAX_UPLOAD_CHANGES )
{
CoreUtils::log( "push " + projectFullName, QStringLiteral( "Maximum amount of changed files exceeded" ) );
break;
}

file.chunks = generateChunkIdsForSize( file.size );

if ( MerginApi::isFileDiffable( filePath ) )
if ( isFileDiffable( *filePath ) )
{
// try to create a diff
QString diffName;
int geodiffRes = GeodiffUtils::createChangeset( transaction.projectDir, filePath, diffName );
int geodiffRes = GeodiffUtils::createChangeset( transaction.projectDir, *filePath, diffName );
QString diffPath = transaction.projectDir + "/.mergin/" + diffName;
QString basePath = transaction.projectDir + "/.mergin/" + filePath;
QString basePath = transaction.projectDir + "/.mergin/" + *filePath;

if ( geodiffRes == GEODIFF_SUCCESS )
{
QByteArray checksumDiff = CoreUtils::calculateChecksum( diffPath );

// TODO: this is ugly. our basefile may not need to have the same checksum as the server's
// basefile (because each of them have applied the diff independently) so we have to fake it
QByteArray checksumBase = oldServerProject.fileInfo( filePath ).checksum.toLatin1();
QByteArray checksumBase = oldServerProject.fileInfo( *filePath ).checksum.toLatin1();

file.diffName = diffName;
file.diffChecksum = QString::fromLatin1( checksumDiff.data(), checksumDiff.size() );
file.diffSize = QFileInfo( diffPath ).size();
file.diffBaseChecksum = QString::fromLatin1( checksumBase.data(), checksumBase.size() );
file.chunks = generateChunkIdsForSize( file.diffSize );

CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 successful: total size %2 bytes" ).arg( filePath ).arg( file.diffSize ) );
CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 successful: total size %2 bytes" ).arg( *filePath ).arg( file.diffSize ) );
}
else
{
// TODO: remove the diff file (if exists)
CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 FAILED with error %2 (will do full upload)" ).arg( filePath ).arg( geodiffRes ) );
CoreUtils::log( "push " + projectFullName, QString( "Geodiff create changeset on %1 FAILED with error %2 (will do full upload)" ).arg( *filePath ).arg( geodiffRes ) );
}
}

if ( mSupportsSelectiveSync && filePath == sMerginConfigFile )
if ( mSupportsSelectiveSync && *filePath == sMerginConfigFile )
{
// problem, we changed selective sync config locally, this is undefined and sync must stop
CoreUtils::log( "push " + projectFullName, QStringLiteral( "PROBLEM! Selective sync config was updated locally, this is undefined and might lead to data loss, aborting push..." ) );
Expand All @@ -475,9 +495,12 @@ void MerginApi::preparePushPayload( const QString &projectFullName )

transaction.pushChanges.updated.append( file );
fileCounter++;
++filePath;
}

for ( QString filePath : transaction.diff.localDeleted )
transaction.diff = localDiff;

for ( const QString &filePath : std::as_const( transaction.diff.localDeleted ) )
{
MerginFile file = findFile( filePath, oldServerProject.files );
transaction.pushChanges.removed.append( file );
Expand All @@ -490,12 +513,12 @@ void MerginApi::preparePushPayload( const QString &projectFullName )
}

qint64 totalSize = 0;
for ( MerginFile file : transaction.pushChanges.added )
for ( const MerginFile &file : transaction.pushChanges.added )
{
totalSize += file.size;
}

for ( MerginFile file : transaction.pushChanges.updated )
for ( const MerginFile &file : transaction.pushChanges.updated )
{
if ( !file.diffName.isEmpty() )
{
Expand All @@ -514,9 +537,6 @@ void MerginApi::preparePushPayload( const QString &projectFullName )
.arg( transaction.pushQueue.count() ).arg( totalSize ) );


// TODO: Check here if the total number of files to upload is not larger than accepted by the server (100)
// TODO: Check here if file size of the individual files to upload is not larger than accepted by server (5 GB for gpkg, 10 GB for other file types)

pushStart( projectFullName );
}

Expand Down
Loading