diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index b6c3f6d7..7a3231f9 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -171,7 +171,7 @@ def get_latest_files_cache(self) -> List[int]: fp.id, pv.project_id ) SELECT - fh.id + fh.id FROM latest_changes ch LEFT OUTER JOIN file_history fh ON (fh.file_path_id = ch.id AND fh.project_version_name = ch.version AND fh.change != 'delete') WHERE fh.id IS NOT NULL; @@ -2115,10 +2115,19 @@ def process_chunks( # based on API version location for uploaded chunks differs if use_shared_chunk_dir: chunk_file = get_chunk_location(chunk_id) + chunk_file_fallback = get_chunk_location( + chunk_id, "/chunks" + ) else: chunk_file = os.path.join( self.upload_dir, "chunks", chunk_id ) + chunk_file_fallback = None + + if chunk_file_fallback and os.path.exists( + chunk_file_fallback + ): + chunk_file = chunk_file_fallback if not os.path.exists(chunk_file): errors[f.path] = FileSyncErrorType.CORRUPTED.value diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index e1c06678..38150db2 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -585,14 +585,14 @@ def get_checkpoints(cls, start: int, end: int) -> List[Checkpoint]: return levels -def get_chunk_location(id: str): +def get_chunk_location(id: str, dir: str | None = None): """ Get file location for chunk on FS Splits the given identifier into two parts where the first two characters of the identifier are the small hash, and the remaining characters is a file identifier. """ - chunk_dir = current_app.config.get("UPLOAD_CHUNKS_DIR") + chunk_dir = dir or current_app.config.get("UPLOAD_CHUNKS_DIR") small_hash = id[:2] file_name = id[2:] return os.path.join(chunk_dir, small_hash, file_name) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index f9b2cf73..479aff72 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -1144,6 +1144,43 @@ def test_create_version_permanent_error_takes_priority(client): assert project.latest_version == 1 +def test_create_version_legacy_chunks_dir(client): + """Test push with chunks found only in legacy fallback chunks directory""" + project = Project.query.filter_by( + workspace_id=test_workspace_id, name=test_project + ).first() + data = {"version": "v1", "changes": _get_changes_without_added(test_project_dir)} + legacy_dir = os.path.join(TMP_DIR, "legacy_chunks") + + def chunk_location(id, dir=None): + # redirect hardcoded legacy "/chunks" directory to writable tmp location + return get_chunk_location(id, legacy_dir if dir == "/chunks" else dir) + + def push(): + with patch( + "mergin.sync.models.get_chunk_location", side_effect=chunk_location + ), patch( + "mergin.sync.public_api_v2_controller.remove_transaction_chunks.delay" + ): + return client.post(f"v2/projects/{project.id}/versions", json=data) + + # mimic chunks were uploaded to legacy directory only + for f in data["changes"]["updated"]: + with open(os.path.join(test_project_dir, f["path"]), "rb") as in_file: + for chunk in f["chunks"]: + chunk_file = get_chunk_location(chunk, legacy_dir) + os.makedirs(os.path.dirname(chunk_file), exist_ok=True) + with open(chunk_file, "wb") as out_file: + out_file.write(in_file.read(CHUNK_SIZE)) + assert not os.path.exists(get_chunk_location(chunk)) + + response = push() + assert response.status_code == 201 + assert response.json["version"] == "v2" + assert project.latest_version == 2 + shutil.rmtree(legacy_dir) + + def test_upload_chunk(client): """Test pushing a chunk to a project""" project = Project.query.filter_by(