diff --git a/mergin/client.py b/mergin/client.py index 1555051..8467a1c 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -67,6 +67,7 @@ int_version, is_version_acceptable, normalize_role, + long_path, ) from .version import __version__ @@ -1237,7 +1238,7 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi # collect required versions from the cache diffs = [] for v in versions_to_fetch[1:]: - diffs.append(mp.fpath_cache(file_history["history"][v]["diff"]["path"], v)) + diffs.append(long_path(mp.fpath_cache(file_history["history"][v]["diff"]["path"], v))) # concatenate diffs, if needed output_dir = os.path.dirname(output_diff) @@ -1377,13 +1378,15 @@ def reset_local_changes(self, directory: str, files_to_reset: typing.List[str] = # remove all added files for file in push_changes["added"]: if all_files or file["path"] in files_to_reset: - os.remove(mp.fpath(file["path"])) + os.remove(long_path(mp.fpath(file["path"]))) # update files get override with previous version for file in push_changes["updated"]: if all_files or file["path"] in files_to_reset: if mp.is_versioned_file(file["path"]): - mp.geodiff.make_copy_sqlite(mp.fpath_meta(file["path"]), mp.fpath(file["path"])) + mp.geodiff.make_copy_sqlite( + long_path(mp.fpath_meta(file["path"])), long_path(mp.fpath(file["path"])) + ) else: files_download.append(file["path"]) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 5210089..2030457 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -25,7 +25,7 @@ from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file +from .utils import cleanup_tmp_dir, save_to_file, long_path from typing import List, Optional # status = download_project_async(...) @@ -93,7 +93,9 @@ def __init__(self, file_path, size, version, diff_only, part_index, download_fil self.version = version # version of the file ("v123") self.diff_only = diff_only # whether downloading diff or full version self.part_index = part_index # index of the chunk - self.download_file_path = download_file_path # full path to a temporary file which will receive the content + self.download_file_path = long_path( + download_file_path + ) # full path to a temporary file which will receive the content def __repr__(self): return "".format( @@ -128,7 +130,9 @@ class DownloadDiffQueueItem: def __init__(self, diff_id, download_file_path): self.diff_id = diff_id # relative path to the file within project - self.download_file_path = download_file_path # full path to a temporary file which will receive the content + self.download_file_path = long_path( + download_file_path + ) # full path to a temporary file which will receive the content self.size = 0 # size of the item in bytes def __repr__(self): @@ -157,7 +161,7 @@ class DownloadFile: """ def __init__(self, dest_file, downloaded_items: typing.List[DownloadQueueItem], size_check=True): - self.dest_file = dest_file # full path to the destination file to be created + self.dest_file = long_path(dest_file) # full path to the destination file to be created self.downloaded_items = downloaded_items # list of pieces of the destination file to be merged self.size_check = size_check # whether we want to do merged file size check @@ -196,7 +200,7 @@ def get_download_items( items = [] for part_index in range(chunks): - download_file_path = os.path.join(file_dir, basename + ".{}".format(part_index)) + download_file_path = long_path(os.path.join(file_dir, basename + ".{}".format(part_index))) size = min(CHUNK_SIZE, file_size - part_index * CHUNK_SIZE) items.append(DownloadQueueItem(file_path, size, file_version, diff_only, part_index, download_file_path)) @@ -419,7 +423,7 @@ def apply(self, directory, mp): # Make a copy of the file to meta dir only if there is no user-specified path for the file. # destination_file is None for full project download and takes a meaningful value for a single file download. if mp.is_versioned_file(self.file_path) and self.destination_file is None: - mp.geodiff.make_copy_sqlite(mp.fpath(self.file_path), mp.fpath_meta(self.file_path)) + mp.geodiff.make_copy_sqlite(long_path(mp.fpath(self.file_path)), long_path(mp.fpath_meta(self.file_path))) class PullJob: @@ -479,7 +483,7 @@ def get_download_diff_files(delta_item: ProjectDeltaChange, target_dir: str) -> result = [] for diff in delta_item.diffs: - dest_file_path = os.path.normpath(os.path.join(target_dir, diff.id)) + dest_file_path = long_path(os.path.normpath(os.path.join(target_dir, diff.id))) download_items = get_download_items(delta_item.path, diff.size, diff.version, target_dir, diff.id, True) result.append(DownloadFile(dest_file_path, download_items)) return result @@ -555,7 +559,7 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: pull_action_type == PullActionType.COPY_CONFLICT and change.type == DeltaChangeType.UPDATE_DIFF ): basefile = mp.fpath_meta(change.path) - if not os.path.exists(basefile): + if not os.path.exists(long_path(basefile)): # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file # or we removed it within previous pull because we failed to apply patch the older version for some reason). # But it's not a problem - we will download the newest version and we're sorted. @@ -722,7 +726,7 @@ def pull_project_finalize(job: PullJob): basefile = job.mp.fpath_meta(file_path) server_file = job.mp.fpath(file_path, job.tmp_dir.name) - shutil.copy(basefile, server_file) + shutil.copy(long_path(basefile), long_path(server_file)) diffs = [job.mp.fpath(f, job.tmp_dir.name) for f in file_diffs] patch_error = job.mp.apply_diffs(server_file, diffs) if patch_error: @@ -735,7 +739,7 @@ def pull_project_finalize(job: PullJob): job.mp.log.error("Diffs we were applying: " + str(diffs)) job.mp.log.error("Removing basefile because it would be corrupted anyway...") job.mp.log.info("--- pull aborted") - os.remove(basefile) + os.remove(long_path(basefile)) raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile)) conflicts = [] job.mp.log.info(f"--- applying pull actions {job.pull_actions}") @@ -829,7 +833,7 @@ def download_diffs_async(mc, project_directory, file_path, versions): download_path=diff.get("path"), diff_only=True, ) - dest_file_path = mp.fpath_cache(diff["path"], version=file["version"]) + dest_file_path = long_path(mp.fpath_cache(diff["path"], version=file["version"])) if os.path.exists(dest_file_path): continue download_files.append(DownloadFile(dest_file_path, items)) diff --git a/mergin/client_push.py b/mergin/client_push.py index 831b59b..bdcf0d1 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -34,7 +34,7 @@ ) from .merginproject import MerginProject, pygeodiff from .editor import filter_changes -from .utils import get_data_checksum, cleanup_tmp_dir +from .utils import get_data_checksum, cleanup_tmp_dir, long_path POST_JSON_HEADERS = {"Content-Type": "application/json"} @@ -114,7 +114,7 @@ def upload_chunk_v2_api(self, data: ByteString, checksum: str): self.mc.upload_chunks_cache.add(checksum, self.server_chunk_id) def upload_blocking(self): - with open(self.file_path, "rb") as file_handle: + with open(long_path(self.file_path), "rb") as file_handle: file_handle.seek(self.chunk_index * UPLOAD_CHUNK_SIZE) data = file_handle.read(UPLOAD_CHUNK_SIZE) checksum_str = get_data_checksum(data) @@ -507,7 +507,7 @@ def remove_diff_files(job: UploadJob) -> None: for change in job.changes.updated: diff = change.get_diff() if diff: - diff_file = job.mp.fpath_meta(diff.path) + diff_file = long_path(job.mp.fpath_meta(diff.path)) if os.path.exists(diff_file): os.remove(diff_file) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 12d798f..0757a96 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -19,6 +19,7 @@ from .utils import ( generate_checksum, is_versioned_file, + long_path, int_version, do_sqlite_checkpoint, unique_path_name, @@ -279,7 +280,7 @@ def is_gpkg_open(self, path): f_extension = os.path.splitext(path)[1] if f_extension != ".gpkg": return False - if os.path.exists(f"{path}-wal"): + if os.path.exists(f"{long_path(path)}-wal"): return True return False @@ -623,16 +624,17 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: delta_item.size = checkpoint_size delta_item.checksum = checkpoint_checksum + diff_location = self.fpath(diff_file, diff_directory) + diff_location_lp = long_path(diff_location) try: - diff_location = self.fpath(diff_file, diff_directory) - self.geodiff.create_changeset(origin_file, current_file, diff_location) - if not self.geodiff.has_changes(diff_location): - os.remove(diff_location) + self.geodiff.create_changeset(long_path(origin_file), long_path(current_file), diff_location_lp) + if not self.geodiff.has_changes(diff_location_lp): + os.remove(diff_location_lp) continue delta_item.checksum = change.get("origin_checksum") delta_item.type = DeltaChangeType.UPDATE_DIFF - os.remove(diff_location) + os.remove(diff_location_lp) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to create changeset for " + path) # probably the database schema has been modified if geodiff cannot create changeset. @@ -677,22 +679,23 @@ def get_push_changes(self): diff_id = str(uuid.uuid4()) diff_name = path + "-diff-" + diff_id diff_file = self.fpath_meta(diff_name) + diff_file_lp = long_path(diff_file) try: - self.geodiff.create_changeset(origin_file, current_file, diff_file) - if self.geodiff.has_changes(diff_file): - diff_size = os.path.getsize(diff_file) + self.geodiff.create_changeset(long_path(origin_file), long_path(current_file), diff_file_lp) + if self.geodiff.has_changes(diff_file_lp): + diff_size = os.path.getsize(diff_file_lp) file["checksum"] = file["origin_checksum"] # need to match basefile on server file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(diff_size / UPLOAD_CHUNK_SIZE))] - file["mtime"] = datetime.fromtimestamp(os.path.getmtime(current_file), tzlocal()) + file["mtime"] = datetime.fromtimestamp(os.path.getmtime(long_path(current_file)), tzlocal()) file["diff"] = { "path": diff_name, - "checksum": generate_checksum(diff_file), + "checksum": generate_checksum(diff_file_lp), "size": diff_size, - "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file), tzlocal()), + "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file_lp), tzlocal()), } else: - if os.path.exists(diff_file): - os.remove(diff_file) + if os.path.exists(diff_file_lp): + os.remove(diff_file_lp) not_updated.append(file) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to create changeset for " + path) @@ -712,9 +715,10 @@ def copy_versioned_file_for_upload(self, f: FileChange, tmp_dir: str) -> str: self.log.info("Making a temporary copy (full upload): " + path) tmp_file = os.path.join(tmp_dir, path) os.makedirs(os.path.dirname(tmp_file), exist_ok=True) - self.geodiff.make_copy_sqlite(self.fpath(path), tmp_file) - f.size = os.path.getsize(tmp_file) - f.checksum = generate_checksum(tmp_file) + tmp_file_lp = long_path(tmp_file) + self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), tmp_file_lp) + f.size = os.path.getsize(tmp_file_lp) + f.checksum = generate_checksum(tmp_file_lp) f.chunks = [str(uuid.uuid4()) for i in range(math.ceil(f.size / UPLOAD_CHUNK_SIZE))] f.upload_file = tmp_file return tmp_file @@ -727,7 +731,7 @@ def get_list_of_push_changes(self, push_changes): changeset = self.fpath_meta(changeset_path) result_file = self.fpath("change_list" + str(idx), self.meta_dir) try: - self.geodiff.list_changes_summary(changeset, result_file) + self.geodiff.list_changes_summary(long_path(changeset), result_file) with open(result_file, "r") as f: change = f.read() changes[file["path"]] = json.loads(change) @@ -761,14 +765,17 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve server_file = self.fpath(path, download_dir) live_file = self.fpath(path) basefile = self.fpath_meta(path) + server_file_lp = long_path(server_file) + live_file_lp = long_path(live_file) + basefile_lp = long_path(basefile) action_type = action.type if action_type == PullActionType.COPY: # simply copy the file from server if is_versioned_file(path): - self.geodiff.make_copy_sqlite(server_file, live_file) - self.geodiff.make_copy_sqlite(server_file, basefile) + self.geodiff.make_copy_sqlite(server_file_lp, live_file_lp) + self.geodiff.make_copy_sqlite(server_file_lp, basefile_lp) else: - shutil.copy(server_file, live_file) + shutil.copy(server_file_lp, live_file_lp) elif action_type == PullActionType.APPLY_DIFF_NO_REBASE: # simply apply the diff without rebase (no local changes or non-conflicting local changes) self.update_without_rebase(path, server_file, live_file, basefile, download_dir) @@ -786,22 +793,22 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve conflicts.append(conflict) if self.is_versioned_file(path): try: - self.geodiff.make_copy_sqlite(server_file, live_file) - self.geodiff.make_copy_sqlite(server_file, basefile) + self.geodiff.make_copy_sqlite(server_file_lp, live_file_lp) + self.geodiff.make_copy_sqlite(server_file_lp, basefile_lp) except pygeodiff.GeoDiffLibError: self.log.info("failed to create SQLite copy for file: " + path) # create unfinished pull copy instead - f_server_unfinished = self.fpath_unfinished_pull(path) - self.geodiff.make_copy_sqlite(server_file, f_server_unfinished) + f_server_unfinished = long_path(self.fpath_unfinished_pull(path)) + self.geodiff.make_copy_sqlite(server_file_lp, f_server_unfinished) else: - shutil.copy(server_file, live_file) + shutil.copy(server_file_lp, live_file_lp) elif action_type == PullActionType.DELETE: # remove local file - if os.path.exists(live_file): - os.remove(live_file) - if self.is_versioned_file(path) and os.path.exists(basefile): - os.remove(basefile) + if os.path.exists(live_file_lp): + os.remove(live_file_lp) + if self.is_versioned_file(path) and os.path.exists(basefile_lp): + os.remove(basefile_lp) return conflicts @@ -828,51 +835,57 @@ def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name): """ self.log.info("updating file with rebase: " + path) - server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile - local_diff = self.fpath(f"{path}-local_diff", temp_dir) + src_lp = long_path(src) + dest_lp = long_path(dest) + basefile_lp = long_path(basefile) + + server_diff = long_path( + self.fpath(f"{path}-server_diff", temp_dir) + ) # diff between server file and local basefile + local_diff = long_path(self.fpath(f"{path}-local_diff", temp_dir)) # temporary backup of file pulled from server for recovery - f_server_backup = self.fpath(f"{path}-server_backup", temp_dir) - self.geodiff.make_copy_sqlite(src, f_server_backup) + f_server_backup = long_path(self.fpath(f"{path}-server_backup", temp_dir)) + self.geodiff.make_copy_sqlite(src_lp, f_server_backup) # create temp backup (ideally with geodiff) of locally modified file if needed later - f_conflict_file = self.fpath(f"{path}-local_backup", temp_dir) + f_conflict_file = long_path(self.fpath(f"{path}-local_backup", temp_dir)) try: - self.geodiff.create_changeset(basefile, dest, local_diff) - self.geodiff.make_copy_sqlite(basefile, f_conflict_file) + self.geodiff.create_changeset(basefile_lp, dest_lp, local_diff) + self.geodiff.make_copy_sqlite(basefile_lp, f_conflict_file) self.geodiff.apply_changeset(f_conflict_file, local_diff) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): self.log.info("backup of local file with geodiff failed - need to do hard copy") - self.geodiff.make_copy_sqlite(dest, f_conflict_file) + self.geodiff.make_copy_sqlite(dest_lp, f_conflict_file) # in case there will be any conflicting operations found during rebase, # they will be stored in a JSON file - if there are no conflicts, the file # won't even be created - rebase_conflicts = unique_path_name( - edit_conflict_file_name(self.fpath(path), user_name, int_version(self.version())) + rebase_conflicts = long_path( + unique_path_name(edit_conflict_file_name(self.fpath(path), user_name, int_version(self.version()))) ) # try to do rebase magic try: - self.geodiff.create_changeset(basefile, src, server_diff) - self.geodiff.rebase(basefile, src, dest, rebase_conflicts) + self.geodiff.create_changeset(basefile_lp, src_lp, server_diff) + self.geodiff.rebase(basefile_lp, src_lp, dest_lp, rebase_conflicts) # make sure basefile is in the same state as remote server file (for calc of push changes) - self.geodiff.apply_changeset(basefile, server_diff) + self.geodiff.apply_changeset(basefile_lp, server_diff) self.log.info("rebase successful!") except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as err: self.log.warning("rebase failed! going to create conflict file") try: # it would not be possible to commit local changes, they need to end up in new conflict file - self.geodiff.make_copy_sqlite(f_conflict_file, dest) + self.geodiff.make_copy_sqlite(f_conflict_file, dest_lp) conflict = self.create_conflicted_copy(path, user_name) # original file synced with server - self.geodiff.make_copy_sqlite(f_server_backup, basefile) - self.geodiff.make_copy_sqlite(f_server_backup, dest) + self.geodiff.make_copy_sqlite(f_server_backup, basefile_lp) + self.geodiff.make_copy_sqlite(f_server_backup, dest_lp) return conflict except pygeodiff.GeoDiffLibError as err: self.log.warning("creation of conflicted copy failed! going to create an unfinished pull") - f_server_unfinished = self.fpath_unfinished_pull(path) + f_server_unfinished = long_path(self.fpath_unfinished_pull(path)) self.geodiff.make_copy_sqlite(f_server_backup, f_server_unfinished) return "" @@ -898,22 +911,27 @@ def update_without_rebase(self, path, src, dest, basefile, temp_dir): :type temp_dir: str """ self.log.info("updating file without rebase: " + path) + src_lp = long_path(src) + dest_lp = long_path(dest) + basefile_lp = long_path(basefile) try: - server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile + server_diff = long_path( + self.fpath(f"{path}-server_diff", temp_dir) + ) # diff between server file and local basefile # TODO: it could happen that basefile does not exist. # It was either never created (e.g. when pushing without geodiff) # or it was deleted by mistake(?) by the user. We should detect that # when starting pull and download it as well - self.geodiff.create_changeset(basefile, src, server_diff) - self.geodiff.apply_changeset(dest, server_diff) - self.geodiff.apply_changeset(basefile, server_diff) + self.geodiff.create_changeset(basefile_lp, src_lp, server_diff) + self.geodiff.apply_changeset(dest_lp, server_diff) + self.geodiff.apply_changeset(basefile_lp, server_diff) self.log.info("update successful") except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): self.log.warning("update failed! going to copy file") # something bad happened and we have failed to patch our local files - this should not happen if there # wasn't a schema change or something similar that geodiff can't handle. - self.geodiff.make_copy_sqlite(src, dest) - self.geodiff.make_copy_sqlite(src, basefile) + self.geodiff.make_copy_sqlite(src_lp, dest_lp) + self.geodiff.make_copy_sqlite(src_lp, basefile_lp) def apply_push_changes(self, changes): """ @@ -929,16 +947,17 @@ def apply_push_changes(self, changes): continue basefile = self.fpath_meta(path) + basefile_lp = long_path(basefile) if k == "removed": - os.remove(basefile) + os.remove(basefile_lp) elif k == "added": - self.geodiff.make_copy_sqlite(self.fpath(path), basefile) + self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), basefile_lp) elif k == "updated": # in case for geopackage cannot be created diff (e.g. forced update with committed changes from wal file) diff = item.get("diff") if not diff: self.log.info("updating basefile (copy) for: " + path) - self.geodiff.make_copy_sqlite(self.fpath(path), basefile) + self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), basefile_lp) else: self.log.info("updating basefile (diff) for: " + path) # better to apply diff to previous basefile to avoid issues with geodiff tmp files @@ -947,7 +966,7 @@ def apply_push_changes(self, changes): if patch_error: # in case of local sync issues it is safier to remove basefile, next time it will be downloaded from server self.log.warning("removing basefile (because of apply diff error) for: " + path) - os.remove(basefile) + os.remove(basefile_lp) else: pass @@ -961,7 +980,8 @@ def create_conflicted_copy(self, file: str, user_name: str): :rtype: str """ src = self.fpath(file) - if not os.path.exists(src): + src_lp = long_path(src) + if not os.path.exists(src_lp): return backup_path = unique_path_name( @@ -969,9 +989,9 @@ def create_conflicted_copy(self, file: str, user_name: str): ) if self.is_versioned_file(file): - self.geodiff.make_copy_sqlite(src, backup_path) + self.geodiff.make_copy_sqlite(src_lp, long_path(backup_path)) else: - shutil.copy(src, backup_path) + shutil.copy(src_lp, long_path(backup_path)) return backup_path def apply_diffs(self, basefile, diffs): @@ -990,9 +1010,10 @@ def apply_diffs(self, basefile, diffs): if not self.is_versioned_file(basefile): return error + basefile_lp = long_path(basefile) for index, diff in enumerate(diffs): try: - self.geodiff.apply_changeset(basefile, diff) + self.geodiff.apply_changeset(basefile_lp, long_path(diff)) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to apply changeset " + diff + " to " + basefile) error = str(e) @@ -1042,12 +1063,12 @@ def resolve_unfinished_pull(self, user_name): self.log.info("resolving unfinished pull") - for root, dirs, files in os.walk(self.unfinished_pull_dir): + for root, dirs, files in os.walk(long_path(self.unfinished_pull_dir)): for file_name in files: - src = os.path.join(root, file_name) - file_path = os.path.relpath(src, self.unfinished_pull_dir) - dest = self.fpath(file_path) - basefile = self.fpath_meta(file_path) + src = os.path.join(root, file_name) # already long-path-prefixed, root came from os.walk above + file_path = os.path.relpath(src, long_path(self.unfinished_pull_dir)) + dest = long_path(self.fpath(file_path)) + basefile = long_path(self.fpath_meta(file_path)) self.log.info("trying to resolve unfinished pull for: " + file_path) @@ -1110,7 +1131,7 @@ def get_geodiff_changes_count(self, diff_rel_path: str): Never raises – diagnostics/logging must not fail. """ - diff_abs = self.fpath_meta(diff_rel_path) + diff_abs = long_path(self.fpath_meta(diff_rel_path)) try: return pygeodiff.GeoDiff().changes_count(diff_abs) except ( diff --git a/mergin/report.py b/mergin/report.py index 5b9cae4..abbf451 100644 --- a/mergin/report.py +++ b/mergin/report.py @@ -7,7 +7,7 @@ from . import ClientError from .merginproject import MerginProject, pygeodiff -from .utils import int_version +from .utils import int_version, long_path try: from qgis.core import ( @@ -243,7 +243,7 @@ def create_report(mc, directory, since, to, out_file): mc.download_file_diffs(directory, f["path"], history_keys) # download full gpkg in "to" version to analyze its schema to determine which col is geometry - full_gpkg = mp.fpath_cache(f["path"], version=to) + full_gpkg = long_path(mp.fpath_cache(f["path"], version=to)) if not os.path.exists(full_gpkg): mc.download_file(directory, f["path"], full_gpkg, to) @@ -263,7 +263,7 @@ def create_report(mc, directory, since, to, out_file): warnings.append(f"Missing diff: {f['path']} was {f['history'][version]['change']} in {version}") continue - v_diff_file = mp.fpath_cache(f["history"][version]["diff"]["path"], version=version) + v_diff_file = long_path(mp.fpath_cache(f["history"][version]["diff"]["path"], version=version)) version_data = versions_map[version] cr = mp.geodiff.read_changeset(v_diff_file) report = changeset_report(cr, schema, mp) diff --git a/mergin/test/test_client_pull.py b/mergin/test/test_client_pull.py index c20fe44..fda8bd0 100644 --- a/mergin/test/test_client_pull.py +++ b/mergin/test/test_client_pull.py @@ -4,6 +4,7 @@ from mergin.common import DeltaChangeType, CHUNK_SIZE from mergin.models import ProjectDeltaChange, ProjectDeltaItemDiff from mergin.client_pull import get_download_diff_files, get_download_items +from mergin.utils import long_path def test_get_diff_download_files(): @@ -25,7 +26,7 @@ def test_get_diff_download_files(): # Check diff f2 = files[0] - assert f2.dest_file == os.path.join(tmp_dir, "diff2") + assert f2.dest_file == long_path(os.path.join(tmp_dir, "diff2")) assert len(f2.downloaded_items) == 1 assert f2.downloaded_items[0].file_path == "data.gpkg" assert f2.downloaded_items[0].size == 20 @@ -41,7 +42,7 @@ def test_get_download_items(): assert items[0].file_path == "small.txt" assert items[0].size == 100 assert items[0].part_index == 0 - assert items[0].download_file_path == os.path.join(tmp_dir, "small.txt.0") + assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "small.txt.0")) # Case 2: Large file (multiple chunks) file_size = int(CHUNK_SIZE * 2.5) @@ -51,17 +52,17 @@ def test_get_download_items(): # Chunk 0 assert items[0].size == CHUNK_SIZE assert items[0].part_index == 0 - assert items[0].download_file_path == os.path.join(tmp_dir, "large.txt.0") + assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.0")) # Chunk 1 assert items[1].size == CHUNK_SIZE assert items[1].part_index == 1 - assert items[1].download_file_path == os.path.join(tmp_dir, "large.txt.1") + assert items[1].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.1")) # Chunk 2 assert items[2].size == int(CHUNK_SIZE * 0.5) assert items[2].part_index == 2 - assert items[2].download_file_path == os.path.join(tmp_dir, "large.txt.2") + assert items[2].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.2")) # Case 3: Empty file items = get_download_items("empty.txt", 0, "v1", tmp_dir) @@ -73,4 +74,4 @@ def test_get_download_items(): assert items[0].diff_only is True assert items[0].file_path == "base.gpkg" assert items[0].size == 50 - assert items[0].download_file_path == os.path.join(tmp_dir, "diff_file.0") + assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "diff_file.0")) diff --git a/mergin/utils.py b/mergin/utils.py index 91796f3..b5bcac9 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -88,10 +88,11 @@ def do_sqlite_checkpoint(path, log=None): """ new_size = None new_checksum = None - if ".gpkg" in path and os.path.exists(f"{path}-wal"): + path_lp = long_path(path) + if ".gpkg" in path and os.path.exists(f"{path_lp}-wal"): if log: log.info("checkpoint - going to add it in " + path) - conn = sqlite3.connect(path) + conn = sqlite3.connect(path_lp) cursor = conn.cursor() cursor.execute("PRAGMA wal_checkpoint=FULL") if log: @@ -99,8 +100,8 @@ def do_sqlite_checkpoint(path, log=None): cursor.execute("VACUUM") conn.commit() conn.close() - new_size = os.path.getsize(path) - new_checksum = generate_checksum(path) + new_size = os.path.getsize(path_lp) + new_checksum = generate_checksum(path_lp) if log: log.info("checkpoint - new size {} checksum {}".format(new_size, new_checksum)) @@ -266,6 +267,25 @@ def is_versioned_file(path: str) -> bool: return f_extension.lower() in diff_extensions +def long_path(path: str) -> str: + """ + Prefix an absolute path with the Windows "\\?\" extended-length marker, so file APIs used by + geodiff/SQLite and Python's own open() can handle long paths without raising an error. + + :param path: absolute or relative path, with either posix or windows separators + :type path: str + :returns: extended-length path on Windows, the unchanged path otherwise + :rtype: str + """ + if os.name != "nt": + return path + backslash = chr(92) + prefix = backslash + backslash + "?" + backslash + if path.startswith(prefix): + return path + return prefix + os.path.abspath(path) + + def is_qgis_file(path: str) -> bool: """ Check if file is a QGIS project file.