diff --git a/LICENSES/CLA-signed-list.md b/LICENSES/CLA-signed-list.md index de5e8e86..1f4c1cc9 100644 --- a/LICENSES/CLA-signed-list.md +++ b/LICENSES/CLA-signed-list.md @@ -22,3 +22,4 @@ C/ My company has custom contribution contract with Lutra Consulting Ltd. or I a * jozef-budac, 30th January 2024 * fernandinand, 13th March 2025 * wonder-sk, 9th February 2026 +* xkello, 26th January 2026 diff --git a/server/mergin/sync/config.py b/server/mergin/sync/config.py index 1113e24f..8a5081ec 100644 --- a/server/mergin/sync/config.py +++ b/server/mergin/sync/config.py @@ -80,3 +80,7 @@ class Configuration(object): EXCLUDED_CLONE_FILENAMES = config( "EXCLUDED_CLONE_FILENAMES", default="qgis_cfg.xml", cast=Csv() ) + # files that should be ignored during extension and MIME type checks + UPLOAD_FILES_WHITELIST = config("UPLOAD_FILES_WHITELIST", default="", cast=Csv()) + # max batch size for fetch projects in batch endpoint + MAX_BATCH_SIZE = config("MAX_BATCH_SIZE", default=100, cast=int) diff --git a/server/mergin/sync/errors.py b/server/mergin/sync/errors.py index 33b80d74..e12d762f 100644 --- a/server/mergin/sync/errors.py +++ b/server/mergin/sync/errors.py @@ -97,6 +97,11 @@ class BigChunkError(ResponseError): detail = f"Chunk size exceeds maximum allowed size {MAX_CHUNK_SIZE} MB" +class BatchLimitError(ResponseError): + code = "BatchLimitExceeded" + detail = f"Batch size exceeds maximum allowed size {Configuration.MAX_BATCH_SIZE}" + + class DiffDownloadError(ResponseError): code = "DiffDownloadError" detail = ( diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index 7dd042d5..e155020a 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -248,6 +248,29 @@ def require_project_by_uuid( return project +def check_project_permissions( + project: Project, permission: ProjectPermissions +) -> int | None: + """Check project permissions and return appropriate HTTP error code if check fails. + :param project: project + :type project: Project + :param permission: permission to check + :type permission: ProjectPermissions + :return: HTTP error code if permission check fails, None otherwise + :rtype: int | None + """ + + if not permission.check(project, current_user): + # logged in - NO, have acccess - NONE, public project - NO + if current_user.is_anonymous: + # we don't want to tell anonymous user if a private project exists + return 404 + # logged in - YES, have access - NO, public project - NO + return 403 + + return None + + def get_upload(transaction_id): upload = Upload.query.get_or_404(transaction_id) # upload to 'removed' projects is forbidden diff --git a/server/mergin/sync/project_handler.py b/server/mergin/sync/project_handler.py index 8299935a..7949dc20 100644 --- a/server/mergin/sync/project_handler.py +++ b/server/mergin/sync/project_handler.py @@ -28,3 +28,13 @@ def get_email_receivers(self, project: Project) -> List[User]: ) .all() ) + + @staticmethod + def get_projects_by_uuids(uuids: List[str]) -> [Project]: + """Gets non-deleted projects""" + return ( + Project.query.filter(Project.id.in_(uuids)) + .filter(Project.storage_params.isnot(None)) + .filter(Project.removed_at.is_(None)) + .all() + ) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 0cbd5e90..7a764e44 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -582,8 +582,9 @@ def get_paginated_projects( public, only_public, ) - result = projects.paginate(page=page, per_page=per_page).items - total = projects.paginate().total + pagination = projects.paginate(page=page, per_page=per_page) + result = pagination.items + total = pagination.total # create user map id:username passed to project schema to minimize queries to db projects_ids = [p.id for p in result] diff --git a/server/mergin/sync/public_api_v2.yaml b/server/mergin/sync/public_api_v2.yaml index b01f6781..b351654f 100644 --- a/server/mergin/sync/public_api_v2.yaml +++ b/server/mergin/sync/public_api_v2.yaml @@ -407,6 +407,53 @@ paths: $ref: "#/components/schemas/ProjectLocked" x-openapi-router-controller: mergin.sync.public_api_v2_controller + + /projects/batch: + post: + tags: + - project + summary: Get multiple projects by UUIDs + operationId: list_batch_projects + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [ids] + properties: + ids: + type: array + description: List of project UUIDs to fetch + items: + $ref: "#/components/schemas/ProjectId" + responses: + "200": + description: Projects returned as a list of simple project objects and/or error objects. + content: + application/json: + schema: + type: object + required: [projects] + properties: + projects: + type: array + items: + oneOf: + - $ref: "#/components/schemas/Project" + - $ref: "#/components/schemas/BatchItemError" + "400": + description: Batch limit exceeded or one or more UUIDs were invalid + content: + application/problem+json: + schema: + $ref: "#/components/schemas/CustomError" + "401": + $ref: "#/components/responses/Unauthorized" + "404": + $ref: "#/components/responses/NotFound" + x-openapi-router-controller: mergin.sync.public_api_v2_controller + /projects/{id}/delta: get: tags: @@ -526,9 +573,7 @@ components: description: UUID of the project required: true schema: - type: string - format: uuid - pattern: \b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b + $ref: "#/components/schemas/ProjectId" WorkspaceId: name: workspace_id in: path @@ -537,6 +582,10 @@ components: schema: type: integer schemas: + ProjectId: + type: string + format: uuid + pattern: \b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b # Errors CustomError: type: object @@ -616,6 +665,17 @@ components: example: code: UploadError detail: "Project version could not be created (UploadError)" + BatchItemError: + type: object + properties: + id: + $ref: "#/components/schemas/ProjectId" + error: + type: integer + example: 404 + required: + - id + - error DiffDownloadError: allOf: - $ref: "#/components/schemas/CustomError" diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index deb4722c..ab2c00a2 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -18,11 +18,13 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import ObjectDeletedError +from .schemas_v2 import BatchErrorSchema, ProjectSchema as ProjectSchemaV2 from ..app import db from ..auth import auth_required from ..auth.models import User from .errors import ( AnotherUploadRunning, + BatchLimitError, BigChunkError, DataSyncError, DiffDownloadError, @@ -43,7 +45,12 @@ project_version_created, push_finished, ) -from .permissions import ProjectPermissions, require_project_by_uuid, projects_query +from .permissions import ( + ProjectPermissions, + check_project_permissions, + require_project_by_uuid, + projects_query, +) from .public_api_controller import catch_sync_failure from .schemas import ( ProjectMemberSchema, @@ -529,3 +536,40 @@ def list_workspace_projects(workspace_id, page, per_page, order_params=None, q=N data = ProjectSchemaV2(many=True).dump(result) return jsonify(projects=data, count=total, page=page, per_page=per_page), 200 + + +def list_batch_projects(body): + """List projects by given list of UUIDs. Limit to 100 projects per request. + + :param ids: List of project UUIDs + :type ids: List[str] + :rtype: Dict[str: List[Project]] + """ + ids = list(dict.fromkeys(body.get("ids", []))) + # remove duplicates while preserving the order + max_batch = current_app.config.get("MAX_BATCH_SIZE", 100) + if len(ids) > max_batch: + return BatchLimitError().response(400) + + projects = current_app.project_handler.get_projects_by_uuids(ids) + by_id = {str(project.id): project for project in projects} + + filtered_projects = [] + for uuid in ids: + project = by_id.get(uuid) + + if not project: + filtered_projects.append( + BatchErrorSchema().dump({"id": uuid, "error": 404}) + ) + continue + + err = check_project_permissions(project, ProjectPermissions.Read) + if err is not None: + filtered_projects.append( + BatchErrorSchema().dump({"id": uuid, "error": err}) + ) + else: + filtered_projects.append(ProjectSchemaV2().dump(project)) + + return jsonify(projects=filtered_projects), 200 diff --git a/server/mergin/sync/schemas_v2.py b/server/mergin/sync/schemas_v2.py index d6b781ee..55b5be52 100644 --- a/server/mergin/sync/schemas_v2.py +++ b/server/mergin/sync/schemas_v2.py @@ -46,3 +46,8 @@ class Meta: "workspace", "role", ) + + +class BatchErrorSchema(ma.Schema): + id = fields.UUID(required=True) + error = fields.Integer(required=True) diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index d3553870..9e89eb7c 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -33,6 +33,8 @@ from flask import current_app from pathlib import Path +from .config import Configuration + # log base for caching strategy, diff checkpoints, etc. LOG_BASE = 4 @@ -357,6 +359,8 @@ def has_trailing_space(filepath: str) -> bool: def is_supported_extension(filepath) -> bool: """Check whether file's extension is supported.""" + if check_skip_validation(filepath): + return True ext = os.path.splitext(filepath)[1].lower() return ext and ext not in FORBIDDEN_EXTENSIONS @@ -499,6 +503,16 @@ def is_supported_extension(filepath) -> bool: ".xnk", } + +def check_skip_validation(file_path: str) -> bool: + """ + Check if we can skip validation for this file path. + Some files are allowed even if they have forbidden extension or mime type. + """ + file_name = os.path.basename(file_path) + return file_name in Configuration.UPLOAD_FILES_WHITELIST + + FORBIDDEN_MIME_TYPES = { "application/x-msdownload", "application/x-sh", @@ -523,6 +537,8 @@ def is_supported_extension(filepath) -> bool: def is_supported_type(filepath) -> bool: """Check whether the file mimetype is supported.""" + if check_skip_validation(filepath): + return True mime_type = get_mimetype(filepath) return mime_type.startswith("image/") or mime_type not in FORBIDDEN_MIME_TYPES diff --git a/server/mergin/tests/test_permissions.py b/server/mergin/tests/test_permissions.py index 230961f0..73bf5ab4 100644 --- a/server/mergin/tests/test_permissions.py +++ b/server/mergin/tests/test_permissions.py @@ -2,15 +2,29 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial +import pytest +from unittest.mock import patch import datetime from flask_login import AnonymousUserMixin -from ..sync.permissions import require_project, ProjectPermissions -from ..sync.models import ProjectRole +from mergin.tests import DEFAULT_USER + +from ..sync.permissions import ( + require_project, + check_project_permissions, + ProjectPermissions, +) +from ..sync.models import Project, ProjectRole from ..auth.models import User from ..app import db from ..config import Configuration -from .utils import add_user, create_project, create_workspace +from .utils import ( + add_user, + create_project, + create_workspace, + login, + logout, +) def test_project_permissions(client): @@ -116,3 +130,47 @@ def test_project_permissions(client): assert ProjectPermissions.All.check(project, user) assert ProjectPermissions.Edit.check(project, user) assert ProjectPermissions.get_user_project_role(project, user) == ProjectRole.OWNER + + +def test_check_project_permissions(client): + """Test check_project_permissions with various permission scenarios.""" + admin = User.query.filter_by(username=DEFAULT_USER[0]).first() + test_workspace = create_workspace() + + private_proj = create_project("batch_private", test_workspace, admin) + public_proj = create_project("batch_public", test_workspace, admin) + + p = Project.query.get(public_proj.id) + p.public = True + db.session.commit() + + priv_proj = Project.query.get(private_proj.id) + pub_proj = Project.query.get(public_proj.id) + + # First user with access to both projects + login(client, DEFAULT_USER[0], DEFAULT_USER[1]) + + with client: + client.get("/") + assert check_project_permissions(priv_proj, ProjectPermissions.Read) is None + assert check_project_permissions(pub_proj, ProjectPermissions.Read) is None + + # Second user with no access to private project (ensure global perms disabled) + with patch.object(Configuration, "GLOBAL_READ", False), patch.object( + Configuration, "GLOBAL_WRITE", False + ), patch.object(Configuration, "GLOBAL_ADMIN", False): + user2 = add_user("user_batch", "password") + login(client, user2.username, "password") + + with client: + client.get("/") + assert check_project_permissions(pub_proj, ProjectPermissions.Read) is None + assert check_project_permissions(priv_proj, ProjectPermissions.Read) == 403 + + # Logged-out (anonymous) user + logout(client) + + with client: + client.get("/") + assert check_project_permissions(priv_proj, ProjectPermissions.Read) == 404 + assert check_project_permissions(pub_proj, ProjectPermissions.Read) is None diff --git a/server/mergin/tests/test_project_handler.py b/server/mergin/tests/test_project_handler.py index 76040ca8..f453e0fb 100644 --- a/server/mergin/tests/test_project_handler.py +++ b/server/mergin/tests/test_project_handler.py @@ -1,3 +1,6 @@ +from datetime import datetime + +from . import DEFAULT_USER from ..sync.models import Project, ProjectRole from .utils import add_user, create_project, create_workspace from ..sync.project_handler import ProjectHandler @@ -51,3 +54,26 @@ def test_email_receivers(client): db.session.commit() receivers = project_handler.get_email_receivers(project) assert len(receivers) == 0 + + +def test_get_projects_by_uuids(client): + """Test getting projects with their UUIDs""" + project_handler = ProjectHandler() + test_workspace = create_workspace() + user = User.query.filter_by(username=DEFAULT_USER[0]).first() + p_found = create_project("p_found", test_workspace, user) + p_removed = create_project("p_removed", test_workspace, user) + p_removed.removed_at = datetime.now() + db.session.commit() + p_other = create_project("p_other", test_workspace, user) + ids = [ + str(p_found.id), + str(p_removed.id), + ] + + projects = project_handler.get_projects_by_uuids(ids) + returned_ids = [str(p.id) for p in projects] + assert str(p_found.id) in returned_ids + assert str(p_removed.id) not in returned_ids + assert str(p_other.id) not in returned_ids + assert len(projects) == 1 diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 528a45d0..85c6ab38 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -51,6 +51,7 @@ import pytest from datetime import datetime, timedelta, timezone import json +import uuid from mergin.app import db from mergin.config import Configuration @@ -1476,3 +1477,85 @@ def test_list_workspace_projects(client): # logout logout(client) assert client.get(url + "?page=1&per_page=10").status_code == 401 + + +def test_list_projects_in_batch(client): + """Test batch project listing endpoint.""" + admin = User.query.filter_by(username=DEFAULT_USER[0]).first() + test_workspace = create_workspace() + + private_proj = create_project("batch_private", test_workspace, admin) + public_proj = create_project("batch_public", test_workspace, admin) + + p = Project.query.get(public_proj.id) + p.public = True + db.session.commit() + + url = "/v2/projects/batch" + priv_id = str(private_proj.id) + pub_id = str(public_proj.id) + + # missing ids -> 400 (connexion validation) + resp = client.post(url, json={}) + assert resp.status_code == 400 + + # invalid UUID -> 400 + resp = client.post(url, json={"ids": ["invalid-uuid", pub_id]}) + assert resp.status_code == 400 + + # returns envelope with projects list + resp = client.post(url, json={"ids": [priv_id, pub_id]}) + assert resp.status_code == 200 + assert "projects" in resp.json + assert isinstance(resp.json["projects"], list) + assert len(resp.json["projects"]) == 2 + # Both projects returned as full objects for admin + for proj in resp.json["projects"]: + assert "id" in proj + assert "name" in proj # full project object + + # Second user with no access to private project + user2 = add_user("user_batch", "password") + login(client, user2.username, "password") + + with patch.object(Configuration, "GLOBAL_READ", False): + resp = client.post(url, json={"ids": [pub_id, priv_id]}) + assert resp.status_code == 200 + projects = resp.json["projects"] + assert len(projects) == 2 + + # public -> full object + pub_result = next(p for p in projects if p.get("id") == pub_id) + assert "name" in pub_result + + # private -> error 403 + priv_result = next(p for p in projects if p.get("id") == priv_id) + assert priv_result["error"] == 403 + + # global permission allows any user to list the project + with patch.object(Configuration, "GLOBAL_READ", True): + resp = client.post(url, json={"ids": [pub_id, priv_id]}) + priv_result = next(p for p in resp.json["projects"] if p.get("id") == priv_id) + assert "name" in priv_result + + # Logged-out (anonymous) user - endpoint allows access to public projects, denies private + logout(client) + resp = client.post(url, json={"ids": [pub_id, priv_id]}) + assert resp.status_code == 200 + projects = resp.json["projects"] + assert len(projects) == 2 + + # public -> full object + pub_result = next(p for p in projects if p.get("id") == pub_id) + assert "name" in pub_result + + # private -> error 404 (anonymous cannot access private) + priv_result = next(p for p in projects if p.get("id") == priv_id) + assert priv_result["error"] == 404 + + # batch size limit: generate more than allowed uuids and expect error + max_batch = client.application.config.get("MAX_BATCH_SIZE", 100) + ids = [str(uuid.uuid4()) for _ in range(max_batch + 1)] + resp = client.post(url, json={"ids": ids}) + assert resp.status_code == 400 + assert resp.json["code"] == "BatchLimitExceeded" diff --git a/server/mergin/tests/test_utils.py b/server/mergin/tests/test_utils.py index 4a0391f9..1f447875 100644 --- a/server/mergin/tests/test_utils.py +++ b/server/mergin/tests/test_utils.py @@ -22,11 +22,14 @@ has_valid_characters, has_valid_first_character, check_filename, + is_supported_extension, + is_supported_type, is_valid_path, get_x_accel_uri, Checkpoint, wkb2wkt, has_trailing_space, + check_skip_validation, ) from ..auth.models import LoginHistory, User from . import json_headers @@ -356,3 +359,46 @@ class TestSchema(Schema): "size": "disk_usage", } assert schema_map == expected_map + + +def test_check_skip_validation(): + ALLOWED_FILES = ["script.js", "config/script.js"] + + # We patch the Configuration class attribute directly + with patch("mergin.sync.utils.Configuration.UPLOAD_FILES_WHITELIST", ALLOWED_FILES): + + # Test allowed files + for file_path in ALLOWED_FILES: + assert check_skip_validation(file_path) + + # Test not allowed files + assert not check_skip_validation("test.py") + assert not check_skip_validation("/some/path/test.py") + assert not check_skip_validation("image.png") + + +def test_is_supported_extension(): + ALLOWED_FILES = ["script.js", "config/script.js"] + + with patch("mergin.sync.utils.Configuration.UPLOAD_FILES_WHITELIST", ALLOWED_FILES): + for file_path in ALLOWED_FILES: + assert is_supported_extension(file_path) + + # Allowed normal file + assert is_supported_extension("image.png") + + # Forbidden file + assert not is_supported_extension("test.js") + + +def test_mime_type_validation_skip(): + ALLOWED_FILES = ["script.js", "config/script.js"] + # Mocking get_mimetype to return forbidden mime type + with patch( + "mergin.sync.utils.get_mimetype", return_value="application/x-python-code" + ), patch("mergin.sync.utils.Configuration.UPLOAD_FILES_WHITELIST", ALLOWED_FILES): + for file_path in ALLOWED_FILES: + assert is_supported_type(file_path) + + # Should be forbidden + assert not is_supported_type("other.js") diff --git a/server/mergin/utils.py b/server/mergin/utils.py index 5d755adc..aa878ffe 100644 --- a/server/mergin/utils.py +++ b/server/mergin/utils.py @@ -60,7 +60,7 @@ def get_order_param( attr = None order_attr = cls.__table__.c.get(col, None) if not isinstance(order_attr, Column): - logging.warning("Ignoring invalid order parameter.") + logging.warning(f"Ignoring invalid order parameter: {order_param}.") return # sort by key in JSON field if attr: diff --git a/web-app/packages/lib/src/assets/sass/theme-base/_mixins.scss b/web-app/packages/lib/src/assets/sass/theme-base/_mixins.scss index 7db81588..076d5c2b 100644 --- a/web-app/packages/lib/src/assets/sass/theme-base/_mixins.scss +++ b/web-app/packages/lib/src/assets/sass/theme-base/_mixins.scss @@ -30,7 +30,7 @@ @mixin invalid-input() { border-color: $inputErrorBorderColor; background-color: $inputErrorBackgroundColor; - box-shadow: inset 0 0 0 1px $inputErrorBorderColor; + box-shadow: inset 0 0 0 2px $inputErrorBorderColor; } @mixin menuitem { diff --git a/web-app/packages/lib/src/assets/sass/themes/mm-theme-light/_extensions.scss b/web-app/packages/lib/src/assets/sass/themes/mm-theme-light/_extensions.scss index fa51ab3c..ccf252bb 100644 --- a/web-app/packages/lib/src/assets/sass/themes/mm-theme-light/_extensions.scss +++ b/web-app/packages/lib/src/assets/sass/themes/mm-theme-light/_extensions.scss @@ -110,6 +110,10 @@ img { color: map-get($colors, 'dark-green'); } +.text-color-dark-gray { + color: map-get($colors, 'dark-gray'); +} + .overflow-wrap-anywhere { overflow-wrap: anywhere; } @@ -124,3 +128,8 @@ img { .p-avatar:not(.p-avatar-lg):not(.p-avatar-xl) { font-size: 0.857rem; } + +.tooltip { + text-decoration-line: underline; + text-decoration-style: dotted; +} diff --git a/web-app/packages/lib/src/assets/sass/themes/mm-theme-light/_variables.scss b/web-app/packages/lib/src/assets/sass/themes/mm-theme-light/_variables.scss index feca7d83..6dea6d06 100644 --- a/web-app/packages/lib/src/assets/sass/themes/mm-theme-light/_variables.scss +++ b/web-app/packages/lib/src/assets/sass/themes/mm-theme-light/_variables.scss @@ -37,7 +37,9 @@ $colors: ( "rose": #FFBABC, "field": #9BD1A9, "sky": #A6CBF4, - "informative": #BEDAF0 + "informative": #BEDAF0, + "dark-gray": #41464C, + "medium-gray": #A0A3A5 ); // Mandatory Designer Variables @@ -115,5 +117,7 @@ $colors: ( --dark-green-color: #{map-get($colors, "dark-green")}; --negative-light-color: #{map-get($colors, "negative-light")}; --earth-color: #{map-get($colors, "earth")}; + --dark-gray-color: #{map-get($colors, "dark-gray")}; + --medium-gray-color: #{map-get($colors, "medium-gray")}; color-scheme: light; } diff --git a/web-app/packages/lib/src/modules/project/views/ProjectsListViewTemplate.vue b/web-app/packages/lib/src/modules/project/views/ProjectsListViewTemplate.vue index 5d91bebc..afa28bed 100644 --- a/web-app/packages/lib/src/modules/project/views/ProjectsListViewTemplate.vue +++ b/web-app/packages/lib/src/modules/project/views/ProjectsListViewTemplate.vue @@ -32,7 +32,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial class="w-full" /> - +