From f52fb919d16f5d723e030b81658ebac05cbd8f5b Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 29 Jan 2026 23:01:12 +0800 Subject: [PATCH] refactor(storage): remove signer, using general file storage - Removed unused app asset download and upload endpoints, along with sandbox archive and file download endpoints. - Updated imports in the file controller to reflect the removal of these endpoints. - Simplified the generator.py file by consolidating the code context field definition. - Enhanced the storage layer with a unified presign wrapper for better handling of presigned URLs. --- api/controllers/console/app/generator.py | 4 +- api/controllers/files/__init__.py | 12 +- api/controllers/files/app_assets_download.py | 77 ----- api/controllers/files/app_assets_upload.py | 61 ---- api/controllers/files/sandbox_archive.py | 76 ----- .../files/sandbox_file_downloads.py | 96 ------ api/controllers/files/storage_download.py | 56 --- api/controllers/files/storage_proxy.py | 102 ++++++ api/core/app_assets/storage.py | 322 ++++-------------- api/core/sandbox/inspector/archive_source.py | 13 +- api/core/sandbox/inspector/runtime_source.py | 2 +- api/core/sandbox/security/__init__.py | 1 - api/core/sandbox/security/archive_signer.py | 152 --------- .../sandbox/security/sandbox_file_signer.py | 155 --------- api/core/sandbox/storage/__init__.py | 6 +- api/core/sandbox/storage/archive_storage.py | 81 +++-- .../sandbox/storage/sandbox_file_storage.py | 83 +++-- .../storage/file_presign_storage.py | 91 +++-- api/services/app_asset_service.py | 6 +- api/services/sandbox/sandbox_service.py | 21 +- .../core/app_assets/test_storage.py | 119 ++++--- 21 files changed, 449 insertions(+), 1087 deletions(-) delete mode 100644 api/controllers/files/app_assets_download.py delete mode 100644 api/controllers/files/app_assets_upload.py delete mode 100644 api/controllers/files/sandbox_archive.py delete mode 100644 api/controllers/files/sandbox_file_downloads.py delete mode 100644 api/controllers/files/storage_download.py create mode 100644 api/controllers/files/storage_proxy.py delete mode 100644 api/core/sandbox/security/__init__.py delete mode 100644 api/core/sandbox/security/archive_signer.py delete mode 100644 api/core/sandbox/security/sandbox_file_signer.py diff --git a/api/controllers/console/app/generator.py b/api/controllers/console/app/generator.py index 63870f8038..d14dd52e4e 100644 --- a/api/controllers/console/app/generator.py +++ b/api/controllers/console/app/generator.py @@ -70,9 +70,7 @@ class ContextGeneratePayload(BaseModel): model_config_data: dict[str, Any] = Field(..., alias="model_config", description="Model configuration") available_vars: list[AvailableVarPayload] = Field(..., description="Available variables from upstream nodes") parameter_info: ParameterInfoPayload = Field(..., description="Target parameter metadata from the frontend") - code_context: CodeContextPayload = Field( - description="Existing code node context for incremental generation" - ) + code_context: CodeContextPayload = Field(description="Existing code node context for incremental generation") class SuggestedQuestionsPayload(BaseModel): diff --git a/api/controllers/files/__init__.py b/api/controllers/files/__init__.py index 77eb012c7c..77bacd6286 100644 --- a/api/controllers/files/__init__.py +++ b/api/controllers/files/__init__.py @@ -15,12 +15,8 @@ api = ExternalApi( files_ns = Namespace("files", description="File operations", path="/") from . import ( - app_assets_download, - app_assets_upload, image_preview, - sandbox_archive, - sandbox_file_downloads, - storage_download, + storage_proxy, tool_files, upload, ) @@ -29,14 +25,10 @@ api.add_namespace(files_ns) __all__ = [ "api", - "app_assets_download", - "app_assets_upload", "bp", "files_ns", "image_preview", - "sandbox_archive", - "sandbox_file_downloads", - "storage_download", + "storage_proxy", "tool_files", "upload", ] diff --git a/api/controllers/files/app_assets_download.py b/api/controllers/files/app_assets_download.py deleted file mode 100644 index 1d829a5671..0000000000 --- a/api/controllers/files/app_assets_download.py +++ /dev/null @@ -1,77 +0,0 @@ -from urllib.parse import quote - -from flask import Response, request -from flask_restx import Resource -from pydantic import BaseModel, Field -from werkzeug.exceptions import Forbidden, NotFound - -from controllers.files import files_ns -from core.app_assets.storage import AppAssetSigner, AssetPath -from extensions.ext_storage import storage - -DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" - - -class AppAssetDownloadQuery(BaseModel): - expires_at: int = Field(..., description="Unix timestamp when the link expires") - nonce: str = Field(..., description="Random string for signature") - sign: str = Field(..., description="HMAC signature") - - -files_ns.schema_model( - AppAssetDownloadQuery.__name__, - AppAssetDownloadQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), -) - - -@files_ns.route("/app-assets/////download") -@files_ns.route( - "/app-assets//////download" -) -class AppAssetDownloadApi(Resource): - def get( - self, - asset_type: str, - tenant_id: str, - app_id: str, - resource_id: str, - sub_resource_id: str | None = None, - ): - args = AppAssetDownloadQuery.model_validate(request.args.to_dict(flat=True)) - - try: - asset_path = AssetPath.from_components( - asset_type=asset_type, - tenant_id=tenant_id, - app_id=app_id, - resource_id=resource_id, - sub_resource_id=sub_resource_id, - ) - except ValueError as exc: - raise Forbidden(str(exc)) from exc - - if not AppAssetSigner.verify_download_signature( - asset_path=asset_path, - expires_at=args.expires_at, - nonce=args.nonce, - sign=args.sign, - ): - raise Forbidden("Invalid or expired download link") - - storage_key = asset_path.get_storage_key() - - try: - generator = storage.load_stream(storage_key) - except FileNotFoundError as exc: - raise NotFound("File not found") from exc - - encoded_filename = quote(storage_key.split("/")[-1]) - - return Response( - generator, - mimetype="application/octet-stream", - direct_passthrough=True, - headers={ - "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}", - }, - ) diff --git a/api/controllers/files/app_assets_upload.py b/api/controllers/files/app_assets_upload.py deleted file mode 100644 index c25bc5e100..0000000000 --- a/api/controllers/files/app_assets_upload.py +++ /dev/null @@ -1,61 +0,0 @@ -from flask import Response, request -from flask_restx import Resource -from pydantic import BaseModel, Field -from werkzeug.exceptions import Forbidden - -from controllers.files import files_ns -from core.app_assets.storage import AppAssetSigner, AssetPath -from services.app_asset_service import AppAssetService - -DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" - - -class AppAssetUploadQuery(BaseModel): - expires_at: int = Field(..., description="Unix timestamp when the link expires") - nonce: str = Field(..., description="Random string for signature") - sign: str = Field(..., description="HMAC signature") - - -files_ns.schema_model( - AppAssetUploadQuery.__name__, - AppAssetUploadQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), -) - - -@files_ns.route("/app-assets/////upload") -@files_ns.route( - "/app-assets//////upload" -) -class AppAssetUploadApi(Resource): - def put( - self, - asset_type: str, - tenant_id: str, - app_id: str, - resource_id: str, - sub_resource_id: str | None = None, - ): - args = AppAssetUploadQuery.model_validate(request.args.to_dict(flat=True)) - - try: - asset_path = AssetPath.from_components( - asset_type=asset_type, - tenant_id=tenant_id, - app_id=app_id, - resource_id=resource_id, - sub_resource_id=sub_resource_id, - ) - except ValueError as exc: - raise Forbidden(str(exc)) from exc - - if not AppAssetSigner.verify_upload_signature( - asset_path=asset_path, - expires_at=args.expires_at, - nonce=args.nonce, - sign=args.sign, - ): - raise Forbidden("Invalid or expired upload link") - - content = request.get_data() - AppAssetService.get_storage().save(asset_path, content) - return Response(status=204) diff --git a/api/controllers/files/sandbox_archive.py b/api/controllers/files/sandbox_archive.py deleted file mode 100644 index 4f5e591a08..0000000000 --- a/api/controllers/files/sandbox_archive.py +++ /dev/null @@ -1,76 +0,0 @@ -from uuid import UUID - -from flask import Response, request -from flask_restx import Resource -from pydantic import BaseModel, Field -from werkzeug.exceptions import Forbidden, NotFound - -from controllers.files import files_ns -from core.sandbox.security.archive_signer import SandboxArchivePath, SandboxArchiveSigner -from extensions.ext_storage import storage - -DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" - - -class SandboxArchiveQuery(BaseModel): - expires_at: int = Field(..., description="Unix timestamp when the link expires") - nonce: str = Field(..., description="Random string for signature") - sign: str = Field(..., description="HMAC signature") - - -files_ns.schema_model( - SandboxArchiveQuery.__name__, - SandboxArchiveQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), -) - - -@files_ns.route("/sandbox-archives///download") -class SandboxArchiveDownloadApi(Resource): - def get(self, tenant_id: str, sandbox_id: str): - args = SandboxArchiveQuery.model_validate(request.args.to_dict(flat=True)) - - try: - archive_path = SandboxArchivePath(tenant_id=UUID(tenant_id), sandbox_id=UUID(sandbox_id)) - except ValueError as exc: - raise Forbidden(str(exc)) from exc - - if not SandboxArchiveSigner.verify_download_signature( - archive_path=archive_path, - expires_at=args.expires_at, - nonce=args.nonce, - sign=args.sign, - ): - raise Forbidden("Invalid or expired download link") - - try: - generator = storage.load_stream(archive_path.get_storage_key()) - except FileNotFoundError as exc: - raise NotFound("Archive not found") from exc - - return Response( - generator, - mimetype="application/gzip", - direct_passthrough=True, - ) - - -@files_ns.route("/sandbox-archives///upload") -class SandboxArchiveUploadApi(Resource): - def put(self, tenant_id: str, sandbox_id: str): - args = SandboxArchiveQuery.model_validate(request.args.to_dict(flat=True)) - - try: - archive_path = SandboxArchivePath(tenant_id=UUID(tenant_id), sandbox_id=UUID(sandbox_id)) - except ValueError as exc: - raise Forbidden(str(exc)) from exc - - if not SandboxArchiveSigner.verify_upload_signature( - archive_path=archive_path, - expires_at=args.expires_at, - nonce=args.nonce, - sign=args.sign, - ): - raise Forbidden("Invalid or expired upload link") - - storage.save(archive_path.get_storage_key(), request.get_data()) - return Response(status=204) diff --git a/api/controllers/files/sandbox_file_downloads.py b/api/controllers/files/sandbox_file_downloads.py deleted file mode 100644 index 7f021d4493..0000000000 --- a/api/controllers/files/sandbox_file_downloads.py +++ /dev/null @@ -1,96 +0,0 @@ -from urllib.parse import quote -from uuid import UUID - -from flask import Response, request -from flask_restx import Resource -from pydantic import BaseModel, Field -from werkzeug.exceptions import Forbidden, NotFound - -from controllers.files import files_ns -from core.sandbox.security.sandbox_file_signer import SandboxFileDownloadPath, SandboxFileSigner -from extensions.ext_storage import storage - -DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" - - -class SandboxFileDownloadQuery(BaseModel): - expires_at: int = Field(..., description="Unix timestamp when the link expires") - nonce: str = Field(..., description="Random string for signature") - sign: str = Field(..., description="HMAC signature") - - -files_ns.schema_model( - SandboxFileDownloadQuery.__name__, - SandboxFileDownloadQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), -) - - -@files_ns.route( - "/sandbox-file-downloads/////download" -) -class SandboxFileDownloadDownloadApi(Resource): - def get(self, tenant_id: str, sandbox_id: str, export_id: str, filename: str): - args = SandboxFileDownloadQuery.model_validate(request.args.to_dict(flat=True)) - - try: - export_path = SandboxFileDownloadPath( - tenant_id=UUID(tenant_id), - sandbox_id=UUID(sandbox_id), - export_id=export_id, - filename=filename, - ) - except ValueError as exc: - raise Forbidden(str(exc)) from exc - - if not SandboxFileSigner.verify_download_signature( - export_path=export_path, - expires_at=args.expires_at, - nonce=args.nonce, - sign=args.sign, - ): - raise Forbidden("Invalid or expired download link") - - try: - generator = storage.load_stream(export_path.get_storage_key()) - except FileNotFoundError as exc: - raise NotFound("File not found") from exc - - encoded_filename = quote(filename.split("/")[-1]) - - return Response( - generator, - mimetype="application/octet-stream", - direct_passthrough=True, - headers={ - "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}", - }, - ) - - -@files_ns.route( - "/sandbox-file-downloads/////upload" -) -class SandboxFileDownloadUploadApi(Resource): - def put(self, tenant_id: str, sandbox_id: str, export_id: str, filename: str): - args = SandboxFileDownloadQuery.model_validate(request.args.to_dict(flat=True)) - - try: - export_path = SandboxFileDownloadPath( - tenant_id=UUID(tenant_id), - sandbox_id=UUID(sandbox_id), - export_id=export_id, - filename=filename, - ) - except ValueError as exc: - raise Forbidden(str(exc)) from exc - - if not SandboxFileSigner.verify_upload_signature( - export_path=export_path, - expires_at=args.expires_at, - nonce=args.nonce, - sign=args.sign, - ): - raise Forbidden("Invalid or expired upload link") - - storage.save(export_path.get_storage_key(), request.get_data()) - return Response(status=204) diff --git a/api/controllers/files/storage_download.py b/api/controllers/files/storage_download.py deleted file mode 100644 index dfa6193a80..0000000000 --- a/api/controllers/files/storage_download.py +++ /dev/null @@ -1,56 +0,0 @@ -from urllib.parse import quote, unquote - -from flask import Response, request -from flask_restx import Resource -from pydantic import BaseModel, Field -from werkzeug.exceptions import Forbidden, NotFound - -from controllers.files import files_ns -from extensions.ext_storage import storage -from extensions.storage.file_presign_storage import FilePresignStorage - -DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" - - -class StorageDownloadQuery(BaseModel): - timestamp: str = Field(..., description="Unix timestamp used in the signature") - nonce: str = Field(..., description="Random string for signature") - sign: str = Field(..., description="HMAC signature") - - -files_ns.schema_model( - StorageDownloadQuery.__name__, - StorageDownloadQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), -) - - -@files_ns.route("/storage//download") -class StorageFileDownloadApi(Resource): - def get(self, filename: str): - filename = unquote(filename) - - args = StorageDownloadQuery.model_validate(request.args.to_dict(flat=True)) - - if not FilePresignStorage.verify_signature( - filename=filename, - timestamp=args.timestamp, - nonce=args.nonce, - sign=args.sign, - ): - raise Forbidden("Invalid or expired download link") - - try: - generator = storage.load_stream(filename) - except FileNotFoundError: - raise NotFound("File not found") - - encoded_filename = quote(filename.split("/")[-1]) - - return Response( - generator, - mimetype="application/octet-stream", - direct_passthrough=True, - headers={ - "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}", - }, - ) diff --git a/api/controllers/files/storage_proxy.py b/api/controllers/files/storage_proxy.py new file mode 100644 index 0000000000..8b7b6cae95 --- /dev/null +++ b/api/controllers/files/storage_proxy.py @@ -0,0 +1,102 @@ +"""Unified file proxy controller for storage operations. + +This controller handles file download and upload operations when the underlying +storage backend doesn't support presigned URLs. It verifies signed proxy URLs +generated by FilePresignStorage and streams files to/from storage. + +Endpoints: + GET /files/storage/{filename}/download - Download a file + PUT /files/storage/{filename}/upload - Upload a file +""" + +from urllib.parse import quote, unquote + +from flask import Response, request +from flask_restx import Resource +from pydantic import BaseModel, Field +from werkzeug.exceptions import Forbidden, NotFound + +from controllers.files import files_ns +from extensions.ext_storage import storage +from extensions.storage.file_presign_storage import FilePresignStorage + +DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}" + + +class StorageProxyQuery(BaseModel): + """Query parameters for storage proxy URLs.""" + + timestamp: str = Field(..., description="Unix timestamp used in the signature") + nonce: str = Field(..., description="Random string for signature") + sign: str = Field(..., description="HMAC signature") + + +files_ns.schema_model( + StorageProxyQuery.__name__, + StorageProxyQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0), +) + + +@files_ns.route("/storage//download") +class StorageFileDownloadApi(Resource): + """Handle file downloads through the proxy.""" + + def get(self, filename: str): + """Download a file from storage. + + Verifies the signed URL and streams the file content back to the client. + """ + filename = unquote(filename) + args = StorageProxyQuery.model_validate(request.args.to_dict(flat=True)) + + if not FilePresignStorage.verify_signature( + filename=filename, + operation="download", + timestamp=args.timestamp, + nonce=args.nonce, + sign=args.sign, + ): + raise Forbidden("Invalid or expired download link") + + try: + generator = storage.load_stream(filename) + except FileNotFoundError: + raise NotFound("File not found") + + encoded_filename = quote(filename.split("/")[-1]) + + return Response( + generator, + mimetype="application/octet-stream", + direct_passthrough=True, + headers={ + "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}", + }, + ) + + +@files_ns.route("/storage//upload") +class StorageFileUploadApi(Resource): + """Handle file uploads through the proxy.""" + + def put(self, filename: str): + """Upload a file to storage. + + Verifies the signed URL and saves the request body to storage. + """ + filename = unquote(filename) + args = StorageProxyQuery.model_validate(request.args.to_dict(flat=True)) + + if not FilePresignStorage.verify_signature( + filename=filename, + operation="upload", + timestamp=args.timestamp, + nonce=args.nonce, + sign=args.sign, + ): + raise Forbidden("Invalid or expired upload link") + + content = request.get_data() + storage.save(filename, content) + + return Response(status=204) diff --git a/api/core/app_assets/storage.py b/api/core/app_assets/storage.py index 50e9d5dac0..17964cb9d5 100644 --- a/api/core/app_assets/storage.py +++ b/api/core/app_assets/storage.py @@ -1,25 +1,31 @@ +"""App assets storage layer. + +This module provides storage abstractions for app assets (draft files, build zips, +resolved assets, skill bundles, source zips, bundle exports/imports). + +Key components: +- AssetPath: Factory for creating typed storage paths +- AppAssetStorage: High-level storage operations with presign support + +All presign operations use the unified FilePresignStorage wrapper, which automatically +falls back to Dify's file proxy when the underlying storage doesn't support presigned URLs. +""" + from __future__ import annotations -import base64 -import hashlib -import hmac -import os -import time -import urllib.parse from abc import ABC, abstractmethod -from collections.abc import Callable, Iterable +from collections.abc import Generator, Iterable from dataclasses import dataclass from typing import Any, ClassVar from uuid import UUID -from configs import dify_config from extensions.storage.base_storage import BaseStorage from extensions.storage.cached_presign_storage import CachedPresignStorage -from libs import rsa +from extensions.storage.file_presign_storage import FilePresignStorage _ASSET_BASE = "app_assets" _SILENT_STORAGE_NOT_FOUND = b"File Not Found" -_ASSET_PATH_REGISTRY: dict[str, tuple[bool, Callable[..., SignedAssetPath]]] = {} +_ASSET_PATH_REGISTRY: dict[str, tuple[bool, Any]] = {} def _require_uuid(value: str, field_name: str) -> None: @@ -29,12 +35,14 @@ def _require_uuid(value: str, field_name: str) -> None: raise ValueError(f"{field_name} must be a UUID") from exc -def register_asset_path(asset_type: str, *, requires_node: bool, factory: Callable[..., SignedAssetPath]) -> None: +def register_asset_path(asset_type: str, *, requires_node: bool, factory: Any) -> None: _ASSET_PATH_REGISTRY[asset_type] = (requires_node, factory) @dataclass(frozen=True) class AssetPathBase(ABC): + """Base class for all asset paths.""" + asset_type: ClassVar[str] tenant_id: str app_id: str @@ -50,49 +58,24 @@ class AssetPathBase(ABC): raise NotImplementedError -class SignedAssetPath(AssetPathBase, ABC): - @abstractmethod - def signature_parts(self) -> tuple[str, str | None]: - """Return (resource_id, sub_resource_id) used for signing. - - sub_resource_id should be None when not applicable. - """ - - @abstractmethod - def proxy_path_parts(self) -> list[str]: - raise NotImplementedError - - @dataclass(frozen=True) -class _DraftAssetPath(SignedAssetPath): +class _DraftAssetPath(AssetPathBase): asset_type: ClassVar[str] = "draft" def get_storage_key(self) -> str: return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/draft/{self.resource_id}" - def signature_parts(self) -> tuple[str, str | None]: - return (self.resource_id, None) - - def proxy_path_parts(self) -> list[str]: - return [self.asset_type, self.tenant_id, self.app_id, self.resource_id] - @dataclass(frozen=True) -class _BuildZipAssetPath(SignedAssetPath): +class _BuildZipAssetPath(AssetPathBase): asset_type: ClassVar[str] = "build-zip" def get_storage_key(self) -> str: return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/artifacts/{self.resource_id}.zip" - def signature_parts(self) -> tuple[str, str | None]: - return (self.resource_id, None) - - def proxy_path_parts(self) -> list[str]: - return [self.asset_type, self.tenant_id, self.app_id, self.resource_id] - @dataclass(frozen=True) -class _ResolvedAssetPath(SignedAssetPath): +class _ResolvedAssetPath(AssetPathBase): asset_type: ClassVar[str] = "resolved" node_id: str @@ -103,58 +86,34 @@ class _ResolvedAssetPath(SignedAssetPath): def get_storage_key(self) -> str: return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/artifacts/{self.resource_id}/resolved/{self.node_id}" - def signature_parts(self) -> tuple[str, str | None]: - return (self.resource_id, self.node_id) - - def proxy_path_parts(self) -> list[str]: - return [self.asset_type, self.tenant_id, self.app_id, self.resource_id, self.node_id] - @dataclass(frozen=True) -class _SkillBundleAssetPath(SignedAssetPath): +class _SkillBundleAssetPath(AssetPathBase): asset_type: ClassVar[str] = "skill-bundle" def get_storage_key(self) -> str: return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/artifacts/{self.resource_id}/skill_artifact_set.json" - def signature_parts(self) -> tuple[str, str | None]: - return (self.resource_id, None) - - def proxy_path_parts(self) -> list[str]: - return [self.asset_type, self.tenant_id, self.app_id, self.resource_id] - @dataclass(frozen=True) -class _SourceZipAssetPath(SignedAssetPath): +class _SourceZipAssetPath(AssetPathBase): asset_type: ClassVar[str] = "source-zip" def get_storage_key(self) -> str: return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/sources/{self.resource_id}.zip" - def signature_parts(self) -> tuple[str, str | None]: - return (self.resource_id, None) - - def proxy_path_parts(self) -> list[str]: - return [self.asset_type, self.tenant_id, self.app_id, self.resource_id] - @dataclass(frozen=True) -class _BundleExportZipAssetPath(SignedAssetPath): +class _BundleExportZipAssetPath(AssetPathBase): asset_type: ClassVar[str] = "bundle-export-zip" def get_storage_key(self) -> str: return f"{_ASSET_BASE}/{self.tenant_id}/{self.app_id}/bundle_exports/{self.resource_id}.zip" - def signature_parts(self) -> tuple[str, str | None]: - return (self.resource_id, None) - - def proxy_path_parts(self) -> list[str]: - return [self.asset_type, self.tenant_id, self.app_id, self.resource_id] - @dataclass(frozen=True) class BundleImportZipPath: - """Path for temporary import zip files. Not signed, uses direct presign URLs only.""" + """Path for temporary import zip files.""" tenant_id: str import_id: str @@ -167,28 +126,30 @@ class BundleImportZipPath: class AssetPath: + """Factory for creating typed asset paths.""" + @staticmethod - def draft(tenant_id: str, app_id: str, node_id: str) -> SignedAssetPath: + def draft(tenant_id: str, app_id: str, node_id: str) -> AssetPathBase: return _DraftAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=node_id) @staticmethod - def build_zip(tenant_id: str, app_id: str, assets_id: str) -> SignedAssetPath: + def build_zip(tenant_id: str, app_id: str, assets_id: str) -> AssetPathBase: return _BuildZipAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=assets_id) @staticmethod - def resolved(tenant_id: str, app_id: str, assets_id: str, node_id: str) -> SignedAssetPath: + def resolved(tenant_id: str, app_id: str, assets_id: str, node_id: str) -> AssetPathBase: return _ResolvedAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=assets_id, node_id=node_id) @staticmethod - def skill_bundle(tenant_id: str, app_id: str, assets_id: str) -> SignedAssetPath: + def skill_bundle(tenant_id: str, app_id: str, assets_id: str) -> AssetPathBase: return _SkillBundleAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=assets_id) @staticmethod - def source_zip(tenant_id: str, app_id: str, workflow_id: str) -> SignedAssetPath: + def source_zip(tenant_id: str, app_id: str, workflow_id: str) -> AssetPathBase: return _SourceZipAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=workflow_id) @staticmethod - def bundle_export_zip(tenant_id: str, app_id: str, export_id: str) -> SignedAssetPath: + def bundle_export_zip(tenant_id: str, app_id: str, export_id: str) -> AssetPathBase: return _BundleExportZipAssetPath(tenant_id=tenant_id, app_id=app_id, resource_id=export_id) @staticmethod @@ -202,7 +163,7 @@ class AssetPath: app_id: str, resource_id: str, sub_resource_id: str | None = None, - ) -> SignedAssetPath: + ) -> AssetPathBase: entry = _ASSET_PATH_REGISTRY.get(asset_type) if not entry: raise ValueError(f"Unsupported asset type: {asset_type}") @@ -224,120 +185,26 @@ register_asset_path("source-zip", requires_node=False, factory=AssetPath.source_ register_asset_path("bundle-export-zip", requires_node=False, factory=AssetPath.bundle_export_zip) -class AppAssetSigner: - SIGNATURE_PREFIX = "app-asset" - SIGNATURE_VERSION = "v1" - OPERATION_DOWNLOAD = "download" - OPERATION_UPLOAD = "upload" - - @classmethod - def create_download_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str) -> str: - return cls._create_signature( - asset_path=asset_path, - operation=cls.OPERATION_DOWNLOAD, - expires_at=expires_at, - nonce=nonce, - ) - - @classmethod - def create_upload_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str) -> str: - return cls._create_signature( - asset_path=asset_path, - operation=cls.OPERATION_UPLOAD, - expires_at=expires_at, - nonce=nonce, - ) - - @classmethod - def verify_download_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str, sign: str) -> bool: - return cls._verify_signature( - asset_path=asset_path, - operation=cls.OPERATION_DOWNLOAD, - expires_at=expires_at, - nonce=nonce, - sign=sign, - ) - - @classmethod - def verify_upload_signature(cls, asset_path: SignedAssetPath, expires_at: int, nonce: str, sign: str) -> bool: - return cls._verify_signature( - asset_path=asset_path, - operation=cls.OPERATION_UPLOAD, - expires_at=expires_at, - nonce=nonce, - sign=sign, - ) - - @classmethod - def _verify_signature( - cls, - *, - asset_path: SignedAssetPath, - operation: str, - expires_at: int, - nonce: str, - sign: str, - ) -> bool: - if expires_at <= 0: - return False - - expected_sign = cls._create_signature( - asset_path=asset_path, - operation=operation, - expires_at=expires_at, - nonce=nonce, - ) - if not hmac.compare_digest(sign, expected_sign): - return False - - current_time = int(time.time()) - if expires_at < current_time: - return False - - if expires_at - current_time > dify_config.FILES_ACCESS_TIMEOUT: - return False - - return True - - @classmethod - def _create_signature(cls, *, asset_path: SignedAssetPath, operation: str, expires_at: int, nonce: str) -> str: - key = cls._tenant_key(asset_path.tenant_id) - message = cls._signature_message( - asset_path=asset_path, - operation=operation, - expires_at=expires_at, - nonce=nonce, - ) - sign = hmac.new(key, message.encode(), hashlib.sha256).digest() - return base64.urlsafe_b64encode(sign).decode() - - @classmethod - def _signature_message(cls, *, asset_path: SignedAssetPath, operation: str, expires_at: int, nonce: str) -> str: - resource_id, sub_resource_id = asset_path.signature_parts() - return ( - f"{cls.SIGNATURE_PREFIX}|{cls.SIGNATURE_VERSION}|{operation}|" - f"{asset_path.asset_type}|{asset_path.tenant_id}|{asset_path.app_id}|" - f"{resource_id}|{sub_resource_id or ''}|{expires_at}|{nonce}" - ) - - @classmethod - def _tenant_key(cls, tenant_id: str) -> bytes: - try: - rsa_key, _ = rsa.get_decrypt_decoding(tenant_id) - except rsa.PrivkeyNotFoundError as exc: - raise ValueError(f"Tenant private key missing for tenant_id={tenant_id}") from exc - private_key = rsa_key.export_key() - return hashlib.sha256(private_key).digest() - - class AppAssetStorage: - _base_storage: BaseStorage + """High-level storage operations for app assets. + + Wraps BaseStorage with: + - FilePresignStorage for presign fallback support + - CachedPresignStorage for URL caching + + Usage: + storage = AppAssetStorage(base_storage, redis_client=redis) + storage.save(asset_path, content) + url = storage.get_download_url(asset_path) + """ + _storage: CachedPresignStorage def __init__(self, storage: BaseStorage, *, redis_client: Any, cache_key_prefix: str = "app_assets") -> None: - self._base_storage = storage + # Wrap with FilePresignStorage for fallback support, then CachedPresignStorage for caching + presign_storage = FilePresignStorage(storage) self._storage = CachedPresignStorage( - storage=storage, + storage=presign_storage, redis_client=redis_client, cache_key_prefix=cache_key_prefix, ) @@ -347,69 +214,44 @@ class AppAssetStorage: return self._storage def save(self, asset_path: AssetPathBase, content: bytes) -> None: - self._storage.save(self.get_storage_key(asset_path), content) + self._storage.save(asset_path.get_storage_key(), content) def load(self, asset_path: AssetPathBase) -> bytes: - return self._storage.load_once(self.get_storage_key(asset_path)) + return self._storage.load_once(asset_path.get_storage_key()) + + def load_stream(self, asset_path: AssetPathBase) -> Generator[bytes, None, None]: + return self._storage.load_stream(asset_path.get_storage_key()) def load_or_none(self, asset_path: AssetPathBase) -> bytes | None: try: - data = self._storage.load_once(self.get_storage_key(asset_path)) + data = self._storage.load_once(asset_path.get_storage_key()) except FileNotFoundError: return None if data == _SILENT_STORAGE_NOT_FOUND: return None return data + def exists(self, asset_path: AssetPathBase) -> bool: + return self._storage.exists(asset_path.get_storage_key()) + def delete(self, asset_path: AssetPathBase) -> None: - self._storage.delete(self.get_storage_key(asset_path)) + self._storage.delete(asset_path.get_storage_key()) - def get_storage_key(self, asset_path: AssetPathBase) -> str: - return asset_path.get_storage_key() + def get_download_url(self, asset_path: AssetPathBase, expires_in: int = 3600) -> str: + return self._storage.get_download_url(asset_path.get_storage_key(), expires_in) - def get_download_url(self, asset_path: SignedAssetPath, expires_in: int = 3600) -> str: - storage_key = self.get_storage_key(asset_path) - try: - return self._storage.get_download_url(storage_key, expires_in) - except NotImplementedError: - pass + def get_download_urls(self, asset_paths: Iterable[AssetPathBase], expires_in: int = 3600) -> list[str]: + storage_keys = [p.get_storage_key() for p in asset_paths] + return self._storage.get_download_urls(storage_keys, expires_in) - return self._generate_signed_proxy_download_url(asset_path, expires_in) - - def get_download_urls( - self, - asset_paths: Iterable[SignedAssetPath], - expires_in: int = 3600, - ) -> list[str]: - asset_paths_list = list(asset_paths) - storage_keys = [self.get_storage_key(asset_path) for asset_path in asset_paths_list] - - try: - return self._storage.get_download_urls(storage_keys, expires_in) - except NotImplementedError: - pass - - return [self._generate_signed_proxy_download_url(asset_path, expires_in) for asset_path in asset_paths_list] - - def get_upload_url( - self, - asset_path: SignedAssetPath, - expires_in: int = 3600, - ) -> str: - storage_key = self.get_storage_key(asset_path) - try: - return self._storage.get_upload_url(storage_key, expires_in) - except NotImplementedError: - pass - - return self._generate_signed_proxy_upload_url(asset_path, expires_in) + def get_upload_url(self, asset_path: AssetPathBase, expires_in: int = 3600) -> str: + return self._storage.get_upload_url(asset_path.get_storage_key(), expires_in) + # Bundle import convenience methods def get_import_upload_url(self, path: BundleImportZipPath, expires_in: int = 3600) -> str: - """Get upload URL for import zip (direct presign, no proxy fallback).""" return self._storage.get_upload_url(path.get_storage_key(), expires_in) def get_import_download_url(self, path: BundleImportZipPath, expires_in: int = 3600) -> str: - """Get download URL for import zip (direct presign, no proxy fallback).""" return self._storage.get_download_url(path.get_storage_key(), expires_in) def delete_import_zip(self, path: BundleImportZipPath) -> None: @@ -420,31 +262,3 @@ class AppAssetStorage: import logging logging.getLogger(__name__).debug("Failed to delete import zip: %s", path.get_storage_key()) - - def _generate_signed_proxy_download_url(self, asset_path: SignedAssetPath, expires_in: int) -> str: - expires_in = min(expires_in, dify_config.FILES_ACCESS_TIMEOUT) - expires_at = int(time.time()) + max(expires_in, 1) - nonce = os.urandom(16).hex() - sign = AppAssetSigner.create_download_signature(asset_path=asset_path, expires_at=expires_at, nonce=nonce) - - base_url = dify_config.FILES_URL - url = self._build_proxy_url(base_url=base_url, asset_path=asset_path, action="download") - query = urllib.parse.urlencode({"expires_at": expires_at, "nonce": nonce, "sign": sign}) - return f"{url}?{query}" - - def _generate_signed_proxy_upload_url(self, asset_path: SignedAssetPath, expires_in: int) -> str: - expires_in = min(expires_in, dify_config.FILES_ACCESS_TIMEOUT) - expires_at = int(time.time()) + max(expires_in, 1) - nonce = os.urandom(16).hex() - sign = AppAssetSigner.create_upload_signature(asset_path=asset_path, expires_at=expires_at, nonce=nonce) - - base_url = dify_config.FILES_URL - url = self._build_proxy_url(base_url=base_url, asset_path=asset_path, action="upload") - query = urllib.parse.urlencode({"expires_at": expires_at, "nonce": nonce, "sign": sign}) - return f"{url}?{query}" - - @staticmethod - def _build_proxy_url(*, base_url: str, asset_path: SignedAssetPath, action: str) -> str: - encoded_parts = [urllib.parse.quote(part, safe="") for part in asset_path.proxy_path_parts()] - path = "/".join(encoded_parts) - return f"{base_url}/files/app-assets/{path}/{action}" diff --git a/api/core/sandbox/inspector/archive_source.py b/api/core/sandbox/inspector/archive_source.py index 8876d878cf..a9b0b360b4 100644 --- a/api/core/sandbox/inspector/archive_source.py +++ b/api/core/sandbox/inspector/archive_source.py @@ -7,9 +7,9 @@ from uuid import UUID, uuid4 from core.sandbox.entities.files import SandboxFileDownloadTicket, SandboxFileNode from core.sandbox.inspector.base import SandboxFileSource -from core.sandbox.security.archive_signer import SandboxArchivePath, SandboxArchiveSigner -from core.sandbox.security.sandbox_file_signer import SandboxFileDownloadPath from core.sandbox.storage import sandbox_file_storage +from core.sandbox.storage.archive_storage import SandboxArchivePath +from core.sandbox.storage.sandbox_file_storage import SandboxFileDownloadPath from core.virtual_environment.__base.exec import CommandExecutionError from core.virtual_environment.__base.helpers import execute from extensions.ext_storage import storage @@ -68,15 +68,14 @@ print(json.dumps(entries)) def _get_archive_download_url(self) -> str: """Get a pre-signed download URL for the sandbox archive.""" + from extensions.storage.file_presign_storage import FilePresignStorage + archive_path = SandboxArchivePath(tenant_id=UUID(self._tenant_id), sandbox_id=UUID(self._sandbox_id)) storage_key = archive_path.get_storage_key() if not storage.exists(storage_key): raise ValueError("Sandbox archive not found") - return SandboxArchiveSigner.build_signed_url( - archive_path=archive_path, - expires_in=self._EXPORT_EXPIRES_IN_SECONDS, - action=SandboxArchiveSigner.OPERATION_DOWNLOAD, - ) + presign_storage = FilePresignStorage(storage.storage_runner) + return presign_storage.get_download_url(storage_key, self._EXPORT_EXPIRES_IN_SECONDS) def _create_zip_sandbox(self) -> ZipSandbox: """Create a ZipSandbox instance for archive operations.""" diff --git a/api/core/sandbox/inspector/runtime_source.py b/api/core/sandbox/inspector/runtime_source.py index 7481169212..052092e00d 100644 --- a/api/core/sandbox/inspector/runtime_source.py +++ b/api/core/sandbox/inspector/runtime_source.py @@ -7,8 +7,8 @@ from uuid import UUID, uuid4 from core.sandbox.entities.files import SandboxFileDownloadTicket, SandboxFileNode from core.sandbox.inspector.base import SandboxFileSource -from core.sandbox.security.sandbox_file_signer import SandboxFileDownloadPath from core.sandbox.storage import sandbox_file_storage +from core.sandbox.storage.sandbox_file_storage import SandboxFileDownloadPath from core.virtual_environment.__base.exec import CommandExecutionError from core.virtual_environment.__base.helpers import execute from core.virtual_environment.__base.virtual_environment import VirtualEnvironment diff --git a/api/core/sandbox/security/__init__.py b/api/core/sandbox/security/__init__.py deleted file mode 100644 index c8c4ebefae..0000000000 --- a/api/core/sandbox/security/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Sandbox security helpers.""" diff --git a/api/core/sandbox/security/archive_signer.py b/api/core/sandbox/security/archive_signer.py deleted file mode 100644 index 5fd48b73db..0000000000 --- a/api/core/sandbox/security/archive_signer.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import annotations - -import base64 -import hashlib -import hmac -import os -import time -import urllib.parse -from dataclasses import dataclass -from uuid import UUID - -from configs import dify_config -from libs import rsa - - -@dataclass(frozen=True) -class SandboxArchivePath: - tenant_id: UUID - sandbox_id: UUID - - def get_storage_key(self) -> str: - return f"sandbox/{self.tenant_id}/{self.sandbox_id}.tar.gz" - - def proxy_path(self) -> str: - return f"{self.tenant_id}/{self.sandbox_id}" - - -class SandboxArchiveSigner: - SIGNATURE_PREFIX = "sandbox-archive" - SIGNATURE_VERSION = "v1" - OPERATION_DOWNLOAD = "download" - OPERATION_UPLOAD = "upload" - - @classmethod - def create_download_signature(cls, archive_path: SandboxArchivePath, expires_at: int, nonce: str) -> str: - return cls._create_signature( - archive_path=archive_path, - operation=cls.OPERATION_DOWNLOAD, - expires_at=expires_at, - nonce=nonce, - ) - - @classmethod - def create_upload_signature(cls, archive_path: SandboxArchivePath, expires_at: int, nonce: str) -> str: - return cls._create_signature( - archive_path=archive_path, - operation=cls.OPERATION_UPLOAD, - expires_at=expires_at, - nonce=nonce, - ) - - @classmethod - def verify_download_signature( - cls, archive_path: SandboxArchivePath, expires_at: int, nonce: str, sign: str - ) -> bool: - return cls._verify_signature( - archive_path=archive_path, - operation=cls.OPERATION_DOWNLOAD, - expires_at=expires_at, - nonce=nonce, - sign=sign, - ) - - @classmethod - def verify_upload_signature(cls, archive_path: SandboxArchivePath, expires_at: int, nonce: str, sign: str) -> bool: - return cls._verify_signature( - archive_path=archive_path, - operation=cls.OPERATION_UPLOAD, - expires_at=expires_at, - nonce=nonce, - sign=sign, - ) - - @classmethod - def _verify_signature( - cls, - *, - archive_path: SandboxArchivePath, - operation: str, - expires_at: int, - nonce: str, - sign: str, - ) -> bool: - if expires_at <= 0: - return False - - expected_sign = cls._create_signature( - archive_path=archive_path, - operation=operation, - expires_at=expires_at, - nonce=nonce, - ) - if not hmac.compare_digest(sign, expected_sign): - return False - - current_time = int(time.time()) - if expires_at < current_time: - return False - - if expires_at - current_time > dify_config.FILES_ACCESS_TIMEOUT: - return False - - return True - - @classmethod - def build_signed_url( - cls, - *, - archive_path: SandboxArchivePath, - expires_in: int, - action: str, - ) -> str: - expires_in = min(expires_in, dify_config.FILES_ACCESS_TIMEOUT) - expires_at = int(time.time()) + max(expires_in, 1) - nonce = os.urandom(16).hex() - sign = cls._create_signature( - archive_path=archive_path, - operation=action, - expires_at=expires_at, - nonce=nonce, - ) - - base_url = dify_config.FILES_URL - url = f"{base_url}/files/sandbox-archives/{archive_path.proxy_path()}/{action}" - query = urllib.parse.urlencode({"expires_at": expires_at, "nonce": nonce, "sign": sign}) - return f"{url}?{query}" - - @classmethod - def _create_signature( - cls, - *, - archive_path: SandboxArchivePath, - operation: str, - expires_at: int, - nonce: str, - ) -> str: - key = cls._tenant_key(str(archive_path.tenant_id)) - message = ( - f"{cls.SIGNATURE_PREFIX}|{cls.SIGNATURE_VERSION}|{operation}|" - f"{archive_path.tenant_id}|{archive_path.sandbox_id}|{expires_at}|{nonce}" - ) - sign = hmac.new(key, message.encode(), hashlib.sha256).digest() - return base64.urlsafe_b64encode(sign).decode() - - @classmethod - def _tenant_key(cls, tenant_id: str) -> bytes: - try: - rsa_key, _ = rsa.get_decrypt_decoding(tenant_id) - except rsa.PrivkeyNotFoundError as exc: - raise ValueError(f"Tenant private key missing for tenant_id={tenant_id}") from exc - private_key = rsa_key.export_key() - return hashlib.sha256(private_key).digest() diff --git a/api/core/sandbox/security/sandbox_file_signer.py b/api/core/sandbox/security/sandbox_file_signer.py deleted file mode 100644 index dd59023ba9..0000000000 --- a/api/core/sandbox/security/sandbox_file_signer.py +++ /dev/null @@ -1,155 +0,0 @@ -from __future__ import annotations - -import base64 -import hashlib -import hmac -import os -import time -import urllib.parse -from dataclasses import dataclass -from uuid import UUID - -from configs import dify_config -from libs import rsa - - -@dataclass(frozen=True) -class SandboxFileDownloadPath: - tenant_id: UUID - sandbox_id: UUID - export_id: str - filename: str - - def get_storage_key(self) -> str: - return f"sandbox_file_downloads/{self.tenant_id}/{self.sandbox_id}/{self.export_id}/{self.filename}" - - def proxy_path(self) -> str: - encoded_parts = [ - urllib.parse.quote(str(self.tenant_id), safe=""), - urllib.parse.quote(str(self.sandbox_id), safe=""), - urllib.parse.quote(self.export_id, safe=""), - urllib.parse.quote(self.filename, safe=""), - ] - return "/".join(encoded_parts) - - -class SandboxFileSigner: - SIGNATURE_PREFIX = "sandbox-file-download" - SIGNATURE_VERSION = "v1" - OPERATION_DOWNLOAD = "download" - OPERATION_UPLOAD = "upload" - - @classmethod - def build_signed_url( - cls, - *, - export_path: SandboxFileDownloadPath, - expires_in: int, - action: str, - ) -> str: - expires_in = min(expires_in, dify_config.FILES_ACCESS_TIMEOUT) - expires_at = int(time.time()) + max(expires_in, 1) - nonce = os.urandom(16).hex() - sign = cls._create_signature( - export_path=export_path, - operation=action, - expires_at=expires_at, - nonce=nonce, - ) - - base_url = dify_config.FILES_URL - url = f"{base_url}/files/sandbox-file-downloads/{export_path.proxy_path()}/{action}" - query = urllib.parse.urlencode({"expires_at": expires_at, "nonce": nonce, "sign": sign}) - return f"{url}?{query}" - - @classmethod - def verify_download_signature( - cls, - *, - export_path: SandboxFileDownloadPath, - expires_at: int, - nonce: str, - sign: str, - ) -> bool: - return cls._verify_signature( - export_path=export_path, - operation=cls.OPERATION_DOWNLOAD, - expires_at=expires_at, - nonce=nonce, - sign=sign, - ) - - @classmethod - def verify_upload_signature( - cls, - *, - export_path: SandboxFileDownloadPath, - expires_at: int, - nonce: str, - sign: str, - ) -> bool: - return cls._verify_signature( - export_path=export_path, - operation=cls.OPERATION_UPLOAD, - expires_at=expires_at, - nonce=nonce, - sign=sign, - ) - - @classmethod - def _verify_signature( - cls, - *, - export_path: SandboxFileDownloadPath, - operation: str, - expires_at: int, - nonce: str, - sign: str, - ) -> bool: - if expires_at <= 0: - return False - - expected_sign = cls._create_signature( - export_path=export_path, - operation=operation, - expires_at=expires_at, - nonce=nonce, - ) - if not hmac.compare_digest(sign, expected_sign): - return False - - current_time = int(time.time()) - if expires_at < current_time: - return False - - if expires_at - current_time > dify_config.FILES_ACCESS_TIMEOUT: - return False - - return True - - @classmethod - def _create_signature( - cls, - *, - export_path: SandboxFileDownloadPath, - operation: str, - expires_at: int, - nonce: str, - ) -> str: - key = cls._tenant_key(str(export_path.tenant_id)) - message = ( - f"{cls.SIGNATURE_PREFIX}|{cls.SIGNATURE_VERSION}|{operation}|" - f"{export_path.tenant_id}|{export_path.sandbox_id}|{export_path.export_id}|{export_path.filename}|" - f"{expires_at}|{nonce}" - ) - digest = hmac.new(key, message.encode(), hashlib.sha256).digest() - return base64.urlsafe_b64encode(digest).decode() - - @classmethod - def _tenant_key(cls, tenant_id: str) -> bytes: - try: - rsa_key, _ = rsa.get_decrypt_decoding(tenant_id) - except rsa.PrivkeyNotFoundError as exc: - raise ValueError(f"Tenant private key missing for tenant_id={tenant_id}") from exc - private_key = rsa_key.export_key() - return hashlib.sha256(private_key).digest() diff --git a/api/core/sandbox/storage/__init__.py b/api/core/sandbox/storage/__init__.py index c7c405204e..b0eeb8940a 100644 --- a/api/core/sandbox/storage/__init__.py +++ b/api/core/sandbox/storage/__init__.py @@ -1,11 +1,13 @@ -from .archive_storage import ArchiveSandboxStorage +from .archive_storage import ArchiveSandboxStorage, SandboxArchivePath from .noop_storage import NoopSandboxStorage -from .sandbox_file_storage import SandboxFileStorage, sandbox_file_storage +from .sandbox_file_storage import SandboxFileDownloadPath, SandboxFileStorage, sandbox_file_storage from .sandbox_storage import SandboxStorage __all__ = [ "ArchiveSandboxStorage", "NoopSandboxStorage", + "SandboxArchivePath", + "SandboxFileDownloadPath", "SandboxFileStorage", "SandboxStorage", "sandbox_file_storage", diff --git a/api/core/sandbox/storage/archive_storage.py b/api/core/sandbox/storage/archive_storage.py index 3b44da5fc9..b67b1b8825 100644 --- a/api/core/sandbox/storage/archive_storage.py +++ b/api/core/sandbox/storage/archive_storage.py @@ -1,18 +1,31 @@ +"""Archive-based sandbox storage for persisting sandbox state. + +This module provides storage operations for sandbox workspace archives (tar.gz), +enabling state persistence across sandbox sessions. + +Storage key format: sandbox/{tenant_id}/{sandbox_id}.tar.gz + +All presign operations use the unified FilePresignStorage wrapper, which automatically +falls back to Dify's file proxy when the underlying storage doesn't support presigned URLs. +""" + +from __future__ import annotations + import logging +from dataclasses import dataclass from uuid import UUID -from core.sandbox.security.archive_signer import SandboxArchivePath, SandboxArchiveSigner from core.virtual_environment.__base.exec import PipelineExecutionError from core.virtual_environment.__base.helpers import pipeline from core.virtual_environment.__base.virtual_environment import VirtualEnvironment -from extensions.ext_storage import storage +from extensions.storage.base_storage import BaseStorage +from extensions.storage.file_presign_storage import FilePresignStorage from .sandbox_storage import SandboxStorage logger = logging.getLogger(__name__) WORKSPACE_DIR = "." - ARCHIVE_DOWNLOAD_TIMEOUT = 60 * 5 ARCHIVE_UPLOAD_TIMEOUT = 60 * 5 @@ -21,40 +34,67 @@ def build_tar_exclude_args(patterns: list[str]) -> list[str]: return [f"--exclude={p}" for p in patterns] +@dataclass(frozen=True) +class SandboxArchivePath: + """Path for sandbox workspace archives.""" + + tenant_id: UUID + sandbox_id: UUID + + def get_storage_key(self) -> str: + return f"sandbox/{self.tenant_id}/{self.sandbox_id}.tar.gz" + + class ArchiveSandboxStorage(SandboxStorage): + """Archive-based storage for sandbox workspace persistence. + + Uses tar.gz archives to save and restore sandbox workspace state. + Requires a presign-capable storage wrapper for generating download/upload URLs. + """ + _tenant_id: str _sandbox_id: str _exclude_patterns: list[str] + _storage: FilePresignStorage - def __init__(self, tenant_id: str, sandbox_id: str, exclude_patterns: list[str] | None = None): + def __init__( + self, + tenant_id: str, + sandbox_id: str, + storage: BaseStorage, + exclude_patterns: list[str] | None = None, + ): self._tenant_id = tenant_id self._sandbox_id = sandbox_id self._exclude_patterns = exclude_patterns or [] + # Wrap with FilePresignStorage for presign fallback support + self._storage = FilePresignStorage(storage) + + @property + def _archive_path(self) -> SandboxArchivePath: + return SandboxArchivePath(UUID(self._tenant_id), UUID(self._sandbox_id)) @property def _storage_key(self) -> str: - return SandboxArchivePath(UUID(self._tenant_id), UUID(self._sandbox_id)).get_storage_key() + return self._archive_path.get_storage_key() @property def _archive_name(self) -> str: return f"{self._sandbox_id}.tar.gz" @property - def _archive_path(self) -> str: + def _archive_tmp_path(self) -> str: return f"/tmp/{self._archive_name}" def mount(self, sandbox: VirtualEnvironment) -> bool: + """Load archive from storage into sandbox workspace.""" if not self.exists(): logger.debug("No archive found for sandbox %s, skipping mount", self._sandbox_id) return False - archive_path = SandboxArchivePath(UUID(self._tenant_id), UUID(self._sandbox_id)) - download_url = SandboxArchiveSigner.build_signed_url( - archive_path=archive_path, - expires_in=ARCHIVE_DOWNLOAD_TIMEOUT, - action=SandboxArchiveSigner.OPERATION_DOWNLOAD, - ) + download_url = self._storage.get_download_url(self._storage_key, ARCHIVE_DOWNLOAD_TIMEOUT) archive_name = self._archive_name + try: ( pipeline(sandbox) @@ -74,13 +114,10 @@ class ArchiveSandboxStorage(SandboxStorage): return True def unmount(self, sandbox: VirtualEnvironment) -> bool: - archive_path = SandboxArchivePath(UUID(self._tenant_id), UUID(self._sandbox_id)) - upload_url = SandboxArchiveSigner.build_signed_url( - archive_path=archive_path, - expires_in=ARCHIVE_UPLOAD_TIMEOUT, - action=SandboxArchiveSigner.OPERATION_UPLOAD, - ) - archive_path = self._archive_path + """Save sandbox workspace to storage as archive.""" + upload_url = self._storage.get_upload_url(self._storage_key, ARCHIVE_UPLOAD_TIMEOUT) + archive_path = self._archive_tmp_path + ( pipeline(sandbox) .add( @@ -105,11 +142,13 @@ class ArchiveSandboxStorage(SandboxStorage): return True def exists(self) -> bool: - return storage.exists(self._storage_key) + """Check if archive exists in storage.""" + return self._storage.exists(self._storage_key) def delete(self) -> None: + """Delete archive from storage.""" try: - storage.delete(self._storage_key) + self._storage.delete(self._storage_key) logger.info("Deleted archive for sandbox %s", self._sandbox_id) except Exception: logger.exception("Failed to delete archive for sandbox %s", self._sandbox_id) diff --git a/api/core/sandbox/storage/sandbox_file_storage.py b/api/core/sandbox/storage/sandbox_file_storage.py index da7c17b402..2d0b5482fc 100644 --- a/api/core/sandbox/storage/sandbox_file_storage.py +++ b/api/core/sandbox/storage/sandbox_file_storage.py @@ -1,23 +1,58 @@ +"""Sandbox file storage for exporting files from sandbox environments. + +This module provides storage operations for files exported from sandbox environments, +including download tickets for both runtime and archive-based file sources. + +Storage key format: sandbox_file_downloads/{tenant_id}/{sandbox_id}/{export_id}/{filename} + +All presign operations use the unified FilePresignStorage wrapper, which automatically +falls back to Dify's file proxy when the underlying storage doesn't support presigned URLs. +""" + from __future__ import annotations +from dataclasses import dataclass from typing import Any +from uuid import UUID -from core.sandbox.security.sandbox_file_signer import SandboxFileDownloadPath, SandboxFileSigner -from extensions.ext_redis import redis_client -from extensions.ext_storage import storage from extensions.storage.base_storage import BaseStorage from extensions.storage.cached_presign_storage import CachedPresignStorage -from extensions.storage.silent_storage import SilentStorage +from extensions.storage.file_presign_storage import FilePresignStorage + + +@dataclass(frozen=True) +class SandboxFileDownloadPath: + """Path for sandbox file exports.""" + + tenant_id: UUID + sandbox_id: UUID + export_id: str + filename: str + + def get_storage_key(self) -> str: + return f"sandbox_file_downloads/{self.tenant_id}/{self.sandbox_id}/{self.export_id}/{self.filename}" class SandboxFileStorage: - _base_storage: BaseStorage + """Storage operations for sandbox file exports. + + Wraps BaseStorage with: + - FilePresignStorage for presign fallback support + - CachedPresignStorage for URL caching + + Usage: + storage = SandboxFileStorage(base_storage, redis_client=redis) + storage.save(download_path, content) + url = storage.get_download_url(download_path) + """ + _storage: CachedPresignStorage def __init__(self, storage: BaseStorage, *, redis_client: Any) -> None: - self._base_storage = storage + # Wrap with FilePresignStorage for fallback support, then CachedPresignStorage for caching + presign_storage = FilePresignStorage(storage) self._storage = CachedPresignStorage( - storage=storage, + storage=presign_storage, redis_client=redis_client, cache_key_prefix="sandbox_file_downloads", ) @@ -26,29 +61,19 @@ class SandboxFileStorage: self._storage.save(download_path.get_storage_key(), content) def get_download_url(self, download_path: SandboxFileDownloadPath, expires_in: int = 3600) -> str: - storage_key = download_path.get_storage_key() - try: - return self._storage.get_download_url(storage_key, expires_in) - except NotImplementedError: - return SandboxFileSigner.build_signed_url( - export_path=download_path, - expires_in=expires_in, - action=SandboxFileSigner.OPERATION_DOWNLOAD, - ) + return self._storage.get_download_url(download_path.get_storage_key(), expires_in) def get_upload_url(self, download_path: SandboxFileDownloadPath, expires_in: int = 3600) -> str: - storage_key = download_path.get_storage_key() - try: - return self._storage.get_upload_url(storage_key, expires_in) - except NotImplementedError: - return SandboxFileSigner.build_signed_url( - export_path=download_path, - expires_in=expires_in, - action=SandboxFileSigner.OPERATION_UPLOAD, - ) + return self._storage.get_upload_url(download_path.get_storage_key(), expires_in) class _LazySandboxFileStorage: + """Lazy initializer for singleton SandboxFileStorage. + + Delays storage initialization until first access, ensuring Flask app + context is available. + """ + _instance: SandboxFileStorage | None def __init__(self) -> None: @@ -56,12 +81,16 @@ class _LazySandboxFileStorage: def _get_instance(self) -> SandboxFileStorage: if self._instance is None: + from extensions.ext_redis import redis_client + from extensions.ext_storage import storage + if not hasattr(storage, "storage_runner"): raise RuntimeError( "Storage is not initialized; call storage.init_app before using sandbox_file_storage" ) self._instance = SandboxFileStorage( - storage=SilentStorage(storage.storage_runner), redis_client=redis_client + storage=storage.storage_runner, + redis_client=redis_client, ) return self._instance @@ -69,4 +98,4 @@ class _LazySandboxFileStorage: return getattr(self._get_instance(), name) -sandbox_file_storage = _LazySandboxFileStorage() +sandbox_file_storage: SandboxFileStorage = _LazySandboxFileStorage() # type: ignore[assignment] diff --git a/api/extensions/storage/file_presign_storage.py b/api/extensions/storage/file_presign_storage.py index ae7dee3033..c42354db98 100644 --- a/api/extensions/storage/file_presign_storage.py +++ b/api/extensions/storage/file_presign_storage.py @@ -1,4 +1,23 @@ -"""Storage wrapper that provides presigned URL support with fallback to signed proxy URLs.""" +"""Storage wrapper that provides presigned URL support with fallback to signed proxy URLs. + +This is the unified presign wrapper for all storage operations. When the underlying +storage backend doesn't support presigned URLs (raises NotImplementedError), it falls +back to generating signed proxy URLs that route through Dify's file proxy endpoints. + +Usage: + from extensions.storage.file_presign_storage import FilePresignStorage + + # Wrap any BaseStorage to add presign support + presign_storage = FilePresignStorage(base_storage) + download_url = presign_storage.get_download_url("path/to/file.txt", expires_in=3600) + upload_url = presign_storage.get_upload_url("path/to/file.txt", expires_in=3600) + +The proxy URLs follow the format: + {FILES_URL}/files/storage/{encoded_filename}/(download|upload)?timestamp=...&nonce=...&sign=... + +Signature format: + HMAC-SHA256(SECRET_KEY, "storage-file|{operation}|{filename}|{timestamp}|{nonce}") +""" import base64 import hashlib @@ -12,59 +31,81 @@ from extensions.storage.storage_wrapper import StorageWrapper class FilePresignStorage(StorageWrapper): - """Storage wrapper that provides presigned URL support. + """Storage wrapper that provides presigned URL support with proxy fallback. If the wrapped storage supports presigned URLs, delegates to it. - Otherwise, generates signed proxy URLs for download. + Otherwise, generates signed proxy URLs for both download and upload operations. """ - SIGNATURE_PREFIX = "storage-download" + SIGNATURE_PREFIX = "storage-file" def get_download_url(self, filename: str, expires_in: int = 3600) -> str: + """Get a presigned download URL, falling back to proxy URL if not supported.""" try: - return super().get_download_url(filename, expires_in) + return self._storage.get_download_url(filename, expires_in) except NotImplementedError: - return self._generate_signed_proxy_url(filename, expires_in) - - def get_upload_url(self, filename: str, expires_in: int = 3600) -> str: - try: - return super().get_upload_url(filename, expires_in) - except NotImplementedError: - return self._generate_signed_upload_url(filename) + return self._generate_signed_proxy_url(filename, "download", expires_in) def get_download_urls(self, filenames: list[str], expires_in: int = 3600) -> list[str]: + """Get presigned download URLs for multiple files.""" try: - return super().get_download_urls(filenames, expires_in) + return self._storage.get_download_urls(filenames, expires_in) except NotImplementedError: - return [self._generate_signed_proxy_url(filename, expires_in) for filename in filenames] + return [self._generate_signed_proxy_url(f, "download", expires_in) for f in filenames] - def _generate_signed_upload_url(self, filename: str) -> str: - # TODO: Implement this - raise NotImplementedError("This storage backend doesn't support pre-signed URLs") + def get_upload_url(self, filename: str, expires_in: int = 3600) -> str: + """Get a presigned upload URL, falling back to proxy URL if not supported.""" + try: + return self._storage.get_upload_url(filename, expires_in) + except NotImplementedError: + return self._generate_signed_proxy_url(filename, "upload", expires_in) - def _generate_signed_proxy_url(self, filename: str, expires_in: int = 3600) -> str: + def _generate_signed_proxy_url(self, filename: str, operation: str, expires_in: int = 3600) -> str: + """Generate a signed proxy URL for file operations. + + Args: + filename: The storage key/path + operation: Either "download" or "upload" + expires_in: URL validity duration in seconds + + Returns: + Signed proxy URL string + """ base_url = dify_config.FILES_URL encoded_filename = urllib.parse.quote(filename, safe="") - url = f"{base_url}/files/storage/{encoded_filename}/download" + url = f"{base_url}/files/storage/{encoded_filename}/{operation}" timestamp = str(int(time.time())) nonce = os.urandom(16).hex() - sign = self._create_signature(filename, timestamp, nonce) + sign = self._create_signature(operation, filename, timestamp, nonce) query = urllib.parse.urlencode({"timestamp": timestamp, "nonce": nonce, "sign": sign}) return f"{url}?{query}" @classmethod - def _create_signature(cls, filename: str, timestamp: str, nonce: str) -> str: + def _create_signature(cls, operation: str, filename: str, timestamp: str, nonce: str) -> str: + """Create HMAC signature for the proxy URL.""" key = dify_config.SECRET_KEY.encode() - msg = f"{cls.SIGNATURE_PREFIX}|{filename}|{timestamp}|{nonce}" + msg = f"{cls.SIGNATURE_PREFIX}|{operation}|{filename}|{timestamp}|{nonce}" sign = hmac.new(key, msg.encode(), hashlib.sha256).digest() return base64.urlsafe_b64encode(sign).decode() @classmethod - def verify_signature(cls, *, filename: str, timestamp: str, nonce: str, sign: str) -> bool: - expected_sign = cls._create_signature(filename, timestamp, nonce) - if sign != expected_sign: + def verify_signature(cls, *, operation: str, filename: str, timestamp: str, nonce: str, sign: str) -> bool: + """Verify the signature of a proxy URL. + + Args: + operation: The operation type ("download" or "upload") + filename: The storage key/path + timestamp: Unix timestamp string from the URL + nonce: Random nonce string from the URL + sign: Signature string from the URL + + Returns: + True if signature is valid and not expired, False otherwise + """ + expected_sign = cls._create_signature(operation, filename, timestamp, nonce) + if not hmac.compare_digest(sign, expected_sign): return False current_time = int(time.time()) diff --git a/api/services/app_asset_service.py b/api/services/app_asset_service.py index 8382d88e85..3efa985620 100644 --- a/api/services/app_asset_service.py +++ b/api/services/app_asset_service.py @@ -18,7 +18,6 @@ from core.app_assets.storage import AppAssetStorage, AssetPath from extensions.ext_database import db from extensions.ext_redis import redis_client from extensions.ext_storage import storage -from extensions.storage.silent_storage import SilentStorage from models.app_asset import AppAssets from models.model import App @@ -43,9 +42,12 @@ class AppAssetService: This method creates an AppAssetStorage each time it's called, ensuring storage.storage_runner is only accessed after init_app. + + The storage is wrapped with FilePresignStorage for presign fallback support + and CachedPresignStorage for URL caching. """ return AppAssetStorage( - storage=SilentStorage(storage.storage_runner), + storage=storage.storage_runner, redis_client=redis_client, cache_key_prefix="app_assets", ) diff --git a/api/services/sandbox/sandbox_service.py b/api/services/sandbox/sandbox_service.py index f462a239b2..6229f0ba14 100644 --- a/api/services/sandbox/sandbox_service.py +++ b/api/services/sandbox/sandbox_service.py @@ -10,6 +10,7 @@ from core.sandbox.initializer.draft_app_assets_initializer import DraftAppAssets from core.sandbox.initializer.skill_initializer import SkillInitializer from core.sandbox.sandbox import Sandbox from core.sandbox.storage.archive_storage import ArchiveSandboxStorage +from extensions.ext_storage import storage from services.app_asset_package_service import AppAssetPackageService from services.app_asset_service import AppAssetService @@ -30,7 +31,7 @@ class SandboxService: if not assets: raise ValueError(f"No assets found for tid={tenant_id}, app_id={app_id}") - storage = ArchiveSandboxStorage(tenant_id, workflow_execution_id) + archive_storage = ArchiveSandboxStorage(tenant_id, workflow_execution_id, storage.storage_runner) sandbox = ( SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type)) .options(sandbox_provider.config) @@ -40,7 +41,7 @@ class SandboxService: .initializer(AppAssetsInitializer(tenant_id, app_id, assets.id)) .initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id)) .initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id)) - .storage(storage, assets.id) + .storage(archive_storage, assets.id) .build() ) @@ -49,8 +50,8 @@ class SandboxService: @classmethod def delete_draft_storage(cls, tenant_id: str, user_id: str) -> None: - storage = ArchiveSandboxStorage(tenant_id, SandboxBuilder.draft_id(user_id)) - storage.delete() + archive_storage = ArchiveSandboxStorage(tenant_id, SandboxBuilder.draft_id(user_id), storage.storage_runner) + archive_storage.delete() @classmethod def create_draft( @@ -66,7 +67,9 @@ class SandboxService: AppAssetPackageService.build_assets(tenant_id, app_id, assets) sandbox_id = SandboxBuilder.draft_id(user_id) - storage = ArchiveSandboxStorage(tenant_id, sandbox_id, exclude_patterns=[AppAssets.PATH]) + archive_storage = ArchiveSandboxStorage( + tenant_id, sandbox_id, storage.storage_runner, exclude_patterns=[AppAssets.PATH] + ) sandbox = ( SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type)) @@ -77,7 +80,7 @@ class SandboxService: .initializer(DraftAppAssetsInitializer(tenant_id, app_id, assets.id)) .initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id)) .initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id)) - .storage(storage, assets.id) + .storage(archive_storage, assets.id) .build() ) @@ -98,7 +101,9 @@ class SandboxService: AppAssetPackageService.build_assets(tenant_id, app_id, assets) sandbox_id = SandboxBuilder.draft_id(user_id) - storage = ArchiveSandboxStorage(tenant_id, sandbox_id, exclude_patterns=[AppAssets.PATH]) + archive_storage = ArchiveSandboxStorage( + tenant_id, sandbox_id, storage.storage_runner, exclude_patterns=[AppAssets.PATH] + ) sandbox = ( SandboxBuilder(tenant_id, SandboxType(sandbox_provider.provider_type)) @@ -109,7 +114,7 @@ class SandboxService: .initializer(DraftAppAssetsInitializer(tenant_id, app_id, assets.id)) .initializer(DifyCliInitializer(tenant_id, user_id, app_id, assets.id)) .initializer(SkillInitializer(tenant_id, user_id, app_id, assets.id)) - .storage(storage, assets.id) + .storage(archive_storage, assets.id) .build() ) diff --git a/api/tests/unit_tests/core/app_assets/test_storage.py b/api/tests/unit_tests/core/app_assets/test_storage.py index 05a26c7415..f5883656a3 100644 --- a/api/tests/unit_tests/core/app_assets/test_storage.py +++ b/api/tests/unit_tests/core/app_assets/test_storage.py @@ -4,9 +4,9 @@ from uuid import uuid4 import pytest from configs import dify_config -from core.app_assets.storage import AppAssetSigner, AppAssetStorage, AssetPath +from core.app_assets.storage import AppAssetStorage, AssetPath from extensions.storage.base_storage import BaseStorage -from libs import rsa +from extensions.storage.file_presign_storage import FilePresignStorage class DummyStorage(BaseStorage): @@ -70,83 +70,96 @@ def test_asset_path_validation(): AssetPath.draft(tenant_id=tenant_id, app_id=app_id, node_id="not-a-uuid") -def test_storage_key_mapping(): - tenant_id = str(uuid4()) - app_id = str(uuid4()) - node_id = str(uuid4()) - - storage = AppAssetStorage(DummyStorage(), redis_client=DummyRedis()) - ref = AssetPath.draft(tenant_id, app_id, node_id) - assert storage.get_storage_key(ref) == ref.get_storage_key() - - -def test_signature_verification(monkeypatch: pytest.MonkeyPatch): - tenant_id = str(uuid4()) - app_id = str(uuid4()) - resource_id = str(uuid4()) - asset_path = AssetPath.draft(tenant_id, app_id, resource_id) - - class _FakeKey: - def export_key(self) -> bytes: - return b"tenant-private-key" - - def _fake_get_decrypt_decoding(_tenant_id: str) -> tuple[_FakeKey, None]: - return _FakeKey(), None - +def test_file_presign_signature_verification(monkeypatch: pytest.MonkeyPatch): + """Test FilePresignStorage signature creation and verification.""" + monkeypatch.setattr(dify_config, "SECRET_KEY", "test-secret-key", raising=False) monkeypatch.setattr(dify_config, "FILES_ACCESS_TIMEOUT", 300, raising=False) - monkeypatch.setattr(rsa, "get_decrypt_decoding", _fake_get_decrypt_decoding) - expires_at = int(time.time()) + 120 - nonce = "nonce" - sign = AppAssetSigner.create_download_signature(asset_path=asset_path, expires_at=expires_at, nonce=nonce) + filename = "test/path/file.txt" + timestamp = str(int(time.time())) + nonce = "test-nonce" - assert AppAssetSigner.verify_download_signature( - asset_path=asset_path, - expires_at=expires_at, + # Test download signature + sign = FilePresignStorage._create_signature("download", filename, timestamp, nonce) + assert FilePresignStorage.verify_signature( + filename=filename, + operation="download", + timestamp=timestamp, nonce=nonce, sign=sign, ) - expired_at = int(time.time()) - 1 - expired_sign = AppAssetSigner.create_download_signature(asset_path=asset_path, expires_at=expired_at, nonce=nonce) - assert not AppAssetSigner.verify_download_signature( - asset_path=asset_path, - expires_at=expired_at, + # Test upload signature + upload_sign = FilePresignStorage._create_signature("upload", filename, timestamp, nonce) + assert FilePresignStorage.verify_signature( + filename=filename, + operation="upload", + timestamp=timestamp, + nonce=nonce, + sign=upload_sign, + ) + + # Test expired signature + expired_timestamp = str(int(time.time()) - 400) + expired_sign = FilePresignStorage._create_signature("download", filename, expired_timestamp, nonce) + assert not FilePresignStorage.verify_signature( + filename=filename, + operation="download", + timestamp=expired_timestamp, nonce=nonce, sign=expired_sign, ) - too_far = int(time.time()) + 3600 - far_sign = AppAssetSigner.create_download_signature(asset_path=asset_path, expires_at=too_far, nonce=nonce) - assert not AppAssetSigner.verify_download_signature( - asset_path=asset_path, - expires_at=too_far, + # Test wrong signature + assert not FilePresignStorage.verify_signature( + filename=filename, + operation="download", + timestamp=timestamp, nonce=nonce, - sign=far_sign, + sign="wrong-signature", ) def test_signed_proxy_url_generation(monkeypatch: pytest.MonkeyPatch): + """Test that AppAssetStorage generates correct proxy URLs when presign is not supported.""" tenant_id = str(uuid4()) app_id = str(uuid4()) resource_id = str(uuid4()) asset_path = AssetPath.draft(tenant_id, app_id, resource_id) - class _FakeKey: - def export_key(self) -> bytes: - return b"tenant-private-key" - - def _fake_get_decrypt_decoding(_tenant_id: str) -> tuple[_FakeKey, None]: - return _FakeKey(), None - + monkeypatch.setattr(dify_config, "SECRET_KEY", "test-secret-key", raising=False) monkeypatch.setattr(dify_config, "FILES_ACCESS_TIMEOUT", 300, raising=False) - monkeypatch.setattr(rsa, "get_decrypt_decoding", _fake_get_decrypt_decoding) monkeypatch.setattr(dify_config, "FILES_URL", "http://files.local", raising=False) storage = AppAssetStorage(DummyStorage(), redis_client=DummyRedis()) url = storage.get_download_url(asset_path, expires_in=120) - assert url.startswith(f"http://files.local/files/app-assets/draft/{tenant_id}/{app_id}/{resource_id}/download?") - assert "expires_at=" in url + # URL should be a proxy URL since DummyStorage doesn't support presign + storage_key = asset_path.get_storage_key() + assert url.startswith("http://files.local/files/storage/") + assert "/download?" in url + assert "timestamp=" in url + assert "nonce=" in url + assert "sign=" in url + + +def test_upload_url_generation(monkeypatch: pytest.MonkeyPatch): + """Test that AppAssetStorage generates correct upload URLs.""" + tenant_id = str(uuid4()) + app_id = str(uuid4()) + resource_id = str(uuid4()) + asset_path = AssetPath.draft(tenant_id, app_id, resource_id) + + monkeypatch.setattr(dify_config, "SECRET_KEY", "test-secret-key", raising=False) + monkeypatch.setattr(dify_config, "FILES_ACCESS_TIMEOUT", 300, raising=False) + monkeypatch.setattr(dify_config, "FILES_URL", "http://files.local", raising=False) + + storage = AppAssetStorage(DummyStorage(), redis_client=DummyRedis()) + url = storage.get_upload_url(asset_path, expires_in=120) + + # URL should be a proxy URL since DummyStorage doesn't support presign + assert url.startswith("http://files.local/files/storage/") + assert "/upload?" in url + assert "timestamp=" in url assert "nonce=" in url assert "sign=" in url