From 2ffb31ac252e0357a7c6ce0062ba753f7fb42ff7 Mon Sep 17 00:00:00 2001 From: Kapozux Date: Mon, 9 Nov 2026 21:10:06 +0800 Subject: [PATCH] fix(datasource): handle CSV extensions case-insensitively --- backend/apps/datasource/api/datasource.py | 6 +-- backend/apps/datasource/utils/excel.py | 8 +++- .../tests/test_datasource_csv_extension.py | 39 +++++++++++++++++++ .../test_datasource_upload_path_traversal.py | 12 ++++++ 4 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 backend/tests/test_datasource_csv_extension.py diff --git a/backend/apps/datasource/api/datasource.py b/backend/apps/datasource/api/datasource.py index db03b2fae..672a19513 100644 --- a/backend/apps/datasource/api/datasource.py +++ b/backend/apps/datasource/api/datasource.py @@ -33,7 +33,7 @@ from ..crud.table import get_tables_by_ds_id from ..models.datasource import CoreDatasource, CreateDatasource, TableObj, CoreTable, CoreField, FieldObj, \ TableSchemaResponse, ColumnSchemaResponse, PreviewResponse, ImportRequest -from ..utils.excel import parse_excel_preview, read_import_dataframe +from ..utils.excel import is_csv_file, parse_excel_preview, read_import_dataframe router = APIRouter(tags=["Datasource"], prefix="/datasource") path = settings.EXCEL_PATH @@ -363,7 +363,7 @@ async def upload_excel(trans: Trans, session: SessionDep, def inner(): sheets = [] engine = get_engine_conn() - if filename.endswith(".csv"): + if is_csv_file(filename): df = pd.read_csv(save_path, engine='c') tableName = f"sheet1_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}" sheets.append({"tableName": tableName, "tableComment": ""}) @@ -633,7 +633,7 @@ def inner(): field_mapping = {f.fieldName: f.fieldType for f in fields} try: df = read_import_dataframe(save_path, sheet_name, field_mapping) - if save_path.endswith(".csv"): + if is_csv_file(save_path): sheet_name = "Sheet1" except Exception as e: raise HTTPException(500, f"{trans('i18n_ds_upload_error')}: {str(e)}") diff --git a/backend/apps/datasource/utils/excel.py b/backend/apps/datasource/utils/excel.py index 2d982097c..6251bb40d 100644 --- a/backend/apps/datasource/utils/excel.py +++ b/backend/apps/datasource/utils/excel.py @@ -22,6 +22,10 @@ } +def is_csv_file(file_path: str) -> bool: + return file_path.lower().endswith(".csv") + + def infer_field_type(dtype) -> str: dtype_str = str(dtype) return FIELD_TYPE_MAP.get(dtype_str, 'string') @@ -51,7 +55,7 @@ def read_import_dataframe(save_path: str, sheet_name: str, field_mapping: dict): column: 'object' if field_type == 'int' else USER_TYPE_TO_PANDAS.get(field_type, 'string') for column, field_type in field_mapping.items() } - if save_path.endswith('.csv'): + if is_csv_file(save_path): df = pd.read_csv(save_path, engine='c', dtype=dtype_dict) else: df = pd.read_excel(save_path, sheet_name=sheet_name, engine='calamine', dtype=dtype_dict) @@ -68,7 +72,7 @@ def read_import_dataframe(save_path: str, sheet_name: str, field_mapping: dict): def parse_excel_preview(save_path: str, max_rows: int = 10): sheets_data = [] - if save_path.endswith(".csv"): + if is_csv_file(save_path): df = pd.read_csv(save_path, engine='c') fields = [] for col in df.columns: diff --git a/backend/tests/test_datasource_csv_extension.py b/backend/tests/test_datasource_csv_extension.py new file mode 100644 index 000000000..d2714ece9 --- /dev/null +++ b/backend/tests/test_datasource_csv_extension.py @@ -0,0 +1,39 @@ +import pandas as pd +import pytest + +from apps.datasource.utils.excel import parse_excel_preview, read_import_dataframe + + +@pytest.fixture(params=["csv", "CSV", "CsV"]) +def stored_csv(tmp_path, request): + path = tmp_path / f"sales_0123456789.{request.param}" + path.write_text("region,units\nEast,3\nWest,7\n", encoding="utf-8") + return str(path) + + +def test_preview_csv_with_any_extension_case(stored_csv): + [sheet] = parse_excel_preview(stored_csv) + assert sheet["sheetName"] == "Sheet1" + assert sheet["rows"] == 2 + assert sheet["data"] == [ + {"region": "East", "units": 3}, + {"region": "West", "units": 7}, + ] + + +def test_import_stored_csv_with_any_extension_case(stored_csv): + frame = read_import_dataframe( + stored_csv, "Sheet1", {"region": "string", "units": "int"} + ) + assert frame["region"].tolist() == ["East", "West"] + assert frame["units"].tolist() == [3, 7] + + +def test_excel_reader_preserves_named_sheet(tmp_path): + path = tmp_path / "sales.CSV.xlsx" + pd.DataFrame({"units": [3]}).to_excel(path, sheet_name="Sales", index=False) + + [sheet] = parse_excel_preview(str(path)) + assert sheet["sheetName"] == "Sales" + frame = read_import_dataframe(str(path), "Sales", {"units": "int"}) + assert frame["units"].tolist() == [3] diff --git a/backend/tests/test_datasource_upload_path_traversal.py b/backend/tests/test_datasource_upload_path_traversal.py index 3505bc782..3ad2a6dd3 100644 --- a/backend/tests/test_datasource_upload_path_traversal.py +++ b/backend/tests/test_datasource_upload_path_traversal.py @@ -122,3 +122,15 @@ def test_normal_names_keep_their_own_stem_and_extension(parse_harness): stored = response.json()["filePath"] assert stored.startswith("a.b_") and stored.endswith(".xlsx"), stored assert (parse_harness.upload_dir / stored).is_file() + + +def test_legacy_upload_imports_mixed_case_csv(harness, monkeypatch): + insert = Mock() + monkeypatch.setattr(datasource, "insert_pg", insert) + response = harness.client.post( + UPLOAD_URL, files={"file": ("orders.CsV", b"units\n3\n7\n", "text/csv")} + ) + assert response.status_code == 200, response.text + insert.assert_called_once() + frame = insert.call_args.args[0] + assert frame.to_dict("records") == [{"units": 3}, {"units": 7}]