From c613fd35d9e6c048d5f6b447af6edf348865fd00 Mon Sep 17 00:00:00 2001 From: phil Date: Sat, 20 Apr 2024 10:58:12 +0530 Subject: [PATCH] Admin basket: update/fix file import, download, delete remove useless AdminBasketFile model definition cleanups, typings --- src/gisaf/api/admin.py | 38 +++++++-- src/gisaf/baskets.py | 166 ++++++++++++++++---------------------- src/gisaf/importers.py | 4 +- src/gisaf/models/admin.py | 49 ++++------- src/gisaf/registry.py | 2 +- 5 files changed, 123 insertions(+), 136 deletions(-) diff --git a/src/gisaf/api/admin.py b/src/gisaf/api/admin.py index 213c573..fe6314e 100644 --- a/src/gisaf/api/admin.py +++ b/src/gisaf/api/admin.py @@ -47,13 +47,23 @@ async def get_basket( async def upload_basket_file( name: str, file: UploadFile, + project_id: int | None = None, + surveyor_id: int | None = None, + equipment_id: int | None = None, + auto_import: bool = False, user: UserRead = Depends(get_current_active_user), - ): +) -> BasketImportResult: try: basket = manager.baskets[name] except KeyError: - raise HTTPException(status.HTTP_404_NOT_FOUND, f'No basket named {name}') - fileItem = await basket.add_files(file, user) + raise HTTPException(status.HTTP_404_NOT_FOUND, f"No basket named {name}") + fileItem = await basket.add_file( + file, + user, + project_id=project_id, + surveyor_id=surveyor_id, + equipment_id=equipment_id, + ) return fileItem @api.get('/basket/download/{name}/{file_id}/{file_name}') @@ -73,7 +83,10 @@ async def download_basket_file( file_record = await basket.get_file(file_id) if file_record is None: raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}') - return FileResponse(file_record.path) + abs_path = basket.base_dir / file_record.path + if not abs_path.exists(): + raise HTTPException(status.HTTP_404_NOT_FOUND, f'File {file_record.name} not found') + return FileResponse(abs_path) @api.get('/basket/import/{basket}/{file_id}') async def import_basket_file( @@ -93,4 +106,19 @@ async def import_basket_file( ## FIXME: shouldn't it be AdminImportError? except ImportError as err: raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, err) - return result \ No newline at end of file + return result + + +@api.get('/basket/delete/{basket}/{file_id}') +async def delete_basket_file( + basket: str, + file_id: int, + user: User = Depends(get_current_active_user), + ) -> None: + if not (user and user.has_role('reviewer')): + raise HTTPException(status.HTTP_401_UNAUTHORIZED) + basket_ = manager.baskets[basket] + file_import = await basket_.get_file(file_id) + if file_import is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}') + await basket_.delete_file(file_import) \ No newline at end of file diff --git a/src/gisaf/baskets.py b/src/gisaf/baskets.py index f273193..0c377be 100644 --- a/src/gisaf/baskets.py +++ b/src/gisaf/baskets.py @@ -1,10 +1,8 @@ from pathlib import Path from aiopath import AsyncPath -from collections import defaultdict -from json import loads -from datetime import datetime import logging -from typing import ClassVar +from typing import ClassVar, Type +from hashlib import md5 # from aiohttp_security import check_permission # from aiohttp.multipart import MultipartReader @@ -17,8 +15,9 @@ from fastapi import UploadFile from gisaf.config import conf from gisaf.database import db_session -from gisaf.importers import RawSurveyImporter, GeoDataImporter, LineWorkImporter, ImportError -from gisaf.models.admin import FileImport, AdminBasketFile, BasketImportResult +from gisaf.importers import (Importer, RawSurveyImporter, GeoDataImporter, + LineWorkImporter, ImportError) +from gisaf.models.admin import FileImport, BasketImportResult from gisaf.models.authentication import UserRead from gisaf.models.survey import Surveyor, Equipment from gisaf.models.project import Project @@ -39,11 +38,12 @@ class Basket: who don't have that role. """ name: ClassVar[str] - importer_class = None - _custom_module = None + importer_class: Type[Importer] + importer: Importer + _custom_module: str | None = None columns: list[str] = ['name', 'time', 'import', 'delete'] upload_fields: list[str] = [] - role = None + role: str | None = None def __init__(self): self.base_dir = Path(conf.admin.basket.base_dir) / self.name @@ -66,7 +66,7 @@ class Basket: async def get_files(self) -> list[FileImport]: async with db_session() as session: data = await session.exec(select(FileImport).where(FileImport.basket==self.name)) - return data.all() + return data.all() # type: ignore # async def get_files_df(self, convert_path=False): # """ @@ -107,9 +107,9 @@ class Basket: async def get_file(self, file_id: int) -> FileImport | None: async with db_session() as session: query = select(FileImport).where(FileImport.id==file_id).options( - joinedload(FileImport.project), - joinedload(FileImport.surveyor), - joinedload(FileImport.equipment), + joinedload(FileImport.project), # type: ignore + joinedload(FileImport.surveyor), # type: ignore + joinedload(FileImport.equipment), # type: ignore ) res = await session.exec(query) try: @@ -136,17 +136,19 @@ class Basket: # file.name = file['name'] # return file - async def delete_file(self, id): - file = await FileImport.get(id) + async def delete_file(self, file: FileImport): if file.dir: path = self.base_dir/file.dir/file.name else: path = self.base_dir/file.name if path.exists(): path.unlink() - await file.delete() + async with db_session() as session: + await session.delete(file) + await session.commit() - async def import_file(self, file_import: FileImport, dry_run=True, **kwargs) -> BasketImportResult: + async def import_file(self, file_import: FileImport, + dry_run=True, **kwargs) -> BasketImportResult: """ Import the file by calling the basket's importer's do_import. Time stamp the FileImport. @@ -164,7 +166,6 @@ class Basket: except Exception as err: logger.exception(err) raise ImportError(f'Unexpected import error (details in the Gisaf logs): {err}') - if not isinstance(result, BasketImportResult): raise ImportError('Import error: the importer did not return a BasketImportResult') # if import_result: @@ -181,9 +182,11 @@ class Basket: # result = BasketImportResult(message=import_result) # else: # result = BasketImportResult(message='Import successful.') + if file_import.time is None: + raise ImportError('No time found in file import') if dry_run: result.time = file_import.time - if not dry_run: + else: ## Save time stamp async with db_session() as session: file_import.time = result.time @@ -191,7 +194,15 @@ class Basket: await session.commit() return result - async def add_files(self, file: UploadFile, user: UserRead): + async def add_file(self, + file: UploadFile, + user: UserRead, + auto_import: bool = False, + dry_run: bool = False, + project_id: int | None = None, + surveyor_id: int | None = None, + equipment_id: int | None = None, + ) -> BasketImportResult: """ File upload to basket. Typically called through an http POST view handler. @@ -203,89 +214,52 @@ class Basket: ## TODO: check if file already exists # assert part.name == 'file' ## Save file on filesystem + if file.filename is None: + raise ImportError('No file name') path = AsyncPath(self.base_dir) / file.filename ## Eventually create the directory await path.parent.mkdir(parents=True, exist_ok=True) + file_content = await file.read() + md5sum = md5(file_content).hexdigest() async with path.open('wb') as f: ## No way to use async to stream the file content to write it? - await f.write(await file.read()) - - ## Read other parts - # parts = defaultdict(None) - # while True: - # part = await reader.next() - # if not part: - # break - # value = (await part.read()).decode() - # if value != 'null': - # parts[part.name] = value - - ## Find ids of project, surveyor, equipment - if 'project' in parts: - project_id = (await Project.query.where(Project.name==parts.get('project')).gino.first()).id - else: - project_id = None - if 'surveyor' in parts: - surveyor_id = (await Surveyor.query.where(Surveyor.name==parts.get('surveyor')).gino.first()).id - else: - surveyor_id = None - if 'equipment' in parts: - equipment_id = (await Equipment.query.where(Equipment.name==parts.get('equipment')).gino.first()).id - else: - equipment_id = None - - ## Save FileImport record - store_keys = [store_key for store_key in parts.keys() - if store_key.startswith('store')] - if len(store_keys) == 1: - store_type_name = parts[store_keys[0]] - else: - store_type_name = None - fileImportRecord = await FileImport( - name=file.file_name, - dir=parts.get('dir', '.'), - basket=self.name, - project_id=project_id, - surveyor_id=surveyor_id, - equipment_id=equipment_id, - store=store_type_name, - status=parts.get('status', None), - ).create() - fileImportRecord.path = self.base_dir/fileImportRecord.dir/fileImportRecord.name - - admin_basket_file = AdminBasketFile( - id=fileImportRecord.id, - name=file.file_name, - status=parts.get('status'), - store=store_type_name, - project=parts.get('project'), - surveyor=parts.get('surveyor'), - equipment=parts.get('equipment'), - ) - + await f.write(file_content) + async with db_session() as session: + fileImportRecord = FileImport( + name=file.filename, + md5=md5sum, + dir=str(self.base_dir), + basket=self.name, + project_id=project_id, + surveyor_id=surveyor_id, + equipment_id=equipment_id, + # store=store_type_name, + # status=parts.get('status', None), + ) + # fileImportRecord.path = self.base_dir / fileImportRecord.dir / fileImportRecord.name + session.add(fileImportRecord) + await session.commit() + await session.refresh(fileImportRecord) + if fileImportRecord.id is None: + raise ImportError('Cannot save (no fileImportRecord.id)') ## Eventually do import - import_result = None - if loads(parts['autoImport']): - ## Get the record from DB with Pandas, for compatibility with import_file - file_import_record = await self.get_file(fileImportRecord.id) - try: - await check_permission(request, 'reviewer') - except HTTPUnauthorized as err: - basket_import_result = BasketImportResult( - message="Cannot import: only a reviewer can do that" - ) + basket_import_result = BasketImportResult( + fileImport=fileImportRecord + ) + if auto_import: + if user.has_role('reviewer'): + ## Get the record from DB, for compatibility with import_file + file_import_record = await self.get_file(fileImportRecord.id) + if file_import_record is None: + basket_import_result.message="Cannot import: file not found" + else: + try: + basket_import_result = await self.import_file(file_import_record, dry_run) + except ImportError as err: + basket_import_result.message=f'Error: {err.args[0]}' else: - dry_run = parts.get('dry_run', False) - try: - basket_import_result = await self.import_file(file_import_record, dry_run) - except ImportError as err: - basket_import_result = BasketImportResult( - message=f'Error: {err.args[0]}' - ) - - admin_basket_file.import_result = basket_import_result - - return admin_basket_file + basket_import_result.message="Cannot import: only a reviewer can do that" + return basket_import_result class MiscGeomBasket(Basket): diff --git a/src/gisaf/importers.py b/src/gisaf/importers.py index e5c83a7..d9e63e1 100644 --- a/src/gisaf/importers.py +++ b/src/gisaf/importers.py @@ -39,7 +39,7 @@ class Importer: The main process is executed by do_import(file) Subclasses should define read_file and process_df. """ - basket = None + basket = None # type hint: baskets.Basket async def do_import(self, file_record, dry_run=False, **kwargs) -> BasketImportResult: """ @@ -181,7 +181,7 @@ class RawSurveyImporter(Importer): ## Import to raw_survey_data table ## PostGis specific: add SRID - gdf['geom'] = gdf['geometry'].apply(lambda g: dumps_wkb(g, srid=conf.raw_survey['srid'], hex=True)) + gdf['geom'] = gdf['geometry'].apply(lambda g: dumps_wkb(g, srid=conf.geo.raw_survey.srid, hex=True)) if not dry_run: await upsert_df(gdf, model) diff --git a/src/gisaf/models/admin.py b/src/gisaf/models/admin.py index e74b5b5..219a733 100644 --- a/src/gisaf/models/admin.py +++ b/src/gisaf/models/admin.py @@ -1,7 +1,8 @@ +from pathlib import Path import re from datetime import datetime, date -from pydantic import BaseModel +from pydantic import BaseModel, computed_field from sqlmodel import Field, Relationship import pandas as pd @@ -47,33 +48,27 @@ class FileImport(Model, table=True): __table_args__ = gisaf_admin.table_args id: int | None = Field(default=None, primary_key=True) - url: str + url: str | None = None ## TODO: Deprecate FileImport.path, in favour of dir + name - path: str + #path: str dir: str name: str md5: str - time: datetime - comment: str - status: str - store: str + time: datetime | None = None + comment: str | None = None + status: str | None = None + store: str | None = None basket: str - project_id: int = Field(foreign_key=gisaf_admin.table('project.id')) + project_id: int | None = Field(foreign_key=gisaf_admin.table('project.id')) project: Project = Relationship() # ALTER TABLE gisaf_admin.file_import add column project_id INT REFERENCES gisaf_admin.project; - surveyor_id: int = Field(foreign_key=gisaf_survey.table('surveyor.id')) + surveyor_id: int | None = Field(foreign_key=gisaf_survey.table('surveyor.id')) surveyor: Surveyor = Relationship() # ALTER TABLE gisaf_admin.file_import add column surveyor_id INT REFERENCES gisaf_survey.surveyor; - equipment_id: int = Field(foreign_key=gisaf_survey.table('equipment.id')) + equipment_id: int | None = Field(foreign_key=gisaf_survey.table('equipment.id')) equipment: Equipment = Relationship() # ALTER TABLE gisaf_admin.file_import add column equipment_id INT REFERENCES gisaf_survey.equipment; - def __str__(self): - return f'{self.path:s} for project id {self.project_id}' - - def __repr__(self): - return f'' - @classmethod def selectinload(cls): return [cls.project, cls.surveyor, cls.equipment] @@ -93,6 +88,10 @@ class FileImport(Model, table=True): df['date'] = pd.to_datetime(dates[0], format='%Y-%m-%d') return df + @computed_field + @property + def path(self) -> Path: + return Path(self.dir) / self.name #def get_parent_dir(self): # split_path = self.path.split(os_path.sep) @@ -158,23 +157,9 @@ class Basket(BasketNameOnly): class BasketImportResult(BaseModel): time: datetime = Field(default_factory=datetime.now) - message: str + message: str | None = None details: dict[str, str | int | float | bool] | None = None - -class AdminBasketFile(BaseModel): - id: int - dir: str - name: str - url: str - md5: str - time: datetime - comment: str - status: str - store: str - project: str - surveyor: str - equipment: str - import_result: str + fileImport: FileImport | None = None class AdminBasket(BaseModel): diff --git a/src/gisaf/registry.py b/src/gisaf/registry.py index 8693ec7..bc108d8 100644 --- a/src/gisaf/registry.py +++ b/src/gisaf/registry.py @@ -681,7 +681,7 @@ class ModelRegistry: 'live': 'is_live', 'zIndex': 'z_index', 'gisType': 'gis_type', - # 'type': 'mapbox_type', + 'type': 'mapbox_type', 'viewableRole': 'viewable_role', }, inplace=True )