diff --git a/src/gisaf/api/admin.py b/src/gisaf/api/admin.py index fe6314e..6ffd603 100644 --- a/src/gisaf/api/admin.py +++ b/src/gisaf/api/admin.py @@ -1,13 +1,13 @@ import logging -from fastapi import (Depends, APIRouter, HTTPException, status, responses, - UploadFile) +from fastapi import Depends, APIRouter, HTTPException, status, responses, UploadFile from fastapi.responses import FileResponse from gisaf.models.admin import AdminBasket, BasketImportResult, BasketNameOnly from gisaf.models.authentication import User, UserRead from gisaf.security import get_current_active_user from gisaf.admin import manager +from gisaf.importers import ImportError logger = logging.getLogger(__name__) @@ -17,20 +17,22 @@ api = APIRouter( responses={404: {"description": "Not found"}}, ) -@api.get('/basket') + +@api.get("/basket") async def get_baskets( user: User = Depends(get_current_active_user), - ) -> list[BasketNameOnly]: +) -> list[BasketNameOnly]: return [ BasketNameOnly(name=name) for name, basket in (await manager.baskets_for_role(user)).items() ] -@api.get('/basket/{name}') + +@api.get("/basket/{name}") async def get_basket( name: str, user: User = Depends(get_current_active_user), - ) -> AdminBasket: +) -> AdminBasket: basket = manager.baskets[name] if basket.role and not user.has_role(basket.role): raise HTTPException(status.HTTP_401_UNAUTHORIZED) @@ -43,7 +45,8 @@ async def get_basket( # projects=getattr(basket, 'projects', None) ) -@api.post('/basket/upload/{name}') + +@api.post("/basket/upload/{name}") async def upload_basket_file( name: str, file: UploadFile, @@ -66,59 +69,62 @@ async def upload_basket_file( ) return fileItem -@api.get('/basket/download/{name}/{file_id}/{file_name}') + +@api.get("/basket/download/{name}/{file_id}/{file_name}") async def download_basket_file( name: str, file_id: int, file_name: str, user: User = Depends(get_current_active_user), - ) -> FileResponse: +) -> FileResponse: try: basket = manager.baskets[name] except KeyError: - raise HTTPException(status.HTTP_404_NOT_FOUND, f'No basket named {name}') + raise HTTPException(status.HTTP_404_NOT_FOUND, f"No basket named {name}") if basket.role: if not user.has_role(basket.role): raise HTTPException(status.HTTP_401_UNAUTHORIZED) file_record = await basket.get_file(file_id) if file_record is None: - raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}') + raise HTTPException(status.HTTP_404_NOT_FOUND, f"No import file id {file_id}") abs_path = basket.base_dir / file_record.path if not abs_path.exists(): - raise HTTPException(status.HTTP_404_NOT_FOUND, f'File {file_record.name} not found') + raise HTTPException( + status.HTTP_404_NOT_FOUND, f"File {file_record.name} not found" + ) return FileResponse(abs_path) -@api.get('/basket/import/{basket}/{file_id}') + +@api.get("/basket/import/{basket}/{file_id}") async def import_basket_file( basket: str, file_id: int, dryRun: bool = False, user: User = Depends(get_current_active_user), - ) -> BasketImportResult: - if not (user and user.has_role('reviewer')): +) -> BasketImportResult: + if not (user and user.has_role("reviewer")): raise HTTPException(status.HTTP_401_UNAUTHORIZED) basket_ = manager.baskets[basket] file_import = await basket_.get_file(file_id) if file_import is None: - raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}') + raise HTTPException(status.HTTP_404_NOT_FOUND, f"No import file id {file_id}") try: result = await basket_.import_file(file_import, dryRun) - ## FIXME: shouldn't it be AdminImportError? except ImportError as err: - raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, err) + raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, str(err)) return result -@api.get('/basket/delete/{basket}/{file_id}') +@api.get("/basket/delete/{basket}/{file_id}") async def delete_basket_file( basket: str, file_id: int, user: User = Depends(get_current_active_user), - ) -> None: - if not (user and user.has_role('reviewer')): +) -> None: + if not (user and user.has_role("reviewer")): raise HTTPException(status.HTTP_401_UNAUTHORIZED) basket_ = manager.baskets[basket] file_import = await basket_.get_file(file_id) if file_import is None: - raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}') - await basket_.delete_file(file_import) \ No newline at end of file + raise HTTPException(status.HTTP_404_NOT_FOUND, f"No import file id {file_id}") + await basket_.delete_file(file_import) diff --git a/src/gisaf/baskets.py b/src/gisaf/baskets.py index 5adffc8..d800062 100644 --- a/src/gisaf/baskets.py +++ b/src/gisaf/baskets.py @@ -15,8 +15,13 @@ from fastapi import UploadFile from gisaf.config import conf from gisaf.database import db_session -from gisaf.importers import (Importer, RawSurveyImporter, GeoDataImporter, - LineWorkImporter, ImportError) +from gisaf.importers import ( + Importer, + RawSurveyImporter, + GeoDataImporter, + LineWorkImporter, + ImportError, +) from gisaf.models.admin import FileImport, BasketImportResult from gisaf.models.authentication import UserRead from gisaf.models.survey import Surveyor, Equipment @@ -24,7 +29,7 @@ from gisaf.models.project import Project logger = logging.getLogger(__name__) -upload_fields_available = ['store', 'status', 'project', 'surveyor', 'equipment'] +upload_fields_available = ["store", "status", "project", "surveyor", "equipment"] class Basket: @@ -37,11 +42,12 @@ class Basket: The basket can have a role. In that case, it will be completely hidden from users who don't have that role. """ + name: ClassVar[str] importer_class: Type[Importer] | None = None importer: Importer _custom_module: str | None = None - columns: list[str] = ['name', 'time', 'import', 'delete'] + columns: list[str] = ["name", "time", "import", "delete"] upload_fields: list[str] = [] role: str | None = None @@ -65,7 +71,9 @@ class Basket: async def get_files(self) -> list[FileImport]: async with db_session() as session: - data = await session.exec(select(FileImport).where(FileImport.basket==self.name)) + data = await session.exec( + select(FileImport).where(FileImport.basket == self.name) + ) return data.all() # type: ignore # async def get_files_df(self, convert_path=False): @@ -106,10 +114,14 @@ class Basket: async def get_file(self, file_id: int) -> FileImport | None: async with db_session() as session: - query = select(FileImport).where(FileImport.id==file_id).options( - joinedload(FileImport.project), # type: ignore - joinedload(FileImport.surveyor), # type: ignore - joinedload(FileImport.equipment), # type: ignore + query = ( + select(FileImport) + .where(FileImport.id == file_id) + .options( + joinedload(FileImport.project), # type: ignore + joinedload(FileImport.surveyor), # type: ignore + joinedload(FileImport.equipment), # type: ignore + ) ) res = await session.exec(query) try: @@ -138,36 +150,43 @@ class Basket: async def delete_file(self, file: FileImport): if file.dir: - path = self.base_dir/file.dir/file.name + path = self.base_dir / file.dir / file.name else: - path = self.base_dir/file.name + path = self.base_dir / file.name if path.exists(): path.unlink() async with db_session() as session: await session.delete(file) await session.commit() - async def import_file(self, file_import: FileImport, - dry_run=True, **kwargs) -> BasketImportResult: + async def import_file( + self, file_import: FileImport, dry_run=True, **kwargs + ) -> BasketImportResult: """ Import the file by calling the basket's importer's do_import. Time stamp the FileImport. Return a BasketImportResult ObjectType """ - if not hasattr(self, 'importer'): + if not hasattr(self, "importer"): return BasketImportResult( - message=f'No import defined/required for {self.name} basket' + message=f"No import defined/required for {self.name} basket" ) result: BasketImportResult try: - result = await self.importer.do_import(file_import, dry_run=dry_run, **kwargs) + result = await self.importer.do_import( + file_import, dry_run=dry_run, **kwargs + ) except ImportError as err: raise err except Exception as err: logger.exception(err) - raise ImportError(f'Unexpected import error (details in the Gisaf logs): {err}') + raise ImportError( + f"Unexpected import error (details in the Gisaf logs): {err}" + ) if not isinstance(result, BasketImportResult): - raise ImportError('Import error: the importer did not return a BasketImportResult') + raise ImportError( + "Import error: the importer did not return a BasketImportResult" + ) # if import_result: # if isinstance(import_result, (tuple, list)): # assert len(import_result) >= 2, \ @@ -182,8 +201,6 @@ class Basket: # result = BasketImportResult(message=import_result) # else: # result = BasketImportResult(message='Import successful.') - if file_import.time is None: - raise ImportError('No time found in file import') if dry_run: result.time = file_import.time else: @@ -194,15 +211,16 @@ class Basket: await session.commit() return result - async def add_file(self, - file: UploadFile, - user: UserRead, - auto_import: bool = False, - dry_run: bool = False, - project_id: int | None = None, - surveyor_id: int | None = None, - equipment_id: int | None = None, - ) -> BasketImportResult: + async def add_file( + self, + file: UploadFile, + user: UserRead, + auto_import: bool = False, + dry_run: bool = False, + project_id: int | None = None, + surveyor_id: int | None = None, + equipment_id: int | None = None, + ) -> BasketImportResult: """ File upload to basket. Typically called through an http POST view handler. @@ -215,13 +233,13 @@ class Basket: # assert part.name == 'file' ## Save file on filesystem if file.filename is None: - raise ImportError('No file name') + raise ImportError("No file name") path = AsyncPath(self.base_dir) / file.filename ## Eventually create the directory await path.parent.mkdir(parents=True, exist_ok=True) file_content = await file.read() md5sum = md5(file_content).hexdigest() - async with path.open('wb') as f: + async with path.open("wb") as f: ## No way to use async to stream the file content to write it? await f.write(file_content) async with db_session() as session: @@ -241,46 +259,48 @@ class Basket: await session.commit() await session.refresh(fileImportRecord) if fileImportRecord.id is None: - raise ImportError('Cannot save (no fileImportRecord.id)') + raise ImportError("Cannot save (no fileImportRecord.id)") ## Eventually do import - basket_import_result = BasketImportResult( - fileImport=fileImportRecord - ) + basket_import_result = BasketImportResult(fileImport=fileImportRecord) if auto_import: - if user.has_role('reviewer'): + if user.has_role("reviewer"): ## Get the record from DB, for compatibility with import_file file_import_record = await self.get_file(fileImportRecord.id) if file_import_record is None: - basket_import_result.message="Cannot import: file not found" + basket_import_result.message = "Cannot import: file not found" else: try: - basket_import_result = await self.import_file(file_import_record, dry_run) + basket_import_result = await self.import_file( + file_import_record, dry_run + ) except ImportError as err: - basket_import_result.message=f'Error: {err.args[0]}' + basket_import_result.message = f"Error: {err.args[0]}" else: - basket_import_result.message="Cannot import: only a reviewer can do that" + basket_import_result.message = ( + "Cannot import: only a reviewer can do that" + ) return basket_import_result class MiscGeomBasket(Basket): - name = 'Misc geo file' + name = "Misc geo file" importer_class = GeoDataImporter - columns = ['name', 'time', 'status', 'store', 'import', 'delete'] - upload_fields = ['store_misc', 'status'] + columns = ["name", "time", "status", "store", "import", "delete"] + upload_fields = ["store_misc", "status"] class LineWorkBasket(Basket): - name = 'Line work' + name = "Line work" importer_class = LineWorkImporter - columns = ['name', 'time', 'status', 'store', 'project', 'import', 'delete'] - upload_fields = ['store_line_work', 'project', 'status'] + columns = ["name", "time", "status", "store", "project", "import", "delete"] + upload_fields = ["store_line_work", "project", "status"] class SurveyBasket(Basket): - name = 'Survey' + name = "Survey" importer_class = RawSurveyImporter - columns = ['name', 'time', 'project', 'surveyor', 'equipment', 'import', 'delete'] - upload_fields = ['project', 'surveyor', 'equipment'] + columns = ["name", "time", "project", "surveyor", "equipment", "import", "delete"] + upload_fields = ["project", "surveyor", "equipment"] standard_baskets = (