philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

Admin: fix issue with import and error handling

Browse source
This commit is contained in:
phil 2024-05-15 15:09:10 +02:00
parent 0bf74b2bba
commit 5a63892640
2 changed files with 98 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

40
src/gisaf/api/admin.py
View file

@ -1,13 +1,13 @@
import logging import logging
from fastapi import (Depends, APIRouter, HTTPException, status, responses, from fastapi import Depends, APIRouter, HTTPException, status, responses, UploadFile
UploadFile)
from fastapi.responses import FileResponse from fastapi.responses import FileResponse
from gisaf.models.admin import AdminBasket, BasketImportResult, BasketNameOnly from gisaf.models.admin import AdminBasket, BasketImportResult, BasketNameOnly
from gisaf.models.authentication import User, UserRead from gisaf.models.authentication import User, UserRead
from gisaf.security import get_current_active_user from gisaf.security import get_current_active_user
from gisaf.admin import manager from gisaf.admin import manager
from gisaf.importers import ImportError
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -17,7 +17,8 @@ api = APIRouter(
responses={404: {"description": "Not found"}}, responses={404: {"description": "Not found"}},
) )
@api.get('/basket')
@api.get("/basket")
async def get_baskets( async def get_baskets(
user: User = Depends(get_current_active_user), user: User = Depends(get_current_active_user),
) -> list[BasketNameOnly]: ) -> list[BasketNameOnly]:
@ -26,7 +27,8 @@ async def get_baskets(
for name, basket in (await manager.baskets_for_role(user)).items() for name, basket in (await manager.baskets_for_role(user)).items()
] ]
@api.get('/basket/{name}')
@api.get("/basket/{name}")
async def get_basket( async def get_basket(
name: str, name: str,
user: User = Depends(get_current_active_user), user: User = Depends(get_current_active_user),
@ -43,7 +45,8 @@ async def get_basket(
# projects=getattr(basket, 'projects', None) # projects=getattr(basket, 'projects', None)
) )
@api.post('/basket/upload/{name}')
@api.post("/basket/upload/{name}")
async def upload_basket_file( async def upload_basket_file(
name: str, name: str,
file: UploadFile, file: UploadFile,
@ -66,7 +69,8 @@ async def upload_basket_file(
) )
return fileItem return fileItem
@api.get('/basket/download/{name}/{file_id}/{file_name}')
@api.get("/basket/download/{name}/{file_id}/{file_name}")
async def download_basket_file( async def download_basket_file(
name: str, name: str,
file_id: int, file_id: int,
@ -76,49 +80,51 @@ async def download_basket_file(
try: try:
basket = manager.baskets[name] basket = manager.baskets[name]
except KeyError: except KeyError:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No basket named {name}') raise HTTPException(status.HTTP_404_NOT_FOUND, f"No basket named {name}")
if basket.role: if basket.role:
if not user.has_role(basket.role): if not user.has_role(basket.role):
raise HTTPException(status.HTTP_401_UNAUTHORIZED) raise HTTPException(status.HTTP_401_UNAUTHORIZED)
file_record = await basket.get_file(file_id) file_record = await basket.get_file(file_id)
if file_record is None: if file_record is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}') raise HTTPException(status.HTTP_404_NOT_FOUND, f"No import file id {file_id}")
abs_path = basket.base_dir / file_record.path abs_path = basket.base_dir / file_record.path
if not abs_path.exists(): if not abs_path.exists():
raise HTTPException(status.HTTP_404_NOT_FOUND, f'File {file_record.name} not found') raise HTTPException(
status.HTTP_404_NOT_FOUND, f"File {file_record.name} not found"
)
return FileResponse(abs_path) return FileResponse(abs_path)
@api.get('/basket/import/{basket}/{file_id}')
@api.get("/basket/import/{basket}/{file_id}")
async def import_basket_file( async def import_basket_file(
basket: str, basket: str,
file_id: int, file_id: int,
dryRun: bool = False, dryRun: bool = False,
user: User = Depends(get_current_active_user), user: User = Depends(get_current_active_user),
) -> BasketImportResult: ) -> BasketImportResult:
if not (user and user.has_role('reviewer')): if not (user and user.has_role("reviewer")):
raise HTTPException(status.HTTP_401_UNAUTHORIZED) raise HTTPException(status.HTTP_401_UNAUTHORIZED)
basket_ = manager.baskets[basket] basket_ = manager.baskets[basket]
file_import = await basket_.get_file(file_id) file_import = await basket_.get_file(file_id)
if file_import is None: if file_import is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}') raise HTTPException(status.HTTP_404_NOT_FOUND, f"No import file id {file_id}")
try: try:
result = await basket_.import_file(file_import, dryRun) result = await basket_.import_file(file_import, dryRun)
## FIXME: shouldn't it be AdminImportError?
except ImportError as err: except ImportError as err:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, err) raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, str(err))
return result return result
@api.get('/basket/delete/{basket}/{file_id}') @api.get("/basket/delete/{basket}/{file_id}")
async def delete_basket_file( async def delete_basket_file(
basket: str, basket: str,
file_id: int, file_id: int,
user: User = Depends(get_current_active_user), user: User = Depends(get_current_active_user),
) -> None: ) -> None:
if not (user and user.has_role('reviewer')): if not (user and user.has_role("reviewer")):
raise HTTPException(status.HTTP_401_UNAUTHORIZED) raise HTTPException(status.HTTP_401_UNAUTHORIZED)
basket_ = manager.baskets[basket] basket_ = manager.baskets[basket]
file_import = await basket_.get_file(file_id) file_import = await basket_.get_file(file_id)
if file_import is None: if file_import is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}') raise HTTPException(status.HTTP_404_NOT_FOUND, f"No import file id {file_id}")
await basket_.delete_file(file_import) await basket_.delete_file(file_import)

90
src/gisaf/baskets.py
View file

@ -15,8 +15,13 @@ from fastapi import UploadFile
from gisaf.config import conf from gisaf.config import conf
from gisaf.database import db_session from gisaf.database import db_session
from gisaf.importers import (Importer, RawSurveyImporter, GeoDataImporter, from gisaf.importers import (
LineWorkImporter, ImportError) Importer,
RawSurveyImporter,
GeoDataImporter,
LineWorkImporter,
ImportError,
)
from gisaf.models.admin import FileImport, BasketImportResult from gisaf.models.admin import FileImport, BasketImportResult
from gisaf.models.authentication import UserRead from gisaf.models.authentication import UserRead
from gisaf.models.survey import Surveyor, Equipment from gisaf.models.survey import Surveyor, Equipment
@ -24,7 +29,7 @@ from gisaf.models.project import Project
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
upload_fields_available = ['store', 'status', 'project', 'surveyor', 'equipment'] upload_fields_available = ["store", "status", "project", "surveyor", "equipment"]
class Basket: class Basket:
@ -37,11 +42,12 @@ class Basket:
The basket can have a role. In that case, it will be completely hidden from users The basket can have a role. In that case, it will be completely hidden from users
who don't have that role. who don't have that role.
""" """
name: ClassVar[str] name: ClassVar[str]
importer_class: Type[Importer] | None = None importer_class: Type[Importer] | None = None
importer: Importer importer: Importer
_custom_module: str | None = None _custom_module: str | None = None
columns: list[str] = ['name', 'time', 'import', 'delete'] columns: list[str] = ["name", "time", "import", "delete"]
upload_fields: list[str] = [] upload_fields: list[str] = []
role: str | None = None role: str | None = None
@ -65,7 +71,9 @@ class Basket:
async def get_files(self) -> list[FileImport]: async def get_files(self) -> list[FileImport]:
async with db_session() as session: async with db_session() as session:
data = await session.exec(select(FileImport).where(FileImport.basket==self.name)) data = await session.exec(
select(FileImport).where(FileImport.basket == self.name)
)
return data.all() # type: ignore return data.all() # type: ignore
# async def get_files_df(self, convert_path=False): # async def get_files_df(self, convert_path=False):
@ -106,11 +114,15 @@ class Basket:
async def get_file(self, file_id: int) -> FileImport | None: async def get_file(self, file_id: int) -> FileImport | None:
async with db_session() as session: async with db_session() as session:
query = select(FileImport).where(FileImport.id==file_id).options( query = (
select(FileImport)
.where(FileImport.id == file_id)
.options(
joinedload(FileImport.project), # type: ignore joinedload(FileImport.project), # type: ignore
joinedload(FileImport.surveyor), # type: ignore joinedload(FileImport.surveyor), # type: ignore
joinedload(FileImport.equipment), # type: ignore joinedload(FileImport.equipment), # type: ignore
) )
)
res = await session.exec(query) res = await session.exec(query)
try: try:
file = res.one() file = res.one()
@ -147,27 +159,34 @@ class Basket:
await session.delete(file) await session.delete(file)
await session.commit() await session.commit()
async def import_file(self, file_import: FileImport, async def import_file(
dry_run=True, **kwargs) -> BasketImportResult: self, file_import: FileImport, dry_run=True, **kwargs
) -> BasketImportResult:
""" """
Import the file by calling the basket's importer's do_import. Import the file by calling the basket's importer's do_import.
Time stamp the FileImport. Time stamp the FileImport.
Return a BasketImportResult ObjectType Return a BasketImportResult ObjectType
""" """
if not hasattr(self, 'importer'): if not hasattr(self, "importer"):
return BasketImportResult( return BasketImportResult(
message=f'No import defined/required for {self.name} basket' message=f"No import defined/required for {self.name} basket"
) )
result: BasketImportResult result: BasketImportResult
try: try:
result = await self.importer.do_import(file_import, dry_run=dry_run, **kwargs) result = await self.importer.do_import(
file_import, dry_run=dry_run, **kwargs
)
except ImportError as err: except ImportError as err:
raise err raise err
except Exception as err: except Exception as err:
logger.exception(err) logger.exception(err)
raise ImportError(f'Unexpected import error (details in the Gisaf logs): {err}') raise ImportError(
f"Unexpected import error (details in the Gisaf logs): {err}"
)
if not isinstance(result, BasketImportResult): if not isinstance(result, BasketImportResult):
raise ImportError('Import error: the importer did not return a BasketImportResult') raise ImportError(
"Import error: the importer did not return a BasketImportResult"
)
# if import_result: # if import_result:
# if isinstance(import_result, (tuple, list)): # if isinstance(import_result, (tuple, list)):
# assert len(import_result) >= 2, \ # assert len(import_result) >= 2, \
@ -182,8 +201,6 @@ class Basket:
# result = BasketImportResult(message=import_result) # result = BasketImportResult(message=import_result)
# else: # else:
# result = BasketImportResult(message='Import successful.') # result = BasketImportResult(message='Import successful.')
if file_import.time is None:
raise ImportError('No time found in file import')
if dry_run: if dry_run:
result.time = file_import.time result.time = file_import.time
else: else:
@ -194,7 +211,8 @@ class Basket:
await session.commit() await session.commit()
return result return result
async def add_file(self, async def add_file(
self,
file: UploadFile, file: UploadFile,
user: UserRead, user: UserRead,
auto_import: bool = False, auto_import: bool = False,
@ -215,13 +233,13 @@ class Basket:
# assert part.name == 'file' # assert part.name == 'file'
## Save file on filesystem ## Save file on filesystem
if file.filename is None: if file.filename is None:
raise ImportError('No file name') raise ImportError("No file name")
path = AsyncPath(self.base_dir) / file.filename path = AsyncPath(self.base_dir) / file.filename
## Eventually create the directory ## Eventually create the directory
await path.parent.mkdir(parents=True, exist_ok=True) await path.parent.mkdir(parents=True, exist_ok=True)
file_content = await file.read() file_content = await file.read()
md5sum = md5(file_content).hexdigest() md5sum = md5(file_content).hexdigest()
async with path.open('wb') as f: async with path.open("wb") as f:
## No way to use async to stream the file content to write it? ## No way to use async to stream the file content to write it?
await f.write(file_content) await f.write(file_content)
async with db_session() as session: async with db_session() as session:
@ -241,46 +259,48 @@ class Basket:
await session.commit() await session.commit()
await session.refresh(fileImportRecord) await session.refresh(fileImportRecord)
if fileImportRecord.id is None: if fileImportRecord.id is None:
raise ImportError('Cannot save (no fileImportRecord.id)') raise ImportError("Cannot save (no fileImportRecord.id)")
## Eventually do import ## Eventually do import
basket_import_result = BasketImportResult( basket_import_result = BasketImportResult(fileImport=fileImportRecord)
fileImport=fileImportRecord
)
if auto_import: if auto_import:
if user.has_role('reviewer'): if user.has_role("reviewer"):
## Get the record from DB, for compatibility with import_file ## Get the record from DB, for compatibility with import_file
file_import_record = await self.get_file(fileImportRecord.id) file_import_record = await self.get_file(fileImportRecord.id)
if file_import_record is None: if file_import_record is None:
basket_import_result.message = "Cannot import: file not found" basket_import_result.message = "Cannot import: file not found"
else: else:
try: try:
basket_import_result = await self.import_file(file_import_record, dry_run) basket_import_result = await self.import_file(
file_import_record, dry_run
)
except ImportError as err: except ImportError as err:
basket_import_result.message=f'Error: {err.args[0]}' basket_import_result.message = f"Error: {err.args[0]}"
else: else:
basket_import_result.message="Cannot import: only a reviewer can do that" basket_import_result.message = (
"Cannot import: only a reviewer can do that"
)
return basket_import_result return basket_import_result
class MiscGeomBasket(Basket): class MiscGeomBasket(Basket):
name = 'Misc geo file' name = "Misc geo file"
importer_class = GeoDataImporter importer_class = GeoDataImporter
columns = ['name', 'time', 'status', 'store', 'import', 'delete'] columns = ["name", "time", "status", "store", "import", "delete"]
upload_fields = ['store_misc', 'status'] upload_fields = ["store_misc", "status"]
class LineWorkBasket(Basket): class LineWorkBasket(Basket):
name = 'Line work' name = "Line work"
importer_class = LineWorkImporter importer_class = LineWorkImporter
columns = ['name', 'time', 'status', 'store', 'project', 'import', 'delete'] columns = ["name", "time", "status", "store", "project", "import", "delete"]
upload_fields = ['store_line_work', 'project', 'status'] upload_fields = ["store_line_work", "project", "status"]
class SurveyBasket(Basket): class SurveyBasket(Basket):
name = 'Survey' name = "Survey"
importer_class = RawSurveyImporter importer_class = RawSurveyImporter
columns = ['name', 'time', 'project', 'surveyor', 'equipment', 'import', 'delete'] columns = ["name", "time", "project", "surveyor", "equipment", "import", "delete"]
upload_fields = ['project', 'surveyor', 'equipment'] upload_fields = ["project", "surveyor", "equipment"]
standard_baskets = ( standard_baskets = (