Admin: fix issue with import and error handling
This commit is contained in:
phil
parent
0bf74b2bba
commit
5a63892640
2 changed files with 98 additions and 72 deletions
|
|
@ -1,13 +1,13 @@
|
|||
import logging
|
||||
|
||||
from fastapi import (Depends, APIRouter, HTTPException, status, responses,
|
||||
UploadFile)
|
||||
from fastapi import Depends, APIRouter, HTTPException, status, responses, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from gisaf.models.admin import AdminBasket, BasketImportResult, BasketNameOnly
|
||||
from gisaf.models.authentication import User, UserRead
|
||||
from gisaf.security import get_current_active_user
|
||||
from gisaf.admin import manager
|
||||
from gisaf.importers import ImportError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -17,20 +17,22 @@ api = APIRouter(
|
|||
responses={404: {"description": "Not found"}},
|
||||
)
|
||||
|
||||
@api.get('/basket')
|
||||
|
||||
@api.get("/basket")
|
||||
async def get_baskets(
|
||||
user: User = Depends(get_current_active_user),
|
||||
) -> list[BasketNameOnly]:
|
||||
) -> list[BasketNameOnly]:
|
||||
return [
|
||||
BasketNameOnly(name=name)
|
||||
for name, basket in (await manager.baskets_for_role(user)).items()
|
||||
]
|
||||
|
||||
@api.get('/basket/{name}')
|
||||
|
||||
@api.get("/basket/{name}")
|
||||
async def get_basket(
|
||||
name: str,
|
||||
user: User = Depends(get_current_active_user),
|
||||
) -> AdminBasket:
|
||||
) -> AdminBasket:
|
||||
basket = manager.baskets[name]
|
||||
if basket.role and not user.has_role(basket.role):
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED)
|
||||
|
|
@ -43,7 +45,8 @@ async def get_basket(
|
|||
# projects=getattr(basket, 'projects', None)
|
||||
)
|
||||
|
||||
@api.post('/basket/upload/{name}')
|
||||
|
||||
@api.post("/basket/upload/{name}")
|
||||
async def upload_basket_file(
|
||||
name: str,
|
||||
file: UploadFile,
|
||||
|
|
@ -66,59 +69,62 @@ async def upload_basket_file(
|
|||
)
|
||||
return fileItem
|
||||
|
||||
@api.get('/basket/download/{name}/{file_id}/{file_name}')
|
||||
|
||||
@api.get("/basket/download/{name}/{file_id}/{file_name}")
|
||||
async def download_basket_file(
|
||||
name: str,
|
||||
file_id: int,
|
||||
file_name: str,
|
||||
user: User = Depends(get_current_active_user),
|
||||
) -> FileResponse:
|
||||
) -> FileResponse:
|
||||
try:
|
||||
basket = manager.baskets[name]
|
||||
except KeyError:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No basket named {name}')
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, f"No basket named {name}")
|
||||
if basket.role:
|
||||
if not user.has_role(basket.role):
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED)
|
||||
file_record = await basket.get_file(file_id)
|
||||
if file_record is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}')
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, f"No import file id {file_id}")
|
||||
abs_path = basket.base_dir / file_record.path
|
||||
if not abs_path.exists():
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, f'File {file_record.name} not found')
|
||||
raise HTTPException(
|
||||
status.HTTP_404_NOT_FOUND, f"File {file_record.name} not found"
|
||||
)
|
||||
return FileResponse(abs_path)
|
||||
|
||||
@api.get('/basket/import/{basket}/{file_id}')
|
||||
|
||||
@api.get("/basket/import/{basket}/{file_id}")
|
||||
async def import_basket_file(
|
||||
basket: str,
|
||||
file_id: int,
|
||||
dryRun: bool = False,
|
||||
user: User = Depends(get_current_active_user),
|
||||
) -> BasketImportResult:
|
||||
if not (user and user.has_role('reviewer')):
|
||||
) -> BasketImportResult:
|
||||
if not (user and user.has_role("reviewer")):
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED)
|
||||
basket_ = manager.baskets[basket]
|
||||
file_import = await basket_.get_file(file_id)
|
||||
if file_import is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}')
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, f"No import file id {file_id}")
|
||||
try:
|
||||
result = await basket_.import_file(file_import, dryRun)
|
||||
## FIXME: shouldn't it be AdminImportError?
|
||||
except ImportError as err:
|
||||
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, err)
|
||||
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, str(err))
|
||||
return result
|
||||
|
||||
|
||||
@api.get('/basket/delete/{basket}/{file_id}')
|
||||
@api.get("/basket/delete/{basket}/{file_id}")
|
||||
async def delete_basket_file(
|
||||
basket: str,
|
||||
file_id: int,
|
||||
user: User = Depends(get_current_active_user),
|
||||
) -> None:
|
||||
if not (user and user.has_role('reviewer')):
|
||||
) -> None:
|
||||
if not (user and user.has_role("reviewer")):
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED)
|
||||
basket_ = manager.baskets[basket]
|
||||
file_import = await basket_.get_file(file_id)
|
||||
if file_import is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}')
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, f"No import file id {file_id}")
|
||||
await basket_.delete_file(file_import)
|
||||
|
|
@ -15,8 +15,13 @@ from fastapi import UploadFile
|
|||
|
||||
from gisaf.config import conf
|
||||
from gisaf.database import db_session
|
||||
from gisaf.importers import (Importer, RawSurveyImporter, GeoDataImporter,
|
||||
LineWorkImporter, ImportError)
|
||||
from gisaf.importers import (
|
||||
Importer,
|
||||
RawSurveyImporter,
|
||||
GeoDataImporter,
|
||||
LineWorkImporter,
|
||||
ImportError,
|
||||
)
|
||||
from gisaf.models.admin import FileImport, BasketImportResult
|
||||
from gisaf.models.authentication import UserRead
|
||||
from gisaf.models.survey import Surveyor, Equipment
|
||||
|
|
@ -24,7 +29,7 @@ from gisaf.models.project import Project
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
upload_fields_available = ['store', 'status', 'project', 'surveyor', 'equipment']
|
||||
upload_fields_available = ["store", "status", "project", "surveyor", "equipment"]
|
||||
|
||||
|
||||
class Basket:
|
||||
|
|
@ -37,11 +42,12 @@ class Basket:
|
|||
The basket can have a role. In that case, it will be completely hidden from users
|
||||
who don't have that role.
|
||||
"""
|
||||
|
||||
name: ClassVar[str]
|
||||
importer_class: Type[Importer] | None = None
|
||||
importer: Importer
|
||||
_custom_module: str | None = None
|
||||
columns: list[str] = ['name', 'time', 'import', 'delete']
|
||||
columns: list[str] = ["name", "time", "import", "delete"]
|
||||
upload_fields: list[str] = []
|
||||
role: str | None = None
|
||||
|
||||
|
|
@ -65,7 +71,9 @@ class Basket:
|
|||
|
||||
async def get_files(self) -> list[FileImport]:
|
||||
async with db_session() as session:
|
||||
data = await session.exec(select(FileImport).where(FileImport.basket==self.name))
|
||||
data = await session.exec(
|
||||
select(FileImport).where(FileImport.basket == self.name)
|
||||
)
|
||||
return data.all() # type: ignore
|
||||
|
||||
# async def get_files_df(self, convert_path=False):
|
||||
|
|
@ -106,10 +114,14 @@ class Basket:
|
|||
|
||||
async def get_file(self, file_id: int) -> FileImport | None:
|
||||
async with db_session() as session:
|
||||
query = select(FileImport).where(FileImport.id==file_id).options(
|
||||
joinedload(FileImport.project), # type: ignore
|
||||
joinedload(FileImport.surveyor), # type: ignore
|
||||
joinedload(FileImport.equipment), # type: ignore
|
||||
query = (
|
||||
select(FileImport)
|
||||
.where(FileImport.id == file_id)
|
||||
.options(
|
||||
joinedload(FileImport.project), # type: ignore
|
||||
joinedload(FileImport.surveyor), # type: ignore
|
||||
joinedload(FileImport.equipment), # type: ignore
|
||||
)
|
||||
)
|
||||
res = await session.exec(query)
|
||||
try:
|
||||
|
|
@ -138,36 +150,43 @@ class Basket:
|
|||
|
||||
async def delete_file(self, file: FileImport):
|
||||
if file.dir:
|
||||
path = self.base_dir/file.dir/file.name
|
||||
path = self.base_dir / file.dir / file.name
|
||||
else:
|
||||
path = self.base_dir/file.name
|
||||
path = self.base_dir / file.name
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
async with db_session() as session:
|
||||
await session.delete(file)
|
||||
await session.commit()
|
||||
|
||||
async def import_file(self, file_import: FileImport,
|
||||
dry_run=True, **kwargs) -> BasketImportResult:
|
||||
async def import_file(
|
||||
self, file_import: FileImport, dry_run=True, **kwargs
|
||||
) -> BasketImportResult:
|
||||
"""
|
||||
Import the file by calling the basket's importer's do_import.
|
||||
Time stamp the FileImport.
|
||||
Return a BasketImportResult ObjectType
|
||||
"""
|
||||
if not hasattr(self, 'importer'):
|
||||
if not hasattr(self, "importer"):
|
||||
return BasketImportResult(
|
||||
message=f'No import defined/required for {self.name} basket'
|
||||
message=f"No import defined/required for {self.name} basket"
|
||||
)
|
||||
result: BasketImportResult
|
||||
try:
|
||||
result = await self.importer.do_import(file_import, dry_run=dry_run, **kwargs)
|
||||
result = await self.importer.do_import(
|
||||
file_import, dry_run=dry_run, **kwargs
|
||||
)
|
||||
except ImportError as err:
|
||||
raise err
|
||||
except Exception as err:
|
||||
logger.exception(err)
|
||||
raise ImportError(f'Unexpected import error (details in the Gisaf logs): {err}')
|
||||
raise ImportError(
|
||||
f"Unexpected import error (details in the Gisaf logs): {err}"
|
||||
)
|
||||
if not isinstance(result, BasketImportResult):
|
||||
raise ImportError('Import error: the importer did not return a BasketImportResult')
|
||||
raise ImportError(
|
||||
"Import error: the importer did not return a BasketImportResult"
|
||||
)
|
||||
# if import_result:
|
||||
# if isinstance(import_result, (tuple, list)):
|
||||
# assert len(import_result) >= 2, \
|
||||
|
|
@ -182,8 +201,6 @@ class Basket:
|
|||
# result = BasketImportResult(message=import_result)
|
||||
# else:
|
||||
# result = BasketImportResult(message='Import successful.')
|
||||
if file_import.time is None:
|
||||
raise ImportError('No time found in file import')
|
||||
if dry_run:
|
||||
result.time = file_import.time
|
||||
else:
|
||||
|
|
@ -194,15 +211,16 @@ class Basket:
|
|||
await session.commit()
|
||||
return result
|
||||
|
||||
async def add_file(self,
|
||||
file: UploadFile,
|
||||
user: UserRead,
|
||||
auto_import: bool = False,
|
||||
dry_run: bool = False,
|
||||
project_id: int | None = None,
|
||||
surveyor_id: int | None = None,
|
||||
equipment_id: int | None = None,
|
||||
) -> BasketImportResult:
|
||||
async def add_file(
|
||||
self,
|
||||
file: UploadFile,
|
||||
user: UserRead,
|
||||
auto_import: bool = False,
|
||||
dry_run: bool = False,
|
||||
project_id: int | None = None,
|
||||
surveyor_id: int | None = None,
|
||||
equipment_id: int | None = None,
|
||||
) -> BasketImportResult:
|
||||
"""
|
||||
File upload to basket.
|
||||
Typically called through an http POST view handler.
|
||||
|
|
@ -215,13 +233,13 @@ class Basket:
|
|||
# assert part.name == 'file'
|
||||
## Save file on filesystem
|
||||
if file.filename is None:
|
||||
raise ImportError('No file name')
|
||||
raise ImportError("No file name")
|
||||
path = AsyncPath(self.base_dir) / file.filename
|
||||
## Eventually create the directory
|
||||
await path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_content = await file.read()
|
||||
md5sum = md5(file_content).hexdigest()
|
||||
async with path.open('wb') as f:
|
||||
async with path.open("wb") as f:
|
||||
## No way to use async to stream the file content to write it?
|
||||
await f.write(file_content)
|
||||
async with db_session() as session:
|
||||
|
|
@ -241,46 +259,48 @@ class Basket:
|
|||
await session.commit()
|
||||
await session.refresh(fileImportRecord)
|
||||
if fileImportRecord.id is None:
|
||||
raise ImportError('Cannot save (no fileImportRecord.id)')
|
||||
raise ImportError("Cannot save (no fileImportRecord.id)")
|
||||
## Eventually do import
|
||||
basket_import_result = BasketImportResult(
|
||||
fileImport=fileImportRecord
|
||||
)
|
||||
basket_import_result = BasketImportResult(fileImport=fileImportRecord)
|
||||
if auto_import:
|
||||
if user.has_role('reviewer'):
|
||||
if user.has_role("reviewer"):
|
||||
## Get the record from DB, for compatibility with import_file
|
||||
file_import_record = await self.get_file(fileImportRecord.id)
|
||||
if file_import_record is None:
|
||||
basket_import_result.message="Cannot import: file not found"
|
||||
basket_import_result.message = "Cannot import: file not found"
|
||||
else:
|
||||
try:
|
||||
basket_import_result = await self.import_file(file_import_record, dry_run)
|
||||
basket_import_result = await self.import_file(
|
||||
file_import_record, dry_run
|
||||
)
|
||||
except ImportError as err:
|
||||
basket_import_result.message=f'Error: {err.args[0]}'
|
||||
basket_import_result.message = f"Error: {err.args[0]}"
|
||||
else:
|
||||
basket_import_result.message="Cannot import: only a reviewer can do that"
|
||||
basket_import_result.message = (
|
||||
"Cannot import: only a reviewer can do that"
|
||||
)
|
||||
return basket_import_result
|
||||
|
||||
|
||||
class MiscGeomBasket(Basket):
|
||||
name = 'Misc geo file'
|
||||
name = "Misc geo file"
|
||||
importer_class = GeoDataImporter
|
||||
columns = ['name', 'time', 'status', 'store', 'import', 'delete']
|
||||
upload_fields = ['store_misc', 'status']
|
||||
columns = ["name", "time", "status", "store", "import", "delete"]
|
||||
upload_fields = ["store_misc", "status"]
|
||||
|
||||
|
||||
class LineWorkBasket(Basket):
|
||||
name = 'Line work'
|
||||
name = "Line work"
|
||||
importer_class = LineWorkImporter
|
||||
columns = ['name', 'time', 'status', 'store', 'project', 'import', 'delete']
|
||||
upload_fields = ['store_line_work', 'project', 'status']
|
||||
columns = ["name", "time", "status", "store", "project", "import", "delete"]
|
||||
upload_fields = ["store_line_work", "project", "status"]
|
||||
|
||||
|
||||
class SurveyBasket(Basket):
|
||||
name = 'Survey'
|
||||
name = "Survey"
|
||||
importer_class = RawSurveyImporter
|
||||
columns = ['name', 'time', 'project', 'surveyor', 'equipment', 'import', 'delete']
|
||||
upload_fields = ['project', 'surveyor', 'equipment']
|
||||
columns = ["name", "time", "project", "surveyor", "equipment", "import", "delete"]
|
||||
upload_fields = ["project", "surveyor", "equipment"]
|
||||
|
||||
|
||||
standard_baskets = (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue