Admin: fix issue with import and error handling
This commit is contained in:
phil
parent
0bf74b2bba
commit
5a63892640
2 changed files with 98 additions and 72 deletions
|
|
@ -15,8 +15,13 @@ from fastapi import UploadFile
|
|||
|
||||
from gisaf.config import conf
|
||||
from gisaf.database import db_session
|
||||
from gisaf.importers import (Importer, RawSurveyImporter, GeoDataImporter,
|
||||
LineWorkImporter, ImportError)
|
||||
from gisaf.importers import (
|
||||
Importer,
|
||||
RawSurveyImporter,
|
||||
GeoDataImporter,
|
||||
LineWorkImporter,
|
||||
ImportError,
|
||||
)
|
||||
from gisaf.models.admin import FileImport, BasketImportResult
|
||||
from gisaf.models.authentication import UserRead
|
||||
from gisaf.models.survey import Surveyor, Equipment
|
||||
|
|
@ -24,7 +29,7 @@ from gisaf.models.project import Project
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
upload_fields_available = ['store', 'status', 'project', 'surveyor', 'equipment']
|
||||
upload_fields_available = ["store", "status", "project", "surveyor", "equipment"]
|
||||
|
||||
|
||||
class Basket:
|
||||
|
|
@ -37,11 +42,12 @@ class Basket:
|
|||
The basket can have a role. In that case, it will be completely hidden from users
|
||||
who don't have that role.
|
||||
"""
|
||||
|
||||
name: ClassVar[str]
|
||||
importer_class: Type[Importer] | None = None
|
||||
importer: Importer
|
||||
_custom_module: str | None = None
|
||||
columns: list[str] = ['name', 'time', 'import', 'delete']
|
||||
columns: list[str] = ["name", "time", "import", "delete"]
|
||||
upload_fields: list[str] = []
|
||||
role: str | None = None
|
||||
|
||||
|
|
@ -65,7 +71,9 @@ class Basket:
|
|||
|
||||
async def get_files(self) -> list[FileImport]:
|
||||
async with db_session() as session:
|
||||
data = await session.exec(select(FileImport).where(FileImport.basket==self.name))
|
||||
data = await session.exec(
|
||||
select(FileImport).where(FileImport.basket == self.name)
|
||||
)
|
||||
return data.all() # type: ignore
|
||||
|
||||
# async def get_files_df(self, convert_path=False):
|
||||
|
|
@ -106,10 +114,14 @@ class Basket:
|
|||
|
||||
async def get_file(self, file_id: int) -> FileImport | None:
|
||||
async with db_session() as session:
|
||||
query = select(FileImport).where(FileImport.id==file_id).options(
|
||||
joinedload(FileImport.project), # type: ignore
|
||||
joinedload(FileImport.surveyor), # type: ignore
|
||||
joinedload(FileImport.equipment), # type: ignore
|
||||
query = (
|
||||
select(FileImport)
|
||||
.where(FileImport.id == file_id)
|
||||
.options(
|
||||
joinedload(FileImport.project), # type: ignore
|
||||
joinedload(FileImport.surveyor), # type: ignore
|
||||
joinedload(FileImport.equipment), # type: ignore
|
||||
)
|
||||
)
|
||||
res = await session.exec(query)
|
||||
try:
|
||||
|
|
@ -138,36 +150,43 @@ class Basket:
|
|||
|
||||
async def delete_file(self, file: FileImport):
|
||||
if file.dir:
|
||||
path = self.base_dir/file.dir/file.name
|
||||
path = self.base_dir / file.dir / file.name
|
||||
else:
|
||||
path = self.base_dir/file.name
|
||||
path = self.base_dir / file.name
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
async with db_session() as session:
|
||||
await session.delete(file)
|
||||
await session.commit()
|
||||
|
||||
async def import_file(self, file_import: FileImport,
|
||||
dry_run=True, **kwargs) -> BasketImportResult:
|
||||
async def import_file(
|
||||
self, file_import: FileImport, dry_run=True, **kwargs
|
||||
) -> BasketImportResult:
|
||||
"""
|
||||
Import the file by calling the basket's importer's do_import.
|
||||
Time stamp the FileImport.
|
||||
Return a BasketImportResult ObjectType
|
||||
"""
|
||||
if not hasattr(self, 'importer'):
|
||||
if not hasattr(self, "importer"):
|
||||
return BasketImportResult(
|
||||
message=f'No import defined/required for {self.name} basket'
|
||||
message=f"No import defined/required for {self.name} basket"
|
||||
)
|
||||
result: BasketImportResult
|
||||
try:
|
||||
result = await self.importer.do_import(file_import, dry_run=dry_run, **kwargs)
|
||||
result = await self.importer.do_import(
|
||||
file_import, dry_run=dry_run, **kwargs
|
||||
)
|
||||
except ImportError as err:
|
||||
raise err
|
||||
except Exception as err:
|
||||
logger.exception(err)
|
||||
raise ImportError(f'Unexpected import error (details in the Gisaf logs): {err}')
|
||||
raise ImportError(
|
||||
f"Unexpected import error (details in the Gisaf logs): {err}"
|
||||
)
|
||||
if not isinstance(result, BasketImportResult):
|
||||
raise ImportError('Import error: the importer did not return a BasketImportResult')
|
||||
raise ImportError(
|
||||
"Import error: the importer did not return a BasketImportResult"
|
||||
)
|
||||
# if import_result:
|
||||
# if isinstance(import_result, (tuple, list)):
|
||||
# assert len(import_result) >= 2, \
|
||||
|
|
@ -182,8 +201,6 @@ class Basket:
|
|||
# result = BasketImportResult(message=import_result)
|
||||
# else:
|
||||
# result = BasketImportResult(message='Import successful.')
|
||||
if file_import.time is None:
|
||||
raise ImportError('No time found in file import')
|
||||
if dry_run:
|
||||
result.time = file_import.time
|
||||
else:
|
||||
|
|
@ -194,15 +211,16 @@ class Basket:
|
|||
await session.commit()
|
||||
return result
|
||||
|
||||
async def add_file(self,
|
||||
file: UploadFile,
|
||||
user: UserRead,
|
||||
auto_import: bool = False,
|
||||
dry_run: bool = False,
|
||||
project_id: int | None = None,
|
||||
surveyor_id: int | None = None,
|
||||
equipment_id: int | None = None,
|
||||
) -> BasketImportResult:
|
||||
async def add_file(
|
||||
self,
|
||||
file: UploadFile,
|
||||
user: UserRead,
|
||||
auto_import: bool = False,
|
||||
dry_run: bool = False,
|
||||
project_id: int | None = None,
|
||||
surveyor_id: int | None = None,
|
||||
equipment_id: int | None = None,
|
||||
) -> BasketImportResult:
|
||||
"""
|
||||
File upload to basket.
|
||||
Typically called through an http POST view handler.
|
||||
|
|
@ -215,13 +233,13 @@ class Basket:
|
|||
# assert part.name == 'file'
|
||||
## Save file on filesystem
|
||||
if file.filename is None:
|
||||
raise ImportError('No file name')
|
||||
raise ImportError("No file name")
|
||||
path = AsyncPath(self.base_dir) / file.filename
|
||||
## Eventually create the directory
|
||||
await path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_content = await file.read()
|
||||
md5sum = md5(file_content).hexdigest()
|
||||
async with path.open('wb') as f:
|
||||
async with path.open("wb") as f:
|
||||
## No way to use async to stream the file content to write it?
|
||||
await f.write(file_content)
|
||||
async with db_session() as session:
|
||||
|
|
@ -241,46 +259,48 @@ class Basket:
|
|||
await session.commit()
|
||||
await session.refresh(fileImportRecord)
|
||||
if fileImportRecord.id is None:
|
||||
raise ImportError('Cannot save (no fileImportRecord.id)')
|
||||
raise ImportError("Cannot save (no fileImportRecord.id)")
|
||||
## Eventually do import
|
||||
basket_import_result = BasketImportResult(
|
||||
fileImport=fileImportRecord
|
||||
)
|
||||
basket_import_result = BasketImportResult(fileImport=fileImportRecord)
|
||||
if auto_import:
|
||||
if user.has_role('reviewer'):
|
||||
if user.has_role("reviewer"):
|
||||
## Get the record from DB, for compatibility with import_file
|
||||
file_import_record = await self.get_file(fileImportRecord.id)
|
||||
if file_import_record is None:
|
||||
basket_import_result.message="Cannot import: file not found"
|
||||
basket_import_result.message = "Cannot import: file not found"
|
||||
else:
|
||||
try:
|
||||
basket_import_result = await self.import_file(file_import_record, dry_run)
|
||||
basket_import_result = await self.import_file(
|
||||
file_import_record, dry_run
|
||||
)
|
||||
except ImportError as err:
|
||||
basket_import_result.message=f'Error: {err.args[0]}'
|
||||
basket_import_result.message = f"Error: {err.args[0]}"
|
||||
else:
|
||||
basket_import_result.message="Cannot import: only a reviewer can do that"
|
||||
basket_import_result.message = (
|
||||
"Cannot import: only a reviewer can do that"
|
||||
)
|
||||
return basket_import_result
|
||||
|
||||
|
||||
class MiscGeomBasket(Basket):
|
||||
name = 'Misc geo file'
|
||||
name = "Misc geo file"
|
||||
importer_class = GeoDataImporter
|
||||
columns = ['name', 'time', 'status', 'store', 'import', 'delete']
|
||||
upload_fields = ['store_misc', 'status']
|
||||
columns = ["name", "time", "status", "store", "import", "delete"]
|
||||
upload_fields = ["store_misc", "status"]
|
||||
|
||||
|
||||
class LineWorkBasket(Basket):
|
||||
name = 'Line work'
|
||||
name = "Line work"
|
||||
importer_class = LineWorkImporter
|
||||
columns = ['name', 'time', 'status', 'store', 'project', 'import', 'delete']
|
||||
upload_fields = ['store_line_work', 'project', 'status']
|
||||
columns = ["name", "time", "status", "store", "project", "import", "delete"]
|
||||
upload_fields = ["store_line_work", "project", "status"]
|
||||
|
||||
|
||||
class SurveyBasket(Basket):
|
||||
name = 'Survey'
|
||||
name = "Survey"
|
||||
importer_class = RawSurveyImporter
|
||||
columns = ['name', 'time', 'project', 'surveyor', 'equipment', 'import', 'delete']
|
||||
upload_fields = ['project', 'surveyor', 'equipment']
|
||||
columns = ["name", "time", "project", "surveyor", "equipment", "import", "delete"]
|
||||
upload_fields = ["project", "surveyor", "equipment"]
|
||||
|
||||
|
||||
standard_baskets = (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue