Admin basket: update/fix file import, download, delete
remove useless AdminBasketFile model definition cleanups, typings
This commit is contained in:
parent
6139f49aae
commit
c613fd35d9
5 changed files with 123 additions and 136 deletions
|
|
@ -1,10 +1,8 @@
|
|||
from pathlib import Path
|
||||
from aiopath import AsyncPath
|
||||
from collections import defaultdict
|
||||
from json import loads
|
||||
from datetime import datetime
|
||||
import logging
|
||||
from typing import ClassVar
|
||||
from typing import ClassVar, Type
|
||||
from hashlib import md5
|
||||
|
||||
# from aiohttp_security import check_permission
|
||||
# from aiohttp.multipart import MultipartReader
|
||||
|
|
@ -17,8 +15,9 @@ from fastapi import UploadFile
|
|||
|
||||
from gisaf.config import conf
|
||||
from gisaf.database import db_session
|
||||
from gisaf.importers import RawSurveyImporter, GeoDataImporter, LineWorkImporter, ImportError
|
||||
from gisaf.models.admin import FileImport, AdminBasketFile, BasketImportResult
|
||||
from gisaf.importers import (Importer, RawSurveyImporter, GeoDataImporter,
|
||||
LineWorkImporter, ImportError)
|
||||
from gisaf.models.admin import FileImport, BasketImportResult
|
||||
from gisaf.models.authentication import UserRead
|
||||
from gisaf.models.survey import Surveyor, Equipment
|
||||
from gisaf.models.project import Project
|
||||
|
|
@ -39,11 +38,12 @@ class Basket:
|
|||
who don't have that role.
|
||||
"""
|
||||
name: ClassVar[str]
|
||||
importer_class = None
|
||||
_custom_module = None
|
||||
importer_class: Type[Importer]
|
||||
importer: Importer
|
||||
_custom_module: str | None = None
|
||||
columns: list[str] = ['name', 'time', 'import', 'delete']
|
||||
upload_fields: list[str] = []
|
||||
role = None
|
||||
role: str | None = None
|
||||
|
||||
def __init__(self):
|
||||
self.base_dir = Path(conf.admin.basket.base_dir) / self.name
|
||||
|
|
@ -66,7 +66,7 @@ class Basket:
|
|||
async def get_files(self) -> list[FileImport]:
|
||||
async with db_session() as session:
|
||||
data = await session.exec(select(FileImport).where(FileImport.basket==self.name))
|
||||
return data.all()
|
||||
return data.all() # type: ignore
|
||||
|
||||
# async def get_files_df(self, convert_path=False):
|
||||
# """
|
||||
|
|
@ -107,9 +107,9 @@ class Basket:
|
|||
async def get_file(self, file_id: int) -> FileImport | None:
|
||||
async with db_session() as session:
|
||||
query = select(FileImport).where(FileImport.id==file_id).options(
|
||||
joinedload(FileImport.project),
|
||||
joinedload(FileImport.surveyor),
|
||||
joinedload(FileImport.equipment),
|
||||
joinedload(FileImport.project), # type: ignore
|
||||
joinedload(FileImport.surveyor), # type: ignore
|
||||
joinedload(FileImport.equipment), # type: ignore
|
||||
)
|
||||
res = await session.exec(query)
|
||||
try:
|
||||
|
|
@ -136,17 +136,19 @@ class Basket:
|
|||
# file.name = file['name']
|
||||
# return file
|
||||
|
||||
async def delete_file(self, id):
|
||||
file = await FileImport.get(id)
|
||||
async def delete_file(self, file: FileImport):
|
||||
if file.dir:
|
||||
path = self.base_dir/file.dir/file.name
|
||||
else:
|
||||
path = self.base_dir/file.name
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
await file.delete()
|
||||
async with db_session() as session:
|
||||
await session.delete(file)
|
||||
await session.commit()
|
||||
|
||||
async def import_file(self, file_import: FileImport, dry_run=True, **kwargs) -> BasketImportResult:
|
||||
async def import_file(self, file_import: FileImport,
|
||||
dry_run=True, **kwargs) -> BasketImportResult:
|
||||
"""
|
||||
Import the file by calling the basket's importer's do_import.
|
||||
Time stamp the FileImport.
|
||||
|
|
@ -164,7 +166,6 @@ class Basket:
|
|||
except Exception as err:
|
||||
logger.exception(err)
|
||||
raise ImportError(f'Unexpected import error (details in the Gisaf logs): {err}')
|
||||
|
||||
if not isinstance(result, BasketImportResult):
|
||||
raise ImportError('Import error: the importer did not return a BasketImportResult')
|
||||
# if import_result:
|
||||
|
|
@ -181,9 +182,11 @@ class Basket:
|
|||
# result = BasketImportResult(message=import_result)
|
||||
# else:
|
||||
# result = BasketImportResult(message='Import successful.')
|
||||
if file_import.time is None:
|
||||
raise ImportError('No time found in file import')
|
||||
if dry_run:
|
||||
result.time = file_import.time
|
||||
if not dry_run:
|
||||
else:
|
||||
## Save time stamp
|
||||
async with db_session() as session:
|
||||
file_import.time = result.time
|
||||
|
|
@ -191,7 +194,15 @@ class Basket:
|
|||
await session.commit()
|
||||
return result
|
||||
|
||||
async def add_files(self, file: UploadFile, user: UserRead):
|
||||
async def add_file(self,
|
||||
file: UploadFile,
|
||||
user: UserRead,
|
||||
auto_import: bool = False,
|
||||
dry_run: bool = False,
|
||||
project_id: int | None = None,
|
||||
surveyor_id: int | None = None,
|
||||
equipment_id: int | None = None,
|
||||
) -> BasketImportResult:
|
||||
"""
|
||||
File upload to basket.
|
||||
Typically called through an http POST view handler.
|
||||
|
|
@ -203,89 +214,52 @@ class Basket:
|
|||
## TODO: check if file already exists
|
||||
# assert part.name == 'file'
|
||||
## Save file on filesystem
|
||||
if file.filename is None:
|
||||
raise ImportError('No file name')
|
||||
path = AsyncPath(self.base_dir) / file.filename
|
||||
## Eventually create the directory
|
||||
await path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_content = await file.read()
|
||||
md5sum = md5(file_content).hexdigest()
|
||||
async with path.open('wb') as f:
|
||||
## No way to use async to stream the file content to write it?
|
||||
await f.write(await file.read())
|
||||
|
||||
## Read other parts
|
||||
# parts = defaultdict(None)
|
||||
# while True:
|
||||
# part = await reader.next()
|
||||
# if not part:
|
||||
# break
|
||||
# value = (await part.read()).decode()
|
||||
# if value != 'null':
|
||||
# parts[part.name] = value
|
||||
|
||||
## Find ids of project, surveyor, equipment
|
||||
if 'project' in parts:
|
||||
project_id = (await Project.query.where(Project.name==parts.get('project')).gino.first()).id
|
||||
else:
|
||||
project_id = None
|
||||
if 'surveyor' in parts:
|
||||
surveyor_id = (await Surveyor.query.where(Surveyor.name==parts.get('surveyor')).gino.first()).id
|
||||
else:
|
||||
surveyor_id = None
|
||||
if 'equipment' in parts:
|
||||
equipment_id = (await Equipment.query.where(Equipment.name==parts.get('equipment')).gino.first()).id
|
||||
else:
|
||||
equipment_id = None
|
||||
|
||||
## Save FileImport record
|
||||
store_keys = [store_key for store_key in parts.keys()
|
||||
if store_key.startswith('store')]
|
||||
if len(store_keys) == 1:
|
||||
store_type_name = parts[store_keys[0]]
|
||||
else:
|
||||
store_type_name = None
|
||||
fileImportRecord = await FileImport(
|
||||
name=file.file_name,
|
||||
dir=parts.get('dir', '.'),
|
||||
basket=self.name,
|
||||
project_id=project_id,
|
||||
surveyor_id=surveyor_id,
|
||||
equipment_id=equipment_id,
|
||||
store=store_type_name,
|
||||
status=parts.get('status', None),
|
||||
).create()
|
||||
fileImportRecord.path = self.base_dir/fileImportRecord.dir/fileImportRecord.name
|
||||
|
||||
admin_basket_file = AdminBasketFile(
|
||||
id=fileImportRecord.id,
|
||||
name=file.file_name,
|
||||
status=parts.get('status'),
|
||||
store=store_type_name,
|
||||
project=parts.get('project'),
|
||||
surveyor=parts.get('surveyor'),
|
||||
equipment=parts.get('equipment'),
|
||||
)
|
||||
|
||||
await f.write(file_content)
|
||||
async with db_session() as session:
|
||||
fileImportRecord = FileImport(
|
||||
name=file.filename,
|
||||
md5=md5sum,
|
||||
dir=str(self.base_dir),
|
||||
basket=self.name,
|
||||
project_id=project_id,
|
||||
surveyor_id=surveyor_id,
|
||||
equipment_id=equipment_id,
|
||||
# store=store_type_name,
|
||||
# status=parts.get('status', None),
|
||||
)
|
||||
# fileImportRecord.path = self.base_dir / fileImportRecord.dir / fileImportRecord.name
|
||||
session.add(fileImportRecord)
|
||||
await session.commit()
|
||||
await session.refresh(fileImportRecord)
|
||||
if fileImportRecord.id is None:
|
||||
raise ImportError('Cannot save (no fileImportRecord.id)')
|
||||
## Eventually do import
|
||||
import_result = None
|
||||
if loads(parts['autoImport']):
|
||||
## Get the record from DB with Pandas, for compatibility with import_file
|
||||
file_import_record = await self.get_file(fileImportRecord.id)
|
||||
try:
|
||||
await check_permission(request, 'reviewer')
|
||||
except HTTPUnauthorized as err:
|
||||
basket_import_result = BasketImportResult(
|
||||
message="Cannot import: only a reviewer can do that"
|
||||
)
|
||||
basket_import_result = BasketImportResult(
|
||||
fileImport=fileImportRecord
|
||||
)
|
||||
if auto_import:
|
||||
if user.has_role('reviewer'):
|
||||
## Get the record from DB, for compatibility with import_file
|
||||
file_import_record = await self.get_file(fileImportRecord.id)
|
||||
if file_import_record is None:
|
||||
basket_import_result.message="Cannot import: file not found"
|
||||
else:
|
||||
try:
|
||||
basket_import_result = await self.import_file(file_import_record, dry_run)
|
||||
except ImportError as err:
|
||||
basket_import_result.message=f'Error: {err.args[0]}'
|
||||
else:
|
||||
dry_run = parts.get('dry_run', False)
|
||||
try:
|
||||
basket_import_result = await self.import_file(file_import_record, dry_run)
|
||||
except ImportError as err:
|
||||
basket_import_result = BasketImportResult(
|
||||
message=f'Error: {err.args[0]}'
|
||||
)
|
||||
|
||||
admin_basket_file.import_result = basket_import_result
|
||||
|
||||
return admin_basket_file
|
||||
basket_import_result.message="Cannot import: only a reviewer can do that"
|
||||
return basket_import_result
|
||||
|
||||
|
||||
class MiscGeomBasket(Basket):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue