philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

Admin basket: update/fix file import, download, delete

Browse source 
remove useless AdminBasketFile model definition
cleanups, typings
This commit is contained in:
phil 2024-04-20 10:58:12 +05:30
parent 6139f49aae
commit c613fd35d9
5 changed files with 123 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

38
src/gisaf/api/admin.py
View file

@ -47,13 +47,23 @@ async def get_basket(
async def upload_basket_file( async def upload_basket_file(
name: str, name: str,
file: UploadFile, file: UploadFile,
project_id: int | None = None,
surveyor_id: int | None = None,
equipment_id: int | None = None,
auto_import: bool = False,
user: UserRead = Depends(get_current_active_user), user: UserRead = Depends(get_current_active_user),
): ) -> BasketImportResult:
try: try:
basket = manager.baskets[name] basket = manager.baskets[name]
except KeyError: except KeyError:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No basket named {name}') raise HTTPException(status.HTTP_404_NOT_FOUND, f"No basket named {name}")
fileItem = await basket.add_files(file, user) fileItem = await basket.add_file(
file,
user,
project_id=project_id,
surveyor_id=surveyor_id,
equipment_id=equipment_id,
)
return fileItem return fileItem
@api.get('/basket/download/{name}/{file_id}/{file_name}') @api.get('/basket/download/{name}/{file_id}/{file_name}')
@ -73,7 +83,10 @@ async def download_basket_file(
file_record = await basket.get_file(file_id) file_record = await basket.get_file(file_id)
if file_record is None: if file_record is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}') raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}')
return FileResponse(file_record.path) abs_path = basket.base_dir / file_record.path
if not abs_path.exists():
raise HTTPException(status.HTTP_404_NOT_FOUND, f'File {file_record.name} not found')
return FileResponse(abs_path)
@api.get('/basket/import/{basket}/{file_id}') @api.get('/basket/import/{basket}/{file_id}')
async def import_basket_file( async def import_basket_file(
@ -93,4 +106,19 @@ async def import_basket_file(
## FIXME: shouldn't it be AdminImportError? ## FIXME: shouldn't it be AdminImportError?
except ImportError as err: except ImportError as err:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, err) raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, err)
return result return result
@api.get('/basket/delete/{basket}/{file_id}')
async def delete_basket_file(
basket: str,
file_id: int,
user: User = Depends(get_current_active_user),
) -> None:
if not (user and user.has_role('reviewer')):
raise HTTPException(status.HTTP_401_UNAUTHORIZED)
basket_ = manager.baskets[basket]
file_import = await basket_.get_file(file_id)
if file_import is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}')
await basket_.delete_file(file_import)

166
src/gisaf/baskets.py
View file

@ -1,10 +1,8 @@
from pathlib import Path from pathlib import Path
from aiopath import AsyncPath from aiopath import AsyncPath
from collections import defaultdict
from json import loads
from datetime import datetime
import logging import logging
from typing import ClassVar from typing import ClassVar, Type
from hashlib import md5
# from aiohttp_security import check_permission # from aiohttp_security import check_permission
# from aiohttp.multipart import MultipartReader # from aiohttp.multipart import MultipartReader
@ -17,8 +15,9 @@ from fastapi import UploadFile
from gisaf.config import conf from gisaf.config import conf
from gisaf.database import db_session from gisaf.database import db_session
from gisaf.importers import RawSurveyImporter, GeoDataImporter, LineWorkImporter, ImportError from gisaf.importers import (Importer, RawSurveyImporter, GeoDataImporter,
from gisaf.models.admin import FileImport, AdminBasketFile, BasketImportResult LineWorkImporter, ImportError)
from gisaf.models.admin import FileImport, BasketImportResult
from gisaf.models.authentication import UserRead from gisaf.models.authentication import UserRead
from gisaf.models.survey import Surveyor, Equipment from gisaf.models.survey import Surveyor, Equipment
from gisaf.models.project import Project from gisaf.models.project import Project
@ -39,11 +38,12 @@ class Basket:
who don't have that role. who don't have that role.
""" """
name: ClassVar[str] name: ClassVar[str]
importer_class = None importer_class: Type[Importer]
_custom_module = None importer: Importer
_custom_module: str | None = None
columns: list[str] = ['name', 'time', 'import', 'delete'] columns: list[str] = ['name', 'time', 'import', 'delete']
upload_fields: list[str] = [] upload_fields: list[str] = []
role = None role: str | None = None
def __init__(self): def __init__(self):
self.base_dir = Path(conf.admin.basket.base_dir) / self.name self.base_dir = Path(conf.admin.basket.base_dir) / self.name
@ -66,7 +66,7 @@ class Basket:
async def get_files(self) -> list[FileImport]: async def get_files(self) -> list[FileImport]:
async with db_session() as session: async with db_session() as session:
data = await session.exec(select(FileImport).where(FileImport.basket==self.name)) data = await session.exec(select(FileImport).where(FileImport.basket==self.name))
return data.all() return data.all() # type: ignore
# async def get_files_df(self, convert_path=False): # async def get_files_df(self, convert_path=False):
# """ # """
@ -107,9 +107,9 @@ class Basket:
async def get_file(self, file_id: int) -> FileImport | None: async def get_file(self, file_id: int) -> FileImport | None:
async with db_session() as session: async with db_session() as session:
query = select(FileImport).where(FileImport.id==file_id).options( query = select(FileImport).where(FileImport.id==file_id).options(
joinedload(FileImport.project), joinedload(FileImport.project), # type: ignore
joinedload(FileImport.surveyor), joinedload(FileImport.surveyor), # type: ignore
joinedload(FileImport.equipment), joinedload(FileImport.equipment), # type: ignore
) )
res = await session.exec(query) res = await session.exec(query)
try: try:
@ -136,17 +136,19 @@ class Basket:
# file.name = file['name'] # file.name = file['name']
# return file # return file
async def delete_file(self, id): async def delete_file(self, file: FileImport):
file = await FileImport.get(id)
if file.dir: if file.dir:
path = self.base_dir/file.dir/file.name path = self.base_dir/file.dir/file.name
else: else:
path = self.base_dir/file.name path = self.base_dir/file.name
if path.exists(): if path.exists():
path.unlink() path.unlink()
await file.delete() async with db_session() as session:
await session.delete(file)
await session.commit()
async def import_file(self, file_import: FileImport, dry_run=True, **kwargs) -> BasketImportResult: async def import_file(self, file_import: FileImport,
dry_run=True, **kwargs) -> BasketImportResult:
""" """
Import the file by calling the basket's importer's do_import. Import the file by calling the basket's importer's do_import.
Time stamp the FileImport. Time stamp the FileImport.
@ -164,7 +166,6 @@ class Basket:
except Exception as err: except Exception as err:
logger.exception(err) logger.exception(err)
raise ImportError(f'Unexpected import error (details in the Gisaf logs): {err}') raise ImportError(f'Unexpected import error (details in the Gisaf logs): {err}')
if not isinstance(result, BasketImportResult): if not isinstance(result, BasketImportResult):
raise ImportError('Import error: the importer did not return a BasketImportResult') raise ImportError('Import error: the importer did not return a BasketImportResult')
# if import_result: # if import_result:
@ -181,9 +182,11 @@ class Basket:
# result = BasketImportResult(message=import_result) # result = BasketImportResult(message=import_result)
# else: # else:
# result = BasketImportResult(message='Import successful.') # result = BasketImportResult(message='Import successful.')
if file_import.time is None:
raise ImportError('No time found in file import')
if dry_run: if dry_run:
result.time = file_import.time result.time = file_import.time
if not dry_run: else:
## Save time stamp ## Save time stamp
async with db_session() as session: async with db_session() as session:
file_import.time = result.time file_import.time = result.time
@ -191,7 +194,15 @@ class Basket:
await session.commit() await session.commit()
return result return result
async def add_files(self, file: UploadFile, user: UserRead): async def add_file(self,
file: UploadFile,
user: UserRead,
auto_import: bool = False,
dry_run: bool = False,
project_id: int | None = None,
surveyor_id: int | None = None,
equipment_id: int | None = None,
) -> BasketImportResult:
""" """
File upload to basket. File upload to basket.
Typically called through an http POST view handler. Typically called through an http POST view handler.
@ -203,89 +214,52 @@ class Basket:
## TODO: check if file already exists ## TODO: check if file already exists
# assert part.name == 'file' # assert part.name == 'file'
## Save file on filesystem ## Save file on filesystem
if file.filename is None:
raise ImportError('No file name')
path = AsyncPath(self.base_dir) / file.filename path = AsyncPath(self.base_dir) / file.filename
## Eventually create the directory ## Eventually create the directory
await path.parent.mkdir(parents=True, exist_ok=True) await path.parent.mkdir(parents=True, exist_ok=True)
file_content = await file.read()
md5sum = md5(file_content).hexdigest()
async with path.open('wb') as f: async with path.open('wb') as f:
## No way to use async to stream the file content to write it? ## No way to use async to stream the file content to write it?
await f.write(await file.read()) await f.write(file_content)
async with db_session() as session:
## Read other parts fileImportRecord = FileImport(
# parts = defaultdict(None) name=file.filename,
# while True: md5=md5sum,
# part = await reader.next() dir=str(self.base_dir),
# if not part: basket=self.name,
# break project_id=project_id,
# value = (await part.read()).decode() surveyor_id=surveyor_id,
# if value != 'null': equipment_id=equipment_id,
# parts[part.name] = value # store=store_type_name,
# status=parts.get('status', None),
## Find ids of project, surveyor, equipment )
if 'project' in parts: # fileImportRecord.path = self.base_dir / fileImportRecord.dir / fileImportRecord.name
project_id = (await Project.query.where(Project.name==parts.get('project')).gino.first()).id session.add(fileImportRecord)
else: await session.commit()
project_id = None await session.refresh(fileImportRecord)
if 'surveyor' in parts: if fileImportRecord.id is None:
surveyor_id = (await Surveyor.query.where(Surveyor.name==parts.get('surveyor')).gino.first()).id raise ImportError('Cannot save (no fileImportRecord.id)')
else:
surveyor_id = None
if 'equipment' in parts:
equipment_id = (await Equipment.query.where(Equipment.name==parts.get('equipment')).gino.first()).id
else:
equipment_id = None
## Save FileImport record
store_keys = [store_key for store_key in parts.keys()
if store_key.startswith('store')]
if len(store_keys) == 1:
store_type_name = parts[store_keys[0]]
else:
store_type_name = None
fileImportRecord = await FileImport(
name=file.file_name,
dir=parts.get('dir', '.'),
basket=self.name,
project_id=project_id,
surveyor_id=surveyor_id,
equipment_id=equipment_id,
store=store_type_name,
status=parts.get('status', None),
).create()
fileImportRecord.path = self.base_dir/fileImportRecord.dir/fileImportRecord.name
admin_basket_file = AdminBasketFile(
id=fileImportRecord.id,
name=file.file_name,
status=parts.get('status'),
store=store_type_name,
project=parts.get('project'),
surveyor=parts.get('surveyor'),
equipment=parts.get('equipment'),
)
## Eventually do import ## Eventually do import
import_result = None basket_import_result = BasketImportResult(
if loads(parts['autoImport']): fileImport=fileImportRecord
## Get the record from DB with Pandas, for compatibility with import_file )
file_import_record = await self.get_file(fileImportRecord.id) if auto_import:
try: if user.has_role('reviewer'):
await check_permission(request, 'reviewer') ## Get the record from DB, for compatibility with import_file
except HTTPUnauthorized as err: file_import_record = await self.get_file(fileImportRecord.id)
basket_import_result = BasketImportResult( if file_import_record is None:
message="Cannot import: only a reviewer can do that" basket_import_result.message="Cannot import: file not found"
) else:
try:
basket_import_result = await self.import_file(file_import_record, dry_run)
except ImportError as err:
basket_import_result.message=f'Error: {err.args[0]}'
else: else:
dry_run = parts.get('dry_run', False) basket_import_result.message="Cannot import: only a reviewer can do that"
try: return basket_import_result
basket_import_result = await self.import_file(file_import_record, dry_run)
except ImportError as err:
basket_import_result = BasketImportResult(
message=f'Error: {err.args[0]}'
)
admin_basket_file.import_result = basket_import_result
return admin_basket_file
class MiscGeomBasket(Basket): class MiscGeomBasket(Basket):

4
src/gisaf/importers.py
View file

@ -39,7 +39,7 @@ class Importer:
The main process is executed by do_import(file) The main process is executed by do_import(file)
Subclasses should define read_file and process_df. Subclasses should define read_file and process_df.
""" """
basket = None basket = None # type hint: baskets.Basket
async def do_import(self, file_record, dry_run=False, **kwargs) -> BasketImportResult: async def do_import(self, file_record, dry_run=False, **kwargs) -> BasketImportResult:
""" """
@ -181,7 +181,7 @@ class RawSurveyImporter(Importer):
## Import to raw_survey_data table ## Import to raw_survey_data table
## PostGis specific: add SRID ## PostGis specific: add SRID
gdf['geom'] = gdf['geometry'].apply(lambda g: dumps_wkb(g, srid=conf.raw_survey['srid'], hex=True)) gdf['geom'] = gdf['geometry'].apply(lambda g: dumps_wkb(g, srid=conf.geo.raw_survey.srid, hex=True))
if not dry_run: if not dry_run:
await upsert_df(gdf, model) await upsert_df(gdf, model)

49
src/gisaf/models/admin.py
View file

@ -1,7 +1,8 @@
from pathlib import Path
import re import re
from datetime import datetime, date from datetime import datetime, date
from pydantic import BaseModel from pydantic import BaseModel, computed_field
from sqlmodel import Field, Relationship from sqlmodel import Field, Relationship
import pandas as pd import pandas as pd
@ -47,33 +48,27 @@ class FileImport(Model, table=True):
__table_args__ = gisaf_admin.table_args __table_args__ = gisaf_admin.table_args
id: int | None = Field(default=None, primary_key=True) id: int | None = Field(default=None, primary_key=True)
url: str url: str | None = None
## TODO: Deprecate FileImport.path, in favour of dir + name ## TODO: Deprecate FileImport.path, in favour of dir + name
path: str #path: str
dir: str dir: str
name: str name: str
md5: str md5: str
time: datetime time: datetime | None = None
comment: str comment: str | None = None
status: str status: str | None = None
store: str store: str | None = None
basket: str basket: str
project_id: int = Field(foreign_key=gisaf_admin.table('project.id')) project_id: int | None = Field(foreign_key=gisaf_admin.table('project.id'))
project: Project = Relationship() project: Project = Relationship()
# ALTER TABLE gisaf_admin.file_import add column project_id INT REFERENCES gisaf_admin.project; # ALTER TABLE gisaf_admin.file_import add column project_id INT REFERENCES gisaf_admin.project;
surveyor_id: int = Field(foreign_key=gisaf_survey.table('surveyor.id')) surveyor_id: int | None = Field(foreign_key=gisaf_survey.table('surveyor.id'))
surveyor: Surveyor = Relationship() surveyor: Surveyor = Relationship()
# ALTER TABLE gisaf_admin.file_import add column surveyor_id INT REFERENCES gisaf_survey.surveyor; # ALTER TABLE gisaf_admin.file_import add column surveyor_id INT REFERENCES gisaf_survey.surveyor;
equipment_id: int = Field(foreign_key=gisaf_survey.table('equipment.id')) equipment_id: int | None = Field(foreign_key=gisaf_survey.table('equipment.id'))
equipment: Equipment = Relationship() equipment: Equipment = Relationship()
# ALTER TABLE gisaf_admin.file_import add column equipment_id INT REFERENCES gisaf_survey.equipment; # ALTER TABLE gisaf_admin.file_import add column equipment_id INT REFERENCES gisaf_survey.equipment;
def __str__(self):
return f'{self.path:s} for project id {self.project_id}'
def __repr__(self):
return f'<gisaf.misc.FileImport (gisaf_admin.file_import) {self.path}>'
@classmethod @classmethod
def selectinload(cls): def selectinload(cls):
return [cls.project, cls.surveyor, cls.equipment] return [cls.project, cls.surveyor, cls.equipment]
@ -93,6 +88,10 @@ class FileImport(Model, table=True):
df['date'] = pd.to_datetime(dates[0], format='%Y-%m-%d') df['date'] = pd.to_datetime(dates[0], format='%Y-%m-%d')
return df return df
@computed_field
@property
def path(self) -> Path:
return Path(self.dir) / self.name
#def get_parent_dir(self): #def get_parent_dir(self):
# split_path = self.path.split(os_path.sep) # split_path = self.path.split(os_path.sep)
@ -158,23 +157,9 @@ class Basket(BasketNameOnly):
class BasketImportResult(BaseModel): class BasketImportResult(BaseModel):
time: datetime = Field(default_factory=datetime.now) time: datetime = Field(default_factory=datetime.now)
message: str message: str | None = None
details: dict[str, str | int | float | bool] | None = None details: dict[str, str | int | float | bool] | None = None
fileImport: FileImport | None = None
class AdminBasketFile(BaseModel):
id: int
dir: str
name: str
url: str
md5: str
time: datetime
comment: str
status: str
store: str
project: str
surveyor: str
equipment: str
import_result: str
class AdminBasket(BaseModel): class AdminBasket(BaseModel):

2
src/gisaf/registry.py
View file

@ -681,7 +681,7 @@ class ModelRegistry:
'live': 'is_live', 'live': 'is_live',
'zIndex': 'z_index', 'zIndex': 'z_index',
'gisType': 'gis_type', 'gisType': 'gis_type',
# 'type': 'mapbox_type', 'type': 'mapbox_type',
'viewableRole': 'viewable_role', 'viewableRole': 'viewable_role',
}, inplace=True }, inplace=True
) )