philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages 6 Wiki Activity Actions

Admin basket: update/fix file import, download, delete

Browse source 
remove useless AdminBasketFile model definition
cleanups, typings
This commit is contained in:
phil 2024-04-20 10:58:12 +05:30
parent 6139f49aae
commit c613fd35d9
5 changed files with 123 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

38
src/gisaf/api/admin.py
View file

@ -47,13 +47,23 @@ async def get_basket(
async def upload_basket_file(
name: str,
file: UploadFile,
project_id: int | None = None,
surveyor_id: int | None = None,
equipment_id: int | None = None,
auto_import: bool = False,
user: UserRead = Depends(get_current_active_user),
):
) -> BasketImportResult:
try:
basket = manager.baskets[name]
except KeyError:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No basket named {name}')
fileItem = await basket.add_files(file, user)
raise HTTPException(status.HTTP_404_NOT_FOUND, f"No basket named {name}")
fileItem = await basket.add_file(
file,
user,
project_id=project_id,
surveyor_id=surveyor_id,
equipment_id=equipment_id,
)
return fileItem
@api.get('/basket/download/{name}/{file_id}/{file_name}')
@ -73,7 +83,10 @@ async def download_basket_file(
file_record = await basket.get_file(file_id)
if file_record is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}')
return FileResponse(file_record.path)
abs_path = basket.base_dir / file_record.path
if not abs_path.exists():
raise HTTPException(status.HTTP_404_NOT_FOUND, f'File {file_record.name} not found')
return FileResponse(abs_path)
@api.get('/basket/import/{basket}/{file_id}')
async def import_basket_file(
@ -93,4 +106,19 @@ async def import_basket_file(
## FIXME: shouldn't it be AdminImportError?
except ImportError as err:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, err)
return result
return result
@api.get('/basket/delete/{basket}/{file_id}')
async def delete_basket_file(
basket: str,
file_id: int,
user: User = Depends(get_current_active_user),
) -> None:
if not (user and user.has_role('reviewer')):
raise HTTPException(status.HTTP_401_UNAUTHORIZED)
basket_ = manager.baskets[basket]
file_import = await basket_.get_file(file_id)
if file_import is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, f'No import file id {file_id}')
await basket_.delete_file(file_import)

166
src/gisaf/baskets.py
View file

@ -1,10 +1,8 @@
from pathlib import Path
from aiopath import AsyncPath
from collections import defaultdict
from json import loads
from datetime import datetime
import logging
from typing import ClassVar
from typing import ClassVar, Type
from hashlib import md5
# from aiohttp_security import check_permission
# from aiohttp.multipart import MultipartReader
@ -17,8 +15,9 @@ from fastapi import UploadFile
from gisaf.config import conf
from gisaf.database import db_session
from gisaf.importers import RawSurveyImporter, GeoDataImporter, LineWorkImporter, ImportError
from gisaf.models.admin import FileImport, AdminBasketFile, BasketImportResult
from gisaf.importers import (Importer, RawSurveyImporter, GeoDataImporter,
LineWorkImporter, ImportError)
from gisaf.models.admin import FileImport, BasketImportResult
from gisaf.models.authentication import UserRead
from gisaf.models.survey import Surveyor, Equipment
from gisaf.models.project import Project
@ -39,11 +38,12 @@ class Basket:
who don't have that role.
"""
name: ClassVar[str]
importer_class = None
_custom_module = None
importer_class: Type[Importer]
importer: Importer
_custom_module: str | None = None
columns: list[str] = ['name', 'time', 'import', 'delete']
upload_fields: list[str] = []
role = None
role: str | None = None
def __init__(self):
self.base_dir = Path(conf.admin.basket.base_dir) / self.name
@ -66,7 +66,7 @@ class Basket:
async def get_files(self) -> list[FileImport]:
async with db_session() as session:
data = await session.exec(select(FileImport).where(FileImport.basket==self.name))
return data.all()
return data.all() # type: ignore
# async def get_files_df(self, convert_path=False):
# """
@ -107,9 +107,9 @@ class Basket:
async def get_file(self, file_id: int) -> FileImport | None:
async with db_session() as session:
query = select(FileImport).where(FileImport.id==file_id).options(
joinedload(FileImport.project),
joinedload(FileImport.surveyor),
joinedload(FileImport.equipment),
joinedload(FileImport.project), # type: ignore
joinedload(FileImport.surveyor), # type: ignore
joinedload(FileImport.equipment), # type: ignore
)
res = await session.exec(query)
try:
@ -136,17 +136,19 @@ class Basket:
# file.name = file['name']
# return file
async def delete_file(self, id):
file = await FileImport.get(id)
async def delete_file(self, file: FileImport):
if file.dir:
path = self.base_dir/file.dir/file.name
else:
path = self.base_dir/file.name
if path.exists():
path.unlink()
await file.delete()
async with db_session() as session:
await session.delete(file)
await session.commit()
async def import_file(self, file_import: FileImport, dry_run=True, **kwargs) -> BasketImportResult:
async def import_file(self, file_import: FileImport,
dry_run=True, **kwargs) -> BasketImportResult:
"""
Import the file by calling the basket's importer's do_import.
Time stamp the FileImport.
@ -164,7 +166,6 @@ class Basket:
except Exception as err:
logger.exception(err)
raise ImportError(f'Unexpected import error (details in the Gisaf logs): {err}')
if not isinstance(result, BasketImportResult):
raise ImportError('Import error: the importer did not return a BasketImportResult')
# if import_result:
@ -181,9 +182,11 @@ class Basket:
# result = BasketImportResult(message=import_result)
# else:
# result = BasketImportResult(message='Import successful.')
if file_import.time is None:
raise ImportError('No time found in file import')
if dry_run:
result.time = file_import.time
if not dry_run:
else:
## Save time stamp
async with db_session() as session:
file_import.time = result.time
@ -191,7 +194,15 @@ class Basket:
await session.commit()
return result
async def add_files(self, file: UploadFile, user: UserRead):
async def add_file(self,
file: UploadFile,
user: UserRead,
auto_import: bool = False,
dry_run: bool = False,
project_id: int | None = None,
surveyor_id: int | None = None,
equipment_id: int | None = None,
) -> BasketImportResult:
"""
File upload to basket.
Typically called through an http POST view handler.
@ -203,89 +214,52 @@ class Basket:
## TODO: check if file already exists
# assert part.name == 'file'
## Save file on filesystem
if file.filename is None:
raise ImportError('No file name')
path = AsyncPath(self.base_dir) / file.filename
## Eventually create the directory
await path.parent.mkdir(parents=True, exist_ok=True)
file_content = await file.read()
md5sum = md5(file_content).hexdigest()
async with path.open('wb') as f:
## No way to use async to stream the file content to write it?
await f.write(await file.read())
## Read other parts
# parts = defaultdict(None)
# while True:
# part = await reader.next()
# if not part:
# break
# value = (await part.read()).decode()
# if value != 'null':
# parts[part.name] = value
## Find ids of project, surveyor, equipment
if 'project' in parts:
project_id = (await Project.query.where(Project.name==parts.get('project')).gino.first()).id
else:
project_id = None
if 'surveyor' in parts:
surveyor_id = (await Surveyor.query.where(Surveyor.name==parts.get('surveyor')).gino.first()).id
else:
surveyor_id = None
if 'equipment' in parts:
equipment_id = (await Equipment.query.where(Equipment.name==parts.get('equipment')).gino.first()).id
else:
equipment_id = None
## Save FileImport record
store_keys = [store_key for store_key in parts.keys()
if store_key.startswith('store')]
if len(store_keys) == 1:
store_type_name = parts[store_keys[0]]
else:
store_type_name = None
fileImportRecord = await FileImport(
name=file.file_name,
dir=parts.get('dir', '.'),
basket=self.name,
project_id=project_id,
surveyor_id=surveyor_id,
equipment_id=equipment_id,
store=store_type_name,
status=parts.get('status', None),
).create()
fileImportRecord.path = self.base_dir/fileImportRecord.dir/fileImportRecord.name
admin_basket_file = AdminBasketFile(
id=fileImportRecord.id,
name=file.file_name,
status=parts.get('status'),
store=store_type_name,
project=parts.get('project'),
surveyor=parts.get('surveyor'),
equipment=parts.get('equipment'),
)
await f.write(file_content)
async with db_session() as session:
fileImportRecord = FileImport(
name=file.filename,
md5=md5sum,
dir=str(self.base_dir),
basket=self.name,
project_id=project_id,
surveyor_id=surveyor_id,
equipment_id=equipment_id,
# store=store_type_name,
# status=parts.get('status', None),
)
# fileImportRecord.path = self.base_dir / fileImportRecord.dir / fileImportRecord.name
session.add(fileImportRecord)
await session.commit()
await session.refresh(fileImportRecord)
if fileImportRecord.id is None:
raise ImportError('Cannot save (no fileImportRecord.id)')
## Eventually do import
import_result = None
if loads(parts['autoImport']):
## Get the record from DB with Pandas, for compatibility with import_file
file_import_record = await self.get_file(fileImportRecord.id)
try:
await check_permission(request, 'reviewer')
except HTTPUnauthorized as err:
basket_import_result = BasketImportResult(
message="Cannot import: only a reviewer can do that"
)
basket_import_result = BasketImportResult(
fileImport=fileImportRecord
)
if auto_import:
if user.has_role('reviewer'):
## Get the record from DB, for compatibility with import_file
file_import_record = await self.get_file(fileImportRecord.id)
if file_import_record is None:
basket_import_result.message="Cannot import: file not found"
else:
try:
basket_import_result = await self.import_file(file_import_record, dry_run)
except ImportError as err:
basket_import_result.message=f'Error: {err.args[0]}'
else:
dry_run = parts.get('dry_run', False)
try:
basket_import_result = await self.import_file(file_import_record, dry_run)
except ImportError as err:
basket_import_result = BasketImportResult(
message=f'Error: {err.args[0]}'
)
admin_basket_file.import_result = basket_import_result
return admin_basket_file
basket_import_result.message="Cannot import: only a reviewer can do that"
return basket_import_result
class MiscGeomBasket(Basket):

4
src/gisaf/importers.py
View file

@ -39,7 +39,7 @@ class Importer:
The main process is executed by do_import(file)
Subclasses should define read_file and process_df.
"""
basket = None
basket = None # type hint: baskets.Basket
async def do_import(self, file_record, dry_run=False, **kwargs) -> BasketImportResult:
"""
@ -181,7 +181,7 @@ class RawSurveyImporter(Importer):
## Import to raw_survey_data table
## PostGis specific: add SRID
gdf['geom'] = gdf['geometry'].apply(lambda g: dumps_wkb(g, srid=conf.raw_survey['srid'], hex=True))
gdf['geom'] = gdf['geometry'].apply(lambda g: dumps_wkb(g, srid=conf.geo.raw_survey.srid, hex=True))
if not dry_run:
await upsert_df(gdf, model)

49
src/gisaf/models/admin.py
View file

@ -1,7 +1,8 @@
from pathlib import Path
import re
from datetime import datetime, date
from pydantic import BaseModel
from pydantic import BaseModel, computed_field
from sqlmodel import Field, Relationship
import pandas as pd
@ -47,33 +48,27 @@ class FileImport(Model, table=True):
__table_args__ = gisaf_admin.table_args
id: int | None = Field(default=None, primary_key=True)
url: str
url: str | None = None
## TODO: Deprecate FileImport.path, in favour of dir + name
path: str
#path: str
dir: str
name: str
md5: str
time: datetime
comment: str
status: str
store: str
time: datetime | None = None
comment: str | None = None
status: str | None = None
store: str | None = None
basket: str
project_id: int = Field(foreign_key=gisaf_admin.table('project.id'))
project_id: int | None = Field(foreign_key=gisaf_admin.table('project.id'))
project: Project = Relationship()
# ALTER TABLE gisaf_admin.file_import add column project_id INT REFERENCES gisaf_admin.project;
surveyor_id: int = Field(foreign_key=gisaf_survey.table('surveyor.id'))
surveyor_id: int | None = Field(foreign_key=gisaf_survey.table('surveyor.id'))
surveyor: Surveyor = Relationship()
# ALTER TABLE gisaf_admin.file_import add column surveyor_id INT REFERENCES gisaf_survey.surveyor;
equipment_id: int = Field(foreign_key=gisaf_survey.table('equipment.id'))
equipment_id: int | None = Field(foreign_key=gisaf_survey.table('equipment.id'))
equipment: Equipment = Relationship()
# ALTER TABLE gisaf_admin.file_import add column equipment_id INT REFERENCES gisaf_survey.equipment;
def __str__(self):
return f'{self.path:s} for project id {self.project_id}'
def __repr__(self):
return f'<gisaf.misc.FileImport (gisaf_admin.file_import) {self.path}>'
@classmethod
def selectinload(cls):
return [cls.project, cls.surveyor, cls.equipment]
@ -93,6 +88,10 @@ class FileImport(Model, table=True):
df['date'] = pd.to_datetime(dates[0], format='%Y-%m-%d')
return df
@computed_field
@property
def path(self) -> Path:
return Path(self.dir) / self.name
#def get_parent_dir(self):
# split_path = self.path.split(os_path.sep)
@ -158,23 +157,9 @@ class Basket(BasketNameOnly):
class BasketImportResult(BaseModel):
time: datetime = Field(default_factory=datetime.now)
message: str
message: str | None = None
details: dict[str, str | int | float | bool] | None = None
class AdminBasketFile(BaseModel):
id: int
dir: str
name: str
url: str
md5: str
time: datetime
comment: str
status: str
store: str
project: str
surveyor: str
equipment: str
import_result: str
fileImport: FileImport | None = None
class AdminBasket(BaseModel):

2
src/gisaf/registry.py
View file

@ -681,7 +681,7 @@ class ModelRegistry:
'live': 'is_live',
'zIndex': 'z_index',
'gisType': 'gis_type',
# 'type': 'mapbox_type',
'type': 'mapbox_type',
'viewableRole': 'viewable_role',
}, inplace=True
)