philorg/gisaf-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases1 Packages6 Wiki Activity Actions

Fix/update baskets:

Browse source 
- get_file
- import
- Basket importers must return BasketImportResult
Add API points
Fix utils delete_df and upsert_df, also making them async friendly
Auth: add helper functions to UserRead
This commit is contained in:
phil 2024-04-09 16:16:04 +05:30
parent 52e1d2135b
commit d2ae5e4d7b
9 changed files with 323 additions and 182 deletions
src/gisaf
baskets.py

158
src/gisaf/baskets.py
View file

@ -1,4 +1,5 @@
from pathlib import Path
from aiopath import AsyncPath
from collections import defaultdict
from json import loads
from datetime import datetime
@ -10,13 +11,15 @@ from typing import ClassVar
# from aiohttp.web import HTTPUnauthorized, HTTPForbidden
from sqlmodel import select
from sqlalchemy.orm import joinedload, QueryableAttribute
from sqlalchemy.exc import NoResultFound
from fastapi import UploadFile
from gisaf.config import conf
from gisaf.utils import ToMigrate
from gisaf.database import db_session
from gisaf.importers import RawSurveyImporter, GeoDataImporter, LineWorkImporter, ImportError
from gisaf.models.admin import FileImport, AdminBasketFile, BasketImportResult
from gisaf.models.authentication import User
from gisaf.models.authentication import UserRead
from gisaf.models.survey import Surveyor, Equipment
from gisaf.models.project import Project
@ -48,7 +51,7 @@ class Basket:
self.importer = self.importer_class()
self.importer.basket = self
async def allowed_for(self, user: User):
async def allowed_for(self, user: UserRead):
"""
Return False if the basket is protected by a role
Request: aiohttp.Request instance
@ -101,23 +104,37 @@ class Basket:
# else:
# return df
async def get_file(self, id):
df = await FileImport.get_df(
where=FileImport.id==id,
with_related=True,
)
df.rename(columns={
'gisaf_admin_project_name': 'project',
'gisaf_survey_surveyor_name': 'surveyor',
'gisaf_survey_equipment_name': 'equipment',
}, inplace=True)
df['dir'] = df.dir.fillna('.')
## Replace path with Path from pathlib
df['path'] = df.apply(lambda fi: self.base_dir/fi['dir']/fi['name'], axis=1)
file = df.iloc[0]
## Override file.name, which otherwise is the index of the item (hack)
file.name = file['name']
return file
async def get_file(self, file_id: int) -> FileImport | None:
async with db_session() as session:
query = select(FileImport).where(FileImport.id==file_id).options(
joinedload(FileImport.project),
joinedload(FileImport.surveyor),
joinedload(FileImport.equipment),
)
res = await session.exec(query)
try:
file = res.one()
except NoResultFound:
return None
else:
return file
# df = await FileImport.get_df(
# where=FileImport.id==id,
# with_related=True,
# )
# df.rename(columns={
# 'gisaf_admin_project_name': 'project',
# 'gisaf_survey_surveyor_name': 'surveyor',
# 'gisaf_survey_equipment_name': 'equipment',
# }, inplace=True)
# df['dir'] = df.dir.fillna('.')
# ## Replace path with Path from pathlib
# df['path'] = df.apply(lambda fi: self.base_dir/fi['dir']/fi['name'], axis=1)
# file = df.iloc[0]
# ## Override file.name, which otherwise is the index of the item (hack)
# file.name = file['name']
# return file
async def delete_file(self, id):
file = await FileImport.get(id)
@ -129,7 +146,7 @@ class Basket:
path.unlink()
await file.delete()
async def import_file(self, file_import, dry_run=True, return_data_info=False, **kwargs):
async def import_file(self, file_import: FileImport, dry_run=True, **kwargs) -> BasketImportResult:
"""
Import the file by calling the basket's importer's do_import.
Time stamp the FileImport.
@ -139,44 +156,42 @@ class Basket:
return BasketImportResult(
message=f'No import defined/required for {self.name} basket'
)
result: BasketImportResult
try:
import_result = await self.importer.do_import(file_import, dry_run=dry_run, **kwargs)
result = await self.importer.do_import(file_import, dry_run=dry_run, **kwargs)
except ImportError as err:
raise
raise err
except Exception as err:
logger.exception(err)
raise ImportError(f'Unexpected import error: {err}')
raise ImportError(f'Unexpected import error (details in the Gisaf logs): {err}')
if isinstance(import_result, BasketImportResult):
result = import_result
else:
if import_result:
if isinstance(import_result, (tuple, list)):
assert len(import_result) >= 2, \
'do_import should return message or (message, details)'
result = BasketImportResult(
message=import_result[0],
details=import_result[1],
)
if len(import_result) > 2:
data = import_result[2]
else:
result = BasketImportResult(message=import_result)
else:
result = BasketImportResult(message='Import successful.')
if not isinstance(result, BasketImportResult):
raise ImportError('Import error: the importer did not return a BasketImportResult')
# if import_result:
# if isinstance(import_result, (tuple, list)):
# assert len(import_result) >= 2, \
# 'do_import should return message or (message, details)'
# result = BasketImportResult(
# message=import_result[0],
# details=import_result[1],
# )
# if len(import_result) > 2:
# data = import_result[2]
# else:
# result = BasketImportResult(message=import_result)
# else:
# result = BasketImportResult(message='Import successful.')
if dry_run:
result.time = file_import.time
else:
if not result.time:
result.time = datetime.now()
if not dry_run:
## Save time stamp
await (await FileImport.get(file_import.id)).update(time=result.time).apply()
if return_data_info:
return result, data
else:
return result
async with db_session() as session:
file_import.time = result.time
session.add(file_import)
await session.commit()
return result
async def add_files(self, reader, request):
async def add_files(self, file: UploadFile, user: UserRead):
"""
File upload to basket.
Typically called through an http POST view handler.
@ -184,37 +199,26 @@ class Basket:
Note that the return dict has eventually numpy types and needs NumpyEncoder
to be json.dump'ed.
"""
raise ToMigrate("basket add_files reader was aiohttp's MultipartReader")
## TODO: multiple items
## TODO: check if file already exists
## First part is the file
part = await reader.next()
assert part.name == 'file'
file_name = part.filename
# assert part.name == 'file'
## Save file on filesystem
size = 0
path = Path(self.base_dir) / file_name
path = AsyncPath(self.base_dir) / file.filename
## Eventually create the directory
path.parent.mkdir(parents=True, exist_ok=True)
with path.open('wb') as f:
while True:
chunk = await part.read_chunk() # 8192 bytes by default.
if not chunk:
break
size += len(chunk)
f.write(chunk)
await path.parent.mkdir(parents=True, exist_ok=True)
async with path.open('wb') as f:
## No way to use async to stream the file content to write it?
await f.write(await file.read())
## Read other parts
parts = defaultdict(None)
while True:
part = await reader.next()
if not part:
break
value = (await part.read()).decode()
if value != 'null':
parts[part.name] = value
# parts = defaultdict(None)
# while True:
# part = await reader.next()
# if not part:
# break
# value = (await part.read()).decode()
# if value != 'null':
# parts[part.name] = value
## Find ids of project, surveyor, equipment
if 'project' in parts:
@ -238,7 +242,7 @@ class Basket:
else:
store_type_name = None
fileImportRecord = await FileImport(
name=file_name,
name=file.file_name,
dir=parts.get('dir', '.'),
basket=self.name,
project_id=project_id,
@ -251,7 +255,7 @@ class Basket:
admin_basket_file = AdminBasketFile(
id=fileImportRecord.id,
name=file_name,
name=file.file_name,
status=parts.get('status'),
store=store_type_name,
project=parts.get('project'),