Skip to content

Commit e82cde4

Browse files
test/style: use actual data.for test cases, implement ruff lint suggestions and formatting. Use fastapi TestClient class
1 parent 373e35e commit e82cde4

12 files changed

Lines changed: 9335 additions & 137 deletions

File tree

‎.github/workflows/hdf-reader-api-composite/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ runs:
3333
- name: Set up Python
3434
uses: actions/setup-python@v4
3535
with:
36-
python-version: "3.10"
36+
python-version: "3.13"
3737

3838
- name: Use Node.js
3939
uses: actions/setup-node@v4

‎src/HDF/data_reader.py‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
import logging
21
import fsspec
32
import rasterio
43
import xarray as xr
5-
from src import api_logger
4+
65
# from fsspec.implementations.local import S3FileSystem
76
from s3fs.core import S3FileSystem
8-
from time import perf_counter
7+
8+
from src import api_logger
9+
910

1011
def open_zarr_store(file_system: S3FileSystem, group='/') -> xr.Dataset:
1112
# Implements example from Luis López using fsspec_params and driver_config for kerchunked zarr store
@@ -18,15 +19,14 @@ def open_zarr_store(file_system: S3FileSystem, group='/') -> xr.Dataset:
1819
backend_kwargs={'consolidated': False, 'group': group},
1920
)
2021

22+
2123
def get_bounding_polygon(file_uri: str):
2224
fs = fsspec.filesystem(
2325
'reference',
24-
**{
25-
'fo': file_uri,
26-
'remote_protocol': 's3',
27-
'compression': 'gzip',
28-
'remote_options': {'anon': False},
29-
},
26+
fo=file_uri,
27+
remote_protocol='s3',
28+
compression='gzip',
29+
remote_options={'anon': False},
3030
# One line required to specify compression here
3131
target_options={'compression': 'gzip'},
3232
)
@@ -50,5 +50,5 @@ def get_crs(data):
5050
api_logger.debug(msg='Reading crs')
5151
spatial_ref = data.variables['spatial_ref']
5252
crs = rasterio.crs.CRS.from_string(spatial_ref.attrs['crs_wkt']) # type: ignore
53-
api_logger.debug(msg=f'Read crs from data: {str(crs)}')
53+
api_logger.debug(msg=f'Read crs from data: {crs!s}')
5454
return crs

‎src/__init__.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,14 @@
33
# (api_logger needs to be created before
44
# the "from . import"'s to avoid a circular import)
55
import logging
6+
67
from . import logger
78

89
api_logger = logger.get_logger(__name__, logging.DEBUG)
910

10-
from . import models # noqa: E402, F401
11-
from . import application # noqa: E402, F401
12-
from .exceptions import MissingFrameException, OutdatedStackException # noqa: E402, F401
13-
from .constants import Polarizations, fsspec_params # noqa: E402, F401
11+
from . import (
12+
application, # noqa: F401
13+
models, # noqa: F401
14+
)
15+
from .constants import Polarizations, fsspec_params # noqa: F401
16+
from .exceptions import MissingFrameException, OutdatedStackException # noqa: F401

‎src/application.py‎

Lines changed: 60 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,25 @@
1-
from fastapi import APIRouter, FastAPI, HTTPException, Response
21
from pathlib import Path
32
from time import perf_counter
4-
import logging
53

6-
from fastapi.responses import JSONResponse
4+
import geopandas as gpd
5+
import s3fs
6+
from fastapi import APIRouter, FastAPI, HTTPException, Response
77
from fastapi.middleware.cors import CORSMiddleware
8-
8+
from fastapi.responses import JSONResponse
99
from pydantic import BaseModel
10+
from shapely import GEOSException, convex_hull
11+
from shapely import wkt as shapely_wkt
12+
from shapely.geometry.base import BaseGeometry
13+
14+
from src import api_logger
1015
from src.exceptions import MissingFrameException
1116
from src.models import (
12-
TimeSeriesModel,
1317
GetFramesModel,
18+
TimeSeriesModel,
1419
)
1520
from src.timeseries import get_timeseries_over_aoi
16-
from src import api_logger
17-
import s3fs
18-
import geopandas as gpd
1921

2022
from . import log_router
21-
from shapely import wkt as shapely_wkt
22-
from shapely import convex_hull
23-
from shapely.geometry.base import BaseGeometry
2423

2524
router = APIRouter(route_class=log_router.LoggingRoute)
2625
app = FastAPI()
@@ -39,30 +38,33 @@
3938
def read_root():
4039
return {'Status': 'A-Okay'}
4140

41+
4242
class ReadLayer(BaseModel):
4343
file_name: str
4444
layer: str
4545

46+
4647
@app.post(
4748
'/timeseries',
4849
description='Returns a json response with values read at "unwrapped_phase", "interferometric_correlation", and "temporal_coherence" layers in provided files at given point',
4950
)
5051
def get_timeseries(series: TimeSeriesModel):
51-
# TODO: REMOVE BEFORE ANOTHER PRODUCTION RELEASE
52-
return HTTPException(
53-
status_code=400,
54-
detail='The test displacement API is temporarily disabled due to unintended usage. For programmatic queries please access the zarr stack store `displacement` layer directly. For more info or questions please contact uso@asf.alaska.edu',
55-
)
52+
# # TODO: REMOVE BEFORE ANOTHER PRODUCTION RELEASE
53+
# return HTTPException(
54+
# status_code=400,
55+
# detail='The test displacement API is temporarily disabled due to unintended usage. For programmatic queries please access the zarr stack store `displacement` layer directly. For more info or questions please contact uso@asf.alaska.edu',
56+
# )
5657
start = perf_counter()
5758
try:
5859
geometry = shapely_wkt.loads(series.wkt)
59-
except BaseException as exc:
60+
except GEOSException as exc:
6061
raise HTTPException(
6162
400, detail=f'Failed to load provided wkt string. Original Error: {exc}'
6263
)
6364
if geometry.geom_type not in ['Polygon', 'Point']:
6465
raise HTTPException(
65-
400, detail=f'Failed to load provided wkt. Must be "POLYGON" or "POINT", got {geometry.geom_type.upper()}'
66+
400,
67+
detail=f'Failed to load provided wkt. Must be "POLYGON" or "POINT", got {geometry.geom_type.upper()}',
6668
)
6769

6870
api_logger.debug(f'wkt loaded: {geometry.wkt}')
@@ -72,8 +74,8 @@ def get_timeseries(series: TimeSeriesModel):
7274
wkt=series.wkt,
7375
frame_id=series.frameId,
7476
polarization=series.polarization.value,
75-
flight_direction=series.flightDirection
76-
)
77+
flight_direction=series.flightDirection,
78+
)
7779

7880
api_logger.info(f'using file store: {file_store}')
7981

@@ -97,10 +99,14 @@ def get_timeseries(series: TimeSeriesModel):
9799
)
98100
def get_frame_ids(wkt_event: GetFramesModel):
99101
return JSONResponse(
100-
{'frame_ids': sorted(_get_frames_for_wkt(wkt_event.wkt, wkt_event.flightDirection)), 'wkt': wkt_event.wkt},
102+
{
103+
'frame_ids': sorted(_get_frames_for_wkt(wkt_event.wkt, wkt_event.flightDirection)),
104+
'wkt': wkt_event.wkt,
105+
},
101106
headers=DEFAULT_HEADERS,
102107
)
103108

109+
104110
@app.post(
105111
'/frame_intersection',
106112
description='Returns the frame ids for a given aoi, splitting any cross frames',
@@ -112,40 +118,53 @@ def frame_intersection(wkt_event: GetFramesModel):
112118
frame_path = Path(__file__).parent.parent / 'data' / 'frames.geojson'
113119
geometry = shapely_wkt.loads(wkt_event.wkt)
114120
gdf = gpd.read_file(frame_path, engine='pyogrio')
115-
frames: gpd.GeoDataFrame = gdf[gdf['frame_id'].isin([int(p) for p in overlapping_frames])] # type: ignore
121+
frames: gpd.GeoDataFrame = gdf[gdf['frame_id'].isin([int(p) for p in overlapping_frames])]
116122
intersection = frames.intersection(geometry)
117123
return JSONResponse(
118-
{_pad_frame_str(frame_id): _simplify_multipolygons(geom).wkt for frame_id, geom in zip(gdf.iloc[intersection.index]['frame_id'], intersection)},
124+
{
125+
_pad_frame_str(frame_id): _simplify_multipolygons(geom).wkt
126+
for frame_id, geom in zip(gdf.iloc[intersection.index]['frame_id'], intersection)
127+
},
119128
headers=DEFAULT_HEADERS,
120129
)
121130

131+
122132
def _simplify_multipolygons(geometry: BaseGeometry) -> BaseGeometry:
123133
"""In case polygon intersects different sections of frame at once and get multi-polygon (swath corners for example), perform convex hull instead"""
124134
if geometry.geom_type == 'MultiPolygon':
125135
geometry = convex_hull(geometry)
126136

127137
return geometry
128138

139+
129140
def _get_file_store_for_wkt(bucket: str, frame_id: str, polarization: str):
130141
return f's3://{bucket}/OPERA_L3_DISP-S1_STACK_V1/OPERA_L3_DISP-S1_IW_F{frame_id}_{polarization}_V1/OPERA_L3_DISP-S1_IW_F{frame_id}_{polarization}_short_wavelength_displacement.zarr.json.gz'
131142

132-
def _get_zarr_store_uri(bucket: str, wkt: str, frame_id: str | None, polarization: str, flight_direction: str):
143+
144+
def _get_zarr_store_uri(
145+
bucket: str, wkt: str, frame_id: str | None, polarization: str, flight_direction: str
146+
):
133147
if isinstance(frame_id, str):
134-
zarr_store = _get_file_store_for_wkt(bucket=bucket, frame_id=frame_id, polarization=polarization)
148+
zarr_store = _get_file_store_for_wkt(
149+
bucket=bucket, frame_id=frame_id, polarization=polarization
150+
)
135151

136152
s3 = s3fs.S3FileSystem(anon=False)
137153
try:
138154
if s3.exists(zarr_store):
139155
return zarr_store
140156
except Exception as exc:
141-
logging.debug(f'Failed to open store "{zarr_store}". Original exception: {exc}')
142-
raise MissingFrameException(
143-
400, f'No data found for the given area of interest {wkt}'
144-
)
157+
api_logger.debug(f'Failed to open store "{zarr_store}". Original exception: {exc}')
158+
raise MissingFrameException(400, f'No data found for the given area of interest {wkt}')
145159
else:
146-
return _get_westmost_file_store_for_wkt(bucket=bucket, wkt=wkt, polarization=polarization, flight_direction=flight_direction)
160+
return _get_westmost_file_store_for_wkt(
161+
bucket=bucket, wkt=wkt, polarization=polarization, flight_direction=flight_direction
162+
)
147163

148-
def _get_westmost_file_store_for_wkt(bucket: str, wkt: str, polarization: str, flight_direction: str):
164+
165+
def _get_westmost_file_store_for_wkt(
166+
bucket: str, wkt: str, polarization: str, flight_direction: str
167+
):
149168
# example:
150169
# s3://asf-cumulus-test-opera-products/OPERA_L3_DISP-S1_STACK_V1/OPERA_L3_DISP-S1_IW_F44328_VV_V1/OPERA_L3_DISP-S1_IW_F44328_VV_short_wavelength_displacement.zarr.json.gz
151170

@@ -162,21 +181,23 @@ def _get_westmost_file_store_for_wkt(bucket: str, wkt: str, polarization: str, f
162181
if s3.exists(store):
163182
available_zarr_stores.append(store)
164183
except Exception as exc:
165-
logging.debug(f'Failed to open store "{store}". Original exception: {exc}')
184+
api_logger.debug(f'Failed to open store "{store}". Original exception: {exc}')
166185

167186
if len(available_zarr_stores) == 0:
168-
raise MissingFrameException(
169-
400, f'No data found for the given area of interest {wkt}'
170-
)
187+
raise MissingFrameException(400, f'No data found for the given area of interest {wkt}')
171188

172-
return available_zarr_stores[0] if flight_direction.upper() == 'DESCENDING' else available_zarr_stores[-1]
189+
return (
190+
available_zarr_stores[0]
191+
if flight_direction.upper() == 'DESCENDING'
192+
else available_zarr_stores[-1]
193+
)
173194

174195

175196
def _get_frames_for_wkt(wkt: str, flight_direction: str) -> list[str]:
176197
frame_path = Path(__file__).parent.parent / 'data' / 'frames.geojson'
177198
geometry = shapely_wkt.loads(wkt)
178199
gdf = gpd.read_file(frame_path, engine='pyogrio')
179-
result = gdf[gdf.orbit_direction==flight_direction][gdf.intersects(geometry)]
200+
result = gdf[gdf.orbit_direction == flight_direction][gdf.intersects(geometry)]
180201
result['west_most_point'] = result.geometry.bounds.minx
181202
result.sort_values(by='west_most_point', ascending=flight_direction == 'DESCENDING')
182203

@@ -190,11 +211,12 @@ def _get_frames_for_wkt(wkt: str, flight_direction: str) -> list[str]:
190211

191212
return [_pad_frame_str(frame) for frame in frame_ids]
192213

214+
193215
def _pad_frame_str(frame: int):
194216
frame_str = str(frame)
195217

196218
# append leading zeros
197219
if len(frame_str) < 5:
198-
frame_str = ('0'*(5-len(frame_str))) + frame_str
220+
frame_str = ('0' * (5 - len(frame_str))) + frame_str
199221

200222
return frame_str

‎src/logger.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import logging
22
import os
3+
34
from pythonjsonlogger import jsonlogger
45

56

@@ -29,7 +30,7 @@ class Colors:
2930

3031
LOGGER_FORMAT = f'{Colors.BOLD}[ %(asctime)s (%(name)s) %(filename)s:%(lineno)d ] %(levelname)s - {Colors.BOLD_OFF}%(message)s'
3132

32-
FORMATS = {
33+
FORMATS = { # noqa: RUF012
3334
logging.DEBUG: Colors.OKBLUE + LOGGER_FORMAT + Colors.END,
3435
logging.INFO: Colors.OKCYAN + LOGGER_FORMAT + Colors.END,
3536
logging.WARNING: Colors.WARNING + LOGGER_FORMAT + Colors.END,
@@ -57,7 +58,7 @@ class AwsStreamFormatter(logging.Formatter):
5758
Custom Logger formatter, used when running this app in AWS
5859
"""
5960

60-
LOGGER_FORMAT = ' '.join(
61+
LOGGER_FORMAT = ' '.join( # noqa: FLY002
6162
[
6263
'%(asctime)s',
6364
'%(name)s',
@@ -69,7 +70,7 @@ class AwsStreamFormatter(logging.Formatter):
6970
]
7071
)
7172

72-
LOGGER_RENAME_FIELDS = {
73+
LOGGER_RENAME_FIELDS = { # noqa: RUF012
7374
'asctime': 'datetime',
7475
'lineno': 'line_number',
7576
'levelname': 'log_level',

‎src/models.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
from typing import Optional
21
from pydantic import BaseModel, Field, field_validator
2+
33
from .constants import Polarizations
44

5+
56
class GetFramesModel(BaseModel):
67
wkt: str
78
flightDirection: str = Field(pattern=r'((ASC)|(DESC))(ENDING)?', default='ASCENDING')
@@ -16,7 +17,8 @@ def pre_validate_flightdir(cls, v: str) -> str:
1617
def post_validate_flightdir(cls, v: str) -> str:
1718
return 'ASCENDING' if v in 'ASCENDING' else 'DESCENDING'
1819

20+
1921
class TimeSeriesModel(GetFramesModel):
2022
bucket: str = Field(pattern=r'[^/^\\]*')
2123
polarization: Polarizations = Polarizations.VV
22-
frameId: Optional[str] = Field(None, pattern=r'^\d{5}$')
24+
frameId: str | None = Field(None, pattern=r'^\d{5}$')

‎src/requirements.txt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
uvicorn==0.28.0
2-
fastapi==0.110.0
1+
anyio==4.15
2+
fastapi==0.141
33
zarr>=2.0.0,<3.0.0
44
xarray==2025.01.2
55
s3fs==2024.12.0
6-
numpy==1.26.4
7-
rasterio==1.3.9
6+
numpy==2
7+
rasterio==1.5.1
88
h5netcdf
99
python-json-logger==2.0.7
1010
debugpy
@@ -15,4 +15,4 @@ cftime
1515
geopandas
1616
rioxarray
1717
dask<=2024.12.1
18-
kerchunk
18+
kerchunk

0 commit comments

Comments
 (0)