Skip to content

fix: cop_dem items download for wekeo_main #2201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
cauriol wants to merge 1 commit into
base: develop
Choose a base branch
from
+107 −41
Draft

fix: cop_dem items download for wekeo_main #2201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions eodag/api/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1550,20 +1550,20 @@ def _search_by_id(
**kwargs,
):
results.data.extend(page_results.data)
# try using crunch to get unique result and stop if found, to avoid unnecessary requests
if (
len(results) > 1
and len(filtered := results.filter_property(id=uid)) == 1
):
results = filtered
break
except Exception as e:
if kwargs.get("raise_errors"):
raise
logger.warning(e)
results.errors.append((plugin.provider, e))
continue

# try using crunch to get unique result
if (
len(results) > 1
and len(filtered := results.filter_property(id=uid)) == 1
):
results = filtered

if len(results) == 1:
if not results[0].collection:
# guess collection from properties
Expand Down
127 changes: 94 additions & 33 deletions eodag/resources/providers.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2954,21 +2954,21 @@
processing:level:
- '{{"processingLevel": "{processing:level}"}}'
- '$.id.`sub(/^[^_]([^_]+)_([^_]+)_([^_]+)_([0-4]+).*/, LEVEL\\4)`'
sar:instrument_mode:
- '{{"sensorMode": "{sar:instrument_mode}"}}'
- '$.id.`sub(/^[^_]([^_]+)_([^_]+)_.*/, \\2)`'
swath:
- '{{"swath": "{swath}"}}'
platform:
- '{{"platformSerialIdentifier": "{platform}"}}'
- '$.null'
polarisation:
- '{{"polarisation": "{polarisation}"}}'
instruments:
- '{{"instrumentShortName": "{instruments#csv_list}"}}'
- '$.null'
order:status:
- '{{"online": "{order:status#get_group_name((?P<true>succeeded)|(?P<false>orderable))}"}}'
- '$.null'
sar:polarizations:
- '{{"polarisationChannels": "{sar:polarizations#csv_list(%26)}"}}'
- '$.null'
sat:relative_orbit:
- '{{"relativeOrbitNumber": "{sat:relative_orbit}"}}'
- '$.null'
missionTakeId:
- '{{"missionTakeId": "{missionTakeId}"}}'
- '$.null'
anchor_orbit_cycle: &orbit_cycle
sat:relative_orbit:
- '{{"relativeOrbitNumber": "{sat:relative_orbit}"}}'
Expand All @@ -2986,7 +2986,7 @@
"startdate": "{id#replace_str(r'^.*_([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9])([0-9][0-9])([0-9][0-9])_.*$',r'\1-\2-\3T\4%3A\5%3A00Z')}",
"enddate": "{id#replace_str(r'^.*_([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9])([0-9][0-9])([0-9][0-9])_.*$',r'\1-\2-\3T\4%3A\5%3A00Z')}"
}}
- '$.id'
- '{$.id#remove_extension}'
search: !plugin
type: WekeoSearch
api_endpoint: https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/search
Expand All @@ -2999,6 +2999,14 @@
next_page_query_obj: '{{"itemsPerPage":{limit},"startIndex":{next_page_token}}}'
max_limit: 200
next_page_token_key: skip
sort:
sort_by_tpl: '{{"sortOrder": "{sort_order}"}}'
sort_param_mapping:
# start_datetime for consistency between providers, but it's unused for the actual sorting
start_datetime: unused
sort_order_mapping:
ascending: ASC
descending: DESC
discover_collections:
fetch_url: null
discover_queryables:
Expand All @@ -3015,7 +3023,7 @@
- '$.geometry'
eodag:default_geometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))'
id:
- '{{"productIdentifier": "{id}"}}'
- '{{"datasetIdentifier": "{id}"}}'
- '{$.id#remove_extension}'
start_datetime:
- '{{"startdate": "{start_datetime}"}}'
Expand All @@ -3042,34 +3050,34 @@
sat:orbit_state:
- '{{"orbitDirection": "{sat:orbit_state}"}}'
- '$.null'
variable:
wekeo_main:variable:
- '{{"variable": "{variable}"}}'
- '$.null'
system:
wekeo_main:system:
- '{{"system": "{system}"}}'
- '$.null'
version:
wekeo_main:version:
- '{{"version": {version}}}'
- '$.null'
region:
wekeo_main:region:
- '{{"region": {region}}}'
- '$.null'
type:
wekeo_main:type:
- '{{"type": "{type}"}}'
- '$.null'
source:
wekeo_main:source:
- '{{"source": {source}}}'
- '$.null'
model:
wekeo_main:model:
- '{{"model": {model}}}'
- '$.null'
level:
wekeo_main:level:
- '{{"level": {level}}}'
- '$.null'
step:
wekeo_main:step:
- '{{"step": {step}}}'
- '$.null'
satellite:
wekeo_main:satellite:
- '{{"satellite": {satellite}}}'
- '$.null'
products:
Expand All @@ -3078,6 +3086,13 @@
product:type: GRD
metadata_mapping:
<<: *s1_sar_params
id:
- |
{{
"startdate": "{id#replace_str(r'^.*_([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9])T([0-9][0-9])([0-9][0-9])([0-9][0-9])_.*$',r'\1-\2-\3T00%3A00%3A00Z')}",
"enddate": "{id#replace_str(r'^.*_([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9])T([0-9][0-9])([0-9][0-9])([0-9][0-9])_.*$',r'\1-\2-\3T23%3A59%3A59Z')}"
}}
- '{$.id#remove_extension}'
eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}.SAFE", "dataset_id": "EO:ESA:DAT:SENTINEL-1"}}'
S1_SAR_RAW:
_collection: EO:ESA:DAT:SENTINEL-1
Expand All @@ -3099,6 +3114,15 @@
processing:level:
- '{{"processingLevel": "{processing:level}"}}'
- '$.id.`sub(/^[^_]([^_]+)_([^_]+)_.*/, S2\\2)`'
platform:
- '{{"platformSerialIdentifier": "{platform}"}}'
- '$.null'
instruments:
- '{{"instrumentShortName": "{instruments#csv_list}"}}'
- '$.null'
sat:relative_orbit:
- '{{"relativeOrbitNumber": "{sat:relative_orbit}"}}'
- '$.null'
eo:cloud_cover:
- '{{"cloudCover": "{eo:cloud_cover}"}}'
- '$.null'
Expand All @@ -3113,9 +3137,28 @@
product:type: SR_2_LAN_HY
processing:level: 2
metadata_mapping:
id:
- |
{{
"startdate": "{id#replace_str(r'^.*[A-Z]_([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9])T([0-9][0-9])([0-9][0-9])([0-9][0-9])_.*$',r'\1-\2-\3T\4%3A00%3A00Z')}",
"enddate": "{id#replace_str(r'^.*[A-Z]_([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9])T([0-9][0-9])([0-9][0-9])([0-9][0-9])_.*$',r'\1-\2-\3T\4%3A59%3A59Z')}"
}}
- '{$.id#remove_extension}'
processing:level:
- '{{"processingLevel": "{processing:level}"}}'
- '2'
platform:
- '{{"platformSerialIdentifier": "{platform}"}}'
- '$.null'
instruments:
- '{{"instrumentShortName": "{instruments#csv_list}"}}'
- '$.null'
sat:relative_orbit:
- '{{"relativeOrbitNumber": "{sat:relative_orbit}"}}'
- '$.null'
eo:cloud_cover:
- '{{"cloudCover": "{eo:cloud_cover}"}}'
- '$.null'
eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}.SEN3", "dataset_id": "EO:ESA:DAT:SENTINEL-3"}}'
S3_LAN_SI:
_collection: EO:ESA:DAT:SENTINEL-3
Expand Down Expand Up @@ -3206,14 +3249,20 @@
eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}.SEN3", "dataset_id": "EO:EUM:DAT:SENTINEL-3:SR_2_WAT___"}}'
S5P_L1B_IR_ALL:
_collection: EO:ESA:DAT:SENTINEL-5P
processing:level: L1B
processing:level: L1b
metadata_mapping:
processing:level:
- '{{"processingLevel": "{processing:level}"}}'
- '$.id.`sub(/^[^_]([^_]+)_([^_]+)_([^_]+)_.*/, \\3)`'
instruments:
- '{{"instrumentShortName": "{instruments#csv_list}"}}'
- '$.null'
processingMode:
- '{{"processingMode": "{processingMode}"}}'
- '$.null'
sat:absolute_orbit:
- '{{"orbitNumber": "{sat:absolute_orbit}"}}'
- '$.null'
eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}.nc", "dataset_id": "EO:ESA:DAT:SENTINEL-5P"}}'
S5P_L2_IR_ALL:
_collection: EO:ESA:DAT:SENTINEL-5P
Expand All @@ -3234,6 +3283,9 @@
_collection: EO:ESA:DAT:COP-DEM
product:type: DGE_30
metadata_mapping:
gsd:
- '{{"spatialResolution": "{gsd}"}}'
- '$.null'
eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:ESA:DAT:COP-DEM"}}'
COP_DEM_GLO30_DTED:
_collection: EO:ESA:DAT:COP-DEM
Expand All @@ -3250,17 +3302,26 @@
CLMS_GLO_NDVI_333M:
_collection: EO:JRC:DAT:CLMS
product:type: vegetation_indices
_product_identifier: ndvi_global_300m_10daily_v1
_dataset_identifier: ndvi_global_300m_10daily_v1
metadata_mapping:
_product_identifier:
- '{{"productIdentifier": "{_product_identifier}"}}'
- '$.null'
_dataset_identifier:
- '{{"datasetIdentifier": "{_dataset_identifier}"}}'
- '$.null'
id: '$.id'
platform:
- '{{"platformShortName": "{platform}"}}'
- '$.null'
instruments:
- '{{"instrumentShortName": "{instruments#csv_list}"}}'
- '$.null'
wekeo_main:format:
- '{{"fileFormat": "{format}"}}'
- '$.null'
eodag:order_link: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/download?{{"location": "{eodag:download_link}","product_id":"{id}", "dataset_id": "EO:JRC:DAT:CLMS"}}'
CLMS_GLO_NDVI_1KM_LTS:
_collection: EO:JRC:DAT:CLMS
product:type: vegetation_indices
_product_identifier: ndvi-lts_global_1km_10daily_v2
_dataset_identifier: ndvi-lts_global_1km_10daily_v2
metadata_mapping_from_product: CLMS_GLO_NDVI_333M
CLMS_CORINE:
_collection: EO:EEA:DAT:CORINE
Expand All @@ -3281,27 +3342,27 @@
CLMS_GLO_FCOVER_333M:
_collection: EO:JRC:DAT:CLMS
product:type: vegetation_properties
_product_identifier: fcover_global_300m_10daily_v1
_dataset_identifier: fcover_global_300m_10daily_v1
metadata_mapping_from_product: CLMS_GLO_NDVI_333M
CLMS_GLO_DMP_333M:
_collection: EO:JRC:DAT:CLMS
product:type: dry-gross_dry_matter_productivity
_product_identifier: dry-gross_dry_matter_productivity/dmp_global_300m_10daily_v1
_dataset_identifier: dmp_global_300m_10daily_v1
metadata_mapping_from_product: CLMS_GLO_NDVI_333M
CLMS_GLO_GDMP_333M:
_collection: EO:JRC:DAT:CLMS
product:type: dry-gross_dry_matter_productivity
_product_identifier: gdmp_global_300m_10daily_v1
_dataset_identifier: gdmp_global_300m_10daily_v1
metadata_mapping_from_product: CLMS_GLO_NDVI_333M
CLMS_GLO_FAPAR_333M:
_collection: EO:JRC:DAT:CLMS
product:type: vegetation_properties
_product_identifier: fapar_global_300m_10daily_v1
_dataset_identifier: fapar_global_300m_10daily_v1
metadata_mapping_from_product: CLMS_GLO_NDVI_333M
CLMS_GLO_LAI_333M:
_collection: EO:JRC:DAT:CLMS
product:type: vegetation_properties
_product_identifier: lai_global_300m_10daily_v1
_dataset_identifier: lai_global_300m_10daily_v1
metadata_mapping_from_product: CLMS_GLO_NDVI_333M
CLMS_HRVPP_ST:
_collection: EO:EEA:DAT:CLMS_HRVPP_ST
Expand Down
7 changes: 6 additions & 1 deletion tests/units/test_core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2331,7 +2331,12 @@ def test_available_sortables(self, mock_auth_session_request):
"usgs_satapi_aws": {"max_sort_params": None, "sortables": []},
"wekeo_cmems": None,
"wekeo_ecmwf": None,
"wekeo_main": None,
"wekeo_main": {
"sortables": [
"start_datetime",
],
"max_sort_params": None,
},
}
sortables = self.dag.available_sortables()
self.assertListEqual(
Expand Down
Loading