diff --git a/ci/requirements-py3.10.yml b/ci/requirements-py3.10.yml index cc11e4cb8a..617d89c755 100644 --- a/ci/requirements-py3.10.yml +++ b/ci/requirements-py3.10.yml @@ -8,8 +8,8 @@ dependencies: - ephem - h5py - numba - - numpy >= 1.16.0 - - pandas >= 0.25.0 + - numpy >= 1.17.3 + - pandas >= 1.3.0 - pip - pytest - pytest-cov diff --git a/ci/requirements-py3.11.yml b/ci/requirements-py3.11.yml index 5bd43c6df7..2ffdd932bd 100644 --- a/ci/requirements-py3.11.yml +++ b/ci/requirements-py3.11.yml @@ -8,8 +8,8 @@ dependencies: - ephem - h5py - numba - - numpy >= 1.16.0 - - pandas >= 0.25.0 + - numpy >= 1.17.3 + - pandas >= 1.3.0 - pip - pytest - pytest-cov diff --git a/ci/requirements-py3.12.yml b/ci/requirements-py3.12.yml index 156a408f48..250a9344c0 100644 --- a/ci/requirements-py3.12.yml +++ b/ci/requirements-py3.12.yml @@ -8,8 +8,8 @@ dependencies: - ephem - h5py - numba - - numpy >= 1.16.0 - - pandas >= 0.25.0 + - numpy >= 1.17.3 + - pandas >= 1.3.0 - pip - pytest - pytest-cov diff --git a/ci/requirements-py3.7-min.yml b/ci/requirements-py3.7-min.yml index 65dd6fa744..6371d5afb9 100644 --- a/ci/requirements-py3.7-min.yml +++ b/ci/requirements-py3.7-min.yml @@ -14,8 +14,8 @@ dependencies: - pip: - dataclasses - h5py==3.1.0 - - numpy==1.16.0 - - pandas==0.25.0 + - numpy==1.17.3 + - pandas==1.3.0 - scipy==1.5.0 - pytest-rerunfailures # conda version is >3.6 - pytest-remotedata # conda package is 0.3.0, needs > 0.3.1 diff --git a/ci/requirements-py3.7.yml b/ci/requirements-py3.7.yml index 49da67f3de..4b175ec532 100644 --- a/ci/requirements-py3.7.yml +++ b/ci/requirements-py3.7.yml @@ -8,8 +8,8 @@ dependencies: - ephem - h5py - numba - - numpy >= 1.16.0 - - pandas >= 0.25.0 + - numpy >= 1.17.3 + - pandas >= 1.3.0 - pip - pytest - pytest-cov diff --git a/ci/requirements-py3.8.yml b/ci/requirements-py3.8.yml index 0f5d63fd4a..814708a911 100644 --- a/ci/requirements-py3.8.yml +++ b/ci/requirements-py3.8.yml @@ -8,8 +8,8 @@ dependencies: - ephem - h5py - numba - - numpy >= 1.16.0 - - pandas >= 0.25.0 + - numpy >= 1.17.3 + - pandas >= 1.3.0 - pip - pytest - pytest-cov diff --git a/ci/requirements-py3.9.yml b/ci/requirements-py3.9.yml index 14151ce47a..24573894b7 100644 --- a/ci/requirements-py3.9.yml +++ b/ci/requirements-py3.9.yml @@ -8,8 +8,8 @@ dependencies: - ephem - h5py - numba - - numpy >= 1.16.0 - - pandas >= 0.25.0 + - numpy >= 1.17.3 + - pandas >= 1.3.0 - pip - pytest - pytest-cov diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst index 39081220f3..5f405d7536 100644 --- a/docs/sphinx/source/reference/iotools.rst +++ b/docs/sphinx/source/reference/iotools.rst @@ -53,6 +53,7 @@ of sources and file formats relevant to solar energy modeling. iotools.get_solcast_historic iotools.get_solcast_forecast iotools.get_solcast_live + iotools.get_solargis A :py:class:`~pvlib.location.Location` object may be created from metadata diff --git a/docs/sphinx/source/whatsnew.rst b/docs/sphinx/source/whatsnew.rst index c614c0de98..7fad810f8e 100644 --- a/docs/sphinx/source/whatsnew.rst +++ b/docs/sphinx/source/whatsnew.rst @@ -6,6 +6,7 @@ What's New These are new features and improvements of note in each release. +.. include:: whatsnew/v0.10.4.rst .. include:: whatsnew/v0.10.3.rst .. include:: whatsnew/v0.10.2.rst .. include:: whatsnew/v0.10.1.rst diff --git a/docs/sphinx/source/whatsnew/v0.10.4.rst b/docs/sphinx/source/whatsnew/v0.10.4.rst index 7dbf7d7d6d..216fed8b17 100644 --- a/docs/sphinx/source/whatsnew/v0.10.4.rst +++ b/docs/sphinx/source/whatsnew/v0.10.4.rst @@ -8,6 +8,8 @@ v0.10.4 (Anticipated March, 2024) Enhancements ~~~~~~~~~~~~ * Added the Huld PV model used by PVGIS (:pull:`1940`) +* Add :py:func:`pvlib.iotools.get_solargis` for retrieving Solargis + irradiance data. (:pull:`1969`) * Added function :py:func:`pvlib.shading.projected_solar_zenith_angle`, a common calculation in shading and tracking. (:issue:`1734`, :pull:`1904`) * Added :py:func:`~pvlib.iotools.get_solrad` for fetching irradiance data from @@ -15,7 +17,6 @@ Enhancements * Added metadata parsing to :py:func:`~pvlib.iotools.read_solrad` to follow the standard iotools convention of returning a tuple of (data, meta). Previously the function only returned a dataframe. (:pull:`1968`) - Bug fixes ~~~~~~~~~ * Fixed an error in solar position calculations when using @@ -33,6 +34,7 @@ Bug fixes ``temperature_model_parameters`` are specified on the passed ``system`` instead of on its ``arrays``. (:issue:`1759`). * :py:func:`pvlib.irradiance.ghi_from_poa_driesse_2023` now correctly makes use of the ``xtol`` argument. Previously, it was ignored. (:issue:`1970`, :pull:`1971`) +* Fixed incorrect unit conversion of precipitable water used for the Solcast iotools functions. * :py:class:`~pvlib.modelchain.ModelChain.infer_temperature_model` now raises a more useful error when the temperature model cannot be inferred (:issue:`1946`) @@ -49,6 +51,8 @@ Documentation Requirements ~~~~~~~~~~~~ +* Minimum version of pandas advanced from 0.25.0 to 1.3.0. (:pull:`1969`) +* Minimum version of numpy advanced from 1.16.0 to 1.17.3. (:pull:`1969`) Contributors @@ -59,5 +63,4 @@ Contributors * Cliff Hansen (:ghuser:`cwhanse`) * Roma Koulikov (:ghuser:`matsuobasho`) * Adam R. Jensen (:ghuser:`AdamRJensen`) -* Kevin Anderson (:ghuser:`kandersolar`) * Peter Dudfield (:ghuser:`peterdudfield`) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 0fbec16c1f..96259ecc24 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -34,3 +34,4 @@ from pvlib.iotools.solcast import get_solcast_live # noqa: F401 from pvlib.iotools.solcast import get_solcast_historic # noqa: F401 from pvlib.iotools.solcast import get_solcast_tmy # noqa: F401 +from pvlib.iotools.solargis import get_solargis # noqa: F401 diff --git a/pvlib/iotools/solargis.py b/pvlib/iotools/solargis.py new file mode 100644 index 0000000000..375c7ed3e8 --- /dev/null +++ b/pvlib/iotools/solargis.py @@ -0,0 +1,214 @@ +"""Functions to retrieve and parse irradiance data from Solargis.""" + +import pandas as pd +import requests +from dataclasses import dataclass +import io + +URL = 'https://solargis.info/ws/rest/datadelivery/request' + + +TIME_RESOLUTION_MAP = { + 5: 'MIN_5', 10: 'MIN_10', 15: 'MIN_15', 30: 'MIN_30', 60: 'HOURLY', + 'PT05M': 'MIN_5', 'PT5M': 'MIN_5', 'PT10M': 'MIN_10', 'PT15M': 'MIN_15', + 'PT30': 'MIN_30', 'PT60M': 'HOURLY', 'PT1H': 'HOURLY', 'P1D': 'DAILY', + 'P1M': 'MONTHLY', 'P1Y': 'YEARLY'} + + +@dataclass +class ParameterMap: + solargis_name: str + pvlib_name: str + conversion: callable = lambda x: x + + +# define the conventions between Solargis and pvlib nomenclature and units +VARIABLE_MAP = [ + # Irradiance (unit varies based on time resolution) + ParameterMap('GHI', 'ghi'), + ParameterMap('GHI_C', 'ghi_clear'), # this is stated in documentation + ParameterMap('GHIc', 'ghi_clear'), # this is used in practice + ParameterMap('DNI', 'dni'), + ParameterMap('DNI_C', 'dni_clear'), + ParameterMap('DNIc', 'dni_clear'), + ParameterMap('DIF', 'dhi'), + ParameterMap('GTI', 'poa_global'), + ParameterMap('GTI_C', 'poa_global_clear'), + ParameterMap('GTIc', 'poa_global_clear'), + # Solar position + ParameterMap('SE', 'solar_elevation'), + # SA -> solar_azimuth (degrees) (different convention) + ParameterMap("SA", "solar_azimuth", lambda x: x + 180), + # Weather / atmospheric parameters + ParameterMap('TEMP', 'temp_air'), + ParameterMap('TD', 'temp_dew'), + # surface_pressure (hPa) -> pressure (Pa) + ParameterMap('AP', 'pressure', lambda x: x*100), + ParameterMap('RH', 'relative_humidity'), + ParameterMap('WS', 'wind_speed'), + ParameterMap('WD', 'wind_direction'), + ParameterMap('INC', 'aoi'), # angle of incidence of direct irradiance + # precipitable_water (kg/m2) -> precipitable_water (cm) + ParameterMap('PWAT', 'precipitable_water', lambda x: x/10), +] + +METADATA_FIELDS = [ + 'issued', 'site name', 'latitude', 'longitude', 'elevation', + 'summarization type', 'summarization period' +] + + +# Variables that use "-9" as nan values +NA_9_COLUMNS = ['GHI', 'GHIc', 'DNI', 'DNIc', 'DIF', 'GTI', 'GIc', 'KT', 'PAR', + 'PREC', 'PWAT', 'SDWE', 'SFWE'] + + +def get_solargis(latitude, longitude, start, end, variables, api_key, + time_resolution, timestamp_type='center', tz='GMT+00', + terrain_shading=True, url=URL, map_variables=True, + timeout=30): + """ + Retrieve irradiance time series data from Solargis. + + The Solargis [1]_ API is described in [2]_. + + Parameters + ---------- + latitude: float + In decimal degrees, between -90 and 90, north is positive (ISO 19115) + longitude: float + In decimal degrees, between -180 and 180, east is positive (ISO 19115) + start : datetime-like + Start date of time series. + end : datetime-like + End date of time series. + variables : list + List of variables to request, see [2]_ for options. + api_key : str + API key. + time_resolution : str, {'PT05M', 'PT10M', 'PT15M', 'PT30', 'PT1H', 'P1D', 'P1M', 'P1Y'} + Time resolution as an integer number of minutes (e.g. 5, 60) + or an ISO 8601 duration string (e.g. "PT05M", "PT60M", "P1M"). + timestamp_type : {'start', 'center', 'end'}, default: 'center' + Labeling of time stamps of the return data. + tz : str, default : 'GMT+00' + Timezone of `start` and `end` in the format "GMT+hh" or "GMT-hh". + terrain_shading : boolean, default: True + Whether to account for horizon shading. + url : str, default : :const:`pvlib.iotools.solargis.URL` + Base url of Solargis API. + map_variables : boolean, default: True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable :const:`VARIABLE_MAP`. + timeout : int or float, default: 30 + Time in seconds to wait for server response before timeout + + Returns + ------- + data : DataFrame + DataFrame containing time series data. + meta : dict + Dictionary containing metadata. + + Raises + ------ + requests.HTTPError + A message from the Solargis server if the request is rejected + + Notes + ----- + Each XML request is limited to retrieving 31 days of data. + + The variable units depends on the time frequency, e.g., the unit for + sub-hourly irradiance data is :math:`W/m^2`, for hourly data it is + :math:`Wh/m^2`, and for daily data it is :math:`kWh/m^2`. + + References + ---------- + .. [1] `Solargis `_ + .. [2] `Solargis API User Guide + `_ + + Examples + -------- + >>> # Retrieve two days of irradiance data from Solargis + >>> data, meta = response = pvlib.iotools.get_solargis( + >>> latitude=48.61259, longitude=20.827079, + >>> start='2022-01-01', end='2022-01-02', + >>> variables=['GHI', 'DNI'], time_resolution='PT05M', api_key='demo') + """ # noqa: E501 + # Use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted + start = pd.to_datetime(start) + end = pd.to_datetime(end) + + headers = {'Content-Type': 'application/xml'} + + # Solargis recommends creating a unique site_id for each location request. + # The site_id does not impact the data retrieval and is used for debugging. + site_id = f"latitude_{latitude}_longitude_{longitude}" + + request_xml = f''' + + + + {timestamp_type.upper()} + {tz} + + ''' # noqa: E501 + + response = requests.post(url + "?key=" + api_key, headers=headers, + data=request_xml.encode('utf8'), timeout=timeout) + + if response.ok is False: + raise requests.HTTPError(response.json()) + + # Parse metadata + header = pd.read_xml(io.StringIO(response.text), parser='etree') + meta_lines = header['metadata'].iloc[0].split('#') + meta_lines = [line.strip() for line in meta_lines] + meta = {} + for line in meta_lines: + if ':' in line: + key = line.split(':')[0].lower() + if key in METADATA_FIELDS: + meta[key] = ':'.join(line.split(':')[1:]) + meta['latitude'] = float(meta['latitude']) + meta['longitude'] = float(meta['longitude']) + meta['altitude'] = float(meta.pop('elevation').replace('m a.s.l.', '')) + + # Parse data + data = pd.read_xml(io.StringIO(response.text), xpath='.//doc:row', + namespaces={'doc': 'http://geomodel.eu/schema/ws/data'}, + parser='etree') + data.index = pd.to_datetime(data['dateTime']) + # when requesting one variable, it is necessary to convert dataframe to str + data = data['values'].astype(str).str.split(' ', expand=True) + data = data.astype(float) + data.columns = header['columns'].iloc[0].split() + + # Replace "-9" with nan values for specific columns + for variable in data.columns: + if variable in NA_9_COLUMNS: + data[variable] = data[variable].replace(-9, pd.NA) + + # rename and convert variables + if map_variables: + for variable in VARIABLE_MAP: + if variable.solargis_name in data.columns: + data.rename( + columns={variable.solargis_name: variable.pvlib_name}, + inplace=True + ) + data[variable.pvlib_name] = data[ + variable.pvlib_name].apply(variable.conversion) + + return data, meta diff --git a/pvlib/iotools/solcast.py b/pvlib/iotools/solcast.py index 4fcee40050..5abd9c724e 100644 --- a/pvlib/iotools/solcast.py +++ b/pvlib/iotools/solcast.py @@ -35,7 +35,7 @@ class ParameterMap: "azimuth", "solar_azimuth", lambda x: -x % 360 ), # precipitable_water (kg/m2) -> precipitable_water (cm) - ParameterMap("precipitable_water", "precipitable_water", lambda x: x*10), + ParameterMap("precipitable_water", "precipitable_water", lambda x: x/10), # zenith -> solar_zenith ParameterMap("zenith", "solar_zenith"), # clearsky diff --git a/pvlib/tests/iotools/test_solargis.py b/pvlib/tests/iotools/test_solargis.py new file mode 100644 index 0000000000..55882e91c5 --- /dev/null +++ b/pvlib/tests/iotools/test_solargis.py @@ -0,0 +1,68 @@ +import pandas as pd +import pytest +import pvlib +import requests +from ..conftest import (RERUNS, RERUNS_DELAY, assert_frame_equal, + assert_index_equal) + + +@pytest.fixture +def hourly_index(): + hourly_index = pd.date_range(start='2022-01-01 00:30+01:00', freq='60min', + periods=24, name='dateTime') + hourly_index.freq = None + return hourly_index + + +@pytest.fixture +def hourly_index_start_utc(): + hourly_index_left_utc = pd.date_range( + start='2023-01-01 00:00+00:00', freq='30min', periods=24*2, + name='dateTime') + hourly_index_left_utc.freq = None + return hourly_index_left_utc + + +@pytest.fixture +def hourly_dataframe(hourly_index): + ghi = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 5.0, 73.0, 152.0, 141.0, 105.0, + 62.0, 65.0, 62.0, 11.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + dni = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 30.0, 233.0, 301.0, 136.0, 32.0, + 0.0, 3.0, 77.0, 5.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + return pd.DataFrame(data={'ghi': ghi, 'dni': dni}, index=hourly_index) + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_solargis(hourly_dataframe): + data, meta = pvlib.iotools.get_solargis( + latitude=48.61259, longitude=20.827079, + start='2022-01-01', end='2022-01-01', + tz='GMT+01', variables=['GHI', 'DNI'], + time_resolution='HOURLY', api_key='demo') + assert_frame_equal(data, hourly_dataframe) + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_solargis_utc_start_timestamp(hourly_index_start_utc): + data, meta = pvlib.iotools.get_solargis( + latitude=48.61259, longitude=20.827079, + start='2023-01-01', end='2023-01-01', + variables=['GTI'], + timestamp_type='start', + time_resolution='MIN_30', + map_variables=False, api_key='demo') + assert 'GTI' in data.columns # assert that variables aren't mapped + assert_index_equal(data.index, hourly_index_start_utc) + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_solargis_http_error(): + # Test if HTTPError is raised if date outside range is specified + with pytest.raises(requests.HTTPError, match="data coverage"): + _, _ = pvlib.iotools.get_solargis( + latitude=48.61259, longitude=20.827079, + start='1920-01-01', end='1920-01-01', # date outside range + variables=['GHI', 'DNI'], time_resolution='HOURLY', api_key='demo') diff --git a/pvlib/tests/iotools/test_solcast.py b/pvlib/tests/iotools/test_solcast.py index 19b00b8611..3879d88b20 100644 --- a/pvlib/tests/iotools/test_solcast.py +++ b/pvlib/tests/iotools/test_solcast.py @@ -174,9 +174,9 @@ def test_get_solcast_tmy( ), pd.DataFrame( [[9.4200e+02, 8.4300e+02, 1.0174e+05, 3.0000e+01, 7.8000e+00, - 3.1600e+02, 1.0100e+03, 2.0000e+00, 4.6000e+00, 1.6400e+02, 90], + 3.1600e+02, 1.0100e+03, 2.0000e+00, 4.6000e+00, 1.6400e+00, 90], [9.3600e+02, 8.3200e+02, 1.0179e+05, 3.0000e+01, 7.9000e+00, - 3.1600e+02, 9.9600e+02, 1.4000e+01, 4.5000e+00, 1.6300e+02, 0]], + 3.1600e+02, 9.9600e+02, 1.4000e+01, 4.5000e+00, 1.6300e+00, 0]], columns=[ 'dni', 'ghi', 'pressure', 'temp_air', 'wind_speed', 'wind_direction', 'poa_global', 'solar_azimuth', diff --git a/pyproject.toml b/pyproject.toml index 75970c9f92..0053f0e568 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,8 +11,8 @@ authors = [ ] requires-python = ">=3.7" dependencies = [ - 'numpy >= 1.16.0', - 'pandas >= 0.25.0', + 'numpy >= 1.17.3', + 'pandas >= 1.3.0', 'pytz', 'requests', 'scipy >= 1.5.0',