diff --git a/README.md b/README.md index 2be4b38eb..6e106290a 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,8 @@ python -m pip install 'pystac[orjson]' ``` If you would like to use a custom `RetryStacIO` class for automatically retrying -network requests when reading with PySTAC, you'll need +network requests when reading with PySTAC, or if you have non-ASCII characters in +your urls you'll need [`urllib3`](https://urllib3.readthedocs.io/en/stable/): ```shell diff --git a/pystac/stac_io.py b/pystac/stac_io.py index fa71dd7c0..4da8aec2e 100644 --- a/pystac/stac_io.py +++ b/pystac/stac_io.py @@ -286,8 +286,9 @@ def read_text_from_href(self, href: str) -> str: """Reads file as a UTF-8 string. If ``href`` has a "scheme" (e.g. if it starts with "https://") then this will - use :func:`urllib.request.urlopen` to open the file and read the contents; - otherwise, :func:`open` will be used to open a local file. + use :func:`urllib.request.urlopen` (or func:`urllib3.request` if available) + to open the file and read the contents; otherwise, :func:`open` will be used + to open a local file. Args: @@ -297,9 +298,19 @@ def read_text_from_href(self, href: str) -> str: if _is_url(href): try: logger.debug(f"GET {href} Headers: {self.headers}") - req = Request(href, headers=self.headers) - with urlopen(req) as f: - href_contents = f.read().decode("utf-8") + if HAS_URLLIB3: + with urllib3.request( + "GET", + href, + headers=self.headers, + preload_content=False, # type: ignore + ) as f: + href_contents = f.read().decode("utf-8") + else: + req = Request(href, headers=self.headers) + with urlopen(req) as f: + href_contents = f.read().decode("utf-8") + except HTTPError as e: raise Exception(f"Could not read uri {href}") from e else: