diff --git a/.circleci/config.yml b/.circleci/config.yml index 88ece28..d4b2d87 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,9 @@ common: &common command: .circleci/install_tippecanoe.sh - run: name: run tox - command: ~/.local/bin/tox + command: | + ~/.local/bin/tox + jobs: "python-3.6": <<: *common @@ -94,4 +96,4 @@ workflows: tags: only: /^[0-9]+.*/ branches: - ignore: /.*/ \ No newline at end of file + ignore: /.*/ diff --git a/docs/parameters.rst b/docs/parameters.rst index 8ced398..15fc2c0 100644 --- a/docs/parameters.rst +++ b/docs/parameters.rst @@ -29,6 +29,7 @@ Here is the full list of configuration parameters you can specify in a ``config. Label Maker expects to receive imagery tiles that are 256 x 256 pixels. You can specific the source of the imagery with one of: A template string for a tiled imagery service. Note that you will generally need an API key to obtain images and there may be associated costs. The above example requires a `Mapbox access token `_. Also see `OpenAerialMap `_ for open imagery. + The access token for TMS image formats can be read from an environment variable https://api.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}.jpg?access_token={ACCESS_TOKEN}" or added directly the imagery string. A GeoTIFF file location. Works with local files: ``'http://oin-hotosm.s3.amazonaws.com/593ede5ee407d70011386139/0/3041615b-2bdb-40c5-b834-36f580baca29.tif'`` @@ -67,4 +68,7 @@ Here is the full list of configuration parameters you can specify in a ``config. An optional list of integers representing the number of pixels to offset imagery. For example ``[15, -5]`` will move the images 15 pixels right and 5 pixels up relative to the requested tile bounds. **tms_image_format**: string - An option string that has the downloaded imagery's format such as `.jpg` or `.png` when it isn't provided by the endpoint + An option string that has the downloaded imagery's format such as `.jpg` or `.png` when it isn't provided by the endpoint + +**over_zoom**: int + An integer greater than 0. If set for XYZ tiles, it will fetch tiles from `zoom` + `over_zoom`, to create higher resolution tiles which fill out the bounds of the original zoom level. diff --git a/label_maker/utils.py b/label_maker/utils.py index 74bd012..c4ab809 100644 --- a/label_maker/utils.py +++ b/label_maker/utils.py @@ -1,18 +1,25 @@ # pylint: disable=unused-argument """Provide utility functions""" +import os from os import path as op from urllib.parse import urlparse, parse_qs -from mercantile import bounds +from mercantile import bounds, Tile, children from PIL import Image +import io import numpy as np import requests import rasterio from rasterio.crs import CRS from rasterio.warp import transform, transform_bounds +from rasterio.windows import Window WGS84_CRS = CRS.from_epsg(4326) +class SafeDict(dict): + def __missing__(self, key): + return '{' + key + '}' + def url(tile, imagery): """Return a tile url provided an imagery template and a tile""" return imagery.replace('{x}', tile[0]).replace('{y}', tile[1]).replace('{z}', tile[2]) @@ -40,11 +47,50 @@ def download_tile_tms(tile, imagery, folder, kwargs): image_format = get_image_format(imagery, kwargs) + if os.environ.get('ACCESS_TOKEN'): + token = os.environ.get('ACCESS_TOKEN') + imagery = imagery.format_map(SafeDict(ACCESS_TOKEN=token)) + r = requests.get(url(tile.split('-'), imagery), auth=kwargs.get('http_auth')) tile_img = op.join(folder, '{}{}'.format(tile, image_format)) - with open(tile_img, 'wb')as w: - w.write(r.content) + tile = tile.split('-') + + over_zoom = kwargs.get('over_zoom') + if over_zoom: + new_zoom = over_zoom + kwargs.get('zoom') + # get children + child_tiles = children(int(tile[0]), int(tile[1]), int(tile[2]), zoom=new_zoom) + child_tiles.sort() + + new_dim = 256 * (2 * over_zoom) + + w_lst = [] + for i in range (2 * over_zoom): + for j in range(2 * over_zoom): + window = Window(i * 256, j * 256, 256, 256) + w_lst.append(window) + + # request children + with rasterio.open(tile_img, 'w', driver='jpeg', height=new_dim, + width=new_dim, count=3, dtype=rasterio.uint8) as w: + for num, t in enumerate(child_tiles): + t = [str(t[0]), str(t[1]), str(t[2])] + r = requests.get(url(t, imagery), + auth=kwargs.get('http_auth')) + img = np.array(Image.open(io.BytesIO(r.content)), dtype=np.uint8) + try: + img = img.reshape((256, 256, 3)) # 4 channels returned from some endpoints, but not all + except ValueError: + img = img.reshape((256, 256, 4)) + img = img[:, :, :3] + img = np.rollaxis(img, 2, 0) + w.write(img, window=w_lst[num]) + else: + r = requests.get(url(tile, imagery), + auth=kwargs.get('http_auth')) + with open(tile_img, 'wb')as w: + w.write(r.content) return tile_img def get_tile_tif(tile, imagery, folder, kwargs): diff --git a/label_maker/validate.py b/label_maker/validate.py index 686a45c..fe086ed 100644 --- a/label_maker/validate.py +++ b/label_maker/validate.py @@ -34,5 +34,6 @@ 'imagery_offset': {'type': 'list', 'schema': {'type': 'integer'}, 'minlength': 2, 'maxlength': 2}, 'split_vals': {'type': 'list', 'schema': {'type': 'float'}}, 'split_names': {'type': 'list', 'schema': {'type': 'string'}}, - 'tms_image_format': {'type': 'string'} + 'tms_image_format': {'type': 'string'}, + 'over_zoom': {'type': 'integer', 'min': 1} } diff --git a/test/fixtures/integration/config_overzoom.integration.json b/test/fixtures/integration/config_overzoom.integration.json new file mode 100644 index 0000000..5dbe8e6 --- /dev/null +++ b/test/fixtures/integration/config_overzoom.integration.json @@ -0,0 +1,24 @@ +{"country": "portugal", + "bounding_box": [ + -9.4575, + 38.8467, + -9.4510, + 38.8513 + ], + "zoom": 17, + "classes": [ + { "name": "Water Tower", "filter": ["==", "man_made", "water_tower"] }, + { "name": "Building", "filter": ["has", "building"] }, + { "name": "Farmland", "filter": ["==", "landuse", "farmland"] }, + { "name": "Ruins", "filter": ["==", "historic", "ruins"] }, + { "name": "Parking", "filter": ["==", "amenity", "parking"] }, + { "name": "Roads", "filter": ["has", "highway"] } + ], + "imagery": "https://api.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}.jpg?access_token={ACCESS_TOKEN}", + "background_ratio": 1, + "ml_type": "classification", + "seed": 19, + "split_names": ["train", "test", "val"], + "split_vals": [0.7, 0.2, 0.1], + "over_zoom": 1 +} diff --git a/test/integration/test_classification_package.py b/test/integration/test_classification_package.py index 50977d2..034d43c 100644 --- a/test/integration/test_classification_package.py +++ b/test/integration/test_classification_package.py @@ -21,6 +21,10 @@ def setUpClass(cls): copyfile('test/fixtures/integration/labels-cl.npz', 'integration-cl-split/labels.npz') copytree('test/fixtures/integration/tiles', 'integration-cl-split/tiles') + + makedirs('integration-cl-overzoom') + copyfile('test/fixtures/integration/labels-cl.npz', 'integration-cl-overzoom/labels.npz') + makedirs('integration-cl-img-f') copyfile('test/fixtures/integration/labels-cl-img-f.npz', 'integration-cl-img-f/labels.npz') copytree('test/fixtures/integration/tiles_png', 'integration-cl-img-f/tiles') @@ -29,6 +33,7 @@ def setUpClass(cls): def tearDownClass(cls): rmtree('integration-cl') rmtree('integration-cl-split') + rmtree('integration-cl-overzoom') rmtree('integration-cl-img-f') def test_cli(self): @@ -80,6 +85,22 @@ def test_cli_3way_split(self): self.assertEqual(data['y_test'].shape, (2, 7)) self.assertEqual(data['y_val'].shape, (1, 7)) + def test_overzoom(self): + """Verify data.npz produced by CLI when overzoom is used""" + cmd = 'label-maker images --dest integration-cl-overzoom --config test/fixtures/integration/config_overzoom.integration.json' + cmd = cmd.split(' ') + subprocess.run(cmd, universal_newlines=True) + + cmd = 'label-maker package --dest integration-cl-overzoom --config test/fixtures/integration/config_overzoom.integration.json' + cmd = cmd.split(' ') + subprocess.run(cmd, universal_newlines=True) + + data = np.load('integration-cl-overzoom/data.npz') + + self.assertEqual(data['x_train'].shape, (6, 512, 512, 3)) + self.assertEqual(data['x_test'].shape, (2, 512, 512, 3)) + self.assertEqual(data['x_val'].shape, (1, 512, 512, 3)) + def test_tms_img_format(self): """Verify data.npz produced by CLI""" diff --git a/tox.ini b/tox.ini index fa91649..ac5ca0b 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ envlist = py37,py36 [testenv] +passenv = ACCESS_TOKEN extras = test commands= python -m pytest --cov label_maker --cov-report term-missing --ignore=venv @@ -46,4 +47,4 @@ include_trailing_comma = True multi_line_output = 3 line_length = 90 known_first_party = label_maker -default_section = THIRDPARTY \ No newline at end of file +default_section = THIRDPARTY