Supertiles

docs/parameters.rst

@@ -44,7 +44,7 @@ Here is the full list of configuration parameters you can specify in a ``config.
 	One of ``'classification'``, ``'object-detection'``, or ``'segmentation'``. This defines the output format for the final label numpy arrays (``y_train`` and ``y_test``).
 
  	``'classification'``
- 		Output is an array of ``len(classes) + 1``. Each array value will be either `1` or `0` based on whether it matches the class at the same index. The additional array element belongs to the background class, which will always be the first element. 
+ 		Output is an array of ``len(classes) + 1``. Each array value will be either `1` or `0` based on whether it matches the class at the same index. The additional array element belongs to the background class, which will always be the first element.
 
  	``'object-detection'``
  		Output is an array of bounding boxes of the form ``[xmin, ymin, width, height, class_index]``. In this case, the values are pixel values measured from the upper left-hand corner (not latitude and longitude values). Each feature is tested against each class, so if a feature matches two or more classes, it will have the corresponding number of bounding boxes created.
@@ -65,3 +65,6 @@ Here is the full list of configuration parameters you can specify in a ``config.
 
 **imagery_offset**:  list of ints
 	An optional list of integers representing the number of pixels to offset imagery. For example ``[15, -5]`` will move the images 15 pixels right and 5 pixels up relative to the requested tile bounds.
+
+**over_zoom**: int
+	An integer greater than 0, if set for XYZ tiles, it will fetch a tiles from a zoom level higher than specified, to fill out the bounds of the original zoom level. 

label_maker/main.py

@@ -50,7 +50,7 @@ def parse_args(args):
     subparsers.add_parser('download', parents=[pparser], help='', formatter_class=dhf)
     l = subparsers.add_parser('labels', parents=[pparser], help='', formatter_class=dhf)
     p = subparsers.add_parser('preview', parents=[pparser], help='', formatter_class=dhf)
-    subparsers.add_parser('images', parents=[pparser], help='', formatter_class=dhf)
+    i = subparsers.add_parser('images', parents=[pparser], help='', formatter_class=dhf)
     subparsers.add_parser('package', parents=[pparser], help='', formatter_class=dhf)
 
     # labels has an optional parameter
@@ -60,6 +60,9 @@ def parse_args(args):
     p.add_argument('-n', '--number', default=5, type=int,
                    help='number of examples images to create per class')
 
+    #i.add_argument('-m', '--multispectral', action='store_true') #this should be in the config not argument
+
+
     # turn namespace into dictinary
     parsed_args = vars(parser.parse_args(args))
 

label_maker/utils.py

@@ -3,13 +3,15 @@
 from os import path as op
 from urllib.parse import urlparse, parse_qs
 
-from mercantile import bounds
+from mercantile import bounds, Tile, children
 from PIL import Image
+import io
 import numpy as np
 import requests
 import rasterio
 from rasterio.crs import CRS
 from rasterio.warp import transform, transform_bounds
+from rasterio.windows import Window
 
 WGS84_CRS = CRS.from_epsg(4326)
 
@@ -31,11 +33,46 @@ def download_tile_tms(tile, imagery, folder, kwargs):
     """Download a satellite image tile from a tms endpoint"""
     o = urlparse(imagery)
     _, image_format = op.splitext(o.path)
-    r = requests.get(url(tile.split('-'), imagery),
-                     auth=kwargs.get('http_auth'))
     tile_img = op.join(folder, '{}{}'.format(tile, image_format))
-    with open(tile_img, 'wb')as w:
-        w.write(r.content)
+    tile = tile.split('-')
+
+
+    over_zoom = kwargs.get('over_zoom') or 0
+    new_zoom = over_zoom + kwargs.get('zoom')
+    if over_zoom:
+        print('creating supertiles')
+        #get children
+        child_tiles = children(int(tile[0]), int(tile[1]), int(tile[2]), zoom=new_zoom)
+        child_tiles.sort()
+
+        new_dim = 256 * (2 * over_zoom)
+
+        w_lst = []
+        for i in range (2 * over_zoom):
+            for j in range(2 * over_zoom):
+                window = Window(i * 256, j * 256, 256, 256)
+                w_lst.append(window)
+
+        #request children
+        with rasterio.open(tile_img, 'w', driver='jpeg', height=new_dim,
+                        width=new_dim, count=3, dtype=rasterio.uint8) as w:
+                for num, t in enumerate(child_tiles):
+                    t = [str(t[0]), str(t[1]), str(t[2])]
+                    r = requests.get(url(t, imagery),
+                                    auth=kwargs.get('http_auth'))
+                    img = np.array(Image.open(io.BytesIO(r.content)), dtype=np.uint8)
+                    try:
+                        img = img.reshape((256, 256, 3)) #4 channels returned from some endpoints, but not all
+                    except ValueError:
+                        img = img.reshape((256, 256, 4))
+                    img = img[:, :, :3]
+                    img = np.rollaxis(img, 2, 0)
+                    w.write(img, window=w_lst[num])
+    else:
+        r = requests.get(url(tile, imagery),
+                         auth=kwargs.get('http_auth'))
+        with open(tile_img, 'wb')as w:
+            w.write(r.content)
     return tile_img
 
 def get_tile_tif(tile, imagery, folder, kwargs):

label_maker/validate.py

@@ -33,5 +33,6 @@
     'seed': {'type': 'integer'},
     'imagery_offset': {'type': 'list', 'schema': {'type': 'integer'}, 'minlength': 2, 'maxlength': 2},
     'split_vals': {'type': 'list', 'schema': {'type': 'float'}},
-    'split_names': {'type': 'list', 'schema': {'type': 'string'}}
+    'split_names': {'type': 'list', 'schema': {'type': 'string'}},
+    'over_zoom': {'type': 'integer', 'min': 1}
 }

test/fixtures/integration/config_3way.integration.json

@@ -19,5 +19,4 @@
   "ml_type": "classification",
   "seed": 19,
   "split_names": ["train", "test", "val"],
-  "split_vals": [0.7, 0.2, 0.1]
-}
+  "split_vals": [0.7, 0.2, 0.1]}

test/fixtures/integration/config_overzoom.integration.json

@@ -0,0 +1,24 @@
+{"country": "portugal",
+  "bounding_box": [
+    -9.4575,
+    38.8467,
+    -9.4510,
+    38.8513
+  ],
+  "zoom": 17,
+  "classes": [
+    { "name": "Water Tower", "filter": ["==", "man_made", "water_tower"] },
+    { "name": "Building", "filter": ["has", "building"] },
+    { "name": "Farmland", "filter": ["==", "landuse", "farmland"] },
+    { "name": "Ruins", "filter": ["==", "historic", "ruins"] },
+    { "name": "Parking", "filter": ["==", "amenity", "parking"] },
+    { "name": "Roads", "filter": ["has", "highway"] }
+  ],
+  "imagery": "https://api.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}.jpg?access_token=TOKEN",
+  "background_ratio": 1,
+  "ml_type": "classification",
+  "seed": 19,
+  "split_names": ["train", "test", "val"],
+  "split_vals": [0.7, 0.2, 0.1],
+  "over_zoom": 1
+}

test/integration/test_classification_package.py

@@ -21,10 +21,15 @@ def setUpClass(cls):
         copyfile('test/fixtures/integration/labels-cl.npz', 'integration-cl-split/labels.npz')
         copytree('test/fixtures/integration/tiles', 'integration-cl-split/tiles')
 
+        makedirs('integration-cl-overzoom')
+        copyfile('test/fixtures/integration/labels-cl.npz', 'integration-cl-overzoom/labels.npz')
+        copytree('test/fixtures/integration/tiles_overzoom', 'integration-cl-overzoom/tiles')
+
     @classmethod
     def tearDownClass(cls):
         rmtree('integration-cl')
         rmtree('integration-cl-split')
+        rmtree('integration-cl-overzoom')
 
     def test_cli(self):
         """Verify data.npz produced by CLI"""
@@ -73,4 +78,16 @@ def test_cli_3way_split(self):
         # validate label data with shapes
         self.assertEqual(data['y_train'].shape, (5, 7))
         self.assertEqual(data['y_test'].shape, (2, 7))
-        self.assertEqual(data['y_val'].shape, (1, 7))
+        self.assertEqual(data['y_val'].shape, (1, 7))
+
+    def test_overzoom(self):
+        """Verify data.npz produced by CLI when overzoom is used"""
+        cmd = 'label-maker package --dest integration-cl-overzoom --config test/fixtures/integration/config_overzoom.integration.json'
+        cmd = cmd.split(' ')
+        subprocess.run(cmd, universal_newlines=True)
+
+        data = np.load('integration-cl-overzoom/data.npz')
+
+        self.assertEqual(data['x_train'].shape, (6, 512, 512, 3))
+        self.assertEqual(data['x_test'].shape, (2, 512, 512, 3))
+        self.assertEqual(data['x_val'].shape, (1, 512, 512, 3))