Merge branch 'dev' into rc

Author: jirikadlec2

docker/Dockerfile.worker

@@ -12,7 +12,7 @@ ENV VERSION=${VERSION:-${DOCKER_TAG:-master}}
 RUN echo "$VERSION" >/etc/qc_tool_version.txt \
     && apt-get -y update \
     && apt-get -y upgrade \
-    && apt-get -y install wget unzip
+    && apt-get -y install wget unzip time
 
 # Install python and bootstrap pip.
 RUN cd /usr/local/src \
@@ -24,7 +24,7 @@ RUN cd /usr/local/src \
 RUN apt-get -y install gdal-bin python3-gdal \
     && apt-get -y install python3-psycopg2 \
     && pip3 install numpy \
-    && pip3 install scikit-image reportlab \
+    && pip3 install scikit-image reportlab requests \
     && pip3 install bottle \
     && pip3 install supervisor
 

src/qc_tool/raster/inspire.py

@@ -2,20 +2,35 @@
 # -*- coding: utf-8 -*-
 
 
-"""
-The last implementation used external inspire validation service at `<http://inspire-geoportal.ec.europa.eu/GeoportalProxyWebServices/resources/INSPIREResourceTester>`_.
-Such service has been discontinued.
-
-Until new implementation come, the check always results in cancelled.
-Cancelled result does not prevent later submittion of the delivery.
-This way it may be avoided adapting and hampering already stable product definitions.
-"""
-
-
 DESCRIPTION = "Metadata are in accord with INSPIRE specification."
 IS_SYSTEM = False
 
 
+def locate_xml_file(layer_filepath):
+    # The INSPIRE XML file can be LAYER.xml or LAYER.tif.xml or LAYER_metadata.xml.
+    # The file can also be located in a "metadata" subdirectory.
+    for xml_filepath in [layer_filepath.parent.joinpath(layer_filepath.name + ".xml"),
+                         layer_filepath.parent.joinpath(layer_filepath.stem + ".xml"),
+                         layer_filepath.parent.joinpath(layer_filepath.stem + "_metadata.xml"),
+                         layer_filepath.parent.joinpath("metadata", layer_filepath.name + ".xml"),
+                         layer_filepath.parent.joinpath("metadata", layer_filepath.stem + ".xml"),
+                         layer_filepath.parent.joinpath("metadata", layer_filepath.stem + "_metadata.xml")]:
+        if xml_filepath.exists():
+            return xml_filepath
+    return None
+
+
 def run_check(params, status):
-    """The check always results in cancelled."""
-    status.cancelled("The check is not implemented yet.")
+    from qc_tool.vector.helper import do_inspire_check
+    from qc_tool.raster.helper import do_raster_layers
+
+    for layer_def in do_raster_layers(params):
+        # Verify if there is any xml INSPIRE metadata file to check.
+        xml_filepath = locate_xml_file(layer_def["src_filepath"])
+        if xml_filepath is None:
+            status.failed("Metadata file for {:s} has not been found.".format(layer_def["src_filepath"].name))
+            return
+
+        # Validate the xml file using INSPIRE validator service
+        export_prefix = "s{:02d}_{:s}_inspire".format(params["step_nr"], layer_def["src_filepath"].stem)
+        do_inspire_check(xml_filepath, export_prefix, params["output_dir"], status)

src/qc_tool/raster/mmu.py

@@ -79,6 +79,22 @@ def patch_touches_cell_with_value(coordinates, tile, neighbour_values):
                 return True
     return False
 
+def patch_touches_raster_edge(coordinates, raster_nrows, raster_ncols):
+    """
+    Takes a list of [row, column] cell coordinates and inspects if at least one of the coordinates
+    is located exactly at the specified edge of the tile.
+    This function can be used for detecting patches touching the raster bounding box.
+    :param coordinates: The patch of same-value raster cells.
+    :param raster_nrows: Number of rows in the whole source raster.
+    :param raster_ncols: Number of columns in the whole source raster.
+    :return: True if at least one cell in the patch is located at source raster edge.
+    """
+    for coor in coordinates:
+        if coor[0] == 0 or coor[1] == 0 or coor[0] == raster_ncols - 1 or coor[1] == raster_nrows - 1:
+            return True
+    return False
+
+
 def export(regions, raster_ds, gpkg_filepath):
     """
     Exports a list of lessMMU region dictionaries to geopackage.
@@ -216,22 +232,22 @@ def run_check(params, status):
                 write_percent(percent_filepath, progress_percent)
 
             if tileRow == 0:
-                # first row
+                # First row
                 yOff = 0
                 yOffInner = 0
                 yOffRelative = 0
                 block_height = BLOCKSIZE + buffer_width
                 block_height_inner = BLOCKSIZE
             else:
-                # middle row
+                # Middle row
                 yOff = tileRow * BLOCKSIZE
                 yOffInner = yOff + buffer_width
                 yOffRelative = buffer_width
                 block_height = blocksize_with_buffer
                 block_height_inner = BLOCKSIZE
 
             if tileRow == last_row:
-                # special case for last row - adjust block width
+                # Last row is a special case - block width must be adjusted.
                 block_height = nRasterRows - yOff
                 block_height_inner = block_height - buffer_width
 
@@ -242,22 +258,22 @@ def run_check(params, status):
             # TILES: ITERATE COLUMNS
             for tileCol in range(nTileCols):
                 if tileCol == 0:
-                    # first column
+                    # First column
                     xOff = 0
                     xOffInner = 0
                     xOffRelative = 0
                     block_width = BLOCKSIZE + buffer_width
                     block_width_inner = BLOCKSIZE
                 else:
-                    # middle column
+                    # Middle column
                     xOff = tileCol * BLOCKSIZE
                     xOffInner = xOff + buffer_width
                     xOffRelative = buffer_width
                     block_width = blocksize_with_buffer
                     block_width_inner = BLOCKSIZE
 
                 if tileCol == last_col:
-                    # special case for last column - adjust block width
+                    # Last column is a special case - block width must be adjusted.
                     block_width = nRasterCols - xOff
                     block_width_inner = block_width - buffer_width
 
@@ -286,7 +302,6 @@ def run_check(params, status):
                 tile_inner = tile_buffered[yOffRelative: yOffRelative + block_height_inner, xOffRelative: xOffRelative + block_width_inner]
 
                 # read inner array (without buffer)
-                # tile_inner = ds.ReadAsArray(xOffInner, yOffInner, block_width_inner, block_height_inner)
 
                 # label the inner array and find patches < MMU
                 labels_inner = measure.label(tile_inner, background=NODATA, neighbors=4)
@@ -327,6 +342,8 @@ def run_check(params, status):
                         regions_lessMMU_except.append(lessMMU_info)
                     elif patch_touches_cell_with_value(r.coords, tile_inner, neighbour_exclude_values):
                         regions_lessMMU_except.append(lessMMU_info)
+                    elif patch_touches_raster_edge(absolute_coords, nRasterRows, nRasterCols):
+                        regions_lessMMU_except.append(lessMMU_info)
                     else:
                         regions_lessMMU.append(lessMMU_info)
 
@@ -338,10 +355,7 @@ def run_check(params, status):
                     msg = msg.format(tr=tileRow, ntr=nTileRows, tc=tileCol, w=block_width, h=block_height)
                     write_progress(progress_filepath, msg)
 
-                # read the outer array expanded by buffer with width=number of pixels in MMU
-
-                # no need, already read ...
-                # tile_buffered = ds.ReadAsArray(xOff, yOff, block_width, block_height)
+                # processing the outer array expanded by buffer with width=number of pixels in MMU
 
                 # optimization: set pixels not within buffer zone (deep inside outer array) to background.
                 inner_buf_startcol = xOffRelative + MMU
@@ -352,11 +366,6 @@ def run_check(params, status):
                     tile_buffered[inner_buf_startrow:inner_buf_endrow,
                                   inner_buf_startcol:inner_buf_endcol] = NODATA
 
-                # no need to reclassify, already reclassified ..
-                # reclassify outer tile array if some patches should be grouped together
-                # if use_reclassify:
-                #     tile_buffered = reclassify_values(tile_buffered, params["groupcodes"])
-
                 labels_buf = measure.label(tile_buffered, background=NODATA, neighbors=4)
                 buf_regions = measure.regionprops(labels_buf)
                 buf_regions_small = [r for r in buf_regions if r.area < MMU]
@@ -387,11 +396,13 @@ def run_check(params, status):
                                         "area": r_buf.area, "value": val,
                                         "coords": absolute_coords}
 
-
+                        # handling special cases (exception patches)
                         if val in exclude_values:
                             regions_lessMMU_except.append(lessMMU_info)
                         elif patch_touches_cell_with_value(r_buf.coords, tile_buffered, neighbour_exclude_values):
                             regions_lessMMU_except.append(lessMMU_info)
+                        elif patch_touches_raster_edge(absolute_coords, nRasterRows, nRasterCols):
+                            regions_lessMMU_except.append(lessMMU_info)
                         else:
                             regions_lessMMU.append(lessMMU_info)
 

src/qc_tool/raster/unzip.py

@@ -2,13 +2,11 @@
 # -*- coding: utf-8 -*-
 
 
-from qc_tool.vector.helper import do_unzip
-
-
 DESCRIPTION = "Delivery file can be unzipped."
 IS_SYSTEM = True
 
 
 def run_check(params, status):
     # Raster layers are unzipped to the temporary directory r_unzip.d.
+    from qc_tool.vector.helper import do_unzip
     do_unzip(params["filepath"], params["tmp_dir"].joinpath("r_unzip.d"), status)