Implementations of inspire checks have been removed.

See docstrings in both checks.
Unusable inspire tests have been removed.
Tests have been adapted to inspire check always returning cancelled.
The code of test_raster_products has been unified.
The code of test_vector_products has been unified.
Nonfunctional skip_inspire_check has been removed from ProductTestCase.

Author: igokr

src/qc_tool/raster/inspire.py

@@ -2,74 +2,20 @@
 # -*- coding: utf-8 -*-
 
 
-import json
-import socket
-from urllib.error import HTTPError
-from urllib.error import URLError
-from urllib.request import Request
-from urllib.request import urlopen
-from xml.etree import ElementTree
+"""
+The last implementation used external inspire validation service at `<http://inspire-geoportal.ec.europa.eu/GeoportalProxyWebServices/resources/INSPIREResourceTester>`_.
+Such service has been discontinued.
+
+Until new implementation come, the check always results in cancelled.
+Cancelled result does not prevent later submittion of the delivery.
+This way it may be avoided adapting and hampering already stable product definitions.
+"""
 
 
 DESCRIPTION = "Metadata are in accord with INSPIRE specification."
 IS_SYSTEM = False
 
-INSPIRE_SERVICE_URL = "http://inspire-geoportal.ec.europa.eu/GeoportalProxyWebServices/resources/INSPIREResourceTester"
-
 
 def run_check(params, status):
-    from qc_tool.raster.helper import do_raster_layers
-
-    for layer_def in do_raster_layers(params):
-        # Find the xml metadata file, it can be .xml or .tif.xml.
-        for xml_filepath in [layer_def["src_filepath"].with_suffix(".xml"),
-                             layer_def["src_filepath"].with_suffix(".tif.xml")]:
-            if xml_filepath.exists():
-                break
-        else:
-            status.failed("Metadata file for {:s} has not been found.".format(layer_def["src_filepath"].name))
-            continue
-
-        # Check if the metadata file is a valid xml document.
-        try:
-            ElementTree.parse(str(xml_filepath))
-        except ElementTree.ParseError:
-            status.failed("Metadata file {:s} is not a valid XML document.".format(xml_filepath.name))
-            continue
-
-        # Post the metadata file content to INSPIRE validator service.
-        try:
-            req = Request(INSPIRE_SERVICE_URL,
-                          data=xml_filepath.read_bytes(),
-                          headers={"Accept": "application/json", "Content-Type": "text/plain"})
-            with urlopen(req, timeout=60) as resp:
-                json_data = json.loads(resp.read().decode("utf-8"))
-        except HTTPError as ex:
-            status.cancelled("Unable to validate metadata of {:s}: {:s}."
-                             .format(xml_filepath.name, str(ex)))
-            continue
-        except URLError:
-            status.cancelled("Unable to validate metadata of {:s}: Service not available."
-                             .format(xml_filepath.name))
-            continue
-        except socket.timeout:
-            status.cancelled("Unable to validate metadata of {:s}: Connection timeout."
-                             .format(xml_filepath.name))
-            continue
-        except Exception as ex:
-            status.cancelled("Unable to validate metadata of {:s}: {:s}"
-                             .format(xml_filepath.name, repr(ex)))
-
-        # The metadata content is valid if the completeness indicator is equal to 100.0.
-        # Warnings are ignored.
-        try:
-            inspire_ok = json_data["value"]["CompletenessIndicator"] == 100
-        except:
-            inspire_ok = False
-
-        if not inspire_ok:
-            status.failed("Metadata file {:s} is not valid.".format(xml_filepath.name))
-            error_filename = "s{:02d}_{:s}_metadata_error.json".format(params["step_nr"], layer_def["src_filepath"].stem)
-            error_filepath = params["output_dir"].joinpath(error_filename)
-            error_filepath.write_text(json.dumps(json_data, indent=4, sort_keys=True))
-            status.add_attachment(error_filepath.name)
+    """The check always results in cancelled."""
+    status.cancelled("The check is not implemented yet.")

src/qc_tool/test/helper.py

@@ -22,9 +22,6 @@ def setUp(self):
         # Set up boundary dir to testing sources.
         CONFIG["boundary_dir"] = TEST_DATA_DIR.joinpath("boundaries")
 
-        # online INSPIRE validator is skipped in tests.
-        CONFIG["skip_inspire_check"] = True
-
 
 class RasterCheckTestCase(TestCase):
     def setUp(self):

src/qc_tool/test/test_raster_check.py

@@ -298,44 +298,6 @@ def test_fail(self):
         self.assertIn("s01_mmu_raster_incorrect_lessmmu_error.gpkg", status.attachment_filenames)
 
 
-@skipIf(CONFIG["skip_inspire_check"], "INSPIRE check has been disabled.")
-class Test_inspire(RasterCheckTestCase):
-    def setUp(self):
-        super().setUp()
-        xml_dir = TEST_DATA_DIR.joinpath("metadata")
-        filepath_good = xml_dir.joinpath("inspire-good.tif")
-        filepath_missing = xml_dir.joinpath("inspire-missing-metadata.tif")
-        filepath_bad = xml_dir.joinpath("inspire-bad.tif")
-        layer_defs = {"layer_good": {"src_filepath": filepath_good, "src_layer_name": filepath_good.name},
-                      "layer_missing": {"src_filepath": filepath_missing, "src_layer_name": filepath_missing.name},
-                      "layer_bad": {"src_filepath": filepath_bad, "src_layer_name": filepath_bad.name}}
-        self.params.update({"raster_layer_defs": layer_defs,
-                            "step_nr": 1,
-                            "output_dir": self.jobdir_manager.output_dir})
-
-    def test(self):
-        from qc_tool.raster.inspire import run_check
-        self.params.update({"layers":["layer_good"]})
-        status = self.status_class()
-        run_check(self.params, status)
-        self.assertEqual("ok", status.status, "INSPIRE raster check should pass for raster with valid metadata file.")
-
-    def test_missing_xml_fail(self):
-        from qc_tool.raster.inspire import run_check
-        self.params.update({"layers": ["layer_missing"]})
-        status = self.status_class()
-        run_check(self.params, status)
-        self.assertEqual("failed", status.status, "INSPIRE raster check should fail for raster with missing xml file.")
-
-    def test_fail(self):
-        from qc_tool.raster.inspire import run_check
-        self.params.update({"layers": ["layer_bad"]})
-        self.params["skip_inspire_check"] = False
-        status = self.status_class()
-        run_check(self.params, status)
-        self.assertEqual("failed", status.status, "INSPIRE raster check should fail for raster with non-compliant xml file.")
-
-
 class Test_color(RasterCheckTestCase):
     def setUp(self):
         super().setUp()

src/qc_tool/test/test_raster_product.py

@@ -21,138 +21,173 @@ def show_messages(self, job_result):
 
     def setUp(self):
         super().setUp()
-
         self.maxDiff = None
         self.raster_data_dir = TEST_DATA_DIR.joinpath("raster")
         self.username = "test_username"
 
-        # All steps are expected to finish with ok status by default.
-        self.expected_step_statuses = ["ok"] * 13
-
-    # High resolution forest type (FTY) - 10m
     def test_fty_2018_010m(self):
+        """High resolution forest type (FTY) - 10m"""
         product_ident = "fty_2018_010m"
         filepath = self.raster_data_dir.joinpath("fty_010m", "FTY_2018_010m_eu_03035_clip.zip")
+
+        expected_step_results = ["ok"] * 15
         # fty_010m has extra checks raster.tile and raster.mmu
-        self.expected_step_statuses = ["ok"] * 15
-        self.expected_step_statuses[12] = "failed"
+        expected_step_results[12] = "failed"
+        expected_step_results[14] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(self.expected_step_statuses, step_statuses, self.show_messages(job_result))
+        self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
 
-    # High resolution forest type (FTY) - 100m
     def test_fty_2018_100m(self):
+        """High resolution forest type (FTY) - 100m"""
         product_ident = "fty_2018_100m"
         filepath = self.raster_data_dir.joinpath("fty_100m", "fty_2018_100m_eu_03035_d02_clip.zip")
+
+        expected_step_results = ["ok"] * 13
+        expected_step_results[12] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(self.expected_step_statuses, step_statuses, self.show_messages(job_result))
+        self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
 
-    # High resolution grassland (GRA) - 10m
     def test_gra_2018_010m(self):
+        """High resolution grassland (GRA) - 10m"""
         product_ident = "gra_2018_010m"
         filepath = self.raster_data_dir.joinpath("gra_010m", "GRA_2018_010m_eu_03035_V1_clip.zip")
 
+        expected_step_results = ["ok"] * 13
         # gra_010m has mismatching attributes and color table
-        self.expected_step_statuses[3] = "failed"
-        self.expected_step_statuses[10] = "failed"
+        expected_step_results[3] = "failed"
+        expected_step_results[10] = "failed"
+        expected_step_results[12] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(self.expected_step_statuses, step_statuses, self.show_messages(job_result))
+        self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
 
-    # High resolution grassland (GRA) - 100m
     def test_gra_2018_100m(self):
+        """High resolution grassland (GRA) - 100m"""
         product_ident = "gra_2018_100m"
         filepath = self.raster_data_dir.joinpath("gra_100m", "GRA_2018_100m_eu_03035_V1_clip.zip")
+
+        expected_step_results = ["ok"] * 13
         # gra_2018_100m has mismatching attributes and color table
-        self.expected_step_statuses[3] = "failed"
-        self.expected_step_statuses[10] = "failed"
+        expected_step_results[3] = "failed"
+        expected_step_results[10] = "failed"
+        expected_step_results[12] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(self.expected_step_statuses, step_statuses, self.show_messages(job_result))
+        self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
 
-    # High resolution imperviousness change (IMC) - 20m
     def test_imc_1518_020m(self):
+        """High resolution imperviousness change (IMC) - 20m"""
         product_ident = "imc_1518_020m"
         filepath = self.raster_data_dir.joinpath("imc_020m", "IMC_1518_020m_eu_03035_d02_clip.zip")
 
+        expected_step_results = ["ok"] * 13
         # imc_1518_020m has mismatching attributes and color table
-        self.expected_step_statuses[3] = "failed"
-        self.expected_step_statuses[10] = "failed"
+        expected_step_results[3] = "failed"
+        expected_step_results[10] = "failed"
+        expected_step_results[12] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(self.expected_step_statuses, step_statuses, self.show_messages(job_result))
+        self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
 
-    # High resolution imperviousness change (IMC) - 100m
     def test_imc_1518_100m(self):
+        """High resolution imperviousness change (IMC) - 100m"""
         product_ident = "imc_1518_100m"
         filepath = self.raster_data_dir.joinpath("imc_100m", "IMC_1518_100m_eu_03035_d02_clip.zip")
 
+        expected_step_results = ["ok"] * 13
         # imc_1518_020m has mismatching attributes, values and color table
-        self.expected_step_statuses[3] = "failed"
-        self.expected_step_statuses[9] = "failed"
-        self.expected_step_statuses[10] = "failed"
+        expected_step_results[3] = "failed"
+        expected_step_results[9] = "failed"
+        expected_step_results[10] = "failed"
+        expected_step_results[12] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(self.expected_step_statuses, step_statuses, self.show_messages(job_result))
+        self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
 
-    # High resolution imperviousness density (IMD) - 10m
     def test_imd_2018_010m(self):
+        """High resolution imperviousness density (IMD) - 10m"""
         product_ident = "imd_2018_010m"
         filepath = self.raster_data_dir.joinpath("imd_010m", "IMD_2018_010m_eu_03035_d04_clip.zip")
 
+        expected_step_results = ["ok"] * 13
         # imd_2018_010m has mismatching attributes
-        self.expected_step_statuses[3] = "failed"
+        expected_step_results[3] = "failed"
+        expected_step_results[12] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(self.expected_step_statuses, step_statuses, self.show_messages(job_result))
+        self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
 
-    # High resolution imperviousness density (IMD) - 100m
     def test_imd_2018_100m(self):
+        """High resolution imperviousness density (IMD) - 100m"""
         product_ident = "imd_2018_100m"
         filepath = self.raster_data_dir.joinpath("imd_100m", "IMD_2018_100m_eu_03035_d02_clip.zip")
 
+        expected_step_results = ["ok"] * 13
         # imd_2018_100m has mismatching attributes
-        self.expected_step_statuses[3] = "failed"
+        expected_step_results[3] = "failed"
+        expected_step_results[12] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(self.expected_step_statuses, step_statuses, self.show_messages(job_result))
+        self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
 
-    # High resolution tree cover density (TCD) - 10m
     def test_tcd_2018_010m(self):
+        """High resolution tree cover density (TCD) - 10m"""
         product_ident = "tcd_2018_010m"
+        filepath = self.raster_data_dir.joinpath("tcd_010m", "TCD_2018_010m_eu_03035_d04_clip.zip")
 
+        expected_step_results = ["ok"] * 13
         # tcd_2018_010m has mismatching colors
-        self.expected_step_statuses[10] = "failed"
-        filepath = self.raster_data_dir.joinpath("tcd_010m", "TCD_2018_010m_eu_03035_d04_clip.zip")
+        expected_step_results[10] = "failed"
+        expected_step_results[12] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(self.expected_step_statuses, step_statuses, self.show_messages(job_result))
+        self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
 
-    # High resolution tree cover density (TCD) - 100m
     def test_tcd_2018_100m(self):
+        """High resolution tree cover density (TCD) - 100m"""
         product_ident = "tcd_2018_100m"
         filepath = self.raster_data_dir.joinpath("tcd_100m", "TCD_2018_100m_eu_03035_d03_clip.zip")
 
+        expected_step_results = ["ok"] * 13
         # tcd_2018_010m has mismatching colors
-        self.expected_step_statuses[10] = "failed"
+        expected_step_results[10] = "failed"
+        expected_step_results[12] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(self.expected_step_statuses, step_statuses, self.show_messages(job_result))
+        self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
 
-    # High resolution small woody features - 5m raster + vector
     def test_swf_2015_100m(self):
+        """High resolution small woody features - 5m raster + vector"""
+        product_ident = "swf_2015_100m"
         filepath = TEST_DATA_DIR.joinpath("raster", "swf_100m", "swf_2015_100m_eu_03035_v1_1.zip")
+
         expected_step_results = ["ok"] * 11
-        job_result = dispatch(self.job_uuid, "user_name", filepath, "swf_2015_100m")
+        expected_step_results[10] = "cancelled"
+
+        job_result = dispatch(self.job_uuid, "user_name", filepath, product_ident)
         step_results = [step_result["status"] for step_result in job_result["steps"]]
         self.assertListEqual(expected_step_results, step_results)
 
-    # General raster product
     def test_general_raster(self):
+        """General raster product"""
         product_ident = "general_raster"
         filepath = self.raster_data_dir.joinpath("general_raster", "general_raster.zip")
-        expected_step_statuses = ["ok"] * 9
+
+        expected_step_results = ["ok"] * 9
+        expected_step_results[8] = "cancelled"
+
         job_result = dispatch(self.job_uuid, self.username, filepath, product_ident)
         step_statuses = [step_result["status"] for step_result in job_result["steps"]]
-        self.assertListEqual(expected_step_statuses, step_statuses)
+        self.assertListEqual(expected_step_results, step_statuses)

src/qc_tool/test/test_vector_check.py

@@ -1711,36 +1711,3 @@ def test_warning(self):
         # We expect ok status and one warning message.
         self.assertEqual("ok", status.status)
         self.assertEqual(1, len(status.messages))
-
-
-@skipIf(CONFIG["skip_inspire_check"], "INSPIRE check has been disabled.")
-class Test_inspire(VectorCheckTestCase):
-    def setUp(self):
-        super().setUp()
-        self.xml_dir = TEST_DATA_DIR.joinpath("metadata")
-        self.params["tmp_dir"] = self.params["jobdir_manager"].tmp_dir
-        self.params["output_dir"] = self.params["jobdir_manager"].output_dir
-        self.params["layers"] = ["layer0"]
-
-
-    def test(self):
-        from qc_tool.vector.inspire import run_check
-        self.params["layer_defs"] = {"layer0": {"src_filepath": self.xml_dir.joinpath("inspire-good.shp")}}
-        status = self.status_class()
-        run_check(self.params, status)
-        self.assertEqual("ok", status.status)
-
-    def test_missing_xml_fail(self):
-        from qc_tool.vector.inspire import run_check
-        self.params["layer_defs"] = {"layer0": {"src_filepath": self.xml_dir.joinpath("inspire-missing-metadata.gdb")}}
-        status = self.status_class()
-        run_check(self.params, status)
-        self.assertEqual("failed", status.status)
-
-    def test_fail(self):
-        from qc_tool.vector.inspire import run_check
-        self.params["layer_defs"] = {"layer0": {"src_filepath": self.xml_dir.joinpath("inspire-bad.shp")}}
-        status = self.status_class()
-        run_check(self.params, status)
-        self.assertEqual("failed", status.status)
-        self.assertIn("inspire-bad_metadata_error.json", status.attachment_filenames)