Skip to content

Commit 6744509

Browse files
jirikadlec2jirikadlec2
committed
Version 1.5.8 (Merge branch 'rc').
2 parents 3bffbe7 + 6ade80c commit 6744509

10 files changed

Lines changed: 188 additions & 63 deletions

File tree

‎docker/docker-compose.eea.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: '3'
22

33
services:
44
frontend:
5-
image: eeacms/copernicus-qctool-frontend:1.5.7
5+
image: eeacms/copernicus-qctool-frontend:1.5.8
66
ports:
77
- 8000:8000
88
environment:
@@ -22,7 +22,7 @@ services:
2222
- qc_tool_frontend:/mnt/qc_tool_frontend
2323

2424
worker:
25-
image: eeacms/copernicus-qctool-worker:1.5.7
25+
image: eeacms/copernicus-qctool-worker:1.5.8
2626
environment:
2727
- PRODUCT_DIRS
2828
- BOUNDARY_DIR=/mnt/qc_tool_boundary/boundaries

‎docker/docker-compose.service_provider.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: '3'
22

33
services:
44
frontend:
5-
image: eeacms/copernicus-qctool-frontend:1.5.7
5+
image: eeacms/copernicus-qctool-frontend:1.5.8
66
ports:
77
- 8000:8000
88
environment:
@@ -18,7 +18,7 @@ services:
1818
- qc_tool_volume:/mnt/qc_tool_volume
1919

2020
worker:
21-
image: eeacms/copernicus-qctool-worker:1.5.7
21+
image: eeacms/copernicus-qctool-worker:1.5.8
2222
environment:
2323
- PRODUCT_DIRS
2424
- BOUNDARY_DIR=/mnt/qc_tool_volume/boundaries

‎product_definitions/n2k.json‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
"nodata_12": "integer",
4141
"changecode": "string",
4242
"area_ha": "real"},
43-
"ignored": []
43+
"ignored": ["shape_area", "shape_length"]
4444
}
4545
},
4646
{
@@ -256,10 +256,11 @@
256256
"final_code_column_name": "maes_4_12",
257257
"comment_column_names": ["comment_06", "comment_12"],
258258
"exception_comments": ["Area size exception (at Natura2000 AoI boundary)",
259-
"Area size exception (inside Natura2000 AoI boundary); Areas related to change",
260-
"Area size exception (inside Natura2000 AoI boundary); Splitted change",
261-
"Area size exception (inside Natura2000 AoI boundary); Braided River System",
262-
"Area size exception (inside Natura2000 AoI boundary); Temporal fluctuation of water level"]
259+
"Area size exception (inside Natura2000 AoI boundary)",
260+
"Areas related to change",
261+
"Splitted change",
262+
"Braided River System",
263+
"Temporal fluctuation of water level"]
263264
}
264265
},
265266
{

‎product_definitions/rpz_2012.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
"area_ha": "real",
3838
"nodata": "integer",
3939
"comment": "string"},
40-
"ignored": []
40+
"ignored": ["shape_area", "shape_length"]
4141
}
4242
},
4343
{

‎product_definitions/rpz_2018.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
"nodata_12": "integer",
4444
"nodata_18": "integer",
4545
"comment": "string"},
46-
"ignored": []
46+
"ignored": ["shape_area", "shape_length"]
4747
}
4848
},
4949
{

‎src/qc_tool/test/test_vector_check.py‎

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1266,8 +1266,8 @@ def test(self):
12661266
cursor.execute("INSERT INTO n2k VALUES (50, 0, 8, 'comment1', NULL, ST_MakeEnvelope(60, 0, 61, 1, 4326));")
12671267
cursor.execute("INSERT INTO n2k VALUES (51, 0, 8, NULL, 'comment2', ST_MakeEnvelope(60, 0, 61, 1, 4326));")
12681268

1269-
cursor.execute("INSERT INTO n2k VALUES (52, 0, 8, 'comment1 nok', NULL, ST_MakeEnvelope(60, 0, 61, 1, 4326));")
1270-
cursor.execute("INSERT INTO n2k VALUES (53, 0, 8, NULL, 'comment2 nok', ST_MakeEnvelope(60, 0, 61, 1, 4326));")
1269+
cursor.execute("INSERT INTO n2k VALUES (52, 0, 8, 'comment_nok1', NULL, ST_MakeEnvelope(60, 0, 61, 1, 4326));")
1270+
cursor.execute("INSERT INTO n2k VALUES (53, 0, 8, NULL, 'comment_nok2', ST_MakeEnvelope(60, 0, 61, 1, 4326));")
12711271
cursor.execute("INSERT INTO n2k VALUES (54, 0, 8, NULL, NULL, ST_MakeEnvelope(60, 0, 61, 1, 4326));")
12721272

12731273
self.params.update({"layer_defs": {"n2k": {"pg_layer_name": "n2k",
@@ -1457,8 +1457,8 @@ def test(self):
14571457

14581458

14591459
class Test_mml(VectorCheckTestCase):
1460-
def test(self):
1461-
from qc_tool.vector.mml import run_check
1460+
def setUp(self):
1461+
super().setUp()
14621462
self.params.update({"layer_defs": {"mml": {"pg_layer_name": "mml",
14631463
"pg_fid_name": "fid",
14641464
"fid_display_name": "row number"}},
@@ -1467,19 +1467,49 @@ def test(self):
14671467
"filter_code": "1",
14681468
"mml": 10.,
14691469
"step_nr": 1})
1470-
cursor = self.params["connection_manager"].get_connection().cursor()
1471-
cursor.execute("CREATE TABLE mml (fid integer, code char(1), geom geometry(Polygon, 4326));")
1472-
cursor.execute("INSERT INTO mml VALUES (1, NULL, ST_MakeEnvelope(0, 0, 5, 1, 4326));")
1473-
cursor.execute("INSERT INTO mml VALUES (2, '1', ST_MakeEnvelope(0, 0, 5, 1, 4326));")
1474-
cursor.execute("INSERT INTO mml VALUES (3, '1', ST_MakeEnvelope(0, 0, 9.999, 1, 4326));")
1475-
cursor.execute("INSERT INTO mml VALUES (4, '1', ST_MakeEnvelope(0, 0, 10, 1, 4326));")
1476-
cursor.execute("INSERT INTO mml VALUES (5, '1', ST_MakeEnvelope(0, 0, 11, 1, 4326));")
1477-
cursor.execute("INSERT INTO mml VALUES (6, '2', ST_MakeEnvelope(0, 0, 5, 1, 4326));")
1470+
self.cursor = self.params["connection_manager"].get_connection().cursor()
1471+
self.cursor.execute("CREATE TABLE mml (fid integer, code char(1), geom geometry(Polygon, 4326));")
1472+
1473+
def test(self):
1474+
from qc_tool.vector.mml import run_check
1475+
self.cursor.execute("INSERT INTO mml VALUES (1, NULL, ST_MakeEnvelope(0, 0, 5, 1, 4326));")
1476+
self.cursor.execute("INSERT INTO mml VALUES (2, '1', ST_MakeEnvelope(0, 0, 5, 1, 4326));")
1477+
self.cursor.execute("INSERT INTO mml VALUES (3, '1', ST_MakeEnvelope(0, 0, 9.999, 1, 4326));")
1478+
self.cursor.execute("INSERT INTO mml VALUES (4, '1', ST_MakeEnvelope(0, 0, 10, 1, 4326));")
1479+
self.cursor.execute("INSERT INTO mml VALUES (5, '1', ST_MakeEnvelope(0, 0, 11, 1, 4326));")
1480+
self.cursor.execute("INSERT INTO mml VALUES (6, '2', ST_MakeEnvelope(0, 0, 5, 1, 4326));")
1481+
status = self.status_class()
1482+
run_check(self.params, status)
1483+
self.assertEqual("ok", status.status)
1484+
self.cursor.execute("SELECT fid FROM s01_mml_warning ORDER BY fid;")
1485+
self.assertListEqual([(2,), (3,)], self.cursor.fetchall())
1486+
1487+
def test_arch(self):
1488+
"""The plain box does not pass the check, however, if we make an arch from it, the check passes."""
1489+
from qc_tool.vector.mml import run_check
1490+
self.cursor.execute("INSERT INTO mml VALUES (1, '1', ST_Difference(ST_MakeEnvelope(0, 0, 9.999, 1, 4326),"
1491+
" ST_MakeEnvelope(0.5, 0, 9.5, 0.5, 4326)));")
1492+
status = self.status_class()
1493+
run_check(self.params, status)
1494+
self.assertEqual("ok", status.status)
1495+
self.cursor.execute("SELECT fid FROM s01_mml_warning ORDER BY fid;")
1496+
self.assertListEqual([], self.cursor.fetchall())
1497+
1498+
def test_inner_ring(self):
1499+
"""If there are two inner rings intersecting in a point, then ST_ApproximateMedialAxis() raises error:
1500+
1501+
psycopg2.InternalError: straight skeleton of Polygon with touching interior rings is not implemented
1502+
1503+
Adapted implementation ignores inner rings, so this test should pass with ok status."""
1504+
from qc_tool.vector.mml import run_check
1505+
self.cursor.execute("INSERT INTO mml VALUES (1, '1', ST_Difference(ST_Difference(ST_MakeEnvelope(0, 0, 4, 4, 4326),"
1506+
" ST_MakeEnvelope(1, 0, 2, 2, 4326)),"
1507+
" ST_MakeEnvelope(2, 2, 3, 3, 4326)));")
14781508
status = self.status_class()
14791509
run_check(self.params, status)
14801510
self.assertEqual("ok", status.status)
1481-
cursor.execute("SELECT fid FROM s01_mml_warning ORDER BY fid;")
1482-
self.assertListEqual([(2,), (3,)], cursor.fetchall())
1511+
self.cursor.execute("SELECT fid FROM s01_mml_warning ORDER BY fid;")
1512+
self.assertListEqual([(1,)], self.cursor.fetchall())
14831513

14841514

14851515
class Test_overlap(VectorCheckTestCase):

‎src/qc_tool/vector/gap_unit.py‎

Lines changed: 99 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,12 @@
66
IS_SYSTEM = False
77

88

9+
POLYGON_MAX_POINTS = 500000
10+
BIG_POLYGON_AREA_TOLERANCE = 0.1
11+
12+
913
def run_check(params, status):
1014
from qc_tool.vector.helper import do_layers
11-
from qc_tool.vector.helper import get_failed_items_message
1215

1316
cursor = params["connection_manager"].get_connection().cursor()
1417

@@ -21,37 +24,111 @@ def run_check(params, status):
2124
sql_params = {"layer_name": layer_def["pg_layer_name"],
2225
"boundary_name": params["layer_defs"]["boundary"]["pg_layer_name"],
2326
"boundary_unit_column_name": params["boundary_unit_column_name"],
27+
"polygon_max_points": POLYGON_MAX_POINTS,
28+
"big_polygon_area_tolerance": BIG_POLYGON_AREA_TOLERANCE,
29+
"boundary_union_table": "s{:02d}_{:s}_boundary_union_table".format(params["step_nr"], layer_def["pg_layer_name"]),
30+
"layer_union_table": "s{:02d}_{:s}_layer_union_table".format(params["step_nr"], layer_def["pg_layer_name"]),
31+
"big_polygon_table": "s{:02d}_{:s}_big_polygon_table".format(params["step_nr"], layer_def["pg_layer_name"]),
32+
"gap_suspect_table": "s{:02d}_{:s}_gap_suspect".format(params["step_nr"], layer_def["pg_layer_name"]),
2433
"gap_warning_table": "s{:02d}_{:s}_gap_warning".format(params["step_nr"], layer_def["pg_layer_name"]),
2534
"unit_warning_table": "s{:02d}_{:s}_unit_warning".format(params["step_nr"], layer_def["pg_layer_name"])}
2635

27-
# Create table of gaps.
28-
sql = ("CREATE TABLE {gap_warning_table} AS"
29-
" SELECT"
30-
" layer_union.{boundary_unit_column_name} AS {boundary_unit_column_name},"
31-
" (ST_Dump(ST_Difference(boundary_union.geom, layer_union.geom))).geom AS geom"
32-
" FROM"
36+
# Create boundary union table.
37+
sql = ("CREATE TABLE {boundary_union_table} AS"
38+
" (SELECT"
39+
" {boundary_unit_column_name},"
40+
" ST_Union(geom) AS geom"
41+
" FROM {boundary_name}"
42+
" WHERE {boundary_unit_column_name} IN (SELECT DISTINCT {boundary_unit_column_name} FROM {layer_name})"
43+
" GROUP BY {boundary_unit_column_name})")
44+
sql = sql.format(**sql_params)
45+
cursor.execute(sql)
46+
47+
48+
# Create layer union table of normally sized polygons.
49+
sql = ("CREATE TABLE {layer_union_table} AS"
3350
" (SELECT"
3451
" {boundary_unit_column_name},"
35-
" ST_Union(geom) AS geom"
52+
" ST_Buffer(ST_Collect(geom), 0) AS geom"
3653
" FROM {layer_name}"
54+
" WHERE ST_NPoints(geom) <= {polygon_max_points}"
3755
" GROUP BY {boundary_unit_column_name}"
38-
" ) AS layer_union"
39-
" INNER JOIN"
40-
" (SELECT"
41-
" {boundary_unit_column_name},"
42-
" ST_Union(geom) AS geom"
43-
" FROM {boundary_name}"
44-
" WHERE {boundary_unit_column_name} IN (SELECT {boundary_unit_column_name} FROM {layer_name})"
45-
" GROUP BY {boundary_unit_column_name}"
46-
" ) AS boundary_union"
47-
" USING ({boundary_unit_column_name});")
56+
" )"
57+
)
4858
sql = sql.format(**sql_params)
4959
cursor.execute(sql)
5060

51-
# Report gaps.
52-
if cursor.rowcount > 0:
53-
status.info("Layer {:s} has {:d} gap(s).".format(layer_def["pg_layer_name"], cursor.rowcount))
54-
status.add_full_table(sql_params["gap_warning_table"])
61+
# Are there any large complex polygons with number of vertices > POLYGON_MAX_POINTS?
62+
sql = "SELECT MAX(ST_NPoints(geom)) FROM {layer_name}"
63+
sql = sql.format(**sql_params)
64+
cursor.execute(sql)
65+
layer_npoints_max = float(cursor.fetchone()[0])
66+
67+
if layer_npoints_max <= POLYGON_MAX_POINTS:
68+
# Normal case, layer does not have any polygon with extremely large number of vertices.
69+
# Create table of all gaps.
70+
sql = ("CREATE TABLE {gap_warning_table} AS"
71+
" SELECT"
72+
" layer_union.{boundary_unit_column_name} AS {boundary_unit_column_name},"
73+
" (ST_Dump(ST_Difference(boundary_union.geom, layer_union.geom))).geom AS geom"
74+
" FROM {layer_union_table} AS layer_union"
75+
" INNER JOIN {boundary_union_table} AS boundary_union"
76+
" USING ({boundary_unit_column_name});")
77+
sql = sql.format(**sql_params)
78+
cursor.execute(sql)
79+
80+
# Report gaps.
81+
if cursor.rowcount > 0:
82+
status.info("Layer {:s} has {:d} gap(s).".format(layer_def["pg_layer_name"], cursor.rowcount))
83+
status.add_full_table(sql_params["gap_warning_table"])
84+
85+
else:
86+
# Special case, layer has polygons with large number of points.
87+
88+
# Create union table of polygons with extremely large numbers of vertices.
89+
sql = ("CREATE TABLE {big_polygon_table} AS"
90+
" (SELECT"
91+
" {boundary_unit_column_name},"
92+
" ST_MemUnion(geom) AS geom"
93+
" FROM {layer_name}"
94+
" WHERE ST_NPoints(geom) > {polygon_max_points}"
95+
" GROUP BY {boundary_unit_column_name}"
96+
" )"
97+
)
98+
sql = sql.format(**sql_params)
99+
cursor.execute(sql)
100+
101+
# Create table of suspect gaps.
102+
sql = ("CREATE TABLE {gap_suspect_table} AS"
103+
" SELECT"
104+
" layer_union.{boundary_unit_column_name} AS {boundary_unit_column_name},"
105+
" (ST_Difference(boundary_union.geom, layer_union.geom)) AS geom"
106+
" FROM {layer_union_table} AS layer_union"
107+
" INNER JOIN {boundary_union_table} AS boundary_union"
108+
" USING ({boundary_unit_column_name});")
109+
sql = sql.format(**sql_params)
110+
cursor.execute(sql)
111+
112+
# Verify if any of the suspect gaps is co-incident with any of the extremely complex polygons.
113+
# The non-coincident suspect gaps are reported a real gaps.
114+
sql = ("CREATE TABLE {gap_warning_table} AS"
115+
" SELECT"
116+
" suspect_gaps.{boundary_unit_column_name} AS {boundary_unit_column_name},"
117+
" suspect_gaps.geom as geom,"
118+
" big_polygons.{boundary_unit_column_name} AS found_polygon"
119+
" FROM {gap_suspect_table} AS suspect_gaps"
120+
" LEFT JOIN {big_polygon_table} AS big_polygons"
121+
" ON ABS(ST_Area(suspect_gaps.geom) - ST_AREA(big_polygons.geom)) < {big_polygon_area_tolerance}"
122+
" WHERE big_polygons.{boundary_unit_column_name} IS NULL;")
123+
124+
sql = sql.format(**sql_params)
125+
cursor.execute(sql)
126+
127+
# Report gaps.
128+
if cursor.rowcount > 0:
129+
status.info("Layer {:s} has {:d} gap(s).".format(layer_def["pg_layer_name"], cursor.rowcount))
130+
status.add_full_table(sql_params["gap_warning_table"])
131+
return
55132

56133
# Create table of excessive items.
57134
sql = ("CREATE TABLE {unit_warning_table} AS"

‎src/qc_tool/vector/mml.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def run_check(params, status):
3737
" ST_Distance(ST_PointN(ST_Boundary(ST_OrientedEnvelope(geom)), 2),"
3838
" ST_PointN(ST_Boundary(ST_OrientedEnvelope(geom)), 3))) < %(mml)s"
3939
" ) AS filtered_table"
40-
" WHERE ST_Length(ST_ApproximateMedialAxis(geom)) <= %(mml)s;")
40+
" WHERE ST_Length(ST_ApproximateMedialAxis(ST_MakePolygon(ST_ExteriorRing(geom)))) <= %(mml)s;")
4141
sql = sql.format(**sql_params)
4242
cursor.execute(sql, sql_execute_params)
4343

‎src/qc_tool/vector/mmu_n2k.py‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -80,18 +80,19 @@ def run_check(params, status):
8080

8181
# Features with specific comments.
8282
if len(params["exception_comments"]) > 0:
83-
sql_execute_params = {"exception_comments": tuple(params["exception_comments"])}
84-
for comment_column_name in params["comment_column_names"]:
85-
sql = ("INSERT INTO {exception_table}"
86-
" SELECT {fid_name}"
87-
" FROM {layer_name}"
88-
" WHERE"
89-
" {comment_column_name} IN %(exception_comments)s"
90-
" AND {fid_name} NOT IN (SELECT {fid_name} FROM {general_table})"
91-
" AND {fid_name} NOT IN (SELECT {fid_name} FROM {exception_table});")
92-
sql_params["comment_column_name"] = comment_column_name
93-
sql = sql.format(**sql_params)
94-
cursor.execute(sql, sql_execute_params)
83+
for exception_comment in params["exception_comments"]:
84+
for comment_column_name in params["comment_column_names"]:
85+
sql = ("INSERT INTO {exception_table}"
86+
" SELECT {fid_name}"
87+
" FROM {layer_name}"
88+
" WHERE"
89+
" {comment_column_name} LIKE '%{exception_comment}%'"
90+
" AND {fid_name} NOT IN (SELECT {fid_name} FROM {general_table})"
91+
" AND {fid_name} NOT IN (SELECT {fid_name} FROM {exception_table});")
92+
sql_params["comment_column_name"] = comment_column_name
93+
sql_params["exception_comment"] = exception_comment
94+
sql = sql.format(**sql_params)
95+
cursor.execute(sql)
9596

9697
# Add exceptions comming from complex change.
9798
# Do that once for initial code and once again for final code.

‎src/qc_tool/worker/report.py‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1+
#!/usr/bin/env python3
12

23

3-
import json
44
from datetime import datetime
55
from os.path import normpath
66
from pathlib import Path
@@ -27,6 +27,19 @@
2727
from qc_tool.common import CONFIG
2828

2929

30+
MAX_WORD_LENGTH = 40
31+
32+
33+
def truncate_long_words(sentence, max_length):
34+
words = sentence.split()
35+
for word in words:
36+
if len(word) > max_length:
37+
for i in range(0, len(word), max_length):
38+
yield word[i:i+max_length]
39+
else:
40+
yield word
41+
42+
3043
def generate_pdf_report(job_report_filepath, job_uuid):
3144
job_report = compile_job_report_data(job_uuid)
3245

@@ -86,7 +99,8 @@ def footer(canvas, doc):
8699
# Add summary table
87100
text.append(Paragraph("", styles["Heading1"]))
88101
text.append(Paragraph("Report summary", styles["Heading2"]))
89-
status_file = ["File name", job_report["filename"]]
102+
wrapped_filename = "\n".join(truncate_long_words(job_report["filename"], MAX_WORD_LENGTH))
103+
status_file = ["File name", wrapped_filename]
90104
status_product = ["Product", job_report["description"]]
91105
display_date = datetime.strptime(job_report["job_finish_date"], TIME_FORMAT).strftime("%Y-%m-%d %H:%M:%S")
92106
status_date = ["Checked on", display_date]
@@ -162,7 +176,9 @@ def footer(canvas, doc):
162176
messages = []
163177
display_messages = []
164178
for message in messages:
165-
display_messages.append(Paragraph(message, style_body))
179+
# Split long words in a message
180+
wrapped_message = " ".join(truncate_long_words(message, MAX_WORD_LENGTH))
181+
display_messages.append(Paragraph(wrapped_message, style_body))
166182

167183
check_info = [display_ident,
168184
display_layers,

0 commit comments

Comments
 (0)