Update hexbin and dsra source (#129)

* Updates to new DSRA source repo and hex processing * changs to DSRA repo references * Update to same ES/Kibana version as FGP env * remove hexbin centroid aggregations * modularize data loading from postgres to ES * PSRA results for national model & add bld agg * load national psra instead of p/t subset * switch from P/T to national layer * update Kibana space and hexgrid * add index prefix * update social_fab loading and add hex * add global fabric * Kibana: update index patterns and logs * Kibana logs & patterns * add retry on ES timeout * ES/Kibana updates * lint style fixes * lint style fixes 2 * lint style fixes 3 * lint style fixes 4 * lint style fixes 5 * Update lintly-flake8.yml ignore E501, line too long. Adds undue burden trying to shorten long SQL queries * lint style fixes 6 Co-authored-by: William Chow <[email protected]>
OpenDRR · Oct 4, 2021 · 8687e64 · 8687e64
1 parent ab45369
commit 8687e64
Show file tree

Hide file tree

Showing 27 changed files with 1,185 additions and 982 deletions.
diff --git a/.github/workflows/lintly-flake8.yml b/.github/workflows/lintly-flake8.yml
@@ -17,4 +17,4 @@ jobs:
           # Fail if "new" violations detected or "any", default "new"
           failIf: new
           # Additional arguments to pass to flake8, default "." (current directory)
-          args: "--ignore=E121,E123 --per-file-ignores=python/gen_pygeoapi_config.py:E501 ."
+          args: "--ignore=E121,E123,E501 --per-file-ignores=python/gen_pygeoapi_config.py:E501 ."
diff --git a/docker-compose-run.yml b/docker-compose-run.yml
@@ -6,7 +6,7 @@ volumes:
 services:
 
     kibana-opendrr:
-        image: kibana:7.7.1
+        image: kibana:7.12.0
 
         environment:
             ELASTICSEARCH_HOSTS: http://elasticsearch-opendrr:9200
@@ -18,7 +18,7 @@ services:
             - elasticsearch-opendrr
 
     elasticsearch-opendrr: 
-        image: elasticsearch:7.7.1
+        image: elasticsearch:7.12.0
 
         environment:
             - discovery.type=single-node

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -22,7 +22,7 @@ services:
 
 
     kibana-opendrr:
-        image: kibana:7.7.1
+        image: kibana:7.12.0
 
         environment:
             ELASTICSEARCH_HOSTS: http://elasticsearch-opendrr:9200
@@ -34,7 +34,7 @@ services:
             - elasticsearch-opendrr
 
     elasticsearch-opendrr: 
-        image: elasticsearch:7.7.1
+        image: elasticsearch:7.12.0
 
         environment:
             - discovery.type=single-node

diff --git a/python/add_data.sh b/python/add_data.sh
diff --git a/python/dsraAllScenariosCduid_postgres2es.py b/python/dsraAllScenariosCduid_postgres2es.py
@@ -0,0 +1,45 @@
+# =================================================================
+# !/bin/bash
+# SPDX-License-Identifier: MIT
+#
+# Copyright (C) 2020-2021 Government of Canada
+#
+# Main Authors: Drew Rotheram <[email protected]>
+#               Joost van Ulden <[email protected]>
+# =================================================================
+
+
+import utils
+
+
+def main():
+    table = utils.PostGISdataset(
+        utils.PostGISConnection(),
+        utils.ESConnection(settings={
+            'settings': {
+                'number_of_shards': 1,
+                'number_of_replicas': 0
+            },
+            'mappings': {
+                'properties': {
+                    'geometry': {
+                        'type': 'geo_shape'
+                    }
+                }
+            }
+        }),
+        view="opendrr_dsra_all_scenarios_cduid",
+        sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
+                    FROM dsra.dsra_all_scenarios_cduid \
+                    ORDER BY dsra_all_scenarios_cduid."cduid" \
+                    LIMIT {limit} \
+                    OFFSET {offset}'
+    )
+
+    table.postgis2es()
+
+    return
+
+
+if __name__ == '__main__':
+    main()
diff --git a/python/dsraAllScenariosCsduid_postgres2es.py b/python/dsraAllScenariosCsduid_postgres2es.py
@@ -0,0 +1,45 @@
+# =================================================================
+# !/bin/bash
+# SPDX-License-Identifier: MIT
+#
+# Copyright (C) 2020-2021 Government of Canada
+#
+# Main Authors: Drew Rotheram <[email protected]>
+#               Joost van Ulden <[email protected]>
+# =================================================================
+
+
+import utils
+
+
+def main():
+    table = utils.PostGISdataset(
+        utils.PostGISConnection(),
+        utils.ESConnection(settings={
+            'settings': {
+                'number_of_shards': 1,
+                'number_of_replicas': 0
+            },
+            'mappings': {
+                'properties': {
+                    'geometry': {
+                        'type': 'geo_shape'
+                    }
+                }
+            }
+        }),
+        view="opendrr_dsra_all_scenarios_csduid",
+        sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
+                    FROM dsra.dsra_all_scenarios_csduid \
+                    ORDER BY dsra_all_scenarios_csduid."csduid" \
+                    LIMIT {limit} \
+                    OFFSET {offset}'
+    )
+
+    table.postgis2es()
+
+    return
+
+
+if __name__ == '__main__':
+    main()
diff --git a/python/dsraAllScenariosDauid_postgres2es.py b/python/dsraAllScenariosDauid_postgres2es.py
@@ -0,0 +1,47 @@
+# =================================================================
+# !/bin/bash
+# SPDX-License-Identifier: MIT
+#
+# Copyright (C) 2020-2021 Government of Canada
+#
+# Main Authors: Drew Rotheram <[email protected]>
+#               Joost van Ulden <[email protected]>
+# =================================================================
+
+
+from utils import ESConnection
+from utils import PostGISdataset
+from utils import PostGISConnection
+
+
+def main():
+    table = PostGISdataset(
+        PostGISConnection(),
+        ESConnection(settings={
+            'settings': {
+                'number_of_shards': 1,
+                'number_of_replicas': 0
+            },
+            'mappings': {
+                'properties': {
+                    'geometry': {
+                        'type': 'geo_shape'
+                    }
+                }
+            }
+        }),
+        view="opendrr_dsra_all_scenarios_dauid",
+        sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
+                    FROM dsra.dsra_all_scenarios_dauid \
+                    ORDER BY dsra_all_scenarios_dauid."dauid" \
+                    LIMIT {limit} \
+                    OFFSET {offset}'
+    )
+
+    table.postgis2es()
+
+    return
+
+
+if __name__ == '__main__':
+    main()
diff --git a/python/dsraAllScenariosEruid_postgres2es.py b/python/dsraAllScenariosEruid_postgres2es.py
@@ -0,0 +1,45 @@
+# =================================================================
+# !/bin/bash
+# SPDX-License-Identifier: MIT
+#
+# Copyright (C) 2020-2021 Government of Canada
+#
+# Main Authors: Drew Rotheram <[email protected]>
+#               Joost van Ulden <[email protected]>
+# =================================================================
+
+
+import utils
+
+
+def main():
+    eruidTable = utils.PostGISdataset(
+        utils.PostGISConnection(),
+        utils.ESConnection(settings={
+            'settings': {
+                'number_of_shards': 1,
+                'number_of_replicas': 0
+            },
+            'mappings': {
+                'properties': {
+                    'geometry': {
+                        'type': 'geo_shape'
+                    }
+                }
+            }
+        }),
+        view="opendrr_dsra_all_scenarios_eruid",
+        sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
+                    FROM dsra.dsra_all_scenarios_eruid \
+                    ORDER BY dsra_all_scenarios_eruid."eruid" \
+                    LIMIT {limit} \
+                    OFFSET {offset}'
+    )
+
+    eruidTable.postgis2es()
+
+    return
+
+
+if __name__ == '__main__':
+    main()
diff --git a/python/dsraAllScenariosSauid_postgres2es.py b/python/dsraAllScenariosSauid_postgres2es.py
@@ -0,0 +1,47 @@
+# =================================================================
+# !/bin/bash
+# SPDX-License-Identifier: MIT
+#
+# Copyright (C) 2020-2021 Government of Canada
+#
+# Main Authors: Drew Rotheram <[email protected]>
+#               Joost van Ulden <[email protected]>
+# =================================================================
+
+
+from utils import ESConnection
+from utils import PostGISdataset
+from utils import PostGISConnection
+
+
+def main():
+    table = PostGISdataset(
+        PostGISConnection(),
+        ESConnection(settings={
+            'settings': {
+                'number_of_shards': 1,
+                'number_of_replicas': 0
+            },
+            'mappings': {
+                'properties': {
+                    'geometry': {
+                        'type': 'geo_shape'
+                    }
+                }
+            }
+        }),
+        view="opendrr_dsra_all_scenarios_sauid",
+        sqlquerystring='SELECT *, ST_AsGeoJSON(geom) \
+                    FROM dsra.dsra_all_scenarios_sauid \
+                    ORDER BY dsra_all_scenarios_sauid."sauid" \
+                    LIMIT {limit} \
+                    OFFSET {offset}'
+    )
+
+    table.postgis2es()
+
+    return
+
+
+if __name__ == '__main__':
+    main()
diff --git a/python/dsra_postgres2es.py b/python/dsra_postgres2es.py
@@ -43,8 +43,8 @@ def main():
     es = Elasticsearch([auth.get('es', 'es_endpoint')],
                        http_auth=(auth.get('es', 'es_un'),
                        auth.get('es', 'es_pw')))
-    if es.indices.exists(view):
-        es.indices.delete(view)
+    if es.indices.exists("opendrr_" + view):
+        es.indices.delete("opendrr_" + view)
     if args.idField.lower() == 'sauid':
         id_field = 'Sauid'
         settings = {
@@ -78,7 +78,7 @@ def main():
                 }
             }
         }
-    es.indices.create(index=view, body=settings, request_timeout=90)
+    es.indices.create(index="opendrr_" + view, body=settings, request_timeout=90)
 
     while True:
         if args.idField.lower() == 'sauid':
@@ -158,7 +158,7 @@ def main():
                                            default=decimal_default)
                 d = json.loads(geojsonobject)
                 helpers.bulk(es,
-                             gendata(d, view, id_field),
+                             gendata(d, "opendrr_" + view, id_field),
                              raise_on_error=False)
 
             else: