Merge pull request #509 from mapswipe/osm_api_to_osmcha

Osm api to osmcha

Author: Hagellach37

.github/workflows/actions.yml

@@ -59,6 +59,7 @@ jobs:
         POSTGRES_PASSWORD: test
         POSTGRES_USER: test
         POSTGRES_DB: test
+        OSMCHA_API_KEY: ${{ secrets.OSMCHA_API_KEY }}
       run: |
         docker-compose run mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/
         docker-compose run mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/integration/

docker-compose.yaml

@@ -73,6 +73,7 @@ services:
             SLACK_TOKEN: '${SLACK_TOKEN}'
             SLACK_CHANNEL: '${SLACK_CHANNEL}'
             SENTRY_DSN: '${SENTRY_DSN}'
+            OSMCHA_API_KEY: '${OSMCHA_API_KEY}'
         depends_on:
             - postgres
         command: mapswipe_workers --verbose run --analysis_type=creation --schedule --time_interval=5
@@ -107,6 +108,7 @@ services:
             SLACK_TOKEN: '${SLACK_TOKEN}'
             SLACK_CHANNEL: '${SLACK_CHANNEL}'
             SENTRY_DSN: '${SENTRY_DSN}'
+            OSMCHA_API_KEY: '${OSMCHA_API_KEY}'
         depends_on:
             - postgres
         command: mapswipe_workers --verbose run --analysis_type=generate-stats --schedule --time_interval=60
@@ -141,6 +143,7 @@ services:
             SLACK_TOKEN: '${SLACK_TOKEN}'
             SLACK_CHANNEL: '${SLACK_CHANNEL}'
             SENTRY_DSN: '${SENTRY_DSN}'
+            OSMCHA_API_KEY: '${OSMCHA_API_KEY}'
         depends_on:
             - postgres
         command: mapswipe_workers --verbose run --analysis_type=firebase-to-postgres --schedule --time_interval=2

example.env

@@ -28,3 +28,6 @@ SLACK_CHANNEL=
 
 # sentry configuration
 SENTRY_DSN=
+
+# osmcha configuration
+OSMCHA_API_KEY=

mapswipe_workers/mapswipe_workers/definitions.py

@@ -14,6 +14,8 @@
 
 OHSOME_API_LINK = "https://api.ohsome.org/v1/"
 OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/"
+OSMCHA_API_LINK = "https://osmcha.org/api/v1/"
+OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"]
 
 # number of geometries for project geometries
 MAX_INPUT_GEOMETRIES = 10

mapswipe_workers/mapswipe_workers/utils/api_calls.py

@@ -7,6 +7,8 @@
 from mapswipe_workers.definitions import (
     OHSOME_API_LINK,
     OSM_API_LINK,
+    OSMCHA_API_KEY,
+    OSMCHA_API_LINK,
     CustomError,
     logger,
 )
@@ -22,12 +24,16 @@ def remove_troublesome_chars(string: str):
     return string
 
 
-def retry_get(url, retries=3, timeout=4):
+def retry_get(url, retries=3, timeout=4, to_osmcha: bool = False):
     """Retry a query for a variable amount of tries."""
     retry = Retry(total=retries)
     with requests.Session() as session:
         session.mount("https://", HTTPAdapter(max_retries=retry))
-        return session.get(url, timeout=timeout)
+        if to_osmcha:
+            headers = {"Authorization": f"Token {OSMCHA_API_KEY}"}
+            return session.get(url, timeout=timeout, headers=headers)
+        else:
+            return session.get(url, timeout=timeout)
 
 
 def geojsonToFeatureCollection(geojson: dict) -> dict:
@@ -49,6 +55,29 @@ def chunks(arr, n_objects):
     ]
 
 
+def query_osmcha(changeset_ids: list, changeset_results):
+    """Get data from changesetId."""
+    id_string = ",".join(map(str, changeset_ids))
+
+    url = OSMCHA_API_LINK + f"changesets/?ids={id_string}"
+    response = retry_get(url, to_osmcha=True)
+    if response.status_code != 200:
+        err = f"osmcha request failed: {response.status_code}"
+        logger.warning(f"{err}")
+        logger.warning(response.json())
+        raise CustomError(err)
+    response = response.json()
+    for feature in response["features"]:
+        changeset_results[int(feature["id"])] = {
+            "username": remove_troublesome_chars(feature["properties"]["user"]),
+            "userid": feature["properties"]["uid"],
+            "comment": remove_troublesome_chars(feature["properties"]["comment"]),
+            "editor": remove_troublesome_chars(feature["properties"]["editor"]),
+        }
+
+    return changeset_results
+
+
 def query_osm(changeset_ids: list, changeset_results):
     """Get data from changesetId."""
     id_string = ",".join(map(str, changeset_ids))
@@ -77,14 +106,17 @@ def query_osm(changeset_ids: list, changeset_results):
             "username": remove_troublesome_chars(username),
             "userid": userid,
             "comment": remove_troublesome_chars(comment),
-            "created_by": remove_troublesome_chars(created_by),
+            "editor": remove_troublesome_chars(created_by),
         }
     return changeset_results
 
 
 def remove_noise_and_add_user_info(json: dict) -> dict:
     """Delete unwanted information from properties."""
     logger.info("starting filtering and adding extra info")
+    batch_size = 100
+
+    # remove noise
     changeset_results = {}
 
     missing_rows = {
@@ -106,21 +138,37 @@ def remove_noise_and_add_user_info(json: dict) -> dict:
         changeset_results[new_properties["changesetId"]] = None
         feature["properties"] = new_properties
 
+    # add info
     len_osm = len(changeset_results.keys())
-    batches = int(len(changeset_results.keys()) / 100) + 1
+    batches = int(len(changeset_results.keys()) / batch_size) + 1
+    logger.info(
+        f"""{len_osm} changesets will be queried in roughly {batches} batches from osmCHA"""  # noqa E501
+    )
+
+    chunk_list = chunks(list(changeset_results.keys()), batch_size)
+    for i, subset in enumerate(chunk_list):
+        changeset_results = query_osmcha(subset, changeset_results)
+        progress = round(100 * ((i + 1) / len(chunk_list)), 1)
+        logger.info(f"finished query {i+1}/{len(chunk_list)}, {progress}")
+
+    missing_ids = [i for i, v in changeset_results.items() if v is None]
+    chunk_list = chunks(missing_ids, batch_size)
+    batches = int(len(missing_ids) / batch_size) + 1
     logger.info(
-        f"""{len_osm} changesets will be queried in roughly {batches} batches"""
+        f"""{len(missing_ids)} changesets where missing from osmCHA and are now queried via osmAPI in {batches} batches"""  # noqa E501
     )
-    chunk_list = chunks(list(changeset_results.keys()), 100)
     for i, subset in enumerate(chunk_list):
         changeset_results = query_osm(subset, changeset_results)
         progress = round(100 * ((i + 1) / len(chunk_list)), 1)
         logger.info(f"finished query {i+1}/{len(chunk_list)}, {progress}")
 
     for feature in json["features"]:
-        changeset = changeset_results[feature["properties"]["changesetId"]]
-        for attribute_name in ["username", "comment", "created_by", "userid"]:
-            feature["properties"][attribute_name] = changeset[attribute_name]
+        changeset = changeset_results[int(feature["properties"]["changesetId"])]
+        for attribute_name in ["username", "comment", "editor", "userid"]:
+            if attribute_name == "userid":
+                feature["properties"][attribute_name] = int(changeset[attribute_name])
+            else:
+                feature["properties"][attribute_name] = changeset[attribute_name]
 
     logger.info("finished filtering and adding extra info")
     if any(x > 0 for x in missing_rows.values()):