diff --git a/api-service/.dockerignore b/api-service/.dockerignore index b512c09d..b83f1fd0 100644 --- a/api-service/.dockerignore +++ b/api-service/.dockerignore @@ -1 +1,3 @@ -node_modules \ No newline at end of file +node_modules +api-service/postman-collection +postman-collection \ No newline at end of file diff --git a/api-service/Dockerfile b/api-service/Dockerfile index 2d0f168d..47f228fd 100644 --- a/api-service/Dockerfile +++ b/api-service/Dockerfile @@ -1,7 +1,8 @@ -FROM --platform=linux/amd64 node:20-alpine +FROM --platform=linux/amd64 node:24.3.0-alpine RUN mkdir -p /opt/api-service COPY ./api-service ./opt/api-service WORKDIR /opt/api-service +RUN rm -rf postman-collection RUN npm install COPY . . EXPOSE 3000 diff --git a/api-service/package.json b/api-service/package.json index 782cda64..e80d65f7 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -22,6 +22,17 @@ "@azure/storage-blob": "^12.17.0", "@google-cloud/storage": "^7.9.0", "@jsonhero/schema-infer": "^0.1.5", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-logs-otlp-http": "^0.53.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.53.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.55.0", + "@opentelemetry/resources": "^1.28.0", + "@opentelemetry/sdk-logs": "^0.53.0", + "@opentelemetry/sdk-metrics": "^1.26.0", + "@opentelemetry/sdk-node": "^0.53.0", + "@opentelemetry/sdk-trace-base": "^1.28.0", + "@opentelemetry/sdk-trace-node": "^1.28.0", + "@opentelemetry/semantic-conventions": "^1.28.0", "@project-sunbird/logger": "^0.0.9", "ajv": "^8.11.2", "ajv-formats": "^2.1.1", @@ -31,6 +42,7 @@ "busboy": "^1.6.0", "compression": "^1.7.4", "dateformat": "2.0.0", + "dayjs": "^1.11.13", "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", "http-status": "^1.5.3", diff --git a/api-service/postman-collection/Obsrv v2 apis.postman_collection.json b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json index b48c505b..cd8d4689 100644 --- a/api-service/postman-collection/Obsrv v2 apis.postman_collection.json +++ b/api-service/postman-collection/Obsrv v2 apis.postman_collection.json @@ -1,11 +1,11 @@ { "info": { - "_postman_id": "d3ffc748-3d71-4395-9298-e065845f7bfb", + "_postman_id": "b94beefb-2a07-49ac-bd5b-07d66fe7ba85", "name": "Obsrv v2 apis", "description": "Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets.", - "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", - "_exporter_id": "26192103", - "_collection_link": "https://speeding-star-177775.postman.co/workspace/2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/26192103-d3ffc748-3d71-4395-9298-e065845f7bfb?action=share&source=collection_link&creator=26192103" + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "406673", + "_collection_link": "https://galactic-flare-389256.postman.co/workspace/config-service-workspace~0ba551f8-be4b-4755-9d4d-bf4578113e5f/collection/406673-b94beefb-2a07-49ac-bd5b-07d66fe7ba85?action=share&source=collection_link&creator=406673" }, "item": [ { @@ -23,14 +23,25 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/datasets/create", + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + }, "description": "This API allows you to create new datasets used by the analytical data source." }, "response": [ @@ -53,7 +64,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -110,7 +132,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -167,7 +200,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "Conflict", "code": 409, @@ -224,7 +268,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "Conflict", "code": 409, @@ -281,7 +336,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "Bad Request", "code": 400, @@ -338,7 +404,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -395,7 +472,18 @@ } } }, - "url": "localhost:3000/v2/datasets/create" + "url": { + "raw": "localhost:3000/v2/datasets/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "create" + ] + } }, "status": "OK", "code": 200, @@ -454,7 +542,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url", + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + }, "description": "This API generates presigned URLs to upload or download files from cloud" }, "response": [ @@ -477,7 +576,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "OK", "code": 200, @@ -535,7 +645,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "OK", "code": 200, @@ -593,7 +714,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "Bad Request", "code": 400, @@ -651,7 +783,18 @@ } } }, - "url": "localhost:3000/v2/files/generate-url" + "url": { + "raw": "localhost:3000/v2/files/generate-url", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "files", + "generate-url" + ] + } }, "status": "Bad Request", "code": 400, @@ -710,7 +853,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update", + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + }, "description": "This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or update transformations and connectors" }, "response": [ @@ -733,7 +887,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "OK", "code": 200, @@ -790,7 +955,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "OK", "code": 200, @@ -847,7 +1023,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Conflict", "code": 409, @@ -904,7 +1091,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Not Found", "code": 404, @@ -961,7 +1159,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Bad Request", "code": 400, @@ -1018,7 +1227,18 @@ } } }, - "url": "localhost:3000/v2/datasets/update" + "url": { + "raw": "localhost:3000/v2/datasets/update", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "update" + ] + } }, "status": "Bad Request", "code": 400, @@ -1100,7 +1320,19 @@ "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" } ], - "url": "localhost:3000/v2/datasets/read/master-test" + "url": { + "raw": "localhost:3000/v2/datasets/read/master-test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "read", + "master-test" + ] + } }, "status": "OK", "code": 200, @@ -1215,7 +1447,7 @@ } ], "url": { - "raw": "localhost:3000/v2/datasets/read/master-test?status=Draft&fields=name,type,id", + "raw": "localhost:3000/v2/datasets/read/master-test?fields=name,type,id", "host": [ "localhost" ], @@ -1227,10 +1459,6 @@ "master-test" ], "query": [ - { - "key": "status", - "value": "Draft" - }, { "key": "fields", "value": "name,type,id" @@ -1486,7 +1714,19 @@ "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" } ], - "url": "localhost:3000/v2/datasets/read/new_telemetry_record.1" + "url": { + "raw": "localhost:3000/v2/datasets/read/new_telemetry_record.1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "read", + "new_telemetry_record.1" + ] + } }, "status": "Not Found", "code": 404, @@ -1540,7 +1780,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list", + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + }, "description": "This API allows you to list all datasets. User can apply filters on dataset status and type." }, "response": [ @@ -1558,7 +1809,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "OK", "code": 200, @@ -1610,7 +1872,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "OK", "code": 200, @@ -1649,7 +1922,7 @@ "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" }, { - "name": "Success: Filter basen on status as string", + "name": "Success: Filter based on status as string", "originalRequest": { "method": "POST", "header": [], @@ -1662,11 +1935,22 @@ } } }, - "url": "localhost:3000/v2/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", "header": [ { "key": "X-Powered-By", @@ -1714,7 +1998,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "OK", "code": 200, @@ -1766,7 +2061,18 @@ } } }, - "url": "localhost:3000/v2/datasets/list" + "url": { + "raw": "localhost:3000/v2/datasets/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "list" + ] + } }, "status": "Bad Request", "code": 400, @@ -1829,7 +2135,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema", + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + }, "description": "This api is used to generate data schema for the given dataset event." }, "response": [ @@ -1856,7 +2173,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema" + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + } }, "status": "OK", "code": 200, @@ -1917,7 +2245,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema" + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + } }, "status": "Bad Request", "code": 400, @@ -1978,7 +2317,18 @@ } } }, - "url": "localhost:3000/v2/datasets/dataschema" + "url": { + "raw": "localhost:3000/v2/datasets/dataschema", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "dataschema" + ] + } }, "status": "Bad Request", "code": 400, @@ -2032,7 +2382,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition", + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + }, "description": "This API allows you to perform status transition between 2 states. Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to Live, Live to Retired and even Delete a dataset." }, "response": [ @@ -2050,14 +2411,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:18:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5948e784-37f9-4a70-85ca-86c9077ee30b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset deleted successfully\",\n \"dataset_id\": \"master.1\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:18:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5948e784-37f9-4a70-85ca-86c9077ee30b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to Delete successful\",\n \"dataset_id\": \"trip-data\"\n }\n}" }, { "name": "Delete failure: Dataset not found to delete", @@ -2073,14 +2445,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Not Found", "code": 404, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:25:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"3cdcf2af-c015-4977-9d66-364e00f1712b\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to delete\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:25:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"3cdcf2af-c015-4977-9d66-364e00f1712b\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found for dataset: master\",\n \"trace\": \"\"\n }\n}" }, { "name": "Live success: Dataset published successfully", @@ -2096,14 +2479,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:21:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"88d62970-97be-472f-9ccc-67f875d69335\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset published successfully\",\n \"dataset_id\": \"telemetry\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:21:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"88d62970-97be-472f-9ccc-67f875d69335\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to Live successful\",\n \"dataset_id\": \"telemetry_record\"\n }\n}" }, { "name": "Live failure: Dataset not found to publish", @@ -2119,14 +2513,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Not Found", "code": 404, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:35:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to publish\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:35:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found for dataset: telemetry-data\",\n \"trace\": \"\"\n }\n}" }, { "name": "Live failure: Dataset in draft state", @@ -2142,14 +2547,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:37:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d56e2ed4-f008-48be-a501-164c19178419\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_NOT_READY_FOR_PUBLISH\",\n \"message\": \"Failed to publish dataset as it is in draft state\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:37:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d56e2ed4-f008-48be-a501-164c19178419\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_LIVE_FAILURE\",\n \"message\": \"Transition failed for dataset: sb-telemetry2 status:Draft with status transition to Live\",\n \"trace\": \"\"\n }\n}" }, { "name": "ReadyToPublish success: Dataset is ready to publish", @@ -2165,7 +2581,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, @@ -2188,7 +2615,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, @@ -2211,14 +2649,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:38:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"351f5a37-87f0-47cd-bebe-e3c001256d0a\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_READYTOPUBLISH_FAILURE\",\n \"message\": \"Failed to mark dataset Ready to publish as it not in draft state\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T15:38:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"351f5a37-87f0-47cd-bebe-e3c001256d0a\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_READYTOPUBLISH_FAILURE\",\n \"message\": \"Transition failed for dataset: telemetry-events status:Retired with status transition to ReadyToPublish\",\n \"trace\": \"\"\n }\n}" }, { "name": "Retire success: Dataset retired successfully", @@ -2234,14 +2683,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:22:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"f2285754-7d5b-4320-943d-797fb136e955\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset retired successfully\",\n \"dataset_id\": \"sb-telemetry\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T12:22:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"f2285754-7d5b-4320-943d-797fb136e955\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset status transition to Retire successful\",\n \"dataset_id\": \"sb-telemetry\"\n }\n}" }, { "name": "Retire Failure: Dataset not found to retire", @@ -2257,14 +2717,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Not Found", "code": 404, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:40:31+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"73befbbd-60e3-48e0-9cfd-cb705dfc2b85\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found to retire\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:40:31+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"73befbbd-60e3-48e0-9cfd-cb705dfc2b85\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset not found for dataset: sb-telemetry2\",\n \"trace\": \"\"\n }\n}" }, { "name": "Retire Failure: Dataset is already retired", @@ -2280,14 +2751,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:42:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"80208169-b1d3-41cd-816b-83fae96a4370\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_ALREADY_RETIRED\",\n \"message\": \"Dataset is already retired\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T15:42:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"80208169-b1d3-41cd-816b-83fae96a4370\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_RETIRE_FAILURE\",\n \"message\": \"Transition failed for dataset: master-telemetrry status:Retired with status transition to Retire\",\n \"trace\": \"\"\n }\n}" }, { "name": "Retire failure: Cannot retire master dataset as it is used by other datasets", @@ -2303,14 +2785,25 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, "_postman_previewlanguage": "json", "header": [], "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:01:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"b88c320a-2c01-4662-a509-bd532a612c05\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_IN_USE\",\n \"message\": \"Failed to retire dataset as it is used by other datasets\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:01:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"b88c320a-2c01-4662-a509-bd532a612c05\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_IN_USE\",\n \"message\": \"Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset\",\n \"trace\": \"\"\n }\n}" }, { "name": "Failure: Invalid request payload provided", @@ -2326,7 +2819,18 @@ } } }, - "url": "localhost:3000/v2/datasets/status-transition" + "url": { + "raw": "localhost:3000/v2/datasets/status-transition", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "status-transition" + ] + } }, "status": "Bad Request", "code": 400, @@ -2334,29 +2838,6 @@ "header": [], "cookie": [], "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:03:56+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"ba4c86bd-b438-4582-b178-2410a5c5dd15\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATASET_STATUS_INVALID_INPUT\",\n \"message\": \"#properties/request/properties/status/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Connection to the db failed", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.status-transition\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry01\",\n \"status\": \"Delete\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/datasets/status-transition" - }, - "status": "Internal Server Error", - "code": 500, - "_postman_previewlanguage": "json", - "header": [], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.status\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T16:24:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"92928434-719f-47d4-9946-1e40ecd53253\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"error\": {\n \"code\": \"DATASET_STATUS_FAILURE\",\n \"message\": \"Failed to perform status transition on datasets\",\n \"trace\": \"\"\n }\n}" } ] }, @@ -2372,7 +2853,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" @@ -2380,7 +2861,7 @@ } }, "url": { - "raw": "localhost:3000/v2/datasets/import?override=true", + "raw": "localhost:3000/v2/datasets/import?overwrite=false", "host": [ "localhost" ], @@ -2392,105 +2873,53 @@ ], "query": [ { - "key": "override", - "value": "true" + "key": "overwrite", + "value": "false" } ] } }, - "response": [] - }, - { - "name": "Dataset export", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v2", - "datasets", - "export", - "v1-copy" - ], - "query": [ - { - "key": "status", - "value": "Live" - } - ] - } - }, - "response": [] - }, - { - "name": "Dataset copy", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"dataset-telemetry\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"bew-copy-live2\"\n }\n }\n}\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/datasets/copy" - }, - "response": [] - } - ], - "description": "The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria." - }, - { - "name": "Connector api's", - "item": [ - { - "name": "Connector list", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3000/v2/connectors/list" - }, "response": [ { - "name": "Failure: Invalid request body, filter option array should not be empty", + "name": "Success: Fully imported dataset", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "true" + } + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -2503,15 +2932,15 @@ }, { "key": "Content-Length", - "value": "399" + "value": "346" }, { "key": "ETag", - "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" + "value": "W/\"15a-SX5o6plRdfy+akEYhbMwq9zl+oU\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:43:14 GMT" + "value": "Tue, 19 Nov 2024 07:34:26 GMT" }, { "key": "Connection", @@ -2523,23 +2952,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:04:26+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"fd87393e-4b1b-4cf0-8e9a-a0529d691cc3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is imported successfully\",\n \"data\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"version_key\": \"1732001666671\"\n }\n }\n}" }, { - "name": "Success: Filtered based on status", + "name": "Success: Partially imported dataset", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "true" + } + ] + } }, "status": "OK", "code": 200, @@ -2555,15 +3006,15 @@ }, { "key": "Content-Length", - "value": "250" + "value": "520" }, { "key": "ETag", - "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" + "value": "W/\"208-vju3WNZ7seMZsQ5AS29Wk0j+yOA\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:55:51 GMT" + "value": "Tue, 19 Nov 2024 07:29:41 GMT" }, { "key": "Connection", @@ -2575,23 +3026,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:59:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"379ced0b-eb06-445f-a7e3-8737cb16c351\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is partially imported\",\n \"data\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"version_key\": \"1732001381825\"\n },\n \"ignoredFields\": {\n \"denorm_fields\": [\n {\n \"config\": {\n \"denorm_key\": \"actor.id\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-dataset\"\n },\n \"details\": \"Master dataset does not exist\"\n }\n ]\n }\n }\n}" }, { - "name": "Success: Filtered based on category", + "name": "Success: Imported new dataset [overwrite=false]", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft\",\n \"dataset_id\": \"sb-telemetryRPF_Draft\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=false", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "false" + } + ] + } }, "status": "OK", "code": 200, @@ -2607,15 +3080,15 @@ }, { "key": "Content-Length", - "value": "2571" + "value": "341" }, { "key": "ETag", - "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" + "value": "W/\"155-UdyHMmAFuHt3xNP/3pFL5e1esfA\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:25:03 GMT" + "value": "Tue, 19 Nov 2024 07:35:32 GMT" }, { "key": "Connection", @@ -2627,26 +3100,48 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:05:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"1a50e1b1-14b0-4178-aba4-9e09a63d8d1d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is imported successfully\",\n \"data\": {\n \"id\": \"sb-telemetry_draft\",\n \"version_key\": \"1732001732045\"\n }\n }\n}" }, { - "name": "Success: Connectors list with all filter options", + "name": "Failure: Import Dataset with id already exists", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=false", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "false" + } + ] + } }, - "status": "OK", - "code": 200, + "status": "Conflict", + "code": 409, "_postman_previewlanguage": "json", "header": [ { @@ -2659,15 +3154,15 @@ }, { "key": "Content-Length", - "value": "4380" + "value": "351" }, { "key": "ETag", - "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" + "value": "W/\"15f-nj5WJ+wiNbI5ViTPWUzGutdBSX4\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:26:32 GMT" + "value": "Tue, 19 Nov 2024 07:31:11 GMT" }, { "key": "Connection", @@ -2679,79 +3174,48 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:01:11+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"1e5b34cd-4473-4439-8cc5-5ff917f47e48\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset with dataset_id: sb-telemetryRPF_draft_data already exists.\"\n }\n}" }, { - "name": "Success: Connectors list without filters", + "name": "Failure: Invalid configs to import", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "http://localhost:3000/v2/connectors/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "4380" - }, - { - "key": "ETag", - "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" - }, - { - "key": "Date", - "value": "Wed, 31 Jul 2024 13:27:37 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=false", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "false" + } + ] } - ], - "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" - } - ] - }, - { - "name": "Connector Read", - "request": { - "method": "GET", - "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" - }, - "response": [ - { - "name": "Success: Read live Connector", - "originalRequest": { - "method": "GET", - "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" }, - "status": "OK", - "code": 200, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -2764,15 +3228,15 @@ }, { "key": "Content-Length", - "value": "3304" + "value": "364" }, { "key": "ETag", - "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" + "value": "W/\"16c-Dxem7prJM89DOqth+bdZbh0UI1g\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 12:47:54 GMT" + "value": "Tue, 19 Nov 2024 07:38:05 GMT" }, { "key": "Connection", @@ -2784,30 +3248,143 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T13:08:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"244318a5-d933-41a5-add6-14f299281069\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_IMPORT_INVALID_CONFIGS\",\n \"message\": \"#properties/request/required must have required property 'id'\"\n }\n}" }, { - "name": "Success: Read Draft Connector", + "name": "Dataset import", "originalRequest": { - "method": "GET", - "header": [], - "url": { - "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", - "protocol": "http", - "host": [ - "localhost" - ], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], "port": "3000", "path": [ "v2", - "connectors", - "read", - "mssql-connector-2.0.0" + "datasets", + "import" ], "query": [ { - "key": "mode", - "value": "edit" + "key": "overwrite", + "value": "true" + } + ] + } + }, + "_postman_previewlanguage": "Text", + "header": [], + "cookie": [], + "body": "" + }, + { + "name": "Dataset import", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.import\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"id\": \"sb-telemetry_draft_data.1\",\n \"dataset_id\": \"sb-telemetryRPF_draft_data\",\n \"name\": \"sb-telemetry_draft_data\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": false,\n \"dedup_config\": {\n \"drop_duplicates\": false,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"extraction_key\": \"events\"\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"dedup_key\": \"id\",\n \"drop_duplicates\": true,\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.ets\"\n }\n ]\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"isRootOrg\": {\n \"type\": \"boolean\",\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"framework\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"resourceType\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"createdBy\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"origin\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"state\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"prevstate\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"query\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": false\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"dataset_id\": \"master-dataset\",\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"keys_config\": {\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"cache_config\": {\n \"redis_db_host\": \"redis-denorm-headless.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [\n \"tag1\"\n ],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1724333643940\",\n \"api_version\": \"v2\",\n \"sample_data\": {},\n \"entry_topic\": \"dev.ingest\",\n \"created_date\": \"2024-07-23T18:35:15.690Z\",\n \"updated_date\": \"2024-08-22T13:34:08.041Z\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/datasets/import?overwrite=true", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "import" + ], + "query": [ + { + "key": "overwrite", + "value": "true" + } + ] + } + }, + "_postman_previewlanguage": "Text", + "header": [], + "cookie": [], + "body": "" + } + ] + }, + { + "name": "Dataset export", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "v1-copy" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } + }, + "response": [ + { + "name": "Success: Live Dataset export success", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/content-data-v2?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "content-data-v2" + ], + "query": [ + { + "key": "status", + "value": "Live" } ] } @@ -2826,15 +3403,15 @@ }, { "key": "Content-Length", - "value": "744" + "value": "5155" }, { "key": "ETag", - "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" + "value": "W/\"1423-Tx46QCrIX8So3vSGVkd865IYeEY\"" }, { "key": "Date", - "value": "Thu, 01 Aug 2024 07:17:12 GMT" + "value": "Tue, 19 Nov 2024 07:16:01 GMT" }, { "key": "Connection", @@ -2846,17 +3423,35 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:46:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ae57c620-2858-40f9-9d8e-33f8f259449d\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"content-data-v2\",\n \"dataset_id\": \"content-data-v2\",\n \"type\": \"master\",\n \"name\": \"content-data-v2\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"key\": \"tripID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"VendorID\": {\n \"key\": \"VendorID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tpep_pickup_datetime\": {\n \"key\": \"tpep_pickup_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"tpep_dropoff_datetime\": {\n \"key\": \"tpep_dropoff_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"passenger_count\": {\n \"key\": \"passenger_count\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"trip_distance\": {\n \"key\": \"trip_distance\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"RatecodeID\": {\n \"key\": \"RatecodeID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"store_and_fwd_flag\": {\n \"key\": \"store_and_fwd_flag\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"PULocationID\": {\n \"key\": \"PULocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"DOLocationID\": {\n \"key\": \"DOLocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"payment_type\": {\n \"key\": \"payment_type\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"primary_passenger\": {\n \"key\": \"primary_passenger\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"email\": {\n \"key\": \"email\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"mobile\": {\n \"key\": \"mobile\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n }\n }\n },\n \"fare_details\": {\n \"key\": \"fare_details\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"fare_amount\": {\n \"key\": \"fare_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"extra\": {\n \"key\": \"extra\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"mta_tax\": {\n \"key\": \"mta_tax\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tip_amount\": {\n \"key\": \"tip_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tolls_amount\": {\n \"key\": \"tolls_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"improvement_surcharge\": {\n \"key\": \"improvement_surcharge\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"total_amount\": {\n \"key\": \"total_amount\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"congestion_surcharge\": {\n \"key\": \"congestion_surcharge\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n }\n }\n }\n },\n \"additionalProperties\": true\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"tripID\",\n \"dedup_period\": 604800\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"content-data-v2\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"container/api-service/user_uploads/chunk-2_addedf.json\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"data_key\": \"\",\n \"partition_key\": \"\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 7\n }\n },\n \"tags\": [],\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"data_version\": 1,\n \"api_version\": \"v2\",\n \"version\": 1,\n \"sample_data\": {\n \"mergedEvent\": {\n \"tripID\": \"ccbb970c-4d10-4247-be0c-a480996f3429\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-26 00:36:10\",\n \"tpep_dropoff_datetime\": \"2023-09-15 00:45:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"263\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jacey_Hintz@yahoo.com\",\n \"mobile\": \"247-492-3370\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.25\",\n \"congestion_surcharge\": \"\"\n }\n }\n },\n \"entry_topic\": \"local.ingest\",\n \"created_date\": \"2024-10-14T06:23:44.588Z\",\n \"updated_date\": \"2024-10-14T06:23:44.588Z\"\n }\n}" }, { - "name": "Failure: Connector not found", + "name": "Success: ReadyToPublish Dataset exported successfully", "originalRequest": { "method": "GET", "header": [], - "url": "http://localhost:3000/v2/connectors/read/postgres-conn" + "url": { + "raw": "localhost:3000/v2/datasets/export/mydataset?status=Draft", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "mydataset" + ], + "query": [ + { + "key": "status", + "value": "Draft" + } + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -2869,15 +3464,15 @@ }, { "key": "Content-Length", - "value": "276" + "value": "5635" }, { "key": "ETag", - "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" + "value": "W/\"1603-tBOO40ZL+TVYU+yC9CWRvXGb7vs\"" }, { "key": "Date", - "value": "Thu, 01 Aug 2024 09:32:48 GMT" + "value": "Tue, 19 Nov 2024 07:18:11 GMT" }, { "key": "Connection", @@ -2889,58 +3484,32 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data Ingest", - "item": [ - { - "name": "Data ingest", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:48:11+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"222847a0-32bf-4c63-be68-76f0e51258af\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mydataset\",\n \"dataset_id\": \"mydataset\",\n \"name\": \"mydataset\",\n \"type\": \"event\",\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n }\n },\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 604800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"key\": \"tripID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"VendorID\": {\n \"key\": \"VendorID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tpep_pickup_datetime\": {\n \"key\": \"tpep_pickup_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"tpep_dropoff_datetime\": {\n \"key\": \"tpep_dropoff_datetime\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\",\n \"isRequired\": false,\n \"resolved\": false\n },\n \"passenger_count\": {\n \"key\": \"passenger_count\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"trip_distance\": {\n \"key\": \"trip_distance\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"RatecodeID\": {\n \"key\": \"RatecodeID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"store_and_fwd_flag\": {\n \"key\": \"store_and_fwd_flag\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"PULocationID\": {\n \"key\": \"PULocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"DOLocationID\": {\n \"key\": \"DOLocationID\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"payment_type\": {\n \"key\": \"payment_type\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"primary_passenger\": {\n \"key\": \"primary_passenger\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"email\": {\n \"key\": \"email\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"mobile\": {\n \"key\": \"mobile\",\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\",\n \"isRequired\": false,\n \"resolved\": true\n }\n },\n \"additionalProperties\": false\n },\n \"fare_details\": {\n \"key\": \"fare_details\",\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"isRequired\": false,\n \"resolved\": true,\n \"properties\": {\n \"fare_amount\": {\n \"key\": \"fare_amount\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"integer\"\n },\n \"1\": {\n \"type\": \"number\"\n }\n }\n },\n \"extra\": {\n \"key\": \"extra\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"number\"\n },\n \"1\": {\n \"type\": \"integer\"\n }\n }\n },\n \"mta_tax\": {\n \"key\": \"mta_tax\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"tip_amount\": {\n \"key\": \"tip_amount\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"number\"\n },\n \"1\": {\n \"type\": \"integer\"\n }\n }\n },\n \"tolls_amount\": {\n \"key\": \"tolls_amount\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"integer\"\n },\n \"1\": {\n \"type\": \"number\"\n }\n }\n },\n \"improvement_surcharge\": {\n \"key\": \"improvement_surcharge\",\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\",\n \"isRequired\": false,\n \"resolved\": true\n },\n \"total_amount\": {\n \"key\": \"total_amount\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true,\n \"isModified\": true,\n \"oneof\": {\n \"0\": {\n \"type\": \"number\"\n },\n \"1\": {\n \"type\": \"integer\"\n }\n }\n },\n \"congestion_surcharge\": {\n \"key\": \"congestion_surcharge\",\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\",\n \"isRequired\": false,\n \"resolved\": true\n }\n },\n \"additionalProperties\": false\n }\n },\n \"additionalProperties\": false\n },\n \"denorm_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"mydataset\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"container/api-service/user_uploads/chunk-1 (4) (1)_5ec855.json\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n },\n \"transformations_config\": [],\n \"connectors_config\": [],\n \"tags\": [],\n \"status\": \"ReadyToPublish\",\n \"version\": 1,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"version_key\": \"1731420131072\",\n \"api_version\": \"v2\",\n \"sample_data\": {\n \"mergedEvent\": {\n \"tripID\": \"4c77e9d5-538d-4eb7-8db1-4c2c32860aa8\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-11-02 00:18:42\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willa67@gmail.com\",\n \"mobile\": \"1-720-981-6399 x77055\"\n },\n \"fare_details\": {\n \"fare_amount\": 7,\n \"extra\": 0.5,\n \"mta_tax\": 0.5,\n \"tip_amount\": 0,\n \"tolls_amount\": 0,\n \"improvement_surcharge\": 0.3,\n \"total_amount\": 8.3,\n \"congestion_surcharge\": 0\n }\n }\n },\n \"entry_topic\": \"local.ingest\",\n \"created_date\": \"2024-11-11T21:31:21.390Z\",\n \"updated_date\": \"2024-11-12T08:32:27.931Z\"\n }\n}" }, - "url": "localhost:3000/v2/data/in/sb-telemetry-imported" - }, - "response": [ { - "name": "Entry topic not found", + "name": "Failure: Dataset not found to export", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/v1-copy?status=Live", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "v1-copy" + ], + "query": [ + { + "key": "status", + "value": "Live" } - } - }, - "url": "localhost:3000/v2/data/in/sample-test" + ] + } }, "status": "Not Found", "code": 404, @@ -2956,15 +3525,15 @@ }, { "key": "Content-Length", - "value": "316" + "value": "301" }, { "key": "ETag", - "value": "W/\"13c-O9iirC/EyneYXQzth7iwEEy1UV4\"" + "value": "W/\"12d-vO+8bWB6kFOrTzeJmTIRlcTcYCQ\"" }, { "key": "Date", - "value": "Thu, 18 Apr 2024 10:17:00 GMT" + "value": "Tue, 19 Nov 2024 07:15:10 GMT" }, { "key": "Connection", @@ -2976,29 +3545,32 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-18T15:47:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"11c32a3a-fdeb-4e00-a9cf-f6433ade5608\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:45:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"fd6fdae2-a002-4c15-81e2-3cce539a2a5a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:v1-copy not found to export\"\n }\n}" }, { - "name": "Failure: schema validation", + "name": "Failure: Invalid dataset state to export", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"mid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n}", - "options": { - "raw": { - "language": "json" + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/datasets/export/new-dataset?status=Draft", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "export", + "new-dataset" + ], + "query": [ + { + "key": "status", + "value": "Draft" } - } - }, - "url": "localhost:3000/v2/data/in/test2" + ] + } }, "status": "Bad Request", "code": 400, @@ -3014,15 +3586,15 @@ }, { "key": "Content-Length", - "value": "300" + "value": "291" }, { "key": "ETag", - "value": "W/\"12c-GNCIUUDxOZD3UfM311sjnmFIgPc\"" + "value": "W/\"123-E2lDLItXYvKnJxGcDJ3Q0OKjqQY\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:03:32 GMT" + "value": "Tue, 19 Nov 2024 07:20:43 GMT" }, { "key": "Connection", @@ -3034,29 +3606,74 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:33:32+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"acf07609-77de-4ab5-81ea-42e41b8b95ff\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_INGESTION_INVALID_INPUT\",\n \"message\": \"#required should have required property 'id'\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Success: Data ingested successfully", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } + "body": "{\n \"id\": \"api.datasets.export\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:50:43+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"2097228c-7872-4217-a06a-0ec08e3dc67d\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXPORT_FAILURE\",\n \"message\": \"Dataset with status:Draft cannot be exported\"\n }\n}" + } + ] + }, + { + "name": "Dataset copy", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"dataset-telemetry\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"bew-copy-live2\"\n }\n }\n}\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/datasets/copy", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "copy" + ] + } + }, + "response": [ + { + "name": "Success: Dataset cloned successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"content-data-v2\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"new-copy-live2\"\n }\n }\n}\n", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/test" + "url": { + "raw": "localhost:3000/v2/datasets/copy", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "copy" + ] + } }, "status": "OK", "code": 200, @@ -3072,15 +3689,15 @@ }, { "key": "Content-Length", - "value": "261" + "value": "296" }, { "key": "ETag", - "value": "W/\"105-UtaCmh0qZMBRBdniNq74Gr+4YAo\"" + "value": "W/\"128-bi4wSfgkyMP13qFE8VPTmgDdgLA\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:05:48 GMT" + "value": "Tue, 19 Nov 2024 07:26:12 GMT" }, { "key": "Connection", @@ -3092,32 +3709,42 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:35:48+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"9a44ac5b-ef82-46f7-92c5-c5b39ef88764\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:56:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"28072631-8e80-45eb-906f-d933a90646d0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"new-copy-live2\",\n \"message\": \"Dataset clone successful\"\n }\n}" }, { - "name": "Success: Data ingested successfully (batch)", + "name": "Failure: Dataset not found to copy", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", - "value": "application/json", - "type": "text" + "value": "application/json" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n },\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n ]\n}", + "raw": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-21T14:30:00Z\",\n \"params\": {\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\"\n },\n \"request\": {\n \"source\": {\n \"datasetId\": \"content-data-v2\",\n \"isLive\": true\n },\n \"destination\": {\n \"datasetId\": \"new-copy-live2\"\n }\n }\n}\n", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/test2" + "url": { + "raw": "localhost:3000/v2/datasets/copy", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "datasets", + "copy" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -3130,15 +3757,15 @@ }, { "key": "Content-Length", - "value": "261" + "value": "328" }, { "key": "ETag", - "value": "W/\"105-ghYxKyKmjCwnqMLgFs+qX269zs0\"" + "value": "W/\"148-jUDshghYf/jOhmmVkvtHlfJT864\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:06:49 GMT" + "value": "Tue, 19 Nov 2024 07:22:41 GMT" }, { "key": "Connection", @@ -3150,32 +3777,75 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:36:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"a220041c-0c28-415a-9687-9fb2e211f8c4\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + "body": "{\n \"id\": \"api.datasets.copy\",\n \"ver\": \"v2\",\n \"ts\": \"2024-11-19T12:52:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"127384e4a-a051-4a9f-9b3f-a64a8034fad7\",\n \"resmsgid\": \"f2fd51bb-fc18-4278-a229-47fa25398e69\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset dataset-telemetry does not exists\"\n }\n}" + } + ] + } + ], + "description": "The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria." + }, + { + "name": "Connector api's", + "item": [ + { + "name": "Connector list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } }, + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } + }, + "response": [ { - "name": "Failure: Dataset not found", + "name": "Failure: Invalid request body, filter option array should not be empty", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/added-tags" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -3188,15 +3858,15 @@ }, { "key": "Content-Length", - "value": "317" + "value": "399" }, { "key": "ETag", - "value": "W/\"13d-a2nqq0Td6EeiGj2lxH+p9BiSz9E\"" + "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:07:29 GMT" + "value": "Tue, 30 Jul 2024 09:43:14 GMT" }, { "key": "Connection", @@ -3208,32 +3878,38 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:37:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"d4db36b4-37b0-4170-a4cf-9d2ae8fa0416\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id not found\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" }, { - "name": "Failure: Entry topic not found", + "name": "Success: Filtered based on status", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/in/sample1" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3246,15 +3922,15 @@ }, { "key": "Content-Length", - "value": "316" + "value": "250" }, { "key": "ETag", - "value": "W/\"13c-auYp3/ERXgtSX5SN4N/gd0cioN0\"" + "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:09:26 GMT" + "value": "Tue, 30 Jul 2024 09:55:51 GMT" }, { "key": "Connection", @@ -3266,60 +3942,35 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:39:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"62309380-3e83-442f-877e-359ed2774bbf\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data query", - "item": [ - { - "name": "Data query", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"Select * from \\\"check\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" }, - "url": "localhost:3000/v2/data/query/test" - }, - "response": [ { - "name": "Success: native query (interval as array type)", + "name": "Success: Filtered based on category", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"dataSource\": {\n \"type\": \"table\",\n \"name\": \"test.1_rollup_month\"\n },\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "OK", "code": 200, @@ -3335,15 +3986,15 @@ }, { "key": "Content-Length", - "value": "313" + "value": "2571" }, { "key": "ETag", - "value": "W/\"139-De+IthAwrGNR+J11CwlNf5RSMmw\"" + "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:17:45 GMT" + "value": "Wed, 31 Jul 2024 13:25:03 GMT" }, { "key": "Connection", @@ -3355,30 +4006,35 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:47:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"134efe35-c096-4cab-ad14-db6a8952f264\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" }, { - "name": "Success: native query (queryType: scan)", + "name": "Success: Connectors list with all filter options", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "OK", "code": 200, @@ -3394,15 +4050,15 @@ }, { "key": "Content-Length", - "value": "313" + "value": "4380" }, { "key": "ETag", - "value": "W/\"139-GxRkXDcEuYT0t/bKOD9V0bicsn0\"" + "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:21:14 GMT" + "value": "Wed, 31 Jul 2024 13:26:32 GMT" }, { "key": "Connection", @@ -3414,30 +4070,35 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"4fc6e3c1-4715-47b7-9137-7713fb2fbe72\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" }, { - "name": "Success: native query (normal intervals & queryType : timeseries)", + "name": "Success: Connectors list without filters", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": \"2020-12-31/2024-01-21\",\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"school_id\"\n }\n },\n \"name\": \"school_id\"\n }\n ]\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "http://localhost:3000/v2/connectors/list", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "list" + ] + } }, "status": "OK", "code": 200, @@ -3453,15 +4114,15 @@ }, { "key": "Content-Length", - "value": "289" + "value": "4380" }, { "key": "ETag", - "value": "W/\"121-s6cWxZurfo7P84IPd5cwnCwxEC4\"" + "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:21:49 GMT" + "value": "Wed, 31 Jul 2024 13:27:37 GMT" }, { "key": "Connection", @@ -3473,33 +4134,53 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75d7ae48-a3db-4367-b50f-34eb99ac3480\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" - }, + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + } + ] + }, + { + "name": "Connector Read", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-connector-1.0.0" + ] + } + }, + "response": [ { - "name": "Failure: Datasource not found in live table", + "name": "Success: Read live Connector", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/data/query/telemetry-eventssss" + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-connector-1.0.0" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3512,15 +4193,15 @@ }, { "key": "Content-Length", - "value": "356" + "value": "3304" }, { "key": "ETag", - "value": "W/\"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE\"" + "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:13:12 GMT" + "value": "Wed, 31 Jul 2024 12:47:54 GMT" }, { "key": "Connection", @@ -3532,33 +4213,36 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:43:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"f922f120-2aea-49af-9a76-7312fe2eb266\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-eventssss not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" }, { - "name": "Failure: Invalid date range", + "name": "Success: Read Draft Connector", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-04-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "mssql-connector-2.0.0" + ], + "query": [ + { + "key": "mode", + "value": "edit" } - } - }, - "url": "localhost:3000/v2/data/query/telemetry-events" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3571,15 +4255,15 @@ }, { "key": "Content-Length", - "value": "371" + "value": "744" }, { "key": "ETag", - "value": "W/\"173-OP8NcbSqLKFO92PIyUmMk0lNsXs\"" + "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:16:29 GMT" + "value": "Thu, 01 Aug 2024 07:17:12 GMT" }, { "key": "Connection", @@ -3591,30 +4275,27 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:46:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"b6434700-dd92-4f64-9250-a22939e753b9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" }, { - "name": "Failure: Datasource not found in druid", + "name": "Failure: Connector not found", "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "name": "Content-Type", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/data/query/test" + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-conn", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-conn" + ] + } }, "status": "Not Found", "code": 404, @@ -3630,15 +4311,15 @@ }, { "key": "Content-Length", - "value": "356" + "value": "276" }, { "key": "ETag", - "value": "W/\"164-XlGTuZDLGKbe6Alm6g23+wLUk4w\"" + "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:20:27 GMT" + "value": "Thu, 01 Aug 2024 09:32:48 GMT" }, { "key": "Connection", @@ -3650,33 +4331,85 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:50:27+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75bd4313-d504-4fd3-92ab-ee2a685beb83\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data Ingest", + "item": [ + { + "name": "Data ingest", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", + "options": { + "raw": { + "language": "json" + } + } }, + "url": { + "raw": "localhost:3000/v2/data/in/sb-telemetry-imported", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "sb-telemetry-imported" + ] + } + }, + "response": [ { - "name": "Success: sql query", + "name": "Entry topic not found", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": \"Select * from \\\"test\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "localhost:3000/v2/data/in/sample-test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "sample-test" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -3689,15 +4422,15 @@ }, { "key": "Content-Length", - "value": "582" + "value": "316" }, { "key": "ETag", - "value": "W/\"246-7TUMqYT3IqZGXE69WJAfYv/tZrI\"" + "value": "W/\"13c-O9iirC/EyneYXQzth7iwEEy1UV4\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:12:12 GMT" + "value": "Thu, 18 Apr 2024 10:17:00 GMT" }, { "key": "Connection", @@ -3709,33 +4442,44 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:42:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2c981011-76da-3000-97f3-eafac939e59f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-18T15:47:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"11c32a3a-fdeb-4e00-a9cf-f6433ade5608\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" }, { - "name": "SQL Failure: dataset not found in druid", + "name": "Failure: schema validation", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", + "raw": "{\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"mid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/test" + "url": { + "raw": "localhost:3000/v2/data/in/test2", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "test2" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -3748,15 +4492,15 @@ }, { "key": "Content-Length", - "value": "356" + "value": "300" }, { "key": "ETag", - "value": "W/\"164-/i6eur3CQhuNnbokC6iAon8EIR0\"" + "value": "W/\"12c-GNCIUUDxOZD3UfM311sjnmFIgPc\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:22:26 GMT" + "value": "Thu, 30 May 2024 18:03:32 GMT" }, { "key": "Connection", @@ -3768,33 +4512,44 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:52:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"19849769-c290-4599-a79e-862dcbce1124\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:33:32+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"acf07609-77de-4ab5-81ea-42e41b8b95ff\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_INGESTION_INVALID_INPUT\",\n \"message\": \"#required should have required property 'id'\",\n \"trace\": \"\"\n }\n}" }, { - "name": "SQL Failure: Datasource not found in live table", + "name": "Success: Data ingested successfully", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", - "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"undefined\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/data/query/undefined" + "url": { + "raw": "localhost:3000/v2/data/in/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "test" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -3807,15 +4562,15 @@ }, { "key": "Content-Length", - "value": "346" + "value": "261" }, { "key": "ETag", - "value": "W/\"15a-pPLEc13PzXoPsR8+egdEub66a5g\"" + "value": "W/\"105-UtaCmh0qZMBRBdniNq74Gr+4YAo\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:23:02 GMT" + "value": "Thu, 30 May 2024 18:05:48 GMT" }, { "key": "Connection", @@ -3827,69 +4582,41 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:53:02+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"530a12f9-04f3-4986-8f52-b3184b9194bd\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource undefined not available for querying\",\n \"trace\": \"\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Data exhaust", - "item": [ - { - "name": "Data exhaust", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "localhost:3000/v2/data/exhaust/beckn-test-data?from=2024-02-01&to=2024-02-22&type=raw", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v2", - "data", - "exhaust", - "beckn-test-data" - ], - "query": [ - { - "key": "from", - "value": "2024-02-01" - }, - { - "key": "to", - "value": "2024-02-22" - }, - { - "key": "type", - "value": "raw" - } - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Query Templates", - "item": [ - { - "name": "Read template", - "request": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/sql_template_11" - }, - "response": [ + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:35:48+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"9a44ac5b-ef82-46f7-92c5-c5b39ef88764\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" + }, { - "name": "Success: JSON template", + "name": "Success: Data ingested successfully (batch)", "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/jsontemplate111" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n },\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/in/test2", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "test2" + ] + } }, "status": "OK", "code": 200, @@ -3905,15 +4632,15 @@ }, { "key": "Content-Length", - "value": "537" + "value": "261" }, { "key": "ETag", - "value": "W/\"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU\"" + "value": "W/\"105-ghYxKyKmjCwnqMLgFs+qX269zs0\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:24:14 GMT" + "value": "Thu, 30 May 2024 18:06:49 GMT" }, { "key": "Connection", @@ -3925,17 +4652,44 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ad1c0a98-1ba3-4203-9fac-59ccaf442347\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:36:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"a220041c-0c28-415a-9687-9fb2e211f8c4\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Data ingested successfully\"\n }\n}" }, { - "name": "Success: SQL template", + "name": "Failure: Dataset not found", "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/sql1" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/in/added-tags", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "added-tags" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -3948,15 +4702,15 @@ }, { "key": "Content-Length", - "value": "528" + "value": "317" }, { "key": "ETag", - "value": "W/\"210-ar2vOqjihmm1SMoqGKcF1ROOKkg\"" + "value": "W/\"13d-a2nqq0Td6EeiGj2lxH+p9BiSz9E\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:24:38 GMT" + "value": "Thu, 30 May 2024 18:07:29 GMT" }, { "key": "Connection", @@ -3968,14 +4722,41 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"70047866-a4b4-4fce-b9c4-4699fcab2dc6\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-05-13T07:29:04.117Z\",\n \"updated_date\": \"2024-05-13T07:29:04.117Z\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:37:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"d4db36b4-37b0-4170-a4cf-9d2ae8fa0416\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id not found\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure : Template not exists", + "name": "Failure: Entry topic not found", "originalRequest": { - "method": "GET", - "header": [], - "url": "localhost:3000/v2/template/read/sql100" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"data\": [\n {\n \"date\": 1693699474326,\n \"school_id\": 10038,\n \"school_category\": \"secondary\",\n \"grade\": 9,\n \"gender\": \"others\",\n \"total_students\": 1193,\n \"students_marked\": 1193,\n \"students_present\": 520,\n \"state_id\": \"15\",\n \"district_id\": \"2003\",\n \"block_id\": \"100\",\n \"cluster_id\": \"624\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/in/sample1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "in", + "sample1" + ] + } }, "status": "Not Found", "code": 404, @@ -3991,15 +4772,15 @@ }, { "key": "Content-Length", - "value": "294" + "value": "316" }, { "key": "ETag", - "value": "W/\"126-zq5j1yor+xr1XFEEbP09fQTDI/k\"" + "value": "W/\"13c-auYp3/ERXgtSX5SN4N/gd0cioN0\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:25:16 GMT" + "value": "Thu, 30 May 2024 18:09:26 GMT" }, { "key": "Connection", @@ -4011,24 +4792,84 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:55:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"958bf0e7-bdb8-4153-abdc-ab84e8004a0e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template sql100 does not exists\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.in\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:39:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"62309380-3e83-442f-877e-359ed2774bbf\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"TOPIC_NOT_FOUND\",\n \"message\": \"Entry topic is not defined\",\n \"trace\": \"\"\n }\n}" } ] - }, + } + ] + }, + { + "name": "Data query", + "item": [ { - "name": "Delete template", + "name": "Data query", "request": { - "method": "DELETE", - "header": [], - "url": "localhost:3000/v2/template/delete/yashash-k" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"Select * from \\\"check\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "response": [ { - "name": "Success: Deleted successfully", + "name": "Success: native query (interval as array type)", "originalRequest": { - "method": "DELETE", - "header": [], - "url": "localhost:3000/v2/template/delete/yashash-k" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"dataSource\": {\n \"type\": \"table\",\n \"name\": \"test.1_rollup_month\"\n },\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"681eb5e4-a6be-4f0b-aafd-9ed41fe1bf97\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "status": "OK", "code": 200, @@ -4044,15 +4885,15 @@ }, { "key": "Content-Length", - "value": "241" + "value": "313" }, { "key": "ETag", - "value": "W/\"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo\"" + "value": "W/\"139-De+IthAwrGNR+J11CwlNf5RSMmw\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:28:27 GMT" + "value": "Thu, 30 May 2024 18:17:45 GMT" }, { "key": "Connection", @@ -4064,17 +4905,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:58:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"3e2859df-f494-4c47-ae64-e2c34f4ef1cb\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Template yashash-k deleted successfully\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:47:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"134efe35-c096-4cab-ad14-db6a8952f264\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" }, { - "name": "Failure: Template does not exists", + "name": "Success: native query (queryType: scan)", "originalRequest": { - "method": "DELETE", - "header": [], - "url": "localhost:3000/v2/template/delete/json_template" + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"\n ]\n },\n \"granularity\": {\n \"type\": \"all\"\n },\n \"aggregations\": [\n {\n \"type\": \"longMin\",\n \"name\": \"a0\",\n \"fieldName\": \"__time\"\n },\n {\n \"type\": \"longMax\",\n \"name\": \"a1\",\n \"fieldName\": \"__time\"\n }\n ],\n \"context\": {\n \"queryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlOuterLimit\": 1001,\n \"sqlQueryId\": \"c2da878a-73d0-4ea1-999e-73bc9b50d034\",\n \"sqlStringifyArrays\": false,\n \"useNativeQueryExplain\": true\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -4087,15 +4956,15 @@ }, { "key": "Content-Length", - "value": "303" + "value": "313" }, { "key": "ETag", - "value": "W/\"12f-99pWw8VTwuVfDAhinC55JXfNyyg\"" + "value": "W/\"139-GxRkXDcEuYT0t/bKOD9V0bicsn0\"" }, { "key": "Date", - "value": "Thu, 30 May 2024 18:28:41 GMT" + "value": "Thu, 30 May 2024 18:21:14 GMT" }, { "key": "Connection", @@ -4107,54 +4976,42 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:58:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"9b7f81fb-6705-4d32-9bd3-139cd5a211b9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template json_template does not exists\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Create query template", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"query_type\": \"sql\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"4fc6e3c1-4715-47b7-9137-7713fb2fbe72\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"a1\": 1694390400000,\n \"a0\": 1694390400000\n }\n }\n ]\n}" }, - "url": "localhost:3000/v2/template/create/json_template_111" - }, - "response": [ { - "name": "Success : Create query template success (json)", + "name": "Success: native query (normal intervals & queryType : timeseries)", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"json11template\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": \"2020-12-31/2024-01-21\",\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"school_id\"\n }\n },\n \"name\": \"school_id\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, "status": "OK", "code": 200, @@ -4170,15 +5027,15 @@ }, { "key": "Content-Length", - "value": "359" + "value": "289" }, { "key": "ETag", - "value": "W/\"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4\"" + "value": "W/\"121-s6cWxZurfo7P84IPd5cwnCwxEC4\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 16:59:01 GMT" + "value": "Thu, 30 May 2024 18:21:49 GMT" }, { "key": "Connection", @@ -4190,32 +5047,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"71372ce0-16b9-4594-8db8-f12eff7e6a42\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:51:49+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75d7ae48-a3db-4367-b50f-34eb99ac3480\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-11T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" }, { - "name": "Success: Create query template success (SQL)", + "name": "Failure: Datasource not found in live table", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"sql1_new\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT COUNT(*) FROM \\\"{{DATASET}}\\\" WHERE \\\"__time\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"table\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/data/query/telemetry-eventssss", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "telemetry-eventssss" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4228,15 +5098,15 @@ }, { "key": "Content-Length", - "value": "339" + "value": "356" }, { "key": "ETag", - "value": "W/\"153-+bOl9+lLq4m+Uttv+Euq8i/zBzE\"" + "value": "W/\"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 16:59:58 GMT" + "value": "Thu, 30 May 2024 18:13:12 GMT" }, { "key": "Connection", @@ -4248,32 +5118,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62aa8c6d-c49c-41c2-9cc8-2f1b02bc2388\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:43:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"f922f120-2aea-49af-9a76-7312fe2eb266\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource telemetry-eventssss not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Template already exists", + "name": "Failure: Invalid date range", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaa\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-04-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/data/query/telemetry-events", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "telemetry-events" + ] + } }, - "status": "Conflict", - "code": 409, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -4286,15 +5169,15 @@ }, { "key": "Content-Length", - "value": "350" + "value": "371" }, { "key": "ETag", - "value": "W/\"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ\"" + "value": "W/\"173-OP8NcbSqLKFO92PIyUmMk0lNsXs\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 17:03:28 GMT" + "value": "Thu, 30 May 2024 18:16:29 GMT" }, { "key": "Connection", @@ -4306,32 +5189,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:33:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"18b6b123-4df5-4124-b6ec-73b667250e1c\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_ALREADY_EXISTS\",\n \"message\": \"Template josnaks-aaa already exists\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:46:29+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"b6434700-dd92-4f64-9250-a22939e753b9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Missing required variables", + "name": "Failure: Datasource not found in druid", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaaq\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-01-31/2023-02-01\"\n ]\n },\n \"granularity\": \"day\",\n \"aggregations\": [\n {\n \"type\": \"filtered\",\n \"aggregator\": {\n \"type\": \"count\",\n \"name\": \"a0\"\n },\n \"filter\": {\n \"type\": \"not\",\n \"field\": {\n \"type\": \"null\",\n \"column\": \"mid\"\n }\n },\n \"name\": \"mid\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4344,15 +5240,15 @@ }, { "key": "Content-Length", - "value": "407" + "value": "356" }, { "key": "ETag", - "value": "W/\"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8\"" + "value": "W/\"164-XlGTuZDLGKbe6Alm6g23+wLUk4w\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 17:05:10 GMT" + "value": "Thu, 30 May 2024 18:20:27 GMT" }, { "key": "Connection", @@ -4364,37 +5260,50 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:35:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62e18342-7e25-4122-8fca-6fb12fac3ff0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID\",\n \"message\": \"Invalid template provided, A template should consist of variables undefined and type of json,sql\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:50:27+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75bd4313-d504-4fd3-92ab-ee2a685beb83\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Schema validation failure", + "name": "Success: sql query", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json", + "name": "Content-Type", "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"test\",\n \"query_type\": \"sq\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"week\"\n },\n \"query\": \"Select * from \\\"test\\\" WHERE __time >= timestamp '2020-12-31' AND __time < TIMESTAMP '2024-01-21' Limit 1\"\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/create" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" }, { "key": "Content-Type", @@ -4402,15 +5311,15 @@ }, { "key": "Content-Length", - "value": "416" + "value": "582" }, { "key": "ETag", - "value": "W/\"1a0-au5PdMUOZbCe2RXYjw+SJZ1EwLs\"" + "value": "W/\"246-7TUMqYT3IqZGXE69WJAfYv/tZrI\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 17:13:57 GMT" + "value": "Thu, 30 May 2024 18:12:12 GMT" }, { "key": "Connection", @@ -4422,45 +5331,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:43:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/oneOf/0/properties/query_type/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "List templates", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n },\n \"sortBy\": [\n {\n \"field\": \"created_date\",\n \"order\": \"desc\"\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:42:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"2c981011-76da-3000-97f3-eafac939e59f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" }, - "url": "localhost:3000/v2/template/list" - }, - "response": [ { - "name": "empty request body", + "name": "SQL Failure: dataset not found in druid", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/data/query/test", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "test" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4473,15 +5382,15 @@ }, { "key": "Content-Length", - "value": "6864" + "value": "356" }, { "key": "ETag", - "value": "W/\"1ad0-xp24UiXXXiFWplmv5Acja7prSYM\"" + "value": "W/\"164-/i6eur3CQhuNnbokC6iAon8EIR0\"" }, { "key": "Date", - "value": "Mon, 29 Apr 2024 13:46:03 GMT" + "value": "Thu, 30 May 2024 18:22:26 GMT" }, { "key": "Connection", @@ -4493,26 +5402,45 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T19:16:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5d41ee6e-2fc6-4353-b6c4-49e068f39b2f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:30:59.381Z\",\n \"updated_date\": \"2024-04-29T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:31:53.976Z\",\n \"updated_date\": \"2024-04-29T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:52:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"19849769-c290-4599-a79e-862dcbce1124\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table day is not available for querying\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Success : order by", + "name": "SQL Failure: Datasource not found in live table", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "name": "Content-Type", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"order\": [\n [\n \"created_date\",\n \"ASC\"\n ]\n ]\n }\n}", + "raw": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\":{\n \"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"context\": {\n \"aggregationLevel\": \"day\"\n },\n \"query\": \"SELECT * FROM \\\"undefined\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1\"\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/data/query/undefined", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "query", + "undefined" + ] + } }, - "status": "OK", - "code": 200, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4525,15 +5453,15 @@ }, { "key": "Content-Length", - "value": "7700" + "value": "346" }, { "key": "ETag", - "value": "W/\"1e14-LtQGbX2UhHK2p4juFd3eddoFjQI\"" + "value": "W/\"15a-pPLEc13PzXoPsR8+egdEub66a5g\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:06:09 GMT" + "value": "Thu, 30 May 2024 18:23:02 GMT" }, { "key": "Connection", @@ -4545,23 +5473,93 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:36:09+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"4393ac57-d441-4be8-b22b-9e4328cab887\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"timeseries\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"aggregations\\\":[{\\\"type\\\":\\\"filtered\\\",\\\"aggregator\\\":{\\\"type\\\":\\\"count\\\",\\\"name\\\":\\\"a0\\\"},\\\"filter\\\":{\\\"type\\\":\\\"not\\\",\\\"field\\\":{\\\"type\\\":\\\"null\\\",\\\"column\\\":\\\"school_id\\\"}},\\\"name\\\":\\\"school_id\\\"}]}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:01.096Z\",\n \"updated_date\": \"2024-04-29T11:29:01.096Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" - }, + "body": "{\n \"id\": \"api.data.out\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:53:02+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"530a12f9-04f3-4986-8f52-b3184b9194bd\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource undefined not available for querying\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data exhaust", + "item": [ + { + "name": "Data exhaust", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/data/exhaust/beckn-test-data?from=2024-02-01&to=2024-02-22&type=raw", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "data", + "exhaust", + "beckn-test-data" + ], + "query": [ + { + "key": "from", + "value": "2024-02-01" + }, + { + "key": "to", + "value": "2024-02-22" + }, + { + "key": "type", + "value": "raw" + } + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Query Templates", + "item": [ + { + "name": "Read template", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/template/read/sql_template_11", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "sql_template_11" + ] + } + }, + "response": [ { - "name": "Success : Filters", + "name": "Success: JSON template", "originalRequest": { - "method": "POST", + "method": "GET", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/template/read/jsontemplate111", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "jsontemplate111" + ] + } }, "status": "OK", "code": 200, @@ -4577,15 +5575,15 @@ }, { "key": "Content-Length", - "value": "2708" + "value": "537" }, { "key": "ETag", - "value": "W/\"a94-N1DxAfP8gWdksidnCf3y626Dg3s\"" + "value": "W/\"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:08:55 GMT" + "value": "Thu, 30 May 2024 18:24:14 GMT" }, { "key": "Connection", @@ -4597,23 +5595,26 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"69da1ef2-c2c5-4f22-bb68-abdf823f0744\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:14+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"ad1c0a98-1ba3-4203-9fac-59ccaf442347\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n }\n}" }, { - "name": "Success : limit and offset", + "name": "Success: SQL template", "originalRequest": { - "method": "POST", + "method": "GET", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"limit\":5,\n \"offset\":0\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/list" + "url": { + "raw": "localhost:3000/v2/template/read/sql1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "sql1" + ] + } }, "status": "OK", "code": 200, @@ -4629,15 +5630,15 @@ }, { "key": "Content-Length", - "value": "1921" + "value": "528" }, { "key": "ETag", - "value": "W/\"781-lpOJppPfhjlnrbwoZ6q194w28Xs\"" + "value": "W/\"210-ar2vOqjihmm1SMoqGKcF1ROOKkg\"" }, { "key": "Date", - "value": "Tue, 30 Apr 2024 06:11:36 GMT" + "value": "Thu, 30 May 2024 18:24:38 GMT" }, { "key": "Connection", @@ -4649,42 +5650,103 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:41:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d1aa35c3-e817-4e2e-85f5-dfd346122192\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n }\n ]\n}" - } - ] - }, - { - "name": "update template", - "request": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:54:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"70047866-a4b4-4fce-b9c4-4699fcab2dc6\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-05-13T07:29:04.117Z\",\n \"updated_date\": \"2024-05-13T07:29:04.117Z\"\n }\n}" }, - "url": "localhost:3000/v2/template/update/sql11template1" + { + "name": "Failure : Template not exists", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "localhost:3000/v2/template/read/sql100", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "read", + "sql100" + ] + } + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "294" + }, + { + "key": "ETag", + "value": "W/\"126-zq5j1yor+xr1XFEEbP09fQTDI/k\"" + }, + { + "key": "Date", + "value": "Thu, 30 May 2024 18:25:16 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:55:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"958bf0e7-bdb8-4153-abdc-ab84e8004a0e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template sql100 does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Delete template", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "localhost:3000/v2/template/delete/yashash-k", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "delete", + "yashash-k" + ] + } }, "response": [ { - "name": "Success: update successful", + "name": "Success: Deleted successfully", "originalRequest": { - "method": "PATCH", + "method": "DELETE", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/delete/yashash-k", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "delete", + "yashash-k" + ] + } }, "status": "OK", "code": 200, @@ -4700,15 +5762,15 @@ }, { "key": "Content-Length", - "value": "314" + "value": "241" }, { "key": "ETag", - "value": "W/\"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k\"" + "value": "W/\"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 05:51:47 GMT" + "value": "Tue, 30 Apr 2024 06:28:27 GMT" }, { "key": "Connection", @@ -4720,26 +5782,29 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:21:47+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e4a6959-0eb9-4fc4-8e6f-2eea534d1384\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Query template updated successfully\",\n \"templateId\": \"sql11template1\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:58:27+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"3e2859df-f494-4c47-ae64-e2c34f4ef1cb\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Template yashash-k deleted successfully\"\n }\n}" }, { - "name": "Failure: required variables not exists to update", + "name": "Failure: Template does not exists", "originalRequest": { - "method": "PATCH", + "method": "DELETE", "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/delete/json_template", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "delete", + "json_template" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4752,15 +5817,15 @@ }, { "key": "Content-Length", - "value": "429" + "value": "303" }, { "key": "ETag", - "value": "W/\"1ad-5sb8WUekFL8s4c1Ink6bUByoHho\"" + "value": "W/\"12f-99pWw8VTwuVfDAhinC55JXfNyyg\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 05:53:54 GMT" + "value": "Thu, 30 May 2024 18:28:41 GMT" }, { "key": "Connection", @@ -4772,26 +5837,79 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:23:54+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"66b95cb3-2ef2-4735-9045-2674da552dbd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Invalid template provided, A template should consist of variables DATASET,STARTDATE,ENDDATE and type of json,sql\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.delete\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-30T23:58:41+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"9b7f81fb-6705-4d32-9bd3-139cd5a211b9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_NOT_EXISTS\",\n \"message\": \"Template json_template does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Create query template", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"query_type\": \"sql\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } }, + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } + }, + "response": [ { - "name": "Failure: Template name validation failure", + "name": "Success : Create query template success (json)", "originalRequest": { - "method": "PATCH", - "header": [], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_. template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"json11template\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -4804,15 +5922,15 @@ }, { "key": "Content-Length", - "value": "405" + "value": "359" }, { "key": "ETag", - "value": "W/\"195-Pvg0z+WwBBq8XA4W0J0QalaIepY\"" + "value": "W/\"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 05:56:59 GMT" + "value": "Mon, 29 Apr 2024 16:59:01 GMT" }, { "key": "Connection", @@ -4824,26 +5942,43 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:26:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d6428fcf-53c9-465d-9431-769218f775b8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Template name should contain alphanumeric characters and single space between characters\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:01+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"71372ce0-16b9-4594-8db8-f12eff7e6a42\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" }, { - "name": "Failure: query_type should when updating query", + "name": "Success: Create query template success (SQL)", "originalRequest": { - "method": "PATCH", - "header": [], + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_. template\",\n \"query\": \"\"\n }\n}", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"sql1_new\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT COUNT(*) FROM \\\"{{DATASET}}\\\" WHERE \\\"__time\\\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -4856,15 +5991,15 @@ }, { "key": "Content-Length", - "value": "412" + "value": "339" }, { "key": "ETag", - "value": "W/\"19c-ETwoh4/x7I2s9qIcbYDNmUId4XQ\"" + "value": "W/\"153-+bOl9+lLq4m+Uttv+Euq8i/zBzE\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 07:02:57 GMT" + "value": "Mon, 29 Apr 2024 16:59:58 GMT" }, { "key": "Connection", @@ -4876,26 +6011,2212 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:32:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c7a8675a-73f2-4764-abba-bfdf9f8b4621\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query_type when property query is present\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:29:58+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62aa8c6d-c49c-41c2-9cc8-2f1b02bc2388\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"message\": \"The query template has been saved successfully\"\n }\n}" }, { - "name": "Failure: query should present when updating query_type", + "name": "Failure: Template already exists", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaa\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTDATE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "350" + }, + { + "key": "ETag", + "value": "W/\"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:03:28 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:33:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"18b6b123-4df5-4124-b6ec-73b667250e1c\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_ALREADY_EXISTS\",\n \"message\": \"Template josnaks-aaa already exists\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Missing required variables", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"josnaks-aaaq\",\n \"query_type\": \"json\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "407" + }, + { + "key": "ETag", + "value": "W/\"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:05:10 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:35:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"62e18342-7e25-4122-8fca-6fb12fac3ff0\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID\",\n \"message\": \"Invalid template provided, A template should consist of variables undefined and type of json,sql\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Schema validation failure", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\":\"test\",\n \"query_type\": \"sq\",\n \"query\": {\n \"queryType\": \"select\",\n \"datasetId\": \"{{DATASET}}\",\n \"intervals\": \"{{STARTTE}}/{{ENDDATE}}\",\n \"limit\": \"{{LIMITS}}\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/create", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "create" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "416" + }, + { + "key": "ETag", + "value": "W/\"1a0-au5PdMUOZbCe2RXYjw+SJZ1EwLs\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 17:13:57 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.create\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T22:43:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/oneOf/0/properties/query_type/enum should be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "List templates", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n },\n \"sortBy\": [\n {\n \"field\": \"created_date\",\n \"order\": \"desc\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } + }, + "response": [ + { + "name": "empty request body", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "6864" + }, + { + "key": "ETag", + "value": "W/\"1ad0-xp24UiXXXiFWplmv5Acja7prSYM\"" + }, + { + "key": "Date", + "value": "Mon, 29 Apr 2024 13:46:03 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-29T19:16:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5d41ee6e-2fc6-4353-b6c4-49e068f39b2f\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:30:59.381Z\",\n \"updated_date\": \"2024-04-29T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:31:53.976Z\",\n \"updated_date\": \"2024-04-29T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n }\n ]\n}" + }, + { + "name": "Success : order by", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"order\": [\n [\n \"created_date\",\n \"ASC\"\n ]\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "7700" + }, + { + "key": "ETag", + "value": "W/\"1e14-LtQGbX2UhHK2p4juFd3eddoFjQI\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:06:09 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:36:09+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"4393ac57-d441-4be8-b22b-9e4328cab887\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n },\n {\n \"template_id\": \"yashash-k\",\n \"template_name\": \"YASHASH k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:39.490Z\",\n \"updated_date\": \"2024-04-25T06:31:39.490Z\"\n },\n {\n \"template_id\": \"yashash-ak\",\n \"template_name\": \"YASHASH ak\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:33:56.501Z\",\n \"updated_date\": \"2024-04-25T06:33:56.501Z\"\n },\n {\n \"template_id\": \"test_template\",\n \"template_name\": \"test template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-26T00:24:34.435Z\",\n \"updated_date\": \"2024-04-26T00:24:34.435Z\"\n },\n {\n \"template_id\": \"jsontemplate\",\n \"template_name\": \"jsontemplate\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:27:32.113Z\",\n \"updated_date\": \"2024-04-28T23:27:32.113Z\"\n },\n {\n \"template_id\": \"jsontemplate1\",\n \"template_name\": \"jsontemplate1\",\n \"query\": \"{\\\"queryType\\\":\\\"timeseries\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"aggregations\\\":[{\\\"type\\\":\\\"filtered\\\",\\\"aggregator\\\":{\\\"type\\\":\\\"count\\\",\\\"name\\\":\\\"a0\\\"},\\\"filter\\\":{\\\"type\\\":\\\"not\\\",\\\"field\\\":{\\\"type\\\":\\\"null\\\",\\\"column\\\":\\\"school_id\\\"}},\\\"name\\\":\\\"school_id\\\"}]}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-28T23:28:35.868Z\",\n \"updated_date\": \"2024-04-28T23:28:35.868Z\"\n },\n {\n \"template_id\": \"jsontemplate111\",\n \"template_name\": \"jsontemplate111\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:16:57.023Z\",\n \"updated_date\": \"2024-04-29T01:16:57.023Z\"\n },\n {\n \"template_id\": \"jso_template\",\n \"template_name\": \"jso template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:19:42.458Z\",\n \"updated_date\": \"2024-04-29T01:19:42.458Z\"\n },\n {\n \"template_id\": \"json_1template\",\n \"template_name\": \"json 1template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:22:19.689Z\",\n \"updated_date\": \"2024-04-29T01:22:19.689Z\"\n },\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"json11template\",\n \"template_name\": \"json11template\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:01.096Z\",\n \"updated_date\": \"2024-04-29T11:29:01.096Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + }, + { + "name": "Success : Filters", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\":{\n \"query_type\":\"sql\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "2708" + }, + { + "key": "ETag", + "value": "W/\"a94-N1DxAfP8gWdksidnCf3y626Dg3s\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:08:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"69da1ef2-c2c5-4f22-bb68-abdf823f0744\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"sql_template\",\n \"template_name\": \"sql template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:24:03.511Z\",\n \"updated_date\": \"2024-04-29T01:24:03.511Z\"\n },\n {\n \"template_id\": \"sql1template\",\n \"template_name\": \"sql1template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:26:28.381Z\",\n \"updated_date\": \"2024-04-29T01:26:28.381Z\"\n },\n {\n \"template_id\": \"sql_template_1\",\n \"template_name\": \"sql template 1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:36:01.592Z\",\n \"updated_date\": \"2024-04-29T01:36:01.592Z\"\n },\n {\n \"template_id\": \"sql_template_11\",\n \"template_name\": \"sql template 11\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T01:39:29.968Z\",\n \"updated_date\": \"2024-04-29T01:39:29.968Z\"\n },\n {\n \"template_id\": \"sql11template\",\n \"template_name\": \"sql11template\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-21T01:30:59.381Z\",\n \"updated_date\": \"2024-04-21T01:30:59.381Z\"\n },\n {\n \"template_id\": \"sql11template1\",\n \"template_name\": \"sql11template1\",\n \"query\": \"\\\"SELECT COUNT(*) FROM \\\\\\\"{{DATASET}}\\\\\\\" WHERE \\\\\\\"__time\\\\\\\" BETWEEN TIMESTAMP \\\\\\\"{{STARTDATE}}\\\\\\\" AND TIMESTAMP \\\\\\\"{{ENDDATE}}\\\\\\\"\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-22T01:31:53.976Z\",\n \"updated_date\": \"2024-04-22T01:31:53.976Z\"\n },\n {\n \"template_id\": \"sql1\",\n \"template_name\": \"sql1\",\n \"query\": \"\\\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\\\"\",\n \"query_type\": \"sql\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-29T11:29:58.759Z\",\n \"updated_date\": \"2024-04-29T11:29:58.759Z\"\n }\n ]\n}" + }, + { + "name": "Success : limit and offset", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"limit\":5,\n \"offset\":0\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/list", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "list" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1921" + }, + { + "key": "ETag", + "value": "W/\"781-lpOJppPfhjlnrbwoZ6q194w28Xs\"" + }, + { + "key": "Date", + "value": "Tue, 30 Apr 2024 06:11:36 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-30T11:41:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d1aa35c3-e817-4e2e-85f5-dfd346122192\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"template_id\": \"josnaksaaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:25:04.706Z\",\n \"updated_date\": \"2024-04-25T06:25:04.706Z\"\n },\n {\n \"template_id\": \"josnaks-aaa\",\n \"template_name\": \"JOSnaks--aaa\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:26:35.749Z\",\n \"updated_date\": \"2024-04-25T06:26:35.749Z\"\n },\n {\n \"template_id\": \"a\",\n \"template_name\": \" a\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:30:50.179Z\",\n \"updated_date\": \"2024-04-25T06:30:50.179Z\"\n },\n {\n \"template_id\": \"yash-k\",\n \"template_name\": \"yash k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:08.821Z\",\n \"updated_date\": \"2024-04-25T06:31:08.821Z\"\n },\n {\n \"template_id\": \"yashas-k\",\n \"template_name\": \"yashas k\",\n \"query\": \"{\\\"queryType\\\":\\\"select\\\",\\\"datasetId\\\":\\\"{{DATASET}}\\\",\\\"intervals\\\":\\\"{{STARTDATE}}/{{ENDDATE}}\\\",\\\"limit\\\":\\\"{{LIMITS}}\\\"}\",\n \"query_type\": \"json\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-04-25T06:31:20.819Z\",\n \"updated_date\": \"2024-04-25T06:31:20.819Z\"\n }\n ]\n}" + } + ] + }, + { + "name": "update template", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "response": [ + { + "name": "Success: update successful", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} lIMIT {{LIMIT}}\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "314" + }, + { + "key": "ETag", + "value": "W/\"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:51:47 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:21:47+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e4a6959-0eb9-4fc4-8e6f-2eea534d1384\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Query template updated successfully\",\n \"templateId\": \"sql11template1\"\n }\n}" + }, + { + "name": "Failure: required variables not exists to update", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "429" + }, + { + "key": "ETag", + "value": "W/\"1ad-5sb8WUekFL8s4c1Ink6bUByoHho\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:53:54 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:23:54+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"66b95cb3-2ef2-4735-9045-2674da552dbd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Invalid template provided, A template should consist of variables DATASET,STARTDATE,ENDDATE and type of json,sql\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Template name validation failure", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_. template\",\n \"query_type\": \"sql\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "405" + }, + { + "key": "ETag", + "value": "W/\"195-Pvg0z+WwBBq8XA4W0J0QalaIepY\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 05:56:59 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T11:26:59+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"d6428fcf-53c9-465d-9431-769218f775b8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"Template name should contain alphanumeric characters and single space between characters\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: query_type should when updating query", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"template_name\": \"sql_update_test_. template\",\n \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "412" + }, + { + "key": "ETag", + "value": "W/\"19c-ETwoh4/x7I2s9qIcbYDNmUId4XQ\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 07:02:57 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:32:57+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c7a8675a-73f2-4764-abba-bfdf9f8b4621\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query_type when property query is present\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: query should present when updating query_type", + "originalRequest": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"json\"\n // \"query\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/update/sql11template1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "update", + "sql11template1" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "412" + }, + { + "key": "ETag", + "value": "W/\"19c-nYBbts9obeaddDUgxH60sH5LgFQ\"" + }, + { + "key": "Date", + "value": "Fri, 10 May 2024 07:04:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:34:55+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c1098b2-d7b3-4d39-98ee-e3e790fd23b4\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query when property query_type is present\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "query template", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } + }, + "response": [ + { + "name": "Failure: invalid date range (native template)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/telemetry-events", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "telemetry-events" + ], + "query": [ + { + "key": "limit", + "value": "1", + "disabled": true + } + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "336" + }, + { + "key": "ETag", + "value": "W/\"150-T/XeSIt7PR7GcGEbET1e8n9zX7k\"" + }, + { + "key": "Date", + "value": "Thu, 02 May 2024 07:29:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-02T12:59:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"4379e16b-2fa3-46a8-8ded-bc53f56283e9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Datasource not found in druid", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "357" + }, + { + "key": "ETag", + "value": "W/\"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:51:46 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:21:46+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b35a7050-b94c-4944-9630-233c9542272e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table hour is not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Datasource not found in live table", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test11\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "343" + }, + { + "key": "ETag", + "value": "W/\"157-RFSDKlFaIxNLaNkfCfKqsUcLowk\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:53:47 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:23:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"3a303dfd-1d95-4788-b1a7-d88809d4dcf3\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource test11 not available for querying\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: invalid date range", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "371" + }, + { + "key": "ETag", + "value": "W/\"173-5xMwCPLPBFPXCvzFzMzFY8weqGs\"" + }, + { + "key": "Date", + "value": "Mon, 13 May 2024 07:58:18 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:28:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"20391fb8-2be8-48b5-a16f-fca150580e97\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Success: JSON template with request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/jsontemplate1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "jsontemplate1" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "301" + }, + { + "key": "ETag", + "value": "W/\"12d-9hKB38iHEwYPT2MgF8puXcq05Ew\"" + }, + { + "key": "Date", + "value": "Tue, 14 May 2024 06:22:24 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:52:24+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-01T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" + }, + { + "name": "Success: SQL template query with request body", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"week\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/template/query/sql_test_1", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "template", + "query", + "sql_test_1" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "594" + }, + { + "key": "ETag", + "value": "W/\"252-GyZnr24ylWKlH/bS4kmbMRx5jes\"" + }, + { + "key": "Date", + "value": "Tue, 14 May 2024 06:03:06 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:33:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"48c194ee-6e73-4ee7-83e6-8b154e441911\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + } + ] + } + ] + }, + { + "name": "Schema validator", + "item": [ + { + "name": "Schema validator", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"datasetId\": \"test_dataset\",\n \"isLive\": true,\n \"event\": {\n \"eid\": \"1\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591794034.0,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"311663b2-d7de-4d46-8803-20407eaa3403\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/data/v2/schema/validate", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "data", + "v2", + "schema", + "validate" + ] + } + }, + "response": [ + { + "name": "Succes: Schema validation successful", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"datasetId\": \"sb-telemetry\",\n \"isLive\": true,\n \"event\": {\n \"eid\": \"1\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591794034.0,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"311663b2-d7de-4d46-8803-20407eaa3403\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/schema/validate", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "schema", + "validate" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "266" + }, + { + "key": "ETag", + "value": "W/\"10a-hzmB3lhC1mzaRhnGxclXsZ1lfHY\"" + }, + { + "key": "Date", + "value": "Thu, 20 Jun 2024 11:19:21 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-20T16:49:21+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"af336811-cfd8-4436-9682-05e0f25e2575\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Success\",\n \"isValid\": true\n }\n}" + }, + { + "name": "Failure: Dataset not exists", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"datasetId\": \"test_dataset\",\n \"isLive\": true,\n \"event\": {\n \"eid\": \"1\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591794034.0,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"311663b2-d7de-4d46-8803-20407eaa3403\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3000/v2/schema/validate", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "schema", + "validate" + ] + } + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "324" + }, + { + "key": "ETag", + "value": "W/\"144-Eg0khgCz78o32V2ogAkVSj5KgLw\"" + }, + { + "key": "Date", + "value": "Tue, 18 Jun 2024 11:39:38 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.schema.validator\",\n \"ver\": \"v2\",\n \"ts\": \"2024-06-18T17:09:38+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"5ad8657b-b556-4563-9d39-e1170f1062e8\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset test_dataset does not exists\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Alert Notification-channels", + "item": [ + { + "name": "search channels", + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "search" + ] + } + }, + "response": [] + }, + { + "name": "Add channel (Slack)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"manager\": \"grafana\",\n \"name\": \"functional-metrics-slack\",\n \"type\": \"slack\",\n \"config\": {\n \"channel\": \"grafana-alerts\",\n \"webhookUrl\": \"https://hooks.slack.com/services/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\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "create" + ] + } + }, + "response": [] + }, + { + "name": "publish channel", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/publish/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "publish", + "c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + ] + } + }, + "response": [] + }, + { + "name": "test channel", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"message\": \"Testing Email integration. If you can read this, it's working!\",\n \"payload\": {\n \"error\": {\n \"selectChannel\": true,\n \"configureChannel\": true\n },\n \"manager\": \"grafana\",\n \"name\": \"udhw\",\n \"type\": \"email\",\n \"config\": {\n \"recipientAddresses\": \"jerald@sanketika.in\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/test", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "test" + ] + } + }, + "response": [] + }, + { + "name": "update channel", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"updated name\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/update/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "update", + "c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + ] + } + }, + "response": [] + }, + { + "name": "delete channel", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/delete/c033ce5e-e3fb-49c2-8eee-ac142da2fe8d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "delete", + "c033ce5e-e3fb-49c2-8eee-ac142da2fe8d" + ] + } + }, + "response": [] + }, + { + "name": "Add channel (Email)", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"manager\": \"grafana\",\n \"name\": \"functional-metrics-email\",\n \"type\": \"email\",\n \"config\": {\n \"recipientAddresses\": \"yravinderkumar33@gmail.com;ravinder@sanketika.in\",\n \"subject\": \"Obsrv Prod Alert\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/notifications/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "notifications", + "create" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Alert silence", + "item": [ + { + "name": "Add Silence", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"startDate\": \"2024-01-11T07:56:23Z\",\n \"endDate\": \"2024-01-11T08:56:23Z\",\n \"alertId\": \"c7464d32-1d8d-4eaf-9b23-1313a3ff8149\",\n \"manager\": \"grafana\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "create" + ] + } + }, + "response": [] + }, + { + "name": "Get Silence", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/get/0ec52a4c-4529-4f70-b6f1-0de4b1e81c13", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "get", + "0ec52a4c-4529-4f70-b6f1-0de4b1e81c13" + ] + } + }, + "response": [] + }, + { + "name": "Delete Silence", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/delete/39c7eec1-db78-4f5d-8e77-38385cc3b7dc", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "delete", + "39c7eec1-db78-4f5d-8e77-38385cc3b7dc" + ] + } + }, + "response": [] + }, + { + "name": "Fetch All Silences", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "search" + ] + } + }, + "response": [] + }, + { + "name": "Edit Request", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"startDate\": \"2023-08-09T10:50:59Z\",\n \"endDate\": \"2023-08-10T10:30:59Z\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/silence/update/da46091b-65db-4df9-85d1-c2341168bcb8", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "silence", + "update", + "da46091b-65db-4df9-85d1-c2341168bcb8" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Alerts Wrapper", + "item": [ + { + "name": "delete rule ", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/delete/0c3e08d4-8e4f-4810-b591-f5a294ee378e", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "delete", + "0c3e08d4-8e4f-4810-b591-f5a294ee378e" + ] + }, + "description": "This URLwill provided access to user to delete any custom rule." + }, + "response": [] + }, + { + "name": "edit rule ", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"description\":\"This alert rule is designed to promptly notify you when one or more servers in your infrastructure become unresponsive or inaccessible.\",\n \"labels\":{\n \"severity\":\"warning\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/update/a6c46c48-79c1-4e34-9552-fe8f13d3c73e", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "update", + "a6c46c48-79c1-4e34-9552-fe8f13d3c73e" + ] + }, + "description": "This URL will provide access to users to edit any properties in the Alert rule and save the changes." + }, + "response": [] + }, + { + "name": "publish rule ", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "" + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/publish/c6c93917-9e5e-4aa6-9986-1c53c45ba7fe", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "publish", + "c6c93917-9e5e-4aa6-9986-1c53c45ba7fe" + ] + } + }, + "response": [] + }, + { + "name": "Add Rule ", + "request": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json, text/plain, */*" + }, + { + "key": "Accept-Language", + "value": "en-GB,en" + }, + { + "key": "Cache-Control", + "value": "no-store" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3A5w5I87Tior-cvNu-SijqRFKGxy_b-WIP.lDKfWAJZbxW0kMaUqj%2B0Ivu%2FvNXXL8S796Fa7%2BNyM9Q; grafana_session=4eb514f6fef4ad6884e47e50254af650; grafana_session_expiry=1687239735" + }, + { + "key": "Origin", + "value": "http://localhost:3001" + }, + { + "key": "Pragma", + "value": "no-store" + }, + { + "key": "Referer", + "value": "http://localhost:3001/alertRules/add" + }, + { + "key": "Sec-Fetch-Dest", + "value": "empty" + }, + { + "key": "Sec-Fetch-Mode", + "value": "cors" + }, + { + "key": "Sec-Fetch-Site", + "value": "same-origin" + }, + { + "key": "Sec-GPC", + "value": "1" + }, + { + "key": "User-Agent", + "value": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"Total Api Calls1\",\n \"manager\":\"grafana\",\n \"description\": \"This alert is set up to notify you when the CPU usage on a host reaches a Low level.\",\n \"expression\": \"(node_total_api_calls) > 20\",\n \"category\": \"Infra\",\n \"severity\":\"warning\",\n \"frequency\": \"1m\",\n \"interval\": \"1m\",\n \"labels\": {\n \"component\": \"api\",\n \"notificationChannel\": \"slack\"\n },\n \"annotations\":{\n \"summary\":\"Host Low CPU usage\"\n },\n \"metadata\": {\n \"query\": [\n {\n \"refId\": \"A\",\n \"datasourceUid\": \"$datasourceUid\",\n \"queryType\": \"\",\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n },\n \"model\": {\n \"refId\": \"A\",\n \"hide\": false,\n \"editorMode\": \"code\",\n \"expr\": \"node_total_api_calls\",\n \"legendFormat\": \"__auto\",\n \"range\": true\n }\n },\n {\n \"refId\": \"B\",\n \"datasourceUid\": \"__expr__\",\n \"queryType\": \"\",\n \"model\": {\n \"refId\": \"B\",\n \"hide\": false,\n \"type\": \"reduce\",\n \"datasource\": {\n \"uid\": \"__expr__\",\n \"type\": \"__expr__\"\n },\n \"conditions\": [\n {\n \"type\": \"query\",\n \"evaluator\": {\n \"params\": [],\n \"type\": \"gt\"\n },\n \"operator\": {\n \"type\": \"and\"\n },\n \"query\": {\n \"params\": [\n \"B\"\n ]\n },\n \"reducer\": {\n \"params\": [],\n \"type\": \"last\"\n }\n }\n ],\n \"reducer\": \"last\",\n \"expression\": \"A\"\n },\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n }\n },\n {\n \"refId\": \"C\",\n \"datasourceUid\": \"__expr__\",\n \"queryType\": \"\",\n \"model\": {\n \"refId\": \"C\",\n \"hide\": false,\n \"type\": \"threshold\",\n \"datasource\": {\n \"uid\": \"__expr__\",\n \"type\": \"__expr__\"\n },\n \"conditions\": [\n {\n \"type\": \"query\",\n \"evaluator\": {\n \"params\": [\n 20\n ],\n \"type\": \"gt\"\n },\n \"operator\": {\n \"type\": \"and\"\n },\n \"query\": {\n \"params\": [\n \"C\"\n ]\n },\n \"reducer\": {\n \"params\": [],\n \"type\": \"last\"\n }\n }\n ],\n \"expression\": \"B\"\n },\n \"relativeTimeRange\": {\n \"from\": 600,\n \"to\": 0\n }\n }\n ]\n }\n}" + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "create" + ] + } + }, + "response": [] + }, + { + "name": "search rule ", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"request\": {\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "search" + ] + } + }, + "response": [] + }, + { + "name": "get specific rule ", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/get/dbbe4fbc-df7c-4ad0-84c0-1b7034d5d8ff", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "get", + "dbbe4fbc-df7c-4ad0-84c0-1b7034d5d8ff" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Alert Metric_alias", + "item": [ + { + "name": "add metric", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"alias\": \"Druid\",\n \"component\": \"MyTest\",\n \"metric\": \"sum(druid_supervisors{state='SUSPENDED'})\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/create", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "create" + ] + } + }, + "response": [] + }, + { + "name": "list metrics", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/search", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "search" + ] + } + }, + "response": [] + }, + { + "name": "update metric", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"alias\":\"Druid native\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/update/48aad878-d59c-4046-aeb5-fb87aefbcfed", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "update", + "48aad878-d59c-4046-aeb5-fb87aefbcfed" + ] + } + }, + "response": [] + }, + { + "name": "remove metric", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{HOST_IP}}/alerts/v1/metric/alias/delete/f168707f-5908-4e2e-9111-e73c41e2700d", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "alerts", + "v1", + "metric", + "alias", + "delete", + "f168707f-5908-4e2e-9111-e73c41e2700d" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Connector registry", + "item": [ + { + "name": "Connector registry upload", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/octet-stream", + "type": "text", + "disabled": true + }, + { + "key": "Content-Type", + "value": "multipart/form-data", + "type": "text" + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "file2", + "type": "file", + "src": "/Users/yashash_kumar/Downloads/object_store_connector-0.1.0.tar", + "disabled": true + }, + { + "key": "file", + "type": "file", + "src": "/Users/yashash_kumar/Downloads/zipfile.tar.gz", + "disabled": true + }, + { + "key": "file3", + "type": "file", + "src": "/Users/yashash_kumar/Downloads/kafka-connector-1.0.0-distribution.tar.gz" + }, + { + "key": "file4", + "value": "hello", + "type": "text", + "disabled": true + } + ] + }, + "url": { + "raw": "localhost:3000/v2/connector/register", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connector", + "register" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Dataset metrics", + "item": [ + { + "name": "Dataset metrics", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":1\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } + }, + "response": [ + { + "name": "Failure: Dataset not found", "originalRequest": { - "method": "PATCH", + "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"template_id\": \"sql11template1\",\n \"template_name\": \"sql_update_test_template\",\n \"query_type\": \"json\"\n // \"query\": \"\"\n }\n}", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":1\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/update/sql11template1" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -4908,15 +8229,15 @@ }, { "key": "Content-Length", - "value": "412" + "value": "330" }, { "key": "ETag", - "value": "W/\"19c-nYBbts9obeaddDUgxH60sH5LgFQ\"" + "value": "W/\"14a-RKjp44K2gCKCdq3dClRy/aYxcd8\"" }, { "key": "Date", - "value": "Fri, 10 May 2024 07:04:55 GMT" + "value": "Wed, 15 Jan 2025 09:45:30 GMT" }, { "key": "Connection", @@ -4928,41 +8249,16 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.update\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-10T12:34:55+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c1098b2-d7b3-4d39-98ee-e3e790fd23b4\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"QUERY_TEMPLATE_INVALID_INPUT\",\n \"message\": \"#properties/request/dependencies should have property query when property query_type is present\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "query template", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:15:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"25107bba-6d88-4e9b-b951-61e29a72b97a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with id test not found in live table\",\n \"trace\": \"\"\n }\n}" }, - "url": "localhost:3000/v2/template/query/sql_test_1" - }, - "response": [ { - "name": "Failure: invalid date range (native template)", + "name": "Failure: Request validation failed", "originalRequest": { "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"test-obsev\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":1\n }\n}", "options": { "raw": { "language": "json" @@ -4970,23 +8266,15 @@ } }, "url": { - "raw": "localhost:3000/v2/template/query/telemetry-events", + "raw": "localhost:3007/v2/dataset/metrics", "host": [ "localhost" ], - "port": "3000", + "port": "3007", "path": [ "v2", - "template", - "query", - "telemetry-events" - ], - "query": [ - { - "key": "limit", - "value": "1", - "disabled": true - } + "dataset", + "metrics" ] } }, @@ -5004,15 +8292,15 @@ }, { "key": "Content-Length", - "value": "336" + "value": "362" }, { "key": "ETag", - "value": "W/\"150-T/XeSIt7PR7GcGEbET1e8n9zX7k\"" + "value": "W/\"16a-pmYCarF3FSxd3J7zVBlhwST+rq8\"" }, { "key": "Date", - "value": "Thu, 02 May 2024 07:29:14 GMT" + "value": "Wed, 15 Jan 2025 09:53:22 GMT" }, { "key": "Connection", @@ -5024,26 +8312,37 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-02T12:59:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"4379e16b-2fa3-46a8-8ded-bc53f56283e9\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:23:22+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a6a26834-bb32-4e32-8dfd-473b0d21c8e8\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\",\n \"trace\": \"\"\n }\n}" }, { - "name": "Failure: Datasource not found in druid", + "name": "Success: Get all the data", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test-obsev\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":10\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -5056,15 +8355,15 @@ }, { "key": "Content-Length", - "value": "357" + "value": "1780" }, { "key": "ETag", - "value": "W/\"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U\"" + "value": "W/\"6f4-nUo39jrIJBlBgZ4cY4UL84W7UNU\"" }, { "key": "Date", - "value": "Mon, 13 May 2024 07:51:46 GMT" + "value": "Wed, 15 Jan 2025 09:54:21 GMT" }, { "key": "Connection", @@ -5076,26 +8375,37 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:21:46+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b35a7050-b94c-4944-9630-233c9542272e\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Dataset test with table hour is not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:24:21+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"ef86ff9c-4342-4cdc-a573-d469cbf436eb\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"category\": \"data_freshness\",\n \"status\": \"Healthy\",\n \"components\": [\n {\n \"type\": \"average_time_difference_in_min\",\n \"threshold\": 5,\n \"value\": 0,\n \"status\": \"Healthy\"\n },\n {\n \"type\": \"freshness_query_time_in_min\",\n \"threshold\": 10,\n \"value\": 0,\n \"status\": \"Healthy\"\n }\n ]\n },\n {\n \"category\": \"data_observability\",\n \"status\": \"Unhealthy\",\n \"components\": [\n {\n \"type\": \"data_observability_health\",\n \"status\": \"Unhealthy\"\n },\n {\n \"type\": \"failure_percentage\",\n \"value\": 35.39823008849557\n },\n {\n \"type\": \"threshold_percentage\",\n \"value\": 5\n },\n {\n \"type\": \"importance_score\",\n \"value\": 0\n }\n ]\n },\n {\n \"category\": \"data_volume\",\n \"components\": [\n {\n \"type\": \"events_per_hour\",\n \"value\": 0\n },\n {\n \"type\": \"events_per_day\",\n \"value\": 95\n },\n {\n \"type\": \"events_per_n_day\",\n \"value\": 765\n },\n {\n \"type\": \"volume_percentage_by_hour\",\n \"value\": 0\n },\n {\n \"type\": \"volume_percentage_by_day\",\n \"value\": -87.58169934640523\n },\n {\n \"type\": \"volume_percentage_by_week\",\n \"value\": 0\n },\n {\n \"type\": \"growth_rate_percentage\",\n \"value\": 0\n }\n ]\n },\n {\n \"category\": \"data_lineage\",\n \"components\": [\n {\n \"type\": \"transformation_success\",\n \"value\": 765\n },\n {\n \"type\": \"dedup_success\",\n \"value\": 384\n },\n {\n \"type\": \"denormalization_success\",\n \"value\": 572\n },\n {\n \"type\": \"total_success\",\n \"value\": 1339\n },\n {\n \"type\": \"total_failed\",\n \"value\": 1034\n },\n {\n \"type\": \"transformation_failed\",\n \"value\": 1\n },\n {\n \"type\": \"dedup_failed\",\n \"value\": 574\n },\n {\n \"type\": \"denorm_failed\",\n \"value\": 193\n }\n ]\n },\n {\n \"category\": \"connectors\",\n \"components\": {\n \"category\": \"connectors\",\n \"components\": [\n {\n \"id\": \"failed\",\n \"type\": \"failed\",\n \"value\": 1608\n },\n {\n \"id\": \"api\",\n \"type\": \"success\",\n \"value\": 765\n }\n ]\n }\n },\n {\n \"category\": \"data_quality\",\n \"components\": [\n {\n \"type\": \"incidents_failed\",\n \"value\": 1034\n },\n {\n \"type\": \"incidents_success\",\n \"value\": 1339\n },\n {\n \"type\": \"total_incidents\",\n \"value\": 2373\n }\n ]\n }\n ]\n}" }, { - "name": "Failure: Datasource not found in live table", + "name": "Success : Get particular category data", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test11\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test-obsev\",\n \"category\": [\n // \"data_freshness\",\n // \"data_observability\",\n // \"data_volume\",\n \"data_lineage\"\n // \"connectors\",\n // \"data_quality\"\n ],\n \"query_time_period\":10\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -5108,15 +8418,15 @@ }, { "key": "Content-Length", - "value": "343" + "value": "594" }, { "key": "ETag", - "value": "W/\"157-RFSDKlFaIxNLaNkfCfKqsUcLowk\"" + "value": "W/\"252-mKqBPcpjk0LPbqNHyghckQWQnHs\"" }, { "key": "Date", - "value": "Mon, 13 May 2024 07:53:47 GMT" + "value": "Wed, 15 Jan 2025 09:55:13 GMT" }, { "key": "Connection", @@ -5128,26 +8438,37 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:23:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"3a303dfd-1d95-4788-b1a7-d88809d4dcf3\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"error\": {\n \"code\": \"DATA_OUT_SOURCE_NOT_FOUND\",\n \"message\": \"Datasource test11 not available for querying\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:25:13+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"10915ec3-a265-4502-bc9b-673562f94512\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"category\": \"data_lineage\",\n \"components\": [\n {\n \"type\": \"transformation_success\",\n \"value\": 765\n },\n {\n \"type\": \"dedup_success\",\n \"value\": 384\n },\n {\n \"type\": \"denormalization_success\",\n \"value\": 572\n },\n {\n \"type\": \"total_success\",\n \"value\": 1339\n },\n {\n \"type\": \"total_failed\",\n \"value\": 1034\n },\n {\n \"type\": \"transformation_failed\",\n \"value\": 1\n },\n {\n \"type\": \"dedup_failed\",\n \"value\": 574\n },\n {\n \"type\": \"denorm_failed\",\n \"value\": 193\n }\n ]\n }\n ]\n}" }, { - "name": "Failure: invalid date range", + "name": "Failure: Failed to fetch metrics", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"hour\",\n \"dataset\":\"telemetry-events\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"test-obsev\",\n \"category\": [\n \"data_freshness\",\n \"data_observability\",\n \"data_volume\",\n \"data_lineage\",\n \"connectors\",\n \"data_quality\"\n ],\n \"query_time_period\":10\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3007/v2/dataset/metrics", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "dataset", + "metrics" + ] + } }, - "status": "Bad Request", - "code": 400, + "status": "Internal Server Error", + "code": 500, "_postman_previewlanguage": "json", "header": [ { @@ -5160,15 +8481,15 @@ }, { "key": "Content-Length", - "value": "371" + "value": "317" }, { "key": "ETag", - "value": "W/\"173-5xMwCPLPBFPXCvzFzMzFY8weqGs\"" + "value": "W/\"13d-xfZKi6jq4UY4l8MNfhQmydAroPc\"" }, { "key": "Date", - "value": "Mon, 13 May 2024 07:58:18 GMT" + "value": "Wed, 15 Jan 2025 09:59:49 GMT" }, { "key": "Connection", @@ -5180,117 +8501,154 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-13T13:28:18+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"20391fb8-2be8-48b5-a16f-fca150580e97\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"DATA_OUT_INVALID_DATE_RANGE\",\n \"message\": \"Invalid date range! make sure your range cannot be more than 30 days\",\n \"trace\": \"\"\n }\n}" + "body": "{\n \"id\": \"api.dataset.metrics\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-15T15:29:49+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"211553f9-ad91-4a4d-a6ba-d77d3e14b7ac\"\n },\n \"responseCode\": \"FAILED\",\n \"error\": {\n \"code\": \"FAILED_TO_FETCH_METRICS\",\n \"message\": \"Error while fetching metrics\",\n \"trace\": \"\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Data Analyze PII", + "item": [ + { + "name": "Data Analyze PII", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"request\": {\n \"id\": \"268cb552-7de9-41cc-a413-a8bd3b64734c\",\n \"dataset_id\": \"taxi-payment-data\",\n \"data\": [\n {\n \"tripID\": \"f0f5186b-3195-486e-9b79-0cd9da895707\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-10-31 00:46:40\",\n \"tpep_dropoff_datetime\": \"2024-10-31 00:53:20\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"151\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger.email\": \"Herminia_Veum@hotmail.com\",\n \"primary_passenger.mobile\": \"200-329-2445 x3611\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } }, + "url": { + "raw": "localhost:3005/v2/data/analyze/pii", + "host": [ + "localhost" + ], + "port": "3005", + "path": [ + "v2", + "data", + "analyze", + "pii" + ] + } + }, + "response": [ { - "name": "Success: JSON template with request body", + "name": "Success - Data Analyze PII", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", - "value": "application/json", - "type": "text" + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" } ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"month\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"request\": {\n \"id\": \"268cb552-7de9-41cc-a413-a8bd3b64734c\",\n \"dataset_id\": \"taxi-payment-data\",\n \"data\": [\n {\n \"tripID\": \"f0f5186b-3195-486e-9b79-0cd9da895707\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-10-31 00:46:40\",\n \"tpep_dropoff_datetime\": \"2024-10-31 00:53:20\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"151\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger.email\": \"Herminia_Veum@hotmail.com\",\n \"primary_passenger.mobile\": \"200-329-2445 x3611\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/jsontemplate1" + "url": { + "raw": "localhost:3005/v2/data/analyze/pii", + "host": [ + "localhost" + ], + "port": "3005", + "path": [ + "v2", + "data", + "analyze", + "pii" + ] + } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, { "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "301" - }, - { - "key": "ETag", - "value": "W/\"12d-9hKB38iHEwYPT2MgF8puXcq05Ew\"" - }, - { - "key": "Date", - "value": "Tue, 14 May 2024 06:22:24 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" + "value": "application/json", + "description": "", + "type": "text" } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:52:24+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"timestamp\": \"2023-09-01T00:00:00.000Z\",\n \"result\": {\n \"school_id\": 0\n }\n }\n ]\n}" + "body": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-16T10:27:56+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"8fd2cfe2-7912-40d1-8710-9b7da9a35bd9\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"field\": \"primary_passenger.email\",\n \"type\": \"internet\",\n \"score\": 0.5,\n \"reason\": [\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m014\",\n \"region\": \"\",\n \"score\": 1\n },\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m004\",\n \"region\": \"\",\n \"score\": 1\n }\n ]\n },\n {\n \"field\": \"primary_passenger.mobile\",\n \"type\": \"phone\",\n \"score\": 0.5,\n \"reason\": [\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m018\",\n \"region\": \"USA\",\n \"score\": 1\n },\n {\n \"code\": \"en\",\n \"resourceKey\": \"pii.descriptions.m005\",\n \"region\": \"\",\n \"score\": 1\n }\n ]\n }\n ]\n}" }, { - "name": "Success: SQL template query with request body", + "name": "Failure - Data Analyze PII", "originalRequest": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"startdate\":\"2020-12-31\",\n \"enddate\":\"2024-12-31\",\n \"aggregationLevel\":\"week\",\n \"dataset\":\"test\",\n \"limit\":5\n }\n}", + "raw": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"1711966306164\",\n \"params\": {\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\"\n },\n \"request\": {\n \"id\": \"268cb552-7de9-41cc-a413-a8bd3b64734c\",\n \"dataset_id\": \"taxi-payment-data\",\n \"data\": [\n {\n \"tripID\": \"f0f5186b-3195-486e-9b79-0cd9da895707\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-10-31 00:46:40\",\n \"tpep_dropoff_datetime\": \"2024-10-31 00:53:20\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"151\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger.email\": \"Herminia_Veum@hotmail.com\",\n \"primary_passenger.mobile\": \"200-329-2445 x3611\"\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "url": "localhost:3000/v2/template/query/sql_test_1" + "url": { + "raw": "localhost:3005/v2/data/analyze/pii", + "host": [ + "localhost" + ], + "port": "3005", + "path": [ + "v2", + "data", + "analyze", + "pii" + ] + } }, - "status": "OK", - "code": 200, + "status": "Internal Server Error", + "code": 500, "_postman_previewlanguage": "json", "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, { "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "594" - }, - { - "key": "ETag", - "value": "W/\"252-GyZnr24ylWKlH/bS4kmbMRx5jes\"" - }, - { - "key": "Date", - "value": "Tue, 14 May 2024 06:03:06 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" + "value": "application/json", + "description": "", + "type": "text" } ], "cookie": [], - "body": "{\n \"id\": \"api.query.template.query\",\n \"ver\": \"v2\",\n \"ts\": \"2024-05-14T11:33:06+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"48c194ee-6e73-4ee7-83e6-8b154e441911\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"__time\": \"2023-09-11T00:00:00.000Z\",\n \"school_category\": \"secondary\",\n \"gender\": \"others\",\n \"state_id\": \"15\",\n \"district_id\": \"2002\",\n \"block_id\": \"70\",\n \"cluster_id\": \"485\",\n \"obsrv.meta.source.connector\": null,\n \"obsrv.meta.source.id\": null,\n \"grade_sum\": 18,\n \"school_id_sum\": 180378,\n \"students_marked_sum\": 12492,\n \"students_present_sum\": 2466,\n \"total_count\": 18,\n \"total_students_sum\": 12492\n }\n ]\n}" + "body": "{\n \"id\": \"api.data.analyze.pii\",\n \"ver\": \"v2\",\n \"ts\": \"2025-01-16T10:29:58+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"e180ecac-8f41-4f21-9a21-0b3a1a368917\",\n \"resmsgid\": \"75aa71f9-ef84-41b9-ae1b-64cc7afd3ffb\"\n },\n \"responseCode\": \"INTERNAL_SERVER_ERROR\",\n \"error\": {\n \"code\": \"ERR_BAD_REQUEST\",\n \"message\": \"Request failed with status code 401\",\n \"trace\": \"\"\n }\n}" } ] } diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 6615ad74..a422ccc6 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -1,23 +1,30 @@ import express, { Application } from "express"; -import {router as v2Router} from "./routes/Router" -import { metricRouter } from "./routes/MetricRouter" -import { druidProxyRouter } from "./routes/DruidProxyRouter" +import { druidProxyRouter } from "./routes/DruidProxyRouter"; +import { metricRouter } from "./routes/MetricRouter"; +import { router as v2Router } from "./routes/Router"; import bodyParser from "body-parser"; -import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; -import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; +import { ResponseHandler } from "./helpers/ResponseHandler"; +import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; +import { OTelService } from "./services/otel/OTelService"; import { alertsRouter } from "./routes/AlertsRouter"; -import { interceptAuditEvents } from "./services/telemetry"; +import { interceptAuditEvents, interceptLogEvents } from "./services/telemetry"; +import _ from "lodash"; + + const app: Application = express(); - +// Initialisation of Open telemetry Service. +(config.otel && _.toLower(config?.otel?.enable) === "true") ? OTelService.init() : console.info("OpenTelemetry Service is Disabled"); + app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.use(errorHandler) app.use(interceptAuditEvents()); +app.use(interceptLogEvents()); app.use("/v2/", v2Router); app.use("/", druidProxyRouter); app.use("/alerts/v1", alertsRouter); diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index b26a6906..9186aad0 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -1,12 +1,27 @@ // These configurations provide settings and values for various aspects of dataset management, data ingestion, and table configurations in a system. -const env = process.env.system_env || "local" +const env = process.env.system_env || "obsrv" + +const getCloudConfigs = () => { + const cloudConfig = process.env.cloud_storage_config; + const cloudProvider = process.env.cloud_storage_provider || "aws"; + if (cloudConfig) { + const parsedCloudConfig = JSON.parse(cloudConfig); + if (cloudProvider === "aws") { + const updatedCloudConfig = { ...parsedCloudConfig, "webIdentityTokenFile": process.env.AWS_WEB_IDENTITY_TOKEN_FILE, "roleArn": process.env.AWS_ROLE_ARN }; + return updatedCloudConfig; + } + return parsedCloudConfig; + } + return {}; +} + export const config = { "env": env, "api_port": process.env.api_port || 3000, "body_parser_limit": process.env.body_parser_limit || "100mb", - "version": "1.0", + "version": process.env.obsrv_version || "1.6.0", "query_api": { "druid": { "host": process.env.druid_host || "http://localhost", @@ -14,7 +29,9 @@ export const config = { "sql_query_path": "/druid/v2/sql/", "native_query_path": "/druid/v2", "list_datasources_path": "/druid/v2/datasources", - "submit_ingestion": "druid/indexer/v1/supervisor" + "submit_ingestion": "druid/indexer/v1/supervisor", + "username": process.env.druid_username || "admin", + "password": process.env.druid_password || "admin123" }, "prometheus": { "url": process.env.prometheus_url || "http://localhost:9090" @@ -42,8 +59,8 @@ export const config = { "connectionTimeout": process.env.kafka_connection_timeout ? parseInt(process.env.kafka_connection_timeout) : 5000 }, "topics": { // Default Kafka topics depend on type of dataset. - "createDataset": `${process.env.system_env || "local"}.ingest`, - "createMasterDataset": `${process.env.system_env || "local"}.masterdata.ingest` + "createDataset": `ingest`, + "createMasterDataset": `masterdata.ingest` } } }, @@ -78,12 +95,14 @@ export const config = { "cloud_config": { "cloud_storage_provider": process.env.cloud_storage_provider || "aws", // Supported providers - AWS, GCP, Azure "cloud_storage_region": process.env.cloud_storage_region || "", // Region for the cloud provider storage - "cloud_storage_config": process.env.cloud_storage_config ? JSON.parse(process.env.cloud_storage_config) : {}, // Respective credentials object for cloud provider. Optional if service account provided + "cloud_storage_config": process.env.cloud_storage_config ? getCloudConfigs() : {}, // Respective credentials object for cloud provider. Optional if service account provided "container": process.env.container || "container", // Storage container/bucket name "container_prefix": process.env.container_prefix || "", // Path to the folder inside container/bucket. Empty if data at root level "storage_url_expiry": process.env.storage_url_expiry ? parseInt(process.env.storage_url_expiry) : 3600, // in seconds, Default 1hr of expiry for Signed URLs. "maxQueryDateRange": process.env.exhaust_query_range ? parseInt(process.env.exhaust_query_range) : 31, // in days. Defines the maximum no. of days the files can be fetched - "exclude_exhaust_types": process.env.exclude_exhaust_types ? process.env.exclude_exhaust_types.split(",") : ["system-stats", "masterdata-system-stats", "system-events",] // list of folder type names to skip exhaust service + "exclude_exhaust_types": process.env.exclude_exhaust_types ? process.env.exclude_exhaust_types.split(",") : ["system-stats", "masterdata-system-stats", "system-events",], // list of folder type names to skip exhaust service + "telemetry_data_path": process.env.telemetry_data_path || "telemetry-data", + "azure_blob_type": process.env.azure_blob_type || "BlockBlob" }, "template_config": { "template_required_variables": process.env.template_required_vars ? process.env.template_required_vars.split(",") : ["DATASET", "STARTDATE", "ENDDATE"], @@ -98,7 +117,7 @@ export const config = { "command_service_config": { "host": process.env.command_service_host || "http://localhost", "port": parseInt(process.env.command_service_port || "8000"), - "path": process.env.command_service_path || "/system/v1/dataset/command" + "paths": JSON.parse(process.env.command_service_paths || '{"dataset":"/system/v1/dataset/command","connector":"/connector/v1/register","analyzePII":"/system/data/v1/analyze/pii"}') }, "flink_job_configs": { "pipeline_merged_job_manager_url": process.env.pipeline_merged_job_manager_url || "http://localhost:8081", @@ -114,5 +133,22 @@ export const config = { "access_token": process.env.grafana_token || "" }, "user_token_public_key": process.env.user_token_public_key || "", - "is_RBAC_enabled": process.env.is_rbac_enabled || "false", + "is_RBAC_enabled": process.env.is_rbac_enabled || "true", + "telemetry_log": process.env.telemetry_log || '{"enable":true,"response_data":false}', + "otel": { + "enable": process.env.otel_enable || "false", + "collector_endpoint": process.env.otel_collector_endpoint || "http://localhost:4318" + }, + "storage_types": process.env.storage_types || '{"lake_house":true,"realtime_store":true}', + "data_observability": { + "default_freshness_threshold": process.env.default_freshness_threshold ? parseInt(process.env.default_freshness_threshold) : 5, // in minutes + "data_out_query_time_period": process.env.data_out_query_time_period || "1d", + "default_query_time_period": process.env.default_query_time_period ? parseInt(process.env.default_query_time_period) : 7, // in days + }, + "alerts_rules": { + "config_path": process.env.alerts_config_path + }, + "dataset_filter_config": { + "status_filter_limit": process.env.status_filter_limit ? parseInt(process.env.status_filter_limit) : 10 // Maximum number of filters allowed in a dataset + } } diff --git a/api-service/src/configs/DatasetConfigDefault.ts b/api-service/src/configs/DatasetConfigDefault.ts index 27bdeda0..5c1802dc 100644 --- a/api-service/src/configs/DatasetConfigDefault.ts +++ b/api-service/src/configs/DatasetConfigDefault.ts @@ -17,7 +17,7 @@ export const defaultDatasetConfig = { } }, "dedup_config": { - "drop_duplicates": true, + "drop_duplicates": false, "dedup_key": "id", "dedup_period": 604800, // 7 days }, diff --git a/api-service/src/configs/alertsConfig.json b/api-service/src/configs/alertsConfig.json new file mode 100644 index 00000000..0bea0170 --- /dev/null +++ b/api-service/src/configs/alertsConfig.json @@ -0,0 +1,184 @@ +{ + "configs": { + "alerts": { + "dataset_metrics_flink": [ + { + "metric": "sum(sum_over_time(flink_taskmanager_job_task_operator_ExtractorJob_dataset_id_extractor_failed_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_ExtractorJob_dataset_id_extractor_duplicate_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_dataset_id_validator_failed_count[5m]))", + "alias": "[DATASET]: Detected high rate of invalid data than expected", + "category": "Processing", + "severity": "critical", + "code": "ALERT_1203", + "description": "The dataset is unhealthy, and the query results may be incorrect", + "summary": "Invalid data has been ingested in the system, preventing it from being processed. Henceforth, queries on this dataset may not return accurate data.", + "frequency": "5m", + "interval": "5m", + "operator": "gt", + "threshold": 0 + }, + { + "metric": "sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_dataset_id_dedup_failed_count[5m]))", + "alias": "[DATASET]: Detected higher rate of duplicate data than expected", + "category": "Processing", + "severity": "warning", + "code": "ALERT_1204", + "description": "The dataset is unhealthy, and the query results may be incorrect", + "summary": "Duplicate data has been ingested in the system, preventing it from being processed. Henceforth, queries on this dataset may not return accurate data.", + "frequency": "5m", + "interval": "5m", + "operator": "gt", + "threshold": 0 + }, + { + "metric": "sum(sum_over_time(flink_taskmanager_job_task_operator_DenormalizerJob_dataset_id_denorm_failed[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_DenormalizerJob_dataset_id_denorm_partial_success[5m]))", + "alias": "[DATASET]: Detected higher incidence of failures during data enrichment.", + "category": "Processing", + "severity": "warning", + "code": "ALERT_1205", + "description": "The dataset is unhealthy, and the query results may be incorrect", + "summary": "The data ingested into the system is failing the enrichment process, which may cause queries on this dataset to return inaccurate data.", + "frequency": "5m", + "interval": "5m", + "operator": "gt", + "threshold": 0 + }, + { + "metric": "sum(sum_over_time(flink_taskmanager_job_task_operator_TransformerJob_dataset_id_transform_failed_count[5m])) + sum(sum_over_time(flink_taskmanager_job_task_operator_TransformerJob_dataset_id_transform_partial_count[5m]))", + "alias": "[DATASET]: Detected higher incidence of failures during data transformations.", + "category": "Processing", + "severity": "warning", + "code": "ALERT_1206", + "description": "The dataset is unhealthy, and the query results may be incorrect", + "summary": "The data ingested into the system is failing the data transformation process, which may cause queries on this dataset to return inaccurate data.", + "frequency": "5m", + "interval": "5m", + "operator": "gt", + "threshold": 0 + }, + { + "metric": "sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_dataset_id_validator_total_count[$__range]))", + "alias": "[DATASET]: No data has been received for the past hour.", + "category": "Processing", + "severity": "warning", + "code": "ALERT_1209", + "description": "The dataset hasn’t received any new data for the past hour, which may affect the querying of the new data.", + "summary": "The dataset has not received any new data, which will impact real-time data processing", + "frequency": "5m", + "interval": "60m", + "operator": "lt", + "threshold": 1 + } + ], + "dataset_metrics_druid": [ + { + "metric": "max(druid_supervisors{supervisor_name=\"dataset_id\", state=\"RUNNING\"} or (0 * absent(druid_supervisors{supervisor_name=\"dataset_id\", state=\"RUNNING\"})))", + "alias": "[DATASET]: Druid supervisor is in an unhealthy state", + "category": "Querying", + "severity": "critical", + "code": "ALERT_1309", + "description": "The dataset is unhealthy, and no new data has been available for querying since the system encountered the issue.", + "summary": "The associated Druid Supervisor is in an unhealthy state, preventing druid ingestion tasks from running. As a result, real-time data cannot be queried.", + "frequency": "5m", + "interval": "5m", + "operator": "lt", + "threshold": 1 + }, + { + "metric": "druid_ingest_events_unparseable_total{dataSource=\"dataset_id\"}", + "alias": "[DATASET]: Detected higher amount of unparseable data.", + "flattened": true, + "category": "Querying", + "severity": "critical", + "code": "ALERT_1308", + "description": "The dataset is unhealthy, and the query results may be incorrect", + "summary": "Unparseable data has been detected in the system, preventing it from being processed. Henceforth, queries on this dataset may not return accurate data until the issue is resolved.", + "frequency": "5m", + "interval": "5m", + "operator": "gt", + "threshold": 0 + }, + { + "metric": "druid_ingest_kafka_lag{dataSource=\"dataset_id\"}", + "alias": "[DATASET]: Detected higher amount of query lag than expected.", + "category": "Querying", + "flattened": true, + "severity": "critical", + "code": "ALERT_1307", + "description": "A large amount of data is still waiting to be processed. This may cause delays in querying the most recent data", + "summary": "High indexer lag in the dataset indicates processing of new data is delayed. Because of this delay, new data isn’t available when querying the dataset.", + "frequency": "5m", + "interval": "60m", + "operator": "gt", + "threshold": 5000000 + }, + { + "metric": "druid_ingest_kafka_lag{dataSource=\"dataset_id\"}", + "alias": "[DATASET]: Druid Supervisor Ingestion Failure Due to Offsets.", + "category": "Querying", + "flattened": true, + "severity": "critical", + "code": "ALERT_1312", + "description": "The dataset is unhealthy, and no new data has been available for querying since the issue occurred", + "summary": "The supervisor is experiencing a negative offset, preventing it from ingesting new data. As a result, real-time data is unavailable for querying.", + "frequency": "5m", + "interval": "5m", + "operator": "lt", + "threshold": 0 + }, + { + "metric": "count(druid_tasks_duration{task_status='FAILED', datasource='dataset_id'}) OR on() vector(0)", + "alias": "[DATASET]: Druid tasks are in an unhealthy state", + "category": "Querying", + "severity": "critical", + "code": "ALERT_1310", + "description": "The dataset is unhealthy, and no new data has been available for querying since the system encountered the issue.", + "summary": "The Druid ingestion tasks are in an unhealthy state, causing data ingestion delays and failures. As a result, real-time data may not be available for querying.", + "frequency": "5m", + "interval": "5m", + "operator": "gt", + "threshold": 0 + } + ], + "api_metric": [ + { + "metric": "sum(sum_over_time(node_failed_api_calls{dataset_id='', id='api.data.out'}[$__range])) or vector(0)", + "alias": "[DATASET]: The Data Query API is encountering more failures to retrieve the data", + "category": "Querying", + "severity": "warning", + "code": "ALERT_1305", + "description": "The dataset has been unavailable for querying data for an extented period", + "summary": "Query failures are preventing access to the dataset, resulting in an inability to retrieve data as expected.", + "frequency": "5m", + "interval": "5m", + "operator": "gt", + "threshold": 0 + }, + { + "metric": "avg(avg_over_time(node_query_response_time{dataset_id='', id='api.data.out'}[$__range])) or vector(0)", + "alias": "[DATASET]: The Data Query API is facing delays in retrieving data", + "category": "Querying", + "severity": "warning", + "code": "ALERT_1306", + "description": "There is a delay in querying the dataset for an extended period.", + "summary": "Delays in queries are affecting access to the dataset, leading to delayed data retrieval.", + "frequency": "5m", + "interval": "5m", + "operator": "gt", + "threshold": 1000 + }, + { + "metric": "sum(sum_over_time(node_failed_api_calls{dataset_id='', id='api.data.in'}[$__range])) or vector(0)", + "alias": "[DATASET]: Failed to ingest data into the system", + "category": "Ingestion", + "severity": "warning", + "code": "ALERT_1101", + "description": "Detected failures while adding new data to the dataset.", + "summary": "Failed to add new data to the dataset, impacting real-time data availability.", + "frequency": "5m", + "interval": "5m", + "operator": "gt", + "threshold": 0 + } + ] + } + } +} \ No newline at end of file diff --git a/api-service/src/connections/commandServiceConnection.ts b/api-service/src/connections/commandServiceConnection.ts index fc77fc77..d07b01b2 100644 --- a/api-service/src/connections/commandServiceConnection.ts +++ b/api-service/src/connections/commandServiceConnection.ts @@ -5,11 +5,14 @@ import { v4 } from "uuid"; const commandHost = _.get(config, ["command_service_config", "host"]) const commandPort = _.get(config, ["command_service_config", "port"]) -const commandPath = _.get(config, ["command_service_config", "path"]) +const commandPaths = _.get(config, ["command_service_config", "paths"]) +const datasetPath = _.get(commandPaths, ["dataset"]) +const connectorRegisterPath = _.get(commandPaths, ["connector"]) +const analyzePIIPath = _.get(commandPaths, ["analyzePII"]) export const commandHttpService = axios.create({ baseURL: `${commandHost}:${commandPort}`, headers: { "Content-Type": "application/json" } }); -export const executeCommand = async (id: string, command: string) => { +export const executeCommand = async (id: string, command: string, userToken: string) => { const payload = { "id": v4(), "data": { @@ -17,9 +20,13 @@ export const executeCommand = async (id: string, command: string) => { "command": command } } - return commandHttpService.post(commandPath, payload) + return commandHttpService.post(datasetPath, payload, { headers: { Authorization: userToken }}) } -export const registerConnector = async (requestBody: any) => { - return commandHttpService.post("/connector/v1/register", requestBody) +export const registerConnector = async (requestBody: any, userToken: string) => { + return commandHttpService.post(connectorRegisterPath, requestBody, { headers: { Authorization: userToken }}) +} + +export const detectPII = async (requestBody: any, userToken: string) => { + return commandHttpService.post(analyzePIIPath, requestBody, { headers: { Authorization: userToken }}) } \ No newline at end of file diff --git a/api-service/src/connections/druidConnection.ts b/api-service/src/connections/druidConnection.ts index 7ca7b2dc..21e5ed2c 100644 --- a/api-service/src/connections/druidConnection.ts +++ b/api-service/src/connections/druidConnection.ts @@ -3,22 +3,50 @@ import * as _ from "lodash"; import { config } from "../configs/Config"; const druidPort = _.get(config, "query_api.druid.port"); const druidHost = _.get(config, "query_api.druid.host"); +const druidUsername = _.get(config, "query_api.druid.username"); +const druidPassword = _.get(config, "query_api.druid.password"); const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; const sqlQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.sql_query_path}`; export const executeNativeQuery = async (payload: any) => { - const queryResult = await axios.post(nativeQueryEndpoint, payload) + const queryResult = await axios.post(nativeQueryEndpoint, payload, + { + auth: { + username: druidUsername, + password: druidPassword, + }, + } + ) return queryResult; } export const executeSqlQuery = async (payload: any) => { - const queryResult = await axios.post(sqlQueryEndpoint, payload) + const queryResult = await axios.post(sqlQueryEndpoint, payload, + { + auth: { + username: druidUsername, + password: druidPassword, + }, + } + ) return queryResult; } export const getDatasourceListFromDruid = async () => { - const existingDatasources = await axios.get(`${config?.query_api?.druid?.host}:${config?.query_api?.druid?.port}${config.query_api.druid.list_datasources_path}`, {}) + const existingDatasources = await axios.get(`${config?.query_api?.druid?.host}:${config?.query_api?.druid?.port}${config.query_api.druid.list_datasources_path}`, { + auth: { + username: druidUsername, + password: druidPassword, + } + }) return existingDatasources; } -export const druidHttpService = axios.create({ baseURL: `${config.query_api.druid.host}:${config.query_api.druid.port}`, headers: { "Content-Type": "application/json" } }); \ No newline at end of file +export const druidHttpService = axios.create({ + baseURL: `${config.query_api.druid.host}:${config.query_api.druid.port}`, + headers: { "Content-Type": "application/json" }, + auth: { + username: druidUsername, + password: druidPassword, + } +}); \ No newline at end of file diff --git a/api-service/src/controllers/Alerts/Metric.ts b/api-service/src/controllers/Alerts/Metric.ts index 6f80f16f..a4189cc0 100644 --- a/api-service/src/controllers/Alerts/Metric.ts +++ b/api-service/src/controllers/Alerts/Metric.ts @@ -11,8 +11,7 @@ const telemetryObject = { type: "metric", ver: "1.0.0" }; const createMetricHandler = async (req: Request, res: Response, next: NextFunction) => { try { const { component } = req.body; - const transformComponent = _.toLower(component); - const metricsBody = await Metrics.create({ ...(req.body), component: transformComponent }); + const metricsBody = await Metrics.create({ ...(req.body), component: component }); updateTelemetryAuditEvent({ request: req, object: { id: metricsBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: metricsBody.dataValues.id } }); } catch (error: any) { diff --git a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts index 4844ac7e..0d062d16 100644 --- a/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts +++ b/api-service/src/controllers/ConnectorRegister/ConnectorRegisterController.ts @@ -8,6 +8,8 @@ import busboy from "busboy"; import { PassThrough } from "stream"; import { registerConnector } from "../../connections/commandServiceConnection"; import { generatePreSignedUrl } from "../GenerateSignedURL/helper"; +import { obsrvError } from "../../types/ObsrvError"; +import { config } from "../../configs/Config"; export const apiId = "api.connector.register"; export const code = "FAILED_TO_REGISTER_CONNECTOR"; @@ -22,7 +24,16 @@ const connectorRegisterController = async (req: Request, res: Response) => { relative_path: uploadStreamResponse[0] } logger.info({ apiId, resmsgid, message: `File uploaded to cloud provider successfully` }) - const registryResponse = await registerConnector(payload); + const downloadUrls = await generatePreSignedUrl("read", [payload.relative_path], "connector") + const urlPayload = { + download_url: _.get(downloadUrls, [0, "preSignedUrl"]), + file_name: _.get(downloadUrls, [0, "fileName"]) + } + if (!urlPayload.download_url) { + throw obsrvError("", "SIGNED_URL_NOT_FOUND", `Failed to generate signed url for path ${payload.relative_path}`, "BAD_REQUEST", 400) + } + const userToken = req.get('authorization') as string; + const registryResponse = await registerConnector(urlPayload, userToken); logger.info({ apiId, resmsgid, message: `Connector registered successfully` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: registryResponse?.data?.message } }) } catch (error: any) { @@ -66,11 +77,18 @@ const uploadStream = async (req: Request) => { const pass = new PassThrough(); file.pipe(pass); const fileBuffer = await streamToBuffer(pass); + + const uploadHeaders: any = { + "Content-Type": info.mimeType, + "Content-Length": fileBuffer.length, + }; + + if (config.cloud_config.cloud_storage_provider === "azure") { + uploadHeaders["x-ms-blob-type"] = config.cloud_config.azure_blob_type; + } + await axios.put(preSignedUrl[0]?.preSignedUrl, fileBuffer, { - headers: { - "Content-Type": info.mimeType, - "Content-Length": fileBuffer.length, - } + headers: uploadHeaders }); } catch (err) { @@ -118,4 +136,4 @@ const extractFileNameFromPath = (filePath: string): string[] => { return filePath.match(regex) || []; }; -export default connectorRegisterController; +export default connectorRegisterController; \ No newline at end of file diff --git a/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts new file mode 100644 index 00000000..7e4a3af8 --- /dev/null +++ b/api-service/src/controllers/DataAnalyzePII/DataAnalyzePIIController.ts @@ -0,0 +1,26 @@ +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import * as _ from "lodash"; +import logger from "../../logger"; +import { detectPII } from "../../connections/commandServiceConnection"; + +const code = "FAILED_TO_DETECT_PII"; +export const dataAnalyzePII = async (req: Request, res: Response) => { + const apiId = _.get(req, 'id') + try { + const userToken = req.get('authorization') as string; + const piiSuggestionsResponse = await detectPII(_.get(req, ['body', 'request']), userToken); + logger.info({apiId , message: `Detected PII successfully` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: _.get(piiSuggestionsResponse, ["data", "result"]) }); + } catch (error: any) { + const errMessage = _.get(error, "response.data.detail") + logger.error(error, apiId, code); + let errorMessage = error; + const statusCode = _.get(error, "status") + if (!statusCode || statusCode == 500) { + errorMessage = { code, message: errMessage || "Failed to detect pii" } + } + ResponseHandler.errorResponse(errorMessage, req, res); + } +} \ No newline at end of file diff --git a/api-service/src/controllers/DataExhaust/DataExhaustController.ts b/api-service/src/controllers/DataExhaust/DataExhaustController.ts index ceeb3358..84291aeb 100644 --- a/api-service/src/controllers/DataExhaust/DataExhaustController.ts +++ b/api-service/src/controllers/DataExhaust/DataExhaustController.ts @@ -11,7 +11,7 @@ import { cloudProvider } from "../../services/CloudServices"; export const dataExhaust = async (req: Request, res: Response) => { const { params } = req; - const { datasetId } = params; + const { dataset_id } = params; const { type }: any = req.query; const momentFormat = "YYYY-MM-DD"; @@ -23,17 +23,17 @@ export const dataExhaust = async (req: Request, res: Response) => { const getFromStorage = async (type: string, dateRange: any, datasetId: string) => { const resData = cloudProvider.getFiles( - config.cloud_config.container, config.cloud_config.container_prefix, type, dateRange, datasetId, + config.cloud_config.container, config.cloud_config.telemetry_data_path, type, dateRange, datasetId, ) return resData || {}; } - if (type && config.cloud_config.exclude_exhaust_types.includes(datasetId)) { + if (type && config.cloud_config.exclude_exhaust_types.includes(dataset_id)) { return ResponseHandler.errorResponse({ statusCode: 404, message: "Record not found", errCode: httpStatus["404_NAME"] }, req, res) } - const datasetRecord = await verifyDatasetExists(datasetId); + const datasetRecord = await verifyDatasetExists(dataset_id); if (datasetRecord === null) { - logger.error(`Dataset with ${datasetId} not found in live table`) + logger.error(`Dataset with ${dataset_id} not found in live table`) return ResponseHandler.errorResponse({ statusCode: 404, message: "Record not found", errCode: httpStatus["404_NAME"] }, req, res) } const dateRange = getDateRange(req); @@ -47,7 +47,7 @@ export const dataExhaust = async (req: Request, res: Response) => { return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid date range! make sure your range cannot be more than ${config.cloud_config.maxQueryDateRange} days`, errCode: "BAD_REQUEST" }, req, res) } - const resData: any = await getFromStorage(type, dateRange, datasetId); + const resData: any = await getFromStorage(type, dateRange, dataset_id); if (_.isEmpty(resData.files)) { logger.error("Date range provided does not have any backup files") return ResponseHandler.errorResponse({ statusCode: 404, message: "Date range provided does not have any backup files", errCode: "NOT_FOUND" }, req, res); diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 46f829e9..4c8b38a7 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -5,68 +5,56 @@ import { schemaValidation } from "../../services/ValidationService"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { send } from "../../connections/kafkaConnection"; import { datasetService } from "../../services/DatasetService"; -import logger from "../../logger"; import { config } from "../../configs/Config"; +import { obsrvError } from "../../types/ObsrvError"; -const errorObject = { - datasetNotFound: { - "message": "Dataset with id not found", - "statusCode": 404, - "errCode": "BAD_REQUEST", - "code": "DATASET_NOT_FOUND" - }, - topicNotFound: { - "message": "Entry topic is not defined", - "statusCode": 404, - "errCode": "BAD_REQUEST", - "code": "TOPIC_NOT_FOUND" +const apiId = "api.data.in"; + +const requestValidation = async (req: Request) => { + const datasetKey = req.params.dataset_id.trim(); + + const isValidSchema = schemaValidation(req.body, validationSchema) + if (!isValidSchema?.isValid) { + throw obsrvError("", "DATA_INGESTION_INVALID_INPUT", isValidSchema?.message, "BAD_REQUEST", 400) } + const dataset = await datasetService.getDatasetWithDatasetkey(datasetKey, ["id", "entry_topic", "api_version", "dataset_config", "dataset_id", "extraction_config"], true) + if (_.isEmpty(dataset)) { + throw obsrvError(datasetKey, "DATASET_NOT_FOUND", `Dataset with id/alias name '${datasetKey}' not found`, "NOT_FOUND", 404) + } + _.set(req, "body.request.dataset_id", dataset.dataset_id); + return dataset } -const apiId = "api.data.in"; const dataIn = async (req: Request, res: Response) => { - const requestBody = req.body; - const datasetId = req.params.datasetId.trim(); - - const isValidSchema = schemaValidation(requestBody, validationSchema) - if (!isValidSchema?.isValid) { - logger.error({ apiId, message: isValidSchema?.message, code: "DATA_INGESTION_INVALID_INPUT" }) - return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_INGESTION_INVALID_INPUT" }, req, res); - } - const dataset = await datasetService.getDataset(datasetId, ["id", "entry_topic", "api_version", "dataset_config"], true) - if (!dataset) { - logger.error({ apiId, message: `Dataset with id ${datasetId} not found in live table`, code: "DATASET_NOT_FOUND" }) - return ResponseHandler.errorResponse(errorObject.datasetNotFound, req, res); - } - const { entry_topic, dataset_config, api_version } = dataset - const entryTopic = api_version !== "v2" ? _.get(dataset_config, "entry_topic") : entry_topic - if (!entryTopic) { - logger.error({ apiId, message: "Entry topic not found", code: "TOPIC_NOT_FOUND" }) - return ResponseHandler.errorResponse(errorObject.topicNotFound, req, res); - } - await send(addMetadataToEvents(datasetId, requestBody), entryTopic) - ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Data ingested successfully" } }); + const dataset = await requestValidation(req) + const { entry_topic, dataset_config, extraction_config, api_version, dataset_id } = dataset + const entryTopic = api_version !== "v2" ? _.get(dataset_config, "entry_topic") : entry_topic + if (!entryTopic) { + throw obsrvError(dataset_id, "TOPIC_NOT_FOUND", `Entry topic not found`, "NOT_FOUND", 404) + } + await send(addMetadataToEvents(dataset_id, req.body, extraction_config), entryTopic) + ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Data ingested successfully" } }); } -const addMetadataToEvents = (datasetId: string, payload: any) => { +const addMetadataToEvents = (datasetId: string, payload: any, extraction_config: any) => { const validData = _.get(payload, "data"); const now = Date.now(); const mid = _.get(payload, "params.msgid"); - const source = { id: "api.data.in", version: config?.version, entry_source: "api" }; - const obsrvMeta = { syncts: now, flags: {}, timespans: {}, error: {}, source: source }; + const source = { connector: 'api', connectorInstance: 'api' }; + const obsrvMeta = { syncts: now, flags: {}, timespans: {}, error: {}, source }; if (Array.isArray(validData)) { - const payloadRef = validData.map((event: any) => { - const payload = { - event, - "obsrv_meta": obsrvMeta, - "dataset": datasetId, - "msgid": mid - } - return payload; - }) - return payloadRef; + const extraction_key: string = _.get(extraction_config, "extraction_key", 'events'); + const dedup_key: string = _.get(extraction_config, "dedup_config.dedup_key", 'id'); + const payload: any = { + "obsrv_meta": obsrvMeta, + "dataset": datasetId, + "msgid": mid + }; + payload[extraction_key] = validData; + payload[dedup_key] = mid + return payload; } else { return ({ diff --git a/api-service/src/controllers/DataMetrics/DataMetricsController.ts b/api-service/src/controllers/DataMetrics/DataMetricsController.ts new file mode 100644 index 00000000..98314878 --- /dev/null +++ b/api-service/src/controllers/DataMetrics/DataMetricsController.ts @@ -0,0 +1,38 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { executeNativeQuery } from "../../connections/druidConnection"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import vaidationSchema from "./DataMetricsValidationSchema.json" +import { schemaValidation } from "../../services/ValidationService"; +import logger from "../../logger"; +import { obsrvError } from "../../types/ObsrvError"; +import axios from "axios"; +import { config } from "../../configs/Config"; + +const getBaseUrl = (url: string) => { + if (_.startsWith(url, "/prom")) return config.query_api.prometheus.url + _.replace(url, "/prom", "") +} + +const dataMetrics = async (req: Request, res: Response) => { + const isValidSchema = schemaValidation(req.body, vaidationSchema); + if (!isValidSchema?.isValid) { + logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) + throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + } + const { query } = req.body || {}; + const endpoint = query.url; + if (_.startsWith(endpoint, "/prom")) { + query.url = getBaseUrl(endpoint) + const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; + const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) + const data = _.get(apiResponse, "data"); + return res.json(data); + } + else { + const query = _.get(req, ["body", "query", "body", "query"]); + const response = await executeNativeQuery(query); + ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); + } +} + +export default dataMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json b/api-service/src/controllers/DataMetrics/DataMetricsValidationSchema.json similarity index 100% rename from api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json rename to api-service/src/controllers/DataMetrics/DataMetricsValidationSchema.json diff --git a/api-service/src/controllers/DataOut/DataOutController.ts b/api-service/src/controllers/DataOut/DataOutController.ts index dd6f93b6..50848043 100644 --- a/api-service/src/controllers/DataOut/DataOutController.ts +++ b/api-service/src/controllers/DataOut/DataOutController.ts @@ -6,22 +6,37 @@ import validationSchema from "./DataOutValidationSchema.json"; import { validateQuery } from "./QueryValidator"; import * as _ from "lodash"; import { executeNativeQuery, executeSqlQuery } from "../../connections/druidConnection"; +import { datasetService } from "../../services/DatasetService"; +import { obsrvError } from "../../types/ObsrvError"; export const apiId = "api.data.out"; +export const query_data = {"data": {}}; + +const requestValidation = async (req: Request) => { + const datasourceKey = req.params?.dataset_id; + const isValidSchema = schemaValidation(req.body, validationSchema); + if (!isValidSchema?.isValid) { + throw obsrvError(datasourceKey, "DATA_OUT_INVALID_INPUT", isValidSchema?.message, "BAD_REQUEST", 400) + } + const datasource = await datasetService.getDatasourceWithKey(datasourceKey, ["datasource_ref", "dataset_id"], true) + if (_.isEmpty(datasource)) { + throw obsrvError(datasourceKey, "DATASET_NOT_FOUND", `Dataset with id/alias name '${datasourceKey}' not found`, "NOT_FOUND", 404) + } + _.set(req, "body.request.dataset_id", datasource.dataset_id); + return datasource +} + const dataOut = async (req: Request, res: Response) => { - const datasetId = req.params?.datasetId; const requestBody = req.body; const msgid = _.get(req, "body.params.msgid"); - const isValidSchema = schemaValidation(requestBody, validationSchema); - if (!isValidSchema?.isValid) { - logger.error({ apiId, datasetId, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) - return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); - } - const isValidQuery: any = await validateQuery(req.body, datasetId); + const dataset = await requestValidation(req) + const { dataset_id: datasetId, datasource_ref } = dataset + const isValidQuery: any = await validateQuery(req.body, datasetId, datasource_ref); const query = _.get(req, "body.query", "") if (isValidQuery === true && _.isObject(query)) { const result = await executeNativeQuery(query); + _.set(query_data, "data", result.data); logger.info({ apiId, msgid, requestBody, datasetId, message: "Native query executed successfully" }) return ResponseHandler.successResponse(req, res, { status: 200, data: result?.data @@ -30,6 +45,7 @@ const dataOut = async (req: Request, res: Response) => { if (isValidQuery === true && _.isString(query)) { const result = await executeSqlQuery({ query }) + _.set(query_data, "data", result.data); logger.info({ apiId, msgid, requestBody, datasetId, message: "SQL query executed successfully" }) return ResponseHandler.successResponse(req, res, { status: 200, data: result?.data diff --git a/api-service/src/controllers/DataOut/DataOutValidationSchema.json b/api-service/src/controllers/DataOut/DataOutValidationSchema.json index 13a37a49..18cd9a91 100644 --- a/api-service/src/controllers/DataOut/DataOutValidationSchema.json +++ b/api-service/src/controllers/DataOut/DataOutValidationSchema.json @@ -5,7 +5,6 @@ "ver", "ts", "params", - "context", "query" ], "properties": { diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index 2a0c64ed..e953ec5a 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -11,6 +11,7 @@ import { obsrvError } from "../../types/ObsrvError"; const parser = new Parser(); const momentFormat = "YYYY-MM-DD HH:MM:SS"; +let datasourceId: string; let dataset_id: string; let requestBody: any; let msgid: string; @@ -19,8 +20,9 @@ const errCode = { invalidDateRange: "DATA_OUT_INVALID_DATE_RANGE" } -export const validateQuery = async (requestPayload: any, datasetId: string) => { +export const validateQuery = async (requestPayload: any, datasetId: string, datasourceRef: string = "") => { requestBody = requestPayload; + datasourceId = datasourceRef; dataset_id = datasetId; msgid = _.get(requestPayload, "params.msgid"); const query = requestPayload?.query; @@ -117,7 +119,7 @@ const setQueryLimits = (queryPayload: any) => { const getDataSourceFromPayload = (queryPayload: any) => { if (_.isString(queryPayload.query)) { - queryPayload?.query.replace(/from\s+["'`]?[\w-]+["'`]?(\s+where\s+)/i, ` from "${dataset_id}"$1`); + queryPayload.query = queryPayload.query.replace(/from\s+["'`]?[\w-]+["'`]?(\s+where\s+)/i, ` from "${dataset_id}"$1`); return dataset_id } if (_.isObject(queryPayload.query)) { @@ -198,16 +200,18 @@ const checkSupervisorAvailability = async (datasourceRef: string) => { const { data } = await druidHttpService.get("/druid/coordinator/v1/loadstatus"); const datasourceAvailability = _.get(data, datasourceRef) if (_.isUndefined(datasourceAvailability)) { + logger.error({ apiId, requestBody, msgid, dataset_id, message: `Segments not published to the metadata store yet, please check the coordinator load status`, code: errCode.notFound }) throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not available for querying", "NOT_FOUND", 404) } if (datasourceAvailability !== 100) { + logger.error({ apiId, requestBody, msgid, dataset_id, message: `Segments not fully published to the metadata store yet, please check the coordinator load status`, code: errCode.notFound }) throw obsrvError("", "DATASOURCE_NOT_FULLY_AVAILABLE", "Datasource not fully available for querying", "RANGE_NOT_SATISFIABLE", 416) } } const setDatasourceRef = async (datasetId: string, payload: any): Promise => { const granularity = _.get(payload, "context.aggregationLevel") - const datasourceRef = await getDataSourceRef(datasetId, granularity); + const datasourceRef = _.isEmpty(datasourceId) ? await getDataSourceRef(datasetId, granularity) : datasourceId; if (!datasourceRef) { throw obsrvError("", "DATASOURCE_NOT_FOUND", "Datasource not found to query", "NOT_FOUND", 404) } @@ -216,7 +220,7 @@ const setDatasourceRef = async (datasetId: string, payload: any): Promise = if (!_.includes(existingDatasources.data, datasourceRef)) { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Dataset ${datasetId} with table ${granularity} is not available for querying`, code: errCode.notFound }) - throw obsrvError("", errCode.notFound, `Dataset ${datasetId} with table ${granularity} is not available for querying`, "NOT_FOUND", 404); + throw obsrvError("", errCode.notFound, `Dataset ${datasetId} with table ${granularity} is not available for querying`, "NOT_FOUND", 404); } if (_.isString(payload?.query)) { payload.query = payload.query.replace(datasetId, datasourceRef) diff --git a/api-service/src/controllers/DatasetAlias/DatasetAlias.ts b/api-service/src/controllers/DatasetAlias/DatasetAlias.ts new file mode 100644 index 00000000..9393ed25 --- /dev/null +++ b/api-service/src/controllers/DatasetAlias/DatasetAlias.ts @@ -0,0 +1,57 @@ +import { Request, Response } from "express"; +import { schemaValidation } from "../../services/ValidationService"; +import DatasetAliasSchema from "./DatasetAliasValidationSchema.json" +import { obsrvError } from "../../types/ObsrvError"; +import _ from "lodash"; +import { datasetService } from "../../services/DatasetService"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; + + +const validateRequest = async (req: Request) => { + + const isRequestValid: Record = schemaValidation(req.body, DatasetAliasSchema) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATASET_ALIAS_INPUT_INVALID", isRequestValid.message, "BAD_REQUEST", 400) + } + + const { dataset_id, alias_name: alias, table_name } = _.get(req, ["body", "request"]) + const nameRegex = /^[a-z0-9.-]+$/; + if (!nameRegex.test(alias)) { + throw obsrvError("", "ALIAS_NAME_INVALID", `Alias name must only contain lowercase letters, numbers, periods, and dashes.`, "BAD_REQUEST", 400); + } + + const dataset = await datasetService.getDataset(dataset_id, ["id", "name"], true); + if (_.isEmpty(dataset)) { + throw obsrvError(dataset_id, "DATASET_NOT_EXISTS", `Dataset does not exists with id:${dataset_id}`, "NOT_FOUND", 404); + } + if (table_name) { + const tableRecord = await datasetService.getDatasource(table_name, ["id", "datasource"]); + if (_.isEmpty(tableRecord)) { + throw obsrvError("", "TABLE_NOT_EXISTS", `Table does not exists with name:${table_name}`, "NOT_FOUND", 404); + } + if (_.isEmpty(_.get(tableRecord, "datasource"))) { + throw obsrvError(dataset_id, "ALIAS_NOT_EXISTS", `Alias name for table ${table_name} does not exists`, "BAD_REQUEST", 400); + } + const datasourceRecords = await datasetService.findDatasources({ datasource: alias }, ["datasource_ref"]) + if (!_.isEmpty(datasourceRecords)) { + throw obsrvError(dataset_id, "ALIAS_NOT_UNIQUE", `Alias name '${alias}' already exists`, "BAD_REQUEST", 400); + } + } + +} + +const datasetAlias = async (req: Request, res: Response) => { + await validateRequest(req) + const { dataset_id, alias_name, table_name } = _.get(req, ["body", "request"]) + const userID = (req as any)?.userID; + if (table_name) { + await datasetService.updateDatasource({ datasource: alias_name, updated_by: userID }, { id: table_name }); + } + else { + await datasetService.updateDatasource({ datasource: alias_name, updated_by: userID }, { dataset_id, is_primary: true, type: "druid" }); + } + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: `Dataset alias name '${alias_name}' updated successfully`, dataset_id } }); +} + +export default datasetAlias; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetAlias/DatasetAliasValidationSchema.json b/api-service/src/controllers/DatasetAlias/DatasetAliasValidationSchema.json new file mode 100644 index 00000000..0dfa16ac --- /dev/null +++ b/api-service/src/controllers/DatasetAlias/DatasetAliasValidationSchema.json @@ -0,0 +1,59 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "api.datasets.alias" + ] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": [ + "msgid" + ], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "alias_name": { + "type": "string", + "minLength": 1 + }, + "table_name": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "dataset_id", + "alias_name" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "ver", + "ts", + "params", + "request" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 9015e9c7..7c53f538 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -4,7 +4,7 @@ import { ResponseHandler } from "../../helpers/ResponseHandler"; import * as _ from "lodash"; import { schemaValidation } from "../../services/ValidationService"; import validationSchema from "./RequestValidationSchema.json"; -import { datasetService, getLiveDatasetConfigs } from "../../services/DatasetService"; +import { datasetService, getLiveDatasetConfigs, validateStorageSupport } from "../../services/DatasetService"; import { updateRecords } from "./DatasetCopyHelper"; import { obsrvError } from "../../types/ObsrvError"; @@ -40,6 +40,7 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); + validateStorageSupport(dataset); const userID = (req as any)?.userID; _.set(dataset, "created_by", userID); _.set(dataset, "updated_by", userID); diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index bb96dac2..b591d120 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -1,7 +1,7 @@ import _ from "lodash"; import { Request, Response } from "express"; import httpStatus from "http-status"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import DatasetCreate from "./DatasetCreateValidationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; import { ResponseHandler } from "../../helpers/ResponseHandler"; @@ -22,12 +22,17 @@ const validateRequest = async (req: Request) => { if (isDataSetExists) { throw obsrvError(datasetId, "DATASET_EXISTS", `Dataset Already exists with id:${datasetId}`, "CONFLICT", 409) } + const isDatasourceExists = await datasetService.checkDatasourceExist(datasetId); + if (isDatasourceExists) { + throw obsrvError(datasetId, "DATASOURCE_EXISTS", `Datasource Already exists with id:${datasetId}`, "CONFLICT", 409) + } const duplicateDenormKeys = datasetService.getDuplicateDenormKey(_.get(req, ["body", "request", "denorm_config"])) if (!_.isEmpty(duplicateDenormKeys)) { throw obsrvError(datasetId, "DATASET_DUPLICATE_DENORM_KEY", "Duplicate denorm output fields found.", "BAD_REQUEST", 400, undefined, {duplicateKeys: duplicateDenormKeys}) } + validateStorageSupport(_.get(req, ["body", "request"])) } const datasetCreate = async (req: Request, res: Response) => { diff --git a/api-service/src/controllers/DatasetExport/DatasetExport.ts b/api-service/src/controllers/DatasetExport/DatasetExport.ts index 3e118639..04799aa1 100644 --- a/api-service/src/controllers/DatasetExport/DatasetExport.ts +++ b/api-service/src/controllers/DatasetExport/DatasetExport.ts @@ -25,6 +25,8 @@ const validateDataset = async (req: Request) => { const migratedConfigs = await datasetService.migrateDatasetV1(dataset_id, datasetRecord) datasetRecord = { ...datasetRecord, ...migratedConfigs } } + + datasetRecord = _.omit(datasetRecord, "alias") return datasetRecord; } diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index e390d8bd..de326bb9 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -2,7 +2,7 @@ import { Request, Response } from "express"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; import _ from "lodash"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import { datasetImportValidation, migrateExportedDatasetV1 } from "./DatasetImportHelper"; import { obsrvError } from "../../types/ObsrvError"; @@ -15,18 +15,23 @@ const datasetImport = async (req: Request, res: Response) => { const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } + const userID = (req as any)?.userID; + _.set(datasetPayload, "created_by", userID); + _.set(datasetPayload, "updated_by", userID); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) - const dataset = await importDataset(updatedDataset, overwrite); + validateStorageSupport(updatedDataset); + const dataset = await importDataset(updatedDataset, overwrite, userID); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); } -const importDataset = async (dataset: Record, overwrite: string | any) => { +const importDataset = async (dataset: Record, overwrite: string | any, userID : string) => { const dataset_id = _.get(dataset,"dataset_id") const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) if (response?.name === "SequelizeUniqueConstraintError") { if (overwrite === "true") { + _.set(dataset, "updated_by", userID); const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(()=>{ throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts index 73907183..9c7a75a6 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -36,6 +36,12 @@ export const datasetImportValidation = async (payload: Record): Pro const datasetConfig = payload.request; + const datasetId = _.get(datasetConfig, "dataset_id") || ""; + const isDatasourceExists = await datasetService.checkDatasourceExist(datasetId); + if (isDatasourceExists) { + throw obsrvError(datasetId, "DATASOURCE_EXISTS", `Datasource Already exists with id:${datasetId}`, "CONFLICT", 409) + } + const connectors = _.get(datasetConfig, "connectors_config", []); const transformations = _.get(datasetConfig, "transformations_config", []); const denormConfig = _.get(datasetConfig, "denorm_config", { denorm_fields: [] }); diff --git a/api-service/src/controllers/DatasetList/DatasetList.ts b/api-service/src/controllers/DatasetList/DatasetList.ts index 82acbf62..56bd9386 100644 --- a/api-service/src/controllers/DatasetList/DatasetList.ts +++ b/api-service/src/controllers/DatasetList/DatasetList.ts @@ -5,17 +5,19 @@ import logger from "../../logger"; import { schemaValidation } from "../../services/ValidationService"; import DatasetCreate from "./DatasetListValidationSchema.json"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { datasetService } from "../../services/DatasetService"; +import { attachDraftConnectors, attachLiveConnectors, datasetService } from "../../services/DatasetService"; import { obsrvError } from "../../types/ObsrvError"; +import { config } from "../../configs/Config"; export const apiId = "api.datasets.list" export const errorCode = "DATASET_LIST_FAILURE" const liveDatasetStatus = ["Live", "Retired", "Purged"] const draftDatasetStatus = ["Draft", "ReadyToPublish"] const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config", "created_date", "updated_date"] +const MAX_STATUS_ARRAY_SIZE = config.dataset_filter_config.status_filter_limit || 10; const datasetList = async (req: Request, res: Response) => { - + const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) if (!isRequestValid.isValid) { throw obsrvError("", "DATASET_LIST_INPUT_INVALID", isRequestValid.message, "BAD_REQUEST", 400) @@ -24,20 +26,28 @@ const datasetList = async (req: Request, res: Response) => { const datasetBody = req.body.request; const datasetList = await listDatasets(datasetBody) const responseData = { data: datasetList, count: _.size(datasetList) } - logger.info({req: req.body, resmsgid: _.get(res, "resmsgid"), message: `Datasets are listed successfully with a dataset count (${_.size(datasetList)})` }) + logger.info({ req: req.body, resmsgid: _.get(res, "resmsgid"), message: `Datasets are listed successfully with a dataset count (${_.size(datasetList)})` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); - + } const listDatasets = async (request: Record): Promise> => { const { filters = {} } = request || {}; const datasetStatus = _.get(filters, "status"); + const connectorFilter = _.get(filters, "connectors"); const status = _.isArray(datasetStatus) ? datasetStatus : _.compact([datasetStatus]) - const draftFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? draftDatasetStatus : _.intersection(status, draftDatasetStatus)); - const liveFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? liveDatasetStatus : _.intersection(status, liveDatasetStatus)); - const liveDatasetList = await datasetService.findDatasets(liveFilters, defaultFields, [["updated_date", "DESC"]]); - const draftDatasetList = await datasetService.findDraftDatasets(draftFilters, defaultFields, [["updated_date", "DESC"]]); + if (status.length > MAX_STATUS_ARRAY_SIZE) { + throw obsrvError("", "DATASET_LIST_INPUT_INVALID", "Status filter array length exceeds the allowed limit", "BAD_REQUEST", 400); + } + const draftFilters = _.omit(_.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? draftDatasetStatus : _.intersection(status, draftDatasetStatus)), "connectors"); + const liveFilters = _.omit(_.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? liveDatasetStatus : _.intersection(status, liveDatasetStatus)), "connectors"); + let liveDatasetList = await datasetService.getLiveDatasets(liveFilters, defaultFields) + let draftDatasetList = await datasetService.findDraftDatasets(draftFilters, [...defaultFields, "data_schema", "validation_config", "dedup_config", "denorm_config", "connectors_config", "version_key"], [["updated_date", "DESC"]]); + if(connectorFilter && !_.isEmpty(connectorFilter)) { + liveDatasetList = await attachLiveConnectors(liveDatasetList, connectorFilter); + draftDatasetList = await attachDraftConnectors(draftDatasetList, connectorFilter); + } return _.compact(_.concat(liveDatasetList, draftDatasetList)); } diff --git a/api-service/src/controllers/DatasetList/DatasetListValidationSchema.json b/api-service/src/controllers/DatasetList/DatasetListValidationSchema.json index 3435d631..dbb0ac32 100644 --- a/api-service/src/controllers/DatasetList/DatasetListValidationSchema.json +++ b/api-service/src/controllers/DatasetList/DatasetListValidationSchema.json @@ -45,6 +45,20 @@ "type": { "type": "string", "enum": ["event", "transaction", "master"] + }, + "connectors": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string", + "default": "all" + } + ] } }, "additionalProperties": false diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json b/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json new file mode 100644 index 00000000..70f14029 --- /dev/null +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetrics.json @@ -0,0 +1,61 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "api.dataset.metrics" + ] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": [ + "msgid" + ], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "dataset_id": { + "type": "string" + }, + "category": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "data_freshness", + "data_observability", + "data_volume", + "data_lineage", + "connectors", + "data_quality" + ] + }, + "minItems": 1 + }, + "query_time_period":{ + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "category", + "dataset_id" + ] + } + }, + "required": ["id", "ver", "ts", "params", "request"] +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts index 903d393c..f84d0b43 100644 --- a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -1,38 +1,81 @@ import { Request, Response } from "express"; -import _ from "lodash"; -import { executeNativeQuery } from "../../connections/druidConnection"; +import * as _ from "lodash" import { ResponseHandler } from "../../helpers/ResponseHandler"; -import vaidationSchema from "./DatasetMetricsValidationSchema.json" -import { schemaValidation } from "../../services/ValidationService"; +import dayjs from 'dayjs'; import logger from "../../logger"; -import { obsrvError } from "../../types/ObsrvError"; -import axios from "axios"; +import { schemaValidation } from "../../services/ValidationService"; +import validationSchema from "./DatasetMetrics.json"; import { config } from "../../configs/Config"; +import { datasetService } from "../../services/DatasetService"; +import { getConnectors, getDataFreshness, getDataLineage, getDataObservability, getDataQuality, getDataVolume } from "../../services/DatasetMetricsService"; -const getBaseUrl = (url: string) => { - if (_.startsWith(url, "/prom")) return config.query_api.prometheus.url + _.replace(url, "/prom", "") -} - +const apiId = "api.dataset.metrics"; const datasetMetrics = async (req: Request, res: Response) => { - const isValidSchema = schemaValidation(req.body, vaidationSchema); + const msgid = _.get(req, "body.params.msgid"); + const requestBody = req.body; + const dataset_id = _.get(req, "body.request.dataset_id"); + const timePeriod = _.get(req, "body.request.query_time_period") || config?.data_observability?.default_query_time_period; + + const { category }: any = req.body.request; + const defaultThreshold = (typeof config?.data_observability?.default_freshness_threshold === 'number' ? config?.data_observability?.default_freshness_threshold : 5) * 60 * 1000; // 5 minutes in milliseconds + const dateFormat = 'YYYY-MM-DDTHH:mm:ss'; + const endDate = dayjs().add(1, 'day').format(dateFormat); + const startDate = dayjs(endDate).subtract(timePeriod, 'day').format(dateFormat); + const intervals = `${startDate}/${endDate}`; + const isValidSchema = schemaValidation(requestBody, validationSchema); + const results = []; + if (!isValidSchema?.isValid) { - logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) - throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + logger.error({ apiId, datasetId: dataset_id, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) + return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); } - const { query } = req.body || {}; - const endpoint = query.url; - if (_.startsWith(endpoint, "/prom")) { - query.url = getBaseUrl(endpoint) - const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; - const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) - const data = _.get(apiResponse, "data"); - return res.json(data); + + const dataset = await datasetService.getDataset(dataset_id, ["id"], true) + if (!dataset) { + logger.error({ apiId, message: `Dataset with id ${dataset_id} not found in live table`, code: "DATASET_NOT_FOUND" }) + return ResponseHandler.errorResponse({ message: `Dataset with id ${dataset_id} not found in live table`, code: "DATASET_NOT_FOUND", statusCode: 404, errCode: "NOT_FOUND" }, req, res); } - else { - const query = _.get(req, ["body", "query", "body", "query"]); - const response = await executeNativeQuery(query); - ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); + + try { + if (!category || category.includes("data_freshness")) { + const dataFreshnessResult = await getDataFreshness(dataset_id, intervals, defaultThreshold); + results.push(dataFreshnessResult); + } + + if (!category || category.includes("data_observability")) { + const dataObservabilityResult = await getDataObservability(dataset_id, intervals); + results.push(dataObservabilityResult); + } + + if (!category || category.includes("data_volume")) { + const dataVolumeResult = await getDataVolume(dataset_id, timePeriod, dateFormat); + results.push(dataVolumeResult); + } + + if (!category || category.includes("data_lineage")) { + const dataLineageResult = await getDataLineage(dataset_id, intervals); + results.push(dataLineageResult); + } + + if (!category || category.includes("connectors")) { + const connectorsResult = await getConnectors(dataset_id, intervals); + results.push(connectorsResult); + } + + if (!category || category.includes("data_quality")) { + const connectorsResult = await getDataQuality(dataset_id, intervals); + results.push(connectorsResult); + } + + logger.info({ apiId, msgid, requestBody, datasetId: dataset_id, message: "Metrics fetched successfully" }) + return ResponseHandler.successResponse(req, res, { status: 200, data: results }); + } + catch (error: any) { + logger.error({ apiId, msgid, requestBody: req?.body, datasetId: dataset_id, message: "Error while fetching metrics", code: "FAILED_TO_FETCH_METRICS", error }); + return ResponseHandler.errorResponse({ message: "Error while fetching metrics", statusCode: 500, errCode: "FAILED", code: "FAILED_TO_FETCH_METRICS" }, req, res); + } + } export default datasetMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetMetrics/queries.ts b/api-service/src/controllers/DatasetMetrics/queries.ts new file mode 100644 index 00000000..ae02e1ac --- /dev/null +++ b/api-service/src/controllers/DatasetMetrics/queries.ts @@ -0,0 +1,402 @@ +import dayjs from "dayjs"; + +export const processingTimeQuery = (intervals: string, dataset_id: string) => ({ + query: { + queryType: "groupBy", + dataSource: "system-events", + intervals: intervals, + granularity: { + type: "all", + timeZone: "UTC" + }, + filter: { + type: "and", + fields: [ + { type: "selector", dimension: "ctx_module", value: "processing" }, + { type: "selector", dimension: "ctx_dataset", value: dataset_id }, + { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, + { type: "selector", dimension: "error_code", value: null } + ] + }, + aggregations: [ + { type: "longSum", name: "processing_time", fieldName: "total_processing_time" }, + { type: "longSum", name: "count", fieldName: "count" } + ], + postAggregations: [ + { + type: "expression", + name: "average_processing_time", + expression: "case_searched((count > 0),(processing_time/count),0)" + } + ] + } +}); + +export const totalEventsQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "longSum", + name: "total_events_count", + fieldName: "count" + } + ] +}); + +export const totalFailedEventsQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "total_failed_events", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_pid", + matchValueType: "STRING", + matchValue: "validator" + }, + { + type: "equals", + column: "error_pdata_status", + matchValueType: "STRING", + matchValue: "failed" + } + ] + }, + name: "total_failed_events" + } + ] +}); + +export const generateTimeseriesQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: "system-events", + intervals: intervals, + granularity: { + type: "all", + timeZone: "UTC" + }, + filter: { + type: "and", + fields: [ + { type: "selector", dimension: "ctx_module", value: "processing" }, + { type: "selector", dimension: "ctx_dataset", value: dataset_id }, + { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, + { type: "selector", dimension: "error_code", value: null } + ] + }, + aggregations: [ + { type: "longSum", name: "count", fieldName: "count" } + ] +}); + +export const generateTimeseriesQueryEventsPerHour = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: "system-events", + intervals: intervals, + granularity: { + type: "all", + timeZone: "UTC" + }, + filter: { + type: "and", + fields: [ + { type: "selector", dimension: "ctx_module", value: "processing" }, + { type: "selector", dimension: "ctx_dataset", value: dataset_id }, + { type: "selector", dimension: "ctx_pdata_pid", value: "router" }, + { type: "selector", dimension: "error_code", value: null } + ] + }, + aggregations: [ + { type: "longSum", name: "count", fieldName: "count" } + ] +}); + +export const dataLineageSuccessQuery = (intervals: string, dataset_id: string, column: string, value: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: column, + matchValueType: "STRING", + matchValue: value + }, + { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + } + ] + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "longSum", + name: "count", + fieldName: "count" + } + ] +}); + +export const generateTransformationFailedQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "count", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_id", + matchValueType: "STRING", + matchValue: "TransformerJob" + }, + { + type: "equals", + column: "error_pdata_status", + matchValueType: "STRING", + matchValue: "failed" + } + ] + }, + name: "count" + } + ] +}); + +export const generateDedupFailedQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "count", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_pid", + matchValueType: "STRING", + matchValue: "dedup" + }, + { + type: "equals", + column: "error_type", + matchValueType: "STRING", + matchValue: "DedupFailed" + } + ] + }, + name: "count" + } + ] +}); + +export const generateDenormFailedQuery = (intervals: string, dataset_id: string) => ({ + queryType: "timeseries", + dataSource: { + type: "table", + name: "system-events" + }, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "filtered", + aggregator: { + type: "longSum", + name: "count", + fieldName: "count" + }, + filter: { + type: "and", + fields: [ + { + type: "equals", + column: "ctx_pdata_pid", + matchValueType: "STRING", + matchValue: "denorm" + }, + { + type: "equals", + column: "error_type", + matchValueType: "STRING", + matchValue: "DenormDataNotFound" + } + ] + }, + name: "count" + } + ] +}); + +export const generateConnectorQuery = (intervals: string, dataset_id: string) => ({ + queryType: "topN", + dataSource: { + type: "table", + name: "system-events" + }, + dimension: { + type: "default", + dimension: "ctx_source_connector", + outputName: "name", + outputType: "STRING" + }, + metric: { + type: "dimension", + ordering: { + type: "lexicographic" + } + }, + threshold: 1001, + intervals: { + type: "intervals", + intervals: [intervals] + }, + filter: { + type: "equals", + column: "ctx_dataset", + matchValueType: "STRING", + matchValue: dataset_id + }, + granularity: { + type: "all", + timeZone: "UTC" + }, + aggregations: [ + { + type: "longSum", + name: "count", + fieldName: "count" + } + ] +}); + +export const generateTotalQueryCallsQuery = (time_period: string) => ({ + end: dayjs().unix(), + query: `sum(sum_over_time(node_total_api_calls{entity="data-out"}[${time_period}]))`, + step: `${time_period}`, + start: dayjs().subtract(1, 'day').unix() +}); + +export const generateDatasetQueryCallsQuery = (dataset: string, time_period: string) => ({ + end: dayjs().unix(), + step: `${time_period}`, + query: `sum(sum_over_time(node_total_api_calls{dataset_id="${dataset}",entity="data-out"}[${time_period}]))`, + start: dayjs().subtract(1, 'day').unix(), +}); \ No newline at end of file diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index d428b02a..d1c91209 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -3,9 +3,13 @@ import httpStatus from "http-status"; import _ from "lodash"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { DatasetDraft } from "../../models/DatasetDraft"; -import { datasetService, getV1Connectors } from "../../services/DatasetService"; +import { datasetService, getUpdatedV2ConnectorsPayload } from "../../services/DatasetService"; import { obsrvError } from "../../types/ObsrvError"; import { cipherService } from "../../services/CipherService"; +import { Dataset } from "../../models/Dataset"; +import { Datasource } from "../../models/Datasource"; +import { userService } from "../../services/UserService"; +import { config } from "../../configs/Config"; export const apiId = "api.datasets.read"; export const errorCode = "DATASET_READ_FAILURE" @@ -16,9 +20,9 @@ export const defaultFields = ["dataset_id", "name", "type", "status", "tags", "v const validateRequest = (req: Request) => { const { dataset_id } = req.params; - const { fields } = req.query; + const { fields, mode } = req.query; const fieldValues = fields ? _.split(fields as string, ",") : []; - const invalidFields = _.difference(fieldValues, Object.keys(DatasetDraft.getAttributes())); + const invalidFields = mode === "edit" ? _.difference(fieldValues, Object.keys(DatasetDraft.getAttributes())) : _.difference(fieldValues, Object.keys(Dataset.getAttributes())); if (!_.isEmpty(invalidFields)) { throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS", `The specified fields [${invalidFields}] in the dataset cannot be found.`, "BAD_REQUEST", 400); } @@ -36,8 +40,8 @@ const datasetRead = async (req: Request, res: Response) => { if (!dataset) { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } - if (dataset.connectors_config) { - dataset.connectors_config = processConnectorsConfig(dataset.connectors_config); + if (!_.isEmpty(dataset.connectors_config)) { + dataset.connectors_config = processConnectorsConfig(dataset.connectors_config); } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } @@ -54,6 +58,14 @@ const readDraftDataset = async (datasetId: string, attributes: string[], userID: const liveDataset = await datasetService.getDataset(datasetId, undefined, true); if (liveDataset) { + if (_.lowerCase(config.is_RBAC_enabled) !== "false") { + const user = await userService.getUser({ id: userID }, ["roles", "user_name"]); + const userRoles = _.get(user, "roles"); + const hasValidRole = _.some(userRoles, (role: string) => ['dataset_manager', 'admin', 'dataset_creator'].includes(role)); + if (!hasValidRole) { + throw obsrvError(datasetId, "UNAUTHORIZED_ACCESS", "Access denied. User does not have permission to perform this action", "FORBIDDEN", 403); + } + } const dataset = await datasetService.createDraftDatasetFromLive(liveDataset, userID) return _.pick(dataset, attributes); } @@ -62,15 +74,17 @@ const readDraftDataset = async (datasetId: string, attributes: string[], userID: } const readDataset = async (datasetId: string, attributes: string[]): Promise => { - const dataset = await datasetService.getDataset(datasetId, attributes, true); + const attrs = _.union(attributes, ["api_version"]) + const dataset = await datasetService.getDataset(datasetId, attrs, true); if (!dataset) { return; } const api_version = _.get(dataset, "api_version") const datasetConfigs: any = {} const transformations_config = await datasetService.getTransformations(datasetId, ["field_key", "transformation_function", "mode", "metadata"]) + const datasourceConfig = await Datasource.findOne({ where: { dataset_id: datasetId, is_primary: true, type: "druid" }, attributes: ["datasource"], raw: true }) + datasetConfigs["alias"] = _.get(datasourceConfig, "datasource") if (api_version !== "v2") { - datasetConfigs["connectors_config"] = await getV1Connectors(datasetId) datasetConfigs["transformations_config"] = _.map(transformations_config, (config) => { const section: any = _.get(config, "metadata.section") || _.get(config, "transformation_function.category"); return { @@ -83,11 +97,12 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise { if (!isRequestValid.isValid) { throw obsrvError("", "DATASET_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) } - const datasetId = _.get(req, ["params", "datasetId"]) + const datasetId = _.get(req, ["params", "dataset_id"]) const isDataSetExists = await datasetService.checkDatasetExists(datasetId); if (!isDataSetExists) { throw obsrvError(datasetId, "DATASET_NOT_FOUND", `Dataset not exists with id:${datasetId}`, httpStatus[httpStatus.NOT_FOUND], 404) @@ -28,13 +28,13 @@ const validateRequest = async (req: Request) => { const datasetReset = async (req: Request, res: Response) => { const category = _.get(req, ["body", "request", "category"]); - const datasetId = _.get(req, ["params"," datasetId"]); - + const datasetId = _.get(req, ["params"," dataset_id"]); + const userToken = req.get('authorization') as string; await validateRequest(req); if (category == "processing") { const pipeLineStatus = await getFlinkHealthStatus() if (pipeLineStatus == HealthStatus.UnHealthy) { - await restartPipeline({ "dataset": { "dataset_id": datasetId } }) + await restartPipeline({ "dataset": { "dataset_id": datasetId } }, userToken) } } else if (category == "query") { const datasources = await datasetService.findDatasources({"dataset_id": datasetId}) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 0197d39e..e09fa5b8 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -1,7 +1,7 @@ import { Request, Response } from "express"; import _ from "lodash"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import { schemaValidation } from "../../services/ValidationService"; import StatusTransitionSchema from "./RequestValidationSchema.json"; import ReadyToPublishSchema from "./ReadyToPublishSchema.json" @@ -50,6 +50,7 @@ const validateDataset = (dataset: any, datasetId: any, action: string) => { const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); + const userToken = req.get('authorization') as string; validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version", "name"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) @@ -64,10 +65,10 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await readyForPublish(dataset, userID); break; case "Live": - await publishDataset(dataset, userID); + await publishDataset(dataset, userID, userToken); break; case "Retire": - await retireDataset(dataset, userID); + await retireDataset(dataset, userID, userToken); break; default: throw obsrvError(dataset.id, "UNKNOWN_STATUS_TRANSITION", "Unknown status transition requested", "BAD_REQUEST", 400) @@ -88,6 +89,7 @@ const deleteDataset = async (dataset: Record) => { const readyForPublish = async (dataset: Record, updated_by: any) => { const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + validateStorageSupport(draftDataset); let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); @@ -133,14 +135,15 @@ const readyForPublish = async (dataset: Record, updated_by: any) => * * @param dataset */ -const publishDataset = async (dataset: Record, userID: any) => { +const publishDataset = async (dataset: Record, userID: any, userToken: string) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record + validateStorageSupport(draftDataset); _.set(draftDataset, ["created_by"], userID); _.set(draftDataset, ["updated_by"], userID); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) - await datasetService.publishDataset(draftDataset) + await datasetService.publishDataset(draftDataset, userToken); } const validateAndUpdateDenormConfig = async (draftDataset: Record) => { @@ -225,11 +228,11 @@ const updateMasterDataConfig = async (draftDataset: Record) => { } } -const retireDataset = async (dataset: Record, updated_by: any) => { +const retireDataset = async (dataset: Record, updated_by: any, userToken: string) => { await canRetireIfMasterDataset(dataset); await datasetService.retireDataset(dataset, updated_by); - await restartPipeline(dataset); + await restartPipeline(dataset, userToken); } @@ -252,8 +255,8 @@ const canRetireIfMasterDataset = async (dataset: Record) => { } } -export const restartPipeline = async (dataset: Record) => { - return executeCommand(dataset.id, "RESTART_PIPELINE") +export const restartPipeline = async (dataset: Record, userToken: string) => { + return executeCommand(dataset.id, "RESTART_PIPELINE", userToken) } export default datasetStatusTransition; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index d76acbb8..33d5a14c 100644 --- a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -317,7 +317,6 @@ } }, "required": [ - "data_key", "timestamp_key" ], "additionalProperties": false @@ -338,11 +337,44 @@ }, "required": [ "redis_db_host", - "redis_db_port" + "redis_db_port", + "redis_db" ], "additionalProperties": false } }, + "if": { + "properties": { + "indexing_config": { + "type": "object", + "anyOf": [ + { + "properties": { + "lakehouse_enabled": { + "const": true + } + } + }, + { + "properties": { + "cache_enabled": { + "const": true + } + } + } + ] + } + } + }, + "then": { + "properties": { + "keys_config": { + "required": [ + "data_key" + ] + } + } + }, "required": [ "indexing_config", "keys_config", diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index e9fea051..c1ef0324 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -5,7 +5,7 @@ import Model from "sequelize/types/model"; import { DatasetStatus } from "../../types/DatasetModels"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { cipherService } from "../../services/CipherService"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, validateStorageSupport } from "../../services/DatasetService"; import { schemaValidation } from "../../services/ValidationService"; import DatasetUpdate from "./DatasetUpdateValidationSchema.json"; import { obsrvError } from "../../types/ObsrvError"; @@ -30,6 +30,7 @@ const validateRequest = async (req: Request) => { throw obsrvError(datasetId, "DATASET_UPDATE_NO_FIELDS", "Provide atleast one field in addition to the dataset_id to update the dataset", "BAD_REQUEST", 400) } + validateStorageSupport(_.get(req, ["body", "request"])) } const validateDataset = (dataset: Record | null, req: Request) => { diff --git a/api-service/src/controllers/DatasourceList/DatasourceList.ts b/api-service/src/controllers/DatasourceList/DatasourceList.ts new file mode 100644 index 00000000..c637d3a0 --- /dev/null +++ b/api-service/src/controllers/DatasourceList/DatasourceList.ts @@ -0,0 +1,43 @@ +import { Request, Response } from "express"; +import * as _ from "lodash"; +import { obsrvError } from "../../types/ObsrvError"; +import { schemaValidation } from "../../services/ValidationService"; +import DatasourceSchema from "./RequestValidationSchema.json"; +import httpStatus from "http-status"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import logger from "../../logger"; +import { datasetService } from "../../services/DatasetService"; + +export const apiId = "api.datasources.list" +export const errorCode = "DATASOURCES_LIST_FAILURE" +const liveDatasourceStatus = ["Live", "Retired"] +const draftDatasourceStatus = ["Draft"] + +const getDatasourceList = async (req: Request, res: Response) => { + + const isRequestValid: Record = schemaValidation(req.body, DatasourceSchema) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATASOURCE_LIST_INPUT_INVALID", isRequestValid.message, "BAD_REQUEST", 400) + } + const datasourceBody = req.body.request; + const datasourceList = await listDatasources(datasourceBody) + const responseData = { data: datasourceList, count: _.size(datasourceList) } + logger.info({ req: req.body, resmsgid: _.get(res, "resmsgid"), message: `Datasources are listed successfully with a datasource count (${_.size(datasourceList)})` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); + +}; + +const listDatasources = async (request: Record): Promise> => { + + const { filters = {} } = request || {}; + const dsStatus = _.get(filters, "status"); + const status = _.isArray(dsStatus) ? dsStatus : _.compact([dsStatus]) + const draftFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? draftDatasourceStatus : _.intersection(status, draftDatasourceStatus)); + const liveFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? liveDatasourceStatus : _.intersection(status, liveDatasourceStatus)); + const liveDatasourceList = await datasetService.findDatasources(liveFilters, ["dataset_id", "datasource", "type", "status", "id", "type", "created_by", "updated_by", "created_date", "updated_date"]); + const draftDatasourceList = await datasetService.findDraftDatasources(draftFilters, ["id", "dataset_id","name", "type", "status", "created_by", "updated_by", "created_date", "updated_date"]); + return _.compact(_.concat(liveDatasourceList, draftDatasourceList)); + +} + +export default getDatasourceList; \ No newline at end of file diff --git a/api-service/src/controllers/DatasourceList/RequestValidationSchema.json b/api-service/src/controllers/DatasourceList/RequestValidationSchema.json new file mode 100644 index 00000000..79951a7d --- /dev/null +++ b/api-service/src/controllers/DatasourceList/RequestValidationSchema.json @@ -0,0 +1,59 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.datasources.list"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "filters": { + "type": "object", + "properties": { + "status": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string", + "enum": ["Draft", "Live", "Retired", "Purged"] + } + }, + { + "type": "string", + "enum": ["Draft", "Live", "Retired"] + } + ] + }, + "type": { + "type": "string", + "enum": ["druid", "datalake"] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false + } + \ No newline at end of file diff --git a/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts index dc1f0863..b710f49e 100644 --- a/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts +++ b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts @@ -5,9 +5,12 @@ import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { ErrorObject } from "../../types/ResponseModel"; import { druidHttpService } from "../../connections/druidConnection"; +import { getDatasourceList } from "../../services/DatasourceService"; +import { AxiosResponse } from "axios"; const apiId = "api.obsrv.data.sql-query"; const errorCode = "SQL_QUERY_FAILURE" +export const result_data = {"data": {}}; export const sqlQuery = async (req: Request, res: Response) => { const resmsgid = _.get(res, "resmsgid"); @@ -24,11 +27,17 @@ export const sqlQuery = async (req: Request, res: Response) => { errCode: "BAD_REQUEST" } as ErrorObject, req, res); } - - const result = await druidHttpService.post(`${config.query_api.druid.sql_query_path}`, req.body, { - headers: { Authorization: authorization }, - }); - + const query = req.body.query as string; + let result: AxiosResponse; + if (isTableSchemaQuery(query)) { + const dataSources = await fetchDruidDataSources(); + result = createMockAxiosResponse(dataSources); + } else { + result = await druidHttpService.post(`${config.query_api.druid.sql_query_path}`, req.body, { + headers: { Authorization: authorization }, + }); + } + _.set(result_data, "data", result.data); logger.info({ messsge: "Successfully fetched data using sql query", apiId, resmsgid }) ResponseHandler.flatResponse(req, res, result) } catch (error: any) { @@ -37,4 +46,37 @@ export const sqlQuery = async (req: Request, res: Response) => { logger.error(error, apiId, code, resmsgid) ResponseHandler.errorResponse(errorMessage, req, res); } -} \ No newline at end of file +} + +const fetchDruidDataSources = async (): Promise<{ TABLE_NAME: string }[]> => { + try { + const dataSources = await getDatasourceList(); + return dataSources + .filter((ds: any) => ds.type === "druid") + .map((ds: any) => ({ TABLE_NAME: ds.dataValues.datasource_ref })); + } catch (error) { + logger.error({ message: "Failed to fetch Druid data sources", error }); + throw new Error("Failed to fetch Druid data sources"); + } +}; + +const isTableSchemaQuery = (sqlQuery?: string): boolean => { + return ( + sqlQuery + ?.trim() + .replace(/\s+/g, " ") + .toUpperCase() === + "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'DRUID'" + ); + }; + + +const createMockAxiosResponse = (data: any): AxiosResponse => { + return { + data, + status: 200, + statusText: "OK", + headers: {}, + config: {}, + } as AxiosResponse; +}; diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 05051461..0c675857 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -44,8 +44,7 @@ export const onGone = (req: any, res: Response) => { } export const onObsrvFailure = (req: any, res: Response,error: ObsrvError) => { - const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) - metricLabels.dataset_id = error.datasetId + const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res, error) const { statusCode = 404 } = res const labels = { ...metricLabels, status: statusCode } duration && setQueryResponseTime({ duration, labels }) @@ -53,12 +52,12 @@ export const onObsrvFailure = (req: any, res: Response,error: ObsrvError) => { incrementFailedApiCalls({ labels }); } -const getMetricLabels = (req: any, res: Response) => { +const getMetricLabels = (req: any, res: Response, errorBody?: ObsrvError) => { const { id, entity, originalUrl, startTime } = req; const { statusCode = 200 } = res const request_size = req.socket.bytesRead const response_size = res.getHeader("content-length"); - const dataset_id = _.get(req, ["body", "request", "dataset_id"]) || _.get(req, ["params", "dataset_id"]) || null + const dataset_id = _.get(req, ["body", "request", "dataset_id"]) || _.get(req, ["params", "dataset_id"]) || _.get(errorBody, "datasetId") || null const duration = getDuration(startTime); const metricLabels = { entity, id, endpoint: originalUrl, dataset_id, status: statusCode, request_size, response_size } return { duration, metricLabels } diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 10fda3e2..bcc9bcd2 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -77,7 +77,7 @@ const checkAccess = (decoded: any, action: string, req: Request, res: Response) const basicToken = (token: string, req: Request, res: Response, next: NextFunction) => { try { - const decoded = jwt.verify(token, config.user_token_public_key); + const decoded = jwt.verify(token, config.user_token_public_key, { algorithms: ['RS256'] }); if (!decoded || !_.isObject(decoded)) { return errorHandler(401, "Token verification failed or invalid token", req, res); @@ -125,8 +125,8 @@ export default { (req as any).userID = "SYSTEM"; return next(); } - - const token = req.get("x-user-token"); + const authHeader = req.headers['authorization']; + const token = authHeader && authHeader.split(' ')[1]; if (!token) { return errorHandler(401, "No token provided", req, res); } diff --git a/api-service/src/middlewares/userPermissions.json b/api-service/src/middlewares/userPermissions.json index 400e218f..a7ecccc4 100644 --- a/api-service/src/middlewares/userPermissions.json +++ b/api-service/src/middlewares/userPermissions.json @@ -4,6 +4,7 @@ "api.datasets.list", "api.datasets.read", "api.datasets.export", + "api.dataset.health", "api.data.out", "api.data.exhaust", "api.alert.list", @@ -12,11 +13,15 @@ "api.alert.silence.list", "api.alert.silence.get", "api.alert.notification.list", - "api.alert.notification.get" + "api.alert.notification.get", + "api.dataset.metrics", + "api.datasources.list", + "api.data.metrics" ], "restricted_dataset_api": [ "api.datasets.reset", - "api.datasets.status-transition" + "api.datasets.status-transition", + "api.connector.register" ], "alert": [ "api.alert.create", @@ -46,11 +51,12 @@ "api.datasets.update", "api.datasets.import", "api.datasets.copy", - "api.dataset.health", - "api.datasets.dataschema" + "api.datasets.dataschema", + "api.datasets.alias" ], "data": [ - "api.data.in" + "api.data.in", + "api.data.analyze.pii" ], "queryTemplate": [ "api.query.template.create", @@ -92,7 +98,8 @@ "schema", "file", "connector", - "sqlQuery" + "sqlQuery", + "data" ], "dataset_manager": [ "general_access", @@ -104,7 +111,8 @@ "file", "connector", "sqlQuery", - "restricted_dataset_api" + "restricted_dataset_api", + "data" ], "admin": [ "general_access", @@ -116,7 +124,12 @@ "file", "connector", "sqlQuery", - "restricted_dataset_api" + "restricted_dataset_api", + "alert", + "metric", + "silence", + "notificationChannel", + "data" ], "operations_admin": [ "alert", diff --git a/api-service/src/models/ConnectorInstances.ts b/api-service/src/models/ConnectorInstances.ts index 14c832d7..f3f1b199 100644 --- a/api-service/src/models/ConnectorInstances.ts +++ b/api-service/src/models/ConnectorInstances.ts @@ -12,10 +12,6 @@ export const ConnectorInstances = sequelize.define("connector_instances", { connector_id: { type: DataTypes.STRING }, - data_format: { - type: DataTypes.STRING, - defaultValue: "json" - }, connector_config: { type: DataTypes.STRING }, diff --git a/api-service/src/models/Table.ts b/api-service/src/models/Table.ts new file mode 100644 index 00000000..fe599e17 --- /dev/null +++ b/api-service/src/models/Table.ts @@ -0,0 +1,68 @@ +import { DataTypes } from 'sequelize'; +import { sequelize } from '../connections/databaseConnection'; + +export const TableDraft = sequelize.define("table_draft", { + id: { + type: DataTypes.TEXT, + primaryKey: true, + allowNull: false + }, + name: { + type: DataTypes.TEXT, + allowNull: false + }, + dataset_id: { + type: DataTypes.TEXT, + allowNull: false + }, + type: { + type: DataTypes.TEXT, + allowNull: false + }, + spec: { + type: DataTypes.JSON, + allowNull: false + }, + spectype: { + type: DataTypes.TEXT, + allowNull: false + }, + fields: { + type: DataTypes.JSON, + allowNull: false + }, + ingestion_spec: { + type: DataTypes.JSON, + allowNull: true + }, + version: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 1 + }, + version_key: { + type: DataTypes.TEXT, + allowNull: false + }, + status: { + type: DataTypes.TEXT, + allowNull: false + }, + created_by: { + type: DataTypes.TEXT, + allowNull: false + }, + updated_by: { + type: DataTypes.TEXT, + allowNull: false + } +}, + { + timestamps: true, + createdAt: "created_date", + updatedAt: "updated_date", + tableName: "table_draft", + } +); + +export default TableDraft; \ No newline at end of file diff --git a/api-service/src/routes/AlertsRouter.ts b/api-service/src/routes/AlertsRouter.ts index 6044e9fe..a46e0f72 100644 --- a/api-service/src/routes/AlertsRouter.ts +++ b/api-service/src/routes/AlertsRouter.ts @@ -5,34 +5,36 @@ import customAlertHandler from "../controllers/Alerts/Alerts"; import metricAliasHandler from "../controllers/Alerts/Metric"; import silenceHandler from "../controllers/Alerts/Silence"; import checkRBAC from "../middlewares/RBAC_middleware"; +import { OperationType, telemetryAuditStart } from "../services/telemetry"; +import telemetryActions from "../telemetry/telemetryActions"; export const alertsRouter = express.Router(); // Notifications alertsRouter.post("/notifications/search", setDataToRequestObject("api.alert.notification.list"), checkRBAC.handler(), notificationHandler.listHandler); -alertsRouter.post("/notifications/create", setDataToRequestObject("api.alert.notification.create"), checkRBAC.handler(), notificationHandler.createHandler); +alertsRouter.post("/notifications/create", setDataToRequestObject("api.alert.notification.create"), telemetryAuditStart({action: telemetryActions.createNotifications, operationType: OperationType.CREATE}), checkRBAC.handler(), notificationHandler.createHandler); alertsRouter.get("/notifications/publish/:id", setDataToRequestObject("api.alert.notification.publish"), checkRBAC.handler(), notificationHandler.publishHandler); alertsRouter.post("/notifications/test", setDataToRequestObject("api.alert.notification.test"), checkRBAC.handler(), notificationHandler.testNotifationChannelHandler); -alertsRouter.patch("/notifications/update/:id", setDataToRequestObject("api.alert.notification.update"), checkRBAC.handler(), notificationHandler.updateHandler); -alertsRouter.delete("/notifications/delete/:id", setDataToRequestObject("api.alert.notification.retire"), checkRBAC.handler(), notificationHandler.retireHandler); +alertsRouter.patch("/notifications/update/:id", setDataToRequestObject("api.alert.notification.update"), telemetryAuditStart({action: telemetryActions.updateNotifications, operationType: OperationType.UPDATE}), checkRBAC.handler(), notificationHandler.updateHandler); +alertsRouter.delete("/notifications/delete/:id", setDataToRequestObject("api.alert.notification.retire"), telemetryAuditStart({action: telemetryActions.deleteNotifications, operationType: OperationType.RETIRE}), checkRBAC.handler(), notificationHandler.retireHandler); alertsRouter.get("/notifications/get/:id", setDataToRequestObject("api.alert.notification.get"), checkRBAC.handler(), notificationHandler.fetchHandler); // alerts -alertsRouter.post("/create", setDataToRequestObject("api.alert.create"), checkRBAC.handler(), customAlertHandler.createAlertHandler); -alertsRouter.get("/publish/:alertId", setDataToRequestObject("api.alert.publish"), checkRBAC.handler(), customAlertHandler.publishAlertHandler); +alertsRouter.post("/create", setDataToRequestObject("api.alert.create"), checkRBAC.handler(),telemetryAuditStart({action: telemetryActions.alertCreate, operationType: OperationType.CREATE}), customAlertHandler.createAlertHandler); +alertsRouter.get("/publish/:alertId", setDataToRequestObject("api.alert.publish"), checkRBAC.handler(), telemetryAuditStart({action: telemetryActions.alertHandlerPublish, operationType: OperationType.GET}), customAlertHandler.publishAlertHandler); alertsRouter.post(`/search`, setDataToRequestObject("api.alert.list"), checkRBAC.handler(), customAlertHandler.searchAlertHandler); -alertsRouter.get("/get/:alertId", setDataToRequestObject("api.alert.getAlertDetails"), checkRBAC.handler(), customAlertHandler.alertDetailsHandler); -alertsRouter.delete("/delete/:alertId", setDataToRequestObject("api.alert.delete"), checkRBAC.handler(), customAlertHandler.deleteAlertHandler); +alertsRouter.get("/get/:alertId", setDataToRequestObject("api.alert.getAlertDetails"), telemetryAuditStart({action: telemetryActions.alertHandlerDetails, operationType: OperationType.CREATE}), checkRBAC.handler(), customAlertHandler.alertDetailsHandler); +alertsRouter.delete("/delete/:alertId", setDataToRequestObject("api.alert.delete"), telemetryAuditStart({action: telemetryActions.deleteAlertHandler, operationType: OperationType.RETIRE}), checkRBAC.handler(), customAlertHandler.deleteAlertHandler); alertsRouter.delete("/delete", setDataToRequestObject("api.alert.delete"), checkRBAC.handler(), customAlertHandler.deleteSystemAlertsHandler); -alertsRouter.patch("/update/:alertId", setDataToRequestObject("api.alert.update"), checkRBAC.handler(), customAlertHandler.updateAlertHandler); +alertsRouter.patch("/update/:alertId", setDataToRequestObject("api.alert.update"), telemetryAuditStart({action: telemetryActions.updateAlertHandler, operationType: OperationType.UPDATE}), checkRBAC.handler(), customAlertHandler.updateAlertHandler); // metrics -alertsRouter.post("/metric/alias/create",setDataToRequestObject("api.metric.add"), checkRBAC.handler(), metricAliasHandler.createMetricHandler); +alertsRouter.post("/metric/alias/create",setDataToRequestObject("api.metric.add"), telemetryAuditStart({action: telemetryActions.createMetricHandler, operationType: OperationType.CREATE}), checkRBAC.handler(), metricAliasHandler.createMetricHandler); alertsRouter.post("/metric/alias/search", setDataToRequestObject("api.metric.list"), checkRBAC.handler(), metricAliasHandler.listMetricsHandler); -alertsRouter.patch("/metric/alias/update/:id", setDataToRequestObject("api.metric.update"), checkRBAC.handler(), metricAliasHandler.updateMetricHandler); -alertsRouter.delete("/metric/alias/delete/:id", setDataToRequestObject("api.metric.remove"), checkRBAC.handler(), metricAliasHandler.deleteMetricHandler); -alertsRouter.delete("/metric/alias/delete", setDataToRequestObject("api.metric.remove"), checkRBAC.handler(), metricAliasHandler.deleteMultipleMetricHandler); +alertsRouter.patch("/metric/alias/update/:id", setDataToRequestObject("api.metric.update"), telemetryAuditStart({action: telemetryActions.updateMetricHandler, operationType: OperationType.UPDATE}), checkRBAC.handler(), metricAliasHandler.updateMetricHandler); +alertsRouter.delete("/metric/alias/delete/:id", setDataToRequestObject("api.metric.remove"), telemetryAuditStart({action: telemetryActions.deleteMetricHandler, operationType: OperationType.RETIRE}), checkRBAC.handler(), metricAliasHandler.deleteMetricHandler); +alertsRouter.delete("/metric/alias/delete", setDataToRequestObject("api.metric.remove"), telemetryAuditStart({action: telemetryActions.deleteMultipleMetricHandler, operationType: OperationType.RETIRE}), checkRBAC.handler(), metricAliasHandler.deleteMultipleMetricHandler); // silence alertsRouter.post("/silence/create",setDataToRequestObject("api.alert.silence.create"), checkRBAC.handler(), silenceHandler.createHandler); diff --git a/api-service/src/routes/DruidProxyRouter.ts b/api-service/src/routes/DruidProxyRouter.ts index fdf73315..ac45710c 100644 --- a/api-service/src/routes/DruidProxyRouter.ts +++ b/api-service/src/routes/DruidProxyRouter.ts @@ -20,4 +20,5 @@ druidProxyRouter.post(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.n druidProxyRouter.get(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet); druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) -druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file +druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) +druidProxyRouter.get(/\/druid\/coordinator(.*)/, setDataToRequestObject("query.wrapper.native.get"), onRequest({entity: Entity.DruidProxy}), wrapperService.forwardNativeGetDatasource) \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 1e39d942..b2d4a3ff 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -27,40 +27,49 @@ import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; -import { OperationType, telemetryAuditStart } from "../services/telemetry"; +import { OperationType, telemetryAuditStart, telemetryLogStart } from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; -import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; import checkRBAC from "../middlewares/RBAC_middleware"; import connectorRegisterController from "../controllers/ConnectorRegister/ConnectorRegisterController"; +import dataMetrics from "../controllers/DataMetrics/DataMetricsController"; +import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; +import { dataAnalyzePII } from "../controllers/DataAnalyzePII/DataAnalyzePIIController"; +import datasetAlias from "../controllers/DatasetAlias/DatasetAlias"; +import getDatasourceList from "../controllers/DatasourceList/DatasourceList"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), dataOut); +router.post("/data/in/:dataset_id", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.ingestEvents, operationType: OperationType.CREATE}), checkRBAC.handler(), dataIn); +router.post("/data/query/:dataset_id", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), telemetryLogStart({action: telemetryActions.sqlQuery, operationType: OperationType.CREATE}), checkRBAC.handler(), dataOut); router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(),DatasetCreate) router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), checkRBAC.handler(), DatasetUpdate) router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), checkRBAC.handler(), DatasetRead) router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), checkRBAC.handler(), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), checkRBAC.handler(), dataExhaust); +router.get("/data/exhaust/:dataset_id", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), checkRBAC.handler(), dataExhaust); router.post("/template/create", setDataToRequestObject("api.query.template.create"), checkRBAC.handler(), createQueryTemplate); router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), checkRBAC.handler(), readQueryTemplate); router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), checkRBAC.handler(), deleteQueryTemplate); router.post("/template/list", setDataToRequestObject("api.query.template.list"), checkRBAC.handler(), listQueryTemplates); router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), checkRBAC.handler(), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), checkRBAC.handler(), eventValidation); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"),telemetryAuditStart({action: telemetryActions.schemaValidate, operationType: OperationType.LIST}), checkRBAC.handler(), eventValidation); router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), checkRBAC.handler(), queryTemplate); -router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), GenerateSignedURL); +router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.generateURL, operationType: OperationType.GET}), checkRBAC.handler(), GenerateSignedURL); router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetReset); -router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DataSchemaGenerator); -router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetExport); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.datasetHealth, operationType: OperationType.CREATE}) ,checkRBAC.handler(), datasetHealth); +router.post("/datasets/reset/:dataset_id", setDataToRequestObject("api.datasets.reset"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.datasetReset, operationType: OperationType.CREATE}), checkRBAC.handler(), datasetReset); +router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createDataschema, operationType: OperationType.CREATE}), checkRBAC.handler(), DataSchemaGenerator); +router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.exportDataset, operationType: OperationType.GET}),checkRBAC.handler(), DatasetExport); router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetCopy); router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsList); router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsRead); -router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetImport); -router.post("/connector/register", setDataToRequestObject("api.connector.register"), onRequest({ entity: Entity.Management }), connectorRegisterController); +router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.importDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetImport); +router.post("/connector/register", setDataToRequestObject("api.connector.register"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.registerConnector, operationType: OperationType.CREATE}),checkRBAC.handler(), connectorRegisterController); +router.post("/datasets/alias", setDataToRequestObject("api.datasets.alias"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.datasetAlias, operationType: OperationType.CREATE}), checkRBAC.handler(), datasetAlias); +router.post("/datasources/list", setDataToRequestObject("api.datasources.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasource, operationType: OperationType.CREATE}), checkRBAC.handler(), getDatasourceList); +router.post("/data/analyze/pii", setDataToRequestObject("api.data.analyze.pii"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.dataAnalyzePii, operationType: OperationType.CREATE}), checkRBAC.handler(), dataAnalyzePII); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); -router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) - +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), telemetryLogStart({action: telemetryActions.dataQuery, operationType: OperationType.CREATE}), checkRBAC.handler(), sqlQuery); +router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), telemetryAuditStart({action: telemetryActions.dataAnalyzePii, operationType: OperationType.CREATE}), checkRBAC.handler(), dataMetrics) +router.post("/dataset/metrics", setDataToRequestObject("api.dataset.metrics"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetMetrix, operationType: OperationType.CREATE}), checkRBAC.handler(), datasetMetrics); +//System API +router.post("/files/url-access", setDataToRequestObject("api.files.generate.url"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.generateURL, operationType: OperationType.GET}), GenerateSignedURL); diff --git a/api-service/src/services/AlertManagerService.ts b/api-service/src/services/AlertManagerService.ts new file mode 100644 index 00000000..7e028d58 --- /dev/null +++ b/api-service/src/services/AlertManagerService.ts @@ -0,0 +1,151 @@ +import _ from 'lodash'; +import { obsrvError } from '../types/ObsrvError'; +import { Metrics } from '../models/Metric'; +import { getAlertByDataset, getAlertPayload, getAlertRule, publishAlert } from './managers'; +import { Alert } from '../models/Alert'; +import { alertConfig } from './AlertsConfigSevice'; +import Transaction from "sequelize/types/transaction"; + +interface MetricConfig { + metric: string; + alias: string; + category: string; + description: string; + frequency: string; + interval: string; + code: string; + severity: string; + summary?: string; + operator: string; + threshold: number; + flattened?: boolean; +} + +class AlertManagerService { + private config: any; + + constructor() { + this.config = alertConfig.find('configs.alerts'); + } + + private getModifiedMetric = (service: string, metric: any, datasetId: string, datasource_ref?: string): any => { + const metricData = _.cloneDeep(metric); + if (service === 'flink') { + const modifiedSubstring = datasetId.replace(/-/g, '_'); + metricData.metric = metricData.metric.replaceAll('dataset_id', modifiedSubstring); + } + else if (service === 'druid') { + metricData.metric = metricData.metric.replaceAll('dataset_id', + metricData.flattened ? (datasource_ref || '').replace(/-/g, '_') : datasource_ref + ); + } + else if (service === 'api') { + metricData.metric = metricData.metric.replaceAll('', datasetId); + } + else { + metricData.metric = metricData.metric.replace('dataset_id', datasetId); + } + return metricData; + } + + private createAlerts = async (params: { datasetId: string; service: string; metric: any; transaction: Transaction, datasource_ref?: string }): Promise => { + const { datasetId, service, metric, transaction, datasource_ref } = params; + const metricData = this.getModifiedMetric(service, metric, datasetId, datasource_ref); + + const dataset_id = datasource_ref ? datasource_ref : datasetId + const metricPayload = { + alias: `${metricData.alias} (${dataset_id})`, + component: metricData.category, + subComponent: datasetId, + metric: metricData.metric, + context: { + datasetId: datasetId, + }, + }; + + const response = await this.createMetric(metricPayload, transaction); + await this.createAlertRule({ datasetId, datasource_ref, metricData, transaction, metricId: response.dataValues.id }); + } + + private createAlertRule = async (params: { + datasetId: string; + metricData: any; + transaction: Transaction; + metricId: string; + datasource_ref?: string | null; + }): Promise => { + const { datasetId, metricData, transaction, metricId, datasource_ref = null } = params; + const dataset = datasource_ref ? datasource_ref : datasetId + const datasetName = dataset.replace(/[-.]/g, ' ').replace(/\b\w/g, c => _.toUpper(c)); + const alertPayload = { + name: metricData.alias.replace('[DATASET]', `[DATASET][${datasetName}]`), + manager: 'grafana', + description: metricData.description, + category: metricData.category, + frequency: metricData.frequency, + interval: metricData.interval, + context: { alertType: 'SYSTEM' }, + labels: { alert_code: metricData.code, component: 'obsrv', dataset: datasetId, table: datasource_ref }, + severity: metricData.severity, + annotations: { + summary: _.get(metricData, 'summary', ''), + }, + metadata: { + queryBuilderContext: { + category: metricData.category, + id: metricId, + subComponent: datasetId, + metric: metricData.metric, + operator: metricData.operator, + threshold: [+metricData.threshold], + metricAlias: metricData.alias, + } + }, + notification: { channels: [] } + }; + + const alertData = getAlertPayload(alertPayload); + await this.createAlert(alertData, transaction); + } + + public publishAlertRule = async (dataset_id: string): Promise => { + const datasetAlerts: any[] = await getAlertByDataset(dataset_id) + for (const alert of datasetAlerts) { + const { id } = alert; + const ruleModel: Record | null = await getAlertRule(id); + if (!ruleModel) { + throw obsrvError(id, 'ALERT_RULE_NOT_FOUND', `Alert rule with id ${id} not found`, 'NOT_FOUND', 404); + } + const rulePayload = ruleModel.toJSON(); + await publishAlert(rulePayload); + } + } + + private createMetric = async (payload: Record, transaction: Transaction) => { + return Metrics.create(payload, { transaction }); + } + + private createAlert = async (alertData: Record, transaction: Transaction) => { + return Alert.create(alertData, { transaction }); + } + + public createDatasetAlertsDraft = async (dataset: Record, transaction: Transaction, datasource_ref: string): Promise => { + const allMetrics = [ + ...this.config.dataset_metrics_flink.map((metric: MetricConfig) => ({ service: 'flink', metric })), + ...this.config.dataset_metrics_druid.map((metric: MetricConfig) => ({ service: 'druid', metric })), + ...this.config.api_metric.map((metric: MetricConfig) => ({ service: 'api', metric })) + ]; + + for (const { service, metric } of allMetrics) { + await this.createAlerts({ + datasetId: dataset.dataset_id, + service, + metric, + transaction, + ...(service === 'druid' ? { datasource_ref } : {}) + }); + } + } +} + +export const alertService = new AlertManagerService(); \ No newline at end of file diff --git a/api-service/src/services/AlertsConfigSevice.ts b/api-service/src/services/AlertsConfigSevice.ts new file mode 100644 index 00000000..2c053080 --- /dev/null +++ b/api-service/src/services/AlertsConfigSevice.ts @@ -0,0 +1,24 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import _ from 'lodash'; +import logger from '../logger'; +import { config } from '../configs/Config'; + +class AlertsConfig { + private config: Record; + private readonly ALERTS_CONFIG_FILE = 'alertsConfig.json'; + + constructor() { + const configDir = config.alerts_rules.config_path || path.resolve(process.cwd(), 'src/configs'); + const configPath = path.join(configDir, this.ALERTS_CONFIG_FILE); + const configContent = fs.readFileSync(configPath, 'utf8'); + this.config = JSON.parse(configContent); + logger.info(`Alerts config loaded from: ${configPath}`); + } + + find(path: string): any { + return _.get(this.config, path.split('.')); + } +} + +export const alertConfig = new AlertsConfig(); \ No newline at end of file diff --git a/api-service/src/services/CloudServices/AWSStorageService.ts b/api-service/src/services/CloudServices/AWSStorageService.ts index d83b65b1..70f44e84 100644 --- a/api-service/src/services/CloudServices/AWSStorageService.ts +++ b/api-service/src/services/CloudServices/AWSStorageService.ts @@ -7,21 +7,40 @@ import { getFileKey } from "../../utils/common" import { FilterDataByDateRange, ICloudService } from "./types"; import { URLAccess } from "../../types/SampleURLModel"; import logger from "../../logger"; +import { fromTokenFile } from "@aws-sdk/credential-providers"; export class AWSStorageService implements ICloudService { client: any; constructor(config: any) { - if (_.get(config, "identity") && _.get(config, "credential") && _.get(config, "region")) { + if (_.get(config, "identity") && _.get(config, "credential") && _.get(config, "region") || _.get(config, "webIdentityTokenFile") && _.get(config, "roleArn")) { const region = _.get(config, "region") const accessKeyId = _.get(config, "identity") const secretAccessKey = _.get(config, "credential") const endpoint = _.get(config, "endpoint") + const webIdentityTokenFile = _.get(config, "webIdentityTokenFile") + const roleArn = _.get(config, "roleArn") + const s3ForcePathStyle = _.get(config, "s3ForcePathStyle") const configuration: any = { region, credentials: { accessKeyId, secretAccessKey } } - if(endpoint) { + if (endpoint) { configuration.endpoint = endpoint; } + if (s3ForcePathStyle) { + configuration.forcePathStyle = s3ForcePathStyle; + } try { - this.client = new S3Client(configuration); + if (_.isEmpty(secretAccessKey) && _.isEmpty(accessKeyId) && !_.isEmpty(webIdentityTokenFile) && !_.isEmpty(roleArn)) { + console.log("Using Instance Metadata") + this.client = new S3Client({ + credentials: fromTokenFile({ + webIdentityTokenFile: webIdentityTokenFile, + roleArn: roleArn + }), + region: region + }); + } else { + console.log("Using AWS Credentials") + this.client = new S3Client(configuration); + } } catch (err) { logger.error(err) diff --git a/api-service/src/services/CloudServices/index.ts b/api-service/src/services/CloudServices/index.ts index c458016d..75b05e29 100644 --- a/api-service/src/services/CloudServices/index.ts +++ b/api-service/src/services/CloudServices/index.ts @@ -14,7 +14,7 @@ const initialiseServiceProvider = (provider: any, config: any): AzureStorageServ return new AzureStorageService(config); case "aws": return new AWSStorageService(config); - case "gcloud": + case "gcp": return new GCPStorageService(config); default: throw new Error(`Client Cloud Service - ${provider} provider is not supported`); diff --git a/api-service/src/services/DatasetMetricsService.ts b/api-service/src/services/DatasetMetricsService.ts new file mode 100644 index 00000000..03bcad0c --- /dev/null +++ b/api-service/src/services/DatasetMetricsService.ts @@ -0,0 +1,236 @@ +import axios from "axios"; +import dayjs from "dayjs"; +import _ from "lodash"; +import { config } from "../configs/Config"; +import { dataLineageSuccessQuery, generateConnectorQuery, generateDatasetQueryCallsQuery, generateDedupFailedQuery, generateDenormFailedQuery, generateTimeseriesQuery, generateTimeseriesQueryEventsPerHour, generateTotalQueryCallsQuery, generateTransformationFailedQuery, processingTimeQuery, totalEventsQuery, totalFailedEventsQuery } from "../controllers/DatasetMetrics/queries"; +import { druidHttpService } from "../connections/druidConnection"; +const druidPort = _.get(config, "query_api.druid.port"); +const druidHost = _.get(config, "query_api.druid.host"); +const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; +const prometheusEndpoint = `${config.query_api.prometheus.url}/api/v1/query_range`; + +export const getDataFreshness = async (dataset_id: string, intervals: string, defaultThreshold: number) => { + const queryPayload = processingTimeQuery(intervals, dataset_id); + const druidResponse = await druidHttpService.post(nativeQueryEndpoint, queryPayload?.query); + const avgProcessingTime = _.get(druidResponse, "data[0].average_processing_time", 0); + const freshnessStatus = avgProcessingTime < defaultThreshold ? "Healthy" : "Unhealthy"; + + return { + category: "data_freshness", + status: freshnessStatus, + components: [ + { + type: "average_time_difference_in_min", + threshold: defaultThreshold / 60000, // convert to minutes + value: avgProcessingTime / 60000, + status: freshnessStatus + }, + { + type: "freshness_query_time_in_min", + threshold: 10, + value: avgProcessingTime / 60000, // convert to minutes + status: freshnessStatus + } + ] + }; +}; + +export const getDataObservability = async (dataset_id: string, intervals: string) => { + const totalEventsPayload = totalEventsQuery(intervals, dataset_id); + const totalFailedEventsPayload = totalFailedEventsQuery(intervals, dataset_id); + const totalQueryCalls = generateTotalQueryCallsQuery(config?.data_observability?.data_out_query_time_period); + const totalQueryCallsAtDatasetLevel = generateDatasetQueryCallsQuery(dataset_id, config?.data_observability?.data_out_query_time_period); + + const [totalEventsResponse, totalFailedEventsResponse, totalApiCallsResponse, totalCallsAtDatasetLevelResponse] = await Promise.all([ + druidHttpService.post(nativeQueryEndpoint, totalEventsPayload), + druidHttpService.post(nativeQueryEndpoint, totalFailedEventsPayload), + axios.request({ url: prometheusEndpoint, method: "GET", params: totalQueryCalls }), + axios.request({ url: prometheusEndpoint, method: "GET", params: totalQueryCallsAtDatasetLevel }) + ]); + + const totalApiCalls = _.map(_.get(totalApiCallsResponse, 'data.data.result'), payload => { + return _.floor(_.get(payload, 'values[0][1]'), 3) || 0 + }) + + const totalApiCallsAtDatasetLevel = _.map(_.get(totalCallsAtDatasetLevelResponse, 'data.data.result'), payload => { + return _.floor(_.get(payload, 'values[0][1]'), 3) || 0 + }) + + const importanceScore = (totalApiCallsAtDatasetLevel[0] / totalApiCalls[0]) * 100; + + const totalEventsCount = _.get(totalEventsResponse, "data[0].result.total_events_count") || 0; + const totalFailedEventsCount = _.get(totalFailedEventsResponse, "data[0].result.total_failed_events") || 0; + let failurePercentage = 0; + if (totalEventsCount > 0) { + failurePercentage = (totalFailedEventsCount / totalEventsCount) * 100; + } + const observabilityStatus = failurePercentage > 5 ? "Unhealthy" : "Healthy"; + + return { + category: "data_observability", + status: observabilityStatus, + components: [ + { + type: "data_observability_health", + status: observabilityStatus + }, + { + type: "failure_percentage", + value: failurePercentage + }, + { + type: "threshold_percentage", + value: 5 + }, + { + type: "importance_score", + value: importanceScore || 0 + } + ] + }; +}; + +export const getDataVolume = async (dataset_id: string, volume_by_days: number, dateFormat: string) => { + const currentHourIntervals = dayjs().subtract(1, "hour").startOf("hour").toISOString() + "/" + dayjs().startOf("hour").toISOString(); + const currentDayIntervals = dayjs().subtract(1, 'day').startOf('day').format(dateFormat) + '/' + dayjs().endOf('day').format(dateFormat); + const currentWeekIntervals = dayjs().subtract(1, 'week').startOf('week').format(dateFormat) + '/' + dayjs().endOf('week').format(dateFormat); + const previousHourIntervals = dayjs().subtract(2, "hour").startOf("hour").toISOString() + '/' + dayjs().startOf("hour").toISOString(); + const previousDayIntervals = dayjs().subtract(2, 'day').startOf('day').format(dateFormat) + '/' + dayjs().subtract(1, 'day').endOf('day').format(dateFormat); + const previousWeekIntervals = dayjs().subtract(2, 'week').startOf('week').format(dateFormat) + '/' + dayjs().subtract(1, 'week').endOf('week').format(dateFormat); + const nDaysIntervals = dayjs().subtract(volume_by_days, 'day').startOf('day').format(dateFormat) + '/' + dayjs().endOf('day').format(dateFormat); + + const currentHourPayload = generateTimeseriesQueryEventsPerHour(currentHourIntervals, dataset_id); + const currentDayPayload = generateTimeseriesQuery(currentDayIntervals, dataset_id); + const currentWeekPayload = generateTimeseriesQuery(currentWeekIntervals, dataset_id); + const previousHourPayload = generateTimeseriesQueryEventsPerHour(previousHourIntervals, dataset_id); + const previousDayPayload = generateTimeseriesQuery(previousDayIntervals, dataset_id); + const previousWeekPayload = generateTimeseriesQuery(previousWeekIntervals, dataset_id); + const nDaysPayload = generateTimeseriesQuery(nDaysIntervals, dataset_id); + const [ + currentHourResponse, currentDayResponse, currentWeekResponse, + previousHourResponse, previousDayResponse, previousWeekResponse, + nDaysResponse + ] = await Promise.all([ + druidHttpService.post(nativeQueryEndpoint, currentHourPayload), + druidHttpService.post(nativeQueryEndpoint, currentDayPayload), + druidHttpService.post(nativeQueryEndpoint, currentWeekPayload), + druidHttpService.post(nativeQueryEndpoint, previousHourPayload), + druidHttpService.post(nativeQueryEndpoint, previousDayPayload), + druidHttpService.post(nativeQueryEndpoint, previousWeekPayload), + druidHttpService.post(nativeQueryEndpoint, nDaysPayload) + ]); + const currentHourCount = _.get(currentHourResponse, "data[0].result.count") || 0; + const currentDayCount = _.get(currentDayResponse, "data[0].result.count") || 0; + const currentWeekCount = _.get(currentWeekResponse, "data[0].result.count") || 0; + const previousHourCount = _.get(previousHourResponse, "data[0].result.count") || 0; + const previousDayCount = _.get(previousDayResponse, "data[0].result.count") || 0; + const previousWeekCount = _.get(previousWeekResponse, "data[0].result.count") || 0; + const nDaysCount = _.get(nDaysResponse, "data[0].result.count") || 0; + + const volumePercentageByHour = previousHourCount > 0 ? ((currentHourCount - previousHourCount) / previousHourCount) * 100 : 0; + const volumePercentageByDay = previousDayCount > 0 ? ((currentDayCount - previousDayCount) / previousDayCount) * 100 : 0; + const volumePercentageByWeek = previousWeekCount > 0 ? ((currentWeekCount - previousWeekCount) / previousWeekCount) * 100 : 0; + + return { + category: "data_volume", + components: [ + { type: "events_per_hour", value: currentHourCount }, + { type: "events_per_day", value: currentDayCount }, + { type: "events_per_n_day", value: nDaysCount }, + { type: "volume_percentage_by_hour", value: volumePercentageByHour }, + { type: "volume_percentage_by_day", value: volumePercentageByDay }, + { type: "volume_percentage_by_week", value: volumePercentageByWeek }, + { type: "growth_rate_percentage", value: volumePercentageByHour } + ] + }; +}; + +export const getDataLineage = async (dataset_id: string, intervals: string) => { + const transformationSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "transformer_status", "success"); + const dedupSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "dedup_status", "success"); + const denormSuccessPayload = dataLineageSuccessQuery(intervals, dataset_id, "denorm_status", "success"); + const totalValidationPayload = dataLineageSuccessQuery(intervals, dataset_id, "ctx_dataset", dataset_id); + const totalValidationFailedPayload = dataLineageSuccessQuery(intervals, dataset_id, "error_pdata_status", "failed"); + const transformationFailedPayload = generateTransformationFailedQuery(intervals, dataset_id); + const dedupFailedPayload = generateDedupFailedQuery(intervals, dataset_id); + const denormFailedPayload = generateDenormFailedQuery(intervals, dataset_id); + + const [ + transformationSuccessResponse, dedupSuccessResponse, denormSuccessResponse, + totalValidationResponse, totalValidationFailedResponse, transformationFailedResponse, dedupFailedResponse, denormFailedResponse + ] = await Promise.all([ + druidHttpService.post(nativeQueryEndpoint, transformationSuccessPayload), + druidHttpService.post(nativeQueryEndpoint, dedupSuccessPayload), + druidHttpService.post(nativeQueryEndpoint, denormSuccessPayload), + druidHttpService.post(nativeQueryEndpoint, totalValidationPayload), + druidHttpService.post(nativeQueryEndpoint, totalValidationFailedPayload), + druidHttpService.post(nativeQueryEndpoint, transformationFailedPayload), + druidHttpService.post(nativeQueryEndpoint, dedupFailedPayload), + druidHttpService.post(nativeQueryEndpoint, denormFailedPayload) + ]); + + // success at each level + const transformationSuccessCount = _.get(transformationSuccessResponse, "data[0].result.count") || 0; + const dedupSuccessCount = _.get(dedupSuccessResponse, "data[0].result.count") || 0; + const denormSuccessCount = _.get(denormSuccessResponse, "data[0].result.count") || 0; + const totalValidationCount = _.get(totalValidationResponse, "data[0].result.count") || 0; + const totalValidationFailedCount = _.get(totalValidationFailedResponse, "data[0].result.count") || 0; + const storageSuccessCount = totalValidationCount - totalValidationFailedCount; + + // failure at each level + const transformationFailedCount = _.get(transformationFailedResponse, "data[0].result.count") || 0; + const dedupFailedCount = _.get(dedupFailedResponse, "data[0].result.count") || 0; + const denormFailedCount = _.get(denormFailedResponse, "data[0].result.count") || 0; + return { + category: "data_lineage", + components: [ + { type: "transformation_success", value: transformationSuccessCount }, + { type: "dedup_success", value: dedupSuccessCount }, + { type: "denormalization_success", value: denormSuccessCount }, + { type: "total_success", value: storageSuccessCount }, + { type: "total_failed", value: totalValidationFailedCount }, + { type: "transformation_failed", value: transformationFailedCount }, + { type: "dedup_failed", value: dedupFailedCount }, + { type: "denorm_failed", value: denormFailedCount } + ] + }; +}; + + +export const getConnectors = async (dataset_id: string, intervals: string) => { + const connectorQueryPayload = generateConnectorQuery(intervals, dataset_id); + const connectorResponse = await druidHttpService.post(nativeQueryEndpoint, connectorQueryPayload); + const connectorsData = _.get(connectorResponse, "data[0].result", []); + const result = { + category: "connectors", + components: connectorsData.map((item: any) => ({ + id: item.name || "failed", + type: item.name === null ? "failed" : "success", + value: item.count + })) + }; + + return result; +}; + +export const getDataQuality = async (dataset_id: string, intervals: string) => { + const totalValidationPayload = dataLineageSuccessQuery(intervals, dataset_id, "ctx_dataset", dataset_id); + const totalValidationFailedPayload = dataLineageSuccessQuery(intervals, dataset_id, "error_pdata_status", "failed"); + const [totalValidationResponse, totalValidationFailedResponse, + ] = await Promise.all([ + druidHttpService.post(nativeQueryEndpoint, totalValidationPayload), + druidHttpService.post(nativeQueryEndpoint, totalValidationFailedPayload), + ]); + const totalValidationCount = _.get(totalValidationResponse, "data[0].result.count") || 0; + const totalValidationFailedCount = _.get(totalValidationFailedResponse, "data[0].result.count") || 0; + const storageSuccessCount = totalValidationCount - totalValidationFailedCount; + + return { + category: "data_quality", + "components": [ + { type: "incidents_failed", value: totalValidationFailedCount }, + { type: "incidents_success", value: storageSuccessCount }, + { type: "total_incidents", value: totalValidationCount } + ] + }; +} \ No newline at end of file diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index e8ce9a35..a140a686 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -18,6 +18,11 @@ import { obsrvError } from "../types/ObsrvError"; import { druidHttpService } from "../connections/druidConnection"; import { tableGenerator } from "./TableGenerator"; import { deleteAlertByDataset, deleteMetricAliasByDataset } from "./managers"; +import { config } from "../configs/Config"; +import { Op } from "sequelize"; +import TableDraft from "../models/Table"; +import { alertService } from "./AlertManagerService"; +import { ConnectorRegistry } from "../models/ConnectorRegistry"; class DatasetService { @@ -25,6 +30,30 @@ class DatasetService { return Dataset.findOne({ where: { id: datasetId }, attributes, raw: raw }); } + getDatasourceWithKey = async (datasourceKey: string, attributes?: string[], raw = false, is_primary?: boolean): Promise => { + const whereCondition: any = { + [Op.or]: [{ datasource: datasourceKey }, { id: datasourceKey }] + }; + + if (is_primary) { + whereCondition.is_primary = true; + } + + return Datasource.findOne({ + where: whereCondition, + attributes, + raw: raw + }); + } + + getDatasetWithDatasetkey = async (datasetKey: string, attributes?: string[], raw = false): Promise => { + const datasource = await this.getDatasourceWithKey(datasetKey, ["datasource_ref", "dataset_id"], true, true) + const dataset_id = !_.isEmpty(datasource) ? _.get(datasource, "dataset_id") : datasetKey + return Dataset.findOne({ + where: { dataset_id }, attributes, raw: raw + }); + } + findDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { return Dataset.findAll({ where, attributes, order, raw: true }) } @@ -49,6 +78,21 @@ class DatasetService { } } + checkDatasourceExist = async (id: string): Promise => { + const datasourceRef = await this.getDatasourceWithKey(id, ["id"], true); + if (_.isEmpty(datasourceRef)) { + const tables = TableDraft.findOne({ where: { id }, attributes: ["id"], raw: true }).catch((err: any) => { + if (err?.original?.code === '42P01') { + logger.warn("Table 'table_draft' does not exist, returning empty array."); + return null + } + throw obsrvError("", "FAILED_TO_FETCH_TABLES", err.message, "SERVER_ERROR", 500, err); + }) + return !_.isEmpty(tables) + } + return true; + } + getDraftDataset = async (dataset_id: string, attributes?: string[]) => { return DatasetDraft.findOne({ where: { dataset_id }, attributes, raw: true }); } @@ -73,10 +117,43 @@ class DatasetService { return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); } + getDatasource = async (datasource_id: string, attributes?: string[]) => { + return Datasource.findOne({ where: { id: datasource_id }, attributes, raw: true }); + } + + getDatasetIdWithDatasource = async (dataSource_ref: string, attributes?: string[]): Promise | any> => { + try { + return Datasource.findOne({ where: { datasource_ref: dataSource_ref }, attributes, raw: true }); + } catch (error) { + console.error("Error fetching dataset ID:", error); + return null; + } + } + + updateDatasource = async (payload: Record, where: Record): Promise> => { + return Datasource.update(payload, { where }); + } + getTransformations = async (dataset_id: string, attributes?: string[]) => { return DatasetTransformations.findAll({ where: { dataset_id }, attributes, raw: true }); } + getLiveDatasets = async (filters: Record, attributes?: string[]): Promise> => { + Dataset.hasMany(Datasource, { foreignKey: 'dataset_id' }); + const datasets = await Dataset.findAll({ + include: [ + { + model: Datasource, + attributes: ['datasource'], + where: { is_primary: true, type: "druid" }, + required: false + }, + ], raw: true, where: filters, attributes, order: [["updated_date", "DESC"]] + }); + const updatedDatasets = _.map(datasets, (dataset) => ({ ...dataset, alias: _.get(dataset, "datasources.datasource") })) + return updatedDatasets; + } + updateDraftDataset = async (draftDataset: Record): Promise> => { await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } }); @@ -138,16 +215,7 @@ class DatasetService { mode: _.get(config, ["mode"]) } }) - const connectorsFields = ["id", "connector_type", "connector_config"] - const connectors = _.includes([DatasetStatus.Live], status) ? await this.getConnectorsV1(dataset_id, connectorsFields) : await this.getDraftConnectors(dataset_id, connectorsFields); - draftDataset["connectors_config"] = _.map(connectors, (config) => { - return { - id: _.get(config, ["id"]), - connector_id: _.get(config, ["connector_type"]), - connector_config: _.get(config, ["connector_config"]), - version: "v1" - } - }) + draftDataset["connectors_config"] = []; draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) draftDataset["sample_data"] = dataset_config?.mergedEvent draftDataset["status"] = DatasetStatus.Draft @@ -179,15 +247,6 @@ class DatasetService { keys_config: { data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key }, cache_config: { redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db } } - const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); - draftDataset["connectors_config"] = _.map(connectors, (config) => { - return { - id: _.get(config, "id"), - connector_id: _.get(config, "connector_type"), - connector_config: _.get(config, "connector_config"), - version: "v1" - } - }) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); draftDataset["transformations_config"] = _.map(transformations, (config) => { const section: any = _.get(config, "metadata.section"); @@ -202,13 +261,14 @@ class DatasetService { mode: _.get(config, "mode") } }) + draftDataset["connectors_config"] = []; draftDataset["api_version"] = "v2" draftDataset["sample_data"] = dataset_config?.mergedEvent draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) } else { - const v1connectors = await getV1Connectors(draftDataset.dataset_id); const v2connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); - draftDataset["connectors_config"] = _.concat(v1connectors, v2connectors) + const updatedConnectorsPayload = getUpdatedV2ConnectorsPayload(v2connectors) + draftDataset["connectors_config"] = updatedConnectorsPayload; const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode"]); draftDataset["transformations_config"] = transformations } @@ -268,7 +328,7 @@ class DatasetService { const transaction = await sequelize.transaction(); try { - await Dataset.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { id: dataset.id }, transaction }); + await Dataset.update({ status: DatasetStatus.Retired, updated_by: updatedBy, alias: null }, { where: { id: dataset.id }, transaction }); await DatasetSourceConfig.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); await Datasource.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); await DatasetTransformations.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); @@ -286,6 +346,16 @@ class DatasetService { return Datasource.findAll({ where, attributes, order, raw: true }) } + findDraftDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { + return TableDraft.findAll({ where, attributes, order, raw: true }).catch((err: any) => { + if (err?.original?.code === '42P01') { + logger.warn("Table 'table_draft' does not exist, returning empty array."); + return []; + } + throw obsrvError("", "FAILED_TO_FETCH_TABLES", err.message, "SERVER_ERROR", 500, err); + }); + } + private deleteDruidSupervisors = async (dataset: Record) => { try { @@ -302,36 +372,49 @@ class DatasetService { } } - publishDataset = async (draftDataset: Record) => { + publishDataset = async (draftDataset: Record, userToken: string) => { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() + const liveDataset = await this.getDataset(draftDataset.dataset_id) try { await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }, transaction }) + let datasource_ref: any if (indexingConfig.olap_store_enabled) { - await this.createDruidDataSource(draftDataset, transaction); + const existingDatasource = await Datasource.findAll({ where: { dataset_id: draftDataset.dataset_id }, raw: true }) as unknown as Record + const getDatasetDatasource = _.find(existingDatasource, datasource => !_.get(datasource, "metadata.aggregated") && _.get(datasource, "metadata.granularity") === "day") + if (!_.isEmpty(getDatasetDatasource)) { + datasource_ref = await this.updateDruidDataSource(draftDataset, transaction, getDatasetDatasource); + } + else { + datasource_ref = await this.createDruidDataSource(draftDataset, transaction); + } } if (indexingConfig.lakehouse_enabled) { const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id", "api_version"], true); - if (liveDataset && liveDataset.api_version === "v2") { await this.updateHudiDataSource(draftDataset, transaction) } else { await this.createHudiDataSource(draftDataset, transaction) } } + if (_.isEmpty(liveDataset)) { + await alertService.createDatasetAlertsDraft(draftDataset, transaction, datasource_ref); + } await transaction.commit() } catch (err: any) { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } - await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET"); - + await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET", userToken); + if (_.isEmpty(liveDataset)) { + await alertService.publishAlertRule(draftDataset.dataset_id) + } } private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { - const {created_by, updated_by} = draftDataset; + const { created_by, updated_by } = draftDataset; const allFields = await tableGenerator.getAllFields(draftDataset, "druid"); const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); @@ -339,11 +422,26 @@ class DatasetService { _.set(draftDatasource, "created_by", created_by); _.set(draftDatasource, "updated_by", updated_by); await DatasourceDraft.upsert(draftDatasource, { transaction }) + return draftDatasource.datasource_ref + } + + private updateDruidDataSource = async (draftDataset: Record, transaction: Transaction, existingDatasource: Record) => { + + const { created_by, updated_by } = draftDataset; + const allFields = await tableGenerator.getAllFields(draftDataset, "druid"); + const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, existingDatasource.datasource_ref); + let draftDatasource = existingDatasource + _.set(draftDatasource, "ingestion_spec", ingestionSpec) + _.set(draftDatasource, "created_by", created_by); + _.set(draftDatasource, "updated_by", updated_by); + _.set(draftDatasource, "type", "druid"); + await DatasourceDraft.upsert(draftDatasource, { transaction }) + return existingDatasource.datasource_ref } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { - const {created_by, updated_by} = draftDataset; + const { created_by, updated_by } = draftDataset; const allFields = await tableGenerator.getAllFieldsHudi(draftDataset, "datalake"); const draftDatasource = this.createDraftDatasource(draftDataset, "datalake"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); @@ -355,12 +453,18 @@ class DatasetService { private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { - const {created_by, updated_by} = draftDataset; + const { created_by, updated_by } = draftDataset; const allFields = await tableGenerator.getAllFieldsHudi(draftDataset, "datalake"); const draftDatasource = this.createDraftDatasource(draftDataset, "datalake"); const dsId = _.join([draftDataset.dataset_id, "events", "datalake"], "_") const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record - const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); + let ingestionSpec = _.get(liveDatasource, "ingestion_spec"); + if (_.isEmpty(ingestionSpec)) { + ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); + } + else { + ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); + } _.set(draftDatasource, "ingestion_spec", ingestionSpec) _.set(draftDatasource, "created_by", created_by); _.set(draftDatasource, "updated_by", updated_by); @@ -372,7 +476,7 @@ class DatasetService { const datasource = _.join([draftDataset.dataset_id, "events"], "_") return { id: _.join([datasource, type], "_"), - datasource: draftDataset.dataset_id, + datasource: _.join([draftDataset.dataset_id, type], "_"), dataset_id: draftDataset.id, datasource_ref: datasource, type @@ -385,30 +489,130 @@ export const getLiveDatasetConfigs = async (dataset_id: string) => { const datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) - const connectorsV2 = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) - const connectorsV1 = await getV1Connectors(dataset_id) - const connectors = _.concat(connectorsV1, connectorsV2) + const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) + const updatedConnectorsPayload = getUpdatedV2ConnectorsPayload(connectors) if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations } if (!_.isEmpty(connectors)) { - datasetRecord["connectors_config"] = connectors + datasetRecord["connectors_config"] = updatedConnectorsPayload } return datasetRecord; } -export const getV1Connectors = async (datasetId: string) => { - const v1connectors = await datasetService.getConnectorsV1(datasetId, ["id", "connector_type", "connector_config"]); - const modifiedV1Connectors = _.map(v1connectors, (config) => { - return { - id: _.get(config, "id"), - connector_id: _.get(config, "connector_type"), - connector_config: _.get(config, "connector_config"), - version: "v1" - } - }) - return modifiedV1Connectors; +export const getUpdatedV2ConnectorsPayload = (connectors: Record) => { + return _.map(connectors, connector => ({ ...connector, "version": "v2" })) +} + +const storageTypes = JSON.parse(config.storage_types) +export const validateStorageSupport = (dataset: Record) => { + const { olap_store_enabled, lakehouse_enabled } = _.get(dataset, ["dataset_config", "indexing_config"]) || {} + const validStorageType = _.keys(storageTypes).filter(key => storageTypes[key] === true); + if (olap_store_enabled && !_.get(storageTypes, "realtime_store") === true) { + throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "realtime_store" is not available. Please use one of the available storage types: ${validStorageType}`, "BAD_REQUEST", 400) + } + if (lakehouse_enabled && !_.get(storageTypes, "lake_house") == true) { + throw obsrvError("", "DATASET_UNSUPPORTED_STORAGE_TYPE", `The storage type "lake_house" is not available. Please use one of the available storage types: ${validStorageType}`, "BAD_REQUEST", 400) + } } +export const attachDraftConnectors = async ( + draftDatasetList: Record[], + connectorFilter: string | string[] +): Promise[]> => { + if (_.isEmpty(draftDatasetList)) { + return []; + } + + const connectorIds = _.uniq( + _.flatMap(draftDatasetList, dataset => + _.map(dataset.connectors_config, 'connector_id') + ) + ); + + if (_.isEmpty(connectorIds)) { + return draftDatasetList.map(dataset => ({ + ...dataset, + connectors_config: [] + })); + } + + const connectorRegistry: any = await ConnectorRegistry.findAll({ + where: { id: connectorIds }, + raw: true, + attributes: ['id', 'name', 'category', 'type'] + }); + + return draftDatasetList.map(dataset => { + let filteredConnectors = dataset.connectors_config; + if (connectorFilter !== 'all') { + const filterArray = _.castArray(connectorFilter); // Ensure it's an array + filteredConnectors = _.filter(filteredConnectors, connector => + filterArray.includes(String(connector.connector_id)) + ); + } + + const enrichedConnectors = filteredConnectors.map((connector: any) => { + const registryDetails = _.find(connectorRegistry, { + id: connector.connector_id + }); + return { + ...connector, + name: registryDetails?.name || null, + category: registryDetails?.category || null, + source: registryDetails?.type || null + }; + }); + + return { + ...dataset, + connectors_config: enrichedConnectors + }; + }); +}; + +export const attachLiveConnectors = async ( + liveDatasetList: Record, + connectorFilter: string | string[] +): Promise[]> => { + if (_.isEmpty(liveDatasetList)) { + return []; + } + + ConnectorRegistry.hasMany(ConnectorInstances, { foreignKey: 'connector_id' }); + const connectorRegistry = await ConnectorRegistry.findAll({ + include: [{ + model: ConnectorInstances, + attributes: ['dataset_id', 'connector_id'], + required: true + }], + raw: true, + attributes: ['id', 'name', 'category', 'type'] + }); + + const filterArray = connectorFilter === 'all' ? null : _.castArray(connectorFilter); + + return liveDatasetList.map((dataset: Record) => { + const datasetId = dataset.dataset_id; + + const filteredConnectors = connectorRegistry.filter((connector: any) => + connector['connector_instances.dataset_id'] === datasetId && + (!filterArray || filterArray.includes(String(connector.id))) + ); + + const enrichedConnectors = filteredConnectors.map((connector: any) => ({ + connector_id: connector.id, + name: connector.name, + category: connector.category, + type: connector.type + })); + + return { + ...dataset, + connectors_config: enrichedConnectors + }; + }); +}; + export const datasetService = new DatasetService(); \ No newline at end of file diff --git a/api-service/src/services/DatasourceService.ts b/api-service/src/services/DatasourceService.ts index 812a5611..b5838900 100644 --- a/api-service/src/services/DatasourceService.ts +++ b/api-service/src/services/DatasourceService.ts @@ -1,16 +1,13 @@ import { Datasource } from "../models/Datasource"; -export const getDatasourceList = async (datasetId: string, raw = false) => { - const dataSource = await Datasource.findAll({ - where: { - dataset_id: datasetId, - }, - raw: raw - }); - return dataSource -} - - +export const getDatasourceList = async (datasetId?: string, raw: boolean = false) => { + const query: any = { raw }; + if (datasetId) { + query.where = { dataset_id: datasetId }; + } + + return Datasource.findAll(query); +}; diff --git a/api-service/src/services/SchemaGenerateService/SchemaHandler.ts b/api-service/src/services/SchemaGenerateService/SchemaHandler.ts index 45d76c9c..ac09027e 100644 --- a/api-service/src/services/SchemaGenerateService/SchemaHandler.ts +++ b/api-service/src/services/SchemaGenerateService/SchemaHandler.ts @@ -43,8 +43,11 @@ export class SchemaHandler { return _.set(schema, `${absolutePath}`, { ...schema[absolutePath], ...{ - type: resolution.value, - oneof: conflict.schema.values.map(key => ({ type: key })), + type: resolution.value || _.first(conflict.schema.values), + oneof: conflict.schema.values.map(key => { + const storeFormat = _.get(dataMappingPaths, key); + return { type: _.get(DataMappings, storeFormat) } + }), } }); } @@ -89,12 +92,13 @@ export class SchemaHandler { } private getArrivalFormat(schema: any, fieldData: any, property: any, type: string) { - const types = _.get(fieldData, type); - types && types.map((item: any) => { - const storeFormat = _.get(dataMappingPaths, item.type); + const types = _.get(fieldData, type) + const propType = _.get(fieldData, "type") + if(types){ + const storeFormat = _.get(dataMappingPaths, propType); _.set(schema, `${property}.arrival_format`, _.first(storeFormat.split("."))); _.set(schema, `${property}.data_type`, _.get(DataMappings, storeFormat)); - }) + } return; } diff --git a/api-service/src/services/SchemaGenerateService/SchemaMapping.json b/api-service/src/services/SchemaGenerateService/SchemaMapping.json index e9fe2a6f..f2461a05 100644 --- a/api-service/src/services/SchemaGenerateService/SchemaMapping.json +++ b/api-service/src/services/SchemaGenerateService/SchemaMapping.json @@ -48,7 +48,7 @@ "datasource": "long" }, "float": { - "jsonSchema": "number", + "jsonSchema": "float", "datasource": "double" }, "long": { @@ -56,7 +56,7 @@ "datasource": "long" }, "double": { - "jsonSchema": "number", + "jsonSchema": "double", "datasource": "double" }, "bigdecimal": { @@ -66,6 +66,10 @@ "epoch":{ "jsonSchema": "integer", "datasource": "long" + }, + "number":{ + "jsonSchema": "double", + "datasource": "double" } } }, diff --git a/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts b/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts index fcaba4a5..176ea74a 100644 --- a/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts +++ b/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts @@ -2,12 +2,22 @@ import _ from "lodash" import { Conflict, ConflictTypes, Suggestion, SuggestionsTemplate } from "../../types/SchemaModel" import constants from "./Constants.json" import { SchemaSuggestionTemplate } from "./Template" +import { dataMappingPaths } from "./SchemaHandler" +import DataMappings from "./SchemaMapping.json"; export class SuggestionTemplate { public createSuggestionTemplate(sample: ConflictTypes[]): SuggestionsTemplate[] { return _.map(sample, (value) => { - const dataTypeSuggestions = this.getSchemaMessageTemplate(value.schema) + let updatedConflicts: any = {} + if (value.schema.conflicts) { + updatedConflicts = _.mapKeys(value.schema.conflicts, (value, key) => { + const storeFormat = _.get(dataMappingPaths, key); + if (!storeFormat) return key + return _.get(DataMappings, storeFormat); + }) + } + const dataTypeSuggestions = this.getSchemaMessageTemplate({ ...value.schema, ...(!_.isEmpty(updatedConflicts) && { conflicts: updatedConflicts }) }) const requiredSuggestions = this.getRequiredMessageTemplate(value.required) const formatSuggestions = this.getPropertyFormatTemplate(value.formats) return { diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index 7030ad95..5fca1bc9 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -58,7 +58,7 @@ class BaseTableGenerator { const properties = this.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { _.set(prop, "name", _.join([denormField.denorm_out_field, prop.name], ".")); - _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); + _.set(prop, "expr", _.replace(prop.expr, "$", "$." + `['${denormField.denorm_out_field}']`)); return prop; }); dataFields.push(...transformProps); @@ -66,7 +66,7 @@ class BaseTableGenerator { } if (!_.isEmpty(transformations_config)) { const transformationFields = _.map(transformations_config, (tf) => ({ - expr: "$." + tf.field_key, + expr: "$." + tf.field_key.split('.').map((fieldpart: string) => `['${fieldpart}']`).join('.'), name: tf.field_key, data_type: tf.transformation_function.datatype, arrival_format: tf.transformation_function.datatype, @@ -171,6 +171,7 @@ class TableGenerator extends BaseTableGenerator { case "boolean": return "string"; case "array": return "json"; case "string": return "string"; + case "double": return "double"; default: return "auto"; } } @@ -224,7 +225,7 @@ class TableGenerator extends BaseTableGenerator { oldColumnSpec.push({ "type": col.type, "name": col.name, - "index": currIndex++ + "index": ++currIndex }) }) } diff --git a/api-service/src/services/WrapperService.ts b/api-service/src/services/WrapperService.ts index c38afb6d..543afd16 100644 --- a/api-service/src/services/WrapperService.ts +++ b/api-service/src/services/WrapperService.ts @@ -3,6 +3,7 @@ import { NextFunction, Request, Response } from "express"; import { config } from "../configs/Config"; import { ResponseHandler } from "../helpers/ResponseHandler"; import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; +import { druidHttpService } from "../connections/druidConnection"; class WrapperService { @@ -19,7 +20,7 @@ class WrapperService { try { // console.log("SQL Request to druid - \n" + JSON.stringify({"ts": Date.now(), body: req.body, headers: req.headers})); const authorization = req?.headers?.authorization; - const result = await axios.post( + const result = await druidHttpService.post( `${config.query_api.druid.host}:${config.query_api.druid.port}${config.query_api.druid.sql_query_path}`, req.body, { headers: { Authorization: authorization }, @@ -38,7 +39,7 @@ class WrapperService { // console.log("Native POST Request to druid - \n" + JSON.stringify({"ts": Date.now(), body: req.body, headers: req.headers, url: req.url})); const headers = req?.headers; const url = req?.url; - const result = await axios.post( + const result = await druidHttpService.post( `${config.query_api.druid.host}:${config.query_api.druid.port}${url}`, req.body, { headers, } ); @@ -55,7 +56,7 @@ class WrapperService { // console.log("Native DEL Request to druid - \n" + JSON.stringify({"ts": Date.now(), body: req.body, headers: req.headers, url: req.url})); const headers = req?.headers; const url = req?.url; - const result = await axios.delete( + const result = await druidHttpService.delete( `${config.query_api.druid.host}:${config.query_api.druid.port}${url}`, { headers, @@ -74,7 +75,7 @@ class WrapperService { // console.log("Native GET Request to druid - \n" + JSON.stringify({"ts": Date.now(), body: req.body, headers: req.headers, url: req.url})); const headers = req?.headers; const url = req?.url; - const result = await axios.get( + const result = await druidHttpService.get( `${config.query_api.druid.host}:${config.query_api.druid.port}${url}`, { headers, @@ -85,6 +86,22 @@ class WrapperService { } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false); } }; + public forwardNativeGetDatasource = async ( + req: Request, + res: Response, + next: NextFunction + ) => { + try { + const headers = req?.headers; + const url = req?.url; + const result = await druidHttpService.get( + `${config.query_api.druid.host}:${config.query_api.druid.port}${url}`, + { headers } + ); + ResponseHandler.flatResponse(req, res, result); + } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false); } + }; + public nativeStatus = async ( req: Request, res: Response, @@ -92,7 +109,7 @@ class WrapperService { ) => { try { // console.log("Native STATUS Request to druid - \n" + JSON.stringify({"ts": Date.now(), body: req.body, headers: req.headers, url: req.url})); - const result = await axios.get( + const result = await druidHttpService.get( `${config.query_api.druid.host}:${config.query_api.druid.port}/status` ); ResponseHandler.flatResponse(req, res, result); @@ -100,7 +117,7 @@ class WrapperService { }; public submitIngestion = async (ingestionSpec: object) => { - return await axios.post(`${config.query_api.druid.host}:${config.query_api.druid.port}/${config.query_api.druid.submit_ingestion}`, ingestionSpec) + return await druidHttpService.post(`${config.query_api.druid.host}:${config.query_api.druid.port}/${config.query_api.druid.submit_ingestion}`, ingestionSpec) } } diff --git a/api-service/src/services/managers/grafana/alert/helpers/index.ts b/api-service/src/services/managers/grafana/alert/helpers/index.ts index 9c577a52..463d70ca 100644 --- a/api-service/src/services/managers/grafana/alert/helpers/index.ts +++ b/api-service/src/services/managers/grafana/alert/helpers/index.ts @@ -16,8 +16,9 @@ const alerts = () => { return grafanaHttpClient.get("/api/prometheus/grafana/api/v1/rules"); }; -const deleteAlertRule = (alertCategory: string) => { - return grafanaHttpClient.delete(`/api/ruler/grafana/api/v1/rules/${alertCategory}/${alertCategory}`); +const deleteAlertRule = async (alertCategory: string) => { + const folderUid = await getFolderUid(alertCategory) + return grafanaHttpClient.delete(`/api/ruler/grafana/api/v1/rules/${folderUid}/${alertCategory}`); }; const getFilteredAlerts = () => { @@ -33,7 +34,7 @@ const deleteAlertFolder = async (folderName: string) => { const checkIfGroupNameExists = async (category: string) => { const response = await getRules(); const rules = _.get(response, "data"); - if(!_.has(rules, category)) return undefined; + if (!_.has(rules, category)) return undefined; return _.find(_.flatMap(_.values(rules)), { name: category, }); @@ -69,8 +70,9 @@ const updateMetadata = (metadata: any, dataSource: string, expression: string) = return metadata; }; -const addGrafanaRule = (payload: Record) => { - return grafanaHttpClient.post(`/api/ruler/grafana/api/v1/rules/${payload.name}`, payload); +const addGrafanaRule = async (payload: Record, category: string) => { + const folderUid = await getFolderUid(category) + return grafanaHttpClient.post(`/api/ruler/grafana/api/v1/rules/${folderUid}`, payload); }; const getFolders = () => { @@ -242,11 +244,27 @@ const getMatchingLabels = async (channels: string[]) => { } } +const getNotificationChannel = async (channels: string[]) => { + const fetchChannel = (id: string) => { + return Notification.findOne({ where: { id } }) + .then(response => response?.toJSON()) + .then(channelMetadata => { + const { name, type } = channelMetadata; + return name; + }) + .catch(() => null); + } + + const [name] = await Promise.all(channels.map(fetchChannel)); + return name; +} + const transformRule = async ({ value, condition, metadata, isGroup }: any) => { const { name, id, interval, category, frequency, labels = {}, annotations = {}, severity, description, notification = {} } = value; const annotationObj = { ...annotations, description: description }; const channels = _.get(notification, "channels") || []; const matchingLabelsForNotification = await getMatchingLabels(channels); + const channel = await getNotificationChannel(channels); const payload = { grafana_alert: { @@ -256,6 +274,7 @@ const transformRule = async ({ value, condition, metadata, isGroup }: any) => { exec_err_state: _.get(metadata, "exec_err_state", "Error"), data: metadata, is_paused: false, + ...(channel && { notification_settings: { receiver: channel } }) }, for: interval, annotations: annotationObj, @@ -281,7 +300,7 @@ const groupRulesByCategory = (payload: Record[]) => { const existing = _.get(accumulator, category) || []; accumulator[category] = [...existing, name]; return accumulator - }, {}); + }, {}); } export { diff --git a/api-service/src/services/managers/grafana/alert/index.ts b/api-service/src/services/managers/grafana/alert/index.ts index 471a104d..c0b5c9f7 100644 --- a/api-service/src/services/managers/grafana/alert/index.ts +++ b/api-service/src/services/managers/grafana/alert/index.ts @@ -22,7 +22,7 @@ const publishAlert = async (payload: Record) => { alertRulePayload = await transformRule({ ...transformPayload, isGroup: true }); } - return addGrafanaRule(alertRulePayload); + return addGrafanaRule(alertRulePayload, payload.category); }; const getAlerts = async (payload: Record) => { @@ -65,7 +65,7 @@ const deleteAlert = async (payload: Record) => { if (_.get(alertCategory, "rules.length") > 1) { const filteredRule = _.filter(alertCategory.rules, (rule) => _.get(rule, "grafana_alert.title") !== name) || []; const filteredGroup = { ...alertCategory, rules: filteredRule }; - return addGrafanaRule(filteredGroup); + return addGrafanaRule(filteredGroup, _.get(alertCategory, "name")); } await deleteAlertRule(category); @@ -103,7 +103,7 @@ const deleteSystemRules = async () => { await deleteAlertFolder(category); } else { const filteredGroup = { ...evaluationGroup, rules: filteredRules }; - await addGrafanaRule(filteredGroup); + await addGrafanaRule(filteredGroup, category); } } catch (err) { console.log(err) diff --git a/api-service/src/services/managers/grafana/notification/channels/email.ts b/api-service/src/services/managers/grafana/notification/channels/email.ts index 06658016..cffd4b49 100644 --- a/api-service/src/services/managers/grafana/notification/channels/email.ts +++ b/api-service/src/services/managers/grafana/notification/channels/email.ts @@ -18,7 +18,7 @@ const getReceiverObject = ({ name, recipientAddresses, message, multipleAddresse secureSettings: {}, type: type, name, - disableResolveMessage: false + disableResolveMessage: true } ] } diff --git a/api-service/src/services/managers/grafana/notification/templates/index.ts b/api-service/src/services/managers/grafana/notification/templates/index.ts index cf00f1c0..9cfdb613 100644 --- a/api-service/src/services/managers/grafana/notification/templates/index.ts +++ b/api-service/src/services/managers/grafana/notification/templates/index.ts @@ -1,5 +1,5 @@ const templates = { - "email": "{{ define \"email\" }}\n{{ range .Alerts.Firing }}\nObsrv - {{.Labels.alertname}}\n{{ end }}\n{{ end }}", + "email": "{{ define \"email\" }}\n{{ range .Alerts.Firing }}\n {{- $severity := .Labels.severity }}\n {{- if eq $severity \"critical\" }}\n 🚨 [{{ $severity | toUpper }}] {{ .Annotations.description }}\n {{- else if eq $severity \"warning\" }}\n ⚠️ [{{ $severity | toUpper }}] {{ .Annotations.description }}\n {{- else }}\n 🟢 [{{ $severity | toUpper }}] {{ .Annotations.description }}\n {{- end }}\n{{ end }}\n{{ end }}", "slack_title": "{{ define \"slack_title\" }}\n[{{ .Status | toUpper }}: {{ if eq .Status \"firing\" }}{{ .Alerts.Firing | len }}{{ else if eq .Status \"resolved\" }}{{.Alerts.Resolved | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join \" \" }}\n{{ end }}", "slack_body": "{{ define \"slack_body\" -}}\n{{- $status := \"resolved\" -}}\n{{ range $index, $alert := .Alerts }}\n{{ if eq $index 0 }}\n Severity: {{ .Labels.severity }}\n Description: {{ .Annotations.description }}\n Status: {{ .Status }}\n{{ end }}\n{{ end }}\n{{ end }}", "teams.alerts.title": "{{ define \"teams.alerts.title\" }}\n{{ .CommonLabels.alertname }}\n{{ end }}", diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts index 585b32b7..2fb29394 100644 --- a/api-service/src/services/managers/index.ts +++ b/api-service/src/services/managers/index.ts @@ -122,8 +122,8 @@ export const deleteSilence = async (payload: Record) => { export const deleteAlertByDataset = async (payload: Record) => { try { - const { name } = payload; - const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) + const { id } = payload; + const alertRulePayload = await Alert.findAll({ where: { "metadata.queryBuilderContext.subComponent": id }, raw: true }) if (!alertRulePayload) throw new Error(constants.ALERTS_NOT_FOUND) for (const payload of alertRulePayload) { await deleteAlertRule(payload, true) @@ -137,18 +137,17 @@ export const deleteAlertByDataset = async (payload: Record) => { export const deleteMetricAliasByDataset = async (payload: Record) => { try { - const { name } = payload; - await Metrics.destroy({ where: { component: "datasets", subComponent: name } }) + const { id } = payload; + await Metrics.destroy({ where: { subComponent: id } }) return constants.METRIC_ALIAS_DELETED_SUCCESSFULLY; } catch (error: any) { throw new Error(constants.METRIC_ALIAS_NOT_DELETED); } } -export const getAlertByDataset = async (payload: Record) => { +export const getAlertByDataset = async (dataset_id: string) => { try { - const { name } = payload; - const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) + const alertRulePayload = await Alert.findAll({ where: { "metadata.queryBuilderContext.subComponent": dataset_id }, raw: true }) if (!alertRulePayload) throw new Error(constants.ALERTS_NOT_FOUND) return alertRulePayload; } catch (error) { @@ -158,8 +157,8 @@ export const getAlertByDataset = async (payload: Record) => { export const getAlertMetricsByDataset = async (payload: Record) => { try { - const { name } = payload; - const metricAliasPayload = await Metrics.findAll({ where: { component: "datasets", subComponent: name }, raw: true }) + const { id } = payload; + const metricAliasPayload = await Metrics.findAll({ where: { subComponent: id }, raw: true }) if (!metricAliasPayload) throw new Error(constants.METRIC_ALIAS_NOT_FOUND) return metricAliasPayload; } catch (error: any) { diff --git a/api-service/src/services/otel/OTelService.ts b/api-service/src/services/otel/OTelService.ts new file mode 100644 index 00000000..65186cbd --- /dev/null +++ b/api-service/src/services/otel/OTelService.ts @@ -0,0 +1,158 @@ +import { Counter, diag, DiagConsoleLogger, DiagLogLevel, Meter, metrics } from '@opentelemetry/api'; +import * as logsAPI from '@opentelemetry/api-logs'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { Resource } from '@opentelemetry/resources'; +import { BatchLogRecordProcessor, LoggerProvider } from '@opentelemetry/sdk-logs'; +import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import logger from '../../logger'; +import * as _ from "lodash"; +import { config } from "../../configs/Config"; +const collectorEndpoint = _.get(config, "otel.collector_endpoint", "http://localhost:4318"); + +export class OTelService { + private static meterProvider: MeterProvider; + private static loggerProvider: LoggerProvider; + private static tracerProvider: NodeTracerProvider; + + public static init() { + this.tracerProvider = this.createTracerProvider(collectorEndpoint); + this.meterProvider = this.createMeterProvider(collectorEndpoint); + this.loggerProvider = this.createLoggerProvider(collectorEndpoint); + + // Register the global tracer, meter, and logger providers + this.tracerProvider.register(); + this.setGlobalMeterProvider(this.meterProvider); + + logger.info("OpenTelemetry Service Initialized"); + + // Add shutdown hook + process.on('SIGTERM', async () => { + await this.tracerProvider.shutdown(); + await this.meterProvider.shutdown(); + await this.loggerProvider.shutdown(); + }); + } + + private static createTracerProvider(endpoint: string) { + const traceExporter = new OTLPTraceExporter({ + url: `${endpoint}/v1/traces`, + }); + + const tracerProvider = new NodeTracerProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + tracerProvider.addSpanProcessor(new BatchSpanProcessor(traceExporter)); + + return tracerProvider; + } + + private static createMeterProvider(endpoint: string) { + const metricExporter = new OTLPMetricExporter({ + url: `${endpoint}/v1/metrics`, + }); + + const meterProvider = new MeterProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + meterProvider.addMetricReader( + new PeriodicExportingMetricReader({ + exporter: metricExporter, + exportIntervalMillis: 10000, + }) + ); + + return meterProvider; + } + + private static createLoggerProvider(endpoint: string) { + const logExporter = new OTLPLogExporter({ + url: `${endpoint}/v1/logs`, + }); + + const loggerProvider = new LoggerProvider({ + resource: this.createServiceResource('obsrv-api-service'), + }); + + loggerProvider.addLogRecordProcessor( + new BatchLogRecordProcessor(logExporter) + ); + + return loggerProvider; + } + + // Helper method to create a Resource with service name + private static createServiceResource(serviceName: string) { + return new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: serviceName, + }); + } + + private static setGlobalMeterProvider(meterProvider: MeterProvider) { + diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO); + diag.info('Registering MeterProvider globally.'); + metrics.setGlobalMeterProvider(meterProvider); + } + + // Method to create a counter metric + public static createCounterMetric(name: string): Counter { + const meter = this.getMeterProvider(); // Use the updated getMeterProvider method + const counter = meter.createCounter(name, { + description: 'Counts the number of API calls', + }); + return counter; + } + + public static getMeterProvider(): Meter { + return this.meterProvider.getMeter('obsrv-api-service'); + } + + public static getLoggerProvider(): LoggerProvider { + return this.loggerProvider; + } + + public static getTracerProvider(): NodeTracerProvider { + return this.tracerProvider; + } + + // Method to record the counter metric + public static recordCounter(counter: Counter, value: number) { + counter.add(value, { + service: 'obsrv-api-service', + }); + } + + + public static generateOTelLog(auditLog: Record, severity: 'INFO' | 'WARN' | 'ERROR', logType?: string) { + if((config.otel && _.toLower(config?.otel?.enable) === "true")){ + const loggerInstance = this.loggerProvider.getLogger('obsrv-api-service'); + + const severityMapping: Record = { + INFO: logsAPI.SeverityNumber.INFO, + WARN: logsAPI.SeverityNumber.WARN, + ERROR: logsAPI.SeverityNumber.ERROR, + }; + + const severityNumber = severityMapping[severity] || logsAPI.SeverityNumber.INFO; + + const logRecord = { + severityNumber, + severityText: severity, + body: JSON.stringify(auditLog), + attributes: { + 'log.type': logType || 'console', + ...auditLog, + }, + }; + loggerInstance.emit(logRecord); + } + } + + +} diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index 8408dfb3..06c4ae15 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -3,13 +3,19 @@ import { v4 } from "uuid"; import _ from "lodash"; import { config as appConfig } from "../configs/Config"; import {send} from "../connections/kafkaConnection" +import { OTelService } from "./otel/OTelService"; +import { Parser } from "node-sql-parser"; +import { datasetService } from "./DatasetService"; +import { result_data } from "../controllers/QueryWrapper/SqlQueryWrapper"; +import { query_data } from "../controllers/DataOut/DataOutController"; +import logger from "../logger"; const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); +const parser = new Parser(); export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } - const getDefaults = (userID:any) => { return { eid: "AUDIT", @@ -50,7 +56,8 @@ const getDefaultEdata = ({ action }: any) => ({ }) const sendTelemetryEvents = async (event: Record) => { - send({ messages: [{ value: JSON.stringify(event) }] }, telemetryTopic).catch(console.log); + OTelService.generateOTelLog(event, 'INFO', 'audit-log'); + send(event, telemetryTopic).catch(console.log); } const transformProps = (body: Record) => { @@ -160,6 +167,20 @@ export const updateTelemetryAuditEvent = ({ currentRecord, request, object = {} } } +export const getAlias = (response: Response, ast: any) => { + const aliasMap: Record = {}; + + _.map(_.get(ast, "columns"), data => { + const columnName = data.expr.column || data.expr.value || data.expr.args?.expr.column || data.expr.args?.expr.value; + const alias = _.get(data, "as"); + + if (alias && columnName) { + aliasMap[alias] = columnName; + } + }); + return aliasMap; +} + export const findAndSetExistingRecord = async ({ dbConnector, table, filters, request, object = {} }: Record) => { const auditEvent = request?.auditEvent; if (dbConnector && table && filters && _.get(auditEvent, "type") === "update") { @@ -181,3 +202,232 @@ export const findAndSetExistingRecord = async ({ dbConnector, table, filters, re } } } + +export const getDefaultLog = (actionType: any, userID: any ) => { + return { + eid: "LOG", + ets: Date.now(), + ver: "1.0.0", + mid: v4(), + actor: { + id: userID, + type: "User" + }, + context:{ + pdata:{ + id : `${env}.api.service`, + ver: `${version}` + } + }, + sid: v4(), + edata:{} + } +} + +const setLogEventType = (operationType: any, request: any) => { + switch (operationType) { + case OperationType.CREATE: { + _.set(request, "logEvent.type", "create"); + break; + } + case OperationType.LIST: { + _.set(request, "logEvent.type", "list"); + break; + } + case OperationType.GET: { + _.set(request, "logEvent.type", "get"); + break; + } + default: + break; + } +} + +export const telemetryLogStart = ({ operationType, action }: any) =>{ + return async ( request: any, response: Response, next: NextFunction) => { + try{ + const user_id = (request as any)?.userID; + request.logEvent = getDefaultLog(action, user_id); + setLogEventType( operationType, request); + } catch (error) { + console.log(error) + } finally { + next(); + } + } +} + +export const getTable = (ast: any) => { + const table = _.map(_.get(ast, "from") , data=> data.table); + return table[0]; +} + +export const getColumn = (columns: any) => { + const column = _.map(columns, data => data.expr.column); + return column.filter(value => value !== undefined); +} + +export const getFilters = (whereClause: any) => { + const conditions: any = []; + _.cloneDeepWith(whereClause, (obj) => { + if (obj && obj.type === 'binary_expr') { + const data = { + column: obj.left.column || obj.left.value, + value: getFilterValue(obj.right.value), + operator: obj.operator + }; + if(obj.left.column || obj.left.value) { + conditions.push(data); + } + } + }); + return conditions; +} + +export const getFilterValue = (data: any) => { + let values = _.map(data, value => value.value); + if (values.filter(Boolean).length != 0) { + return values; + } + return data; +} + +export const setLogResponse = (telemetryLogEvent: any, request: Request, response: Response, ast: any) => { + const logEvent: any = _.get(request, "logEvent") || {}; + const size = response.getHeaders()["content-length"]; + const result_value = _.get(result_data, "data"); + const query_value = _.get(query_data, "data"); + const result: any = (!_.isEmpty(result_value) ? result_value : (!_.isEmpty(query_value) ? query_value : undefined)); + _.set(telemetryLogEvent, "edata.query_metadata.response.size", !isNaN(Number(size)) ? Number(size) : size); + _.set(telemetryLogEvent, "edata.query_metadata.response.duration", Date.now() - logEvent.ets); + const telemetryLog = JSON.parse(appConfig.telemetry_log); + if (telemetryLog.response_data) { + const responseData = !_.isEmpty(result) ? getResponseData(result, ast, response) : []; + _.set(telemetryLogEvent, "edata.query_metadata.response.data", responseData); + } +} + +export const getMetrics = (columns: any) => { + const metrics: any = []; + _.map(columns, data => { + if(! data.expr.args?.expr) return; + metrics.push({ + name: data.expr.name, + column: data.expr.args.expr.column || data.expr.args.expr.value + }); + }); + return metrics; +} + +export const getSelectedColumns = (ast: any) => { + const columns = (_.get(ast, "columns"))?.map((col: { expr: { value: string; column: string; args: any } }) => { + return col.expr.column || col.expr.value || col.expr.args?.expr.column || col.expr.args?.expr.value; + }); + return columns.filter((value: any) => value !== undefined); +} + +export const getMetricData = ( ast: any, data: any) => { + const metrics = getMetrics(_.get(ast, "columns")); + const alias = getAlias(data, ast); + const keys = Object.keys(data); + const metric_key: any = _.map(keys, key => { + if (key.startsWith("EXPR") || (key in alias)) { + return key; + } + }).filter(value => value !== undefined) + const metricData = metric_key.reduce((acc: any, key: any, index: any) => { + if (alias && key in alias) { + key = alias[key]; + } + acc[key] = metrics[index]; + return acc; + }, {} as Record); + return metricData; +} + +export const getDatasetId = async (data: any) => { + const data_response = await datasetService.getDatasetIdWithDatasource(data, ["dataset_id"]); + return data_response.dataset_id; +} + +export const setLogEdata = async ( request: Request, response: Response ) => { + try{ + const {edata = {}}: any = _.get(request, "logEvent") || {}; + const userID = (request as any)?.userID || "SYSTEM"; + const telemetryLogEvent: any = getDefaultLog(edata.action,userID); + const query = request.body?.query || request.body.querySql?.query || null; + if (!query || typeof query !== "string") { + throw new Error("Invalid or missing SQL query"); + } + let ast: any; + try { + ast = parser.astify(query); + } catch (error) { + logger.error("SQL parsing failed", { query, error }); + throw new Error("Invalid SQL query"); + } + const table = getTable(ast); + _.set(telemetryLogEvent, "edata", edata); + const dataset_id = await getDatasetId(table); + _.set(telemetryLogEvent, "edata.dataset_id", dataset_id); + _.set(telemetryLogEvent, "edata.query_metadata.table", table); + const columns: any = _.get(ast, "columns"); + _.set(telemetryLogEvent, "edata.query_metadata.dimensions", getColumn(columns)); + _.set(telemetryLogEvent, "edata.query_metadata.metrics", getMetrics(columns) || []); + _.set(telemetryLogEvent, "edata.query_metadata.filters", getFilters(_.get(ast, "where"))); + setLogResponse(telemetryLogEvent, request, response, ast); + return telemetryLogEvent + } + catch (error) { + console.log(error); + } +} + +export const processLogEvents = async (request: Request, response: Response) => { + const telemetryLogEvent: any = await setLogEdata(request, response); + sendTelemetryEvents(telemetryLogEvent); +} + +export const interceptLogEvents = () => { + return (request: Request, response: Response, next: NextFunction) => { + response.on("finish", () => { + JSON.parse(appConfig.telemetry_log).enable && _.get(request, "logEvent") && processLogEvents(request, response); + }); + next(); + } +} + +export const getResponseData = (data: any, ast: any, response: Response) => { + const result: any = { aggregates: {}, values: {} }; + if(!data) { + return result; + } + const alias = getAlias(response, ast); + const metricData = getMetricData(ast, data[0]); + _.forEach(data, (item: any) => { + Object.entries(item).forEach(([key, value]: any) => { + let finalKey = key; + if (finalKey.startsWith("EXPR")) { + finalKey = _.get(metricData, `${finalKey}.column`) || finalKey; + } + finalKey = alias?.[key] || key; + + const metric = metricData[finalKey]; + + if (metric && metric.name) { + const column = metric.column || finalKey; + if (!_.has(result.aggregates, column)) { + result.aggregates[column] = []; + } + result.aggregates[column].push(!isNaN(Number(value)) ? Number(value) : value); + } else { + if (!_.has(result.values, finalKey)) { + result.values[finalKey] = []; + } + result.values[finalKey].push(!isNaN(Number(value)) ? Number(value) : value); + } + }); + }); + + return result; +} \ No newline at end of file diff --git a/api-service/src/telemetry/telemetryActions.ts b/api-service/src/telemetry/telemetryActions.ts index befe6100..78a9ced5 100644 --- a/api-service/src/telemetry/telemetryActions.ts +++ b/api-service/src/telemetry/telemetryActions.ts @@ -21,4 +21,29 @@ export default { "copyDataset": "dataset:copy", "readConnectors": "connectors:read", "listConnectors": "connectors:list", + "importDataset": "dataset:import", + "exportDataset": "dataset:export", + "registerConnector": "connectors:create", + "createDataschema": "dataschema:create", + "dataQuery": "data:query:sql", + "datasetAlias": "dataset:alias:create", + "datasetHealth": "dataset.health:get", + "datasetMetrix": "dataset.metrix:get", + "dataAnalyzePii": "data.analyze.pii:create", + "datasetReset": "dataset.reset", + "generateURL": "files.url:create", + "schemaValidate": "validate:schema:get", + "alertCreate": "alerts:create", + "alertSearch": "alert.search:get", + "updateAlertHandler": "alert.handler:update", + "deleteAlertHandler": "alert.handler:delete", + "alertHandlerPublish": "alert.heandler:create", + "alertHandlerDetails": "alert.handler:get", + "createMetricHandler": "metric.handler:create", + "deleteMetricHandler": "metric.handler:delete", + "updateMetricHandler": "metric.handler:update", + "deleteMultipleMetricHandler": "multiple.metric.handler:delete", + "createNotifications": "notifications:create", + "updateNotifications": "notifications:update", + "deleteNotifications": "notifications:delete" } \ No newline at end of file diff --git a/api-service/swagger-doc/openapi_v2.yml b/api-service/swagger-doc/openapi_v2.yml index d4cd25b5..1fcdf368 100644 --- a/api-service/swagger-doc/openapi_v2.yml +++ b/api-service/swagger-doc/openapi_v2.yml @@ -1,36 +1,38 @@ openapi: 3.0.0 info: - title: Obsrv API V2 + title: Obsrv v2 apis description: >- Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to analyze different types of events, as well as to manage data sources and datasets. version: 1.0.0 servers: - - url: ":3000" + - url: http://localhost:3000 + - url: http://{{host_ip}} tags: - - name: Dataset CRUD APIs + - name: Dataset API's description: >- The Dataset APIs facilitate efficient management of datasets by enabling users to create, read, and update dataset records, along with the capability to list multiple records based on specific criteria. + - name: Connector API's - name: Data Ingest - - name: Data query - - name: Data exhaust + - name: Data Query - name: Query Templates - - name: Dataset copy - - name: Schema validator + - name: Alert Notification Channels + - name: Alert Silence + - name: Alerts Wrapper + - name: Alert Metric Alias paths: /v2/datasets/create: post: tags: - - Dataset CRUD APIs - summary: Dataset create + - Dataset API's + summary: Create Dataset description: >- This API allows you to create new datasets used by the analytical data source. requestBody: - required: true content: application/json: schema: @@ -42,8 +44,8 @@ paths: params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - dataset_id: telemetry_record - type: dataset + dataset_id: telemetry_record-t4 + type: event name: sb-telemetry validation_config: validate: true @@ -78,148 +80,418 @@ paths: denorm_fields: - denorm_key: eid denorm_out_field: userdata + dataset_id: master-telemetry transformations_config: - - field_key: eid + - field_key: email transformation_function: type: mask - expr: eid - condition: null + expr: mid + datatype: string + category: pii mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - dataset_config: - data_key: mid - timestamp_key: ets - files_upload_path: ["telemetry.json"] tags: - tag1 - - tag2 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '287' + ETag: + schema: + type: string + example: W/"11f-uBTr0zBIIFpz/sdLJx6WQf0rAbQ" + Date: + schema: + type: string + example: Mon, 15 Jul 2024 13:14:09 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Dataset Success + summary: 'Success: Dataset created successfullly' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T17:56:06+05:30' + ts: '2024-07-15T18:44:08+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 78hj80j9-d61e-4d4f-be78-181834eeff6d + resmsgid: 276c042c-0f23-4b26-9b10-6fe48bbc2d3a responseCode: OK result: - id: sb-telemetry.1 - version_key: '1713442037275' + id: telemetry_record-t4 + version_key: '1721049248930' example-1: - summary: Master Dataset Success + summary: 'Success: Master dataset created successfully' + value: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T08:36:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 845076be-d9e7-4246-bb8e-07ae0ce59d1e + responseCode: OK + result: + id: telemetry_record-master + version_key: '1721099200603' + example-2: + summary: 'Success: Minimal request body' + value: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T18:14:59+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 9e207f4f-2be6-4a45-ab78-213bea272ae0 + responseCode: OK + result: + id: telemetry_events + version_key: '1721133899306' + example-3: + summary: 'Success: Dataset created successfully with all fields' value: id: api.datasets.create ver: v2 - ts: '2024-04-16T17:56:06+05:30' + ts: '2024-07-17T18:19:53+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 78hj80j9-d61e-4d4f-be78-181834eeff6d + resmsgid: 505fb3bc-ae32-4f5b-a931-adec4d1d84ba responseCode: OK result: - id: sb-telemetry-master.1 - version_key: '1713442037275' + id: telemetry_record-t41 + version_key: '1721220593027' '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '362' + ETag: + schema: + type: string + example: W/"16a-Jn1DYy5EYoYF/Syd3f9LOvOK0lI" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:09:00 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.create + ver: v2 + ts: '2024-07-16T08:39:00+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a07de860-dcbc-4ff6-822e-34b47635c8a3 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'dataset_id' + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '337' + ETag: + schema: + type: string + example: W/"151-a7dJ9XBUyT3AXNxl1TPcraxMX08" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:07:28 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: Dataset contains duplicate denorm out field' + summary: 'Failure: Master dataset already exists' value: - id: api.datasets.create + id: api.files.generate-url ver: v2 - ts: '2024-04-16T17:59:06+05:30' + ts: '2024-07-16T08:37:28+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 54c5b692-dc37-432e-b556-7f396d7c9e07 - responseCode: BAD_REQUEST + resmsgid: 138b796b-1b68-481a-a59d-1cb695c1adc9 + responseCode: CONFLICT + result: {} error: - code: DATASET_DUPLICATE_DENORM_KEY - message: Duplicate denorm key found - trace: '' + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-master example-1: - summary: 'Failure: Invalid request payload provided' + summary: 'Failure: Dataset already exists' value: - id: api.datasets.create + id: api.files.generate-url ver: v2 - ts: '2024-04-16T18:00:34+05:30' + ts: '2024-07-16T08:38:05+05:30' params: status: FAILED - resmsgid: 615c1e4c-8c19-44fd-b29c-c235e7cbb5f0 - responseCode: BAD_REQUEST + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bf62693c-3aa4-42ce-a5ea-4bde340740f5 + responseCode: CONFLICT + result: {} error: - code: DATASET_INVALID_INPUT - message: >- - #additionalProperties should NOT have additional - properties - trace: '' - '409': - description: Conflict + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-t4 + /v2/files/generate-url: + post: + tags: + - Dataset API's + summary: Generate Presigned URLs + description: This API generates presigned URLs to upload or download files from cloud + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.files.generate-url + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: write + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '1344' + ETag: + schema: + type: string + example: W/"540-790rZel+H/rDwgvZRxvlUmZ8Gpc" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 02:56:19 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.create - ver: v2 - ts: '2024-04-16T17:56:06+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 76612ad1-241b-4ce3-8af4-88db860697f4 - responseCode: CONFLICT - error: - code: DATASET_EXISTS - message: Dataset already exists - trace: '' - '500': - description: Internal Server Error + examples: + example-0: + summary: 'Success: Generate put url' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T08:26:19+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5306f309-4a15-458e-89e2-29d8ac0835d4 + responseCode: OK + result: + - filePath: >- + test-connector/api-service/user_uploads/telemetry_10d595.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject + - filePath: >- + test-connector/api-service/user_uploads/school_data_33109a.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject + example-1: + summary: 'Success: Generate get url' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T09:31:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 009c0b2d-8acd-40b0-a807-bbacf9242771 + responseCode: OK + result: + - filePath: test-connector/api-service/user_uploads/telemetry.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject + - filePath: >- + test-connector/api-service/user_uploads/school_data.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '355' + ETag: + schema: + type: string + example: W/"163-9oQYJJEaBH3mJAnzDHXn2MxE848" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 03:03:04 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.create - ver: v2 - ts: '2024-04-16T18:02:44+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: dd1c0e11-fb4c-484c-81fe-82c9e2eee053 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_CREATION_FAILURE - message: Failed to create dataset - trace: '' + examples: + example-0: + summary: 'Failure: limit exceeds' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T08:33:04+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d3a606ca-47d0-4746-95a1-c8692e749959 + responseCode: BAD_REQUEST + error: + code: FILES_URL_GENERATION_LIMIT_EXCEED + message: 'Pre-signed URL generation failed: limit exceeded.' + trace: '' + example-1: + summary: 'Failure: Invalid request' + value: + id: api.files.generate-url + ver: v2 + ts: '2024-07-16T09:31:10+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: c3e9da1c-09f3-4a3b-84ec-a19efc68b856 + responseCode: BAD_REQUEST + error: + code: FILES_GENERATE_URL_INPUT_INVALID + message: >- + #properties/request/properties/access/enum must be equal + to one of the allowed values + trace: '' /v2/datasets/update: patch: tags: - - Dataset CRUD APIs - summary: Dataset update + - Dataset API's + summary: Update Dataset description: >- This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or - update transformations + update transformations and connectors requestBody: - required: true content: application/json: schema: @@ -231,8 +503,8 @@ paths: params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - dataset_id: telemetry_record - version_key: '1717073955640' + dataset_id: telemetry_record-t4 + version_key: '1721135455988' name: sb-telemetry validation_config: validate: true @@ -242,279 +514,254 @@ paths: extraction_key: events dedup_config: drop_duplicates: true - dedup_key: id + dedup_key: ipid dedup_config: drop_duplicates: true - dedup_key: eid + dedup_key: mid data_schema: $schema: https://json-schema.org/draft/2020-12/schema type: object properties: - eid: + midpid: type: string - ets: + arrival_format: text + data_type: string + miduwi: + type: integer + arrival_format: number + data_type: epoch + mid: + type: string + arrival_format: text + data_type: string + sid: type: string - required: - - eid + arrival_format: text + data_type: string additionalProperties: true denorm_config: denorm_fields: - - values: + - value: denorm_key: eid denorm_out_field: userdata - dataset_id: master-telemetry action: remove - - values: + - value: denorm_key: eid denorm_out_field: edata - dataset_id: trip-record - action: add - transformation_config: - - values: - field_key: eid + dataset_id: trip-details + action: upsert + transformations_config: + - value: + field_key: email transformation_function: type: mask - expr: eid - condition: null + expr: mid + datatype: string + category: pii + mode: Strict + action: upsert + - value: + field_key: email_id + transformation_function: + type: mask + expr: mid + datatype: string + category: pii mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - action: add - dataset_config: - data_key: eid - timestamp_key: ets - tags: - - values: - - tag1 - - tag2 action: remove - - values: - - tag3 - - tag4 - action: add + tags: [] + connectors_config: + - value: + connector_id: jdbc + connector_config: + source_database_type: postgresql + source_database_host: postgresql-hl.postgresql.svc.cluster.local.master + source_database_port: 5432 + source_database_name: obsrv_sample_datasets_1 + source_database_username: postgres + source_database_pwd: postgres + table: new_york_taxi_data + timestamp-column: tpep_pickup_datetime + batch-size: 100 + max-batches: 2 + operations_config: + polling_interval: periodic + schedule: twice + action: upsert + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '331' + ETag: + schema: + type: string + example: W/"14b-fNmMHDpT4Ka5pwuzbYvZo7jECEo" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 13:00:45 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - example: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T00:16:13+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 4b2c0c22-f765-46f9-80af-33d0442db5ca - responseCode: OK - result: - message: Dataset is updated successfully - id: sb-telemetry.1 - version_key: '1713465973004' + examples: + example-0: + summary: 'Success: Minimal dataset update' + value: + id: api.datasets.update + ver: v2 + ts: '2024-07-16T18:30:45+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 354f1fec-0c39-42ee-a52a-49552f847c11 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134845559' + example-1: + summary: 'Success: Updated successfully' + value: + id: api.datasets.update + ver: v2 + ts: '2024-07-16T18:27:55+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 6d835f07-aed5-4e8b-81c2-2142cfb55c52 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134675878' '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '369' + ETag: + schema: + type: string + example: W/"171-iNJoyWUecOEsXbHZwx6rld3Sr1I" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:59:21 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: Invalid payload provided' + summary: 'Failure: Invalid request' value: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:22:12+05:30' + ts: '2024-07-16T18:29:21+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9408e137-ada8-48b8-99e9-90d5cdd35e35 + resmsgid: 7d31672b-e5c3-4a6d-afac-d9d78011bcde responseCode: BAD_REQUEST + result: {} error: code: DATASET_UPDATE_INPUT_INVALID message: >- - #properties/request/properties/name/type should be - string - trace: '' + #properties/request/required must have required property + 'dataset_id' example-1: - summary: 'Failure: No field provided along with dataset_id' + summary: 'Failure: No fields are provided to update' value: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:23:06+05:30' + ts: '2024-07-16T18:32:44+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 347f465f-f100-4095-9801-38c4943380c0 + resmsgid: bf99b1e1-7694-4be0-ba5d-e347764736de responseCode: BAD_REQUEST + result: {} error: code: DATASET_UPDATE_NO_FIELDS message: >- Provide atleast one field in addition to the dataset_id to update the dataset - trace: '' - example-2: - summary: 'Failure: Cannot update as dataset not in draft state' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:25:57+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: a9dfc926-c893-4ec3-82b8-bc8601d928a9 - responseCode: BAD_REQUEST - error: - code: DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE - message: Dataset cannot be updated as it is not in draft state - trace: '' - example-3: - summary: 'Failure: Dataset contains duplicate denorm out field' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:29:25+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 865b9bef-9b6d-467c-9065-6689126b0f42 - responseCode: BAD_REQUEST - error: - code: DATASET_DUPLICATE_DENORM_KEY - message: Dataset contains duplicate denorm out keys:[userdata] - trace: '' - example-4: - summary: 'Failure: Dataset tags to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:32:05+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9e9ff0bc-d660-4c4e-a2b0-cb11246a6961 - responseCode: BAD_REQUEST - error: - code: DATASET_TAGS_EXISTS - message: Dataset tags already exist - trace: '' - example-5: - summary: 'Failure: Dataset transformations to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:34:46+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: e0a5cc26-032e-4be8-81ef-c7e76867c9c1 - responseCode: BAD_REQUEST - error: - code: DATASET_TRANSFORMATIONS_EXIST - message: Dataset transformations already exists - trace: '' - example-6: - summary: 'Failure: Dataset denorm fields to add already exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:38:57+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 0b807580-a2b5-4a44-adc4-80b35b4b6529 - responseCode: BAD_REQUEST - error: - code: DATASET_DENORM_EXISTS - message: Denorm fields already exist - trace: '' '404': description: Not Found - content: - application/json: + headers: + X-Powered-By: schema: - type: object - examples: - example-0: - summary: 'Failure: Dataset does not exists to update' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:20:39+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ebcf0a62-7c95-40f5-adf2-87ab90a40a80 - responseCode: NOT_FOUND - error: - code: DATASET_NOT_EXISTS - message: Dataset does not exists to update - trace: '' - example-1: - summary: 'Failure: Dataset tags to remove do not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:33:33+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: c29e7d0f-3eaf-4972-8804-c172db7b6f13 - responseCode: NOT_FOUND - error: - code: DATASET_TAGS_DO_NOT_EXIST - message: Dataset tags do not exist to remove - trace: '' - example-2: - summary: 'Failure: Dataset transformations to update do not exists' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:36:04+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 0b5f515b-1958-4597-b603-b4038d4e5846 - responseCode: NOT_FOUND - error: - code: DATASET_TRANSFORMATIONS_DO_NOT_EXIST - message: Dataset transformations do not exist to update - trace: '' - example-3: - summary: 'Failure: Dataset transformations to remove do not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:37:20+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 69d32411-4252-4b7f-8afa-6d436928c3d3 - responseCode: NOT_FOUND - error: - code: DATASET_TRANSFORMATIONS_DO_NOT_EXIST - message: Dataset transformations do not exist to remove - trace: '' - example-4: - summary: 'Failure: Dataset denorm to remove does not exist' - value: - id: api.datasets.update - ver: v2 - ts: '2024-04-19T12:40:07+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ce78e393-2b74-46da-a70b-2abd06ad3270 - responseCode: NOT_FOUND - error: - code: DATASET_DENORM_DO_NOT_EXIST - message: Denorm fields do not exist to remove - trace: '' - '409': - description: Conflict + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '340' + ETag: + schema: + type: string + example: W/"154-4I5VyTBINyYBZZM8Ge9Cnqz2xBY" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:58:30 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -522,20 +769,47 @@ paths: example: id: api.datasets.update ver: v2 - ts: '2024-04-24T12:30:18+05:30' + ts: '2024-07-16T18:28:30+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: fe4a438d-441c-4497-a3ec-a0cee2058b48 - responseCode: CONFLICT + resmsgid: bf64703c-bb6b-41bf-bc1a-c85373efd925 + responseCode: NOT_FOUND + result: {} error: - code: DATASET_OUTDATED - message: >- - The dataset is outdated. Please try to fetch latest changes - of the dataset and perform the updates - trace: '' - '500': - description: Internal Server Error + code: DATASET_NOT_EXISTS + message: Dataset does not exists with id:telemetry_record-t41 + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '383' + ETag: + schema: + type: string + example: W/"17f-JnlFVLXyuhwx9KbxYWDRB4mmvVw" + Date: + schema: + type: string + example: Tue, 16 Jul 2024 12:53:16 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -543,578 +817,230 @@ paths: example: id: api.datasets.update ver: v2 - ts: '2024-04-19T12:41:33+05:30' + ts: '2024-07-16T18:23:16+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 451daa85-9cb8-459a-b681-a256c6e912f0 - responseCode: INTERNAL_SERVER_ERROR + resmsgid: 02fe03f6-c4c4-48f6-9d84-a32cd52f4c13 + responseCode: CONFLICT + result: {} error: - code: DATASET_UPDATE_FAILURE - message: Failed to update dataset - trace: '' + code: DATASET_OUTDATED + message: >- + The dataset is outdated. Please try to fetch latest changes + of the dataset and perform the updates /v2/datasets/read/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset get: tags: - - Dataset CRUD APIs - summary: Dataset read + - Dataset API's + summary: Read Dataset description: >- This API allows you to read dataset from the requested dataset_id. User can request for the specific fields and status of the dataset through the request params. By default, the API returns the dataset of status - "Live". - This API accepts the parameter mode=edit to read the draft dataset. If a draft dataset is not found, - it creates one using the live dataset and returns the dataset details. -
-
- Valid fields that user can request are - "dataset_id,id,name,type,validation_config,extraction_config,dedup_config,data_schema,router_config,denorm_config,transformation_config,dataset_config,tags,status,version,created_by,updated_by,created_date,updated_date,published_date" + "Live". This API accepts the parameter mode=edit to read the draft + dataset. If a draft dataset is not found, it creates one using the live + dataset and returns the dataset details. parameters: - - name: dataset_id - example: sb_telemetry - in: path - required: true - schema: - type: string - - name: status - example: Draft - in: query - required: false - schema: - type: string - - name: fields - example: name - in: query - required: false + - name: Cookie + in: header schema: type: string + example: >- + connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4 - name: mode - example: edit in: query - required: false schema: type: string + example: edit responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '501' + ETag: + schema: + type: string + example: W/"1f5-p+b/6r0nHRFhgr5+URzxk4d/CSg" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:08:55 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Read Draft dataset + summary: 'Success: Read live dataset' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:43:58+05:30' + ts: '2024-07-17T17:38:55+05:30' params: status: SUCCESS - resmsgid: d02c643a-d51d-4c67-8a42-9f43e61f459e + resmsgid: 8c8a2852-54bc-43fb-b063-7f359d11930a responseCode: OK result: - id: sb-telemetry.1 - dataset_id: sb-telemetry - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: userdata - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: mid - timestamp_key: ets - entry_topic: local.ingest + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft - version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713510527662' - created_date: '2024-04-18T23:45:00.389Z' - updated_date: '2024-04-19T01:38:47.670Z' - transformations_config: - - field_key: eid - transformation_function: {} - mode: Strict - metadata: {} + redis_db: 54 example-1: - summary: Read Live dataset + summary: 'Success: Read draft dataset' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:44:54+05:30' + ts: '2024-07-17T17:41:00+05:30' params: status: SUCCESS - resmsgid: 3d0efbc5-dea8-4815-a749-b9f9ae16ccfe + resmsgid: 96fd4f42-fa84-4730-bc79-d241a4e335a1 responseCode: OK result: - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 dataset_config: data_key: '' - timestamp_key: obsrv_meta.syncts - exclude_fields: [] - entry_topic: local.ingest + timestamp_key: ets + entry_topic: beckn-test-data redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] - status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - version: 1 + file_upload_path: [] + exclude_fields: [] example-2: - summary: Read specific fields from the dataset + summary: 'Success: Read specific column' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:45:44+05:30' + ts: '2024-07-17T17:42:16+05:30' params: status: SUCCESS - resmsgid: da752dfe-0d88-4dd6-a6a8-d858f5960f7e + resmsgid: 02a6b03a-8bf3-4e37-8dcd-859d3e8f904e responseCode: OK result: - name: sb-telemetry - type: dataset - id: sb-telemetry.1 + name: master-test + type: master + id: master-test example-3: - summary: Read version_key from the dataset + summary: 'Success: Read version_key' value: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:47:48+05:30' + ts: '2024-07-17T17:45:37+05:30' params: status: SUCCESS - resmsgid: a3ad1e17-671f-4294-a834-2ee3f255c9b3 + resmsgid: 805e848a-d260-47c3-b55c-fc9b8323719e responseCode: OK result: - version_key: '1713510527662' + version_key: '1718791650227' example-4: - summary: 'Read Dataset: Create a draft dataset on edit live' + summary: >- + Success: Read from draft, if not present cerate draft from + live dataset and then read value: id: api.datasets.read ver: v2 - ts: '2024-06-18T14:44:54+05:30' + ts: '2024-07-17T17:49:28+05:30' params: status: SUCCESS - resmsgid: 9f8f6a80-f056-424a-97c4-179e2b9199f2 + resmsgid: da70e25b-6ad0-49a7-a39d-340d1d0c46a7 responseCode: OK result: - id: master-telemetry - dataset_id: master-telemetry - type: master-dataset - name: master-telemetry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: http://json-schema.org/draft-04/schema# - type: object - properties: - actor: - type: object - properties: - id: - type: string - arrival_format: text - data_type: string - type: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - eid: - type: string - arrival_format: text - data_type: string - edata: - type: object - properties: - data: - type: object - properties: - oldFirstAccessedOn: - type: string - arrival_format: text - data_type: string - updatedBy: - type: string - arrival_format: text - data_type: string - lastAccessedOn: - type: string - arrival_format: text - data_type: string - content_id: - type: string - arrival_format: text - data_type: string - oldLastAccessedOn: - type: string - arrival_format: text - data_type: string - progress: - type: number - arrival_format: number - data_type: number - previousProgress: - type: number - arrival_format: number - data_type: number - contentType: - type: string - arrival_format: text - data_type: string - resourceType: - type: string - arrival_format: text - data_type: string - objectId: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - type: - type: string - arrival_format: text - data_type: string - props: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - ver: - type: string - arrival_format: text - data_type: string - syncts: - type: number - arrival_format: number - data_type: number - '@timestamp': - type: string - arrival_format: text - data_type: string - ets: - type: number - arrival_format: number - data_type: number - context: - type: object - properties: - channel: - type: string - arrival_format: text - data_type: string - pdata: - type: object - properties: - id: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - env: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - flags: - type: object - properties: - ex_processed: - type: boolean - arrival_format: boolean - data_type: boolean - pp_validation_processed: - type: boolean - arrival_format: boolean - data_type: boolean - pp_duplicate_skipped: - type: boolean - arrival_format: boolean - data_type: boolean - arrival_format: object - data_type: object - additionalProperties: true - mid: - type: string - arrival_format: text - data_type: string - type: - type: string - arrival_format: text - data_type: string - additionalProperties: true - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetry - dataset_config: - data_key: ver - timestamp_key: obsrv_meta.syncts - entry_topic: local.masterdata.ingest - redis_db_host: localhost - redis_db_port: 6379 - index_data: true - redis_db: 70 - file_upload_path: [] - configurations: - indexConfiguration: - index: - Event Arrival Time: obsrv_meta.syncts - rollupSuggestions: {} - processing: - dedupKeys: [] - dropDuplicates: - - 'Yes' - - 'No' - mergedEvent: - $schema: http://json-schema.org/draft-04/schema# - type: object - properties: - actor: - type: object - properties: - id: - type: string - type: - type: string - required: - - id - - type - eid: - type: string - edata: - type: object - properties: - data: - type: object - properties: - oldFirstAccessedOn: - type: string - updatedBy: - type: string - lastAccessedOn: - type: string - content_id: - type: string - oldLastAccessedOn: - type: string - progress: - type: number - previousProgress: - type: number - contentType: - type: string - resourceType: - type: string - objectId: - type: string - required: - - oldFirstAccessedOn - - updatedBy - - lastAccessedOn - - content_id - - oldLastAccessedOn - - progress - - previousProgress - - contentType - - resourceType - - objectId - type: - type: string - props: - type: string - required: - - data - - type - - props - ver: - type: string - syncts: - type: number - '@timestamp': - type: string - ets: - type: number - context: - type: object - properties: - channel: - type: string - pdata: - type: object - properties: - id: - type: string - required: - - id - env: - type: string - required: - - channel - - pdata - - env - flags: - type: object - properties: - ex_processed: - type: boolean - pp_validation_processed: - type: boolean - pp_duplicate_skipped: - type: boolean - required: - - ex_processed - - pp_validation_processed - - pp_duplicate_skipped - mid: - type: string - type: - type: string - required: - - actor - - eid - - edata - - ver - - syncts - - '@timestamp' - - ets - - context - - flags - - mid - - type + dataset_id: sample1 + name: sample1 + type: event + status: Live tags: [] - created_by: SYSTEM - updated_by: SYSTEM - version_key: '1718702094143' - version: 1 - status: Draft + version: 2 api_version: v2 - transformations_config: [] + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + keys_config: + data_key: '' + timestamp_key: time + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 0 '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '307' + ETag: + schema: + type: string + example: W/"133-TQ9WpmutsrDcTNkRRmbWOhUChMk" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:20:17 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -1122,19 +1048,48 @@ paths: example: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:46:03+05:30' + ts: '2024-07-17T17:50:17+05:30' params: status: FAILED - resmsgid: 302fec39-5070-4464-b02f-56b1d0418147 + resmsgid: bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd responseCode: BAD_REQUEST + result: {} error: code: DATASET_INVALID_FIELDS message: >- The specified fields [newname] in the dataset cannot be found. - trace: '' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '304' + ETag: + schema: + type: string + example: W/"130-JL/oBB+GUHTrBp278giBHRvO71I" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:21:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -1142,43 +1097,26 @@ paths: example: id: api.datasets.read ver: v2 - ts: '2024-04-19T12:46:14+05:30' + ts: '2024-07-17T17:51:12+05:30' params: status: FAILED - resmsgid: cd71f8c7-6aa9-4b3a-9825-b2caab67fe8c + resmsgid: 3aad3842-a76e-4fe8-b635-c7fef5f318f9 responseCode: NOT_FOUND + result: {} error: code: DATASET_NOT_FOUND - message: Dataset with the given dataset_id not found - trace: '' - '500': - description: Internal Server Error - content: - application/json: - schema: - type: object - example: - id: api.datasets.read - ver: v2 - ts: '2024-04-19T12:46:34+05:30' - params: - status: FAILED - resmsgid: 29d82020-101d-4506-8fb8-6f3603057064 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_READ_FAILURE - message: Failed to read dataset - trace: '' + message: >- + Dataset with the given dataset_id:new_telemetry_record.1 not + found /v2/datasets/list: post: tags: - - Dataset CRUD APIs - summary: Dataset List + - Dataset API's + summary: List Datasets description: >- This API allows you to list all datasets. User can apply filters on - dataset status and type, sort the datasets as per requested order. + dataset status and type. requestBody: - required: true content: application/json: schema: @@ -1191,70 +1129,65 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: filters: - type: master-dataset + status: + - Live responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '24340' + ETag: + schema: + type: string + example: W/"5f14-Cq3tfdk3YuXhXtjub1V0q8YVdC4" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:25:36 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Dataset list success when no filters provided + summary: 'Success: Lists all when no filters provided' value: id: api.datasets.list ver: v2 - ts: '2024-04-19T12:54:56+05:30' + ts: '2024-07-17T17:55:36+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: e7a2c5c1-5aa9-4422-81a5-6f197d3b689e + resmsgid: 97efe04d-e981-493d-9ee7-a6dad6887d64 responseCode: OK result: data: - - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts @@ -1264,427 +1197,222 @@ paths: redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' + tags: [] version: 1 - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: tpep_pickup_datetime exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event + status: Live tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' + tags: [] version: 1 - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: skill + data_key: userid timestamp_key: '' exclude_fields: [] entry_topic: local.masterdata.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - - id: two.1 - dataset_id: two - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + index_data: true + redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: mid + data_key: '' timestamp_key: ets - entry_topic: local.ingest - redis_db_host: localhost + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local redis_db_port: 6379 index_data: true redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713509698978' - created_date: '2024-04-15T02:22:48.270Z' - updated_date: '2024-04-15T02:24:03.308Z' - transformations_config: - - field_key: eid - transformation_function: - type: mask - expr: eid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - field_key: cid - transformation_function: - type: mask - expr: eid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: telemetry01.1 - dataset_id: telemetry01 - name: telemetry01 - type: dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: - redis_db_host: localhost + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + index_data: false + redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: data_key: '' - timestamp_key: obsrv_meta.syncts + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 + - dataset_id: sample1 + name: sample1 + type: event + status: Live tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T00:57:37.581Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: old - name: j - skill_id: abc - Skill_id: bbc - activePage: 3 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T00:54:06.404Z' - updated_date: '2024-04-11T06:28:24.526Z' - - id: sb-telemetry.1 - dataset_id: sb-telemetry - name: sb-telemetry - type: dataset - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: true - dedup_key: mid - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - eid: - type: string - ver: - type: string - required: - - eid - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: - - denorm_key: actor.id - denorm_out_field: userdata - - denorm_key: actor.id - denorm_out_field: mid - - denorm_key: actor.id - denorm_out_field: edata - router_config: - topic: '' + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 dataset_config: - data_key: mid - timestamp_key: ets + data_key: '' + timestamp_key: date + exclude_fields: [] entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true redis_db: 0 - tags: - - tag3 - - tag4 - status: Draft + - dataset_id: taxt_trip + name: taxt_trip + type: event + status: Live + tags: [] version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: null - client_state: {} - version_key: '1713510527662' - created_date: '2024-04-18T23:45:00.389Z' - updated_date: '2024-04-19T01:38:47.670Z' - transformations_config: - - field_key: eid - transformation_function: {} - mode: Strict - metadata: {} - count: 6 - example-1: - summary: Dataset list success when status filter provided as array - value: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:55:39+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 836f853f-9ad8-4567-a9bb-74c5d3bba25d - responseCode: OK - result: - data: - - id: telemetry01 - dataset_id: telemetry01 - type: dataset - name: telemetry01 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts @@ -1694,208 +1422,139 @@ paths: redis_db_port: 6379 index_data: true redis_db: 0 - tags: [] + - dataset_id: beckn-test-data + name: beckn-test-data + type: event status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T06:27:37.557Z' - updated_date: '2024-04-11T06:27:37.557Z' + tags: [] version: 1 - transformations_config: - - field_key: mid - transformation_function: - type: mask - expr: mid - condition: null - mode: Strict - metadata: - _transformationType: mask - _transformedFieldDataType: string - _transformedFieldSchemaType: string - section: transformation - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + index_data: true + redis_db: 0 + - dataset_id: telemetry_record-t4 + name: sb-telemetry + type: event + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_events + name: sb-telemetry + type: event + status: Draft + tags: [] + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_record-master + name: sb-telemetry + type: master + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 dataset_config: - data_key: skill + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: generate-schema + name: generate-schema + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' timestamp_key: '' exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish tags: [] - status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' version: 1 - count: 2 - example-2: - summary: Dataset list success when status filter provided as string - value: - id: api.datasets.list - ver: v2 - ts: '2024-04-19T12:56:13+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 469ef16b-4211-4b71-b063-bbd81cc9df02 - responseCode: OK - result: - data: - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry + api_version: null dataset_config: - data_key: skill - timestamp_key: '' + data_key: '' + timestamp_key: obsrv_meta.syncts exclude_fields: [] - entry_topic: local.masterdata.ingest + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 index_data: true - redis_db: 4 + redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - - id: telemetry01.1 - dataset_id: telemetry01 - name: telemetry01 - type: dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: id - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - validation_mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: false - denorm_config: + api_version: null + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest redis_db_host: localhost redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: telemetry01 + index_data: true + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null dataset_config: data_key: '' timestamp_key: obsrv_meta.syncts @@ -1905,1453 +1564,6520 @@ paths: redis_db_port: 6379 index_data: true redis_db: 0 + - dataset_id: trip-test + name: trip-test + type: event + status: Draft tags: [] - status: Publish version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T00:57:37.581Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: old - name: j - skill_id: abc - Skill_id: bbc - activePage: 3 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T00:54:06.404Z' - updated_date: '2024-04-11T06:28:24.526Z' - count: 2 - example-3: - summary: Dataset list success when type filter provided - value: + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample-trip-details + name: sample-trip-details + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 30 + example-1: + summary: 'Success: Filter based on status as array' + value: id: api.datasets.list ver: v2 - ts: '2024-04-19T12:57:28+05:30' + ts: '2024-07-17T17:57:38+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 31aba5bc-8492-45ce-be0e-8c52d8716014 + responseCode: OK + result: + data: + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: taxt_trip + name: taxt_trip + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + count: 16 + example-2: + summary: 'Success: Filter based on status as string' + value: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T17:59:18+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a08c7ea0-bb1c-4998-b47d-a76e38e87e31 + responseCode: OK + result: + data: + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 8 + example-3: + summary: 'Success: Filter based on dataset type' + value: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T18:00:41+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 0d1ff2de-42c9-4192-b75d-84f711dbfb55 + responseCode: OK + result: + data: + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + count: 2 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '312' + ETag: + schema: + type: string + example: W/"138-XQplwhrgIYKIg0qtQdRCYWIGTNM" + Date: + schema: + type: string + example: Wed, 17 Jul 2024 12:32:26 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.list + ver: v2 + ts: '2024-07-17T18:02:26+05:30' + params: + status: FAILED + resmsgid: add9dbe0-f362-4f99-890c-3387c998a049 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_LIST_INPUT_INVALID + message: >- + #properties/params/required must have required property + 'msgid' + /v2/datasets/dataschema: + post: + tags: + - Dataset API's + summary: Schema Generation + description: This api is used to generate data schema for the given dataset event. + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + data: + - eid: IMPRESSION + ets: 1672657002221 + ver: '3.0' + mid: IMPRESSION:2b5834e196f485c17c4e49d292af43c0 + actor: + id: 0c45959486f579c24854d40a225d6161 + type: User + context: + channel: '01268904781886259221' + pdata: + id: staging.diksha.portal + ver: 5.1.0 + pid: sunbird-portal + env: public + sid: 23850c90-8a8c-11ed-95d0-276800e1048c + did: 0c45959486f579c24854d40a225d6161 + cdata: [] + rollup: + l1: '01268904781886259221' + uid: anonymous + object: {} + tags: + - '01268904781886259221' + edata: + type: view + pageid: login + subtype: pageexit + uri: >- + https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=254efd70-6b89-4f7d-868b-5c957f54174e&redirect_uri=https%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1&scope=openid&response_type=code&version=4 + visits: [] + syncts: 1672657005814 + '@timestamp': '2023-01-02T10:56:45.814Z' + flags: + ex_processed: true + - eid: IMPRESSION + ets: 1672656997928 + ver: '3.0' + mid: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: POST + - url: /v1/org/search + - duration: 0 + - status: OK + - eid: LOG + ets: 1672656998024 + ver: '3.0' + mid: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + type=system, message=EXIT LOG: method : POST, url: + /v1/org/search , For Operation : orgSearch, + params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + errmsg=Invalid value null for parameter hashTagId. + Please provide a valid value., + resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + err=UOS_ORGSER0017, status=FAILED, + responseCode=400}]}} + params: [] + - eid: LOG + ets: 1672657004961 + ver: '3.0' + mid: f34112c7242a3e3a26f0015796b029c2 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: f34112c7242a3e3a26f0015796b029c2 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + ElasticSearchRestHighImpl:search: calling search for + index org_alias, with query = + {"from":0,"size":250,"query":{"bool":{"must":[{"term":{"isTenant.raw":{"value":true,"boost":1.0}}},{"term":{"slug.raw":{"value":"ntp","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":[],"excludes":[]}} + params: [] + - eid: LOG + ets: 1672657006595 + ver: '3.0' + mid: d23ff123-40f0-4262-a69b-b75b46d315a1 + actor: + id: 930a3994-cbe7-4e84-936f-4974096af6f2 + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: d23ff123-40f0-4262-a69b-b75b46d315a1 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, + type=system, message=ENTRY LOG: method : GET, url: + /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 + , For Operation : getUserRolesById, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006611 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, + type=system, message=ENTRY LOG: method : GET, url: + /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , + For Operation : getUserProfileV5, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006620 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user_roles + WHERE userId=?; + params: [] + - eid: LOG + ets: 1672657006645 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: GET + - url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 + - duration: 0 + - status: OK + - eid: LOG + ets: 1672657007238 + ver: '3.0' + mid: d4d34fde-c407-efb6-03bd-9f892ca0f114 + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.portal + pid: learner-service + ver: 5.0.0 + env: User + did: d904c90d9f81ddac20141b94ddd606a0 + cdata: + - id: d4d34fde-c407-efb6-03bd-9f892ca0f114 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user WHERE + id=?; + params: [] + config: + dataset: financial_transactions + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Cookie + in: header + schema: + type: string + example: >- + connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '10760' + ETag: + schema: + type: string + example: W/"2a08-QF5x1q0kIlfE9XU/pa9IboJuY8I" + Date: + schema: + type: string + example: Mon, 22 Jul 2024 07:02:50 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:32:50+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1309aea0-9a97-46e9-bc5e-a16a8a7fb624 + responseCode: OK + result: + schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' format + type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + suggestions: + - message: >- + The Property 'mid' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.mid + arrival_format: text + data_type: string + actor: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'actor.id' appears to be 'uuid' + format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.actor.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + env: + type: string + arrival_format: text + data_type: string + sid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'sid'. The property sid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.sid + - message: >- + The Property 'context.sid' appears to be + 'uuid' format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.context.properties.sid + arrival_format: text + data_type: string + did: + type: string + arrival_format: text + data_type: string + cdata: + type: array + items: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'context.cdata[*].id' + appears to be 'uuid' format type. + advice: >- + Suggest to not to index the high + cardinal columns + resolutionType: DEDUP + severity: LOW + path: >- + properties.context.properties.cdata.items.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: array + data_type: array + rollup: + type: object + properties: + l1: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at + property: 'l1'. The property l1: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System + has updated the property schema to + optional + resolutionType: OPTIONAL + severity: MEDIUM + path: >- + properties.context.properties.rollup.properties.l1 + arrival_format: text + data_type: string + arrival_format: object + data_type: object + uid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uid'. The property uid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.uid + arrival_format: text + data_type: string + arrival_format: object + data_type: object + object: + type: object + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'object'. The property object: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.object + arrival_format: object + data_type: object + tags: + type: array + items: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'tags'. The property tags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.tags + arrival_format: array + data_type: array + edata: + type: object + properties: + type: + type: string + arrival_format: text + data_type: string + pageid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'pageid'. The property pageid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.pageid + arrival_format: text + data_type: string + subtype: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'subtype'. The property subtype: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.subtype + arrival_format: text + data_type: string + uri: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uri'. The property uri: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.uri + - message: >- + The Property 'edata.uri' appears to be 'uri' + format type. + severity: '' + path: properties.edata.properties.uri + arrival_format: text + data_type: string + visits: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'visits'. The property visits: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.visits + arrival_format: array + data_type: array + level: + type: string + arrival_format: text + data_type: string + message: + type: string + arrival_format: text + data_type: string + params: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + arrival_format: array + data_type: array + arrival_format: object + data_type: object + syncts: + type: integer + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'syncts'. The property syncts: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.syncts + - message: >- + The Property 'syncts' appears to be 'epoch' format + type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + '@timestamp': + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + '@timestamp'. The property @timestamp: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.@timestamp + - message: >- + The Property '@timestamp' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.@timestamp + arrival_format: text + data_type: date-time + flags: + type: object + properties: + ex_processed: + type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'ex_processed'. The property ex_processed: + only 1 time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags.properties.ex_processed + arrival_format: boolean + data_type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'flags'. The property flags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags + arrival_format: object + data_type: object + additionalProperties: true + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + mid: + path: $.mid + cardinality: 67 + index: false + actor.id: + path: $.actor.properties.id + cardinality: 56 + index: false + context.sid: + path: $.context.properties.sid + cardinality: 11 + index: true + edata.uri: + path: $.edata.properties.uri + cardinality: 11 + index: true + context.cdata[*].id: + path: $.context.properties.cdata.items.properties.id + cardinality: 62 + index: false + processing: + dedupKeys: + - mid + - context.cdata[*].id + - actor.id + dropDuplicates: + - 'Yes' + - 'No' + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '364' + ETag: + schema: + type: string + example: W/"16c-tfKVtCWTjNkWCtH8cFw1RrzbgV0" + Date: + schema: + type: string + example: Mon, 22 Jul 2024 07:03:47 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Invalid request body' + value: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:33:47+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bbcc86c2-042d-4f77-bb6e-e1c9116df570 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'data' + example-1: + summary: 'Failure: Invalid request (config not provided)' + value: + id: api.datasets.dataschema + ver: v2 + ts: '2024-07-22T12:35:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1f856c5e-37f0-41e9-96fb-642471228da2 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'config' + /v2/datasets/status-transition: + post: + tags: + - Dataset API's + summary: Dataset Status Transition + description: >- + This API allows you to perform status transition between 2 states. + Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to + Live, Live to Retired and even Delete a dataset. + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.status-transition + ver: v2 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + dataset_id: telemetry-events + status: ReadyToPublish + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Delete success: Deleted dataset successfully' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-05-30T12:18:54+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5948e784-37f9-4a70-85ca-86c9077ee30b + responseCode: OK + result: + message: Dataset status transition to Delete successful + dataset_id: trip-data + example-1: + summary: 'Live success: Dataset published successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:21:42+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 88d62970-97be-472f-9ccc-67f875d69335 + responseCode: OK + result: + message: Dataset status transition to Live successful + dataset_id: telemetry_record + example-2: + summary: 'ReadyToPublish success: Dataset is ready to publish' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:30:04+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 84858e85-6a97-43cb-b8e9-17a7e0a43365 + responseCode: OK + result: + message: Dataset status transition to ReadyToPublish successful + dataset_id: telemetry-events + example-3: + summary: 'Retire success: Dataset retired successfully' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:22:58+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: f2285754-7d5b-4320-943d-797fb136e955 + responseCode: OK + result: + message: Dataset status transition to Retire successful + dataset_id: sb-telemetry + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Live failure: Dataset in draft state' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-05-30T15:37:43+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d56e2ed4-f008-48be-a501-164c19178419 + responseCode: BAD_REQUEST + error: + code: DATASET_LIVE_FAILURE + message: >- + Transition failed for dataset: sb-telemetry2 + status:Draft with status transition to Live + trace: '' + example-1: + summary: 'ReadyToPublish failure: Incomplete dataset configs' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:36:16+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: a504565b-41ff-4c0f-9d64-f96df9ed89bb + responseCode: BAD_REQUEST + error: + code: DATASET_CONFIGS_INVALID + message: >- + #properties/denorm_config/properties/denorm_fields/items/required + must have required property 'dataset_name' + trace: '' + example-2: + summary: 'ReadyToPublish failure: Dataset not in draft state' + value: + id: api.datasets.status-transition + ver: v2 + ts: '2024-06-18T15:38:14+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 351f5a37-87f0-47cd-bebe-e3c001256d0a + responseCode: BAD_REQUEST + error: + code: DATASET_READYTOPUBLISH_FAILURE + message: >- + Transition failed for dataset: telemetry-events + status:Retired with status transition to ReadyToPublish + trace: '' + example-3: + summary: 'Retire Failure: Dataset is already retired' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:42:18+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 80208169-b1d3-41cd-816b-83fae96a4370 + responseCode: BAD_REQUEST + error: + code: DATASET_RETIRE_FAILURE + message: >- + Transition failed for dataset: master-telemetrry + status:Retired with status transition to Retire + trace: '' + example-4: + summary: >- + Retire failure: Cannot retire master dataset as it is used + by other datasets + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:01:41+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: b88c320a-2c01-4662-a509-bd532a612c05 + responseCode: BAD_REQUEST + error: + code: DATASET_IN_USE + message: >- + Failed to retire dataset as it is in use. Please retire + or delete dependent datasets before retiring this + dataset + trace: '' + example-5: + summary: 'Failure: Invalid request payload provided' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T16:03:56+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: ba4c86bd-b438-4582-b178-2410a5c5dd15 + responseCode: BAD_REQUEST + error: + code: DATASET_STATUS_INVALID_INPUT + message: >- + #properties/request/properties/status/enum should be + equal to one of the allowed values + trace: '' + '404': + description: Not Found + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Delete failure: Dataset not found to delete' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T12:25:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 3cdcf2af-c015-4977-9d66-364e00f1712b + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: master' + trace: '' + example-1: + summary: 'Live failure: Dataset not found to publish' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:35:59+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: telemetry-data' + trace: '' + example-2: + summary: 'Retire Failure: Dataset not found to retire' + value: + id: api.datasets.status + ver: v2 + ts: '2024-05-30T15:40:31+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 73befbbd-60e3-48e0-9cfd-cb705dfc2b85 + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: 'Dataset not found for dataset: sb-telemetry2' + trace: '' + /v2/datasets/import: + post: + tags: + - Dataset API's + summary: Dataset import + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-05-21T14:30:00Z' + params: + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + request: + id: sb-telemetry_draft_data + dataset_id: sb-telemetryRPF_draft_data + name: sb-telemetry_draft_data + type: event + extraction_config: + is_batch_event: false + dedup_config: + drop_duplicates: false + dedup_key: id + dedup_period: 604800 + extraction_key: events + validation_config: + validate: true + mode: Strict + dedup_config: + dedup_key: id + drop_duplicates: true + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + syncts: + type: integer + arrival_format: number + data_type: integer + ets: + type: integer + arrival_format: number + data_type: epoch + suggestions: + - message: >- + The Property 'ets' appears to be 'date-time' + format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.ets + flags: + type: object + properties: + ex_processed: + type: boolean + arrival_format: boolean + data_type: boolean + pp_validation_processed: + type: boolean + arrival_format: boolean + data_type: boolean + pp_duplicate_skipped: + type: boolean + arrival_format: boolean + data_type: boolean + user_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + device_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + loc_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + content_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + coll_denorm: + type: boolean + arrival_format: boolean + data_type: boolean + arrival_format: object + data_type: object + additionalProperties: false + mid: + type: string + suggestions: + - message: >- + The Property 'mid' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.mid + arrival_format: text + data_type: string + actor: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'actor.id' appears to be 'uuid' + format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.actor.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + edata: + type: object + properties: + visits: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + type: + type: string + arrival_format: text + data_type: string + duration: + type: number + arrival_format: number + data_type: number + size: + type: integer + arrival_format: number + data_type: integer + query: + type: string + arrival_format: text + data_type: string + filters: + type: object + properties: + objectType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + version: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + status: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + id: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + isRootOrg: + type: boolean + arrival_format: boolean + data_type: boolean + trackable.enabled: + type: string + arrival_format: text + data_type: string + channel: + type: object + arrival_format: object + data_type: object + additionalProperties: false + framework: + type: object + arrival_format: object + data_type: object + additionalProperties: false + resourceType: + type: object + arrival_format: object + data_type: object + additionalProperties: false + identifier: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + contentType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + mimeType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + hashTagId: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + compatibilityLevel: + type: object + properties: + min: + type: integer + arrival_format: number + data_type: integer + max: + type: integer + arrival_format: number + data_type: integer + arrival_format: object + data_type: object + additionalProperties: false + createdBy: + type: string + arrival_format: text + data_type: string + mediaType: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + origin: + type: string + arrival_format: text + data_type: string + primaryCategory: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + trackable: + enabled: + type: string + arrival_format: object + data_type: object + additionalProperties: false + sort: + type: object + properties: + lastUpdatedOn: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + topn: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + pageid: + type: string + arrival_format: text + data_type: string + uri: + type: string + arrival_format: text + data_type: string + subtype: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + data: + type: string + arrival_format: text + data_type: string + uaspec: + type: object + properties: + agent: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + system: + type: string + arrival_format: text + data_type: string + platform: + type: string + arrival_format: text + data_type: string + raw: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + state: + type: string + arrival_format: text + data_type: string + props: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + prevstate: + type: string + arrival_format: text + data_type: string + dspec: + type: object + properties: + os: + type: string + arrival_format: text + data_type: string + make: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + idisk: + type: number + arrival_format: number + data_type: number + edisk: + type: number + arrival_format: number + data_type: number + scrn: + type: number + arrival_format: number + data_type: number + camera: + type: string + arrival_format: text + data_type: string + cpu: + type: string + arrival_format: text + data_type: string + sims: + type: integer + arrival_format: number + data_type: integer + webview: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + extra: + type: object + properties: + pos: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + values: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + query: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + mode: + type: string + arrival_format: text + data_type: string + duration: + type: string + arrival_format: object + data_type: object + additionalProperties: false + '@timestamp': + type: string + suggestions: + - message: >- + The Property '@timestamp' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.@timestamp + arrival_format: text + data_type: date-time + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + env: + type: string + arrival_format: text + data_type: string + sid: + type: string + arrival_format: text + data_type: string + rollup: + type: object + properties: + l1: + type: string + arrival_format: text + data_type: string + l2: + type: string + arrival_format: text + data_type: string + l3: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + cdata: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + did: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + object: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + rollup: + type: object + properties: + l1: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + version: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: false + tags: + type: array + items: + type: object + properties: + key: + type: string + value: + type: string + additionalProperties: true + arrival_format: array + data_type: array + additionalProperties: false + denorm_config: + redis_db_host: redis-denorm-headless.redis.svc.cluster.local + redis_db_port: 6379 + denorm_fields: + - denorm_key: actor.id + dataset_id: master-dataset + denorm_out_field: userdata + router_config: + topic: sb-telemetry + dataset_config: + keys_config: + timestamp_key: obsrv_meta.syncts + data_key: '' + partition_key: '' + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + cache_config: + redis_db_host: redis-denorm-headless.redis.svc.cluster.local + redis_db_port: 6379 + redis_db: 0 + file_upload_path: [] + transformations_config: [] + connectors_config: [] + tags: + - tag1 + status: ReadyToPublish + version: 1 + created_by: SYSTEM + updated_by: SYSTEM + version_key: '1724333643940' + api_version: v2 + sample_data: {} + entry_topic: dev.ingest + created_date: '2024-07-23T18:35:15.690Z' + updated_date: '2024-08-22T13:34:08.041Z' + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: overwrite + in: query + schema: + type: boolean + example: 'false' + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '346' + ETag: + schema: + type: string + example: W/"15a-SX5o6plRdfy+akEYhbMwq9zl+oU" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:34:26 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Fully imported dataset' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:04:26+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: fd87393e-4b1b-4cf0-8e9a-a0529d691cc3 + responseCode: OK + result: + message: Dataset is imported successfully + data: + id: sb-telemetry_draft_data + version_key: '1732001666671' + example-1: + summary: 'Success: Partially imported dataset' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T12:59:41+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 7d9aeb08-2952-485f-9944-4d118da7842e + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 379ced0b-eb06-445f-a7e3-8737cb16c351 + responseCode: OK + result: + message: Dataset is partially imported + data: + id: sb-telemetry_draft_data + version_key: '1732001381825' + ignoredFields: + denorm_fields: + - config: + denorm_key: actor.id + denorm_out_field: userdata + dataset_id: master-dataset + details: Master dataset does not exist + example-2: + summary: 'Success: Imported new dataset [overwrite=false]' + value: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:05:32+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 1a50e1b1-14b0-4178-aba4-9e09a63d8d1d responseCode: OK result: + message: Dataset is imported successfully data: - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema + id: sb-telemetry_draft + version_key: '1732001732045' + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '364' + ETag: + schema: + type: string + example: W/"16c-Dxem7prJM89DOqth+bdZbh0UI1g" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:38:05 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:08:05+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 244318a5-d933-41a5-add6-14f299281069 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_IMPORT_INVALID_CONFIGS + message: >- + #properties/request/required must have required property + 'id' + '409': + description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '351' + ETag: + schema: + type: string + example: W/"15f-nj5WJ+wiNbI5ViTPWUzGutdBSX4" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:31:11 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.import + ver: v2 + ts: '2024-11-19T13:01:11+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 1e5b34cd-4473-4439-8cc5-5ff917f47e48 + responseCode: CONFLICT + result: {} + error: + code: DATASET_EXISTS + message: >- + Dataset with dataset_id: sb-telemetryRPF_draft_data already + exists. + /v2/datasets/export/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset + get: + tags: + - Dataset API's + summary: Export Dataset + parameters: + - name: status + in: query + schema: + type: string + example: Live + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '5155' + ETag: + schema: + type: string + example: W/"1423-Tx46QCrIX8So3vSGVkd865IYeEY" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:16:01 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Live Dataset export success' + value: + id: api.datasets.export + ver: v2 + ts: '2024-11-19T12:46:01+05:30' + params: + status: SUCCESS + resmsgid: ae57c620-2858-40f9-9d8e-33f8f259449d + responseCode: OK + result: + id: content-data-v2 + dataset_id: content-data-v2 + type: master + name: content-data-v2 + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + tripID: + key: tripID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: false + VendorID: + key: VendorID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tpep_pickup_datetime: + key: tpep_pickup_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + tpep_dropoff_datetime: + key: tpep_dropoff_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + passenger_count: + key: passenger_count + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + trip_distance: + key: trip_distance + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + RatecodeID: + key: RatecodeID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + store_and_fwd_flag: + key: store_and_fwd_flag + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + PULocationID: + key: PULocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + DOLocationID: + key: DOLocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + payment_type: + key: payment_type + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + primary_passenger: + key: primary_passenger type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true properties: - skill: + email: + key: email type: string arrival_format: text data_type: string - name: + isRequired: false + resolved: true + mobile: + key: mobile type: string arrival_format: text data_type: string - skill_id: + isRequired: false + resolved: true + fare_details: + key: fare_details + type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true + properties: + fare_amount: + key: fare_amount type: string arrival_format: text data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry - dataset_config: - data_key: skill - timestamp_key: '' - exclude_fields: [] - entry_topic: local.masterdata.ingest - redis_db_host: localhost - redis_db_port: 6379 - index_data: true - redis_db: 4 - tags: [] - status: Publish - version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - count: 1 - example-4: - summary: Dataset list success based on sortBy values + isRequired: false + resolved: true + extra: + key: extra + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + mta_tax: + key: mta_tax + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tip_amount: + key: tip_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tolls_amount: + key: tolls_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + improvement_surcharge: + key: improvement_surcharge + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + total_amount: + key: total_amount + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + congestion_surcharge: + key: congestion_surcharge + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + additionalProperties: true + dedup_config: + drop_duplicates: true + dedup_key: tripID + dedup_period: 604800 + denorm_config: + redis_db_host: localhost + redis_db_port: 6379 + denorm_fields: [] + router_config: + topic: content-data-v2 + dataset_config: + file_upload_path: + - >- + container/api-service/user_uploads/chunk-2_addedf.json + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + keys_config: + timestamp_key: tpep_pickup_datetime + data_key: '' + partition_key: '' + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 7 + tags: [] + status: Live + created_by: SYSTEM + updated_by: SYSTEM + data_version: 1 + api_version: v2 + version: 1 + sample_data: + mergedEvent: + tripID: ccbb970c-4d10-4247-be0c-a480996f3429 + VendorID: '1' + tpep_pickup_datetime: '2023-02-26 00:36:10' + tpep_dropoff_datetime: '2023-09-15 00:45:45' + passenger_count: '2' + trip_distance: '1.70' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '238' + DOLocationID: '263' + payment_type: '1' + primary_passenger: + email: Jacey_Hintz@yahoo.com + mobile: 247-492-3370 + fare_details: + fare_amount: '8.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2.45' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '12.25' + congestion_surcharge: '' + entry_topic: local.ingest + created_date: '2024-10-14T06:23:44.588Z' + updated_date: '2024-10-14T06:23:44.588Z' + example-1: + summary: 'Success: ReadyToPublish Dataset exported successfully' value: - id: api.datasets.list + id: api.datasets.export ver: v2 - ts: '2024-04-19T12:58:27+05:30' + ts: '2024-11-19T12:48:11+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 270a4e50-f7e5-4562-ae75-7a967866d065 + resmsgid: 222847a0-32bf-4c63-be68-76f0e51258af responseCode: OK result: - data: - - id: master-telemetrry - dataset_id: master-telemetrry - type: master-dataset - name: master-telemetrry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema + id: mydataset + dataset_id: mydataset + name: mydataset + type: event + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + validation_config: + validate: true + mode: Strict + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_period: 604800 + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + tripID: + key: tripID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: false + VendorID: + key: VendorID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + tpep_pickup_datetime: + key: tpep_pickup_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + tpep_dropoff_datetime: + key: tpep_dropoff_datetime + type: string + arrival_format: text + data_type: date-time + isRequired: false + resolved: false + passenger_count: + key: passenger_count + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + trip_distance: + key: trip_distance + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + RatecodeID: + key: RatecodeID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + store_and_fwd_flag: + key: store_and_fwd_flag + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + PULocationID: + key: PULocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + DOLocationID: + key: DOLocationID + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + payment_type: + key: payment_type + type: string + arrival_format: text + data_type: string + isRequired: false + resolved: true + primary_passenger: + key: primary_passenger type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true properties: - skill: - type: string - arrival_format: text - data_type: string - name: + email: + key: email type: string arrival_format: text data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - Skill_id: + isRequired: false + resolved: true + mobile: + key: mobile type: string arrival_format: text data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry - dataset_config: - data_key: skill - timestamp_key: '' - exclude_fields: [] - entry_topic: local.masterdata.ingest - redis_db_host: localhost - redis_db_port: 6379 - index_data: true - redis_db: 4 - tags: [] - status: Live - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-04-11T09:16:36.904Z' - updated_date: '2024-04-11T09:16:36.904Z' - version: 1 - - id: master-telemetrry.1 - dataset_id: master-telemetrry - name: master-telemetrry - type: master-dataset - extraction_config: - is_batch_event: false - extraction_key: '' - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - validation_config: - validate: true - mode: Strict - dedup_config: - drop_duplicates: false - dedup_key: '' - dedup_period: 604800 - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema + isRequired: false + resolved: true + additionalProperties: false + fare_details: + key: fare_details type: object + arrival_format: object + data_type: object + isRequired: false + resolved: true properties: - skill: - type: string - arrival_format: text - data_type: string - name: - type: string - arrival_format: text - data_type: string - skill_id: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: - redis_db_host: localhost - redis_db_port: 6379 - denorm_fields: [] - router_config: - topic: master-telemetrry - dataset_config: - data_key: skill - timestamp_key: '' - exclude_fields: [] - entry_topic: local.masterdata.ingest - redis_db_host: localhost - redis_db_port: 6379 - index_data: true - redis_db: 4 - tags: [] - status: Publish - version: 1 - created_by: SYSTEM - updated_by: SYSTEM - published_date: '2024-04-11T03:46:36.938Z' - client_state: - metadata: - conflicts: {} - event: - mergedEvent: - skill: new - name: j - skill_id: aabc - Skill_id: bbc - activePage: 1 - pages: {} - version_key: '1713509698978' - created_date: '2024-04-11T03:45:57.251Z' - updated_date: '2024-04-11T09:21:11.167Z' - count: 2 + fare_amount: + key: fare_amount + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: integer + '1': + type: number + extra: + key: extra + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + mta_tax: + key: mta_tax + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + tip_amount: + key: tip_amount + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + tolls_amount: + key: tolls_amount + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: integer + '1': + type: number + improvement_surcharge: + key: improvement_surcharge + type: number + arrival_format: number + data_type: number + isRequired: false + resolved: true + total_amount: + key: total_amount + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + isModified: true + oneof: + '0': + type: number + '1': + type: integer + congestion_surcharge: + key: congestion_surcharge + type: integer + arrival_format: number + data_type: integer + isRequired: false + resolved: true + additionalProperties: false + additionalProperties: false + denorm_config: + redis_db_host: localhost + redis_db_port: 6379 + denorm_fields: [] + router_config: + topic: mydataset + dataset_config: + file_upload_path: + - >- + container/api-service/user_uploads/chunk-1 (4) + (1)_5ec855.json + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 0 + transformations_config: [] + connectors_config: [] + tags: [] + status: ReadyToPublish + version: 1 + created_by: SYSTEM + updated_by: SYSTEM + version_key: '1731420131072' + api_version: v2 + sample_data: + mergedEvent: + tripID: 4c77e9d5-538d-4eb7-8db1-4c2c32860aa8 + VendorID: '2' + tpep_pickup_datetime: '2023-11-02 00:18:42' + tpep_dropoff_datetime: '2023-11-02 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Willa67@gmail.com + mobile: 1-720-981-6399 x77055 + fare_details: + fare_amount: 7 + extra: 0.5 + mta_tax: 0.5 + tip_amount: 0 + tolls_amount: 0 + improvement_surcharge: 0.3 + total_amount: 8.3 + congestion_surcharge: 0 + entry_topic: local.ingest + created_date: '2024-11-11T21:31:21.390Z' + updated_date: '2024-11-12T08:32:27.931Z' '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '291' + ETag: + schema: + type: string + example: W/"123-E2lDLItXYvKnJxGcDJ3Q0OKjqQY" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:20:43 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object example: - id: api.datasets.list + id: api.datasets.export ver: v2 - ts: '2024-04-19T12:58:47+05:30' + ts: '2024-11-19T12:50:43+05:30' params: status: FAILED - resmsgid: f9e1fae9-4660-409d-bb4d-e29888983d4b + resmsgid: 2097228c-7872-4217-a06a-0ec08e3dc67d responseCode: BAD_REQUEST + result: {} error: - code: DATASET_LIST_INPUT_INVALID - message: >- - #properties/params/additionalProperties should NOT have - additional properties - trace: '' - '500': - description: Internal Server Error + code: DATASET_EXPORT_FAILURE + message: Dataset with status:Draft cannot be exported + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '301' + ETag: + schema: + type: string + example: W/"12d-vO+8bWB6kFOrTzeJmTIRlcTcYCQ" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:15:10 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object example: - id: api.datasets.list + id: api.datasets.export ver: v2 - ts: '2024-04-19T12:59:15+05:30' + ts: '2024-11-19T12:45:10+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 81c3a4a5-e8ac-41de-be00-6d0300ecb106 - responseCode: INTERNAL_SERVER_ERROR + resmsgid: fd6fdae2-a002-4c15-81e2-3cce539a2a5a + responseCode: NOT_FOUND + result: {} error: - code: DATASET_LIST_FAILURE - message: Failed to list dataset - trace: '' - /v2/datasets/diff/{dataset_id}: - get: + code: DATASET_NOT_FOUND + message: >- + Dataset with the given dataset_id:v1-copy not found to + export + /v2/datasets/copy: + post: tags: - - Dataset CRUD APIs - summary: Dataset Diff - description: >- - This API allows you to return the difference between the live and draft - dataset by returning additions, deletions and modifications in the - dataset. + - Dataset API's + summary: Clone Dataset + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: v2 + ts: '2024-05-21T14:30:00Z' + params: + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + request: + source: + datasetId: dataset-telemetry + isLive: true + destination: + datasetId: bew-copy-live2 parameters: - - name: dataset_id - example: telemetry - in: path - required: true + - name: Content-Type + in: header schema: type: string + example: application/json + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '296' + ETag: + schema: + type: string + example: W/"128-bi4wSfgkyMP13qFE8VPTmgDdgLA" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:26:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: v2 + ts: '2024-11-19T12:56:12+05:30' + params: + status: SUCCESS + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: 28072631-8e80-45eb-906f-d933a90646d0 + responseCode: OK + result: + dataset_id: new-copy-live2 + message: Dataset clone successful + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '328' + ETag: + schema: + type: string + example: W/"148-jUDshghYf/jOhmmVkvtHlfJT864" + Date: + schema: + type: string + example: Tue, 19 Nov 2024 07:22:41 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.datasets.copy + ver: v2 + ts: '2024-11-19T12:52:41+05:30' + params: + status: FAILED + msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + resmsgid: f2fd51bb-fc18-4278-a229-47fa25398e69 + responseCode: NOT_FOUND + result: {} + error: + code: DATASET_NOT_EXISTS + message: Dataset dataset-telemetry does not exists + /v2/connectors/list: + post: + tags: + - Connector API's + summary: List Connectors + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: + - Draft responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '250' + ETag: + schema: + type: string + example: W/"fa-+eWKIfUxsWBGuJy23qSucgLXke4" + Date: + schema: + type: string + example: Tue, 30 Jul 2024 09:55:51 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: Data Format Diff + summary: 'Success: Filtered based on status' value: - id: api.datasets.diff + id: api.connectors.list ver: v2 - ts: '2024-05-30T16:55:25+05:30' + ts: '2024-07-30T15:25:51+05:30' params: status: SUCCESS - resmsgid: c58aa5f7-f182-4dff-9ee9-16f508e53c51 + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: f506e725-eed4-41df-86dc-2477d5c4d19a responseCode: OK result: - additions: - - type: dataFormat - items: - - name: extraction_key - value: mid - - name: batch_id - value: id - - name: dedup_config.dedup_key - value: id - modifications: - - type: dataFormat - items: - - name: is_batch_event - value: - from: false - to: true - deletions: [] + data: [] + count: 0 example-1: - summary: Timestamp and data key diff + summary: 'Success: Filtered based on category' value: - id: api.datasets.diff + id: api.connectors.list ver: v2 - ts: '2024-05-30T17:00:44+05:30' + ts: '2024-07-31T18:55:03+05:30' params: status: SUCCESS - resmsgid: 8ec5fa57-5105-4793-ae96-3a7e0550dbe9 + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab responseCode: OK result: - additions: [] - modifications: - - type: timestamp - value: - from: syncts - to: ets - - type: dataKey - value: - from: eid - to: mid - deletions: [] + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + count: 4 example-2: - summary: Data Schema Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-28T15:40:48+05:30' - params: - status: SUCCESS - resmsgid: c364416d-b903-4af6-a39b-a4e52c4bf6b5 - responseCode: OK - result: - additions: - - type: dataSchema - items: - - name: eid_1 - value: - arrivalFormat: boolean - dataType: boolean - absolutePath: $.properties.eid_1 - isRequired: false - modifications: - - type: dataSchema - items: - - field: eid - name: type - value: - from: string - to: boolean - - field: eid - name: arrivalFormat - value: - from: text - to: boolean - - field: eid - name: dataType - value: - from: string - to: boolean - deletions: - - type: dataSchema - items: - - name: ver - example-3: - summary: Data Validation Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:05:28+05:30' - params: - status: SUCCESS - resmsgid: ad6b4977-c341-4193-a42f-9b45993e5f8e - responseCode: OK - result: - additions: [] - modifications: - - type: validation - items: - - field: mode - value: - from: IgnoreNewFields - to: Strict - deletions: [] - example-4: - summary: Dataset Dedup diff + summary: 'Success: Connectors list with all filter options' value: - id: api.datasets.diff + id: api.connectors.list ver: v2 - ts: '2024-05-30T17:07:58+05:30' + ts: '2024-07-31T18:56:32+05:30' params: status: SUCCESS - resmsgid: e59b841b-e35f-462d-adb1-31018d718f6a - responseCode: OK - result: - additions: - - type: dedup - items: - - name: dedup_key - value: mid - modifications: - - type: dedup - items: - - name: drop_duplicates - value: - from: false - to: true - deletions: [] - example-5: - summary: Dataset Denorm Diff - value: - id: api.datasets.diff - ver: v2 - ts: '2024-05-30T17:17:35+05:30' - params: - status: SUCCESS - resmsgid: ee558c17-b3a6-41cc-9b07-64105053394e + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 11a2f537-bd98-405b-97e5-0f0d5b86b2c3 responseCode: OK result: - additions: - - type: denorm - items: - - name: syncts - value: - denorm_key: actor.id - dataset_name: mid - modifications: [] - deletions: - - type: denorm - items: - - name: mid - value: - denorm_key: actor.id - dataset_name: mid - example-6: - summary: Dataset transformation diff + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' + count: 7 + example-3: + summary: 'Success: Connectors list without filters' value: - id: api.datasets.diff + id: api.connectors.list ver: v2 - ts: '2024-05-30T17:25:12+05:30' + ts: '2024-07-31T18:57:37+05:30' params: status: SUCCESS - resmsgid: f4389891-9c09-4ccd-affe-39970590045b + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: c2467e01-0a2d-401c-aa3d-dd16b804f723 responseCode: OK result: - additions: - - type: transformations - items: - - name: skill_id - value: - transformation_function: - type: mask - condition: null - expr: skill_id - mode: Strict - metadata: - section: pii - modifications: - - type: transformations - items: - - name: new - field: transformation_function.type - value: - from: jsonata - to: encrypt - metadata: - section: transformation - deletions: - - type: transformations - items: - - name: mid - value: - metadata: - section: transformation - - name: name - value: - metadata: - section: transformation - - name: neew - value: - metadata: - section: additionalFields - /v2/files/generate-url: - post: - tags: - - Dataset CRUD APIs - summary: Files generate URL - description: >- - This API allows you to generate pre signed urls from cloud providers for - getting and putting files. On access: write, upload urls are generated to put files and on access: read, download urls are generated to get files. - requestBody: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' + count: 7 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '399' + ETag: + schema: + type: string + example: W/"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k" + Date: + schema: + type: string + example: Tue, 30 Jul 2024 09:43:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: - examples: - example1: - summary: Generate upload URL - value: - id: api.files.generate-url - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: write - example2: - summary: Generate download URL - value: - id: api.files.generate-url - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: read + schema: + type: object + example: + id: api.connectors.list + ver: v2 + ts: '2024-07-30T15:13:14+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 02fadde0-8c59-4420-8ab3-56474b01670b + responseCode: BAD_REQUEST + result: {} + error: + code: CONNECTORS_LIST_INPUT_INVALID + message: >- + #properties/request/properties/filters/properties/status/minItems + must NOT have fewer than 1 items + /v2/connectors/read/{connector_id}: + parameters: + - name: connector_id + in: path + required: true + schema: + type: string + description: Unique identifier for the connector + get: + tags: + - Connector API's + summary: Read Connector responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '3304' + ETag: + schema: + type: string + example: W/"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28" + Date: + schema: + type: string + example: Wed, 31 Jul 2024 12:47:54 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Success: Generate Put URL' + summary: 'Success: Read live Connector' value: - id: api.files.generate-url + id: api.connectors.read ver: v2 - ts: '2024-05-30T12:04:15+05:30' + ts: '2024-07-31T18:17:54+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: a79f1260-1b34-4b96-9c12-cf0f5600e9f3 + resmsgid: 7587f564-c2d7-49a8-9e56-dc56f6808ced responseCode: OK result: - - filePath: >- - obsrv-onest/api-service/user_uploads/telemetry_b16bb2.json - fileName: telemetry.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/telemetry_b16bb2.json?sv=2024-05-04&st=2024-05-30T06%3A34%3A15Z&se=2024-05-30T06%3A44%3A15Z&sr=b&sp=w&sig=0zWi7R9wqUT6uW0QuN%2F0nyvaloSCqRPTvzHn%2BpkCWdk%3D - - filePath: >- - obsrv-onest/api-service/user_uploads/school_data_14092a.json - fileName: school_data.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/school_data_14092a.json?sv=2024-05-04&st=2024-05-30T06%3A34%3A15Z&se=2024-05-30T06%3A44%3A15Z&sr=b&sp=w&sig=35g%2F46r%2FOI490%2B8pKry9mEbAgXOXznmCyw7L%2FSLITM0%3D + id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from any + Postgres Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + ui_spec: + schema: + type: object + properties: + connector_config: + title: Connector Config + type: object + encrypt: true + properties: + databaseType: + type: string + title: Database Type + enum: + - PostgreSQL + - MySQL + fieldDescription: + - type: string + description: '' + dependencies: + databaseType: + oneOf: + - properties: + databaseType: + enum: + - PostgreSQL + - MySQL + connection_info: + title: Connection Information + type: object + properties: + host: + type: string + title: Database Host + pattern: >- + /^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}(?:/[^\s]*)?|localhost(?:/[^\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/ + fieldDescription: + - type: string + description: '' + port: + type: number + title: Database Port + minimum: 1 + maximum: 65535 + fieldDescription: + - type: string + description: '' + name: + type: string + title: Database Name + pattern: ^[a-zA-Z0-9_]{1,64}$ + fieldDescription: + - type: string + description: '' + username: + type: string + title: Database Username + fieldDescription: + - type: string + description: '' + password: + type: string + title: Database Password + fieldDescription: + - type: string + description: '' + schemaInfo: + title: Schema Information + type: object + properties: + table: + title: Table Name + type: string + pattern: ^[a-zA-Z_][a-zA-Z0-9_]{0,62}$ + fieldDescription: + - type: string + description: '' + timestampColumn: + title: Timestamp Column + type: string + fieldDescription: + - type: string + description: '' + operations_config: + title: Operations Configuration + type: object + properties: + batch_size: + type: number + title: Batch Size + default: 100 + fieldDescription: + - type: string + description: '' + max_batches: + type: number + title: Maximum Batches + default: 10 + fieldDescription: + - type: string + description: '' + pollingInterval: + type: string + title: Polling Interval + enum: + - Once + - Periodic + fieldDescription: + - type: string + description: Select polling interval + dependencies: + pollingInterval: + oneOf: + - properties: + pollingInterval: + enum: + - Periodic + schedule: + type: string + title: Schedule + enum: + - Hourly + - Daily + - Weekly + - Monthly + fieldDescription: + - type: string + description: '' + required: + - schedule + properties: + connector_config: + connection_info: + password: + ui:widget: password + operations_config: + batch_size: + ui:readonly: true + max_batches: + ui:readonly: true + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' example-1: - summary: 'Success: Generate Get URL' + summary: 'Success: Read Draft Connector' value: - id: api.files.generate-url + id: api.connectors.read ver: v2 - ts: '2024-05-30T12:05:47+05:30' + ts: '2024-08-01T12:47:12+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 141717fb-9b94-463d-b0f1-1bab40b187cc + resmsgid: b6fcfb05-246c-4a1b-9eb1-27497ee9b80b responseCode: OK result: - - filePath: obsrv-onest/api-service/user_uploads/telemetry.json - fileName: telemetry.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/telemetry.json?sv=2024-05-04&st=2024-05-30T06%3A35%3A47Z&se=2024-05-30T06%3A45%3A47Z&sr=b&sp=r&sig=UEWKMP9wcETNKA1sXo%2BVI6qZ7Iqk%2BJg6kglaEXz7%2B1Y%3D - - filePath: obsrv-onest/api-service/user_uploads/school_data.json - fileName: school_data.json - preSignedUrl: >- - https://obsrvonest64fd7c37.blob.core.windows.net/obsrv-onest/obsrv-onest/api-service/user_uploads/school_data.json?sv=2024-05-04&st=2024-05-30T06%3A35%3A47Z&se=2024-05-30T06%3A45%3A47Z&sr=b&sp=r&sig=jXzootTAoUAWLYKF9AYwKh9Mzl%2FVctlebTBHVz%2BbaKM%3D - '400': - description: Bad Request + id: mssql-connector-2.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 2.0.0 + description: >- + The MS SQL Connector is used to move data from any MS + SQL Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Draft + ui_spec: {} + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '276' + ETag: + schema: + type: string + example: W/"114-izVC8DsHdeSfau/USVJvnIqZIMQ" + Date: + schema: + type: string + example: Thu, 01 Aug 2024 09:32:48 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object - examples: - example-0: - summary: 'Failure: Invalid access value requested' - value: - id: api.files.generate-url - ver: v2 - ts: '2024-05-30T12:08:23+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: b1da48a0-5381-49a1-9d5f-ce6c7c65fdcf - responseCode: BAD_REQUEST - error: - code: FILES_GENERATE_URL_INPUT_INVALID - message: >- - #properties/request/properties/access/enum should be - equal to one of the allowed values - trace: '' - example-1: - summary: >- - Failure: Number of file url requested exceeds the limit - configured - value: - id: api.files.generate-url - ver: v2 - ts: '2024-05-30T12:10:29+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 5828abd2-2934-40ef-8fa1-a9bca1e7e24e - responseCode: BAD_REQUEST - error: - code: FILES_URL_GENERATION_LIMIT_EXCEED - message: 'Pre-signed URL generation failed: limit exceeded.' - trace: '' - /v2/obsrv/data/sql-query: + example: + id: api.connectors.read + ver: v2 + ts: '2024-08-01T15:02:48+05:30' + params: + status: FAILED + resmsgid: 712e7298-99f8-4694-9011-4232fcfd664a + responseCode: NOT_FOUND + result: {} + error: + code: CONNECTOR_NOT_FOUND + message: 'Connector not found: postgres-conn' + /v2/connector/register: post: tags: - - Dataset CRUD APIs - summary: SQL Wrapper - description: This api is a wrapper api, used to query the data. + - Connector API's + summary: Register Connector + description: Registers a connector by uploading a file. + operationId: registerConnector requestBody: + required: true content: - application/json: + multipart/form-data: schema: type: object - example: - query: >- - SELECT "channel","flags", COUNT(*) AS "Count" FROM "wikipedia" - WHERE "page" IS NOT NULL GROUP BY 1, 2 ORDER BY 3 DESC + properties: + file3: + type: string + format: binary + description: Connector distribution file to upload. responses: '200': - description: OK - content: - application/json: - schema: - type: object - example: - - channel: '#en.wikipedia' - flags: '' - Count: 4776 - - channel: '#fr.wikipedia' - flags: '!M' - Count: 31 - - channel: '#fa.wikipedia' - flags: B - Count: 30 - - channel: '#vi.wikipedia' - flags: 'N' - Count: 9 - - channel: '#ar.wikipedia' - flags: '!N' - Count: 8 - - channel: '#da.wikipedia' - flags: '!' - Count: 8 - - channel: '#hy.wikipedia' - flags: '!N' - Count: 8 - - channel: '#id.wikipedia' - flags: '!N' - Count: 8 - - channel: '#eu.wikipedia' - flags: M - Count: 3 - - channel: '#fa.wikipedia' - flags: '!N' - Count: 3 - - channel: '#fi.wikipedia' - flags: 'N' - Count: 3 - - channel: '#he.wikipedia' - flags: NB - Count: 3 - - channel: '#hi.wikipedia' - flags: '' - Count: 1 - - channel: '#hr.wikipedia' - flags: '!' - Count: 1 - - channel: '#hr.wikipedia' - flags: '!N' - Count: 1 - '400': - description: Bad Request + description: Connector registered successfully. content: application/json: schema: type: object - example: - id: api.obsrv.data.sql-query - ver: v2 - ts: '2024-05-30T16:31:57+05:30' - params: - status: FAILED - resmsgid: 7709ef5b-d7f3-48a5-bf6c-cf243fb6d2eb - responseCode: BAD_REQUEST - error: - code: SQL_QUERY_EMPTY_REQUEST - message: Failed to query. Invalid request - trace: '' + properties: + id: + type: string + example: api.connector.register + ver: + type: string + example: v2 + ts: + type: string + format: date-time + example: "2024-11-19T09:28:47+00:00" + params: + type: object + properties: + status: + type: string + example: SUCCESS + resmsgid: + type: string + example: fc2ce24f-a333-479f-931c-024d61039801 + responseCode: + type: string + example: OK + result: + type: object + properties: + message: + type: string + example: connector registered successfully. '500': - description: Internal Server Error + description: Internal server error while registering the connector. content: application/json: schema: type: object - example: - id: api.obsrv.data.sql-query - ver: v2 - ts: '2024-05-29T13:03:44+05:30' - params: - status: FAILED - resmsgid: 0b4b7168-2b70-406b-84ce-c320b65559eb - responseCode: INTERNAL_SERVER_ERROR - error: - code: ERR_BAD_RESPONSE - message: Request failed with status code 500 - trace: '' - /v2/datasets/status-transition: + properties: + id: + type: string + example: api.connector.register + ver: + type: string + example: v2 + ts: + type: string + format: date-time + example: "2024-11-19T09:50:21+00:00" + params: + type: object + properties: + status: + type: string + example: FAILED + resmsgid: + type: string + example: ebaad9d7-c0d4-47e4-8cde-87fb25eef739 + responseCode: + type: string + example: INTERNAL_SERVER_ERROR + error: + type: object + properties: + code: + type: string + example: FAILED_TO_REGISTER_CONNECTOR + message: + type: string + example: Failed to Extract the File + trace: + type: string + example: "" + /v2/data/in/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset post: tags: - - Dataset CRUD APIs - summary: Dataset Status Transition - description: >- - This API allows you to perform status transition between 2 states. - Allowed status transition are: -
- - Draft to ReadyToPublish when the dataset configuration is complete and configuration is validated, ReadyToPublish to Live to make dataset live and make it actively running within the pipeline, Live to Retired when dataset is needed to be stopped and retired.User can even Delete the dataset of Draft and ReadyToPublish status. + - Data Ingest + summary: Ingest Data requestBody: - content: - application/json: - examples: - example1: - summary: Transition to ReadyToPublish - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: ReadyToPublish - example2: - summary: Transition to Live - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Live - example3: - summary: Transition to Retired - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Retire - example4: - summary: Delete a draft dataset - value: - id: api.datasets.status-transition - ver: v2 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - dataset_id: telemetry-events - status: Delete + content: + application/json: + schema: + type: object + example: + id: api.data.in + ver: v2 + ts: '1711966306164' + params: + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + data: + eid: INTERACT + date: '2022-01-01' + ver: '3.0' + syncts: 1668591949682 + ets: 1668591949682 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '261' + ETag: + schema: + type: string + example: W/"105-UtaCmh0qZMBRBdniNq74Gr+4YAo" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:05:48 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Delete success: Deleted dataset successfully' - value: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T12:18:54+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 5948e784-37f9-4a70-85ca-86c9077ee30b - responseCode: OK - result: - message: Dataset deleted successfully - dataset_id: master.1 - example-1: - summary: 'Live success: Dataset published successfully' - value: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T12:21:42+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 88d62970-97be-472f-9ccc-67f875d69335 - responseCode: OK - result: - message: Dataset published successfully - dataset_id: telemetry - example-2: - summary: 'ReadyToPublish success: Dataset is ready to publish' + summary: 'Success: Data ingested successfully' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:30:04+05:30' + ts: '2024-05-30T23:35:48+05:30' params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 84858e85-6a97-43cb-b8e9-17a7e0a43365 - responseCode: OK - result: - message: Dataset status transition to ReadyToPublish successful - dataset_id: telemetry-events - example-3: - summary: 'Retire success: Dataset retired successfully' + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 9a44ac5b-ef82-46f7-92c5-c5b39ef88764 + responseCode: OK + result: + message: Data ingested successfully + example-1: + summary: 'Success: Data ingested successfully (batch)' value: - id: api.datasets.status + id: api.data.in ver: v2 - ts: '2024-05-30T12:22:58+05:30' + ts: '2024-05-30T23:36:49+05:30' params: status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: f2285754-7d5b-4320-943d-797fb136e955 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: a220041c-0c28-415a-9687-9fb2e211f8c4 responseCode: OK result: - message: Dataset retired successfully - dataset_id: sb-telemetry + message: Data ingested successfully '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '300' + ETag: + schema: + type: string + example: W/"12c-GNCIUUDxOZD3UfM311sjnmFIgPc" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:03:32 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.data.in + ver: v2 + ts: '2024-05-30T23:33:32+05:30' + params: + status: FAILED + resmsgid: acf07609-77de-4ab5-81ea-42e41b8b95ff + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_INGESTION_INVALID_INPUT + message: '#required should have required property ''id''' + trace: '' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '316' + ETag: + schema: + type: string + example: W/"13c-O9iirC/EyneYXQzth7iwEEy1UV4" + Date: + schema: + type: string + example: Thu, 18 Apr 2024 10:17:00 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Live failure: Dataset in draft state' + summary: Entry topic not found value: - id: api.datasets.status + id: api.data.in ver: v2 - ts: '2024-05-30T15:37:43+05:30' + ts: '2024-04-18T15:47:00+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: d56e2ed4-f008-48be-a501-164c19178419 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 11c32a3a-fdeb-4e00-a9cf-f6433ade5608 responseCode: BAD_REQUEST + result: {} error: - code: DATASET_NOT_READY_FOR_PUBLISH - message: Failed to publish dataset as it is in draft state + code: TOPIC_NOT_FOUND + message: Entry topic is not defined trace: '' example-1: - summary: 'ReadyToPublish failure: Incomplete dataset configs' + summary: 'Failure: Dataset not found' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:36:16+05:30' + ts: '2024-05-30T23:37:29+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: a504565b-41ff-4c0f-9d64-f96df9ed89bb + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: d4db36b4-37b0-4170-a4cf-9d2ae8fa0416 responseCode: BAD_REQUEST + result: {} error: - code: DATASET_CONFIGS_INVALID - message: >- - #properties/denorm_config/properties/denorm_fields/items/required - must have required property 'dataset_name' + code: DATASET_NOT_FOUND + message: Dataset with id not found trace: '' example-2: - summary: 'ReadyToPublish failure: Dataset not in draft state' + summary: 'Failure: Entry topic not found' value: - id: api.datasets.status-transition + id: api.data.in ver: v2 - ts: '2024-06-18T15:38:14+05:30' + ts: '2024-05-30T23:39:26+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 351f5a37-87f0-47cd-bebe-e3c001256d0a + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 62309380-3e83-442f-877e-359ed2774bbf responseCode: BAD_REQUEST + result: {} error: - code: DATASET_READYTOPUBLISH_FAILURE - message: >- - Failed to mark dataset Ready to publish as it not in - draft state + code: TOPIC_NOT_FOUND + message: Entry topic is not defined trace: '' - example-3: - summary: 'Retire Failure: Dataset is already retired' + /v2/data/query/{dataset_id}: + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + description: Unique identifier for the dataset + post: + tags: + - Data Query + summary: Query Data + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.data.out + ver: v2 + ts: '1711966306164' + params: + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + context: + aggregationLevel: day + query: >- + Select * from "check" WHERE __time >= timestamp '2020-12-31' + AND __time < TIMESTAMP '2024-01-21' Limit 1 + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '313' + ETag: + schema: + type: string + example: W/"139-De+IthAwrGNR+J11CwlNf5RSMmw" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:17:45 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: native query (interval as array type)' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T15:42:18+05:30' + ts: '2024-05-30T23:47:45+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 80208169-b1d3-41cd-816b-83fae96a4370 - responseCode: BAD_REQUEST - error: - code: DATASET_ALREADY_RETIRED - message: Dataset is already retired - trace: '' - example-4: + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 134efe35-c096-4cab-ad14-db6a8952f264 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + a1: 1694390400000 + a0: 1694390400000 + example-1: + summary: 'Success: native query (queryType: scan)' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:51:14+05:30' + params: + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 4fc6e3c1-4715-47b7-9137-7713fb2fbe72 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + a1: 1694390400000 + a0: 1694390400000 + example-2: summary: >- - Retire failure: Cannot retire master dataset as it is used - by other datasets + Success: native query (normal intervals & queryType : + timeseries) value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T16:01:41+05:30' + ts: '2024-05-30T23:51:49+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: b88c320a-2c01-4662-a509-bd532a612c05 - responseCode: BAD_REQUEST - error: - code: DATASET_IN_USE - message: Failed to retire dataset as it is used by other datasets - trace: '' - example-5: - summary: 'Failure: Invalid request payload provided' + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75d7ae48-a3db-4367-b50f-34eb99ac3480 + responseCode: OK + result: + - timestamp: '2023-09-11T00:00:00.000Z' + result: + school_id: 0 + example-3: + summary: 'Success: sql query' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T16:03:56+05:30' + ts: '2024-05-30T23:42:12+05:30' params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: ba4c86bd-b438-4582-b178-2410a5c5dd15 - responseCode: BAD_REQUEST - error: - code: DATASET_STATUS_INVALID_INPUT - message: >- - #properties/request/properties/status/enum should be - equal to one of the allowed values - trace: '' + status: SUCCESS + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 2c981011-76da-3000-97f3-eafac939e59f + responseCode: OK + result: + - __time: '2023-09-11T00:00:00.000Z' + school_category: secondary + gender: others + state_id: '15' + district_id: '2002' + block_id: '70' + cluster_id: '485' + obsrv.meta.source.connector: null + obsrv.meta.source.id: null + grade_sum: 18 + school_id_sum: 180378 + students_marked_sum: 12492 + students_present_sum: 2466 + total_count: 18 + total_students_sum: 12492 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '371' + ETag: + schema: + type: string + example: W/"173-OP8NcbSqLKFO92PIyUmMk0lNsXs" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:16:29 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:46:29+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: b6434700-dd92-4f64-9250-a22939e753b9 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_OUT_INVALID_DATE_RANGE + message: >- + Invalid date range! make sure your range cannot be more than + 30 days + trace: '' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '356' + ETag: + schema: + type: string + example: W/"164-DSmPP0WJI5ISEqIw3U3B1NFXxVE" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:13:12 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Delete failure: Dataset not found to delete' + summary: 'Failure: Datasource not found in live table' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T12:25:36+05:30' + ts: '2024-05-30T23:43:12+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 3cdcf2af-c015-4977-9d66-364e00f1712b + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: f922f120-2aea-49af-9a76-7312fe2eb266 responseCode: NOT_FOUND + result: {} error: - code: DATASET_NOT_FOUND - message: Dataset not found to delete + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Datasource telemetry-eventssss not available for + querying trace: '' example-1: - summary: 'Live failure: Dataset not found to publish' - value: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T15:35:59+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 427b3b1a-a0d2-4255-91d9-04ee4a1f0e3c - responseCode: NOT_FOUND - error: - code: DATASET_NOT_FOUND - message: Dataset not found to publish - trace: '' - example-2: - summary: 'Retire Failure: Dataset not found to retire' + summary: 'Failure: Datasource not found in druid' value: - id: api.datasets.status + id: api.data.out ver: v2 - ts: '2024-05-30T15:40:31+05:30' + ts: '2024-05-30T23:50:27+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 73befbbd-60e3-48e0-9cfd-cb705dfc2b85 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75bd4313-d504-4fd3-92ab-ee2a685beb83 responseCode: NOT_FOUND + result: {} error: - code: DATASET_NOT_FOUND - message: Dataset not found to retire - trace: '' - '500': - description: Internal Server Error - content: - application/json: - schema: - type: object - example: - id: api.datasets.status - ver: v2 - ts: '2024-05-30T16:24:12+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 92928434-719f-47d4-9946-1e40ecd53253 - responseCode: INTERNAL_SERVER_ERROR - error: - code: DATASET_STATUS_FAILURE - message: Failed to perform status transition on datasets - trace: '' - /v2/data/in/{datasetId}: - post: - tags: - - Data Ingest - parameters: - - required: true - schema: - title: datasetId - type: string - name: datasetId - in: path - summary: Data ingest - description: >- - This API allows you to ingest events to a data streaming platform in individual or batch format. - requestBody: - content: - application/json: - examples: - example1: - summary: Ingest individual event - value: - id: api.data.in - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - data: - eid: INTERACT - date: '2022-01-01' - ver: '3.0' - syncts: 1668591949682 - ets: 1668591949682 - example2: - summary: Ingest batch event - value: - id: api.data.in - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - data: [{ "eid": "INTERACT","date": "2022-01-01","ver": "3.0","syncts": 1668591949682}] - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: Data ingest successful(individual) - value: - id: api.data.in - ver: v2 - ts: 1711966306164 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - message: Data ingested successfully - example-1: - summary: Data ingest successfully(batch) - value: - id: api.data.in - ver: v2 - ts: 1711966352987 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - message: Data ingested successfully - '400': - description: Bad Request - content: - application/json: - schema: - type: object - example: - id: api.data.in - ver: v2 - ts: 1711966410766 - params: - status: FAILED - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: DATA_INGESTION_INVALID_INPUT - message: "#required should have required property 'id'" - trace: '' - '404': - description: Not Found - content: - application/json: - schema: - type: object - examples: - example-0: - summary: Dataset not found - value: - id: api.data.in - ver: v2 - ts: 1711966446364 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: DATASET_NOT_FOUND - message: Dataset with id not found - trace: '' - example-1: - summary: Entry topic not found - value: - id: api.data.in - ver: v2 - ts: 1711975482739 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 98020cf9-4239-4b6e-beea-c687d21fafa3 - responseCode: BAD_REQUEST - error: - code: ENTRY_TOPIC_NOT_FOUND - message: Entry topic is not defined - trace: '' - /v2/data/query/{datasetId}: - post: - tags: - - Data query - summary: Data query - parameters: - - required: true - schema: - title: datasetId - type: string - name: datasetId - in: path - description: >- - This API allows you to query your datasource. - requestBody: - content: - application/json: - examples: - example1: - summary: SQL query - value: - id: api.data.out - ver: v2 - ts: "1711966306164" - params: - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - context: - datasetId: rollups-configured - table: day - query: >- - SELECT * FROM "rollups-configured" WHERE __time >= TIMESTAMP - '2020-12-31' AND __time < TIMESTAMP '2024-01-21' LIMIT 1 - example2: - summary: Native query - value: - id: api.data.out - ver: v2 - ts: "1711966306164" - params: - mid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - context: - datasetId: rollups-configured - table: day - query: - quertType: scan - dataSource: rollups-configured - intervals: - type: intervals - intervals: ["2022-01-01/2022-02-01"] - granularity: day - columns: ["eid","__time"] - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Success: sql query' - value: - id: api.data.out - ver: v2 - ts: 1711971972855 - params: - status: "SUCCESS" - msgid: "e180ecac-8f41-4f21-9a21-0b3a1a368917" - resmsgid: "ba6391e2-8c35-4c3f-8883-f503b4ffd6a5" - responseCode: OK - result: - - __time: '2022-01-01T00:00:00.000Z' - eid: '1' - ver: '3.0' - flags_ex_processed: 'true' - flags_pp_validation_processed: 'true' - flags_pp_duplicate_skipped: 'true' - flags_device_denorm: 'true' - flags_user_denorm: 'true' - flags_loc_denorm: 'true' - derivedlocationdata_district: AGRA - derivedlocationdata_from: user-profile - derivedlocationdata_state: Uttar Pradesh - mid: 6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0 - type: events - actor_type: User - actor_id: 311663b2-d7de-4d46-8803-20407eaa3403 - edata_type: session - userdata_subject: null - userdata_district: AGRA - userdata_usersubtype: hm - userdata_grade: null - userdata_usersignintype: Self-Signed-In - userdata_usertype: administrator - userdata_userlogintype: administrator - userdata_state: Uttar Pradesh - devicedata_statecustomcode: '29' - devicedata_country: India - devicedata_iso3166statecode: IN-KA - devicedata_city: Bengaluru - devicedata_countrycode: IN - devicedata_state: Karnataka - devicedata_devicespec_idisk: '106.47' - devicedata_devicespec_webview: 107.0.5304.105 - devicedata_devicespec_os: Android 12 - devicedata_devicespec_scrn: '6.53' - devicedata_devicespec_sims: '-1' - devicedata_devicespec_cpu: "abi: arm64-v8a processor\t: 0 " - devicedata_devicespec_id: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 - devicedata_devicespec_camera: '' - devicedata_devicespec_edisk: '106.27' - devicedata_devicespec_make: vivo 1915 - devicedata_statecode: KA - devicedata_districtcustom: BENGALURU URBAN SOUTH - devicedata_statecustomname: Karnataka - devicedata_userdeclared_district: AGRA - devicedata_userdeclared_state: Uttar Pradesh - context_cdata_id: a3c784f0-61d8-43e4-a92a-373fd4338c1d - context_cdata_type: UserSession - context_env: sdk - context_channel: '0126796199493140480' - context_pdata_id: preprod.diksha.app - context_pdata_pid: sunbird.app - context_pdata_ver: 4.10.1023preproduction - context_sid: a3c784f0-61d8-43e4-a92a-373fd4338c1d - context_did: ac4ad4ac3feda0f2b17835b81e736c88c194dc89 - context_rollup_l1: '0126796199493140480' - object_id: '' - object_type: '' - object_version: '' - object_rollup: '{}' - count: 3 - edata_duration_sum: null - example-1: - summary: 'Success: native query' - value: - id: api.data.out - ver: v2 - ts: 1711973520793 - params: - status: SUCCESS - errmsg: '' - responseCode: OK - result: - - timestamp: '2022-01-01T00:00:00.000Z' - result: - mid: 1 - '400': - description: Bad Request - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: invalid date range (sql)' - value: - id: api.data.out - ver: v2 - ts: 1711972193426 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: INVALID_DATE_RANGE - message: Invalid date range! make sure your range cannot be more than 30 days - trace: "" - example-1: - summary: 'Failure: Invalid date range (native)' - value: - id: api.data.out - ver: v2 - ts: 1711972501847 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: INVALID_DATE_RANGE - message: Invalid date range! make sure your range cannot be more than 30 days - trace: "" - '404': - description: Not Found - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: Datasource not found in druid' - value: - id: api.data.out - ver: v2 - ts: 1711972085550 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip not found - trace: "" - example-1: - summary: 'Failure: Datasource not found in live table' - value: - id: api.data.out - ver: v2 - ts: 1711975034302 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip not found - trace: "" - example-2: - summary: 'Failure: Datasource not found in live table (native)' - value: - id: api.data.out - ver: v2 - ts: 1711974978635 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip.1_day not available for querying - trace: "" - example-3: - summary: 'Failure: Datasource not found in druid(native)' - value: - id: api.data.out - ver: v2 - ts: 1711972569115 - params: - status: FAILED - msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 - resmsgid: 5aaf81d1-675d-4071-883a-f92d96656aa4 - responseCode: BAD_REQUEST - error: - code: DATASOURCE_NOT_FOUND - message: Datasource taxt_trip.1_day not available for querying - trace: "" + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Dataset test with table day is not available for + querying + trace: '' + example-2: + summary: 'SQL Failure: dataset not found in druid' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:52:26+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 19849769-c290-4599-a79e-862dcbce1124 + responseCode: NOT_FOUND + result: {} + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: >- + Dataset test with table day is not available for + querying + trace: '' + example-3: + summary: 'SQL Failure: Datasource not found in live table' + value: + id: api.data.out + ver: v2 + ts: '2024-05-30T23:53:02+05:30' + params: + status: FAILED + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 530a12f9-04f3-4986-8f52-b3184b9194bd + responseCode: NOT_FOUND + result: {} + error: + code: DATA_OUT_SOURCE_NOT_FOUND + message: Datasource undefined not available for querying + trace: '' /v2/template/read/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template get: tags: - Query Templates - summary: Read template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path + summary: Read Template responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '537' + ETag: + schema: + type: string + example: W/"219-BOO8L4HgRkX4zEKoNwUwfDQA+uU" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:24:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3362,10 +8088,10 @@ paths: value: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:09:55+05:30' + ts: '2024-05-30T23:54:14+05:30' params: status: SUCCESS - resmsgid: 8ef3a9d0-8658-47dd-b6cb-a4b1777a1ea4 + resmsgid: ad1c0a98-1ba3-4203-9fac-59ccaf442347 responseCode: OK result: template_id: jsontemplate111 @@ -3382,24 +8108,54 @@ paths: value: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:10:54+05:30' + ts: '2024-05-30T23:54:38+05:30' params: status: SUCCESS - resmsgid: 656185ef-527d-4b60-ac19-263da85142b8 + resmsgid: 70047866-a4b4-4fce-b9c4-4699fcab2dc6 responseCode: OK result: template_id: sql1 template_name: sql1 query: >- - "SELECT * FROM {{DATASET}} WHERE __time BETWEEN - TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}" + "SELECT COUNT(*) FROM \"{{DATASET}}\" WHERE \"__time\" + BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP + {{ENDDATE}} lIMIT {{LIMIT}}" query_type: sql created_by: SYSTEM updated_by: SYSTEM - created_date: '2024-04-29T11:29:58.759Z' - updated_date: '2024-04-29T11:29:58.759Z' + created_date: '2024-05-13T07:29:04.117Z' + updated_date: '2024-05-13T07:29:04.117Z' '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '294' + ETag: + schema: + type: string + example: W/"126-zq5j1yor+xr1XFEEbP09fQTDI/k" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:25:16 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3407,30 +8163,60 @@ paths: example: id: api.query.template.read ver: v2 - ts: '2024-04-30T12:11:45+05:30' + ts: '2024-05-30T23:55:16+05:30' params: status: FAILED - resmsgid: bc6a4621-4ac5-4959-b467-1134ea874960 + resmsgid: 958bf0e7-bdb8-4153-abdc-ab84e8004a0e responseCode: NOT_FOUND + result: {} error: code: QUERY_TEMPLATE_NOT_EXISTS message: Template sql100 does not exists trace: '' /v2/template/delete/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template delete: tags: - Query Templates - summary: Delete template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path + summary: Delete Template responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '241' + ETag: + schema: + type: string + example: W/"f1-FblV17jkZ3FSGFt5MHx6s6dlMuo" + Date: + schema: + type: string + example: Tue, 30 Apr 2024 06:28:27 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3447,6 +8233,35 @@ paths: message: Template yashash-k deleted successfully '404': description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '303' + ETag: + schema: + type: string + example: W/"12f-99pWw8VTwuVfDAhinC55JXfNyyg" + Date: + schema: + type: string + example: Thu, 30 May 2024 18:28:41 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3454,27 +8269,21 @@ paths: example: id: api.query.template.delete ver: v2 - ts: '2024-04-29T18:29:09+05:30' + ts: '2024-05-30T23:58:41+05:30' params: status: FAILED - resmsgid: 6c62ff4d-0a63-4384-a365-7bbef951f288 + resmsgid: 9b7f81fb-6705-4d32-9bd3-139cd5a211b9 responseCode: NOT_FOUND + result: {} error: code: QUERY_TEMPLATE_NOT_EXISTS message: Template json_template does not exists trace: '' - /v2/template/create/{template_id}: + /v2/template/create: post: tags: - Query Templates summary: Create query template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path requestBody: content: application/json: @@ -3493,9 +8302,44 @@ paths: datasetId: '{{DATASET}}' intervals: '{{STARTDATE}}/{{ENDDATE}}' limit: '{{LIMITS}}' + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '359' + ETag: + schema: + type: string + example: W/"167-HIMd6+dVF/Wyu6lcmb/+68O4AY4" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 16:59:01 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3533,6 +8377,35 @@ paths: message: The query template has been saved successfully '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '407' + ETag: + schema: + type: string + example: W/"197-y0n7/XzKhcV9HKqgPNj2eo8bzh8" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 17:05:10 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3549,6 +8422,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 62e18342-7e25-4122-8fca-6fb12fac3ff0 responseCode: BAD_REQUEST + result: {} error: code: QUERY_TEMPLATE_INVALID message: >- @@ -3566,6 +8440,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: d2b598b5-62c1-4c5d-b0b3-5d7d109a2bc2 responseCode: BAD_REQUEST + result: {} error: code: QUERY_TEMPLATE_INVALID_INPUT message: >- @@ -3574,6 +8449,35 @@ paths: trace: '' '409': description: Conflict + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '350' + ETag: + schema: + type: string + example: W/"15e-FDXFj2WIyZ1MVllwsiSJoBKU4GQ" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 17:03:28 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -3587,6 +8491,7 @@ paths: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d resmsgid: 18b6b123-4df5-4124-b6ec-73b667250e1c responseCode: CONFLICT + result: {} error: code: QUERY_TEMPLATE_ALREADY_EXISTS message: Template josnaks-aaa already exists @@ -3595,7 +8500,7 @@ paths: post: tags: - Query Templates - summary: List templates + summary: List Templates requestBody: content: application/json: @@ -3616,6 +8521,35 @@ paths: responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '6864' + ETag: + schema: + type: string + example: W/"1ad0-xp24UiXXXiFWplmv5Acja7prSYM" + Date: + schema: + type: string + example: Mon, 29 Apr 2024 13:46:03 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -4176,17 +9110,17 @@ paths: created_date: '2024-04-25T06:31:20.819Z' updated_date: '2024-04-25T06:31:20.819Z' /v2/template/update/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template patch: tags: - Query Templates - summary: update template - parameters: - - required: true - schema: - title: template_id - type: string - name: template_id - in: path + summary: Update Template requestBody: content: application/json: @@ -4207,6 +9141,35 @@ paths: responses: '200': description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '314' + ETag: + schema: + type: string + example: W/"13a-jsb3kdb5RR9P3vnOhZWsAWEr37k" + Date: + schema: + type: string + example: Fri, 10 May 2024 05:51:47 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: @@ -4214,136 +9177,958 @@ paths: example: id: api.query.template.update ver: v2 - ts: '2024-05-02T13:16:31+05:30' + ts: '2024-05-10T11:21:47+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: ab869754-d2f0-4b9a-b38b-04d9e5901b5d + resmsgid: 9e4a6959-0eb9-4fc4-8e6f-2eea534d1384 responseCode: OK result: message: Query template updated successfully templateId: sql11template1 '400': description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '429' + ETag: + schema: + type: string + example: W/"1ad-5sb8WUekFL8s4c1Ink6bUByoHho" + Date: + schema: + type: string + example: Fri, 10 May 2024 05:53:54 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: required variables not exists to update' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T11:23:54+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 66b95cb3-2ef2-4735-9045-2674da552dbd + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + Invalid template provided, A template should consist of + variables DATASET,STARTDATE,ENDDATE and type of json,sql + trace: '' + example-1: + summary: 'Failure: Template name validation failure' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T11:26:59+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: d6428fcf-53c9-465d-9431-769218f775b8 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + Template name should contain alphanumeric characters and + single space between characters + trace: '' + example-2: + summary: 'Failure: query_type should when updating query' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T12:32:57+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: c7a8675a-73f2-4764-abba-bfdf9f8b4621 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/dependencies should have property + query_type when property query is present + trace: '' + example-3: + summary: 'Failure: query should present when updating query_type' + value: + id: api.query.template.update + ver: v2 + ts: '2024-05-10T12:34:55+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 2c1098b2-d7b3-4d39-98ee-e3e790fd23b4 + responseCode: BAD_REQUEST + result: {} + error: + code: QUERY_TEMPLATE_INVALID_INPUT + message: >- + #properties/request/dependencies should have property + query when property query_type is present + trace: '' + /v2/template/query/{template_id}: + parameters: + - name: template_id + in: path + required: true + schema: + type: string + description: Unique identifier for the template + post: + tags: + - Query Templates + summary: Query Template + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.query.template.query + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + startdate: '2020-12-31' + enddate: '2024-12-31' + aggregationLevel: month + dataset: test + limit: 5 + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '301' + ETag: + schema: + type: string + example: W/"12d-9hKB38iHEwYPT2MgF8puXcq05Ew" + Date: + schema: + type: string + example: Tue, 14 May 2024 06:22:24 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: JSON template with request body' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-14T11:52:24+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d + responseCode: OK + result: + - timestamp: '2023-09-01T00:00:00.000Z' + result: + school_id: 0 + example-1: + summary: 'Success: SQL template query with request body' + value: + id: api.query.template.query + ver: v2 + ts: '2024-05-14T11:33:06+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 48c194ee-6e73-4ee7-83e6-8b154e441911 + responseCode: OK + result: + - __time: '2023-09-11T00:00:00.000Z' + school_category: secondary + gender: others + state_id: '15' + district_id: '2002' + block_id: '70' + cluster_id: '485' + obsrv.meta.source.connector: null + obsrv.meta.source.id: null + grade_sum: 18 + school_id_sum: 180378 + students_marked_sum: 12492 + students_present_sum: 2466 + total_count: 18 + total_students_sum: 12492 + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '336' + ETag: + schema: + type: string + example: W/"150-T/XeSIt7PR7GcGEbET1e8n9zX7k" + Date: + schema: + type: string + example: Thu, 02 May 2024 07:29:14 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 content: application/json: schema: type: object examples: example-0: - summary: 'Failure: query should present when query_type provided' + summary: 'Failure: invalid date range (native template)' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:30:36+05:30' + ts: '2024-05-02T12:59:14+05:30' params: status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: f11f5287-77cd-43f0-b7f6-6d6b94b9cea2 + resmsgid: 4379e16b-2fa3-46a8-8ded-bc53f56283e9 responseCode: BAD_REQUEST + result: {} error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_INVALID_DATE_RANGE message: >- - #properties/request/dependencies should have property - query when property query_type is present + Invalid date range! make sure your range cannot be more + than 30 days trace: '' example-1: - summary: 'Failure: query_type should present when query is given' - value: - id: api.query.template.update - ver: v2 - ts: '2024-05-02T11:33:23+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 937cf790-867b-450b-afc9-86a96d02f636 - responseCode: BAD_REQUEST - error: - code: QUERY_TEMPLATE_INVALID_INPUT - message: >- - #properties/request/dependencies should have property - query_type when property query is present - trace: '' - example-2: - summary: 'Failure: template name validation failure' + summary: 'Failure: invalid date range' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:34:18+05:30' + ts: '2024-05-13T13:28:18+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 9f3ae5aa-2e98-4204-8f87-b11e5bb3865a + resmsgid: 20391fb8-2be8-48b5-a16f-fca150580e97 responseCode: BAD_REQUEST error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_INVALID_DATE_RANGE message: >- - Template name should contain alphanumeric characters and - single space between characters + Invalid date range! make sure your range cannot be more + than 30 days trace: '' - example-3: - summary: 'Failure: required template_id' + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '357' + ETag: + schema: + type: string + example: W/"165-Q7Qi9SUmHUwU75fy/RFrXL9Pp3U" + Date: + schema: + type: string + example: Mon, 13 May 2024 07:51:46 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Datasource not found in druid' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T11:38:16+05:30' + ts: '2024-05-13T13:21:46+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 94b71f80-bd1e-4283-8f3d-83833b60d08c - responseCode: BAD_REQUEST + resmsgid: b35a7050-b94c-4944-9630-233c9542272e + responseCode: NOT_FOUND error: - code: QUERY_TEMPLATE_INVALID_INPUT + code: DATA_OUT_SOURCE_NOT_FOUND message: >- - #properties/request/required should have required - property 'template_id' + Dataset test with table hour is not available for + querying trace: '' - example-4: - summary: 'Failure: required variables not exists' + example-1: + summary: 'Failure: Datasource not found in live table' value: - id: api.query.template.update + id: api.query.template.query ver: v2 - ts: '2024-05-02T13:13:22+05:30' + ts: '2024-05-13T13:23:47+05:30' params: status: FAILED msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: dff3c22a-c198-4d1c-b87a-07d725f868f7 - responseCode: BAD_REQUEST + resmsgid: 3a303dfd-1d95-4788-b1a7-d88809d4dcf3 + responseCode: NOT_FOUND error: - code: QUERY_TEMPLATE_INVALID - message: >- - Invalid template provided, A template should consist of - variables DATASET,STARTDATE,ENDDATE and type of json,sql + code: DATA_OUT_SOURCE_NOT_FOUND + message: Datasource test11 not available for querying trace: '' - /v2/template/query/{template_id}: + /alerts/v1/notifications/search: + post: + tags: + - Alert Notification Channels + summary: Search Notificaation Channels + requestBody: + content: {} + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/create: + post: + tags: + - Alert Notification Channels + summary: Configure channel + requestBody: + content: + application/json: + schema: + type: object + example: + manager: grafana + name: functional-metrics-email + type: email + config: + recipientAddresses: yravinderkumar33@gmail.com;ravinder@sanketika.in + subject: Obsrv Prod Alert + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/publish/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alert Notification Channels + summary: Publish Channel + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/test: + post: + tags: + - Alert Notification Channels + summary: Verify Configured Channel + requestBody: + content: + application/json: + schema: + type: object + example: + message: Testing Email integration. If you can read this, it's working! + payload: + error: + selectChannel: true + configureChannel: true + manager: grafana + name: udhw + type: email + config: + recipientAddresses: jerald@sanketika.in + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alert Notification Channels + summary: Update Specific Channel + requestBody: + content: + application/json: + schema: + type: object + example: + name: updated name + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/notifications/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert Notification Channels + summary: Delete Channel + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/create: + post: + tags: + - Alert Silence + summary: Add Silence + requestBody: + content: + application/json: + schema: + type: object + example: + startDate: '2024-01-11T07:56:23Z' + endDate: '2024-01-11T08:56:23Z' + alertId: c7464d32-1d8d-4eaf-9b23-1313a3ff8149 + manager: grafana + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/get/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alert Silence + summary: Get Silence + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert Silence + summary: Delete Silence + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/search: + get: + tags: + - Alert Silence + summary: Fetch All Silences + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/silence/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alert Silence + summary: Modify Silence Request + requestBody: + content: + application/json: + schema: + type: object + example: + startDate: '2023-08-09T10:50:59Z' + endDate: '2023-08-10T10:30:59Z' + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/delete/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alerts Wrapper + summary: 'Delete Configured Alert Rules ' + description: This URLwill provided access to user to delete any custom rule. + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + patch: + tags: + - Alerts Wrapper + summary: 'Update Rule ' + description: >- + This URL will provide access to users to edit any properties in the + Alert rule and save the changes. + requestBody: + content: + application/json: + schema: + type: object + example: + description: >- + This alert rule is designed to promptly notify you when one or + more servers in your infrastructure become unresponsive or + inaccessible. + labels: + severity: warning + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/publish/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alerts Wrapper + summary: 'Publish Alert Rules ' + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/create: + post: + tags: + - Alerts Wrapper + summary: 'Add Rule ' + requestBody: + content: + '*/*': + schema: + type: string + example: >- + "{\n \"name\": \"Total Api Calls1\",\n + \"manager\":\"grafana\",\n \"description\": \"This alert is + set up to notify you when the CPU usage on a host reaches a Low + level.\",\n \"expression\": \"(node_total_api_calls) > + 20\",\n \"category\": \"Infra\",\n + \"severity\":\"warning\",\n \"frequency\": \"1m\",\n + \"interval\": \"1m\",\n \"labels\": {\n \"component\": + \"api\",\n \"notificationChannel\": \"slack\"\n + },\n \"annotations\":{\n \"summary\":\"Host Low CPU + usage\"\n },\n \"metadata\": {\n \"query\": + [\n {\n \"refId\": + \"A\",\n \"datasourceUid\": + \"$datasourceUid\",\n \"queryType\": + \"\",\n \"relativeTimeRange\": + {\n \"from\": 600,\n + \"to\": 0\n },\n \"model\": + {\n \"refId\": \"A\",\n + \"hide\": false,\n \"editorMode\": + \"code\",\n \"expr\": + \"node_total_api_calls\",\n \"legendFormat\": + \"__auto\",\n \"range\": + true\n }\n },\n + {\n \"refId\": \"B\",\n + \"datasourceUid\": \"__expr__\",\n \"queryType\": + \"\",\n \"model\": {\n + \"refId\": \"B\",\n \"hide\": + false,\n \"type\": + \"reduce\",\n \"datasource\": + {\n \"uid\": + \"__expr__\",\n \"type\": + \"__expr__\"\n },\n + \"conditions\": [\n + {\n \"type\": + \"query\",\n \"evaluator\": + {\n \"params\": + [],\n \"type\": + \"gt\"\n + },\n \"operator\": + {\n \"type\": + \"and\"\n + },\n \"query\": + {\n \"params\": + [\n + \"B\"\n + ]\n },\n + \"reducer\": {\n \"params\": + [],\n \"type\": + \"last\"\n }\n + }\n ],\n \"reducer\": + \"last\",\n \"expression\": + \"A\"\n },\n + \"relativeTimeRange\": {\n \"from\": + 600,\n \"to\": 0\n + }\n },\n {\n \"refId\": + \"C\",\n \"datasourceUid\": + \"__expr__\",\n \"queryType\": + \"\",\n \"model\": {\n + \"refId\": \"C\",\n \"hide\": + false,\n \"type\": + \"threshold\",\n \"datasource\": + {\n \"uid\": + \"__expr__\",\n \"type\": + \"__expr__\"\n },\n + \"conditions\": [\n + {\n \"type\": + \"query\",\n \"evaluator\": + {\n \"params\": + [\n + 20\n + ],\n \"type\": + \"gt\"\n + },\n \"operator\": + {\n \"type\": + \"and\"\n + },\n \"query\": + {\n \"params\": + [\n + \"C\"\n + ]\n },\n + \"reducer\": {\n \"params\": + [],\n \"type\": + \"last\"\n }\n + }\n ],\n \"expression\": + \"B\"\n },\n + \"relativeTimeRange\": {\n \"from\": + 600,\n \"to\": 0\n + }\n }\n ]\n }\n}" + parameters: + - name: Accept + in: header + schema: + type: string + example: application/json, text/plain, */* + - name: Accept-Language + in: header + schema: + type: string + example: en-GB,en + - name: Cache-Control + in: header + schema: + type: string + example: no-store + - name: Connection + in: header + schema: + type: string + example: keep-alive + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Cookie + in: header + schema: + type: string + example: >- + connect.sid=s%3A5w5I87Tior-cvNu-SijqRFKGxy_b-WIP.lDKfWAJZbxW0kMaUqj%2B0Ivu%2FvNXXL8S796Fa7%2BNyM9Q; + grafana_session=4eb514f6fef4ad6884e47e50254af650; + grafana_session_expiry=1687239735 + - name: Origin + in: header + schema: + type: string + example: http://localhost:3001 + - name: Pragma + in: header + schema: + type: string + example: no-store + - name: Referer + in: header + schema: + type: string + example: http://localhost:3001/alertRules/add + - name: Sec-Fetch-Dest + in: header + schema: + type: string + example: empty + - name: Sec-Fetch-Mode + in: header + schema: + type: string + example: cors + - name: Sec-Fetch-Site + in: header + schema: + type: string + example: same-origin + - name: Sec-GPC + in: header + schema: + type: integer + example: '1' + - name: User-Agent + in: header + schema: + type: string + example: >- + Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like + Gecko) Chrome/114.0.0.0 Safari/537.36 + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/search: post: tags: - - Query Templates - summary: query template - parameters: - - required: true + - Alerts Wrapper + summary: 'Search Alert Rule' + requestBody: + content: + application/json: + schema: + type: object + example: + request: {} + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/get/{alert_id}: + parameters: + - name: alert_id + in: path + required: true + schema: + type: string + description: Unique identifier for the alert + get: + tags: + - Alerts Wrapper + summary: 'Get Alert Rule' + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/metric/alias/create: + post: + tags: + - Alert Metric Alias + summary: Create Custom Metric + requestBody: + content: + application/json: + schema: + type: object + example: + alias: Druid + component: MyTest + metric: sum(druid_supervisors{state='SUSPENDED'}) + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/metric/alias/search: + post: + tags: + - Alert Metric Alias + summary: List Create Metrics + requestBody: + content: + application/json: + schema: + type: object + example: {} + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/metric/alias/update/{alert_id}: + parameters: + - name: alert_id + in: path + required: true schema: - title: template_id type: string - name: template_id + description: Unique identifier for the alert + patch: + tags: + - Alert Metric Alias + summary: Modify Metric + requestBody: + content: + application/json: + schema: + type: object + example: + alias: Druid native + responses: + '200': + description: Successful response + content: + application/json: {} + /alerts/v1/metric/alias/delete/{alert_id}: + parameters: + - name: alert_id in: path + required: true + schema: + type: string + description: Unique identifier for the alert + delete: + tags: + - Alert Metric Alias + summary: Delete Metric + responses: + '200': + description: Successful response + content: + application/json: {} + /v2/dataset/metrics: + post: + tags: + - Dataset metrics + summary: Dataset metrics requestBody: content: application/json: schema: type: object example: - id: api.query.template.query + id: api.dataset.metrics ver: v2 ts: '2024-04-10T16:10:50+05:30' params: msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d request: - startdate: '2020-12-31' - enddate: '2024-12-31' - aggregationLevel: month - dataset: test - limit: 5 + dataset_id: test + category: + - data_freshness + - data_observability + - data_volume + - data_lineage + - connectors + - data_quality + query_time_period: 1 responses: '200': description: OK @@ -4353,148 +10138,222 @@ paths: type: object examples: example-0: - summary: 'Success: JSON template with request body' + summary: 'Success: Get all the data' value: - id: api.query.template.query + id: api.dataset.metrics ver: v2 - ts: '2024-05-14T11:52:24+05:30' + ts: '2025-01-15T15:24:21+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: b65e0130-5ba4-49f1-bc6a-8a7d66d1a02d + resmsgid: ef86ff9c-4342-4cdc-a573-d469cbf436eb responseCode: OK result: - - timestamp: '2023-09-01T00:00:00.000Z' - result: - school_id: 0 + - category: data_freshness + status: Healthy + components: + - type: average_time_difference_in_min + threshold: 5 + value: 0 + status: Healthy + - type: freshness_query_time_in_min + threshold: 10 + value: 0 + status: Healthy + - category: data_observability + status: Unhealthy + components: + - type: data_observability_health + status: Unhealthy + - type: failure_percentage + value: 35.39823008849557 + - type: threshold_percentage + value: 5 + - type: importance_score + value: 0 + - category: data_volume + components: + - type: events_per_hour + value: 0 + - type: events_per_day + value: 95 + - type: events_per_n_day + value: 765 + - type: volume_percentage_by_hour + value: 0 + - type: volume_percentage_by_day + value: -87.58169934640523 + - type: volume_percentage_by_week + value: 0 + - type: growth_rate_percentage + value: 0 + - category: data_lineage + components: + - type: transformation_success + value: 765 + - type: dedup_success + value: 384 + - type: denormalization_success + value: 572 + - type: total_success + value: 1339 + - type: total_failed + value: 1034 + - type: transformation_failed + value: 1 + - type: dedup_failed + value: 574 + - type: denorm_failed + value: 193 + - category: connectors + components: + - id: failed + type: failed + value: 1608 + - id: api + type: success + value: 765 + - category: data_quality + components: + - type: incidents_failed + value: 1034 + - type: incidents_success + value: 1339 + - type: total_incidents + value: 2373 example-1: - summary: 'Success: SQL template query with request body' + summary: 'Success : Get particular category data' value: - id: api.query.template.query + id: api.dataset.metrics ver: v2 - ts: '2024-05-14T11:33:06+05:30' + ts: '2025-01-15T15:25:13+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 48c194ee-6e73-4ee7-83e6-8b154e441911 + resmsgid: 10915ec3-a265-4502-bc9b-673562f94512 responseCode: OK result: - - __time: '2023-09-11T00:00:00.000Z' - school_category: secondary - gender: others - state_id: '15' - district_id: '2002' - block_id: '70' - cluster_id: '485' - obsrv.meta.source.connector: null - obsrv.meta.source.id: null - grade_sum: 18 - school_id_sum: 180378 - students_marked_sum: 12492 - students_present_sum: 2466 - total_count: 18 - total_students_sum: 12492 + - category: data_lineage + components: + - type: transformation_success + value: 765 + - type: dedup_success + value: 384 + - type: denormalization_success + value: 572 + - type: total_success + value: 1339 + - type: total_failed + value: 1034 + - type: transformation_failed + value: 1 + - type: dedup_failed + value: 574 + - type: denorm_failed + value: 193 '400': description: Bad Request content: application/json: schema: type: object - examples: - example-0: - summary: 'Failure: invalid date range (native template)' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-02T12:59:14+05:30' - params: - status: FAILED - resmsgid: 4379e16b-2fa3-46a8-8ded-bc53f56283e9 - responseCode: BAD_REQUEST - error: - code: DATA_OUT_INVALID_DATE_RANGE - message: >- - Invalid date range! make sure your range cannot be more - than 30 days - trace: '' - example-1: - summary: 'Failure: invalid date range' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:28:18+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 20391fb8-2be8-48b5-a16f-fca150580e97 - responseCode: BAD_REQUEST - error: - code: DATA_OUT_INVALID_DATE_RANGE - message: >- - Invalid date range! make sure your range cannot be more - than 30 days - trace: '' + example: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:23:22+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a6a26834-bb32-4e32-8dfd-473b0d21c8e8 + responseCode: BAD_REQUEST + error: + code: DATA_OUT_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'dataset_id' + trace: '' '404': description: Not Found content: application/json: schema: type: object - examples: - example-0: - summary: 'Failure: Datasource not found in druid' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:21:46+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: b35a7050-b94c-4944-9630-233c9542272e - responseCode: NOT_FOUND - error: - code: DATA_OUT_SOURCE_NOT_FOUND - message: >- - Dataset test with table hour is not available for - querying - trace: '' - example-1: - summary: 'Failure: Datasource not found in live table' - value: - id: api.query.template.query - ver: v2 - ts: '2024-05-13T13:23:47+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 3a303dfd-1d95-4788-b1a7-d88809d4dcf3 - responseCode: NOT_FOUND - error: - code: DATA_OUT_SOURCE_NOT_FOUND - message: Datasource test11 not available for querying - trace: '' - /v2/dataset/copy: + example: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:15:30+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 25107bba-6d88-4e9b-b951-61e29a72b97a + responseCode: NOT_FOUND + error: + code: DATASET_NOT_FOUND + message: Dataset with id test not found in live table + trace: '' + '500': + description: Internal Server Error + content: + application/json: + schema: + type: object + example: + id: api.dataset.metrics + ver: v2 + ts: '2025-01-15T15:29:49+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 211553f9-ad91-4a4d-a6ba-d77d3e14b7ac + responseCode: FAILED + error: + code: FAILED_TO_FETCH_METRICS + message: Error while fetching metrics + trace: '' + /v2/data/analyze/pii: post: tags: - - Dataset copy - summary: Dataset copy + - Data Analyze PII + summary: Data Analyze PII requestBody: content: application/json: schema: type: object example: - id: api.datasets.copy - ver: '1.0' - ts: '2024-05-21T14:30:00Z' + id: api.data.analyze.pii + ver: v2 + ts: '1711966306164' params: - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 request: - source: - datasetId: test_dataset - isLive: true - destination: - datasetId: test-dataset-copy-1 + id: 268cb552-7de9-41cc-a413-a8bd3b64734c + dataset_id: taxi-payment-data + data: + - tripID: f0f5186b-3195-486e-9b79-0cd9da895707 + VendorID: '1' + tpep_pickup_datetime: '2024-10-31 00:46:40' + tpep_dropoff_datetime: '2024-10-31 00:53:20' + passenger_count: '1' + trip_distance: '1.50' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '151' + DOLocationID: '239' + payment_type: '1' + primary_passenger.email: Herminia_Veum@hotmail.com + primary_passenger.mobile: 200-329-2445 x3611 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + - name: Authorization + in: header + schema: + type: string + example: Bearer {{token}} responses: '200': description: OK @@ -4503,52 +10362,55 @@ paths: schema: type: object example: - id: api.dataset.copy + id: api.data.analyze.pii ver: v2 - ts: '2024-05-31T10:57:11+05:30' + ts: '2025-01-16T10:27:56+05:30' params: status: SUCCESS - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: 77080c18-67b3-489c-af7b-9739709e3c4b + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 8fd2cfe2-7912-40d1-8710-9b7da9a35bd9 responseCode: OK result: - dataset_id: sample1_copy.1 - message: Dataset clone successful - '400': - description: Bad Request - content: - application/json: - schema: - type: object - example: - id: api.dataset.copy - ver: v2 - ts: '2024-05-31T10:56:26+05:30' - params: - status: FAILED - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: '' - responseCode: BAD_REQUEST - error: - code: DATASET_ALREADY_EXISTS - message: Dataset with id sample1 already exists - trace: '' - '404': - description: Not Found + - field: primary_passenger.email + type: internet + score: 0.5 + reason: + - code: en + resourceKey: pii.descriptions.m014 + region: '' + score: 1 + - code: en + resourceKey: pii.descriptions.m004 + region: '' + score: 1 + - field: primary_passenger.mobile + type: phone + score: 0.5 + reason: + - code: en + resourceKey: pii.descriptions.m018 + region: USA + score: 1 + - code: en + resourceKey: pii.descriptions.m005 + region: '' + score: 1 + '500': + description: Internal Server Error content: application/json: schema: type: object example: - id: api.dataset.copy + id: api.data.analyze.pii ver: v2 - ts: '2024-05-31T10:59:59+05:30' + ts: '2025-01-16T10:29:58+05:30' params: status: FAILED - msgid: 127384e4a-a051-4a9f-9b3f-a64a8034fad7 - resmsgid: '' - responseCode: NOT_FOUND + msgid: e180ecac-8f41-4f21-9a21-0b3a1a368917 + resmsgid: 75aa71f9-ef84-41b9-ae1b-64cc7afd3ffb + responseCode: INTERNAL_SERVER_ERROR error: - code: DATASET_NOT_EXISTS - message: Dataset sample does not exists - trace: '' + code: ERR_BAD_REQUEST + message: Request failed with status code 401 + trace: '' \ No newline at end of file diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 74fc9cca..e8f2af12 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -10,6 +10,7 @@ metadata: name: {{ printf "%s-%s" $jobName $component }} namespace: {{ include "base.namespace" $ }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/connector: {{ .Values.connector_id }} annotations: checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} @@ -27,11 +28,16 @@ spec: labels: app.kubernetes.io/name: {{ include "common.names.name" . }} app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + app.kubernetes.io/connector: {{ .Values.connector_id }} component: {{ printf "%s-%s" $jobName $component }} annotations: checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} {{- if .Values.serviceAccount.create}} serviceAccountName: {{ include "base.serviceaccountname" . }} {{- end }} @@ -55,16 +61,21 @@ spec: env: - name: FLINK_PROPERTIES value: |+ - jobmanager.rpc.address: {{ $jobName }}-jobmanager - taskmanager.rpc.port=6122 - taskmanager.numberOfTaskSlots: 2 + web.submit.enable: false metrics.reporters: prom metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory metrics.reporter.prom.host: {{ $jobName }}-taskmanager - metrics.reporter.prom.port: 9251 + metrics.reporter.prom.port: {{ .Values.taskmanager.prom_port }} + jobmanager.rpc.address: {{ $jobName }}-jobmanager + taskmanager.rpc.port: {{ .Values.taskmanager.rpc_port }} + {{- with .Values.flink_conf }} + {{- toYaml . | nindent 16 }} + {{- end }} ports: - containerPort: {{ .Values.taskmanager.rpc_port }} name: rpc + - containerPort: {{ .Values.taskmanager.prom_port }} + name: prom {{- toYaml (index .Values.flink_resources $component )| nindent 10 }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} @@ -87,6 +98,7 @@ metadata: name: {{ printf "%s-%s" $jobName $component }} namespace: {{ include "base.namespace" . }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/connector: {{ .Values.connector_id }} annotations: checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} @@ -104,11 +116,15 @@ spec: labels: app.kubernetes.io/name: {{ include "common.names.name" . }} app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} - component: {{ printf "%s-%s" $jobName $component }} + app.kubernetes.io/connector: {{ .Values.connector_id }} annotations: - checksum/config: {{ .Files.Glob "cFonfigs/*" | toYaml | sha256sum }} + checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} serviceAccountName: {{ include "base.serviceaccountname" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} @@ -131,12 +147,30 @@ spec: env: - name: FLINK_PROPERTIES value: |+ - jobmanager.rpc.address: {{ $jobName }}-jobmanager - jobmanager.rpc.port=6123 + web.submit.enable: false metrics.reporters: prom metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory metrics.reporter.prom.host: {{ $jobName }}-jobmanager - metrics.reporter.prom.port: 9250 + metrics.reporter.prom.port: {{ .Values.jobmanager.prom_port }} + jobmanager.rpc.address: {{ $jobName }}-jobmanager + blob.server.port: {{ .Values.jobmanager.blob_port }} + queryable-state.server.ports: {{ .Values.jobmanager.query_port }} + {{- with .Values.flink_conf }} + {{- toYaml . | nindent 16 }} + {{- end }} + ports: + {{- range .Values.service.ports }} + - name: {{ .name }} + containerPort: {{ .targetPort }} + {{- end }} + - name: prom + containerPort: {{ .Values.jobmanager.prom_port }} + - name: blob + containerPort: {{ .Values.jobmanager.blob_port }} + - name: query + containerPort: {{ .Values.jobmanager.query_port }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - name: connector-config-volume mountPath: /data/flink/conf/connectors-scala-config.conf @@ -165,26 +199,31 @@ spec: - /bin/bash - -c - | + export PYTHONPATH=/opt/flink/.local/lib/python3.10/site-packages:$PYTHONPATH; + rm -rf /data/connectors/*; /usr/bin/python3 /data/connectors-init/connector.py; sleep 30s; + {{- range $jobData.connector_instance_ids }} /opt/flink/bin/flink run -d -m \ - {{ $jobName }}-jobmanager.{{ include "base.namespace" . }}.svc.cluster.local:8081 \ + {{ $jobName }}-jobmanager.{{ include "base.namespace" $ }}.svc.cluster.local:8081 \ /data/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ --config.file.path /data/flink/conf/connectors-scala-config.conf \ - --metadata.id {{ index $jobData "connector_id" }} \ - {{- if eq .Values.checkpoint_store_type "azure" }} - "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ - {{- end }} - {{- if and (eq .Values.checkpoint_store_type "s3") (ne .Values.s3_auth_type "serviceAccount") }} - "-Ds3.access-key={{ .Values.s3_access_key }}" \ - "-Ds3.secret-key={{ .Values.s3_secret_key }}" \ - "-Ds3.endpoint={{ .Values.s3_endpoint }}" \ - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}" \ - {{- end }} - {{- if eq .Values.checkpoint_store_type "gcs" }} + --connector.instance.id {{ . }} \ + {{- if eq $.Values.checkpoint_store_type "azure" }} + "-Dfs.azure.account.key.{{ $.Values.global.azure_storage_account_name }}.blob.core.windows.net={{ $.Values.global.azure_storage_account_key }}" \ + {{- end }} + {{- if and (eq $.Values.checkpoint_store_type "s3") (ne $.Values.s3_auth_type "serviceAccount") }} + "-Ds3.access-key={{ $.Values.s3_access_key }}" \ + "-Ds3.secret-key={{ $.Values.s3_secret_key }}" \ + "-Ds3.endpoint={{ $.Values.s3_endpoint }}" \ + "-Ds3.path.style.access={{ $.Values.s3_path_style_access }}" \ + {{- end }} + {{- if eq $.Values.checkpoint_store_type "gcs" }} "-Dgoogle.cloud.auth.service.account.enable=true" \ - {{- end }} - ; sleep 10s; echo "Job submitted"; tail -f /dev/null; + {{- end }} + ; + {{- end }} + sleep 10s; echo "Jobs submitted"; tail -f /dev/null ports: {{- range .Values.service.ports }} - name: {{ .name }} diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index ed51316d..6b055ed3 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -4,7 +4,7 @@ fullnameOverride: "" replicaCount: 1 -namespace: "flink" +namespace: "flink-connectors" commonLabels: system.processing: "true" release: monitoring @@ -16,7 +16,7 @@ commonLabels: # docker pull sunbirded.azurecr.io/sunbird-datapipeline:release-4.9.0_RC4_1 registry: sanketikahub repository: flink-connectors -tag: 1.17.2-scala_2.12-java11 +tag: 1.20-scala_2.12-java11 imagePullSecrets: [] imagePullPolicy: IfNotPresent @@ -242,10 +242,37 @@ serviceMonitor: jobLabel: "app.kubernetes.io/name" port: prom - # override flink_jobs # flink_jobs: +connector_id: "" +connector_instance_ids: [] commonAnnotations: - reloader.stakater.com/auto: "true" \ No newline at end of file + reloader.stakater.com/auto: "true" + +# flink_conf: +# state.savepoints.dir: file:///tmp +# taskmanager.bind-host: 0.0.0.0 +# taskmanager.memory.flink.size: 1024m +# taskmanager.memory.network.fraction: 0.1 +# taskmanager.numberOfTaskSlots: 1 +# taskmanager.rpc.port: 6122 +# rest.address: 0.0.0.0 +# rest.bind-address: 0.0.0.0 + +flink_conf: + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + jobmanager.bind-host: 0.0.0.0 + jobmanager.memory.flink.size: 1024m + jobmanager.numberOfTaskSlots: 1 + jobmanager.execution.failover-strategy: region + parallelism.default: 1 + taskmanager.memory.flink.size: 1024m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.network.fraction: 0.1 + state.savepoints.dir: file:///tmp + state.checkpoint.interval: 60000 + state.checkpoint.pause.between.seconds: 5000 + query.server.port: 6125 \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index f22dcaef..5c8ffd3f 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -1,9 +1,12 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "base.cronReleaseName" . }} + name: {{ .Release.Name }} namespace: {{ include "base.namespace" . }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/dataset: {{ .Values.dataset_id }} + app.kubernetes.io/connector: {{ .Values.connector_id }} + app.kubernetes.io/connector_instance: {{ .Values.instance_id }} {{- if .Values.commonAnnotations }} annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} @@ -19,7 +22,14 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 12 }} {{- end }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 12 }} + app.kubernetes.io/dataset: {{ .Values.dataset_id }} + app.kubernetes.io/connector: {{ .Values.connector_id }} + app.kubernetes.io/connector_instance: {{ .Values.instance_id }} spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} serviceAccountName: {{ .Values.serviceAccount.name }} restartPolicy: {{ .Values.restartPolicy }} securityContext: @@ -70,7 +80,7 @@ spec: - | # Wait for the Spark pod to be ready SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') - kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master=local[*] --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" {{- end }} {{- with .Values.sidecars }} {{- toYaml . | nindent 12 }} diff --git a/command-service/helm-charts/spark-connector-cron/values.yaml b/command-service/helm-charts/spark-connector-cron/values.yaml index ceca090a..8069fe0e 100644 --- a/command-service/helm-charts/spark-connector-cron/values.yaml +++ b/command-service/helm-charts/spark-connector-cron/values.yaml @@ -22,10 +22,10 @@ commonAnnotations: {} podAnnotations: {} -podSecurityContext: {} - # runAsNonRoot: true - # runAsUser: 1001 - # fsGroup: 1001 +podSecurityContext: + runAsNonRoot: true + runAsUser: 1001 + fsGroup: 1001 securityContext: {} # readOnlyRootFilesystem: false @@ -142,6 +142,9 @@ instance_id: nyt-psql.1 main_class: org.sunbird.obsrv.connector.JDBCConnector main_file: jdbc-connector-1.0.0.jar +dataset_id: "" +connector_id: "" + ## Object Store Connector # technology: python # connector-source: object_store_connector-0.1.0 diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py deleted file mode 100644 index 3b6a82b9..00000000 --- a/command-service/src/command/alert_manager_command.py +++ /dev/null @@ -1,221 +0,0 @@ -import json - -from command.icommand import ICommand -from config import Config -from model.data_models import Action, ActionResponse, CommandPayload -from service.db_service import DatabaseService -from service.http_service import HttpService - - -class AlertManagerService(ICommand): - - def __init__( - self, config: Config, db_service: DatabaseService, http_service: HttpService - ): - self.config = config - self.db_service = db_service - self.http_service = http_service - self.metrics = self.config.find("alert_manager.metrics") - self.masterdata_metrics = self.config.find("alert_manager.masterdata_metrics") - self.object_connector_metrics = self.config.find( - "alert_manager.object_connector_metrics" - ) - self.jdbc_connector_metrics = self.config.find( - "alert_manager.jdbc_connector_metrics" - ) - self.config_service_host = self.config.find("config_service.host") - self.config_service_port = self.config.find("config_service.port") - self.base_url = ( - f"http://{self.config_service_host}:{self.config_service_port}/alerts/v1" - ) - - def execute(self, command_payload: CommandPayload, action: Action): - dataset = self.get_dataset(dataset_id=command_payload.dataset_id) - if dataset is None: - return ActionResponse( - status="ERROR", - status_code=500, - error_message=f"Dataset {command_payload.dataset_id} does not exist", - ) - - dataset_source_config = self.get_dataset_source_config( - dataset_id=command_payload.dataset_id - ) - - for metric in self.metrics: - for service, metrics in metric.items(): - for metric_info in metrics: - self.create_alert_metric( - payload=command_payload, - service=service, - metric=metric_info, - dataset_name=dataset["name"], - ) - - for config in dataset_source_config: - if config["connector_type"] == "object": - for metric in self.object_connector_metrics: - self.create_alert_metric( - payload=command_payload, - service=service, - metric=metric, - dataset_name=dataset["name"], - ) - if config["connector_type"] == "jdbc": - for metric in self.jdbc_connector_metrics: - self.create_alert_metric( - payload=command_payload, - service=service, - metric=metric, - dataset_name=dataset["name"], - ) - - if dataset["type"] == "master-dataset": - for metric_info in self.masterdata_metrics: - self.create_alert_metric( - payload=command_payload, - service=service, - metric=metric_info, - dataset_name=dataset["name"], - ) - return ActionResponse(status="OK", status_code=200) - - def get_dataset(self, dataset_id: str) -> str: - query = f"SELECT * FROM datasets WHERE dataset_id= %s" - params = (dataset_id,) - result = self.db_service.execute_select_one(sql=query, params=params) - return result - - def get_dataset_source_config(self, dataset_id: str) -> str: - query = f"SELECT * FROM dataset_source_config WHERE dataset_id= %s" - params = (dataset_id,) - result = self.db_service.execute_select_all(sql=query, params=params) - return result - - def get_modified_metric( - self, service: str, metric: dict, payload: CommandPayload - ) -> dict: - if service == "flink": - substring = f"{payload.dataset_id}" - modified_substring = substring.replace("-", "_") - modified_metric = metric["metric"].replace("dataset_id", modified_substring) - metric["metric"] = modified_metric - return metric - else: - metric["metric"] = metric["metric"].replace( - "dataset_id", payload.dataset_id - ) - return metric - - def create_alert_metric( - self, payload: CommandPayload, service: str, metric: dict, dataset_name: str - ) -> ActionResponse: - metric_url = f"{self.base_url}/metric/alias/create" - - metric_data = self.get_modified_metric( - service=service, metric=metric, payload=payload - ) - - prom_metric = metric_data["metric"] - metric_alias = f"{metric_data['alias']} ({payload.dataset_id})" - # Metric api payload - metric_body = json.dumps( - { - "alias": metric_alias, - "component": "datasets", - "subComponent": dataset_name, - "metric": prom_metric, - "context": { - "datasetId": payload.dataset_id, - }, - } - ) - - response = self.http_service.post( - url=metric_url, - body=metric_body, - headers={"Content-Type": "application/json"}, - ) - if response.status == 200: - self.create_alert_rule( - payload={"dataset_name": dataset_name, "metric_data": metric_data} - ) - else: - error_data = json.loads(response.body) - if "params" in error_data and "errmsg" in error_data["params"]: - error_message = error_data["params"]["errmsg"] - else: - error_message = "Failed to publish alerts" - return ActionResponse( - status="ERROR", - status_code=500, - error_message=f"Error creating alert metric {metric_alias}: {error_message}", - ) - - def create_alert_rule(self, payload: dict) -> ActionResponse: - dataset_name = payload["dataset_name"] - prom_metric = payload["metric_data"]["metric"] - description = payload["metric_data"]["description"] - metric_alias = payload["metric_data"]["alias"] - frequency = payload["metric_data"]["frequency"] - interval = payload["metric_data"]["interval"] - operator = payload["metric_data"]["operator"] - threshold = payload["metric_data"]["threshold"] - - alert_body = json.dumps( - { - "name": f"{dataset_name}_{metric_alias}", - "manager": "grafana", - "description": description - or f"Automated alert set up for dataset {dataset_name}", - "category": "datasets", - "frequency": frequency, - "interval": interval, - "context": {"alertType": "SYSTEM"}, - "labels": {"component": "obsrv"}, - "metadata": { - "queryBuilderContext": { - "category": "datasets", - "subComponent": dataset_name, - "metric": prom_metric, - "operator": operator, - "threshold": [+threshold], - "metricAlias": metric_alias, - } - }, - "notification": {"channels": []}, - } - ) - - response = self.http_service.post( - url=f"{self.base_url}/create", - body=alert_body, - headers={"Content-Type": "application/json"}, - ) - result = json.loads(response.body) - alert_id = result["result"]["id"] - if response.status == 200: - self.publish_alert_rule(alert_id=alert_id) - else: - error_data = json.loads(response.body) - if "params" in error_data and "errmsg" in error_data["params"]: - error_message = error_data["params"]["errmsg"] - else: - error_message = "Failed to publish alerts" - return ActionResponse( - status="ERROR", - status_code=500, - error_message=f"Error creating alert rule for {dataset_name}: {error_message}", - ) - - def publish_alert_rule(self, alert_id: str) -> ActionResponse: - endpoint = f"/publish/{alert_id}" - url = self.base_url + endpoint - try: - response = self.http_service.get(url=url) - except Exception as e: - return ActionResponse( - status="ERROR", - status_code=500, - message=f"Error publishing alert rule for {alert_id}: {e}", - ) diff --git a/command-service/src/command/command_executor.py b/command-service/src/command/command_executor.py index dffa8155..2ca2dea7 100644 --- a/command-service/src/command/command_executor.py +++ b/command-service/src/command/command_executor.py @@ -3,7 +3,6 @@ import psycopg2 from urllib3.exceptions import MaxRetryError, NewConnectionError -from command.alert_manager_command import AlertManagerService from command.connector_command import ConnectorCommand from command.dataset_command import DatasetCommand from command.db_command import DBCommand @@ -36,11 +35,6 @@ def __init__(self): db_service=self.db_service, http_service=self.http_service, ) - self.alert_manager_command = AlertManagerService( - config=self.config_obj, - db_service=self.db_service, - http_service=self.http_service, - ) self.dataset_command = DatasetCommand( db_service=self.db_service, telemetry_service=self.telemetry_service, @@ -63,9 +57,6 @@ def __init__(self): self.action_commands[Action.CREATE_KAFKA_TOPIC.name] = self.create_kafka_topic self.action_commands[Action.SUBMIT_INGESTION_TASKS.name] = self.druid_command self.action_commands[Action.DEPLOY_CONNECTORS.name] = self.connector_command - self.action_commands[Action.CREATE_ALERT_METRIC.name] = ( - self.alert_manager_command - ) self.action_commands[Action.CREATE_AUDIT_EVENT.name] = self.audit_event_command self.logger = logging.getLogger() diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 8f5b145e..2a2ea031 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -8,6 +8,9 @@ from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType from model.db_models import ConnectorInstance from service.db_service import DatabaseService +import time +from random import choice +from string import ascii_lowercase class ConnectorCommand(ICommand): @@ -38,15 +41,12 @@ def execute(self, command_payload: CommandPayload, action: Action): def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): result = None - self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"], active_connectors, dataset_id) + self._stop_connector_jobs(is_masterdata, dataset_id) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) return result - def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, dataset_id): - print(f"Uninstalling jobs for {namespace}..") - base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] - + def _stop_connector_jobs(self, is_masterdata, dataset_id): # managed_releases = [] # connector_jar_config = self.config.find("connector_job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -57,34 +57,66 @@ def _stop_connector_jobs(self, is_masterdata, namespace, active_connectors, data # for release in masterdata_jar_config: # managed_releases.append(release["release_name"]) - helm_ls_cmd = ["helm", "ls", "--namespace", namespace] - helm_ls_result = subprocess.run( - helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ## Clear all spark cronjobs for the dataset + spark_namespace = self.connector_job_config["spark"]["namespace"] + deployed_cron_jobs_cmd = [ + "kubectl", "get", "cronjobs", "-n", spark_namespace, + "--selector", f"app.kubernetes.io/dataset={dataset_id}", + "-o", "jsonpath=\"{.items[*].metadata.name}\"" + ] + + deployed_cron_jobs_result = subprocess.run( + deployed_cron_jobs_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - if helm_ls_result.returncode == 0: - jobs = helm_ls_result.stdout.decode().splitlines()[1:] - job_names = {job.split()[0] for job in jobs if base_helm_chart in job} - spark_connector = {connector.id for connector in active_connectors if connector.connector_runtime == "spark"} - for release_name in spark_connector: - if release_name in job_names: - print(f"Uninstalling job {release_name} related to dataset'{dataset_id}'...") - helm_uninstall_cmd = [ - "helm", - "uninstall", - release_name, - "--namespace", - namespace, - ] - helm_uninstall_result = subprocess.run( - helm_uninstall_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if helm_uninstall_result.returncode == 0: - print(f"Successfully uninstalled job '{release_name}'...") - else: - print(f"Error uninstalling job '{release_name}': {helm_uninstall_result.stderr.decode()}") - + + if deployed_cron_jobs_result.returncode == 0 and len(deployed_cron_jobs_result.stdout.decode().replace("\"", "").splitlines()): + jobs = deployed_cron_jobs_result.stdout.decode().replace("\"", "").splitlines()[0].split() + for job in jobs: + print(f"Uninstalling job {job} related to dataset'{dataset_id}'...") + helm_uninstall_cmd = [ + "helm", + "uninstall", + job, + "--namespace", + spark_namespace, + ] + print("Uninstall command: ", " ".join(helm_uninstall_cmd)) + helm_uninstall_result = subprocess.run( + helm_uninstall_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if helm_uninstall_result.returncode == 0: + print(f"Successfully uninstalled job '{job}'...") + else: + print(f"Error uninstalling job '{job}': {helm_uninstall_result.stderr.decode()}") + + ## Clear all flink connectors that are not active + flink_namespace = self.connector_job_config["flink"]["namespace"] + registered_connectors = self._get_registered_connectors(runtime="flink") + print("Registered flink connectors: ", registered_connectors) + for connector in registered_connectors: + if connector[1] == 0: + ### Uninstall the helm chart + helm_uninstall_cmd = [ + "helm", + "uninstall", + connector[0].replace(".", "-").replace("_", "-"), + "--namespace", + flink_namespace, + ] + + print("Uninstall command: ", " ".join(helm_uninstall_cmd)) + helm_uninstall_result = subprocess.run( + helm_uninstall_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if helm_uninstall_result.returncode == 0: + print(f"Successfully uninstalled deployment '{connector[0]}'...") + else: + print(f"Error uninstalling deployment '{connector}': {helm_uninstall_result.stderr.decode()}") + def _install_jobs(self, dataset_id, active_connectors, is_masterdata): result = None for connector in active_connectors: @@ -100,6 +132,9 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): ) break + if result is None: + result = ActionResponse(status="OK", status_code=200) + # if is_masterdata: # print("Installing masterdata job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -111,7 +146,6 @@ def _perform_flink_install(self, dataset_id, connector_instance): err = None result = None release_name = connector_instance.connector_id - runtime = connector_instance.connector_runtime namespace = self.connector_job_config["flink"]["namespace"] job_name = release_name.replace(".", "-") helm_ls_cmd = ["helm", "ls", "--namespace", namespace] @@ -120,89 +154,88 @@ def _perform_flink_install(self, dataset_id, connector_instance): helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - if helm_ls_result.returncode == 0: - jobs = helm_ls_result.stdout.decode() + if helm_ls_result.returncode == 0 and self._get_live_instances(runtime="flink", connector_instance=connector_instance): + connector_source = connector_instance.connector_source + # Get all live instances for this connector + query = """ + SELECT id FROM connector_instances + WHERE status = 'Live' AND connector_id = %s + """ + params = (connector_instance.connector_id,) + instances = self.db_service.execute_select_all(sql=query, params=params) + + if not instances: + print(f"No live instances found for connector {connector_instance.connector_id}") + return ActionResponse( + status="ERROR", + status_code=400, + error_message="NO_LIVE_INSTANCES_FOUND" + ) - deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) - if deployment_exists: - restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( - namespace=namespace, job_name=job_name + # Get instance IDs + instance_ids = [str(inst["id"]) for inst in instances] + + flink_jobs = dict() + flink_jobs[job_name] = { + "enabled": "true", + "connector_id": connector_instance.connector_id, + "connector_instance_ids": instance_ids, + "source": connector_source.get("source"), + "main_program": connector_source.get("main_program") + } + + flink_jobs_json = json.dumps(flink_jobs) + nodeSelector = self.config.find("node_selector") + security_contexts = self.config.find("flink_connector_container_security_context") + pod_security_context = self.config.find("flink_connector_pod_security_context") + task_manager_limits = { + "cpu": len(instance_ids), + "memory": "{}Mi".format(len(instance_ids)*1024) + } + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + job_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set", "namespace={}".format(namespace), + "--set", "connector_id={}".format(connector_instance.connector_id), + "--set", "flink_conf.taskmanager\\.numberOfTaskSlots={}".format(len(instance_ids)), + "--set-json", f"""flink_jobs={flink_jobs_json.replace(" ", "")}""", + "--set-json", f"""nodeSelector={json.dumps(nodeSelector)}""", + "--set-json", f"""securityContext={json.dumps(security_contexts)}""", + "--set-json", f"""podSecurityContext={json.dumps(pod_security_context)}""", + "--set-json", f"""flink_resources.taskmanager.resources.limits={json.dumps(task_manager_limits)}""" + ] + + print("flink connector installation: ", " ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + print(helm_install_result) + + if helm_install_result.returncode == 0: + print(f"Job '{job_name}' deployment succeeded...") + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", ) - print("Restart command: ", restart_cmd) - # Run the helm command - helm_install_result = subprocess.run( - restart_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True, + print( + f"Error installing job '{job_name}': {helm_install_result.stderr.decode()}" ) - if helm_install_result.returncode == 0: - print(f"Job {job_name} restart succeeded...") - else: - err = True - return ActionResponse( - status="ERROR", - status_code=500, - error_message="FLINK_HELM_LIST_EXCEPTION", - ) - print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") - - if err is None: - result = ActionResponse(status="OK", status_code=200) - - return result - else: - if self._get_live_instances(runtime="flink", connector_instance=connector_instance): - connector_source = connector_instance.connector_source - flink_jobs = dict() - flink_jobs[job_name] = { - "enabled": "true", - "connector_id": connector_instance.connector_id, - "source": connector_source.get("source"), - "main_program": connector_source.get("main_program") - } - - set_json_value = json.dumps(flink_jobs) - helm_install_cmd = [ - "helm", - "upgrade", - "--install", - job_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", - "--namespace", - namespace, - "--create-namespace", - "--set-json", - f"""flink_jobs={set_json_value.replace(" ", "")}""" - ] - - print("flink connector installation: ", " ".join(helm_install_cmd)) - - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - - print(helm_install_result) - - if helm_install_result.returncode == 0: - print(f"Job '{job_name}' deployment succeeded...") - else: - err = True - result = ActionResponse( - status="ERROR", - status_code=500, - error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", - ) - print( - f"Error installing job '{job_name}': {helm_install_result.stderr.decode()}" - ) - - if err is None: - result = ActionResponse(status="OK", status_code=200) - - return result - else: - self._stop_connector_jobs(is_masterdata=False, namespace="flink") + + if err is None: + result = ActionResponse(status="OK", status_code=200) + + return result else: print(f"Error checking Flink deployments: {helm_ls_result.stderr.decode()}") return ActionResponse( @@ -214,7 +247,10 @@ def _perform_flink_install(self, dataset_id, connector_instance): def _perform_spark_install(self, dataset_id, connector_instance): err = None result = None - release_name = connector_instance.id + release_name = "{}-{}".format( + (dataset_id + "_" + connector_instance.connector_id).lower().replace("_", "-")[:54], + "".join(choice(ascii_lowercase) for i in range(6)) + ) connector_source = connector_instance.connector_source schedule = connector_instance.operations_config["schedule"] @@ -226,6 +262,9 @@ def _perform_spark_install(self, dataset_id, connector_instance): } namespace = self.connector_job_config["spark"]["namespace"] + nodeSelector = self.config.find("node_selector") + container_security_context = self.config.find("spark_connector_container_security_context") + pod_security_context = self.config.find("spark_connector_pod_security_context") helm_install_cmd = [ "helm", @@ -237,9 +276,15 @@ def _perform_spark_install(self, dataset_id, connector_instance): namespace, "--create-namespace", "--set", + "namespace={}".format(namespace), + "--set", "technology={}".format(connector_instance.technology), "--set", - "instance_id={}".format(release_name), + "dataset_id={}".format(dataset_id), + "--set", + "connector_id={}".format(connector_instance.connector_id), + "--set", + "instance_id={}".format(connector_instance.id), "--set", "connector_source={}".format(connector_source["source"]), "--set", @@ -247,7 +292,13 @@ def _perform_spark_install(self, dataset_id, connector_instance): "--set", "main_file={}".format(connector_source["main_program"]), "--set", - "cronSchedule={}".format(schedule_configs[schedule]) + "cronSchedule={}".format(schedule_configs[schedule]), + "--set-json", + f"""nodeSelector={json.dumps(nodeSelector)}""", + "--set-json", + f"""securityContext={json.dumps(container_security_context)}""", + "--set-json", + f"""podSecurityContext={json.dumps(pod_security_context)}""" ] print("spark connector installation:", " ".join(helm_install_cmd)) @@ -304,7 +355,6 @@ def _get_masterdata_details(self, dataset_id): return is_masterdata - ## TODO: check for connector_id as well def _get_live_instances(self, runtime, connector_instance): has_live_instances = False query = f""" @@ -321,48 +371,14 @@ def _get_live_instances(self, runtime, connector_instance): return has_live_instances - # def _perform_install(self, release): - # err = None - # result = None - # release_name = release["release_name"] - # helm_install_cmd = [ - # "helm", - # "upgrade", - # "--install", - # release_name, - # self.connector_job_chart_dir, - # "--namespace", - # self.connector_job_ns, - # "--create-namespace", - # "--set", - # "file.path={}".format(release["jar"]), - # "--set", - # "class.name={}".format(release["class"]), - # "--set", - # "job.name={}".format(release_name), - # "--set", - # "args={}".format(",".join(release["args"])), - # "--set", - # "schedule={}".format(release["schedule"]), - # ] - # helm_install_result = subprocess.run( - # helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - # ) - # if helm_install_result.returncode == 0: - # print(f"Job {release_name} deployment succeeded...") - # else: - # err = True - # result = ActionResponse( - # status="ERROR", - # status_code=500, - # error_message="FLINK_HELM_INSTALLATION_EXCEPTION", - # ) - # print( - # f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" - # ) - - # if err is None: - # result = ActionResponse(status="OK", status_code=200) - - # return result - + def _get_registered_connectors(self, runtime): + query = f""" + SELECT cr.id, COUNT(ci.id) AS instance_count + FROM connector_registry cr + LEFT JOIN connector_instances ci ON (cr.id = ci.connector_id AND ci.status = %s) + WHERE cr.runtime = %s + GROUP BY cr.id + """ + params = (DatasetStatusType.Live.name, runtime) + rows = self.db_service.execute_select_all(sql=query, params=params) + return rows \ No newline at end of file diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index 40edf5f7..6aa9bae9 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -39,52 +39,19 @@ def __init__(self): "connector_registry.metadata_file_name" ) self.ui_spec_file_name = self.config.find("connector_registry.ui_spec_file") - self.dataset_api_url = self.config.find("dataset_api.host").strip("/") - self.pre_signed_url = self.config.find("dataset_api.pre_signed_url").strip("/") - def register(self, rel_path: str) -> RegistryResponse: + def register(self, download_url: str, rel_path: str) -> RegistryResponse: try: download_file_path = os.path.join(self.download_path, rel_path) file_extension = rel_path.split(".")[-1] # if not os.path.exists(download_file_path): - http_service = HttpService() - print( - f"Connector Registry | Received request to register connector: {rel_path} | {self.dataset_api_url}/{self.pre_signed_url}" - ) - dataset_api_request = {"request": {"files": [rel_path], "access": "read", "type": "connector"}} - dataset_api_response = http_service.post( - url=f"{self.dataset_api_url}/{self.pre_signed_url}", - body=json.dumps(dataset_api_request), - headers={"Content-Type": "application/json"} - ) - print( - f"Connector Registry | Dataset API Response: {dataset_api_response.body}" - ) - - if dataset_api_response.status != 200: - return RegistryResponse( - status="failure", - message="dataset api failed to generate read url.", - statusCode=status.HTTP_400_BAD_REQUEST, - ) - - dataset_api_response_data = json.loads(dataset_api_response.body) - url = dataset_api_response_data.get("result", [{}])[0].get( - "preSignedUrl", None - ) - if not url: - return RegistryResponse( - status="failure", - message="dataset api failed to generate read url.", - statusCode=status.HTTP_400_BAD_REQUEST, - ) self.cleanup_download_path() os.makedirs(self.download_path, exist_ok=True) # download the file - download_status = self.download_file(url, download_file_path) + download_status = self.download_file(download_url, download_file_path) print(f"Connector Registry | Download status: {download_status}") if not download_status: return RegistryResponse( @@ -214,9 +181,6 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) - subprocess.run(["rm", "-rf", self.extraction_path]) - subprocess.run(["rm", "-rf", self.download_path]) - if not success: return RegistryResponse( status="failure", @@ -224,13 +188,16 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, ) result.append(registry_meta.to_dict()) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + return RegistryResponse( status="success", connector_info=result, message="Connectors registered successfully", statusCode=status.HTTP_200_OK ) - else: connector_id = ( self.metadata.get("metadata", {}).get("id", "").replace(" ", "-") @@ -453,13 +420,14 @@ def copy_connector_to_runtime(self, runtime: str, connector_source: str): def copy_connector_to_spark(self, connector_source: str): print(f"Connector Registry | copying {connector_source} to spark") + spark_namespace = self.config.find("connector_jobs")["spark"]["namespace"] ## get name of the spark pod using kubectl spark_pod_cmd = [ "kubectl", "get", "pods", "-n", - "spark", + spark_namespace, "-l", "app.kubernetes.io/name=spark,app.kubernetes.io/component=master", "-o", @@ -480,13 +448,31 @@ def copy_connector_to_spark(self, connector_source: str): spark_pod = spark_pod_result.stdout.decode("utf-8").replace("'", "") print(f"Connector Registry | spark_pod: {spark_pod}") + ## remove the connector from spark if it already exists + remove_cmd = [ + "kubectl", + "exec", + f"pod/{spark_pod}", + "-n", + spark_namespace, + "--", + "rm", + "-rf", + f"/data/connectors/{connector_source}", + ] + + remove_result = subprocess.run(remove_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(f"Connector Registry | remove_result: {remove_result}") + if remove_result.returncode != 0: + print("Connector Registry | failed to remove the connector from spark") + ## copy the connector to the spark pod under /data/connectors/{source} source_path = os.path.join(self.extraction_path, connector_source) copy_cmd = [ "kubectl", "cp", f"{source_path}", - f"spark/{spark_pod}:/data/connectors/{connector_source}", + f"{spark_namespace}/{spark_pod}:/data/connectors/{connector_source}", ] copy_result = subprocess.run(copy_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -504,7 +490,7 @@ def copy_connector_to_spark(self, connector_source: str): "exec", f"pod/{spark_pod}", "-n", - "spark", + spark_namespace, "--", "bash", "-c", diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py index d0433884..1e9c3059 100644 --- a/command-service/src/command/dataset_command.py +++ b/command-service/src/command/dataset_command.py @@ -21,12 +21,7 @@ def __init__( ): self.db_service = db_service self.telemetry_service = telemetry_service - self.http_service = http_service self.config = config - self.http_service = http_service - self.config_service_host = self.config.find("config_service.host") - self.config_service_port = self.config.find("config_service.port") - self.base_url = f"http://{self.config_service_host}:{self.config_service_port}/v2/datasets/export" def _get_draft_dataset_record(self, dataset_id): query = f""" @@ -63,19 +58,36 @@ def _check_for_live_record(self, dataset_id): return live_dataset, data_version return None, None + def _get_live_dataset_for_audit(self, dataset): + result = {} + if dataset is not None: + dataset_dict = dataset.__dict__ + result["dataset"] = dataset_dict + connector_instances = self.db_service.execute_select_all( + sql="SELECT * FROM connector_instances WHERE dataset_id = %s", + params=(dataset.dataset_id,) + ) + if connector_instances is not None: + result["dataset"]["connectors_config"] = connector_instances + + transformations = self.db_service.execute_select_all( + sql="SELECT * FROM dataset_transformations WHERE dataset_id = %s", + params=(dataset.dataset_id,) + ) + if transformations is not None: + result["dataset"]["transformations_config"] = transformations + return result + def audit_live_dataset(self, command_payload: CommandPayload, ts: int): dataset_id = command_payload.dataset_id dataset_record, data_version = self._check_for_live_record(dataset_id) - url=self.base_url + '/{}'.format(dataset_id) - export_dataset = self.http_service.get( - url=url - ) - if export_dataset.status == 200: - result = json.loads(export_dataset.body) + live_dataset_result = self._get_live_dataset_for_audit(dataset_record) + live_dataset = live_dataset_result["dataset"] + if live_dataset is not None: object_ = Object( dataset_id, dataset_record.type, dataset_record.data_version ) - live_dataset_property = Property("dataset:export", result["result"], "") + live_dataset_property = Property("dataset:export", live_dataset, "") draft_property = Property( "draft-dataset:status", DatasetStatusType.ReadyToPublish.name, diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 6279102f..0a52a171 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -188,8 +188,9 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): current_timestamp, current_timestamp, json.dumps(draft_datasource.metadata).replace("'", "''"), + True, - draft_datasource.datasource, + draft_datasource.datasource_ref, json.dumps(draft_datasource.ingestion_spec), draft_datasource.type, json.dumps(draft_datasource.retention_period).replace("'", "''"), @@ -201,11 +202,12 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): current_timestamp, json.dumps(draft_datasource.metadata).replace("'", "''"), DatasetStatusType.Live.name, + True ) insert_query = f""" INSERT INTO datasources(id, datasource, dataset_id, datasource_ref, ingestion_spec, type, retention_period, archival_policy, purge_policy, backup_config, status, created_by, updated_by, created_date, - updated_date, published_date, metadata) + updated_date, published_date, metadata, is_primary) VALUES ( %s, %s, @@ -223,6 +225,7 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): %s, %s, %s, + %s, %s ) ON CONFLICT (id) DO UPDATE @@ -237,7 +240,8 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): updated_date = %s, published_date = %s, metadata = %s, - status = %s; + status = %s, + is_primary = %s; """ result = self.db_service.execute_upsert(sql=insert_query, params=params) print( @@ -308,54 +312,7 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): status = %s; """ result = self.db_service.execute_upsert(sql=insert_query, params=params) - print( - f"Connector[v2] Instance record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully..." - ) - else: - params = ( - connector_config.id, - dataset_id, - connector_config.connector_id, - json.dumps(connector_config.connector_config).replace("'", "''"), - DatasetStatusType.Live.name, - draft_dataset_record.get('created_by'), - draft_dataset_record.get('updated_by'), - current_timestamp, - current_timestamp, - current_timestamp, - - json.dumps(connector_config.connector_config).replace("'", "''"), - draft_dataset_record.get('updated_by'), - current_timestamp, - current_timestamp, - DatasetStatusType.Live.name, - ) - insert_query = f""" - INSERT INTO dataset_source_config(id, dataset_id, connector_type, connector_config, - status, created_by, updated_by, created_date, updated_date, published_date) - VALUES ( - %s, - %s, - %s, - %s, - %s, - %s, - %s, - %s, - %s, - %s - ) - ON CONFLICT (id) DO UPDATE - SET connector_config = %s, - updated_by = %s, - updated_date = %s, - published_date = %s, - status = %s; - """ - result = self.db_service.execute_upsert(sql=insert_query, params=params) - print( - f"Connector[v1] record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully..." - ) + print(f"Connector[v2] Instance record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully...") return result diff --git a/command-service/src/command/druid_command.py b/command-service/src/command/druid_command.py index 73d5bf86..48fbbf21 100644 --- a/command-service/src/command/druid_command.py +++ b/command-service/src/command/druid_command.py @@ -5,6 +5,7 @@ from model.data_models import Action, ActionResponse, CommandPayload from service.db_service import DatabaseService from service.http_service import HttpService +import base64 class DruidCommand(ICommand): @@ -19,6 +20,16 @@ def __init__( router_post = self.config.find("druid.router_port") self.supervisor_endpoint = self.config.find("druid.supervisor_endpoint") self.router_url = f"{router_host}:{router_post}/druid" + self._auth = None + + @property + def auth(self): + """Lazy initialization of auth header.""" + if self._auth is None: + username = self.config.find("druid.username") + password = self.config.find("druid.password") + self._auth = base64.b64encode(f"{username}:{password}".encode()).decode() + return self._auth def execute(self, command_payload: CommandPayload, action: Action): if action == Action.SUBMIT_INGESTION_TASKS.name: @@ -42,7 +53,10 @@ def _submit_ingestion_task(self, dataset_id): response = self.http_service.post( url=f"{self.router_url}/{self.supervisor_endpoint}", body=ingestion_spec, - headers={"Content-Type": "application/json"} + headers={ + "Content-Type": "application/json", + "Authorization": f"Basic {self.auth}" + } ) if response.status != 200: task_submitted = 0 diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index 80276238..44ea498e 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -20,7 +20,6 @@ commands: - SUBMIT_INGESTION_TASKS - START_PIPELINE_JOBS - DEPLOY_CONNECTORS - - CREATE_ALERT_METRIC - CREATE_AUDIT_EVENT RESTART_PIPELINE: workflow: @@ -29,83 +28,6 @@ commands: workflow: - DEPLOY_CONNECTORS -alert_manager: - metrics: - - flink: - - metric: "flink_taskmanager_job_task_operator_ExtractorJob_dataset_id_failed_event_count" - alias: "Number of Failed Extraction Events" - description: "This alert tracks how many events failed the extraction stage" - frequency: 5m - interval: 5m - operator: "gt" - threshold: 100 - - metric: "flink_taskmanager_job_task_operator_ExtractorJob_dataset_id_duplicate_extraction_count" - alias: "Number of Duplicate Extraction Events" - description: "This alert tracks how many duplicate events were found during extraction stage" - frequency: 5m - interval: 5m - operator: "gt" - threshold: 100 - - metric: "flink_taskmanager_job_task_operator_PipelinePreprocessorJob_dataset_id_failed_event_count" - alias: "Number of Failed Preprocessing Events" - description: "This alert tracks how many events failed the preprocessing stage" - frequency: 5m - interval: 5m - operator: "gt" - threshold: 100 - - metric: "flink_taskmanager_job_task_operator_PipelinePreprocessorJob_dataset_id_duplicate_event_count" - alias: "Number of Duplicate Preprocessing Events" - description: "This alert tracks how many duplicate events were found during preprocessing stage" - frequency: 5m - interval: 5m - operator: "gt" - threshold: 100 - - metric: "flink_taskmanager_job_task_operator_PipelinePreprocessorJob_dataset_id_validation_failed_event_count" - alias: "Number of Failed Validation Events" - description: "This alert tracks how many events failed the validation stage" - frequency: 5m - interval: 5m - operator: "gt" - threshold: 100 - object_connector_metrics: - - metric: "sum_over_time(ObjectDiscoveryJob_cloud_authentication_failure{datasetId='dataset_id'}[1h])" - alias: "Cloud Authentication Failure" - description: "This alert tracks whether the cloud authentication failed for the dataset" - frequency: 1h - interval: 1h - operator: "gt" - threshold: 1 - - metric: "sum_over_time(ObjectProcessorJob_object_tag_update_failure{datasetId='dataset_id'}[1h])" - alias: "Update Tag Failure" - description: "This alert tracks whether the tag update failed for the dataset" - frequency: 1h - interval: 1h - operator: "gt" - threshold: 1 - - metric: "sum_over_time(ObjectDiscoveryJob_num_new_objects{datasetId='dataset_id'}[1d])" - alias: "Number Of New Objects" - description: "This alert tracks whether any new objects are processed. If not, then this alert is fired" - frequency: 1d - interval: 1h - operator: "eq" - threshold: 0 - jdbc_connector_metrics: - - metric: "sum_over_time(JDBCConnectorJob_failure_count{datasetId='dataset_id'}[1d])" - alias: "Number of Failed Records" - description: "This alert tracks whether any records failed to be ingested into the database" - frequency: 1d - interval: 1h - operator: "gt" - threshold: 10 - masterdata_metrics: - - metric: "MasterDataProcessorIndexerJob_failure_dataset_count{datasetId='dataset_id'}" - alias: "Master Dataset failed to index" - description: "This alert tracks whether the given master dataset failed to index data" - frequency: 240m - interval: 5m - operator: "gt" - threshold: 0 - postgres: db_host: localhost db_port: 5432 @@ -113,14 +35,12 @@ postgres: db_password: postgres database: obsrv -config_service: - host: localhost - port: 4000 - druid: router_host: http://localhost router_port: 8888 supervisor_endpoint: indexer/v1/supervisor + username: admin + password: admin123 helm_charts_base_dir: ../helm-charts @@ -175,7 +95,7 @@ connector_jobs: namespace: spark base_helm_chart: spark-connector-cron flink: - namespace: flink + namespace: flink-connectors base_helm_chart: flink-connector connector_registry: @@ -194,6 +114,8 @@ prometheus: endpoint: /api/v1/admin/tsdb/snapshot backup_prefix: prometheus_backups -dataset_api: - host: "http://localhost:3000" - pre_signed_url: "v2/files/generate-url" \ No newline at end of file +node_selector: + +container_security_context: + +pod_security_context: diff --git a/command-service/src/model/data_models.py b/command-service/src/model/data_models.py index 2ab1611d..7ad18171 100644 --- a/command-service/src/model/data_models.py +++ b/command-service/src/model/data_models.py @@ -17,7 +17,6 @@ class Action(Enum): CREATE_KAFKA_TOPIC = "CREATE_KAFKA_TOPIC" MAKE_DATASOURCE_ACTIVE = "MAKE_DATASOURCE_ACTIVE" START_PIPELINE_JOBS = "START_PIPELINE_JOBS" - CREATE_ALERT_METRIC = "CREATE_ALERT_METRIC" DEPLOY_CONNECTORS = "DEPLOY_CONNECTORS" CREATE_AUDIT_EVENT = "CREATE_AUDIT_EVENT" diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index 65abb534..87901326 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -24,6 +24,10 @@ class DatasetsLive: updated_date: datetime | None = None denorm_config: dict | None = None published_date: datetime | None = None + api_version: str = "v2" + version: int + sample_data: dict | None = None + entry_topic: str = "ingest" @dataclass diff --git a/command-service/src/routes.py b/command-service/src/routes.py index d94ce922..e0c3b5e1 100644 --- a/command-service/src/routes.py +++ b/command-service/src/routes.py @@ -170,13 +170,14 @@ async def register_connector(req: FastAPIRequest): response.params.msgid = data.get("params", {}).get("msgid", str(uuid.uuid4())) # Get the connector relative path in data - rel_path: str = data.get("relative_path", None) + download_url: str = data.get("download_url", None) + file_name: str = data.get("file_name", None) print( - f"Connector Registry | Received request to register connector: {rel_path}" + f"Connector Registry | Received request to register connector: {file_name}" ) - if not rel_path: + if not download_url: response.params.status = "Failure" return JSONResponse( { @@ -185,13 +186,13 @@ async def register_connector(req: FastAPIRequest): "ts": response.ts, "params": response.params.to_dict(), "responseCode": status.HTTP_400_BAD_REQUEST, - "error": {"message": "connector relative path is missing."}, + "error": {"message": "connector download path is missing."}, }, status_code=status.HTTP_400_BAD_REQUEST, ) executor = ConnectorRegistry() - result = executor.register(rel_path) + result = executor.register(download_url, file_name) response.params.status = result.status content = { diff --git a/system-rules-ingestor/.dockerignore b/system-rules-ingestor/.dockerignore new file mode 100644 index 00000000..30bc1627 --- /dev/null +++ b/system-rules-ingestor/.dockerignore @@ -0,0 +1 @@ +/node_modules \ No newline at end of file diff --git a/system-rules-ingestor/Dockerfile b/system-rules-ingestor/Dockerfile new file mode 100644 index 00000000..20a2c2fc --- /dev/null +++ b/system-rules-ingestor/Dockerfile @@ -0,0 +1,6 @@ +FROM --platform=linux/amd64 node:20.10-alpine +WORKDIR /opt/app +COPY ./package.json . +RUN yarn install --silent +COPY . . +CMD ["yarn", "run", "start"] \ No newline at end of file diff --git a/system-rules-ingestor/helpers/index.js b/system-rules-ingestor/helpers/index.js new file mode 100644 index 00000000..2d150dc2 --- /dev/null +++ b/system-rules-ingestor/helpers/index.js @@ -0,0 +1,86 @@ +const _ = require('lodash'); +const jsYaml = require('js-yaml'); +const fs = require('fs'); +const axios = require('axios'); +const axiosRetry = require('axios-retry').default; + +axiosRetry(axios, { retries: 3 }); + +const datasetServiceUrl = process.env.datasetServiceUrl || "http://localhost:3000"; +const folder = process.env.folder || './alerting'; +const alertSource = process.env.source || "system-rule-ingestor-job" + +const publishRule = (payload) => axios.get(`${datasetServiceUrl}/alerts/v1/publish/${_.get(payload, 'data.result.id')}`); + +const deleteRule = (payload) => axios.delete(`${datasetServiceUrl}/alerts/v1/delete`, payload); + +const deleteMetricAlias = (payload) => axios.delete(`${datasetServiceUrl}/alerts/v1/metric/alias/delete`, payload); + +const sleep = (timer) => new Promise((resolve) => setTimeout(() => resolve(), timer)); + +const createRule = (payload) => { + const { rule } = payload; + return axios.post(`${datasetServiceUrl}/alerts/v1/create`, rule); +} + +const createMetricAlias = (payload) => { + const { metricAlias } = payload; + return axios.post(`${datasetServiceUrl}/alerts/v1/metric/alias/create`, metricAlias); +} + +const transformRule = (metadata) => { + const { name, description, query, severity = "warning", operator, threshold, category, frequency, interval, labels = {}, annotations = {}, alertName } = metadata + const defaultLabels = { component: 'obsrv', type: category, alertSource: alertSource, dataset: "all" } + + const rulePayload = { + name, "manager": "grafana", description, category, interval, frequency, labels: { ...defaultLabels, ...labels }, annotations, severity, + "metadata": { + "queryBuilderContext": { + "category": category, + "metric": query, + "operator": operator, + "threshold": threshold, + "metricAlias": name + } + }, + "context": { "alertType": "SYSTEM", "alertSource": alertSource }, + "notification": {} + } + + const metricAliasPayload = { + "alias": name, + "component": category, + "metric": query, + "context": { "alertType": "SYSTEM", "alertSource": alertSource }, + } + + return { + rule: rulePayload, + metricAlias: metricAliasPayload + } +} + +const getAllRuleFiles = () => fs.readdirSync(folder); + +const convertRulesToJSON = (fileName) => { + const rulesFile = fs.readFileSync(`${folder}/${fileName}`, { encoding: 'utf-8' }); + const rulesInJSON = jsYaml.load(rulesFile); + return rulesInJSON || []; +} + +const logError = (error) => { + console.log(_.get(error, 'response.data') || _.get(error, 'message')) +} + +module.exports = { + publishRule, + deleteRule, + deleteMetricAlias, + sleep, + createRule, + createMetricAlias, + transformRule, + getAllRuleFiles, + convertRulesToJSON, + logError +} \ No newline at end of file diff --git a/system-rules-ingestor/index.js b/system-rules-ingestor/index.js new file mode 100644 index 00000000..b90aef71 --- /dev/null +++ b/system-rules-ingestor/index.js @@ -0,0 +1,73 @@ +const _ = require('lodash'); + +const { publishRule, deleteRule, deleteMetricAlias, sleep, createRule, createMetricAlias, transformRule, convertRulesToJSON, getAllRuleFiles, logError } = require('./helpers'); + +const delay = process.env.delay || 5000; +const alertSource = process.env.source || "system-rule-ingestor-job" + +/** + * @description Delete all the existing system alert rules. Also delete from the alerting manager + * + */ +const cleanExistingSystemAlerts = () => { + const payload = { data: { "filters": { "context.alertType": "SYSTEM", "context.alertSource": alertSource } } } + return deleteRule(payload); +} + +/** + * @description Delete all the existing system rules metric aliases + * + */ +const cleanExistingSystemAlias = () => { + const payload = { data: { "filters": { "context.alertType": "SYSTEM", "context.alertSource": alertSource } } } + return deleteMetricAlias(payload); +} + +/** + * @description Clean all existing system rules and metric alias + */ +const cleanup = async () => { + return cleanExistingSystemAlias() + .then(_ => cleanExistingSystemAlerts()) + .catch(logError) +} + +/** + * @description Create Alias -> Create Rule -> Publish Rule + * @param {*} payload + * @return {*} + */ +const createAndPublishRule = async (payload) => { + try { + const ruleName = _.get(payload, 'rule.name') + console.log("[CREATE-ALIAS]", ruleName); + const metricResponse = await createMetricAlias(payload).catch(logError); + const metricId = _.get(metricResponse, 'data.result.id'); + _.set(payload, 'rule.metadata.queryBuilderContext.id', metricId); + console.log("[CREATE-RULE]", ruleName); + const response = await createRule(payload) + console.log("[PUBLISH-RULE]", ruleName); + await publishRule(response); + return sleep(delay); + } catch (error) { + logError(error); + } +} + +const init = async () => { + try { + await cleanup(); + const alertRuleFiles = getAllRuleFiles(); + for (let file of alertRuleFiles) { + const rulesMetadata = await convertRulesToJSON(file); + const transformedRulesMetadata = _.flatten(_.map(rulesMetadata, transformRule)); + for (let rule of transformedRulesMetadata) { + await createAndPublishRule(rule); + } + } + } catch (error) { + logError(error) + } +} + +init(); \ No newline at end of file diff --git a/system-rules-ingestor/package.json b/system-rules-ingestor/package.json new file mode 100644 index 00000000..2f0d3d0a --- /dev/null +++ b/system-rules-ingestor/package.json @@ -0,0 +1,19 @@ +{ + "name": "system-rules-ingestor", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "start": "node index.js", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "ravinder kumar", + "license": "ISC", + "dependencies": { + "axios": "^1.5.0", + "axios-retry": "^4.0.0", + "js-yaml": "^4.1.0", + "lodash": "^4.17.21" + } +} \ No newline at end of file