From 0b2549d5202e108b5cccdc29175ff099dbe18aab Mon Sep 17 00:00:00 2001 From: lance allen Date: Thu, 23 Aug 2018 14:16:23 -0700 Subject: [PATCH] first commit from local project --- .gitignore | 9 + Pipfile | 14 ++ Pipfile.lock | 344 ++++++++++++++++++++++++++++++++++++ README.md | 34 ++++ app.py | 41 +++++ config.example.py | 11 ++ functions.py | 70 ++++++++ prebuild.py | 175 ++++++++++++++++++ zappa_settings.example.json | 24 +++ 9 files changed, 722 insertions(+) create mode 100644 .gitignore create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 README.md create mode 100644 app.py create mode 100644 config.example.py create mode 100644 functions.py create mode 100644 prebuild.py create mode 100644 zappa_settings.example.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1a01630 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.venv +venv +.env +.idea +*.zip +*.tar +*.tar.gz +config.py +zappa_settings.json diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..4d6f1ea --- /dev/null +++ b/Pipfile @@ -0,0 +1,14 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +pyramid = "*" +zappa = "*" +boto3 = "*" + +[dev-packages] + +[requires] +python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..05e496b --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,344 @@ +{ + "_meta": { + "hash": { + "sha256": "55e0fb489f47c86e4c53aaf61bbbb75bb2628725636eb1c7ca4dbbc4a09145e6" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "argcomplete": { + "hashes": [ + "sha256:c079ceb0b72d4d4e03531ed77e6071babb9d42c3f790d7def2c41295b4990b44", + "sha256:d97b7f3cfaa4e494ad59ed6d04c938fc5ed69b590bd8f53274e258fb1119bd1b" + ], + "version": "==1.9.3" + }, + "base58": { + "hashes": [ + "sha256:93fa54b615a7c406701a56e3d11c3a5defdbcd371f36c0452f1ac77623e42d16", + "sha256:c5fe8b00fab798b4a3393da6235bdecb143db505833e3f979890f7c6fc99f651" + ], + "version": "==1.0.0" + }, + "boto3": { + "hashes": [ + "sha256:08f268d6eb3347061384e144121dcca1e454a7a8b6c8424a23d3a312cdebab68", + "sha256:ce462e7505c03c3e6708ce6f264ac43d478886082af703ff69c502592df5d4f3" + ], + "index": "pypi", + "version": "==1.7.58" + }, + "botocore": { + "hashes": [ + "sha256:17a88a578161dc12ecf14950afa93a354cf009380977921f7f52891acc5e751a", + "sha256:e0e6b6d1fdbce81c28151136ee919d2cdeee13041559710cd5c93d7e4035a455" + ], + "version": "==1.10.58" + }, + "certifi": { + "hashes": [ + "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7", + "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0" + ], + "version": "==2018.4.16" + }, + "cfn-flip": { + "hashes": [ + "sha256:9c61039c71995ab204c005ec46d47d0f7a109e9f1b6d63569397f8bc648a8151" + ], + "version": "==1.0.3" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", + "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" + ], + "version": "==6.7" + }, + "docutils": { + "hashes": [ + "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", + "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", + "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + ], + "version": "==0.14" + }, + "durationpy": { + "hashes": [ + "sha256:5ef9416b527b50d722f34655becfb75e49228eb82f87b855ed1911b3314b5408" + ], + "version": "==0.5" + }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "version": "==0.16.0" + }, + "futures": { + "hashes": [ + "sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265", + "sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1" + ], + "version": "==3.2.0" + }, + "hjson": { + "hashes": [ + "sha256:1d1727faa6aaef2973921877125a3ab7c5f6d34b93233179d01770f41fab51f9" + ], + "version": "==3.0.1" + }, + "hupper": { + "hashes": [ + "sha256:20387760e4d32bd4813c2cabc8e51d92b2c22c546102a0af182c33c152cd7ede", + "sha256:6b8133e9c5cc0a8ec422a29ef3b38aea2c49a809a0af73f419a78a7015b32615" + ], + "version": "==1.3" + }, + "idna": { + "hashes": [ + "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", + "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + ], + "version": "==2.7" + }, + "jmespath": { + "hashes": [ + "sha256:6a81d4c9aa62caf061cb517b4d9ad1dd300374cd4706997aff9cd6aedd61fc64", + "sha256:f11b4461f425740a1d908e9a3f7365c3d2e569f6ca68a2ff8bc5bcd9676edd63" + ], + "version": "==0.9.3" + }, + "kappa": { + "hashes": [ + "sha256:4b5b372872f25d619e427e04282551048dc975a107385b076b3ffc6406a15833", + "sha256:4d6b7b3accce4a0aaaac92b36237a6304f0f2fffbbe3caea3f7c9f52d12c9989" + ], + "version": "==0.6.0" + }, + "lambda-packages": { + "hashes": [ + "sha256:b5e3b81ecef5f7c1b0903b5c40813536ba2343a33868a567e4e4ff1e26243406" + ], + "version": "==0.20.0" + }, + "pastedeploy": { + "hashes": [ + "sha256:39973e73f391335fac8bc8a8a95f7d34a9f42e2775600ce2dc518d93b37ef943", + "sha256:d5858f89a255e6294e63ed46b73613c56e3b9a2d82a42f1df4d06c8421a9e3cb" + ], + "version": "==1.5.2" + }, + "placebo": { + "hashes": [ + "sha256:8aa848b892924786fa5e37e75524e8ec039b7d54860d35c51ffb4ed3e30590c5" + ], + "version": "==0.8.1" + }, + "plaster": { + "hashes": [ + "sha256:215c921a438b5349931fd7df9a5a11a3572947f20f4bc6dd622ac08f1c3ba249", + "sha256:8351c7c7efdf33084c1de88dd0f422cbe7342534537b553c49b857b12d98c8c3" + ], + "version": "==1.0" + }, + "plaster-pastedeploy": { + "hashes": [ + "sha256:71e29b0ab90df8343bca5f0debe4706f0f8147308a78922c8c26e8252809bce4", + "sha256:c231130cb86ae414084008fe1d1797db7e61dc5eaafb5e755de21387c27c6fae" + ], + "version": "==0.6" + }, + "pyramid": { + "hashes": [ + "sha256:600f12e0d11211a55c2da970120af33214f77607ed45caba6af6c891afeaa771", + "sha256:cf89a48cb899291639686bf3d4a883b39e496151fa4871fb83cc1a3200d5b925" + ], + "index": "pypi", + "version": "==1.9.2" + }, + "python-dateutil": { + "hashes": [ + "sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca", + "sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c" + ], + "markers": "python_version >= '2.7'", + "version": "==2.6.1" + }, + "python-slugify": { + "hashes": [ + "sha256:57a385df7a1c6dbd15f7666eaff0ff29d3f60363b228b1197c5308ed3ba5f824", + "sha256:c3733135d3b184196fdb8844f6a74bbfb9cf6720d1dcce3254bdc434353f938f" + ], + "version": "==1.2.4" + }, + "pyyaml": { + "hashes": [ + "sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8", + "sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736", + "sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f", + "sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608", + "sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8", + "sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab", + "sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7", + "sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3", + "sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1", + "sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6", + "sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8", + "sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4", + "sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca", + "sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269" + ], + "version": "==3.12" + }, + "repoze.lru": { + "hashes": [ + "sha256:0429a75e19380e4ed50c0694e26ac8819b4ea7851ee1fc7583c8572db80aff77", + "sha256:f77bf0e1096ea445beadd35f3479c5cff2aa1efe604a133e67150bc8630a62ea" + ], + "version": "==0.7" + }, + "requests": { + "hashes": [ + "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", + "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" + ], + "version": "==2.19.1" + }, + "s3transfer": { + "hashes": [ + "sha256:90dc18e028989c609146e241ea153250be451e05ecc0c2832565231dacdf59c1", + "sha256:c7a9ec356982d5e9ab2d4b46391a7d6a950e2b04c472419f5fdec70cc0ada72f" + ], + "version": "==0.1.13" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + ], + "version": "==1.11.0" + }, + "toml": { + "hashes": [ + "sha256:8e86bd6ce8cc11b9620cb637466453d94f5d57ad86f17e98a98d1f73e3baab2d" + ], + "version": "==0.9.4" + }, + "tqdm": { + "hashes": [ + "sha256:ba650e08b8b102923a05896bf9d7e1c9cdc20b484156df0511a4bbf1f6b6f89b", + "sha256:fa6d2ea6285f56e75d7efe9259805deadc450f16066a1f82ad0629ea9be2cd0f" + ], + "version": "==4.19.1" + }, + "translationstring": { + "hashes": [ + "sha256:4ee44cfa58c52ade8910ea0ebc3d2d84bdcad9fa0422405b1801ec9b9a65b72d", + "sha256:e26c7bf383413234ed442e0980a2ebe192b95e3745288a8fd2805156d27515b4" + ], + "version": "==1.3" + }, + "troposphere": { + "hashes": [ + "sha256:aecc32359326634c9911ae4bea05d308822b827787926dcc038c153410ce380b" + ], + "version": "==2.3.1" + }, + "unidecode": { + "hashes": [ + "sha256:72f49d3729f3d8f5799f710b97c1451c5163102e76d64d20e170aedbbd923582", + "sha256:8c33dd588e0c9bc22a76eaa0c715a5434851f726131bd44a6c26471746efabf5" + ], + "version": "==1.0.22" + }, + "urllib3": { + "hashes": [ + "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", + "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" + ], + "version": "==1.23" + }, + "venusian": { + "hashes": [ + "sha256:757162c5f907e18571b6ab41b7673e5bf18cc8715abf8164292eaef4f1610668", + "sha256:9902e492c71a89a241a18b2f9950bea7e41d025cc8f3af1ea8d8201346f8577d" + ], + "version": "==1.1.0" + }, + "webob": { + "hashes": [ + "sha256:1fe722f2ab857685fc96edec567dc40b1875b21219b3b348e58cd8c4d5ea7df3", + "sha256:263690003a3e092ca1ec4df787f93feb0004e39d7bac9cba2c19a552c765894b" + ], + "version": "==1.8.2" + }, + "werkzeug": { + "hashes": [ + "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", + "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" + ], + "version": "==0.14.1" + }, + "wheel": { + "hashes": [ + "sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c", + "sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f" + ], + "version": "==0.31.1" + }, + "wsgi-request-logger": { + "hashes": [ + "sha256:445d7ec52799562f812006394d0b4a7064b37084c6ea6bd74ea7a2136c97ed83" + ], + "version": "==0.4.6" + }, + "zappa": { + "hashes": [ + "sha256:cb70195801efb8ae50c78d5180640afe1bcb9317d3c11da97b6d3588b90aea89", + "sha256:d841b2ca57c80c2fe5cd9098763ab26f59580658838eb77fcdefeed894543d45" + ], + "index": "pypi", + "version": "==0.46.1" + }, + "zope.deprecation": { + "hashes": [ + "sha256:7d52e134bbaaa0d72e1e2bc90f0587f1adc116c4bdf15912afaf2f1e8856b224", + "sha256:c83cfef3085d10dcb07de5a59a2d95713865befa46e0e88784c5648610fba789" + ], + "version": "==4.3.0" + }, + "zope.interface": { + "hashes": [ + "sha256:21506674d30c009271fe68a242d330c83b1b9d76d62d03d87e1e9528c61beea6", + "sha256:3d184aff0756c44fff7de69eb4cd5b5311b6f452d4de28cb08343b3f21993763", + "sha256:467d364b24cb398f76ad5e90398d71b9325eb4232be9e8a50d6a3b3c7a1c8789", + "sha256:57c38470d9f57e37afb460c399eb254e7193ac7fb8042bd09bdc001981a9c74c", + "sha256:9ada83f4384bbb12dedc152bcdd46a3ac9f5f7720d43ac3ce3e8e8b91d733c10", + "sha256:a1daf9c5120f3cc6f2b5fef8e1d2a3fb7bbbb20ed4bfdc25bc8364bc62dcf54b", + "sha256:e6b77ae84f2b8502d99a7855fa33334a1eb6159de45626905cb3e454c023f339", + "sha256:e881ef610ff48aece2f4ee2af03d2db1a146dc7c705561bd6089b2356f61641f", + "sha256:f41037260deaacb875db250021fe883bf536bf6414a4fd25b25059b02e31b120" + ], + "version": "==4.5.0" + } + }, + "develop": {} +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..788d3b3 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# owntracks-aws-slt + +Serverless location tracking HTTP endpoints for use with [Owntracks](https://owntracks.org). Uses several [AWS](https://aws.amazon.com) services: + +* **API Gateway** - Provides HTTP endpoints to point Owntracks client at +* **Lambda** - Processes incoming data streams and performs queries, cleanups, notifications, etc +* **S3** - Stores incoming messages + +The backend code is written in Python 3.6 and makes use of the [Pyramid](https://pylonsproject.org) web framework. + +Everything is deployed and managed using the [Zappa](https://github.com/Miserlou/Zappa) serverless management framework. + +## Requirements + +Before you can use this tool, you must have these prerequisites: + +* Amazon Web Services account +* Administrative IAM API key pair configured on your computer +* Pipenv installed on your computer +* Python 3.6 installed on your computer + +## Setup + +Assuming you have all the requirements met, the following steps will create everything needed: + +```bash +$ git clone https://github.com/lalanza808/owntracks-aws-slt.git +$ cd owntracks-aws-slt +$ pipenv install --python 3.6 +$ zappa init +$ zappa deploy live +``` + +The Zappa output should provide you with an endpoint for API Gateway - the HTTP endpoints with Python Lambda scripts being triggered behind them. diff --git a/app.py b/app.py new file mode 100644 index 0000000..24cc818 --- /dev/null +++ b/app.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +import functions as lambda_functions +import config as app_config + +from wsgiref.simple_server import make_server +from pyramid.view import view_config, view_defaults +from pyramid.config import Configurator +from configparser import ConfigParser +from pathlib import Path + + +# Generate routes and views for each device and build the app +def configure_app(): + with Configurator() as config: + for device_name in app_config.devices: + device_key = app_config.devices[device_name] + config.add_route( + device_name, + '/device/{}'.format(device_key) + ) + config.add_view( + lambda_functions.ingest, + route_name=device_name, + renderer='json' + ) + app = config.make_wsgi_app() + return app + + +# API Gateway/Zappa function handler +def generate_wsgi_app(app, environ): + wsgi_app = configure_app() + return wsgi_app(app, environ) + + +# Local web server for development +if __name__ == '__main__': + print("[+] Starting local web server on port 8080...") + server = make_server('0.0.0.0', 8080, configure_app()) + server.serve_forever() diff --git a/config.example.py b/config.example.py new file mode 100644 index 0000000..e8ee95d --- /dev/null +++ b/config.example.py @@ -0,0 +1,11 @@ +devices = { + 'my_device': 'device_secret_string' +} + +backend = { + 's3': { + 'name': 'MYLOGBUCKETNAME', + 'region': 'us-west-2', + 'retention': 365 + } +} diff --git a/functions.py b/functions.py new file mode 100644 index 0000000..7539e6e --- /dev/null +++ b/functions.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +from datetime import datetime +from json import dumps +import config as app_config +import boto3 +from pprint import pprint + +bucket_name = app_config.backend['s3']['name'] +bucket_region = app_config.backend['s3']['region'] + +athena = boto3.client('athena') + +query_tmpl = ''' + SELECT + * + FROM + history + WHERE + token = '%(token)s' + AND + year = %(year)d + AND + month = %(month)d + AND + day = %(day)d + ORDER BY + timestamp asc +''' + +response = athena.start_query_execution( + QueryString=query, + QueryExecutionContext={ + 'Database': 'locations' + }, + ResultConfiguration={ + 'OutputLocation': output_location + } + ) + +print('Execution scheduled with ID -> ', response['QueryExecutionId']) + +def ingest(request): + """Ingest incoming JSON data from Owntracks devices into S3""" + now = datetime.now() + device_name = request.matched_route.name + + # https://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1 + if request.method == "POST": + s3 = boto3.client("s3") + json_data = request.json_body + json_data["device_name"] = device_name + s3.put_object( + ACL="private", + Body=dumps(json_data), + Bucket=bucket_name, + Key="year={}/month={}/day={}/{}.json".format( + now.year, now.month, now.day, json_data["tst"] + ), + ServerSideEncryption='AES256' + ) + response_code = 202 + else: + response_code = 204 + + return { + "response": response_code, + "method": request.method, + "device": device_name + } diff --git a/prebuild.py b/prebuild.py new file mode 100644 index 0000000..72b41f9 --- /dev/null +++ b/prebuild.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python + +from botocore.client import ClientError +from datetime import datetime +from json import dumps +import config as app_config +import boto3 + + +bucket_name = app_config.backend["s3"]["name"] +bucket_region = app_config.backend["s3"]["region"] +bucket_retention = app_config.backend["s3"]["retention"] +account_id = boto3.client("sts").get_caller_identity().get("Account") +glue_role_name = "AWSGlue-{}".format(bucket_name) +glue_policy_name = "AWSGlue-{}-ReadOnly".format(bucket_name) +glue_policy_arn = "arn:aws:iam::{account_id}:policy/{policy_name}".format(account_id=account_id, policy_name=glue_policy_name) +glue_managed_policy = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" +glue_assume_policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} +glue_custom_policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:Get*", + "Resource": "arn:aws:s3:::{}/*".format(bucket_name) + } + ] +} + + +def create_bucket(): + """Create the s3 bucket used for capturing log data if it doesn"t exist already""" + s3 = boto3.resource("s3") + # s3c = boto3.client("s3") + # s3c.put_bucket_lifecycle_configuration( + # Bucket=bucket_name, + # LifecycleConfiguration={ + # "Rules": [ + # { + # "Expiration": { + # "Days": bucket_retention, + # "ExpiredObjectDeleteMarker": True + # }, + # "Status": "Enabled" + # }, + # ] + # } + # ) + + try: + s3.meta.client.head_bucket(Bucket=bucket_name) + + except ClientError: + s3.create_bucket( + ACL="private", + Bucket=bucket_name, + CreateBucketConfiguration={ + "LocationConstraint": bucket_region + } + ) + + return + + +def create_glue_iam(): + """Create the required IAM roles for AWS Glue to assume""" + iam = boto3.client("iam") + res = boto3.resource("iam") + + # Ensure IAM role exists + try: + res.meta.client.get_role(RoleName=glue_role_name) + pass + except ClientError: + iam.create_role( + RoleName=glue_role_name, + AssumeRolePolicyDocument=dumps(glue_assume_policy) + ) + + # Ensure custom IAM policy exists + try: + res.meta.client.get_policy(PolicyArn=glue_policy_arn) + pass + except ClientError: + iam.create_policy( + PolicyName=glue_policy_name, + PolicyDocument=dumps(glue_custom_policy) + ) + + # Ensure custom Glue policy is attached to role + try: + res.meta.client.attach_role_policy( + RoleName=glue_role_name, + PolicyArn=glue_policy_arn + ) + pass + except ClientError: + iam.attach_role_policy( + RoleName=glue_role_name, + PolicyArn=glue_policy_arn + ) + + # Ensure managed Glue policy is attached to role + try: + res.meta.client.attach_role_policy( + RoleName=glue_role_name, + PolicyArn=glue_managed_policy + ) + pass + except ClientError: + iam.attach_role_policy( + RoleName=glue_role_name, + PolicyArn=glue_managed_policy + ) + + return + + +def create_glue(): + """Sets up the Glue database and crawler""" + glue = boto3.client("glue") + + # Ensure Glue database exists + try: + glue.get_database(Name=bucket_name) + pass + except ClientError: + glue.create_database( + DatabaseInput={ + "Name": bucket_name, + "Description": "Owntracks SLT database" + } + ) + + # Ensure Glue crawler exists and run it if you create it + try: + glue.get_crawler(Name=bucket_name) + pass + except ClientError: + glue.create_crawler( + Name=bucket_name, + Description="Owntracks SLT data crawler", + Role=glue_role_name, + DatabaseName=bucket_name, + Targets={ + "S3Targets": [{ + "Path": "s3://{}/".format(bucket_name) + }] + } + ) + glue.start_crawler( + Name=bucket_name + ) + + return + + +def setup(): + print("[+] Setting up S3 bucket resources") + create_bucket() + print("[+] Setting up Glue IAM resources") + create_glue_iam() + print("[+] Setting up Glue resources") + create_glue() diff --git a/zappa_settings.example.json b/zappa_settings.example.json new file mode 100644 index 0000000..797a33e --- /dev/null +++ b/zappa_settings.example.json @@ -0,0 +1,24 @@ +{ + "live": { + "app_function": "app.generate_wsgi_app", + "aws_region": "us-west-2", + "profile_name": "default", + "project_name": "MYPROJECTNAME", + "prebuild_script": "prebuild.setup", + "runtime": "python3.6", + "s3_bucket": "MYZAPPABUCKETNAME", + "keep_warm": false, + "extra_permissions": [{ + "Effect": "Allow", + "Action": [ + "s3:Get*", + "s3:Put*" + ], + "Resource": ["arn:aws:s3:::MYLOGBUCKETNAME"] + }], + "tags": { + "Project": "lzahq-ha" + }, + "timeout_seconds": 10 + } +}