commit 0b2549d5202e108b5cccdc29175ff099dbe18aab Author: lance allen Date: Thu Aug 23 14:16:23 2018 -0700 first commit from local project diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1a01630 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.venv +venv +.env +.idea +*.zip +*.tar +*.tar.gz +config.py +zappa_settings.json diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..4d6f1ea --- /dev/null +++ b/Pipfile @@ -0,0 +1,14 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +pyramid = "*" +zappa = "*" +boto3 = "*" + +[dev-packages] + +[requires] +python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..05e496b --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,344 @@ +{ + "_meta": { + "hash": { + "sha256": "55e0fb489f47c86e4c53aaf61bbbb75bb2628725636eb1c7ca4dbbc4a09145e6" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "argcomplete": { + "hashes": [ + "sha256:c079ceb0b72d4d4e03531ed77e6071babb9d42c3f790d7def2c41295b4990b44", + "sha256:d97b7f3cfaa4e494ad59ed6d04c938fc5ed69b590bd8f53274e258fb1119bd1b" + ], + "version": "==1.9.3" + }, + "base58": { + "hashes": [ + "sha256:93fa54b615a7c406701a56e3d11c3a5defdbcd371f36c0452f1ac77623e42d16", + "sha256:c5fe8b00fab798b4a3393da6235bdecb143db505833e3f979890f7c6fc99f651" + ], + "version": "==1.0.0" + }, + "boto3": { + "hashes": [ + "sha256:08f268d6eb3347061384e144121dcca1e454a7a8b6c8424a23d3a312cdebab68", + "sha256:ce462e7505c03c3e6708ce6f264ac43d478886082af703ff69c502592df5d4f3" + ], + "index": "pypi", + "version": "==1.7.58" + }, + "botocore": { + "hashes": [ + "sha256:17a88a578161dc12ecf14950afa93a354cf009380977921f7f52891acc5e751a", + "sha256:e0e6b6d1fdbce81c28151136ee919d2cdeee13041559710cd5c93d7e4035a455" + ], + "version": "==1.10.58" + }, + "certifi": { + "hashes": [ + "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7", + "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0" + ], + "version": "==2018.4.16" + }, + "cfn-flip": { + "hashes": [ + "sha256:9c61039c71995ab204c005ec46d47d0f7a109e9f1b6d63569397f8bc648a8151" + ], + "version": "==1.0.3" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", + "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" + ], + "version": "==6.7" + }, + "docutils": { + "hashes": [ + "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", + "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", + "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + ], + "version": "==0.14" + }, + "durationpy": { + "hashes": [ + "sha256:5ef9416b527b50d722f34655becfb75e49228eb82f87b855ed1911b3314b5408" + ], + "version": "==0.5" + }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "version": "==0.16.0" + }, + "futures": { + "hashes": [ + "sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265", + "sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1" + ], + "version": "==3.2.0" + }, + "hjson": { + "hashes": [ + "sha256:1d1727faa6aaef2973921877125a3ab7c5f6d34b93233179d01770f41fab51f9" + ], + "version": "==3.0.1" + }, + "hupper": { + "hashes": [ + "sha256:20387760e4d32bd4813c2cabc8e51d92b2c22c546102a0af182c33c152cd7ede", + "sha256:6b8133e9c5cc0a8ec422a29ef3b38aea2c49a809a0af73f419a78a7015b32615" + ], + "version": "==1.3" + }, + "idna": { + "hashes": [ + "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", + "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + ], + "version": "==2.7" + }, + "jmespath": { + "hashes": [ + "sha256:6a81d4c9aa62caf061cb517b4d9ad1dd300374cd4706997aff9cd6aedd61fc64", + "sha256:f11b4461f425740a1d908e9a3f7365c3d2e569f6ca68a2ff8bc5bcd9676edd63" + ], + "version": "==0.9.3" + }, + "kappa": { + "hashes": [ + "sha256:4b5b372872f25d619e427e04282551048dc975a107385b076b3ffc6406a15833", + "sha256:4d6b7b3accce4a0aaaac92b36237a6304f0f2fffbbe3caea3f7c9f52d12c9989" + ], + "version": "==0.6.0" + }, + "lambda-packages": { + "hashes": [ + "sha256:b5e3b81ecef5f7c1b0903b5c40813536ba2343a33868a567e4e4ff1e26243406" + ], + "version": "==0.20.0" + }, + "pastedeploy": { + "hashes": [ + "sha256:39973e73f391335fac8bc8a8a95f7d34a9f42e2775600ce2dc518d93b37ef943", + "sha256:d5858f89a255e6294e63ed46b73613c56e3b9a2d82a42f1df4d06c8421a9e3cb" + ], + "version": "==1.5.2" + }, + "placebo": { + "hashes": [ + "sha256:8aa848b892924786fa5e37e75524e8ec039b7d54860d35c51ffb4ed3e30590c5" + ], + "version": "==0.8.1" + }, + "plaster": { + "hashes": [ + "sha256:215c921a438b5349931fd7df9a5a11a3572947f20f4bc6dd622ac08f1c3ba249", + "sha256:8351c7c7efdf33084c1de88dd0f422cbe7342534537b553c49b857b12d98c8c3" + ], + "version": "==1.0" + }, + "plaster-pastedeploy": { + "hashes": [ + "sha256:71e29b0ab90df8343bca5f0debe4706f0f8147308a78922c8c26e8252809bce4", + "sha256:c231130cb86ae414084008fe1d1797db7e61dc5eaafb5e755de21387c27c6fae" + ], + "version": "==0.6" + }, + "pyramid": { + "hashes": [ + "sha256:600f12e0d11211a55c2da970120af33214f77607ed45caba6af6c891afeaa771", + "sha256:cf89a48cb899291639686bf3d4a883b39e496151fa4871fb83cc1a3200d5b925" + ], + "index": "pypi", + "version": "==1.9.2" + }, + "python-dateutil": { + "hashes": [ + "sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca", + "sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c" + ], + "markers": "python_version >= '2.7'", + "version": "==2.6.1" + }, + "python-slugify": { + "hashes": [ + "sha256:57a385df7a1c6dbd15f7666eaff0ff29d3f60363b228b1197c5308ed3ba5f824", + "sha256:c3733135d3b184196fdb8844f6a74bbfb9cf6720d1dcce3254bdc434353f938f" + ], + "version": "==1.2.4" + }, + "pyyaml": { + "hashes": [ + "sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8", + "sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736", + "sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f", + "sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608", + "sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8", + "sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab", + "sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7", + "sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3", + "sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1", + "sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6", + "sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8", + "sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4", + "sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca", + "sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269" + ], + "version": "==3.12" + }, + "repoze.lru": { + "hashes": [ + "sha256:0429a75e19380e4ed50c0694e26ac8819b4ea7851ee1fc7583c8572db80aff77", + "sha256:f77bf0e1096ea445beadd35f3479c5cff2aa1efe604a133e67150bc8630a62ea" + ], + "version": "==0.7" + }, + "requests": { + "hashes": [ + "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", + "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" + ], + "version": "==2.19.1" + }, + "s3transfer": { + "hashes": [ + "sha256:90dc18e028989c609146e241ea153250be451e05ecc0c2832565231dacdf59c1", + "sha256:c7a9ec356982d5e9ab2d4b46391a7d6a950e2b04c472419f5fdec70cc0ada72f" + ], + "version": "==0.1.13" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + ], + "version": "==1.11.0" + }, + "toml": { + "hashes": [ + "sha256:8e86bd6ce8cc11b9620cb637466453d94f5d57ad86f17e98a98d1f73e3baab2d" + ], + "version": "==0.9.4" + }, + "tqdm": { + "hashes": [ + "sha256:ba650e08b8b102923a05896bf9d7e1c9cdc20b484156df0511a4bbf1f6b6f89b", + "sha256:fa6d2ea6285f56e75d7efe9259805deadc450f16066a1f82ad0629ea9be2cd0f" + ], + "version": "==4.19.1" + }, + "translationstring": { + "hashes": [ + "sha256:4ee44cfa58c52ade8910ea0ebc3d2d84bdcad9fa0422405b1801ec9b9a65b72d", + "sha256:e26c7bf383413234ed442e0980a2ebe192b95e3745288a8fd2805156d27515b4" + ], + "version": "==1.3" + }, + "troposphere": { + "hashes": [ + "sha256:aecc32359326634c9911ae4bea05d308822b827787926dcc038c153410ce380b" + ], + "version": "==2.3.1" + }, + "unidecode": { + "hashes": [ + "sha256:72f49d3729f3d8f5799f710b97c1451c5163102e76d64d20e170aedbbd923582", + "sha256:8c33dd588e0c9bc22a76eaa0c715a5434851f726131bd44a6c26471746efabf5" + ], + "version": "==1.0.22" + }, + "urllib3": { + "hashes": [ + "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", + "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" + ], + "version": "==1.23" + }, + "venusian": { + "hashes": [ + "sha256:757162c5f907e18571b6ab41b7673e5bf18cc8715abf8164292eaef4f1610668", + "sha256:9902e492c71a89a241a18b2f9950bea7e41d025cc8f3af1ea8d8201346f8577d" + ], + "version": "==1.1.0" + }, + "webob": { + "hashes": [ + "sha256:1fe722f2ab857685fc96edec567dc40b1875b21219b3b348e58cd8c4d5ea7df3", + "sha256:263690003a3e092ca1ec4df787f93feb0004e39d7bac9cba2c19a552c765894b" + ], + "version": "==1.8.2" + }, + "werkzeug": { + "hashes": [ + "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", + "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" + ], + "version": "==0.14.1" + }, + "wheel": { + "hashes": [ + "sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c", + "sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f" + ], + "version": "==0.31.1" + }, + "wsgi-request-logger": { + "hashes": [ + "sha256:445d7ec52799562f812006394d0b4a7064b37084c6ea6bd74ea7a2136c97ed83" + ], + "version": "==0.4.6" + }, + "zappa": { + "hashes": [ + "sha256:cb70195801efb8ae50c78d5180640afe1bcb9317d3c11da97b6d3588b90aea89", + "sha256:d841b2ca57c80c2fe5cd9098763ab26f59580658838eb77fcdefeed894543d45" + ], + "index": "pypi", + "version": "==0.46.1" + }, + "zope.deprecation": { + "hashes": [ + "sha256:7d52e134bbaaa0d72e1e2bc90f0587f1adc116c4bdf15912afaf2f1e8856b224", + "sha256:c83cfef3085d10dcb07de5a59a2d95713865befa46e0e88784c5648610fba789" + ], + "version": "==4.3.0" + }, + "zope.interface": { + "hashes": [ + "sha256:21506674d30c009271fe68a242d330c83b1b9d76d62d03d87e1e9528c61beea6", + "sha256:3d184aff0756c44fff7de69eb4cd5b5311b6f452d4de28cb08343b3f21993763", + "sha256:467d364b24cb398f76ad5e90398d71b9325eb4232be9e8a50d6a3b3c7a1c8789", + "sha256:57c38470d9f57e37afb460c399eb254e7193ac7fb8042bd09bdc001981a9c74c", + "sha256:9ada83f4384bbb12dedc152bcdd46a3ac9f5f7720d43ac3ce3e8e8b91d733c10", + "sha256:a1daf9c5120f3cc6f2b5fef8e1d2a3fb7bbbb20ed4bfdc25bc8364bc62dcf54b", + "sha256:e6b77ae84f2b8502d99a7855fa33334a1eb6159de45626905cb3e454c023f339", + "sha256:e881ef610ff48aece2f4ee2af03d2db1a146dc7c705561bd6089b2356f61641f", + "sha256:f41037260deaacb875db250021fe883bf536bf6414a4fd25b25059b02e31b120" + ], + "version": "==4.5.0" + } + }, + "develop": {} +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..788d3b3 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# owntracks-aws-slt + +Serverless location tracking HTTP endpoints for use with [Owntracks](https://owntracks.org). Uses several [AWS](https://aws.amazon.com) services: + +* **API Gateway** - Provides HTTP endpoints to point Owntracks client at +* **Lambda** - Processes incoming data streams and performs queries, cleanups, notifications, etc +* **S3** - Stores incoming messages + +The backend code is written in Python 3.6 and makes use of the [Pyramid](https://pylonsproject.org) web framework. + +Everything is deployed and managed using the [Zappa](https://github.com/Miserlou/Zappa) serverless management framework. + +## Requirements + +Before you can use this tool, you must have these prerequisites: + +* Amazon Web Services account +* Administrative IAM API key pair configured on your computer +* Pipenv installed on your computer +* Python 3.6 installed on your computer + +## Setup + +Assuming you have all the requirements met, the following steps will create everything needed: + +```bash +$ git clone https://github.com/lalanza808/owntracks-aws-slt.git +$ cd owntracks-aws-slt +$ pipenv install --python 3.6 +$ zappa init +$ zappa deploy live +``` + +The Zappa output should provide you with an endpoint for API Gateway - the HTTP endpoints with Python Lambda scripts being triggered behind them. diff --git a/app.py b/app.py new file mode 100644 index 0000000..24cc818 --- /dev/null +++ b/app.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +import functions as lambda_functions +import config as app_config + +from wsgiref.simple_server import make_server +from pyramid.view import view_config, view_defaults +from pyramid.config import Configurator +from configparser import ConfigParser +from pathlib import Path + + +# Generate routes and views for each device and build the app +def configure_app(): + with Configurator() as config: + for device_name in app_config.devices: + device_key = app_config.devices[device_name] + config.add_route( + device_name, + '/device/{}'.format(device_key) + ) + config.add_view( + lambda_functions.ingest, + route_name=device_name, + renderer='json' + ) + app = config.make_wsgi_app() + return app + + +# API Gateway/Zappa function handler +def generate_wsgi_app(app, environ): + wsgi_app = configure_app() + return wsgi_app(app, environ) + + +# Local web server for development +if __name__ == '__main__': + print("[+] Starting local web server on port 8080...") + server = make_server('0.0.0.0', 8080, configure_app()) + server.serve_forever() diff --git a/config.example.py b/config.example.py new file mode 100644 index 0000000..e8ee95d --- /dev/null +++ b/config.example.py @@ -0,0 +1,11 @@ +devices = { + 'my_device': 'device_secret_string' +} + +backend = { + 's3': { + 'name': 'MYLOGBUCKETNAME', + 'region': 'us-west-2', + 'retention': 365 + } +} diff --git a/functions.py b/functions.py new file mode 100644 index 0000000..7539e6e --- /dev/null +++ b/functions.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +from datetime import datetime +from json import dumps +import config as app_config +import boto3 +from pprint import pprint + +bucket_name = app_config.backend['s3']['name'] +bucket_region = app_config.backend['s3']['region'] + +athena = boto3.client('athena') + +query_tmpl = ''' + SELECT + * + FROM + history + WHERE + token = '%(token)s' + AND + year = %(year)d + AND + month = %(month)d + AND + day = %(day)d + ORDER BY + timestamp asc +''' + +response = athena.start_query_execution( + QueryString=query, + QueryExecutionContext={ + 'Database': 'locations' + }, + ResultConfiguration={ + 'OutputLocation': output_location + } + ) + +print('Execution scheduled with ID -> ', response['QueryExecutionId']) + +def ingest(request): + """Ingest incoming JSON data from Owntracks devices into S3""" + now = datetime.now() + device_name = request.matched_route.name + + # https://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1 + if request.method == "POST": + s3 = boto3.client("s3") + json_data = request.json_body + json_data["device_name"] = device_name + s3.put_object( + ACL="private", + Body=dumps(json_data), + Bucket=bucket_name, + Key="year={}/month={}/day={}/{}.json".format( + now.year, now.month, now.day, json_data["tst"] + ), + ServerSideEncryption='AES256' + ) + response_code = 202 + else: + response_code = 204 + + return { + "response": response_code, + "method": request.method, + "device": device_name + } diff --git a/prebuild.py b/prebuild.py new file mode 100644 index 0000000..72b41f9 --- /dev/null +++ b/prebuild.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python + +from botocore.client import ClientError +from datetime import datetime +from json import dumps +import config as app_config +import boto3 + + +bucket_name = app_config.backend["s3"]["name"] +bucket_region = app_config.backend["s3"]["region"] +bucket_retention = app_config.backend["s3"]["retention"] +account_id = boto3.client("sts").get_caller_identity().get("Account") +glue_role_name = "AWSGlue-{}".format(bucket_name) +glue_policy_name = "AWSGlue-{}-ReadOnly".format(bucket_name) +glue_policy_arn = "arn:aws:iam::{account_id}:policy/{policy_name}".format(account_id=account_id, policy_name=glue_policy_name) +glue_managed_policy = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" +glue_assume_policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} +glue_custom_policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:Get*", + "Resource": "arn:aws:s3:::{}/*".format(bucket_name) + } + ] +} + + +def create_bucket(): + """Create the s3 bucket used for capturing log data if it doesn"t exist already""" + s3 = boto3.resource("s3") + # s3c = boto3.client("s3") + # s3c.put_bucket_lifecycle_configuration( + # Bucket=bucket_name, + # LifecycleConfiguration={ + # "Rules": [ + # { + # "Expiration": { + # "Days": bucket_retention, + # "ExpiredObjectDeleteMarker": True + # }, + # "Status": "Enabled" + # }, + # ] + # } + # ) + + try: + s3.meta.client.head_bucket(Bucket=bucket_name) + + except ClientError: + s3.create_bucket( + ACL="private", + Bucket=bucket_name, + CreateBucketConfiguration={ + "LocationConstraint": bucket_region + } + ) + + return + + +def create_glue_iam(): + """Create the required IAM roles for AWS Glue to assume""" + iam = boto3.client("iam") + res = boto3.resource("iam") + + # Ensure IAM role exists + try: + res.meta.client.get_role(RoleName=glue_role_name) + pass + except ClientError: + iam.create_role( + RoleName=glue_role_name, + AssumeRolePolicyDocument=dumps(glue_assume_policy) + ) + + # Ensure custom IAM policy exists + try: + res.meta.client.get_policy(PolicyArn=glue_policy_arn) + pass + except ClientError: + iam.create_policy( + PolicyName=glue_policy_name, + PolicyDocument=dumps(glue_custom_policy) + ) + + # Ensure custom Glue policy is attached to role + try: + res.meta.client.attach_role_policy( + RoleName=glue_role_name, + PolicyArn=glue_policy_arn + ) + pass + except ClientError: + iam.attach_role_policy( + RoleName=glue_role_name, + PolicyArn=glue_policy_arn + ) + + # Ensure managed Glue policy is attached to role + try: + res.meta.client.attach_role_policy( + RoleName=glue_role_name, + PolicyArn=glue_managed_policy + ) + pass + except ClientError: + iam.attach_role_policy( + RoleName=glue_role_name, + PolicyArn=glue_managed_policy + ) + + return + + +def create_glue(): + """Sets up the Glue database and crawler""" + glue = boto3.client("glue") + + # Ensure Glue database exists + try: + glue.get_database(Name=bucket_name) + pass + except ClientError: + glue.create_database( + DatabaseInput={ + "Name": bucket_name, + "Description": "Owntracks SLT database" + } + ) + + # Ensure Glue crawler exists and run it if you create it + try: + glue.get_crawler(Name=bucket_name) + pass + except ClientError: + glue.create_crawler( + Name=bucket_name, + Description="Owntracks SLT data crawler", + Role=glue_role_name, + DatabaseName=bucket_name, + Targets={ + "S3Targets": [{ + "Path": "s3://{}/".format(bucket_name) + }] + } + ) + glue.start_crawler( + Name=bucket_name + ) + + return + + +def setup(): + print("[+] Setting up S3 bucket resources") + create_bucket() + print("[+] Setting up Glue IAM resources") + create_glue_iam() + print("[+] Setting up Glue resources") + create_glue() diff --git a/zappa_settings.example.json b/zappa_settings.example.json new file mode 100644 index 0000000..797a33e --- /dev/null +++ b/zappa_settings.example.json @@ -0,0 +1,24 @@ +{ + "live": { + "app_function": "app.generate_wsgi_app", + "aws_region": "us-west-2", + "profile_name": "default", + "project_name": "MYPROJECTNAME", + "prebuild_script": "prebuild.setup", + "runtime": "python3.6", + "s3_bucket": "MYZAPPABUCKETNAME", + "keep_warm": false, + "extra_permissions": [{ + "Effect": "Allow", + "Action": [ + "s3:Get*", + "s3:Put*" + ], + "Resource": ["arn:aws:s3:::MYLOGBUCKETNAME"] + }], + "tags": { + "Project": "lzahq-ha" + }, + "timeout_seconds": 10 + } +}