first commit from local project
lance allen
6 years ago
commit
0b2549d520
@ -0,0 +1,9 @@
|
||||
.venv
|
||||
venv
|
||||
.env
|
||||
.idea
|
||||
*.zip
|
||||
*.tar
|
||||
*.tar.gz
|
||||
config.py
|
||||
zappa_settings.json
|
||||
@ -0,0 +1,14 @@
|
||||
[[source]]
|
||||
url = "https://pypi.org/simple"
|
||||
verify_ssl = true
|
||||
name = "pypi"
|
||||
|
||||
[packages]
|
||||
pyramid = "*"
|
||||
zappa = "*"
|
||||
boto3 = "*"
|
||||
|
||||
[dev-packages]
|
||||
|
||||
[requires]
|
||||
python_version = "3.6"
|
||||
@ -0,0 +1,344 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "55e0fb489f47c86e4c53aaf61bbbb75bb2628725636eb1c7ca4dbbc4a09145e6"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.6"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"name": "pypi",
|
||||
"url": "https://pypi.org/simple",
|
||||
"verify_ssl": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"argcomplete": {
|
||||
"hashes": [
|
||||
"sha256:c079ceb0b72d4d4e03531ed77e6071babb9d42c3f790d7def2c41295b4990b44",
|
||||
"sha256:d97b7f3cfaa4e494ad59ed6d04c938fc5ed69b590bd8f53274e258fb1119bd1b"
|
||||
],
|
||||
"version": "==1.9.3"
|
||||
},
|
||||
"base58": {
|
||||
"hashes": [
|
||||
"sha256:93fa54b615a7c406701a56e3d11c3a5defdbcd371f36c0452f1ac77623e42d16",
|
||||
"sha256:c5fe8b00fab798b4a3393da6235bdecb143db505833e3f979890f7c6fc99f651"
|
||||
],
|
||||
"version": "==1.0.0"
|
||||
},
|
||||
"boto3": {
|
||||
"hashes": [
|
||||
"sha256:08f268d6eb3347061384e144121dcca1e454a7a8b6c8424a23d3a312cdebab68",
|
||||
"sha256:ce462e7505c03c3e6708ce6f264ac43d478886082af703ff69c502592df5d4f3"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.7.58"
|
||||
},
|
||||
"botocore": {
|
||||
"hashes": [
|
||||
"sha256:17a88a578161dc12ecf14950afa93a354cf009380977921f7f52891acc5e751a",
|
||||
"sha256:e0e6b6d1fdbce81c28151136ee919d2cdeee13041559710cd5c93d7e4035a455"
|
||||
],
|
||||
"version": "==1.10.58"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7",
|
||||
"sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0"
|
||||
],
|
||||
"version": "==2018.4.16"
|
||||
},
|
||||
"cfn-flip": {
|
||||
"hashes": [
|
||||
"sha256:9c61039c71995ab204c005ec46d47d0f7a109e9f1b6d63569397f8bc648a8151"
|
||||
],
|
||||
"version": "==1.0.3"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
|
||||
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
|
||||
],
|
||||
"version": "==3.0.4"
|
||||
},
|
||||
"click": {
|
||||
"hashes": [
|
||||
"sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d",
|
||||
"sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b"
|
||||
],
|
||||
"version": "==6.7"
|
||||
},
|
||||
"docutils": {
|
||||
"hashes": [
|
||||
"sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6",
|
||||
"sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274",
|
||||
"sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6"
|
||||
],
|
||||
"version": "==0.14"
|
||||
},
|
||||
"durationpy": {
|
||||
"hashes": [
|
||||
"sha256:5ef9416b527b50d722f34655becfb75e49228eb82f87b855ed1911b3314b5408"
|
||||
],
|
||||
"version": "==0.5"
|
||||
},
|
||||
"future": {
|
||||
"hashes": [
|
||||
"sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb"
|
||||
],
|
||||
"version": "==0.16.0"
|
||||
},
|
||||
"futures": {
|
||||
"hashes": [
|
||||
"sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265",
|
||||
"sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1"
|
||||
],
|
||||
"version": "==3.2.0"
|
||||
},
|
||||
"hjson": {
|
||||
"hashes": [
|
||||
"sha256:1d1727faa6aaef2973921877125a3ab7c5f6d34b93233179d01770f41fab51f9"
|
||||
],
|
||||
"version": "==3.0.1"
|
||||
},
|
||||
"hupper": {
|
||||
"hashes": [
|
||||
"sha256:20387760e4d32bd4813c2cabc8e51d92b2c22c546102a0af182c33c152cd7ede",
|
||||
"sha256:6b8133e9c5cc0a8ec422a29ef3b38aea2c49a809a0af73f419a78a7015b32615"
|
||||
],
|
||||
"version": "==1.3"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e",
|
||||
"sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16"
|
||||
],
|
||||
"version": "==2.7"
|
||||
},
|
||||
"jmespath": {
|
||||
"hashes": [
|
||||
"sha256:6a81d4c9aa62caf061cb517b4d9ad1dd300374cd4706997aff9cd6aedd61fc64",
|
||||
"sha256:f11b4461f425740a1d908e9a3f7365c3d2e569f6ca68a2ff8bc5bcd9676edd63"
|
||||
],
|
||||
"version": "==0.9.3"
|
||||
},
|
||||
"kappa": {
|
||||
"hashes": [
|
||||
"sha256:4b5b372872f25d619e427e04282551048dc975a107385b076b3ffc6406a15833",
|
||||
"sha256:4d6b7b3accce4a0aaaac92b36237a6304f0f2fffbbe3caea3f7c9f52d12c9989"
|
||||
],
|
||||
"version": "==0.6.0"
|
||||
},
|
||||
"lambda-packages": {
|
||||
"hashes": [
|
||||
"sha256:b5e3b81ecef5f7c1b0903b5c40813536ba2343a33868a567e4e4ff1e26243406"
|
||||
],
|
||||
"version": "==0.20.0"
|
||||
},
|
||||
"pastedeploy": {
|
||||
"hashes": [
|
||||
"sha256:39973e73f391335fac8bc8a8a95f7d34a9f42e2775600ce2dc518d93b37ef943",
|
||||
"sha256:d5858f89a255e6294e63ed46b73613c56e3b9a2d82a42f1df4d06c8421a9e3cb"
|
||||
],
|
||||
"version": "==1.5.2"
|
||||
},
|
||||
"placebo": {
|
||||
"hashes": [
|
||||
"sha256:8aa848b892924786fa5e37e75524e8ec039b7d54860d35c51ffb4ed3e30590c5"
|
||||
],
|
||||
"version": "==0.8.1"
|
||||
},
|
||||
"plaster": {
|
||||
"hashes": [
|
||||
"sha256:215c921a438b5349931fd7df9a5a11a3572947f20f4bc6dd622ac08f1c3ba249",
|
||||
"sha256:8351c7c7efdf33084c1de88dd0f422cbe7342534537b553c49b857b12d98c8c3"
|
||||
],
|
||||
"version": "==1.0"
|
||||
},
|
||||
"plaster-pastedeploy": {
|
||||
"hashes": [
|
||||
"sha256:71e29b0ab90df8343bca5f0debe4706f0f8147308a78922c8c26e8252809bce4",
|
||||
"sha256:c231130cb86ae414084008fe1d1797db7e61dc5eaafb5e755de21387c27c6fae"
|
||||
],
|
||||
"version": "==0.6"
|
||||
},
|
||||
"pyramid": {
|
||||
"hashes": [
|
||||
"sha256:600f12e0d11211a55c2da970120af33214f77607ed45caba6af6c891afeaa771",
|
||||
"sha256:cf89a48cb899291639686bf3d4a883b39e496151fa4871fb83cc1a3200d5b925"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.9.2"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca",
|
||||
"sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c"
|
||||
],
|
||||
"markers": "python_version >= '2.7'",
|
||||
"version": "==2.6.1"
|
||||
},
|
||||
"python-slugify": {
|
||||
"hashes": [
|
||||
"sha256:57a385df7a1c6dbd15f7666eaff0ff29d3f60363b228b1197c5308ed3ba5f824",
|
||||
"sha256:c3733135d3b184196fdb8844f6a74bbfb9cf6720d1dcce3254bdc434353f938f"
|
||||
],
|
||||
"version": "==1.2.4"
|
||||
},
|
||||
"pyyaml": {
|
||||
"hashes": [
|
||||
"sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8",
|
||||
"sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736",
|
||||
"sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f",
|
||||
"sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608",
|
||||
"sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8",
|
||||
"sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab",
|
||||
"sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7",
|
||||
"sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3",
|
||||
"sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1",
|
||||
"sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6",
|
||||
"sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8",
|
||||
"sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4",
|
||||
"sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca",
|
||||
"sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269"
|
||||
],
|
||||
"version": "==3.12"
|
||||
},
|
||||
"repoze.lru": {
|
||||
"hashes": [
|
||||
"sha256:0429a75e19380e4ed50c0694e26ac8819b4ea7851ee1fc7583c8572db80aff77",
|
||||
"sha256:f77bf0e1096ea445beadd35f3479c5cff2aa1efe604a133e67150bc8630a62ea"
|
||||
],
|
||||
"version": "==0.7"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1",
|
||||
"sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a"
|
||||
],
|
||||
"version": "==2.19.1"
|
||||
},
|
||||
"s3transfer": {
|
||||
"hashes": [
|
||||
"sha256:90dc18e028989c609146e241ea153250be451e05ecc0c2832565231dacdf59c1",
|
||||
"sha256:c7a9ec356982d5e9ab2d4b46391a7d6a950e2b04c472419f5fdec70cc0ada72f"
|
||||
],
|
||||
"version": "==0.1.13"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
|
||||
"sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
|
||||
],
|
||||
"version": "==1.11.0"
|
||||
},
|
||||
"toml": {
|
||||
"hashes": [
|
||||
"sha256:8e86bd6ce8cc11b9620cb637466453d94f5d57ad86f17e98a98d1f73e3baab2d"
|
||||
],
|
||||
"version": "==0.9.4"
|
||||
},
|
||||
"tqdm": {
|
||||
"hashes": [
|
||||
"sha256:ba650e08b8b102923a05896bf9d7e1c9cdc20b484156df0511a4bbf1f6b6f89b",
|
||||
"sha256:fa6d2ea6285f56e75d7efe9259805deadc450f16066a1f82ad0629ea9be2cd0f"
|
||||
],
|
||||
"version": "==4.19.1"
|
||||
},
|
||||
"translationstring": {
|
||||
"hashes": [
|
||||
"sha256:4ee44cfa58c52ade8910ea0ebc3d2d84bdcad9fa0422405b1801ec9b9a65b72d",
|
||||
"sha256:e26c7bf383413234ed442e0980a2ebe192b95e3745288a8fd2805156d27515b4"
|
||||
],
|
||||
"version": "==1.3"
|
||||
},
|
||||
"troposphere": {
|
||||
"hashes": [
|
||||
"sha256:aecc32359326634c9911ae4bea05d308822b827787926dcc038c153410ce380b"
|
||||
],
|
||||
"version": "==2.3.1"
|
||||
},
|
||||
"unidecode": {
|
||||
"hashes": [
|
||||
"sha256:72f49d3729f3d8f5799f710b97c1451c5163102e76d64d20e170aedbbd923582",
|
||||
"sha256:8c33dd588e0c9bc22a76eaa0c715a5434851f726131bd44a6c26471746efabf5"
|
||||
],
|
||||
"version": "==1.0.22"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf",
|
||||
"sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5"
|
||||
],
|
||||
"version": "==1.23"
|
||||
},
|
||||
"venusian": {
|
||||
"hashes": [
|
||||
"sha256:757162c5f907e18571b6ab41b7673e5bf18cc8715abf8164292eaef4f1610668",
|
||||
"sha256:9902e492c71a89a241a18b2f9950bea7e41d025cc8f3af1ea8d8201346f8577d"
|
||||
],
|
||||
"version": "==1.1.0"
|
||||
},
|
||||
"webob": {
|
||||
"hashes": [
|
||||
"sha256:1fe722f2ab857685fc96edec567dc40b1875b21219b3b348e58cd8c4d5ea7df3",
|
||||
"sha256:263690003a3e092ca1ec4df787f93feb0004e39d7bac9cba2c19a552c765894b"
|
||||
],
|
||||
"version": "==1.8.2"
|
||||
},
|
||||
"werkzeug": {
|
||||
"hashes": [
|
||||
"sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c",
|
||||
"sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b"
|
||||
],
|
||||
"version": "==0.14.1"
|
||||
},
|
||||
"wheel": {
|
||||
"hashes": [
|
||||
"sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c",
|
||||
"sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f"
|
||||
],
|
||||
"version": "==0.31.1"
|
||||
},
|
||||
"wsgi-request-logger": {
|
||||
"hashes": [
|
||||
"sha256:445d7ec52799562f812006394d0b4a7064b37084c6ea6bd74ea7a2136c97ed83"
|
||||
],
|
||||
"version": "==0.4.6"
|
||||
},
|
||||
"zappa": {
|
||||
"hashes": [
|
||||
"sha256:cb70195801efb8ae50c78d5180640afe1bcb9317d3c11da97b6d3588b90aea89",
|
||||
"sha256:d841b2ca57c80c2fe5cd9098763ab26f59580658838eb77fcdefeed894543d45"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.46.1"
|
||||
},
|
||||
"zope.deprecation": {
|
||||
"hashes": [
|
||||
"sha256:7d52e134bbaaa0d72e1e2bc90f0587f1adc116c4bdf15912afaf2f1e8856b224",
|
||||
"sha256:c83cfef3085d10dcb07de5a59a2d95713865befa46e0e88784c5648610fba789"
|
||||
],
|
||||
"version": "==4.3.0"
|
||||
},
|
||||
"zope.interface": {
|
||||
"hashes": [
|
||||
"sha256:21506674d30c009271fe68a242d330c83b1b9d76d62d03d87e1e9528c61beea6",
|
||||
"sha256:3d184aff0756c44fff7de69eb4cd5b5311b6f452d4de28cb08343b3f21993763",
|
||||
"sha256:467d364b24cb398f76ad5e90398d71b9325eb4232be9e8a50d6a3b3c7a1c8789",
|
||||
"sha256:57c38470d9f57e37afb460c399eb254e7193ac7fb8042bd09bdc001981a9c74c",
|
||||
"sha256:9ada83f4384bbb12dedc152bcdd46a3ac9f5f7720d43ac3ce3e8e8b91d733c10",
|
||||
"sha256:a1daf9c5120f3cc6f2b5fef8e1d2a3fb7bbbb20ed4bfdc25bc8364bc62dcf54b",
|
||||
"sha256:e6b77ae84f2b8502d99a7855fa33334a1eb6159de45626905cb3e454c023f339",
|
||||
"sha256:e881ef610ff48aece2f4ee2af03d2db1a146dc7c705561bd6089b2356f61641f",
|
||||
"sha256:f41037260deaacb875db250021fe883bf536bf6414a4fd25b25059b02e31b120"
|
||||
],
|
||||
"version": "==4.5.0"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
# owntracks-aws-slt
|
||||
|
||||
Serverless location tracking HTTP endpoints for use with [Owntracks](https://owntracks.org). Uses several [AWS](https://aws.amazon.com) services:
|
||||
|
||||
* **API Gateway** - Provides HTTP endpoints to point Owntracks client at
|
||||
* **Lambda** - Processes incoming data streams and performs queries, cleanups, notifications, etc
|
||||
* **S3** - Stores incoming messages
|
||||
|
||||
The backend code is written in Python 3.6 and makes use of the [Pyramid](https://pylonsproject.org) web framework.
|
||||
|
||||
Everything is deployed and managed using the [Zappa](https://github.com/Miserlou/Zappa) serverless management framework.
|
||||
|
||||
## Requirements
|
||||
|
||||
Before you can use this tool, you must have these prerequisites:
|
||||
|
||||
* Amazon Web Services account
|
||||
* Administrative IAM API key pair configured on your computer
|
||||
* Pipenv installed on your computer
|
||||
* Python 3.6 installed on your computer
|
||||
|
||||
## Setup
|
||||
|
||||
Assuming you have all the requirements met, the following steps will create everything needed:
|
||||
|
||||
```bash
|
||||
$ git clone https://github.com/lalanza808/owntracks-aws-slt.git
|
||||
$ cd owntracks-aws-slt
|
||||
$ pipenv install --python 3.6
|
||||
$ zappa init
|
||||
$ zappa deploy live
|
||||
```
|
||||
|
||||
The Zappa output should provide you with an endpoint for API Gateway - the HTTP endpoints with Python Lambda scripts being triggered behind them.
|
||||
@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import functions as lambda_functions
|
||||
import config as app_config
|
||||
|
||||
from wsgiref.simple_server import make_server
|
||||
from pyramid.view import view_config, view_defaults
|
||||
from pyramid.config import Configurator
|
||||
from configparser import ConfigParser
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# Generate routes and views for each device and build the app
|
||||
def configure_app():
|
||||
with Configurator() as config:
|
||||
for device_name in app_config.devices:
|
||||
device_key = app_config.devices[device_name]
|
||||
config.add_route(
|
||||
device_name,
|
||||
'/device/{}'.format(device_key)
|
||||
)
|
||||
config.add_view(
|
||||
lambda_functions.ingest,
|
||||
route_name=device_name,
|
||||
renderer='json'
|
||||
)
|
||||
app = config.make_wsgi_app()
|
||||
return app
|
||||
|
||||
|
||||
# API Gateway/Zappa function handler
|
||||
def generate_wsgi_app(app, environ):
|
||||
wsgi_app = configure_app()
|
||||
return wsgi_app(app, environ)
|
||||
|
||||
|
||||
# Local web server for development
|
||||
if __name__ == '__main__':
|
||||
print("[+] Starting local web server on port 8080...")
|
||||
server = make_server('0.0.0.0', 8080, configure_app())
|
||||
server.serve_forever()
|
||||
@ -0,0 +1,11 @@
|
||||
devices = {
|
||||
'my_device': 'device_secret_string'
|
||||
}
|
||||
|
||||
backend = {
|
||||
's3': {
|
||||
'name': 'MYLOGBUCKETNAME',
|
||||
'region': 'us-west-2',
|
||||
'retention': 365
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from datetime import datetime
|
||||
from json import dumps
|
||||
import config as app_config
|
||||
import boto3
|
||||
from pprint import pprint
|
||||
|
||||
bucket_name = app_config.backend['s3']['name']
|
||||
bucket_region = app_config.backend['s3']['region']
|
||||
|
||||
athena = boto3.client('athena')
|
||||
|
||||
query_tmpl = '''
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
history
|
||||
WHERE
|
||||
token = '%(token)s'
|
||||
AND
|
||||
year = %(year)d
|
||||
AND
|
||||
month = %(month)d
|
||||
AND
|
||||
day = %(day)d
|
||||
ORDER BY
|
||||
timestamp asc
|
||||
'''
|
||||
|
||||
response = athena.start_query_execution(
|
||||
QueryString=query,
|
||||
QueryExecutionContext={
|
||||
'Database': 'locations'
|
||||
},
|
||||
ResultConfiguration={
|
||||
'OutputLocation': output_location
|
||||
}
|
||||
)
|
||||
|
||||
print('Execution scheduled with ID -> ', response['QueryExecutionId'])
|
||||
|
||||
def ingest(request):
|
||||
"""Ingest incoming JSON data from Owntracks devices into S3"""
|
||||
now = datetime.now()
|
||||
device_name = request.matched_route.name
|
||||
|
||||
# https://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1
|
||||
if request.method == "POST":
|
||||
s3 = boto3.client("s3")
|
||||
json_data = request.json_body
|
||||
json_data["device_name"] = device_name
|
||||
s3.put_object(
|
||||
ACL="private",
|
||||
Body=dumps(json_data),
|
||||
Bucket=bucket_name,
|
||||
Key="year={}/month={}/day={}/{}.json".format(
|
||||
now.year, now.month, now.day, json_data["tst"]
|
||||
),
|
||||
ServerSideEncryption='AES256'
|
||||
)
|
||||
response_code = 202
|
||||
else:
|
||||
response_code = 204
|
||||
|
||||
return {
|
||||
"response": response_code,
|
||||
"method": request.method,
|
||||
"device": device_name
|
||||
}
|
||||
@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from botocore.client import ClientError
|
||||
from datetime import datetime
|
||||
from json import dumps
|
||||
import config as app_config
|
||||
import boto3
|
||||
|
||||
|
||||
bucket_name = app_config.backend["s3"]["name"]
|
||||
bucket_region = app_config.backend["s3"]["region"]
|
||||
bucket_retention = app_config.backend["s3"]["retention"]
|
||||
account_id = boto3.client("sts").get_caller_identity().get("Account")
|
||||
glue_role_name = "AWSGlue-{}".format(bucket_name)
|
||||
glue_policy_name = "AWSGlue-{}-ReadOnly".format(bucket_name)
|
||||
glue_policy_arn = "arn:aws:iam::{account_id}:policy/{policy_name}".format(account_id=account_id, policy_name=glue_policy_name)
|
||||
glue_managed_policy = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
|
||||
glue_assume_policy = {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"Service": "glue.amazonaws.com"
|
||||
},
|
||||
"Action": "sts:AssumeRole"
|
||||
}
|
||||
]
|
||||
}
|
||||
glue_custom_policy = {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": "s3:Get*",
|
||||
"Resource": "arn:aws:s3:::{}/*".format(bucket_name)
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def create_bucket():
|
||||
"""Create the s3 bucket used for capturing log data if it doesn"t exist already"""
|
||||
s3 = boto3.resource("s3")
|
||||
# s3c = boto3.client("s3")
|
||||
# s3c.put_bucket_lifecycle_configuration(
|
||||
# Bucket=bucket_name,
|
||||
# LifecycleConfiguration={
|
||||
# "Rules": [
|
||||
# {
|
||||
# "Expiration": {
|
||||
# "Days": bucket_retention,
|
||||
# "ExpiredObjectDeleteMarker": True
|
||||
# },
|
||||
# "Status": "Enabled"
|
||||
# },
|
||||
# ]
|
||||
# }
|
||||
# )
|
||||
|
||||
try:
|
||||
s3.meta.client.head_bucket(Bucket=bucket_name)
|
||||
|
||||
except ClientError:
|
||||
s3.create_bucket(
|
||||
ACL="private",
|
||||
Bucket=bucket_name,
|
||||
CreateBucketConfiguration={
|
||||
"LocationConstraint": bucket_region
|
||||
}
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
|
||||
def create_glue_iam():
|
||||
"""Create the required IAM roles for AWS Glue to assume"""
|
||||
iam = boto3.client("iam")
|
||||
res = boto3.resource("iam")
|
||||
|
||||
# Ensure IAM role exists
|
||||
try:
|
||||
res.meta.client.get_role(RoleName=glue_role_name)
|
||||
pass
|
||||
except ClientError:
|
||||
iam.create_role(
|
||||
RoleName=glue_role_name,
|
||||
AssumeRolePolicyDocument=dumps(glue_assume_policy)
|
||||
)
|
||||
|
||||
# Ensure custom IAM policy exists
|
||||
try:
|
||||
res.meta.client.get_policy(PolicyArn=glue_policy_arn)
|
||||
pass
|
||||
except ClientError:
|
||||
iam.create_policy(
|
||||
PolicyName=glue_policy_name,
|
||||
PolicyDocument=dumps(glue_custom_policy)
|
||||
)
|
||||
|
||||
# Ensure custom Glue policy is attached to role
|
||||
try:
|
||||
res.meta.client.attach_role_policy(
|
||||
RoleName=glue_role_name,
|
||||
PolicyArn=glue_policy_arn
|
||||
)
|
||||
pass
|
||||
except ClientError:
|
||||
iam.attach_role_policy(
|
||||
RoleName=glue_role_name,
|
||||
PolicyArn=glue_policy_arn
|
||||
)
|
||||
|
||||
# Ensure managed Glue policy is attached to role
|
||||
try:
|
||||
res.meta.client.attach_role_policy(
|
||||
RoleName=glue_role_name,
|
||||
PolicyArn=glue_managed_policy
|
||||
)
|
||||
pass
|
||||
except ClientError:
|
||||
iam.attach_role_policy(
|
||||
RoleName=glue_role_name,
|
||||
PolicyArn=glue_managed_policy
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
|
||||
def create_glue():
|
||||
"""Sets up the Glue database and crawler"""
|
||||
glue = boto3.client("glue")
|
||||
|
||||
# Ensure Glue database exists
|
||||
try:
|
||||
glue.get_database(Name=bucket_name)
|
||||
pass
|
||||
except ClientError:
|
||||
glue.create_database(
|
||||
DatabaseInput={
|
||||
"Name": bucket_name,
|
||||
"Description": "Owntracks SLT database"
|
||||
}
|
||||
)
|
||||
|
||||
# Ensure Glue crawler exists and run it if you create it
|
||||
try:
|
||||
glue.get_crawler(Name=bucket_name)
|
||||
pass
|
||||
except ClientError:
|
||||
glue.create_crawler(
|
||||
Name=bucket_name,
|
||||
Description="Owntracks SLT data crawler",
|
||||
Role=glue_role_name,
|
||||
DatabaseName=bucket_name,
|
||||
Targets={
|
||||
"S3Targets": [{
|
||||
"Path": "s3://{}/".format(bucket_name)
|
||||
}]
|
||||
}
|
||||
)
|
||||
glue.start_crawler(
|
||||
Name=bucket_name
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
|
||||
def setup():
|
||||
print("[+] Setting up S3 bucket resources")
|
||||
create_bucket()
|
||||
print("[+] Setting up Glue IAM resources")
|
||||
create_glue_iam()
|
||||
print("[+] Setting up Glue resources")
|
||||
create_glue()
|
||||
@ -0,0 +1,24 @@
|
||||
{
|
||||
"live": {
|
||||
"app_function": "app.generate_wsgi_app",
|
||||
"aws_region": "us-west-2",
|
||||
"profile_name": "default",
|
||||
"project_name": "MYPROJECTNAME",
|
||||
"prebuild_script": "prebuild.setup",
|
||||
"runtime": "python3.6",
|
||||
"s3_bucket": "MYZAPPABUCKETNAME",
|
||||
"keep_warm": false,
|
||||
"extra_permissions": [{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:Get*",
|
||||
"s3:Put*"
|
||||
],
|
||||
"Resource": ["arn:aws:s3:::MYLOGBUCKETNAME"]
|
||||
}],
|
||||
"tags": {
|
||||
"Project": "lzahq-ha"
|
||||
},
|
||||
"timeout_seconds": 10
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue