init

.gitignore

@@ -0,0 +1,109 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+config.py
+.terraform
+*.tfstate
+*.tfstate.backup

README.md

@@ -0,0 +1,69 @@
+# flask-mdi
+Flask based mobile device ingestion application
+
+# Setup
+
+All of the below assumes you have an active/working default AWSCLI profile configured on your machine. Running `aws sts get-caller-identity` in a fresh terminal will give you an indication.
+
+First deploy a Kinesis Stream:
+
+```bash
+aws kinesis create-stream \
+    --stream-name mdi-test \
+    --shard-count 1 \
+    --region us-east-1
+```
+
+Then configure the app:
+
+```bash
+cp mdi/config.{example.py,py}
+vim mdi/config.py
+# Modify the config file as needed
+```
+
+Install dependencies and run it:
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install .
+export FLASK_APP=mdi/app.py
+export FLASK_SECRETS=config.py
+export FLASK_ENV=development
+export FLASK_DEBUG=1
+flask run
+```
+
+# Testing
+
+Open another tab/window in your terminal and `curl` the Flask app - there's only a single endpoint, `/`
+
+```bash
+# GET returns simple JSON body and 200, no data is stored
+curl localhost:5000/ -X GET
+{
+  "app": "mdi",
+  "message": "Welcome",
+  "status": "success",
+  "version": "0.0.1"
+}
+
+# Empty payload on POST returns error, 400
+curl localhost:5000/ -X POST
+{
+  "app": "mdi",
+  "message": "No JSON data provided",
+  "status": "error",
+  "version": "0.0.1"
+}
+
+# Any payload on POST returns 202
+curl localhost:5000/ -X POST -d '{"hello": "world"}' -H "Content-Type: application/json"
+{
+  "app": "mdi",
+  "message": "Data stored for processing",
+  "status": "success",
+  "version": "0.0.1"
+}
+```

mdi/__init__.py

@@ -0,0 +1 @@
+from mdi._version import __version__

mdi/_version.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

mdi/app.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+from json import dumps as json_dumps
+from socket import gethostname
+from boto3 import client as boto3_client
+from flask import Flask, jsonify, request, make_response
+from datetime import datetime
+from mdi import __version__
+from mdi import config
+
+
+# Setup Flask application
+app = Flask(__name__)
+app.config.from_envvar('FLASK_SECRETS')
+
+@app.route('/', methods=["GET", "POST"])
+def index():
+    now = datetime.now()
+
+    if request.method == "POST":
+
+        if not request.data:
+            return render_result(400, "error", "No JSON data provided")
+
+        json_data = request.get_json()
+        json_data["timestamp"] = int(now.timestamp())
+        json_data["datestamp"] = now.strftime("%Y-%m-%d %H:%M:%S")
+        if not json_data.get("device_name"):
+            json_data["device_name"] = gethostname()
+
+        print(f"Using payload: {json_data}")
+
+        publish_kinesis(config.STREAM_NAME, config.REGION, json_data, json_data["device_name"])
+        write_fs(config.DATA_PATH, json_data["timestamp"], json_data)
+
+        return render_result(202, "success", "Data stored for processing")
+    else:
+        return render_result(200, "success", "Welcome")
+
+
+def render_result(code, status, message):
+    response = make_response(jsonify({
+        "status": status,
+        "message": message,
+        "app": config.APP_NAME,
+        "version": __version__
+    }), code)
+    return response
+
+def write_fs(path, name, data):
+    fs_path = f"{path}/{name}.json"
+    print(f"Writing payload to {fs_path}")
+    with open(fs_path, "w") as f:
+        f.write(json_dumps(data))
+    print("Payload written to filesystem")
+
+def publish_kinesis(stream, region, data, pk):
+    print("Publishing payload to Kinesis")
+    kinesis = boto3_client("kinesis", region_name=region)
+    res = kinesis.put_record(
+        StreamName=stream,
+        Data=json_dumps(data),
+        PartitionKey=pk
+    )
+    print(f"Kinesis response: {res}")
+
+if __name__ == "__main__":
+    app.run()

mdi/config.example.py

@@ -0,0 +1,4 @@
+APP_NAME = "mdi"
+STREAM_NAME = "mdi-test"
+DATA_PATH = "/tmp"
+REGION = "us-east-1"

setup.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+from setuptools import setup, find_packages
+from mdi import __version__
+
+NAME = "mdi"
+DESCRIPTION = "Flask application for ingesting data from mobile devices"
+URL = "https://github.com/lalanza808/flask-mdi.git"
+EMAIL = "lance@lzahq.tech"
+AUTHOR = "Lance Allen"
+REQUIRES_PYTHON = ">=3.6.0"
+VERSION = __version__
+REQUIRED = [
+    "boto3==1.9.74",
+    "Flask==1.0.2"
+]
+EXTRAS = {}
+TESTS = []
+SETUP = []
+
+setup(
+    name=NAME,
+    version=VERSION,
+    description=DESCRIPTION,
+    author=AUTHOR,
+    author_email=EMAIL,
+    include_package_data=True,
+    extras_require=EXTRAS,
+    install_requires=REQUIRED,
+    setup_requires=SETUP,
+    tests_require=TESTS,
+    packages=find_packages(exclude=['ez_setup']),
+    zip_safe=False
+)

util/dev

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+source .venv/bin/activate
+export FLASK_APP=mdi/app.py
+export FLASK_SECRETS=config.py
+export FLASK_ENV=development
+export FLASK_DEBUG=1
+flask run