You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
181 lines
4.8 KiB
Python
181 lines
4.8 KiB
Python
#!/usr/bin/env python
|
|
|
|
from botocore.client import ClientError
|
|
from datetime import datetime
|
|
from json import dumps
|
|
import config as app_config
|
|
import boto3
|
|
|
|
|
|
bucket_name = app_config.backend["s3"]["name"]
|
|
bucket_region = app_config.backend["s3"]["region"]
|
|
bucket_retention = app_config.backend["s3"]["retention"]
|
|
# account_id = boto3.client("sts").get_caller_identity().get("Account")
|
|
# glue_role_name = "AWSGlue-{}".format(bucket_name)
|
|
# glue_policy_name = "AWSGlue-{}-ReadOnly".format(bucket_name)
|
|
# glue_policy_arn = "arn:aws:iam::{account_id}:policy/{policy_name}".format(
|
|
# account_id=account_id,
|
|
# policy_name=glue_policy_name
|
|
# )
|
|
# glue_managed_policy = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
|
|
# glue_assume_policy = {
|
|
# "Version": "2012-10-17",
|
|
# "Statement": [
|
|
# {
|
|
# "Effect": "Allow",
|
|
# "Principal": {
|
|
# "Service": "glue.amazonaws.com"
|
|
# },
|
|
# "Action": "sts:AssumeRole"
|
|
# }
|
|
# ]
|
|
# }
|
|
# glue_custom_policy = {
|
|
# "Version": "2012-10-17",
|
|
# "Statement": [
|
|
# {
|
|
# "Effect": "Allow",
|
|
# "Action": "s3:Get*",
|
|
# "Resource": "arn:aws:s3:::{}/*".format(bucket_name)
|
|
# }
|
|
# ]
|
|
# }
|
|
|
|
|
|
def create_bucket():
|
|
"""Create the s3 bucket used for capturing log data if it doesn"t exist already"""
|
|
s3 = boto3.resource("s3")
|
|
s3client = boto3.client("s3")
|
|
|
|
try:
|
|
s3.meta.client.head_bucket(Bucket=bucket_name)
|
|
except ClientError:
|
|
s3.create_bucket(
|
|
ACL="private",
|
|
Bucket=bucket_name,
|
|
CreateBucketConfiguration={
|
|
"LocationConstraint": bucket_region
|
|
}
|
|
)
|
|
s3client.put_bucket_lifecycle_configuration(
|
|
Bucket=bucket_name,
|
|
LifecycleConfiguration={
|
|
"Rules": [
|
|
{
|
|
"Expiration": {
|
|
"Days": bucket_retention
|
|
},
|
|
"Filter": {
|
|
"Prefix": ""
|
|
},
|
|
"ID": "{}-day-retention".format(bucket_retention),
|
|
"Status": "Enabled"
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
return
|
|
|
|
|
|
def create_glue_iam():
|
|
"""Create the required IAM roles for AWS Glue to assume"""
|
|
iam = boto3.client("iam")
|
|
res = boto3.resource("iam")
|
|
|
|
# Ensure IAM role exists
|
|
try:
|
|
res.meta.client.get_role(RoleName=glue_role_name)
|
|
pass
|
|
except ClientError:
|
|
iam.create_role(
|
|
RoleName=glue_role_name,
|
|
AssumeRolePolicyDocument=dumps(glue_assume_policy)
|
|
)
|
|
|
|
# Ensure custom IAM policy exists
|
|
try:
|
|
res.meta.client.get_policy(PolicyArn=glue_policy_arn)
|
|
pass
|
|
except ClientError:
|
|
iam.create_policy(
|
|
PolicyName=glue_policy_name,
|
|
PolicyDocument=dumps(glue_custom_policy)
|
|
)
|
|
|
|
# Ensure custom Glue policy is attached to role
|
|
try:
|
|
res.meta.client.attach_role_policy(
|
|
RoleName=glue_role_name,
|
|
PolicyArn=glue_policy_arn
|
|
)
|
|
pass
|
|
except ClientError:
|
|
iam.attach_role_policy(
|
|
RoleName=glue_role_name,
|
|
PolicyArn=glue_policy_arn
|
|
)
|
|
|
|
# Ensure managed Glue policy is attached to role
|
|
try:
|
|
res.meta.client.attach_role_policy(
|
|
RoleName=glue_role_name,
|
|
PolicyArn=glue_managed_policy
|
|
)
|
|
pass
|
|
except ClientError:
|
|
iam.attach_role_policy(
|
|
RoleName=glue_role_name,
|
|
PolicyArn=glue_managed_policy
|
|
)
|
|
|
|
return
|
|
|
|
|
|
def create_glue():
|
|
"""Sets up the Glue database and crawler"""
|
|
glue = boto3.client("glue")
|
|
|
|
# Ensure Glue database exists
|
|
try:
|
|
glue.get_database(Name=bucket_name)
|
|
pass
|
|
except ClientError:
|
|
glue.create_database(
|
|
DatabaseInput={
|
|
"Name": bucket_name,
|
|
"Description": "Owntracks SLT database"
|
|
}
|
|
)
|
|
|
|
# Ensure Glue crawler exists and run it if you create it
|
|
try:
|
|
glue.get_crawler(Name=bucket_name)
|
|
pass
|
|
except ClientError:
|
|
glue.create_crawler(
|
|
Name=bucket_name,
|
|
Description="Owntracks SLT data crawler",
|
|
Role=glue_role_name,
|
|
DatabaseName=bucket_name,
|
|
Targets={
|
|
"S3Targets": [{
|
|
"Path": "s3://{}/".format(bucket_name)
|
|
}]
|
|
}
|
|
)
|
|
glue.start_crawler(
|
|
Name=bucket_name
|
|
)
|
|
|
|
return
|
|
|
|
|
|
def setup():
|
|
print("[+] Setting up S3 bucket resources")
|
|
create_bucket()
|
|
# print("[+] Setting up Glue IAM resources")
|
|
# create_glue_iam()
|
|
# print("[+] Setting up Glue resources")
|
|
# create_glue()
|