|
|
|
@ -10,57 +10,45 @@ import boto3
|
|
|
|
|
bucket_name = app_config.backend["s3"]["name"]
|
|
|
|
|
bucket_region = app_config.backend["s3"]["region"]
|
|
|
|
|
bucket_retention = app_config.backend["s3"]["retention"]
|
|
|
|
|
account_id = boto3.client("sts").get_caller_identity().get("Account")
|
|
|
|
|
glue_role_name = "AWSGlue-{}".format(bucket_name)
|
|
|
|
|
glue_policy_name = "AWSGlue-{}-ReadOnly".format(bucket_name)
|
|
|
|
|
glue_policy_arn = "arn:aws:iam::{account_id}:policy/{policy_name}".format(account_id=account_id, policy_name=glue_policy_name)
|
|
|
|
|
glue_managed_policy = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
|
|
|
|
|
glue_assume_policy = {
|
|
|
|
|
"Version": "2012-10-17",
|
|
|
|
|
"Statement": [
|
|
|
|
|
{
|
|
|
|
|
"Effect": "Allow",
|
|
|
|
|
"Principal": {
|
|
|
|
|
"Service": "glue.amazonaws.com"
|
|
|
|
|
},
|
|
|
|
|
"Action": "sts:AssumeRole"
|
|
|
|
|
}
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
glue_custom_policy = {
|
|
|
|
|
"Version": "2012-10-17",
|
|
|
|
|
"Statement": [
|
|
|
|
|
{
|
|
|
|
|
"Effect": "Allow",
|
|
|
|
|
"Action": "s3:Get*",
|
|
|
|
|
"Resource": "arn:aws:s3:::{}/*".format(bucket_name)
|
|
|
|
|
}
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
# account_id = boto3.client("sts").get_caller_identity().get("Account")
|
|
|
|
|
# glue_role_name = "AWSGlue-{}".format(bucket_name)
|
|
|
|
|
# glue_policy_name = "AWSGlue-{}-ReadOnly".format(bucket_name)
|
|
|
|
|
# glue_policy_arn = "arn:aws:iam::{account_id}:policy/{policy_name}".format(
|
|
|
|
|
# account_id=account_id,
|
|
|
|
|
# policy_name=glue_policy_name
|
|
|
|
|
# )
|
|
|
|
|
# glue_managed_policy = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
|
|
|
|
|
# glue_assume_policy = {
|
|
|
|
|
# "Version": "2012-10-17",
|
|
|
|
|
# "Statement": [
|
|
|
|
|
# {
|
|
|
|
|
# "Effect": "Allow",
|
|
|
|
|
# "Principal": {
|
|
|
|
|
# "Service": "glue.amazonaws.com"
|
|
|
|
|
# },
|
|
|
|
|
# "Action": "sts:AssumeRole"
|
|
|
|
|
# }
|
|
|
|
|
# ]
|
|
|
|
|
# }
|
|
|
|
|
# glue_custom_policy = {
|
|
|
|
|
# "Version": "2012-10-17",
|
|
|
|
|
# "Statement": [
|
|
|
|
|
# {
|
|
|
|
|
# "Effect": "Allow",
|
|
|
|
|
# "Action": "s3:Get*",
|
|
|
|
|
# "Resource": "arn:aws:s3:::{}/*".format(bucket_name)
|
|
|
|
|
# }
|
|
|
|
|
# ]
|
|
|
|
|
# }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_bucket():
|
|
|
|
|
"""Create the s3 bucket used for capturing log data if it doesn"t exist already"""
|
|
|
|
|
s3 = boto3.resource("s3")
|
|
|
|
|
# s3c = boto3.client("s3")
|
|
|
|
|
# s3c.put_bucket_lifecycle_configuration(
|
|
|
|
|
# Bucket=bucket_name,
|
|
|
|
|
# LifecycleConfiguration={
|
|
|
|
|
# "Rules": [
|
|
|
|
|
# {
|
|
|
|
|
# "Expiration": {
|
|
|
|
|
# "Days": bucket_retention,
|
|
|
|
|
# "ExpiredObjectDeleteMarker": True
|
|
|
|
|
# },
|
|
|
|
|
# "Status": "Enabled"
|
|
|
|
|
# },
|
|
|
|
|
# ]
|
|
|
|
|
# }
|
|
|
|
|
# )
|
|
|
|
|
s3client = boto3.client("s3")
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
s3.meta.client.head_bucket(Bucket=bucket_name)
|
|
|
|
|
|
|
|
|
|
except ClientError:
|
|
|
|
|
s3.create_bucket(
|
|
|
|
|
ACL="private",
|
|
|
|
@ -69,6 +57,23 @@ def create_bucket():
|
|
|
|
|
"LocationConstraint": bucket_region
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
s3client.put_bucket_lifecycle_configuration(
|
|
|
|
|
Bucket=bucket_name,
|
|
|
|
|
LifecycleConfiguration={
|
|
|
|
|
"Rules": [
|
|
|
|
|
{
|
|
|
|
|
"Expiration": {
|
|
|
|
|
"Days": bucket_retention
|
|
|
|
|
},
|
|
|
|
|
"Filter": {
|
|
|
|
|
"Prefix": ""
|
|
|
|
|
},
|
|
|
|
|
"ID": "{}-day-retention".format(bucket_retention),
|
|
|
|
|
"Status": "Enabled"
|
|
|
|
|
}
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
@ -169,7 +174,7 @@ def create_glue():
|
|
|
|
|
def setup():
|
|
|
|
|
print("[+] Setting up S3 bucket resources")
|
|
|
|
|
create_bucket()
|
|
|
|
|
print("[+] Setting up Glue IAM resources")
|
|
|
|
|
create_glue_iam()
|
|
|
|
|
print("[+] Setting up Glue resources")
|
|
|
|
|
create_glue()
|
|
|
|
|
# print("[+] Setting up Glue IAM resources")
|
|
|
|
|
# create_glue_iam()
|
|
|
|
|
# print("[+] Setting up Glue resources")
|
|
|
|
|
# create_glue()
|
|
|
|
|