
---
AWSTemplateFormatVersion: '2010-09-09'
# Sample CFN YAML to demonstrate creating a s3 event crawler
#
# Parameters section contains names that are substituted in the Resources section
# These parameters are the names the resources created in the Data Catalog
Parameters:                                                                                                       
  CFNCrawlerName:
    Type: String
    Default: cfn-s3-event-crawler
  CFNCrawlerScheduleCronExpression:
    Type: String
    Default: cron(0 0 * * ? *)
  CFNDatabaseName:
    Type: String
    Default: cfn-s3-event-database
  CFNTablePrefixName:
    Type: String
    Default: cfn_sample_
  CFNS3BucketName:
    Type: String
    Default: customer-data-us-west-2
  CFNS3FolderName:
    Type: String
    Default: test
  CFNSNSTopicName:
    Type: String
    Default: cfn-s3-event-topic
  CFNSQSQueueName:
    Type: String
    Default: cfn-s3-event-queue
#
# Resources section defines metadata for the Crawler
Resources:
#Create IAM Role assumed by the crawler. 
  # Create an SNS topic
  CFNSNSTopic:
    Type: AWS::SNS::Topic
    Properties:
      TopicName: !Ref CFNSNSTopicName

  SNSTopicPolicy:
    Type: AWS::SNS::TopicPolicy
    Properties:
      PolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal: 
              Service: s3.amazonaws.com
            Action: sns:Publish
            Resource: !Ref CFNSNSTopic
            Condition:
              ArnEquals:
                aws:SourceArn: !Join ["", ['arn:aws:s3:::', !Ref CFNS3BucketName]]
      Topics:
        - !Ref CFNSNSTopic

 # Create an SQS queue
  CFNSQSQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: !Ref CFNSQSQueueName

  SQSQueuePolicy:
    Type: AWS::SQS::QueuePolicy
    Properties:
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Sid: Allow-SNS-SendMessage
            Effect: Allow
            Principal: "*"
            Action: sqs:SendMessage
            Resource: !Ref CFNSQSQueue
            Condition:
              ArnEquals:
                aws:SourceArn: !Ref CFNSNSTopic
      Queues:
        - !Ref CFNSQSQueue

  SnsSubscription:
    Type: AWS::SNS::Subscription
    Properties:
      Protocol: sqs
      Endpoint: !GetAtt CFNSQSQueue.Arn
      TopicArn: !Ref CFNSNSTopic

  CFNRoleCrawler:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          -
            Effect: "Allow"
            Principal:
              Service:
                - "glue.amazonaws.com"
            Action:
              - "sts:AssumeRole"
      Path: "/"
      Policies:
        -
          PolicyName: "cfn-s3-event"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              -
                Effect: "Allow"
                Action: 
                - s3:GetObject
                - s3:PutObject
                Resource: !Join ["", ['arn:aws:s3:::', !Ref CFNS3BucketName, '*']]
              -
                Effect: "Allow"
                Action: 
                - sqs:*
                Resource: !GetAtt CFNSQSQueue.Arn
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole 
 
 # Create event notifications for given s3 bucket
  S3CustomResource:
    Type: 'Custom::S3CustomResource'
    Properties:
      ServiceToken: !GetAtt CustomResourceLambdaFunction.Arn
      SNSTopicArn: !Ref CFNSNSTopic
      Bucket: !Ref CFNS3BucketName
      FolderName: !Ref CFNS3FolderName

  
  LambdaIAMRole:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: /
      Policies:
        - PolicyName: root
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 's3:GetBucketNotification'
                  - 's3:PutBucketNotification'
                Resource: !Sub 'arn:aws:s3:::${CFNS3BucketName}'
              - Effect: Allow
                Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:PutLogEvents'
                Resource: 'arn:aws:logs:*:*:*'  
  
  CustomResourceLambdaFunction:
    Type: 'AWS::Lambda::Function'
    Properties:
      Handler: index.lambda_handler
      Role: !GetAtt LambdaIAMRole.Arn
      Code:
        ZipFile: |

            from __future__ import print_function
            import json
            import boto3
            import cfnresponse
            
            SUCCESS = "SUCCESS"
            FAILED = "FAILED"
            
            print('Loading function')
            s3 = boto3.resource('s3')
            
            def lambda_handler(event, context):
                print("Received event: " + json.dumps(event, indent=2))
                responseData={}
                # Retrieve parameters
                
                try:
                    if event['RequestType'] == 'Delete':
                        print("Request Type:",event['RequestType'])
                        Bucket=event['ResourceProperties']['Bucket']
                        delete_notification(Bucket)
                        print("Sending response to custom resource after Delete")
                    elif event['RequestType'] == 'Create' or event['RequestType'] == 'Update':
                        print("Request Type:",event['RequestType'])
                        SNSTopicArn=event['ResourceProperties']['SNSTopicArn']
                        Bucket=event['ResourceProperties']['Bucket']
                        FolderName=event['ResourceProperties']['FolderName']
                        add_notification(SNSTopicArn, Bucket, FolderName)
                        responseData={'Bucket':Bucket}
                        print("Sending response to custom resource")
                    responseStatus = 'SUCCESS'
                except Exception as e:
                    print('Failed to process:', e)
                    responseStatus = 'FAILED'
                    responseData = {'Failure': 'Something bad happened.'}
                cfnresponse.send(event, context, responseStatus, responseData)

            def add_notification(SNSTopicArn, Bucket, FolerName):
                bucket_notification = s3.BucketNotification(Bucket)
                response = bucket_notification.put(
                  NotificationConfiguration={
                    'TopicConfigurations': [
                      {
                          'TopicArn': SNSTopicArn,
                          'Events': [
                              's3:ObjectCreated:*',
                              's3:ObjectRemoved:*'
                          ],
                          'Filter' :  {'Key': {'FilterRules': [{'Name': 'prefix', 'Value': FolerName}]}}
                      }
                    ]
                  }
                )
                print("Put request completed....")
              
            def delete_notification(Bucket):
                bucket_notification = s3.BucketNotification(Bucket)
                response = bucket_notification.put(
                    NotificationConfiguration={}
                )
                print("Delete request completed....")
      Runtime: python3.9
      Timeout: 50
    
 # Create a database to contain tables created by the crawler
  CFNDatabase:
    Type: AWS::Glue::Database
    Properties:
      CatalogId: !Ref AWS::AccountId
      DatabaseInput:
        Name: !Ref CFNDatabaseName
        Description: "AWS Glue container to hold metadata tables for the flights crawler"
 #Create a crawler to crawl the flights data on a public S3 bucket
  CFNCrawler:
    Type: AWS::Glue::Crawler
    Properties:
      Name: !Ref CFNCrawlerName
      Role: !GetAtt CFNRoleCrawler.Arn
      #Classifiers: none, use the default classifier
      Description: AWS Glue crawler to crawl the specified s3 path
      #Schedule: none, use default run-on-demand
      DatabaseName: !Ref CFNDatabaseName
      RecrawlPolicy:
        RecrawlBehavior: "CRAWL_EVENT_MODE"
      Targets:
        S3Targets:
          # Public S3 bucket with the flights data
          - Path: !Join [ "/", ["s3:/", !Ref CFNS3BucketName, !Ref CFNS3FolderName] ]
            EventQueueArn: !GetAtt CFNSQSQueue.Arn
      TablePrefix: !Ref CFNTablePrefixName
      Schedule:
        ScheduleExpression: !Ref CFNCrawlerScheduleCronExpression
      SchemaChangePolicy:
        UpdateBehavior: "UPDATE_IN_DATABASE"
        DeleteBehavior: "LOG"
      Configuration: "{\"Version\":1.0,\"CrawlerOutput\":{\"Partitions\":{\"AddOrUpdateBehavior\":\"InheritFromTable\"},\"Tables\":{\"AddOrUpdateBehavior\":\"MergeNewColumns\"}}}"
