Data-Engineering-Covid19-ETL/CF-Covid_DataPipeline.yaml at master · skyprince999/Data-Engineering-Covid19-ETL · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
AWSTemplateFormatVersion: "2010-09-09"
Resources:
  BackupS3Bucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: "es-covidtweets-err"

  CovidESDomain:
    Type: AWS::Elasticsearch::Domain
    Properties:
      DomainName: "covidtweets"
      ElasticsearchClusterConfig:
        DedicatedMasterEnabled: false
        InstanceCount: 4
        ZoneAwarenessEnabled: false
        InstanceType: "t2.small.elasticsearch"
      EBSOptions:
        EBSEnabled: true
        Iops: 0
        VolumeSize: 35
        VolumeType: "gp2"
      ElasticsearchVersion: "7.7"
      SnapshotOptions:
        AutomatedSnapshotStartHour: "0"
      DomainEndpointOptions:
        EnforceHTTPS: true

      AccessPolicies:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              AWS: "*"
            Action: "es:*"
            Resource: !Sub "arn:aws:es:us-east-1:$(AWS::AccountId):domain/covidtweets/*"
            Condition:
              IpAddress:
                aws:SourceIP: "*"
      AdvancedOptions:
        rest.action.multi.allow_explicit_index: "true"


  COVIDSTREAM:
    Type: AWS::KinesisFirehose::DeliveryStream
    Properties:
        DeliveryStreamName: "covid-stream"
        DeliveryStreamType: "DirectPut"
        ElasticsearchDestinationConfiguration:
            BufferingHints:
                IntervalInSeconds: 300
                SizeInMBs: 5
            DomainARN: !GetAtt CovidESDomain.Arn
            IndexName: "covidtweets"
            RoleARN: <Replace with RoleARN>
            S3BackupMode: "FailedDocumentsOnly"
            S3Configuration:
                BucketARN: !GetAtt BackupS3Bucket.Arn
                BufferingHints:
                  IntervalInSeconds: 300
                  SizeInMBs: 5
                RoleARN: <Replace with RoleARN>