WhiskerRAG/template_task.yml at main · neovateai/WhiskerRAG · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Description: Task Function with SQS Integration

Globals:
  Function:
    Timeout: 300

Parameters:
  s3TempBucketName:
    Type: String
    Description: Environment marker for S3_TEMP_BUCKET_NAME
    Default: "1"

  openAIKey:
    Type: String
    Description: Environment marker for OPENAI_API_KEY
    Default: "1"

  supabaseServiceKey:
    Type: String
    Description: Environment marker for SUPABASE_SERVICE_KEY
    Default: "1"

  supabaseUrl:
    Type: String
    Description: Environment marker for SUPABASE_URL
    Default: "1"

  chunkTableName:
    Type: String
    Description: Environment marker for CHUNK_TABLE_NAME
    Default: "chunk"

  taskTableName:
    Type: String
    Description: Environment marker for TASK_TABLE_NAME
    Default: "task"

Resources:
  InputQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: !Sub '${AWS::StackName}-input-queue'
      VisibilityTimeout: 300

  InputQueuePolicy:
    Type: AWS::SQS::QueuePolicy
    Properties:
      Queues:
        - !Ref InputQueue
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal: "*"
            Action: 'sqs:SendMessage'
            Resource: !GetAtt InputQueue.Arn

  TaskFunction:
    Type: AWS::Serverless::Function
    Properties:
      PackageType: Image
      MemorySize: 512
      FunctionUrlConfig:
        AuthType: NONE
      Environment:
        Variables:
          AWS_ACCOUNT_ID: !Ref AWS::AccountId
          S3_TEMP_BUCKET_NAME: !Ref s3TempBucketName
          OPENAI_API_KEY: !Ref openAIKey
          SUPABASE_SERVICE_KEY: !Ref supabaseServiceKey
          SUPABASE_URL: !Ref supabaseUrl
          CHUNK_TABLE_NAME: !Ref chunkTableName
          TASK_TABLE_NAME: !Ref taskTableName
          GIT_PYTHON_REFRESH: quiet
          GIT_EXEC_PATH: /usr/bin
      Events:
        SQSTrigger:
          Type: SQS
          Properties:
            Queue: !GetAtt InputQueue.Arn
            BatchSize: 5          # 5 messages at a time
      Policies:
        - Statement:
            - Sid: BedrockInvokePolicy
              Effect: Allow
              Action:
                - bedrock:InvokeModelWithResponseStream
              Resource: '*'
            - Sid: SQSPolicy
              Effect: Allow
              Action:
                - sqs:SendMessage
                - sqs:ReceiveMessage
                - sqs:DeleteMessage
                - sqs:GetQueueAttributes
              Resource:
                - !GetAtt InputQueue.Arn
            - Sid: AllObjectActions
              Effect: Allow
              Action:
                - s3:PutObject
                - s3:GetObject
                - s3:DeleteObject
              Resource:
                - !Sub 'arn:aws:s3:::${s3TempBucketName}/*'
      Tracing: Active
    Metadata:
      DockerContext: lambda_task_subscriber
      Dockerfile: ../docker/Dockerfile.aws.task
      DockerTag: v1

Outputs:
  TaskFunctionUrl:
    Description: "Function URL for SQS Subscriptio function"
    Value: !GetAtt TaskFunctionUrl.FunctionUrl
  TaskFunction:
    Description: "SQS Subscription Function Lambda Function ARN"
    Value: !GetAtt TaskFunction.Arn
  InputQueueUrl:
    Description: "URL of the Input SQS Queue"
    Value: !Ref InputQueue
  InputQueueArn:
    Description: "ARN of the Input SQS Queue"
    Value: !GetAtt InputQueue.Arn