1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
import { S3Client } from "@aws-sdk/client-s3";
import { SQSClient } from "@aws-sdk/client-sqs";
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
import { Args, args } from "@/consumer/args";
import { DynamoWidgetDAO, S3WidgetDAO } from "@/consumer/dao";
import { S3WidgetKeyGenerator } from "@/consumer/dao/keys";
import { ConsoleTracingLogger } from "@/shared/log";
import { ConsumerRequestProcessor, WidgetRequestProcessor } from "@/consumer/processor";
import {
S3WidgetRequestRetriever,
WidgetRequestRetriever,
SQSWidgetRequestRetriever,
} from "@/consumer/retrievers";
import { MainLoop } from "@/consumer/Loop";
const main = async (args: Args) => {
const mainLogger = new ConsoleTracingLogger("main");
const s3Client = new S3Client();
const sqsClient = new SQSClient();
const dynamoClient = new DynamoDBClient();
const retrievers: WidgetRequestRetriever[] = [];
if (!args.sourcebucket && !args.sourcequeue) {
throw new Error("Must specify source to poll.");
}
if (args.sourcebucket) {
mainLogger.info(
`constructing s3 retriever for Bucket=(${args.sourcebucket})`,
);
const retriever = new S3WidgetRequestRetriever(
args.sourcebucket,
s3Client,
mainLogger.createChild("S3WidgetRequestRetriever"),
);
retrievers.push(retriever);
}
if (args.sourcequeue) {
mainLogger.info(
`constructing sqs retriever for Bucket=(${args.sourcequeue})`,
);
const retriever = new SQSWidgetRequestRetriever(
args.sourcequeue,
sqsClient,
mainLogger.createChild("SQSWidgetRequestRetriever"),
);
retrievers.push(retriever);
}
if (!args.storagebucket && !args.storagetable) {
throw new Error("Must specify a target data store for Widgets");
}
const processors: WidgetRequestProcessor[] = [];
if (args.storagebucket) {
if (!args.storageprefix) {
throw new Error("must specify widget prefix location for storage bucket");
}
mainLogger.info(
`constructing s3 processor for Bucket=(${args.storagebucket})`,
);
const s3KeyGenerator = new S3WidgetKeyGenerator().withPrefix(
args.storageprefix,
);
const s3WidgetDao = new S3WidgetDAO(
args.storagebucket,
s3Client,
s3KeyGenerator,
mainLogger.createChild("S3WidgetDAO"),
);
const s3RequestProcessor = new ConsumerRequestProcessor(
s3WidgetDao,
mainLogger.createChild("ConsumerRequestProcessor => (S3)"),
);
processors.push(s3RequestProcessor);
}
if (args.storagetable) {
mainLogger.info(
`constructing dynamo processor for Table=(${args.storagetable})`,
);
const dynamoWidgetDao = new DynamoWidgetDAO(
args.storagetable,
dynamoClient,
mainLogger.createChild("DynamoWidgetDAO"),
);
const dynamoRequestProcessor = new ConsumerRequestProcessor(
dynamoWidgetDao,
mainLogger.createChild("ConsumerRequestProcessor => (Dynamo)"),
);
processors.push(dynamoRequestProcessor);
}
const maxPollAttempts = 15;
await MainLoop(retrievers, processors, maxPollAttempts, 200, mainLogger);
mainLogger.info("EXITING!");
};
main(args);
|
