Skip to content

Commit b846c41

Browse files
dohakidohaki
authored
feat: split modes dynamic modules (#87)
1 parent 5e18b13 commit b846c41

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+517
-261
lines changed

‎.env.sample

+32-16
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
########################
2+
# SHARED ENV VARIABLES #
3+
########################
14
PORT=3001
25
DB_HOST=postgres
36
DB_PORT=5432
@@ -7,33 +10,46 @@ DB_DATABASE_NAME=scraperapi
710
REDIS_HOST=redis
811
REDIS_PORT=6379
912
REDIS_PASSWORD=password
13+
JWT_SECRET=secret
14+
# MerkleDistributor overrides
15+
MERKLE_DISTRIBUTOR_CHAIN_ID=
16+
MERKLE_DISTRIBUTOR_ADDRESS=
17+
# set the list of operation modes ("normal", "test", "scraper")
18+
RUN_MODES=normal
19+
20+
########################
21+
# NORMAL ENV VARIABLES #
22+
########################
23+
DISTRIBUTOR_PROOFS_CACHE_SECONDS_DURATION=
24+
REFERRALS_SUMMARY_CACHE_SECONDS_DURATION=
25+
# the Discord app's credentials obtained from https://discord.com/developers/applications.
26+
DISCORD_CLIENT_ID=clientId
27+
DISCORD_CLIENT_SECRET=clientSecret
28+
# the url accessed after the Discord authorization processed is fulfilled
29+
DISCORD_REDIRECT_URI=http://localhost
30+
AMPLITUDE_API_KEY=
31+
32+
#########################
33+
# SCRAPER ENV VARIABLES #
34+
#########################
35+
1036
WEB3_NODE_URL_1=https://mainnet.infura.io/v3/
1137
WEB3_NODE_URL_10=https://optimism-mainnet.infura.io/v3/
1238
WEB3_NODE_URL_288=https://boba-mainnet.gateway.pokt.network/v1/lb/
1339
WEB3_NODE_URL_42161=https://arbitrum-mainnet.infura.io/v3/
1440
WEB3_NODE_URL_137=https://polygon-mainnet.infura.io/v3/
1541
WEB3_NODE_URL_5=https://goerli.infura.io/v3/
42+
# the timestamp after which the referral address is extracted using the delimiter
1643
REFERRAL_DELIMITER_START_TIMESTAMP=1657290720
44+
# enable fetching SpokePool events from the contracts
1745
ENABLE_SPOKE_POOLS_EVENTS_PROCESSING=false
46+
# enable fetching MerkleDistributor events from the contracts
47+
ENABLE_MERKLE_DISTRIBUTOR_EVENTS_PROCESSING=false
48+
# enable the refresh of the referrals materialized view.
1849
ENABLE_REFERRALS_MATERIALIZED_VIEW_REFRESH=false
1950
# specify the strategy used for updating sticky referrals. Valid options: queue | cron | disable
2051
STICKY_REFERRAL_ADDRESSES_MECHANISM=queue
21-
DISABLE_CRONS=true
2252
# Following distances in blocks to guarantee finality on each chain. Format should be a map chainId -> blocks.
2353
# E.g. { "137": 100 }
2454
FOLLOWING_DISTANCES=
25-
JWT_SECRET=secret
26-
# DISCORD_CLIENT_ID and DISCORD_CLIENT_SECRET are the credentials of the Discord app and they are obtained from https://discord.com/developers/applications.
27-
DISCORD_CLIENT_ID=clientId
28-
DISCORD_CLIENT_SECRET=clientSecret
29-
# the url accessed after the Discord authorization processed is fulfilled
30-
DISCORD_REDIRECT_URI=http://localhost
31-
32-
# MerkleDistributor overrides
33-
MERKLE_DISTRIBUTOR_CHAIN_ID=
34-
MERKLE_DISTRIBUTOR_ADDRESS=
35-
ENABLE_MERKLE_DISTRIBUTOR_EVENTS_PROCESSING=false
36-
37-
DISTRIBUTOR_PROOFS_CACHE_SECONDS_DURATION=0
38-
REFERRALS_SUMMARY_CACHE_SECONDS_DURATION=0
39-
AMPLITUDE_API_KEY=
55+
DISABLE_CRONS=true

‎.github/workflows/lint-build-test.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ jobs:
4343
DISCORD_REDIRECT_URI: http://localhost
4444
DISTRIBUTOR_PROOFS_CACHE_SECONDS_DURATION: 0
4545
REFERRALS_SUMMARY_CACHE_SECONDS_DURATION: 0
46+
RUN_MODES: normal,test,scraper
4647
steps:
4748
- uses: actions/checkout@v3
4849
- uses: ./.github/actions/setup

‎README.md

+51
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,54 @@ git rebase master
3636
- `feature-branch` -> `master`: The branch will be deployed on the production environment for testing purposes. Most of the time, all feature branches will be merged into `stage` branch before the `master` branch.
3737
3838
3. Update the `stage` branch with the latest master whenever is needed
39+
40+
## Run modes
41+
42+
The application has the ability the load modules and dependencies depending on predefined running modes that can be one or multiple of: scraper, normal, test. Use the RUN_MODES env variable to configure the behaviour of the application:
43+
44+
```
45+
RUN_MODES=normal,test
46+
```
47+
48+
In order to configure a module to behave differently depending on the running mode, use the static `forRoot` method like in the example below:
49+
50+
```ts
51+
@Module({})
52+
export class ExampleModule {
53+
static forRoot(moduleOptions: ModuleOptions): DynamicModule {
54+
let module: DynamicModule = { module: ExampleModule, providers: [], controllers: [], imports: [], exports: [] };
55+
56+
if (moduleOptions.runModes.includes(RunMode.Normal)) {
57+
module = {
58+
...module,
59+
controllers: [...module.controllers, ...],
60+
providers: [...module.providers, ...],
61+
imports: [...module.imports, ...],
62+
exports: [...module.exports, ...],
63+
};
64+
}
65+
66+
if (moduleOptions.runModes.includes(RunMode.Scraper)) {
67+
module = {
68+
...module,
69+
controllers: [...module.controllers, ...],
70+
providers: [...module.providers, ...],
71+
imports: [...module.imports, ...],
72+
exports: [...module.exports, ...],
73+
};
74+
}
75+
76+
if (moduleOptions.runModes.includes(RunMode.Test)) {
77+
module = {
78+
...module,
79+
controllers: [...module.controllers, ...],
80+
providers: [...module.providers, ...],
81+
imports: [...module.imports, ...],
82+
exports: [...module.exports, ...],
83+
};
84+
}
85+
86+
return module;
87+
}
88+
}
89+
```

‎src/app.module.ts

+45-32
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import { BullModule } from "@nestjs/bull";
2-
import { CacheModule, MiddlewareConsumer, Module } from "@nestjs/common";
2+
import { CacheModule, MiddlewareConsumer, Module, DynamicModule } from "@nestjs/common";
33
import { ConfigModule } from "@nestjs/config";
44
import { ScheduleModule } from "@nestjs/schedule";
55
import { TypeOrmModule } from "@nestjs/typeorm";
66
import { LogsMiddleware } from "./logging.interceptor";
7-
import configuration from "./modules/configuration";
7+
import configuration, { configValues } from "./modules/configuration";
88
import { DatabaseModule } from "./modules/database/database.module";
99
import { TypeOrmDefaultConfigService } from "./modules/database/database.providers";
1010
import { HealthModule } from "./modules/health/health.module";
@@ -18,38 +18,51 @@ import { DepositModule } from "./modules/deposit/module";
1818
import { AuthModule } from "./modules/auth/auth.module";
1919
import { UserModule } from "./modules/user/module";
2020
import { AirdropModule } from "./modules/airdrop/module";
21+
import { ModuleOptions, RunMode } from "./dynamic-module";
2122

22-
@Module({
23-
imports: [
24-
ConfigModule.forRoot({
25-
ignoreEnvFile: false,
26-
ignoreEnvVars: false,
27-
isGlobal: true,
28-
expandVariables: true,
29-
load: [configuration],
30-
}),
31-
TypeOrmModule.forRootAsync({
32-
imports: [DatabaseModule],
33-
useExisting: TypeOrmDefaultConfigService,
34-
}),
35-
HealthModule,
36-
Web3Module,
37-
ScraperModule,
38-
BullModule.forRootAsync({
39-
imports: [MessagingModule],
40-
useExisting: BullConfigService,
41-
}),
42-
ReferralModule,
43-
MarketPriceModule,
44-
ScheduleModule.forRoot(),
45-
DepositModule,
46-
AuthModule,
47-
UserModule,
48-
AirdropModule,
49-
CacheModule.register({ isGlobal: true }),
50-
],
51-
})
23+
@Module({})
5224
export class AppModule {
25+
static forRoot(moduleOptions: ModuleOptions): DynamicModule {
26+
const imports = [
27+
ConfigModule.forRoot({
28+
ignoreEnvFile: false,
29+
ignoreEnvVars: false,
30+
isGlobal: true,
31+
expandVariables: true,
32+
load: [configuration],
33+
}),
34+
TypeOrmModule.forRootAsync({
35+
imports: [DatabaseModule],
36+
useExisting: TypeOrmDefaultConfigService,
37+
}),
38+
HealthModule,
39+
Web3Module,
40+
ReferralModule.forRoot(moduleOptions),
41+
MarketPriceModule.forRoot(moduleOptions),
42+
ScheduleModule.forRoot(),
43+
DepositModule.forRoot(moduleOptions),
44+
AuthModule.forRoot(moduleOptions),
45+
UserModule.forRoot(moduleOptions),
46+
AirdropModule.forRoot(moduleOptions),
47+
CacheModule.register({ isGlobal: true }),
48+
];
49+
50+
if (moduleOptions.runModes.includes(RunMode.Scraper)) {
51+
imports.push(
52+
ScraperModule.forRoot(moduleOptions),
53+
BullModule.forRootAsync({
54+
imports: [MessagingModule],
55+
useExisting: BullConfigService,
56+
}),
57+
);
58+
}
59+
60+
return {
61+
module: AppModule,
62+
imports,
63+
};
64+
}
65+
5366
configure(consumer: MiddlewareConsumer) {
5467
consumer.apply(LogsMiddleware).forRoutes("*");
5568
}

‎src/dynamic-module.ts

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
export enum RunMode {
2+
Scraper = "scraper",
3+
Normal = "normal",
4+
Test = "test",
5+
}
6+
7+
export interface ModuleOptions {
8+
runModes: RunMode[];
9+
}

‎src/main.ts

+4-1
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,12 @@ import { AppModule } from "./app.module";
66
import { AppConfig } from "./modules/configuration/configuration.service";
77
import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger";
88
import { ValidationPipe } from "./validation.pipe";
9+
import { configValues } from "./modules/configuration";
910

1011
async function bootstrap() {
11-
const app = await NestFactory.create<NestExpressApplication>(AppModule);
12+
const app = await NestFactory.create<NestExpressApplication>(
13+
AppModule.forRoot({ runModes: configValues().app.runModes }),
14+
);
1215
const config = app.get(AppConfig);
1316
const port = config.values.app.port;
1417

‎src/modules/scraper/adapter/db/claim-fixture.ts renamed to ‎src/modules/airdrop/adapter/db/claim-fixture.ts

File renamed without changes.

‎src/modules/airdrop/entry-points/http/controller.ts

+2
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ export class AirdropController {
7777
),
7878
)
7979
@ApiTags("airdrop")
80+
@ApiBearerAuth()
8081
async feedWalletRewards(
8182
@UploadedFiles() files: { walletRewards?: Express.Multer.File[]; communityRewards?: Express.Multer.File[] },
8283
) {
@@ -96,6 +97,7 @@ export class AirdropController {
9697
}),
9798
)
9899
@ApiTags("airdrop")
100+
@ApiBearerAuth()
99101
uploadMerkleDistributorRecipients(@UploadedFile() file: Express.Multer.File) {
100102
return this.airdropService.processMerkleDistributorRecipientsFile(file);
101103
}

‎src/modules/scraper/model/claim.entity.ts renamed to ‎src/modules/airdrop/model/claim.entity.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import {
88
UpdateDateColumn,
99
ManyToOne,
1010
} from "typeorm";
11-
import { MerkleDistributorWindow } from "../../airdrop/model/merkle-distributor-window.entity";
11+
import { MerkleDistributorWindow } from "./merkle-distributor-window.entity";
1212

1313
@Entity()
1414
@Unique("UK_claim_windowIndex_accountIndex", ["windowIndex", "accountIndex"])

‎src/modules/airdrop/model/merkle-distributor-window.entity.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { Column, CreateDateColumn, Entity, OneToMany, PrimaryGeneratedColumn, Unique } from "typeorm";
22
import { MerkleDistributorRecipient } from "./merkle-distributor-recipient.entity";
3-
import { Claim } from "../../scraper/model/claim.entity";
3+
import { Claim } from "./claim.entity";
44

55
@Entity()
66
// Don't allow duplicates of the window index

‎src/modules/airdrop/module.ts

+48-25
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Module } from "@nestjs/common";
1+
import { Module, Provider, DynamicModule } from "@nestjs/common";
22
import { TypeOrmModule } from "@nestjs/typeorm";
33

44
import { UserModule } from "../user/module";
@@ -11,32 +11,55 @@ import { AirdropController } from "./entry-points/http/controller";
1111

1212
import { CommunityRewards } from "./model/community-rewards.entity";
1313
import { WalletRewards } from "./model/wallet-rewards.entity";
14-
import { Deposit } from "../scraper/model/deposit.entity";
14+
import { Deposit } from "../deposit/model/deposit.entity";
1515
import { MerkleDistributorWindowFixture } from "./adapter/db/merkle-distributor-window-fixture";
1616
import { MerkleDistributorWindow } from "./model/merkle-distributor-window.entity";
1717
import { MerkleDistributorRecipient } from "./model/merkle-distributor-recipient.entity";
1818
import { MerkleDistributorRecipientFixture } from "./adapter/db/merkle-distributor-recipient";
19+
import { ModuleOptions, RunMode } from "../../dynamic-module";
20+
import { ClaimFixture } from "./adapter/db/claim-fixture";
21+
import { Claim } from "./model/claim.entity";
1922

20-
@Module({
21-
providers: [
22-
AirdropService,
23-
WalletRewardsFixture,
24-
CommunityRewardsFixture,
25-
MerkleDistributorWindowFixture,
26-
MerkleDistributorRecipientFixture,
27-
],
28-
controllers: [AirdropController],
29-
imports: [
30-
TypeOrmModule.forFeature([
31-
CommunityRewards,
32-
WalletRewards,
33-
Deposit,
34-
MerkleDistributorWindow,
35-
MerkleDistributorRecipient,
36-
]),
37-
UserModule,
38-
AppConfigModule,
39-
],
40-
exports: [],
41-
})
42-
export class AirdropModule {}
23+
@Module({})
24+
export class AirdropModule {
25+
static forRoot(moduleOptions: ModuleOptions): DynamicModule {
26+
let module: DynamicModule = { module: AirdropModule, providers: [], controllers: [], imports: [], exports: [] };
27+
28+
if (moduleOptions.runModes.includes(RunMode.Normal) || moduleOptions.runModes.includes(RunMode.Test)) {
29+
module = {
30+
...module,
31+
providers: [...module.providers, AirdropService, ClaimFixture],
32+
controllers: [...module.controllers, AirdropController],
33+
imports: [
34+
...module.imports,
35+
TypeOrmModule.forFeature([
36+
CommunityRewards,
37+
WalletRewards,
38+
Deposit,
39+
MerkleDistributorWindow,
40+
MerkleDistributorRecipient,
41+
Claim,
42+
]),
43+
UserModule.forRoot(moduleOptions),
44+
AppConfigModule,
45+
],
46+
};
47+
}
48+
49+
if (moduleOptions.runModes.includes(RunMode.Test)) {
50+
module = {
51+
...module,
52+
providers: [
53+
...module.providers,
54+
ClaimFixture,
55+
WalletRewardsFixture,
56+
CommunityRewardsFixture,
57+
MerkleDistributorWindowFixture,
58+
MerkleDistributorRecipientFixture,
59+
],
60+
};
61+
}
62+
63+
return module;
64+
}
65+
}

‎src/modules/airdrop/services/airdrop-service.ts

+1-3
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import BigNumber from "bignumber.js";
66
import { DataSource, IsNull, Not, QueryFailedError, Repository } from "typeorm";
77
import { Cache } from "cache-manager";
88

9-
import { Deposit } from "../../scraper/model/deposit.entity";
9+
import { Deposit } from "../../deposit/model/deposit.entity";
1010
import { CommunityRewards } from "../model/community-rewards.entity";
1111
import { WalletRewards } from "../model/wallet-rewards.entity";
1212
import { Token } from "../../web3/model/token.entity";
@@ -36,8 +36,6 @@ export class AirdropService {
3636
@InjectRepository(CommunityRewards) private communityRewardsRepository: Repository<CommunityRewards>,
3737
@InjectRepository(WalletRewards) private walletRewardsRepository: Repository<WalletRewards>,
3838
@InjectRepository(Deposit) private depositRepository: Repository<Deposit>,
39-
@InjectRepository(Deposit) private merkleDistributorWindowRepository: Repository<MerkleDistributorWindow>,
40-
@InjectRepository(Deposit) private merkleDistributorRecipientRepository: Repository<MerkleDistributorRecipient>,
4139
private userService: UserService,
4240
private appConfig: AppConfig,
4341
private dataSource: DataSource,

0 commit comments

Comments
 (0)