Skip to content

Commit 7bcfd7b

Browse files
ibodrovibodrov
committed
concord-server-db: move process log objects into a separate changelog
1 parent 62fe47a commit 7bcfd7b

File tree

16 files changed

+570
-316
lines changed

16 files changed

+570
-316
lines changed

‎server/db/pom.xml‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
<properties>
1717
<db.image>library/postgres:10.4-alpine</db.image>
1818
<db.baseDir>${project.build.directory}/db</db.baseDir>
19-
<db.changeLogPath>com/walmartlabs/concord/server/db/liquibase.xml</db.changeLogPath>
19+
<mainDb.changeLogPath>com/walmartlabs/concord/server/db/mainDb.xml</mainDb.changeLogPath>
20+
<logDb.changeLogPath>com/walmartlabs/concord/server/db/logDb.xml</logDb.changeLogPath>
2021
<db.host>localhost</db.host>
2122
<db.username>postgres</db.username>
2223
<db.password>q1</db.password>
@@ -164,14 +165,27 @@
164165
<version>${liquibase.version}</version>
165166
<executions>
166167
<execution>
168+
<id>main-db</id>
167169
<phase>generate-sources</phase>
168170
<goals>
169171
<goal>update</goal>
170172
</goals>
173+
<configuration>
174+
<changeLogFile>src/main/resources/${mainDb.changeLogPath}</changeLogFile>
175+
</configuration>
176+
</execution>
177+
<execution>
178+
<id>log-db</id>
179+
<phase>generate-sources</phase>
180+
<goals>
181+
<goal>update</goal>
182+
</goals>
183+
<configuration>
184+
<changeLogFile>src/main/resources/${logDb.changeLogPath}</changeLogFile>
185+
</configuration>
171186
</execution>
172187
</executions>
173188
<configuration>
174-
<changeLogFile>src/main/resources/${db.changeLogPath}</changeLogFile>
175189
<driver>org.postgresql.Driver</driver>
176190
<url>jdbc:postgresql://${db.host}:${db.port}/postgres</url>
177191
<username>${db.username}</username>

‎server/db/src/main/java/com/walmartlabs/concord/db/DatabaseModule.java‎

Lines changed: 61 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
import javax.inject.Singleton;
2929
import javax.sql.DataSource;
30+
import java.lang.annotation.Annotation;
3031
import java.util.Comparator;
3132
import java.util.Set;
3233

@@ -61,55 +62,101 @@ public String getChangeLogPath() {
6162

6263
@Override
6364
public int order() {
64-
// we expect the server's DB to be migrated first
65+
// we expect the main DB to be migrated first
6566
return 0;
6667
}
6768

6869
@Override
6970
public String toString() {
70-
return "server-db";
71+
return "main-db";
72+
}
73+
};
74+
}
75+
76+
@Provides
77+
@LogDB
78+
@Singleton
79+
public DatabaseChangeLogProvider logDbChangeLogProvider() {
80+
return new DatabaseChangeLogProvider() {
81+
@Override
82+
public String getChangeLogPath() {
83+
return "com/walmartlabs/concord/server/db/logDb.xml";
84+
}
85+
86+
@Override
87+
public int order() {
88+
// we expect the log DB to be migrated second
89+
return 1;
90+
}
91+
92+
@Override
93+
public String toString() {
94+
return "log-db";
7195
}
7296
};
7397
}
7498

7599
@Provides
76100
@MainDB
77101
@Singleton
78-
public DataSource appDataSource(@MainDB DatabaseConfiguration cfg,
102+
public DataSource mainDbDataSource(@MainDB DatabaseConfiguration cfg,
103+
MetricRegistry metricRegistry,
104+
Set<DatabaseChangeLogProvider> changeLogProviders) {
105+
106+
DataSource ds = DataSourceUtils.createDataSource(cfg, "app" /* not called "main" for backward compatibility */, cfg.username(), cfg.password(), metricRegistry);
107+
if (migrateDb) {
108+
migrateDb(changeLogProviders, ds, cfg, MainDB.class);
109+
}
110+
return ds;
111+
}
112+
113+
@Provides
114+
@LogDB
115+
@Singleton
116+
public DataSource logDataSource(@LogDB DatabaseConfiguration cfg,
79117
MetricRegistry metricRegistry,
80118
Set<DatabaseChangeLogProvider> changeLogProviders) {
81119

82-
DataSource ds = DataSourceUtils.createDataSource(cfg, "app", cfg.username(), cfg.password(), metricRegistry);
83-
120+
DataSource ds = DataSourceUtils.createDataSource(cfg, "log", cfg.username(), cfg.password(), metricRegistry);
84121
if (migrateDb) {
85-
changeLogProviders.stream()
86-
// can't inject a set of objects with the same qualifier, filter manually
87-
.filter(p -> p.getClass().getAnnotation(MainDB.class) != null)
88-
.sorted(Comparator.comparingInt(DatabaseChangeLogProvider::order))
89-
.forEach(p -> DataSourceUtils.migrateDb(ds, p, cfg.changeLogParameters()));
122+
migrateDb(changeLogProviders, ds, cfg, LogDB.class);
90123
}
91-
92124
return ds;
93125
}
94126

95127
@Provides
96128
@JsonStorageDB
97129
@Singleton
98-
public DataSource inventoryDataSource(@JsonStorageDB DatabaseConfiguration cfg, MetricRegistry metricRegistry) {
130+
public DataSource jsonStorageDbDataSource(@JsonStorageDB DatabaseConfiguration cfg, MetricRegistry metricRegistry) {
99131
return DataSourceUtils.createDataSource(cfg, "inventory", cfg.username(), cfg.password(), metricRegistry);
100132
}
101133

102134
@Provides
103135
@MainDB
104136
@Singleton
105-
public Configuration appJooqConfiguration(@MainDB DataSource ds) {
137+
public Configuration mainDbJooqConfiguration(@MainDB DataSource ds) {
138+
return DataSourceUtils.createJooqConfiguration(ds);
139+
}
140+
141+
@Provides
142+
@LogDB
143+
@Singleton
144+
public Configuration logDbJooqConfiguration(@LogDB DataSource ds) {
106145
return DataSourceUtils.createJooqConfiguration(ds);
107146
}
108147

109148
@Provides
110149
@JsonStorageDB
111150
@Singleton
112-
public Configuration inventoryJooqConfiguration(@JsonStorageDB DataSource ds) {
151+
public Configuration jsonStorageDbJooqConfiguration(@JsonStorageDB DataSource ds) {
113152
return DataSourceUtils.createJooqConfiguration(ds);
114153
}
154+
155+
private static void migrateDb(Set<DatabaseChangeLogProvider> changeLogProviders, DataSource ds, DatabaseConfiguration cfg, Class<? extends Annotation> annotation) {
156+
changeLogProviders.stream()
157+
// can't inject a set of objects with the same qualifier, filter manually
158+
.filter(p -> p.getClass().getAnnotation(annotation) != null)
159+
.sorted(Comparator.comparingInt(DatabaseChangeLogProvider::order))
160+
.forEach(p -> DataSourceUtils.migrateDb(ds, p, cfg.changeLogParameters()));
161+
}
115162
}

‎server/db/src/main/java/com/walmartlabs/concord/db/LogDB.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package com.walmartlabs.concord.db;
2+
3+
/*-
4+
* *****
5+
* Concord
6+
* -----
7+
* Copyright (C) 2017 - 2019 Walmart Inc.
8+
* -----
9+
* Licensed under the Apache License, Version 2.0 (the "License");
10+
* you may not use this file except in compliance with the License.
11+
* You may obtain a copy of the License at
12+
*
13+
* http://www.apache.org/licenses/LICENSE-2.0
14+
*
15+
* Unless required by applicable law or agreed to in writing, software
16+
* distributed under the License is distributed on an "AS IS" BASIS,
17+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18+
* See the License for the specific language governing permissions and
19+
* limitations under the License.
20+
* =====
21+
*/
22+
23+
import javax.inject.Qualifier;
24+
import java.lang.annotation.Retention;
25+
import java.lang.annotation.RetentionPolicy;
26+
27+
@Retention(RetentionPolicy.RUNTIME)
28+
@Qualifier
29+
public @interface LogDB {
30+
}

‎server/db/src/main/java/com/walmartlabs/concord/db/MainDBChangeLogProvider.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ public class MainDBChangeLogProvider implements DatabaseChangeLogProvider {
2525

2626
@Override
2727
public String getChangeLogPath() {
28-
return "com/walmartlabs/concord/server/db/liquibase.xml";
28+
return "com/walmartlabs/concord/server/db/mainDb.xml";
2929
}
3030

3131
@Override

‎server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb-v1.49.0.xml‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<databaseChangeLog
3+
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
6+
7+
<changeSet id="1490010" author="[email protected]">
8+
<validCheckSum>7:b70de8118884fd89025bb9ed8a8b0572</validCheckSum>
9+
<createTable tableName="PROCESS_LOG_SEGMENTS">
10+
<column name="INSTANCE_ID" type="uuid">
11+
<constraints nullable="false"/>
12+
</column>
13+
<column name="INSTANCE_CREATED_AT" type="timestamp">
14+
<constraints nullable="false"/>
15+
</column>
16+
<column name="SEGMENT_ID" type="bigserial" autoIncrement="true">
17+
<constraints nullable="false"/>
18+
</column>
19+
<column name="SEGMENT_NAME" type="text">
20+
<constraints nullable="false"/>
21+
</column>
22+
<column name="CORRELATION_ID" type="uuid">
23+
<constraints nullable="true"/>
24+
</column>
25+
<column name="SEGMENT_TS" type="timestamp">
26+
<constraints nullable="false"/>
27+
</column>
28+
<column name="SEGMENT_STATUS" type="text">
29+
<constraints nullable="true"/>
30+
</column>
31+
<column name="SEGMENT_ERRORS" type="int">
32+
<constraints nullable="true"/>
33+
</column>
34+
<column name="SEGMENT_WARN" type="int">
35+
<constraints nullable="true"/>
36+
</column>
37+
</createTable>
38+
</changeSet>
39+
40+
<changeSet id="1490011" author="[email protected]">
41+
<createIndex tableName="PROCESS_LOG_SEGMENTS" indexName="IDX_PLS_IDS">
42+
<column name="INSTANCE_ID"/>
43+
<column name="INSTANCE_CREATED_AT"/>
44+
</createIndex>
45+
</changeSet>
46+
47+
<changeSet id="1490020" author="[email protected]">
48+
<validCheckSum>7:10ab12d5b908de504f897e6e2a773036</validCheckSum>
49+
<createTable tableName="PROCESS_LOG_DATA">
50+
<column name="INSTANCE_ID" type="uuid">
51+
<constraints nullable="false"/>
52+
</column>
53+
<column name="INSTANCE_CREATED_AT" type="timestamp">
54+
<constraints nullable="false"/>
55+
</column>
56+
<column name="SEGMENT_ID" type="bigserial" autoIncrement="true">
57+
<constraints nullable="false"/>
58+
</column>
59+
<column name="LOG_RANGE" type="int4range">
60+
<constraints nullable="false"/>
61+
</column>
62+
<column name="SEGMENT_RANGE" type="int4range">
63+
<constraints nullable="false"/>
64+
</column>
65+
<column name="CHUNK_DATA" type="longblob">
66+
<constraints nullable="false"/>
67+
</column>
68+
<column name="LOG_SEQ" type="bigserial" autoIncrement="true">
69+
<constraints nullable="false"/>
70+
</column>
71+
</createTable>
72+
</changeSet>
73+
74+
<changeSet id="1490030" author="[email protected]">
75+
<createIndex tableName="PROCESS_LOG_DATA" indexName="IDX_PLD_IDS">
76+
<column name="INSTANCE_ID"/>
77+
<column name="INSTANCE_CREATED_AT"/>
78+
<column name="SEGMENT_ID"/>
79+
</createIndex>
80+
</changeSet>
81+
82+
<changeSet id="1490040" author="[email protected]">
83+
<createProcedure dbms="postgresql">
84+
create or replace function PROCESS_LOG_DATA_SEGMENT_NEXT_RANGE(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_SEGMENT_ID bigint, P_DATA_LEN int)
85+
returns int4range as $$
86+
declare
87+
R_START int;
88+
begin
89+
select coalesce(max(upper(SEGMENT_RANGE)), 0) into R_START
90+
from PROCESS_LOG_DATA
91+
where
92+
INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT and SEGMENT_ID = P_SEGMENT_ID;
93+
94+
if R_START is null then
95+
R_START := 0;
96+
end if;
97+
98+
return int4range(R_START, R_START + P_DATA_LEN);
99+
end;
100+
$$ language plpgsql;
101+
</createProcedure>
102+
103+
<createProcedure dbms="postgresql">
104+
create or replace function PROCESS_LOG_DATA_NEXT_RANGE(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_DATA_LEN int)
105+
returns int4range as $$
106+
declare
107+
R_START int;
108+
begin
109+
select coalesce(max(upper(LOG_RANGE)), 0) into R_START
110+
from PROCESS_LOG_DATA
111+
where
112+
INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT;
113+
114+
if R_START is null then
115+
R_START := 0;
116+
end if;
117+
118+
return int4range(R_START, R_START + P_DATA_LEN);
119+
end;
120+
$$ language plpgsql;
121+
</createProcedure>
122+
</changeSet>
123+
124+
<changeSet id="1490050" author="[email protected]">
125+
<createProcedure dbms="postgresql">
126+
create or replace function PROCESS_LOG_DATA_SEGMENT_LAST_N_BYTES(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_SEGMENT_ID bigint, P_DATA_LEN int)
127+
returns int4range as $$
128+
declare
129+
R_START int;
130+
begin
131+
select coalesce(max(upper(SEGMENT_RANGE)), 0) into R_START
132+
from PROCESS_LOG_DATA
133+
where
134+
INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT and SEGMENT_ID = P_SEGMENT_ID;
135+
136+
if R_START is null then
137+
R_START := 0;
138+
end if;
139+
140+
return int4range(R_START - P_DATA_LEN, R_START);
141+
end;
142+
$$ language plpgsql;
143+
</createProcedure>
144+
</changeSet>
145+
146+
<changeSet id="1490060" author="[email protected]">
147+
<createProcedure dbms="postgresql">
148+
create or replace function PROCESS_LOG_DATA_LAST_N_BYTES(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_DATA_LEN int)
149+
returns int4range as $$
150+
declare
151+
R_START int;
152+
begin
153+
select coalesce(max(upper(LOG_RANGE)), 0) into R_START
154+
from PROCESS_LOG_DATA
155+
where
156+
INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT;
157+
158+
if R_START is null then
159+
R_START := 0;
160+
end if;
161+
162+
return int4range(R_START - P_DATA_LEN, R_START);
163+
end;
164+
$$ language plpgsql;
165+
</createProcedure>
166+
</changeSet>
167+
</databaseChangeLog>

0 commit comments

Comments
 (0)