Skip to content

Commit fb6f861

Browse files
mchmarnymchmarny
committed
Refactored to K8s configsets and secrets
1 parent 23fdfea commit fb6f861

17 files changed

+224
-159
lines changed

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,5 @@ _testmain.go
2424
*.prof
2525

2626
.DS_Store
27-
signal
27+
tsignal
2828
auth.json

‎Dockerfile

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
FROM golang
2-
COPY ./ /go/src/github.com/mchmarny/signal
3-
WORKDIR /go/src/github.com/mchmarny/signal
2+
COPY ./ /go/src/github.com/mchmarny/tsignal
3+
WORKDIR /go/src/github.com/mchmarny/tsignal
44
RUN go get ./
55
RUN go build
6-
CMD ./signal
6+
CMD ./tsignal

‎build

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
GOOS=linux go build -a --ldflags '-extldflags "-static"' -tags netgo -installsuffix netgo -o signal .
1+
GOOS=linux go build -a --ldflags '-extldflags "-static"' -tags netgo -installsuffix netgo -o tsignal .

‎build-container

-3
This file was deleted.

‎configmaps/europe-west1.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: cluster
5+
data:
6+
region: europe-west1

‎configmaps/us-west1.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: cluster
5+
data:
6+
region: us-west1

‎container-publish

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
gcloud container builds submit --tag gcr.io/cpe-data/tsignal:latest .

‎deployments/signal.yaml

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
apiVersion: extensions/v1beta1
2+
kind: Deployment
3+
metadata:
4+
labels:
5+
app: signal
6+
name: signal
7+
spec:
8+
replicas: 1
9+
template:
10+
metadata:
11+
labels:
12+
app: signal
13+
name: signal
14+
spec:
15+
containers:
16+
- name: signal
17+
image: gcr.io/cpe-data/tsignal:latest
18+
imagePullPolicy: Always
19+
env:
20+
- name: T_CONSUMER_KEY
21+
valueFrom:
22+
secretKeyRef:
23+
name: signal-tw-key
24+
key: T_CONSUMER_KEY
25+
- name: T_CONSUMER_SECRET
26+
valueFrom:
27+
secretKeyRef:
28+
name: signal-tw-secret
29+
key: T_CONSUMER_SECRET
30+
- name: T_ACCESS_TOKEN
31+
valueFrom:
32+
secretKeyRef:
33+
name: signal-tw-token
34+
key: T_ACCESS_TOKEN
35+
- name: T_ACCESS_SECRET
36+
valueFrom:
37+
secretKeyRef:
38+
name: signal-tw-access
39+
key: T_ACCESS_SECRET
40+
- name: GCLOUD_PROJECT
41+
valueFrom:
42+
secretKeyRef:
43+
name: signal-gcloud-project
44+
key: GCLOUD_PROJECT
45+
- name: GCLOUD_INSTANCE
46+
valueFrom:
47+
secretKeyRef:
48+
name: signal-spanner-instance
49+
key: GCLOUD_INSTANCE
50+
- name: GCLOUD_DB
51+
valueFrom:
52+
secretKeyRef:
53+
name: signal-spanner-db
54+
key: GCLOUD_DB
55+
volumeMounts:
56+
- name: "service-account"
57+
mountPath: "/var/run/secret/cloud.google.com"
58+
- name: "certs"
59+
mountPath: "/etc/ssl/certs"
60+
volumes:
61+
- name: "service-account"
62+
secret:
63+
secretName: "signal-sa"
64+
- name: "certs"
65+
hostPath:
66+
path: "/etc/ssl/certs"

‎scripts/build.sh

-17
This file was deleted.

‎scripts/cluster.sh

+21-9
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,29 @@ gcloud container clusters create "${APP_NAME}-cluster" \
1111
--disk-size "100" \
1212
--scopes default,cloud-platform,logging-write,monitoring-write \
1313
--num-nodes "1" \
14+
--zone $GCLOUD_ZONE \
1415
--network "default" \
1516
--enable-cloud-logging \
1617
--enable-cloud-monitoring
1718

19+
# connect
20+
gcloud container clusters get-credentials "${APP_NAME}-cluster" \
21+
--zone $GCLOUD_ZONE --project $GCLOUD_PROJECT
1822

19-
# define env vars
20-
# T_CONSUMER_KEY=${T_CONSUMER_KEY}
21-
# T_CONSUMER_SECRET=${T_CONSUMER_SECRET}
22-
# T_ACCESS_TOKEN=${T_ACCESS_TOKEN}
23-
# T_ACCESS_SECRET=${T_ACCESS_SECRET}
24-
# GCLOUD_PROJECT=${GCLOUD_PROJECT}
25-
# GCLOUD_INSTANCE=${GCLOUD_INSTANCE}
26-
# GCLOUD_DB=${GCLOUD_DB}
27-
# GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS}
23+
# configs
24+
kubectl create configmap signal-config --from-file configmaps/us-west1.yaml
25+
26+
27+
# populate secrets
28+
kubectl create secret generic signal-tw-key --from-literal=T_CONSUMER_KEY=$T_CONSUMER_KEY
29+
kubectl create secret generic signal-tw-secret --from-literal=T_CONSUMER_SECRET=$T_CONSUMER_SECRET
30+
kubectl create secret generic signal-tw-token --from-literal=T_ACCESS_TOKEN=$T_ACCESS_TOKEN
31+
kubectl create secret generic signal-tw-access --from-literal=T_ACCESS_SECRET=$T_ACCESS_SECRET
32+
33+
kubectl create secret generic signal-gcloud-project --from-literal=GCLOUD_PROJECT=$GCLOUD_PROJECT
34+
kubectl create secret generic signal-spanner-instance --from-literal=GCLOUD_INSTANCE=$GCLOUD_INSTANCE
35+
kubectl create secret generic signal-spanner-db --from-literal=GCLOUD_DB=$GCLOUD_DB
36+
kubectl create secret generic signal-sa --from-file auth.json
37+
38+
# deploy
39+
# kubectl create -f deployments/signal.yaml

‎scripts/config.sh

+2
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,13 @@
77
# export T_ACCESS_SECRET=""
88

99
# Google
10+
export APP_DIR="$(dirname "$0")"
1011
export APP_NAME="tsignal"
1112
export GCLOUD_PROJECT="cpe-data"
1213
export GOOGLE_APPLICATION_CREDENTIALS="./auth.json"
1314
export GCLOUD_INSTANCE="${APP_NAME}"
1415
export GCLOUD_DB="db"
16+
export GCLOUD_ZONE="us-west1-b"
1517
export GCLOUD_SA_NAME="${APP_NAME}-sa"
1618
export GCLOUD_SA_EMAIL="${GCLOUD_SA_NAME}@${GCLOUD_PROJECT}.iam.gserviceaccount.com"
1719

‎scripts/query.sql

-124
This file was deleted.

‎scripts/setup.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@
33
DIR="$(dirname "$0")"
44
. "${DIR}/config.sh"
55

6+
67
gcloud beta spanner instances create ${GCLOUD_INSTANCE} \
7-
--config=${GCLOUD_REGION} \
8+
--config=regional-us-central1 \
89
--description="${GCLOUD_INSTANCE} Instance" \
910
--nodes=1
1011

‎sql/counts.sql

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
-- Data summary
2+
SELECT * FROM
3+
(SELECT count(*) StockCount from Stocks),
4+
(SELECT count(*) PriceCount, MAX(SampleOn) LastPrice from Prices),
5+
(SELECT count(*) AuthorCount, MAX(UpdatedOn) LastAuthorUpUpdate from Authors),
6+
(SELECT count(*) PostCount, MAX(PostedOn) LastPost from Posts)

‎sql/price-per-hr.sql

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
-- Price per stock per hour
2+
SELECT p.Symbol,
3+
p.PostHour,
4+
ROUND(MIN(c.AskPrice),2) MinAskPrice,
5+
ROUND(AVG(c.AskPrice),2) AvgAskPrice,
6+
ROUND(MAX(c.AskPrice),2) MaxAskPrice,
7+
ROUND(AVG(p.SentimentScore),3) AvgSentiment,
8+
ROUND(AVG(p.WeightedScore),2) WeightedScore,
9+
SUM(p.PostCount) PostCount,
10+
SUM(p.PositiveCount) Positives,
11+
SUM(p.NegativeCount) Negatives,
12+
SUM(p.NoiseCount) Noise
13+
FROM (
14+
SELECT r.Symbol,
15+
FORMAT_TIMESTAMP("%F-%k", r.PostedOn)
16+
PostHour, r.SentimentScore,
17+
-- Using Friend Count as a weight multiplier for significant sentiment, else 0
18+
(1 + a.FriendCount / 1000) * CASE
19+
-- if strong then full credit for score
20+
WHEN ABS(r.SentimentScore) > 0.6 THEN r.SentimentScore
21+
-- if poor give them 50% credit for the sentiment score
22+
WHEN ABS(r.SentimentScore) >= 0.3 AND ABS(r.SentimentScore) <= 0.6 THEN r.SentimentScore * 0.5
23+
-- else it is noise to ignore
24+
ELSE 0 END WeightedScore,
25+
1 PostCount,
26+
-- for positic/negatives using 0.3 significant sentiment filter, else noise
27+
CASE WHEN r.SentimentScore >= 0.3 THEN 1 ELSE 0 END PositiveCount,
28+
CASE WHEN r.SentimentScore <= -0.3 THEN 1 ELSE 0 END NegativeCount,
29+
CASE WHEN r.SentimentScore > -0.3 AND r.SentimentScore < 0.3 THEN 1 ELSE 0 END NoiseCount
30+
FROM Posts r
31+
JOIN Authors a ON r.Username = a.Username
32+
) p
33+
LEFT JOIN (
34+
SELECT Symbol, FORMAT_TIMESTAMP("%F-%k", SampleOn) AskHour, AskPrice
35+
FROM Prices
36+
) c ON p.Symbol = c.Symbol AND p.PostHour = c.AskHour
37+
GROUP BY p.Symbol, p.PostHour
38+
ORDER BY p.Symbol, p.PostHour DESC

‎sql/price-per-stock-hr.sql

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
-- Same as above but for a given stock
2+
SELECT p.Symbol,
3+
p.PostHour,
4+
ROUND(MIN(c.AskPrice),2) MinAskPrice,
5+
ROUND(AVG(c.AskPrice),2) AvgAskPrice,
6+
ROUND(MAX(c.AskPrice),2) MaxAskPrice,
7+
ROUND(AVG(p.SentimentScore),3) AvgSentiment,
8+
ROUND(AVG(p.WeightedScore),2) WeightedScore,
9+
SUM(p.PostCount) PostCount,
10+
SUM(p.PositiveCount) Positives,
11+
SUM(p.NegativeCount) Negatives,
12+
SUM(p.NoiseCount) Noise
13+
FROM (
14+
SELECT r.Symbol,
15+
FORMAT_TIMESTAMP("%F-%k", r.PostedOn)
16+
PostHour, r.SentimentScore,
17+
-- Using Friend Count as a weight multiplier for significant sentiment, else 0
18+
(1 + a.FriendCount / 1000) * CASE
19+
-- if strong then full credit for score
20+
WHEN ABS(r.SentimentScore) > 0.6 THEN r.SentimentScore
21+
-- if poor give them 50% credit for the sentiment score
22+
WHEN ABS(r.SentimentScore) >= 0.3 AND ABS(r.SentimentScore) <= 0.6 THEN r.SentimentScore * 0.5
23+
-- else it is noise to ignore
24+
ELSE 0 END WeightedScore,
25+
1 PostCount,
26+
-- for positic/negatives using 0.3 significant sentiment filter, else noise
27+
CASE WHEN r.SentimentScore >= 0.3 THEN 1 ELSE 0 END PositiveCount,
28+
CASE WHEN r.SentimentScore <= -0.3 THEN 1 ELSE 0 END NegativeCount,
29+
CASE WHEN r.SentimentScore > -0.3 AND r.SentimentScore < 0.3 THEN 1 ELSE 0 END NoiseCount
30+
FROM Posts r
31+
JOIN Authors a ON r.Username = a.Username
32+
WHERE r.Symbol = 'MSFT'
33+
) p
34+
LEFT JOIN (
35+
SELECT Symbol, FORMAT_TIMESTAMP("%F-%k", SampleOn) AskHour, AskPrice
36+
FROM Prices
37+
) c ON p.Symbol = c.Symbol AND p.PostHour = c.AskHour
38+
GROUP BY p.Symbol, p.PostHour
39+
ORDER BY p.Symbol, p.PostHour DESC

0 commit comments

Comments
 (0)