Skip to content

Commit 64c41af

Browse files
yangawsyangaws
authored
Add region support to BOM/SIN/LHR/YUL/SFO (#75)
1 parent f242ef0 commit 64c41af

File tree

11 files changed

+231
-9
lines changed

11 files changed

+231
-9
lines changed

‎CHANGELOG.rst‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@
22
CHANGELOG
33
=========
44

5-
1.2.1dev
6-
========
5+
1.2.1
6+
=====
77

8-
* Encode DenseMatrix and SparseMatrix in probobuf Record format
8+
* spark/pyspark: encode DenseMatrix and SparseMatrix in probobuf Record format
9+
* spark/pyspark: add new region support for BOM/SIN/LHR/YUL/SFO
910

1011
1.2.0
1112
=====

‎sagemaker-pyspark-sdk/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
pyspark
1+
pyspark==2.3.2
22
numpy
33
pytest

‎sagemaker-pyspark-sdk/setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from setuptools import setup
88

99

10-
VERSION = "1.2.0"
10+
VERSION = "1.2.1"
1111

1212
TEMP_PATH = "deps"
1313
JARS_TARGET = os.path.join(TEMP_PATH, "jars")

‎sagemaker-spark-sdk/build.sbt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ scalaVersion := "2.11.7"
1919
// to change the version of spark add -DSPARK_VERSION=2.x.x when running sbt
2020
// for example: "sbt -DSPARK_VERSION=2.1.1 clean compile test doc package"
2121
val sparkVersion = System.getProperty("SPARK_VERSION", "2.2.0")
22-
version := "spark_" + sparkVersion + "-1.2.0"
22+
version := "spark_" + sparkVersion + "-1.2.1"
2323

2424
lazy val SageMakerSpark = (project in file("."))
2525

‎sagemaker-spark-sdk/src/main/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/ImageURIProvider.scala‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,12 @@ private[algorithms] object SagerMakerRegionAccountMaps {
4141
Regions.AP_NORTHEAST_2.getName -> "835164637446",
4242
Regions.EU_CENTRAL_1.getName -> "664544806723",
4343
Regions.AP_SOUTHEAST_2.getName -> "712309505854",
44-
Regions.GovCloud.getName -> "226302683700"
44+
Regions.GovCloud.getName -> "226302683700",
45+
Regions.AP_SOUTH_1.getName -> "991648021394",
46+
Regions.AP_SOUTHEAST_1.getName -> "475088953585",
47+
Regions.CA_CENTRAL_1.getName -> "469771592824",
48+
Regions.EU_WEST_2.getName -> "644912444149",
49+
Regions.US_WEST_1.getName -> "632365934929"
4550
)
4651

4752
// For LDA
@@ -53,7 +58,13 @@ private[algorithms] object SagerMakerRegionAccountMaps {
5358
Regions.AP_NORTHEAST_1.getName -> "258307448986",
5459
Regions.AP_NORTHEAST_2.getName -> "293181348795",
5560
Regions.EU_CENTRAL_1.getName -> "353608530281",
56-
Regions.AP_SOUTHEAST_2.getName -> "297031611018"
61+
Regions.AP_SOUTHEAST_2.getName -> "297031611018",
62+
Regions.GovCloud.getName -> "226302683700",
63+
Regions.AP_SOUTH_1.getName -> "991648021394",
64+
Regions.AP_SOUTHEAST_1.getName -> "475088953585",
65+
Regions.CA_CENTRAL_1.getName -> "469771592824",
66+
Regions.EU_WEST_2.getName -> "644912444149",
67+
Regions.US_WEST_1.getName -> "632365934929"
5768
)
5869

5970
// For XGBoost
@@ -66,7 +77,12 @@ private[algorithms] object SagerMakerRegionAccountMaps {
6677
Regions.AP_NORTHEAST_2.getName -> "306986355934",
6778
Regions.EU_CENTRAL_1.getName -> "813361260812",
6879
Regions.AP_SOUTHEAST_2.getName -> "544295431143",
69-
Regions.GovCloud.getName -> "226302683700"
80+
Regions.GovCloud.getName -> "226302683700",
81+
Regions.AP_SOUTH_1.getName -> "991648021394",
82+
Regions.AP_SOUTHEAST_1.getName -> "475088953585",
83+
Regions.CA_CENTRAL_1.getName -> "469771592824",
84+
Regions.EU_WEST_2.getName -> "644912444149",
85+
Regions.US_WEST_1.getName -> "632365934929"
7086
)
7187
}
7288

‎sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/FactorizationMachinesSageMakerEstimatorTests.scala‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,31 @@ class FactorizationMachinesSageMakerEstimatorTests extends FlatSpec with Mockito
101101
createFactorizationMachinesBinaryClassifier(region = Regions.GovCloud.getName)
102102
assert(estimatorGovCloud.trainingImage ==
103103
"226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/factorization-machines:1")
104+
105+
val estimatorAPSouth1 =
106+
createFactorizationMachinesBinaryClassifier(region = Regions.AP_SOUTH_1.getName)
107+
assert(estimatorAPSouth1.trainingImage ==
108+
"991648021394.dkr.ecr.ap-south-1.amazonaws.com/factorization-machines:1")
109+
110+
val estimatorAPSouthEast1 =
111+
createFactorizationMachinesBinaryClassifier(region = Regions.AP_SOUTHEAST_1.getName)
112+
assert(estimatorAPSouthEast1.trainingImage ==
113+
"475088953585.dkr.ecr.ap-southeast-1.amazonaws.com/factorization-machines:1")
114+
115+
val estimatorEUWest2 =
116+
createFactorizationMachinesBinaryClassifier(region = Regions.EU_WEST_2.getName)
117+
assert(estimatorEUWest2.trainingImage ==
118+
"644912444149.dkr.ecr.eu-west-2.amazonaws.com/factorization-machines:1")
119+
120+
val estimatorCACentral1 =
121+
createFactorizationMachinesBinaryClassifier(region = Regions.CA_CENTRAL_1.getName)
122+
assert(estimatorCACentral1.trainingImage ==
123+
"469771592824.dkr.ecr.ca-central-1.amazonaws.com/factorization-machines:1")
124+
125+
val estimatorUSWest1 =
126+
createFactorizationMachinesBinaryClassifier(region = Regions.US_WEST_1.getName)
127+
assert(estimatorUSWest1.trainingImage ==
128+
"632365934929.dkr.ecr.us-west-1.amazonaws.com/factorization-machines:1")
104129
}
105130

106131
it should "use the correct defaults for regressor" in {
@@ -152,6 +177,31 @@ class FactorizationMachinesSageMakerEstimatorTests extends FlatSpec with Mockito
152177
createFactorizationMachinesRegressor(region = Regions.GovCloud.getName)
153178
assert(estimatorGovCloud.trainingImage ==
154179
"226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/factorization-machines:1")
180+
181+
val estimatorAPSouth1 =
182+
createFactorizationMachinesRegressor(region = Regions.AP_SOUTH_1.getName)
183+
assert(estimatorAPSouth1.trainingImage ==
184+
"991648021394.dkr.ecr.ap-south-1.amazonaws.com/factorization-machines:1")
185+
186+
val estimatorAPSouthEast1 =
187+
createFactorizationMachinesRegressor(region = Regions.AP_SOUTHEAST_1.getName)
188+
assert(estimatorAPSouthEast1.trainingImage ==
189+
"475088953585.dkr.ecr.ap-southeast-1.amazonaws.com/factorization-machines:1")
190+
191+
val estimatorEUWest2 =
192+
createFactorizationMachinesRegressor(region = Regions.EU_WEST_2.getName)
193+
assert(estimatorEUWest2.trainingImage ==
194+
"644912444149.dkr.ecr.eu-west-2.amazonaws.com/factorization-machines:1")
195+
196+
val estimatorCACentral1 =
197+
createFactorizationMachinesRegressor(region = Regions.CA_CENTRAL_1.getName)
198+
assert(estimatorCACentral1.trainingImage ==
199+
"469771592824.dkr.ecr.ca-central-1.amazonaws.com/factorization-machines:1")
200+
201+
val estimatorUSWest1 =
202+
createFactorizationMachinesRegressor(region = Regions.US_WEST_1.getName)
203+
assert(estimatorUSWest1.trainingImage ==
204+
"632365934929.dkr.ecr.us-west-1.amazonaws.com/factorization-machines:1")
155205
}
156206

157207
it should "setFeatureDim" in {

‎sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/KMeansSageMakerEstimatorTests.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,26 @@ class KMeansSageMakerEstimatorTests extends FlatSpec with Matchers with MockitoS
8888
val estimatorGovCloud = createKMeansEstimator(region = Regions.GovCloud.getName)
8989
assert(estimatorGovCloud.trainingImage ==
9090
"226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/kmeans:1")
91+
92+
val estimatorAPSouth1 = createKMeansEstimator(region = Regions.AP_SOUTH_1.getName)
93+
assert(estimatorAPSouth1.trainingImage ==
94+
"991648021394.dkr.ecr.ap-south-1.amazonaws.com/kmeans:1")
95+
96+
val estimatorAPSouthEast1 = createKMeansEstimator(region = Regions.AP_SOUTHEAST_1.getName)
97+
assert(estimatorAPSouthEast1.trainingImage ==
98+
"475088953585.dkr.ecr.ap-southeast-1.amazonaws.com/kmeans:1")
99+
100+
val estimatorEUWest2 = createKMeansEstimator(region = Regions.EU_WEST_2.getName)
101+
assert(estimatorEUWest2.trainingImage ==
102+
"644912444149.dkr.ecr.eu-west-2.amazonaws.com/kmeans:1")
103+
104+
val estimatorCACentral1 = createKMeansEstimator(region = Regions.CA_CENTRAL_1.getName)
105+
assert(estimatorCACentral1.trainingImage ==
106+
"469771592824.dkr.ecr.ca-central-1.amazonaws.com/kmeans:1")
107+
108+
val estimatorUSWest1 = createKMeansEstimator(region = Regions.US_WEST_1.getName)
109+
assert(estimatorUSWest1.trainingImage ==
110+
"632365934929.dkr.ecr.us-west-1.amazonaws.com/kmeans:1")
91111
}
92112

93113
it should "setK" in {

‎sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/LDASageMakerEstimatorTests.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,26 @@ class LDASageMakerEstimatorTests extends FlatSpec with MockitoSugar {
8686
createLDAEstimator(region = Regions.AP_SOUTHEAST_2.getName)
8787
assert(estimatorAPSouthEast2.trainingImage ==
8888
"297031611018.dkr.ecr.ap-southeast-2.amazonaws.com/lda:1")
89+
90+
val estimatorAPSouth1 = createLDAEstimator(region = Regions.AP_SOUTH_1.getName)
91+
assert(estimatorAPSouth1.trainingImage ==
92+
"991648021394.dkr.ecr.ap-south-1.amazonaws.com/lda:1")
93+
94+
val estimatorAPSouthEast1 = createLDAEstimator(region = Regions.AP_SOUTHEAST_1.getName)
95+
assert(estimatorAPSouthEast1.trainingImage ==
96+
"475088953585.dkr.ecr.ap-southeast-1.amazonaws.com/lda:1")
97+
98+
val estimatorEUWest2 = createLDAEstimator(region = Regions.EU_WEST_2.getName)
99+
assert(estimatorEUWest2.trainingImage ==
100+
"644912444149.dkr.ecr.eu-west-2.amazonaws.com/lda:1")
101+
102+
val estimatorCACentral1 = createLDAEstimator(region = Regions.CA_CENTRAL_1.getName)
103+
assert(estimatorCACentral1.trainingImage ==
104+
"469771592824.dkr.ecr.ca-central-1.amazonaws.com/lda:1")
105+
106+
val estimatorUSWest1 = createLDAEstimator(region = Regions.US_WEST_1.getName)
107+
assert(estimatorUSWest1.trainingImage ==
108+
"632365934929.dkr.ecr.us-west-1.amazonaws.com/lda:1")
89109
}
90110

91111
it should "setFeatureDim" in {

‎sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/LinearLearnerSageMakerEstimatorTests.scala‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,31 @@ class LinearLearnerSageMakerEstimatorTests extends FlatSpec with MockitoSugar {
106106
createLinearLearnerBinaryClassifier(region = Regions.GovCloud.getName)
107107
assert(estimatorGovCloud.trainingImage ==
108108
"226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/linear-learner:1")
109+
110+
val estimatorAPSouth1 =
111+
createLinearLearnerBinaryClassifier(region = Regions.AP_SOUTH_1.getName)
112+
assert(estimatorAPSouth1.trainingImage ==
113+
"991648021394.dkr.ecr.ap-south-1.amazonaws.com/linear-learner:1")
114+
115+
val estimatorAPSouthEast1 =
116+
createLinearLearnerBinaryClassifier(region = Regions.AP_SOUTHEAST_1.getName)
117+
assert(estimatorAPSouthEast1.trainingImage ==
118+
"475088953585.dkr.ecr.ap-southeast-1.amazonaws.com/linear-learner:1")
119+
120+
val estimatorEUWest2 =
121+
createLinearLearnerBinaryClassifier(region = Regions.EU_WEST_2.getName)
122+
assert(estimatorEUWest2.trainingImage ==
123+
"644912444149.dkr.ecr.eu-west-2.amazonaws.com/linear-learner:1")
124+
125+
val estimatorCACentral1 =
126+
createLinearLearnerBinaryClassifier(region = Regions.CA_CENTRAL_1.getName)
127+
assert(estimatorCACentral1.trainingImage ==
128+
"469771592824.dkr.ecr.ca-central-1.amazonaws.com/linear-learner:1")
129+
130+
val estimatorUSWest1 =
131+
createLinearLearnerBinaryClassifier(region = Regions.US_WEST_1.getName)
132+
assert(estimatorUSWest1.trainingImage ==
133+
"632365934929.dkr.ecr.us-west-1.amazonaws.com/linear-learner:1")
109134
}
110135

111136
it should "use the correct defaults for multiclass classifier" in {
@@ -161,6 +186,31 @@ class LinearLearnerSageMakerEstimatorTests extends FlatSpec with MockitoSugar {
161186
createLinearLearnerMultiClassClassifier(region = Regions.GovCloud.getName)
162187
assert(estimatorGovCloud.trainingImage ==
163188
"226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/linear-learner:1")
189+
190+
val estimatorAPSouth1 =
191+
createLinearLearnerMultiClassClassifier(region = Regions.AP_SOUTH_1.getName)
192+
assert(estimatorAPSouth1.trainingImage ==
193+
"991648021394.dkr.ecr.ap-south-1.amazonaws.com/linear-learner:1")
194+
195+
val estimatorAPSouthEast1 =
196+
createLinearLearnerMultiClassClassifier(region = Regions.AP_SOUTHEAST_1.getName)
197+
assert(estimatorAPSouthEast1.trainingImage ==
198+
"475088953585.dkr.ecr.ap-southeast-1.amazonaws.com/linear-learner:1")
199+
200+
val estimatorEUWest2 =
201+
createLinearLearnerMultiClassClassifier(region = Regions.EU_WEST_2.getName)
202+
assert(estimatorEUWest2.trainingImage ==
203+
"644912444149.dkr.ecr.eu-west-2.amazonaws.com/linear-learner:1")
204+
205+
val estimatorCACentral1 =
206+
createLinearLearnerMultiClassClassifier(region = Regions.CA_CENTRAL_1.getName)
207+
assert(estimatorCACentral1.trainingImage ==
208+
"469771592824.dkr.ecr.ca-central-1.amazonaws.com/linear-learner:1")
209+
210+
val estimatorUSWest1 =
211+
createLinearLearnerMultiClassClassifier(region = Regions.US_WEST_1.getName)
212+
assert(estimatorUSWest1.trainingImage ==
213+
"632365934929.dkr.ecr.us-west-1.amazonaws.com/linear-learner:1")
164214
}
165215

166216
it should "use the correct defaults for regressor" in {
@@ -212,6 +262,31 @@ class LinearLearnerSageMakerEstimatorTests extends FlatSpec with MockitoSugar {
212262
createLinearLearnerRegressor(region = Regions.GovCloud.getName)
213263
assert(estimatorGovCloud.trainingImage ==
214264
"226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/linear-learner:1")
265+
266+
val estimatorAPSouth1 =
267+
createLinearLearnerRegressor(region = Regions.AP_SOUTH_1.getName)
268+
assert(estimatorAPSouth1.trainingImage ==
269+
"991648021394.dkr.ecr.ap-south-1.amazonaws.com/linear-learner:1")
270+
271+
val estimatorAPSouthEast1 =
272+
createLinearLearnerRegressor(region = Regions.AP_SOUTHEAST_1.getName)
273+
assert(estimatorAPSouthEast1.trainingImage ==
274+
"475088953585.dkr.ecr.ap-southeast-1.amazonaws.com/linear-learner:1")
275+
276+
val estimatorEUWest2 =
277+
createLinearLearnerRegressor(region = Regions.EU_WEST_2.getName)
278+
assert(estimatorEUWest2.trainingImage ==
279+
"644912444149.dkr.ecr.eu-west-2.amazonaws.com/linear-learner:1")
280+
281+
val estimatorCACentral1 =
282+
createLinearLearnerRegressor(region = Regions.CA_CENTRAL_1.getName)
283+
assert(estimatorCACentral1.trainingImage ==
284+
"469771592824.dkr.ecr.ca-central-1.amazonaws.com/linear-learner:1")
285+
286+
val estimatorUSWest1 =
287+
createLinearLearnerRegressor(region = Regions.US_WEST_1.getName)
288+
assert(estimatorUSWest1.trainingImage ==
289+
"632365934929.dkr.ecr.us-west-1.amazonaws.com/linear-learner:1")
215290
}
216291

217292
it should "setFeatureDim" in {

‎sagemaker-spark-sdk/src/test/scala/com/amazonaws/services/sagemaker/sparksdk/algorithms/PCASageMakerEstimatorTests.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,26 @@ class PCASageMakerEstimatorTests extends FlatSpec with MockitoSugar {
6868
val estimatorGovCloud = createPCAEstimator(region = Regions.GovCloud.getName)
6969
assert(estimatorGovCloud.trainingImage ==
7070
"226302683700.dkr.ecr.us-gov-west-1.amazonaws.com/pca:1")
71+
72+
val estimatorAPSouth1 = createPCAEstimator(region = Regions.AP_SOUTH_1.getName)
73+
assert(estimatorAPSouth1.trainingImage ==
74+
"991648021394.dkr.ecr.ap-south-1.amazonaws.com/pca:1")
75+
76+
val estimatorAPSouthEast1 = createPCAEstimator(region = Regions.AP_SOUTHEAST_1.getName)
77+
assert(estimatorAPSouthEast1.trainingImage ==
78+
"475088953585.dkr.ecr.ap-southeast-1.amazonaws.com/pca:1")
79+
80+
val estimatorEUWest2 = createPCAEstimator(region = Regions.EU_WEST_2.getName)
81+
assert(estimatorEUWest2.trainingImage ==
82+
"644912444149.dkr.ecr.eu-west-2.amazonaws.com/pca:1")
83+
84+
val estimatorCACentral1 = createPCAEstimator(region = Regions.CA_CENTRAL_1.getName)
85+
assert(estimatorCACentral1.trainingImage ==
86+
"469771592824.dkr.ecr.ca-central-1.amazonaws.com/pca:1")
87+
88+
val estimatorUSWest1 = createPCAEstimator(region = Regions.US_WEST_1.getName)
89+
assert(estimatorUSWest1.trainingImage ==
90+
"632365934929.dkr.ecr.us-west-1.amazonaws.com/pca:1")
7191
}
7292

7393
it should "use the correct defaults" in {

0 commit comments

Comments
 (0)